From 088e680d572918d0bbc04d75a424b2e1b37b83f5 Mon Sep 17 00:00:00 2001 From: M Platypus Date: Thu, 13 Aug 2026 12:43:19 -0400 Subject: [PATCH 01/14] fix: radar_classification run() was dispatching to main_debug, not main --- .../tests/test_radar_entrypoint_uses_main.py | 24 +++++++++++++++++++ .../references/radar_classification/train.py | 2 +- 2 files changed, 25 insertions(+), 1 deletion(-) create mode 100644 tinyml-tinyverse/tests/test_radar_entrypoint_uses_main.py diff --git a/tinyml-tinyverse/tests/test_radar_entrypoint_uses_main.py b/tinyml-tinyverse/tests/test_radar_entrypoint_uses_main.py new file mode 100644 index 0000000..8c05441 --- /dev/null +++ b/tinyml-tinyverse/tests/test_radar_entrypoint_uses_main.py @@ -0,0 +1,24 @@ +"""Regression test: radar_classification.train.run() must dispatch to main(), +not main_debug() (a leftover notebook-parity harness that silently skips +quantization, AMP, and torch.compile for every radar training run).""" +from unittest.mock import patch + +from tinyml_tinyverse.references.radar_classification import train as radar_train + + +def test_run_dispatches_to_main_not_main_debug(): + with patch.object(radar_train, "run_distributed") as mock_run_distributed: + fake_args = object() + radar_train.run(fake_args) + + mock_run_distributed.assert_called_once() + dispatched_fn, dispatched_args = mock_run_distributed.call_args[0] + + assert dispatched_fn is radar_train.main, ( + f"run() dispatched to {dispatched_fn.__name__!r}, expected 'main'. " + "main_debug() never applies quantization_wrapped_model, " + "compile_model_if_enabled/apply_hardware_defaults, or " + "resume_from_checkpoint -- wiring run() to it silently drops quantized " + "training and hardware acceleration for every radar run." + ) + assert dispatched_args is fake_args diff --git a/tinyml-tinyverse/tinyml_tinyverse/references/radar_classification/train.py b/tinyml-tinyverse/tinyml_tinyverse/references/radar_classification/train.py index 3fb0e9b..79749ad 100644 --- a/tinyml-tinyverse/tinyml_tinyverse/references/radar_classification/train.py +++ b/tinyml-tinyverse/tinyml_tinyverse/references/radar_classification/train.py @@ -521,7 +521,7 @@ def main_debug(gpu, args): def run(args): """Run training with optional distributed mode.""" - run_distributed(main_debug, args) + run_distributed(main, args) if __name__ == "__main__": From 57bb0527a355e65a041f9278c87aa586d7c86ac5 Mon Sep 17 00:00:00 2001 From: M Platypus Date: Thu, 13 Aug 2026 12:52:12 -0400 Subject: [PATCH 02/14] docs: record post-fix MPS vs CPU benchmark for radar training Post-fix (main() live per Task 1): CPU 0.621s/epoch, MPS 1.066s/epoch (1.72x slower) -- gap did not close vs pre-fix's 1.90x. Root cause: compile_model_if_enabled/apply_hardware_defaults are never called anywhere in radar_classification/train.py (only wired into the timeseries_* reference scripts), so torch.compile/AMP were off for all four runs regardless of which function run() dispatches to. Co-Authored-By: Claude Sonnet 5 --- ...026-08-13-radar-training-entrypoint-fix.md | 181 ++++++++++++++++++ 1 file changed, 181 insertions(+) create mode 100644 docs/superpowers/plans/2026-08-13-radar-training-entrypoint-fix.md diff --git a/docs/superpowers/plans/2026-08-13-radar-training-entrypoint-fix.md b/docs/superpowers/plans/2026-08-13-radar-training-entrypoint-fix.md new file mode 100644 index 0000000..b63c921 --- /dev/null +++ b/docs/superpowers/plans/2026-08-13-radar-training-entrypoint-fix.md @@ -0,0 +1,181 @@ +# Radar Training Entrypoint Fix Implementation Plan + +> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. + +**Goal:** Wire the radar classification training script's `run()` entrypoint to the fully-featured `main()` function instead of the leftover `main_debug()` harness, then confirm on real hardware that this closes the MPS-vs-CPU performance gap it currently causes. + +**Architecture:** `run_distributed(main_debug, args)` becomes `run_distributed(main, args)` in `radar_classification/train.py`. `main()` already exists and is fully wired to the shared, hardened training infrastructure (`quantization_wrapped_model`, `compile_model_if_enabled`/`apply_hardware_defaults`, `resume_from_checkpoint`, `create_data_loaders`) — it is simply never called today. No new code is needed for the fix itself; the risk is regression in radar-specific behavior that only `main_debug` currently exercises, which Task 1's test and manual run guard against. + +**Tech Stack:** Python 3.10, PyTorch 2.7.1, pytest + +## Global Constraints + +- Python `==3.10.*` +- No new dependencies +- `main_debug()` itself is left in place (unreferenced) — it's the author's own notebook-parity debug harness per its docstring, not this plan's concern to delete +- The fix must not change CLI argument names or defaults — only which internal function `run()` dispatches to + +## Context: Why This Is Needed + +`BaseRadarModelTraining.run()` (`tinyml-modelmaker/tinyml_modelmaker/ai_modules/radar/training/tinyml_tinyverse/radar_base.py`) calls `self.train_module.run(args)` for both the float-training pass and the quantized-training pass. `train_module` is wired to `tinyml-tinyverse/tinyml_tinyverse/references/radar_classification/train.py`, whose `run()` is: + +```python +def run(args): + """Run training with optional distributed mode.""" + run_distributed(main_debug, args) +``` + +`main_debug()` is a hand-rolled duplicate of `main()` (its own comment: *"Following as close as possible steps from jupyter notebook to test if model learning plateau is coming from training loop"*). Confirmed by reading both functions side by side: + +- `main_debug()` hardcodes `phase = 'FloatTrain'` unconditionally and never calls `quantization_wrapped_model` — the "QuantTrain" argv pass `radar_base.py` builds produces a second float-training run instead, logged under the wrong phase label, so `get_radar_classification_log_summary_regex()`'s `QuantTrain`-specific regexes never match anything. +- `main_debug()` never calls `compile_model_if_enabled` or `apply_hardware_defaults` (PRs #21–23) — no AMP, no `torch.compile`, regardless of device or hardware. +- `main_debug()` never calls `resume_from_checkpoint`. + +Empirically confirmed this has a real, measurable performance cost, not just a theoretical one. Benchmarked `LINEAR_4L_PC` (the one registered radar model) on a synthetic 5-class radar-shaped fixture, 30 epochs, via the actual `radar_classification.train.run(args)` entrypoint as currently shipped: + +| Device | Time/epoch | +|---|---| +| CPU | 0.602s | +| MPS | 1.145s (**1.90x slower than CPU**) | + +MPS being slower than CPU is the expected signature of a small-op-heavy graph with no kernel fusion — exactly what `apply_hardware_defaults`/`compile_model_if_enabled` exists to fix elsewhere in the codebase. Task 2 re-runs this same benchmark once `main()` is live, to confirm and quantify the improvement. + +--- + +## File Map + +| Action | Path | Responsibility | +|--------|------|-----------------| +| Modify | `tinyml-tinyverse/tinyml_tinyverse/references/radar_classification/train.py:522-524` | `run()` dispatches to `main`, not `main_debug` | +| Create | `tinyml-tinyverse/tests/test_radar_entrypoint_uses_main.py` | Regression test pinning the dispatch target | + +--- + +## Task 1: Fix `run()` dispatch target, with regression test + +**Files:** +- Modify: `tinyml-tinyverse/tinyml_tinyverse/references/radar_classification/train.py` (function `run`, currently lines 522-524) +- Create: `tinyml-tinyverse/tests/test_radar_entrypoint_uses_main.py` + +**Interfaces:** +- Consumes: `tinyml_tinyverse.references.radar_classification.train.main`, `.main_debug`, `.run`, `.run_distributed` (all already defined in the module — no new interfaces) + +- [ ] **Step 1: Write the failing test** + +```python +"""Regression test: radar_classification.train.run() must dispatch to main(), +not main_debug() (a leftover notebook-parity harness that silently skips +quantization, AMP, and torch.compile for every radar training run).""" +from unittest.mock import patch + +from tinyml_tinyverse.references.radar_classification import train as radar_train + + +def test_run_dispatches_to_main_not_main_debug(): + with patch.object(radar_train, "run_distributed") as mock_run_distributed: + fake_args = object() + radar_train.run(fake_args) + + mock_run_distributed.assert_called_once() + dispatched_fn, dispatched_args = mock_run_distributed.call_args[0] + + assert dispatched_fn is radar_train.main, ( + f"run() dispatched to {dispatched_fn.__name__!r}, expected 'main'. " + "main_debug() never applies quantization_wrapped_model, " + "compile_model_if_enabled/apply_hardware_defaults, or " + "resume_from_checkpoint -- wiring run() to it silently drops quantized " + "training and hardware acceleration for every radar run." + ) + assert dispatched_args is fake_args +``` + +- [ ] **Step 2: Run test to verify it fails** + +Run: `cd tinyml-tinyverse && python -m pytest tests/test_radar_entrypoint_uses_main.py -v` +Expected: FAIL — `dispatched_fn` is `main_debug`, not `main` + +- [ ] **Step 3: Fix the dispatch target** + +In `tinyml-tinyverse/tinyml_tinyverse/references/radar_classification/train.py`, change: + +```python +def run(args): + """Run training with optional distributed mode.""" + run_distributed(main_debug, args) +``` + +to: + +```python +def run(args): + """Run training with optional distributed mode.""" + run_distributed(main, args) +``` + +- [ ] **Step 4: Run test to verify it passes** + +Run: `cd tinyml-tinyverse && python -m pytest tests/test_radar_entrypoint_uses_main.py -v` +Expected: PASS + +- [ ] **Step 5: Manual end-to-end sanity check** + +`main()` has never actually executed for radar before — confirm it runs clean on a real (if synthetic) dataset before trusting the unit test alone. Build a small fixture (5 class dirs of CSVs under `/classes/`, balanced `annotations/instances_{train,test,val}_list.txt`) and drive `train.get_args_parser().parse_args([...]); train.run(args)` directly with `--device cpu`, a few epochs, `--quantization 0`. Confirm it completes and exports `model.onnx` without error. Then repeat with `--quantization 1` (previously silently a no-op under `main_debug`) and confirm the log now shows `QuantTrain` phase entries and a second exported model under the quantization output dir. + +- [ ] **Step 6: Commit** + +```bash +cd tinyml-tinyverse +git add tinyml_tinyverse/references/radar_classification/train.py tests/test_radar_entrypoint_uses_main.py +git commit -m "fix: radar_classification run() was dispatching to main_debug, not main" +``` + +--- + +## Task 2: Re-benchmark MPS vs CPU with the fix in place + +**Files:** +- None modified — this is a measurement task using the fixture/driver already built during investigation (`bench_radar.py`, `make_radar_fixture.py` in the session scratchpad — recreate if not available) + +**Interfaces:** +- Consumes: `radar_classification.train.run(args)` (now dispatching to `main`, per Task 1) + +- [ ] **Step 1: Re-run the same benchmark used to characterize the bug** + +Using the same synthetic fixture (5 classes, balanced annotation lists, `LINEAR_4L_PC`, batch size 16, 30 epochs) and the same driver pattern that produced the pre-fix numbers (constructs `argv`, calls `train.get_args_parser().parse_args(argv)`, times `train.run(args)`), run once with `--device cpu` and once with `--device mps`. + +Run: `python bench_radar.py cpu` then `python bench_radar.py mps` (with `PYTORCH_ENABLE_MPS_FALLBACK=1` set) +Expected: both complete without error; timing printed as `total=...s for N epochs -> ...s/epoch` + +- [ ] **Step 2: Record the comparison** + +Append a short results table to this plan (or a sibling `RESULTS.md` next to it) with pre-fix vs post-fix CPU and MPS per-epoch timing, and note whether `apply_hardware_defaults` actually enabled `torch.compile`/AMP for this run (check the training log for the relevant INFO lines from `compile_model_if_enabled`/`apply_hardware_defaults`). + +- [ ] **Step 3: Commit** + +```bash +git add docs/superpowers/plans/2026-08-13-radar-training-entrypoint-fix.md +git commit -m "docs: record post-fix MPS vs CPU benchmark for radar training" +``` + +--- + +## Results + +Same fixture (5-class synthetic radar data), same driver (`bench_radar.py`, `LINEAR_4L_PC`, batch size 16, 30 epochs), run through `radar_classification.train.run(args)` — which now dispatches to `main()` per Task 1's fix. + +| Run | Device | Time/epoch | Total (30 epochs) | MPS vs CPU | +|---|---|---|---|---| +| Pre-fix (`main_debug`, historical) | CPU | 0.602s | — | — | +| Pre-fix (`main_debug`, historical) | MPS | 1.145s | — | **1.90x slower** | +| Post-fix (`main`) | CPU | 0.621s | 18.62s | — | +| Post-fix (`main`) | MPS | 1.066s | 31.99s | **1.72x slower** | + +**The gap did not close.** MPS is still substantially slower than CPU after the fix — 1.72x, essentially the same order as the pre-fix 1.90x. The ~9% wobble between the two ratios is consistent with ordinary run-to-run variance (different process, thermal state, etc.), not a structural change. + +**Why: `compile_model_if_enabled`/`apply_hardware_defaults` never engage for radar, in either `main()` or `main_debug()`.** Grepping both run logs (`radar_out_cpu/run.log`, `radar_out_mps/run.log`) for the exact strings `compile_model_if_enabled` emits on success (`"Compiling model with torch.compile"`), on quantization skip (`"compile_model is enabled but quantization is also enabled"`), and on failure (`"torch.compile failed"`) — zero matches in either log. Reading the code confirms why: + +- `radar_classification/train.py`'s `main()` (lines 187-334) calls `quantization_wrapped_model` (line 269) and `resume_from_checkpoint` (line 257) — those parts of Task 1's fix are real and now active — but it never imports or calls `compile_model_if_enabled` at all. Contrast with `timeseries_classification/train.py`, `timeseries_forecasting/train.py`, `timeseries_anomalydetection/train.py`, and `timeseries_regression/train.py`, which all import it from `..common.train_base` and call it before their training loops. `image_classification/train.py` and `audio_classification/train.py` also never call it — this looks like a scope gap in PR #22 ("compile-hardening", commit `dc8eeb2`), which only touched the four `timeseries_*` scripts, not the `radar_classification`/`image_classification`/`audio_classification` scripts. +- `apply_hardware_defaults` isn't part of `tinyml-tinyverse` at all — it lives in `tinyml-modelmaker/tinyml_modelmaker/utils/hardware_defaults.py` and is wired only into `tinyml_modelmaker/ai_modules/timeseries/params.py` (commit `edbabba`, "wire apply_hardware_defaults into timeseries init_params"). It runs one layer up, in modelmaker's argv-construction step, before `train.py` ever sees `--compile-model`/`--native-amp`. `bench_radar.py` calls `radar_classification.train.get_args_parser().parse_args(argv)` directly, bypassing modelmaker entirely, so this layer was never in play for either the pre-fix or post-fix benchmark regardless. +- Net effect: `--compile-model` defaults to `0` and `--native-amp` defaults to `False` in `get_base_args_parser()` (`train_base.py:200,221`), nothing in the radar call path ever overrides them, and even if it did, `main()` has no code path that would act on them. torch.compile and AMP were off for all four runs (pre-fix CPU/MPS, post-fix CPU/MPS) — the comparison is apples-to-apples on that axis, just not for the reason the plan's premise expected. + +**Bottom line:** Task 1's fix is still correct and worth having — it wires up quantization (`QuantTrain` phase now actually runs quantization-aware training instead of a mislabeled second float pass) and checkpoint resume, both real, previously-silent gaps. But it does not touch torch.compile/AMP for radar, because that wiring was never built for the radar script in the first place (unlike the `timeseries_*` family). The MPS-slower-than-CPU result — expected for a small-op-heavy graph with no kernel fusion — persists post-fix because the mechanism that would fix it (`compile_model_if_enabled`) is not in `radar_classification/train.py`'s call graph at all. Closing this gap would require a separate change: wiring `compile_model_if_enabled` (and optionally AMP via `get_amp_context`/`get_grad_scaler`) into `radar_classification/train.py::main()`, mirroring what the `timeseries_*` scripts already do — out of scope for this plan. From 459752ddb0c921bc1dccfc0d7711118096f80d92 Mon Sep 17 00:00:00 2001 From: M Platypus Date: Thu, 13 Aug 2026 19:36:57 -0400 Subject: [PATCH 03/14] docs: add top-of-plan correction note for radar entrypoint fix Whole-plan review flagged that the Goal/Architecture sections still asserted the disproven premise (main() wired to compile_model_if_enabled) 130+ lines above the actual Results. Ledger and Results were already honest; this fixes the presentation for a top-down reader. --- .../plans/2026-08-13-radar-training-entrypoint-fix.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/docs/superpowers/plans/2026-08-13-radar-training-entrypoint-fix.md b/docs/superpowers/plans/2026-08-13-radar-training-entrypoint-fix.md index b63c921..ccc17e2 100644 --- a/docs/superpowers/plans/2026-08-13-radar-training-entrypoint-fix.md +++ b/docs/superpowers/plans/2026-08-13-radar-training-entrypoint-fix.md @@ -8,6 +8,8 @@ **Tech Stack:** Python 3.10, PyTorch 2.7.1, pytest +> **Correction (see ## Results below):** the Goal and Architecture above assumed `main()` was already wired to `compile_model_if_enabled`/`apply_hardware_defaults`. It is not. Task 1's fix is real (quantization and checkpoint-resume now work) but does **not** close the MPS-vs-CPU gap — see Results for the actual post-fix numbers and root cause. + ## Global Constraints - Python `==3.10.*` From e4727493c4a9fc2c422f17731eb1350a19d96948 Mon Sep 17 00:00:00 2001 From: M Platypus Date: Thu, 13 Aug 2026 21:20:43 -0400 Subject: [PATCH 04/14] feat: wire compile_model_if_enabled into radar_classification main() --- ...-13-compile-hardening-radar-image-audio.md | 304 ++++++++++++++++++ .../tests/test_radar_compile_wired.py | 15 + .../references/radar_classification/train.py | 2 + 3 files changed, 321 insertions(+) create mode 100644 docs/superpowers/plans/2026-08-13-compile-hardening-radar-image-audio.md create mode 100644 tinyml-tinyverse/tests/test_radar_compile_wired.py diff --git a/docs/superpowers/plans/2026-08-13-compile-hardening-radar-image-audio.md b/docs/superpowers/plans/2026-08-13-compile-hardening-radar-image-audio.md new file mode 100644 index 0000000..88106eb --- /dev/null +++ b/docs/superpowers/plans/2026-08-13-compile-hardening-radar-image-audio.md @@ -0,0 +1,304 @@ +# Extend compile_model_if_enabled to Radar, Image, and Audio Classification Implementation Plan + +> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. + +**Goal:** Wire `compile_model_if_enabled` (the `torch.compile` warmup-and-fallback helper added by PR #22, currently only called from the four `timeseries_*` reference scripts) into `radar_classification`, `image_classification`, and `audio_classification`'s `main()` functions, so all seven reference training scripts get the same hardware-acceleration path — and measure whether it actually helps each one. + +**Architecture:** Each of the three target `main()` functions already has the exact structural shape `compile_model_if_enabled` expects: `move_model_to_device(model, device, logger)` immediately followed by `setup_distributed_model(model, args, device)`, with `dataset.X` (post feature-extraction) available in scope. The fix is the same one-line insertion in all three files, matching `timeseries_classification/train.py:275` verbatim: +```python +model = compile_model_if_enabled(model, args, logger, input_shape=(1,) + dataset.X.shape[1:]) +``` +placed between those two calls, plus adding `compile_model_if_enabled` to each file's import from `..common.train_base`. No changes to `compile_model_if_enabled` itself or to `train_base.py`. + +**Tech Stack:** Python 3.10, PyTorch 2.7.1, pytest + +## Global Constraints + +- Python `==3.10.*` +- No new dependencies +- Zero change to behavior when `--compile-model` is not set (the default, `0`) — `compile_model_if_enabled` already no-ops in that case; do not add any new conditionals around the call +- Do not touch `compile_model_if_enabled` itself, `train_base.py`, or any file outside the three target `train.py` files (+ their new/modified test files) +- Each task's benchmark must reuse the same measurement methodology as the radar entrypoint fix plan (`docs/superpowers/plans/2026-08-13-radar-training-entrypoint-fix.md`) — synthetic fixture, same device/epoch/batch-size reporting style — so results are comparable across all four now-benchmarked modules (timeseries already had compile; radar was benchmarked without it in that plan; this plan adds image and audio, and re-benchmarks radar with it now on) +- **Verification runs on two machines, not one:** local Mac (CPU vs MPS, `~/.venv-tinyml`) AND GX10 (CPU vs CUDA — `NVIDIA GB10`, `~/jupyterlab/.venv`, torch 2.9.0+cu130, reachable via `ssh -i ~/.ssh/gx10_key martin@gx10-singularity.skunk-mercat.ts.net`). Implementer subagents are not expected to have GX10 SSH access — the controller runs the GX10 leg directly after each task's local implementation is approved, and appends those results alongside the local ones in that task's Results section before considering the task's verification complete. + +## Context: Why This Is Needed + +While closing out the radar entrypoint fix (`docs/superpowers/plans/2026-08-13-radar-training-entrypoint-fix.md`), the whole-plan review found that fixing radar's `run()` dispatch (making `main()` live) did **not** close its MPS-vs-CPU performance gap, because `compile_model_if_enabled`/`apply_hardware_defaults` were never wired into radar's `main()` at all — only into the four `timeseries_*` scripts (PR #22, commit `dc8eeb2`). The same review flagged that `image_classification` and `audio_classification` have the identical gap: + +``` +grep -n "compile_model_if_enabled" tinyml-tinyverse/tinyml_tinyverse/references/image_classification/train.py +tinyml-tinyverse/tinyml_tinyverse/references/audio_classification/train.py +``` +— zero matches in either file, confirmed directly against source, independent of the radar plan's own investigation. + +Rather than fix radar alone and leave image/audio in the same state, this plan closes the gap everywhere it exists in one coordinated pass, so the three currently-uncompiled reference scripts converge on the same behavior as `timeseries_classification`, `timeseries_regression`, `timeseries_forecasting`, and `timeseries_anomalydetection` already have. + +**Known unresolved question, not assumed:** whether `torch.compile` actually helps on hardware this small (radar's `LINEAR_4L_PC` is a 4-layer linear/BatchNorm model — the radar plan measured MPS at ~1.9x slower than CPU with no compile, and the theory is that fused kernels reduce per-op dispatch overhead, but this has not been confirmed for a model this size). Each task's benchmark step exists specifically to answer this per-module — report what you measure, not what the theory predicts. + +--- + +## File Map + +| Action | Path | Responsibility | +|--------|------|-----------------| +| Modify | `tinyml-tinyverse/tinyml_tinyverse/references/radar_classification/train.py` | Wire `compile_model_if_enabled` into `main()` | +| Modify | `tinyml-tinyverse/tinyml_tinyverse/references/image_classification/train.py` | Wire `compile_model_if_enabled` into `main()` | +| Modify | `tinyml-tinyverse/tinyml_tinyverse/references/audio_classification/train.py` | Wire `compile_model_if_enabled` into `main()` | +| Create | `tinyml-tinyverse/tests/test_radar_compile_wired.py` | Regression test: radar's `main()` calls `compile_model_if_enabled` | +| Create | `tinyml-tinyverse/tests/test_image_classification_compile_wired.py` | Same, image classification | +| Create | `tinyml-tinyverse/tests/test_audio_classification_compile_wired.py` | Same, audio classification | + +--- + +## Task 1: Wire `compile_model_if_enabled` into radar_classification, with test + benchmark + +**Files:** +- Modify: `tinyml-tinyverse/tinyml_tinyverse/references/radar_classification/train.py` (imports ~line 65-89; insertion point currently lines 251-254, between `move_model_to_device` and `setup_distributed_model`) +- Create: `tinyml-tinyverse/tests/test_radar_compile_wired.py` + +**Interfaces:** +- Consumes: `tinyml_tinyverse.references.common.train_base.compile_model_if_enabled` (existing, unmodified — signature `compile_model_if_enabled(model, args, logger, input_shape=None)`, returns the (possibly compiled) model) +- Produces: nothing new consumed by later tasks — Tasks 2 and 3 are independent, same pattern applied to different files + +- [ ] **Step 1: Write the failing test** + +```python +"""Regression test: radar_classification.train.main() must call +compile_model_if_enabled, matching the pattern already used in the 4 +timeseries_* reference scripts (PR #22). Without this, --compile-model is +silently a no-op for radar regardless of what the caller requests.""" +import inspect + +from tinyml_tinyverse.references.radar_classification import train as radar_train + + +def test_main_calls_compile_model_if_enabled(): + main_source = inspect.getsource(radar_train.main) + assert "compile_model_if_enabled(" in main_source, ( + "main() does not call compile_model_if_enabled -- torch.compile/AMP " + "hardware acceleration would silently not apply to radar training." + ) +``` + +- [ ] **Step 2: Run test to verify it fails** + +Run: `cd tinyml-tinyverse && python -m pytest tests/test_radar_compile_wired.py -v` +Expected: FAIL — `"compile_model_if_enabled("` not present in `main`'s source + +- [ ] **Step 3: Add the import and the call** + +In `tinyml-tinyverse/tinyml_tinyverse/references/radar_classification/train.py`, add `compile_model_if_enabled` to the existing import from `..common.train_base` (around line 65-89 — find the multi-line `from ..common.train_base import (...)` block and add it as a new entry, alphabetically or grouped with `move_model_to_device`/`compile_model_if_enabled`-adjacent imports per the file's existing style). + +Then in `main()`, change: +```python + move_model_to_device(model, device, logger) + criterion = nn.CrossEntropyLoss(label_smoothing=args.label_smoothing) + + model, model_without_ddp, model_ema = setup_distributed_model(model, args, device) +``` +to: +```python + move_model_to_device(model, device, logger) + model = compile_model_if_enabled(model, args, logger, input_shape=(1,) + dataset.X.shape[1:]) + criterion = nn.CrossEntropyLoss(label_smoothing=args.label_smoothing) + + model, model_without_ddp, model_ema = setup_distributed_model(model, args, device) +``` + +- [ ] **Step 4: Run test to verify it passes** + +Run: `cd tinyml-tinyverse && python -m pytest tests/test_radar_compile_wired.py -v` +Expected: PASS + +- [ ] **Step 5: Manual end-to-end sanity check with --compile-model 1** + +Using the synthetic radar fixture from the entrypoint-fix plan (regenerate via `make_radar_fixture.py` if not present — see that plan's Task 1 for the recipe), drive `train.run(args)` directly with `--device cpu --compile-model 1`, a few epochs. Confirm it completes without error and the log shows `compile_model_if_enabled`'s own INFO lines (check `train_base.py`'s `compile_model_if_enabled` for the exact log message text first — grep the run log for it). Then confirm `--compile-model 0` (the default) still behaves identically to before this change — no compile-related log lines, same training behavior. + +- [ ] **Step 6: Benchmark CPU vs MPS with compile now enabled** + +Reuse the benchmark driver pattern from the entrypoint-fix plan (`bench_radar.py` in the session scratchpad), but add `--compile-model 1` to the argv. Run once on `cpu`, once on `mps` (`PYTORCH_ENABLE_MPS_FALLBACK=1`), same 30 epochs / batch size 16 / `LINEAR_4L_PC` as the entrypoint-fix plan's benchmark, so the numbers are directly comparable. Record: does compile change the CPU number, the MPS number, or the ratio between them, versus the entrypoint-fix plan's post-fix-no-compile numbers (CPU 0.621s/epoch, MPS 1.066s/epoch, 1.72x)? Report what you measure even if compile makes things worse or has no effect — that's a legitimate finding for a model this small, not a task failure. + +- [ ] **Step 7: Record results and commit** + +Append a `## Results (Task 1: radar)` section to this plan doc with the benchmark table and a short explanation. + +```bash +cd tinyml-tinyverse +git add tinyml_tinyverse/references/radar_classification/train.py tests/test_radar_compile_wired.py +git add ../docs/superpowers/plans/2026-08-13-compile-hardening-radar-image-audio.md +git commit -m "feat: wire compile_model_if_enabled into radar_classification main()" +``` + +--- + +## Task 2: Wire `compile_model_if_enabled` into image_classification, with test + benchmark + +**Files:** +- Modify: `tinyml-tinyverse/tinyml_tinyverse/references/image_classification/train.py` (imports ~line 92-111; insertion point currently lines 329-332, between `move_model_to_device` and `setup_distributed_model`) +- Create: `tinyml-tinyverse/tests/test_image_classification_compile_wired.py` + +**Interfaces:** +- Consumes: same `compile_model_if_enabled` as Task 1 — independent of Task 1, do not wait for it or reuse its branch + +- [ ] **Step 1: Write the failing test** + +```python +"""Regression test: image_classification.train.main() must call +compile_model_if_enabled, matching the timeseries_* pattern (PR #22).""" +import inspect + +from tinyml_tinyverse.references.image_classification import train as image_train + + +def test_main_calls_compile_model_if_enabled(): + main_source = inspect.getsource(image_train.main) + assert "compile_model_if_enabled(" in main_source, ( + "main() does not call compile_model_if_enabled -- torch.compile/AMP " + "hardware acceleration would silently not apply to image classification training." + ) +``` + +- [ ] **Step 2: Run test to verify it fails** + +Run: `cd tinyml-tinyverse && python -m pytest tests/test_image_classification_compile_wired.py -v` +Expected: FAIL + +- [ ] **Step 3: Add the import and the call** + +Add `compile_model_if_enabled` to the existing `from ..common.train_base import (...)` block (~line 92-111). Then change: +```python + move_model_to_device(model, device, logger) + criterion = nn.CrossEntropyLoss(label_smoothing=args.label_smoothing) + + model, model_without_ddp, model_ema = setup_distributed_model(model, args, device) +``` +to: +```python + move_model_to_device(model, device, logger) + model = compile_model_if_enabled(model, args, logger, input_shape=(1,) + dataset.X.shape[1:]) + criterion = nn.CrossEntropyLoss(label_smoothing=args.label_smoothing) + + model, model_without_ddp, model_ema = setup_distributed_model(model, args, device) +``` +**Note the indentation in this file is one level deeper than radar/audio (this block sits inside an outer block in image_classification) — match the surrounding indentation exactly, don't copy radar's column position verbatim.** + +**Watch for:** this file has an `args.nn_for_feature_extraction` branch elsewhere (used at export time, line ~450-454) that chooses between `dataset.X_raw.shape` and `dataset.X.shape` depending on whether an NN is used for feature extraction. Confirm which shape the model actually consumes *at the point where you're inserting the compile call* (before or after any feature-extraction wrapping) — if `nn_for_feature_extraction` changes what the raw model's forward() expects at this point in `main()`, use the matching shape instead of assuming `dataset.X.shape` unconditionally. If unsure after reading the surrounding ~50 lines, ask before proceeding — this is exactly the kind of task-specific judgment call the brief can't resolve for you. + +- [ ] **Step 4: Run test to verify it passes** + +Run: `cd tinyml-tinyverse && python -m pytest tests/test_image_classification_compile_wired.py -v` +Expected: PASS + +- [ ] **Step 5: Manual end-to-end sanity check with --compile-model 1** + +Build or reuse a small synthetic image classification fixture (check `tinyml-tinyverse/tests/` for an existing image dataset test fixture/helper before building one from scratch — this repo likely already has one given image_classification has existing tests). Drive `train.run(args)` with `--device cpu --compile-model 1`, a few epochs. Confirm completion and compile-related log lines present. Confirm `--compile-model 0` still behaves as before. + +- [ ] **Step 6: Benchmark CPU vs MPS with compile enabled** + +Same methodology as Task 1 Step 6, adapted to whichever image model this benchmark uses (pick the smallest/fastest registered image model available, to keep iteration time reasonable) and image_classification's own CLI args. This is the first CPU-vs-MPS benchmark for image classification in either plan — there's no prior "no-compile" baseline to compare against from the earlier work, so also run once with `--compile-model 0` on both devices first to get that baseline, then `--compile-model 1` to see the delta, in the same benchmark session. + +- [ ] **Step 7: Record results and commit** + +Append `## Results (Task 2: image_classification)` to this plan doc. + +```bash +cd tinyml-tinyverse +git add tinyml_tinyverse/references/image_classification/train.py tests/test_image_classification_compile_wired.py +git add ../docs/superpowers/plans/2026-08-13-compile-hardening-radar-image-audio.md +git commit -m "feat: wire compile_model_if_enabled into image_classification main()" +``` + +--- + +## Task 3: Wire `compile_model_if_enabled` into audio_classification, with test + benchmark + +**Files:** +- Modify: `tinyml-tinyverse/tinyml_tinyverse/references/audio_classification/train.py` (imports ~line 92-111; insertion point currently lines 311-313, between `move_model_to_device` and `setup_distributed_model`) +- Create: `tinyml-tinyverse/tests/test_audio_classification_compile_wired.py` + +**Interfaces:** +- Consumes: same `compile_model_if_enabled` — independent of Tasks 1 and 2 + +- [ ] **Step 1: Write the failing test** + +```python +"""Regression test: audio_classification.train.main() must call +compile_model_if_enabled, matching the timeseries_* pattern (PR #22).""" +import inspect + +from tinyml_tinyverse.references.audio_classification import train as audio_train + + +def test_main_calls_compile_model_if_enabled(): + main_source = inspect.getsource(audio_train.main) + assert "compile_model_if_enabled(" in main_source, ( + "main() does not call compile_model_if_enabled -- torch.compile/AMP " + "hardware acceleration would silently not apply to audio classification training." + ) +``` + +- [ ] **Step 2: Run test to verify it fails** + +Run: `cd tinyml-tinyverse && python -m pytest tests/test_audio_classification_compile_wired.py -v` +Expected: FAIL + +- [ ] **Step 3: Add the import and the call** + +Add `compile_model_if_enabled` to the existing `from ..common.train_base import (...)` block (~line 92-111). Then change: +```python + move_model_to_device(model, device, logger) + criterion = nn.CrossEntropyLoss(label_smoothing=args.label_smoothing) + model, model_without_ddp, model_ema = setup_distributed_model(model, args, device) +``` +to: +```python + move_model_to_device(model, device, logger) + model = compile_model_if_enabled(model, args, logger, input_shape=(1,) + dataset.X.shape[1:]) + criterion = nn.CrossEntropyLoss(label_smoothing=args.label_smoothing) + model, model_without_ddp, model_ema = setup_distributed_model(model, args, device) +``` + +Same caveat as Task 2 applies here — audio_classification also has an `nn_for_feature_extraction` / `X_raw` vs `X` distinction at export time (line ~424-427). Check whether it affects what shape the model expects at this earlier insertion point before assuming `dataset.X.shape` unconditionally. + +- [ ] **Step 4: Run test to verify it passes** + +Run: `cd tinyml-tinyverse && python -m pytest tests/test_audio_classification_compile_wired.py -v` +Expected: PASS + +- [ ] **Step 5: Manual end-to-end sanity check with --compile-model 1** + +Same approach as Task 2 Step 5, adapted to audio_classification's dataset/CLI. Check `tinyml-tinyverse/tests/` for existing audio fixture helpers first. + +- [ ] **Step 6: Benchmark CPU vs MPS with compile enabled** + +Same methodology as Task 2 Step 6 (baseline `--compile-model 0` then `--compile-model 1`, both devices), adapted to audio_classification's smallest registered model. + +- [ ] **Step 7: Record results and commit** + +Append `## Results (Task 3: audio_classification)` to this plan doc. + +```bash +cd tinyml-tinyverse +git add tinyml_tinyverse/references/audio_classification/train.py tests/test_audio_classification_compile_wired.py +git add ../docs/superpowers/plans/2026-08-13-compile-hardening-radar-image-audio.md +git commit -m "feat: wire compile_model_if_enabled into audio_classification main()" +``` + +--- + +## Results (Task 1: radar) + +**Wiring:** `compile_model_if_enabled` is now imported and called in `radar_classification/train.py`'s `main()`, between `move_model_to_device` and `setup_distributed_model`, exactly matching the `timeseries_*` pattern. Regression test `tests/test_radar_compile_wired.py` passes; `--compile-model 0` (default) produces zero compile-related log lines and identical behavior to before this change; `--compile-model 1` produces `compile_model_if_enabled`'s own log line (`Compiling model with torch.compile (backend=aot_eager)`) on both CPU and MPS, with no fallback warnings on either device — compilation and the warmup forward pass succeed cleanly for `LINEAR_4L_PC` on this hardware. + +**Benchmark** — same fixture, `LINEAR_4L_PC`, batch size 16, 30 epochs, synthetic radar fixture, as the entrypoint-fix plan's post-fix-no-compile baseline: + +| Config | Device | s/epoch | CPU/MPS ratio | +|---|---|---|---| +| No compile (entrypoint-fix plan baseline) | CPU | 0.621 | 1.72x | +| No compile (entrypoint-fix plan baseline) | MPS | 1.066 | | +| `--compile-model 1` (this task) | CPU | 0.765 | 1.70x | +| `--compile-model 1` (this task) | MPS | 1.303 | | + +**Finding: `torch.compile` makes radar training slower on both devices, and does not close the CPU/MPS gap.** With compile enabled, CPU slows by ~23% (0.621s -> 0.765s/epoch) and MPS slows by ~22% (1.066s -> 1.303s/epoch). The CPU/MPS ratio is essentially unchanged (1.72x -> 1.70x) since both devices use the same `aot_eager` backend (the backend-selection logic in `compile_model_if_enabled` only routes CUDA to `inductor`; both CPU and MPS get `aot_eager`) and both slow down by a similar proportion. + +This is consistent with `LINEAR_4L_PC` being a tiny 4-layer linear/BatchNorm model: `torch.compile`'s per-call dynamo tracing, guard-checking, and graph-dispatch overhead is fixed cost per training step, and for a model this small there isn't enough per-op dispatch overhead in the eager path for kernel fusion to recoup that cost. The theory in this plan's "Known unresolved question" (fused kernels reducing dispatch overhead) does not hold for this model size — compile is a net loss here, not neutral or beneficial. Wiring it in is still correct (it makes `--compile-model` functional, matching the other reference scripts, and lets a caller opt in for larger/more compute-bound models where the tradeoff may differ), but the flag should not be turned on by default for radar's small linear models based on this evidence. diff --git a/tinyml-tinyverse/tests/test_radar_compile_wired.py b/tinyml-tinyverse/tests/test_radar_compile_wired.py new file mode 100644 index 0000000..fd9f6af --- /dev/null +++ b/tinyml-tinyverse/tests/test_radar_compile_wired.py @@ -0,0 +1,15 @@ +"""Regression test: radar_classification.train.main() must call +compile_model_if_enabled, matching the pattern already used in the 4 +timeseries_* reference scripts (PR #22). Without this, --compile-model is +silently a no-op for radar regardless of what the caller requests.""" +import inspect + +from tinyml_tinyverse.references.radar_classification import train as radar_train + + +def test_main_calls_compile_model_if_enabled(): + main_source = inspect.getsource(radar_train.main) + assert "compile_model_if_enabled(" in main_source, ( + "main() does not call compile_model_if_enabled -- torch.compile/AMP " + "hardware acceleration would silently not apply to radar training." + ) diff --git a/tinyml-tinyverse/tinyml_tinyverse/references/radar_classification/train.py b/tinyml-tinyverse/tinyml_tinyverse/references/radar_classification/train.py index 79749ad..f63d873 100644 --- a/tinyml-tinyverse/tinyml_tinyverse/references/radar_classification/train.py +++ b/tinyml-tinyverse/tinyml_tinyverse/references/radar_classification/train.py @@ -82,6 +82,7 @@ save_checkpoint, handle_export_only, move_model_to_device, + compile_model_if_enabled, log_training_time, apply_output_int_default, get_output_int_flag, @@ -250,6 +251,7 @@ def main(gpu, args): return move_model_to_device(model, device, logger) + model = compile_model_if_enabled(model, args, logger, input_shape=(1,) + dataset.X.shape[1:]) criterion = nn.CrossEntropyLoss(label_smoothing=args.label_smoothing) model, model_without_ddp, model_ema = setup_distributed_model(model, args, device) From b27a579db8bbf47770f4a33ca97df769ed506dbf Mon Sep 17 00:00:00 2001 From: M Platypus Date: Thu, 13 Aug 2026 22:01:06 -0400 Subject: [PATCH 05/14] docs: add GX10 (CUDA) benchmark leg for Task 1 radar compile wiring CUDA beats CPU 4.8x on GX10 even without compile (opposite of the Mac's MPS-loses-to-CPU picture). torch.compile's inductor backend failed to build on GX10 (Triton/gcc CUDA-codegen toolchain issue) and fell back to eager cleanly per the existing warmup-fallback mechanism -- so the CUDA compile question is still open there. CPU's aot_eager backend engaged successfully and gave a small (~3%) win, opposite in sign from the Mac's ~23% CPU loss for the same backend on the same model. --- ...-13-compile-hardening-radar-image-audio.md | 26 +++++++++++++++++++ 1 file changed, 26 insertions(+) diff --git a/docs/superpowers/plans/2026-08-13-compile-hardening-radar-image-audio.md b/docs/superpowers/plans/2026-08-13-compile-hardening-radar-image-audio.md index 88106eb..f9aff53 100644 --- a/docs/superpowers/plans/2026-08-13-compile-hardening-radar-image-audio.md +++ b/docs/superpowers/plans/2026-08-13-compile-hardening-radar-image-audio.md @@ -302,3 +302,29 @@ git commit -m "feat: wire compile_model_if_enabled into audio_classification mai **Finding: `torch.compile` makes radar training slower on both devices, and does not close the CPU/MPS gap.** With compile enabled, CPU slows by ~23% (0.621s -> 0.765s/epoch) and MPS slows by ~22% (1.066s -> 1.303s/epoch). The CPU/MPS ratio is essentially unchanged (1.72x -> 1.70x) since both devices use the same `aot_eager` backend (the backend-selection logic in `compile_model_if_enabled` only routes CUDA to `inductor`; both CPU and MPS get `aot_eager`) and both slow down by a similar proportion. This is consistent with `LINEAR_4L_PC` being a tiny 4-layer linear/BatchNorm model: `torch.compile`'s per-call dynamo tracing, guard-checking, and graph-dispatch overhead is fixed cost per training step, and for a model this small there isn't enough per-op dispatch overhead in the eager path for kernel fusion to recoup that cost. The theory in this plan's "Known unresolved question" (fused kernels reducing dispatch overhead) does not hold for this model size — compile is a net loss here, not neutral or beneficial. Wiring it in is still correct (it makes `--compile-model` functional, matching the other reference scripts, and lets a caller opt in for larger/more compute-bound models where the tradeoff may differ), but the flag should not be turned on by default for radar's small linear models based on this evidence. + +**GX10 leg (controller-run, per Global Constraints):** same fixture/model/batch-size/epochs, `~/jupyterlab/.venv` (torch 2.9.0+cu130), `NVIDIA GB10`. GX10's clone was stale (`5035057`, pre-dating this whole plan) — updated to `origin/integration` (`e472749`) first. Environment needed several missing leaf packages installed (`--no-deps`, no `torch`/`torchvision` touched): `tabulate`, `torcheval`, `torchinfo`, `colorama`, `onnx`, `onnxruntime`, `protobuf`, `ml_dtypes`, `cryptography`, `PyWavelets`, `opencv-python`, `onnxscript`, `onnx_ir`. `cmsisdsp` and `torchaudio` were stubbed via `sys.modules` in the benchmark driver instead of installed for real — both are only needed transitively (by `timeseries_dataset.py`'s FFT path and `audio_dataset.py` respectively, eagerly imported by `datasets/__init__.py`) and never touched by radar's own code path; `cmsisdsp` in particular requires a multi-minute native CMSIS-DSP C build with no prebuilt ARM64 wheel, unrelated to what this benchmark measures. + +| Config | Device | s/epoch | +|---|---|---| +| No compile | CPU | 4.093 | +| No compile | CUDA | 0.856 | +| `--compile-model 1` | CPU | 3.972 | +| `--compile-model 1` | CUDA | 0.943 | + +CPU and CUDA aren't directly comparable to the Mac's CPU/MPS numbers (different, much less GX10-tuned CPU vs Apple Silicon CPU) — the useful comparison is each device against itself, with vs. without compile, on the same machine. + +**Finding: on GX10, CUDA beats CPU by 4.8x even with no compile** (0.856s vs 4.093s/epoch) — a completely different picture from the Mac, where MPS lost to CPU. GX10's GPU has enough throughput advantage over its CPU that it wins decisively on this tiny model without any fusion help. + +**Finding: `torch.compile` on CUDA never actually ran — it failed and silently fell back to eager.** The run log (`~/bench_radar/radar_out_cuda_compile1/run.log`) shows: +``` +INFO: root.main: Compiling model with torch.compile (backend=inductor) +WARNING: root.main: torch.compile failed (or failed its warmup pass), falling back to eager mode: +CalledProcessError: Command '['/usr/bin/gcc', '.../cuda_utils.c', '-O3', '-shared', '-fPIC', ... +-lcuda', '-L.../triton/backends/nvidia/lib', ...]' returned non-zero exit status 1. +``` +Triton's CUDA-kernel codegen fails to build `cuda_utils.c` via `gcc` on this box — a toolchain issue local to this environment (missing header/lib path for Triton's nvidia backend), not a `compile_model_if_enabled` defect. The `compile_model_if_enabled` warmup-and-fallback mechanism (built earlier this session, `docs/superpowers/plans/2026-07-28-compile-warmup-fallback.md`) caught the failure exactly as designed and fell back cleanly — training was not interrupted. This means the measured "compile=1, CUDA, 0.943s/epoch" number is **eager mode plus the one-time cost of a failed compile attempt**, not a real compiled-vs-uncompiled comparison; the CUDA compile question remains genuinely open on this hardware until the Triton/gcc toolchain issue is fixed here. + +**Finding: CPU's `aot_eager` backend did engage successfully on GX10** (`INFO: root.main: Compiling model with torch.compile (backend=aot_eager)`, no fallback warning) and produced a small, real win (4.093 -> 3.972s/epoch, ~3%) — consistent in direction with the Mac's CPU-side aot_eager numbers being a similarly fixed-cost/small-model regime, though the Mac's own CPU result was a ~23% *loss*, not a ~3% gain — the two machines don't even agree on aot_eager's sign for this model, underscoring how workload-and-hardware-specific this tradeoff is. + +**Net implication for the "should compile be on by default" question:** no clean "yes" or "no" emerges. Mac CPU/MPS: compile hurts on both. GX10 CPU: compile helps marginally. GX10 CUDA: unknown — the only device compile theoretically helps most (via `inductor`, not `aot_eager`) is the one device where it couldn't even run here. Recommend leaving `--compile-model` opt-in (its current default) rather than drawing a default-on conclusion from this data, and separately investigating the GX10 Triton/gcc build failure if CUDA compile behavior is worth knowing for real. From bf261a7ac241c5c4d4bce9032c814acce78a46ed Mon Sep 17 00:00:00 2001 From: M Platypus Date: Thu, 13 Aug 2026 22:14:41 -0400 Subject: [PATCH 06/14] feat: wire compile_model_if_enabled into image_classification main() --- ...-13-compile-hardening-radar-image-audio.md | 27 +++++++++++++++++++ ...test_image_classification_compile_wired.py | 13 +++++++++ .../references/image_classification/train.py | 2 ++ 3 files changed, 42 insertions(+) create mode 100644 tinyml-tinyverse/tests/test_image_classification_compile_wired.py diff --git a/docs/superpowers/plans/2026-08-13-compile-hardening-radar-image-audio.md b/docs/superpowers/plans/2026-08-13-compile-hardening-radar-image-audio.md index f9aff53..3bd95ab 100644 --- a/docs/superpowers/plans/2026-08-13-compile-hardening-radar-image-audio.md +++ b/docs/superpowers/plans/2026-08-13-compile-hardening-radar-image-audio.md @@ -328,3 +328,30 @@ Triton's CUDA-kernel codegen fails to build `cuda_utils.c` via `gcc` on this box **Finding: CPU's `aot_eager` backend did engage successfully on GX10** (`INFO: root.main: Compiling model with torch.compile (backend=aot_eager)`, no fallback warning) and produced a small, real win (4.093 -> 3.972s/epoch, ~3%) — consistent in direction with the Mac's CPU-side aot_eager numbers being a similarly fixed-cost/small-model regime, though the Mac's own CPU result was a ~23% *loss*, not a ~3% gain — the two machines don't even agree on aot_eager's sign for this model, underscoring how workload-and-hardware-specific this tradeoff is. **Net implication for the "should compile be on by default" question:** no clean "yes" or "no" emerges. Mac CPU/MPS: compile hurts on both. GX10 CPU: compile helps marginally. GX10 CUDA: unknown — the only device compile theoretically helps most (via `inductor`, not `aot_eager`) is the one device where it couldn't even run here. Recommend leaving `--compile-model` opt-in (its current default) rather than drawing a default-on conclusion from this data, and separately investigating the GX10 Triton/gcc build failure if CUDA compile behavior is worth knowing for real. + +--- + +## Results (Task 2: image_classification) + +**Wiring:** `compile_model_if_enabled` is now imported and called in `image_classification/train.py`'s `main()`, between `move_model_to_device` and `setup_distributed_model`, at the same indentation level as the surrounding `try:` block (one level deeper than radar/audio, per the brief's note). Regression test `tests/test_image_classification_compile_wired.py` passes; `--compile-model 0` (default) produces zero compile-related log lines and identical behavior to before this change; `--compile-model 1` produces `compile_model_if_enabled`'s own log line (`Compiling model with torch.compile (backend=aot_eager)`) on both CPU and MPS, with no fallback warnings on either device. + +**X vs X_raw judgment call:** at the insertion point (right after `move_model_to_device`), `model` is still the plain model returned by `models.get_model(args.model, variables, num_classes, input_features=input_features, ...)` (or `torch.load` for `--load-saved-model`), constructed from `variables = dataset.X.shape[1]` / `input_features = dataset.X.shape[2]` a few lines earlier (line 291-292) — confirmed by the existing `torchinfo.summary(model, summary_input_shape)` call at line 320-322, which also uses `dataset.X.shape[1:]` for this same pre-wrap model. The `nn_for_feature_extraction` / `X_raw` vs `X` branch the brief flagged only appears later (lines 342-350), where `model` gets wrapped in `NeuralNetworkWithPreprocess` — that wrapping happens *after* our insertion point, not before it. So regardless of `args.nn_for_feature_extraction`, the raw model being compiled at this point in `main()` always consumes `dataset.X.shape[1:]`-shaped input; the brief's suggested `input_shape=(1,) + dataset.X.shape[1:]` is correct unconditionally, and no branching on `nn_for_feature_extraction` was needed. + +**Fixture:** no ready-made image dataset fixture/driver existed in `tinyml-tinyverse/tests/` (the closest prior art, `tests/test_train_best_epoch_bugs_vision_audio.py`, fully mocks `get_model`/`create_data_loaders` and never drives real image data end-to-end). Built a small synthetic fixture instead: N class folders of 28x28 grayscale PNGs (`//*.png`, matching `GenericImageDataset`'s fallback-discovery layout — no `annotations/*_list.txt` needed since `--dataset` stays at its `folder` default, not `modelmaker`), each class centered on a distinct mean pixel intensity plus noise. Model: `CNN_LENET5`, the smallest of the three registered image models (~a few hundred params vs. `CNN_IMG_MOBILENETV1/2_58K_NPU`'s ~58K), whose spec hard-codes a 28x28x1 input (`Linear(in_features=400, ...)` after two conv+pool blocks) — fixture images were generated at exactly that size. Driver script (`bench_image.py` in the session scratchpad) builds argv for `train.get_args_parser().parse_args(argv)` and times `train.run(args)`, same pattern as `bench_radar.py`. + +**Manual E2E check (Step 5):** 3-class fixture (12 images/class), 2 epochs, batch size 8. `--device cpu --compile-model 0` (both pre- and post-code-change): completes cleanly, exports `model.onnx`, zero compile-related log lines. `--device cpu --compile-model 1`: completes cleanly, exports `model.onnx`, log shows `INFO: root.main: Compiling model with torch.compile (backend=aot_eager)` with no fallback warning. + +**Benchmark (Step 6)** — 5-class fixture (20 images/class = 100 images), `CNN_LENET5`, batch size 16, 30 epochs, `--device cpu` / `--device mps` (`PYTORCH_ENABLE_MPS_FALLBACK=1`), each config run 3-4 times to check stability: + +| Config | Device | s/epoch (steady-state avg) | compile engaged? | +|---|---|---|---| +| `--compile-model 0` (baseline) | CPU | 0.037 (n=3, range 0.037-0.037) | n/a (no compile lines in log) | +| `--compile-model 0` (baseline) | MPS | 0.073 (n=3 steady-state, range 0.069-0.077) | n/a (no compile lines in log) | +| `--compile-model 1` | CPU | 0.083 (n=3, range 0.083-0.084) | yes, `aot_eager`, no fallback | +| `--compile-model 1` | MPS | 0.094 (n=3 steady-state, range 0.092-0.096) | yes, `aot_eager`, no fallback | + +**Finding: `torch.compile` makes image_classification training slower on both devices, same direction as radar, but with a smaller relative hit on MPS than on CPU.** CPU regresses ~124% (0.037s -> 0.083s/epoch) — proportionally worse than radar's ~23% CPU regression. MPS regresses only ~28% (0.073s -> 0.094s/epoch) — much milder than radar's ~22% MPS regression was *relatively* similar in magnitude, but here the CPU/MPS gap actually **narrows** with compile on: 0.073/0.037 = 1.98x (MPS slower, no compile) vs. 0.094/0.083 = 1.13x (MPS slower, with compile). CPU still wins in absolute terms either way, but compile shrinks rather than preserves the ratio — the opposite of radar, where the ratio stayed essentially flat (1.72x -> 1.70x). `CNN_LENET5` has real conv/pool ops (unlike radar's pure linear/BatchNorm stack), so there is at least some op-fusion opportunity for `aot_eager` to exploit on MPS, but for a model this tiny (~28x28 inputs, batch 16, a few hundred parameters) the fixed per-step dynamo tracing/guard overhead still dominates on both devices — it just dominates less on MPS relative to MPS's already-slower eager baseline. + +**Methodology note — MPS cold-start artifact:** the very first MPS invocation of the whole benchmark session (`--compile-model 0`, run 1) measured 0.171s/epoch, 2.3-4.6x slower than every subsequent MPS run (compile-0 or compile-1) in the same session, each a fresh process. All later MPS runs (4 for compile-0, 4 for compile-1, alternating) clustered tightly (0.069-0.077 and 0.090-0.096 respectively). This looks like a one-time cost paid on the first-ever MPS/Metal invocation in the session (e.g. Metal shader compilation cache being cold on disk, not per-process) rather than genuine compile-vs-no-compile signal, since it appeared on a `--compile-model 0` run. The table above reports the steady-state average (excluding that one outlier); the raw run-by-run numbers are preserved in the session scratchpad driver output for reference. This is the same class of measurement caveat Task 1 flagged for GX10 (environment-specific first-run cost, not part of the compile question itself) — worth keeping in mind for any future single-shot MPS benchmark on this or other modules. + +**Bottom line:** wiring `compile_model_if_enabled` into `image_classification` is correct and makes `--compile-model` functional for this script, matching the other reference scripts and Task 1's radar wiring. As with radar, the evidence here does not support turning `--compile-model` on by default for this small a model: it's a net loss on both CPU and MPS for `CNN_LENET5` at this input size/batch size, though the loss is proportionally smaller on MPS than on CPU (unlike radar, where both devices regressed by a similar percentage). A larger, more compute-bound image model (e.g. `CNN_IMG_MOBILENETV1_58K_NPU`/`CNN_IMG_MOBILENETV2_58K_NPU`) was not benchmarked here (the brief calls for the smallest/fastest model to keep iteration time reasonable) and might show a different tradeoff, since more per-op dispatch overhead in eager mode gives kernel fusion more to recoup. diff --git a/tinyml-tinyverse/tests/test_image_classification_compile_wired.py b/tinyml-tinyverse/tests/test_image_classification_compile_wired.py new file mode 100644 index 0000000..66137e1 --- /dev/null +++ b/tinyml-tinyverse/tests/test_image_classification_compile_wired.py @@ -0,0 +1,13 @@ +"""Regression test: image_classification.train.main() must call +compile_model_if_enabled, matching the timeseries_* pattern (PR #22).""" +import inspect + +from tinyml_tinyverse.references.image_classification import train as image_train + + +def test_main_calls_compile_model_if_enabled(): + main_source = inspect.getsource(image_train.main) + assert "compile_model_if_enabled(" in main_source, ( + "main() does not call compile_model_if_enabled -- torch.compile/AMP " + "hardware acceleration would silently not apply to image classification training." + ) diff --git a/tinyml-tinyverse/tinyml_tinyverse/references/image_classification/train.py b/tinyml-tinyverse/tinyml_tinyverse/references/image_classification/train.py index c154cd2..369a06e 100644 --- a/tinyml-tinyverse/tinyml_tinyverse/references/image_classification/train.py +++ b/tinyml-tinyverse/tinyml_tinyverse/references/image_classification/train.py @@ -109,6 +109,7 @@ save_checkpoint, handle_export_only, move_model_to_device, + compile_model_if_enabled, log_training_time, apply_output_int_default, get_output_int_flag, @@ -327,6 +328,7 @@ def main(gpu, args): return move_model_to_device(model, device, logger) + model = compile_model_if_enabled(model, args, logger, input_shape=(1,) + dataset.X.shape[1:]) criterion = nn.CrossEntropyLoss(label_smoothing=args.label_smoothing) model, model_without_ddp, model_ema = setup_distributed_model(model, args, device) From 729cbe44138793677ad86fec045bb383829c8599 Mon Sep 17 00:00:00 2001 From: M Platypus Date: Thu, 13 Aug 2026 22:19:26 -0400 Subject: [PATCH 07/14] docs: add GX10 (CUDA) benchmark leg for Task 2 image_classification compile wiring CUDA still beats CPU on GX10 without compile, but only 1.3x here (vs radar's 4.8x) -- CNN_LENET5's conv/pool ops give GX10's CPU real work to do, unlike radar's pure linear stack. Same Triton/gcc inductor build failure reproduces identically to Task 1, confirming it's an environment-level GX10 toolchain issue, not model-specific. CPU aot_eager regressed ~104% here, a bigger relative hit than either machine saw for radar's smaller model. --- ...6-08-13-compile-hardening-radar-image-audio.md | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/docs/superpowers/plans/2026-08-13-compile-hardening-radar-image-audio.md b/docs/superpowers/plans/2026-08-13-compile-hardening-radar-image-audio.md index 3bd95ab..b00484e 100644 --- a/docs/superpowers/plans/2026-08-13-compile-hardening-radar-image-audio.md +++ b/docs/superpowers/plans/2026-08-13-compile-hardening-radar-image-audio.md @@ -355,3 +355,18 @@ Triton's CUDA-kernel codegen fails to build `cuda_utils.c` via `gcc` on this box **Methodology note — MPS cold-start artifact:** the very first MPS invocation of the whole benchmark session (`--compile-model 0`, run 1) measured 0.171s/epoch, 2.3-4.6x slower than every subsequent MPS run (compile-0 or compile-1) in the same session, each a fresh process. All later MPS runs (4 for compile-0, 4 for compile-1, alternating) clustered tightly (0.069-0.077 and 0.090-0.096 respectively). This looks like a one-time cost paid on the first-ever MPS/Metal invocation in the session (e.g. Metal shader compilation cache being cold on disk, not per-process) rather than genuine compile-vs-no-compile signal, since it appeared on a `--compile-model 0` run. The table above reports the steady-state average (excluding that one outlier); the raw run-by-run numbers are preserved in the session scratchpad driver output for reference. This is the same class of measurement caveat Task 1 flagged for GX10 (environment-specific first-run cost, not part of the compile question itself) — worth keeping in mind for any future single-shot MPS benchmark on this or other modules. **Bottom line:** wiring `compile_model_if_enabled` into `image_classification` is correct and makes `--compile-model` functional for this script, matching the other reference scripts and Task 1's radar wiring. As with radar, the evidence here does not support turning `--compile-model` on by default for this small a model: it's a net loss on both CPU and MPS for `CNN_LENET5` at this input size/batch size, though the loss is proportionally smaller on MPS than on CPU (unlike radar, where both devices regressed by a similar percentage). A larger, more compute-bound image model (e.g. `CNN_IMG_MOBILENETV1_58K_NPU`/`CNN_IMG_MOBILENETV2_58K_NPU`) was not benchmarked here (the brief calls for the smallest/fastest model to keep iteration time reasonable) and might show a different tradeoff, since more per-op dispatch overhead in eager mode gives kernel fusion more to recoup. + +**GX10 leg (controller-run):** same fixture/model/batch-size, 30 epochs, `~/jupyterlab/.venv` (torch 2.9.0+cu130), `NVIDIA GB10`. GX10's clone fast-forwarded cleanly to `bf261a7` this time (no reset needed, unlike Task 1's stale-clone situation). Same `cmsisdsp`/`torchaudio` `sys.modules` stubs as Task 1 (still unrelated to image_classification's own code path); no new missing packages this time — everything Task 1 installed already covered it. + +| Config | Device | s/epoch | compile engaged? | +|---|---|---|---| +| No compile | CPU | 0.097 | n/a | +| No compile | CUDA | 0.074 | n/a | +| `--compile-model 1` | CPU | 0.198 | yes, `aot_eager`, no fallback | +| `--compile-model 1` | CUDA | 0.113 | attempted `inductor`, failed, fell back to eager (same Triton/gcc `cuda_utils.c` build failure as Task 1) | + +**Finding: CUDA beats CPU on GX10 even without compile, but only by 1.3x here** (0.074 vs 0.097s/epoch) — a much smaller margin than radar's 4.8x. `CNN_LENET5`'s conv/pool ops give GX10's CPU (a many-core Grace ARM chip, not a phone-class CPU) enough to work with that the GPU's advantage narrows considerably for a model this tiny, compared to radar's pure-linear workload. + +**Finding: `aot_eager` on CPU is a bigger loss on GX10 than on the Mac for this model** — 0.097 -> 0.198s/epoch is a ~104% regression on GX10's CPU, roughly consistent in direction and rough magnitude with the Mac's own CPU regression for image_classification (~124%) and both are much worse than either machine's CPU regression for radar. Small-CNN `aot_eager` compilation overhead appears to be a bigger relative cost than small-linear-net overhead was, on both machines. + +**Finding: the same CUDA/Triton/gcc build failure from Task 1 reproduces identically here** — confirms it's an environment-level GX10 toolchain issue (Triton's nvidia backend `cuda_utils.c` failing to compile via `gcc`), not something specific to radar's model or code path. This will block any real `inductor`-backend measurement on GX10 across all modules until fixed there; worth investigating separately if CUDA compile behavior needs to be known for real, rather than re-discovering the same failure in Task 3's audio benchmark. From faad5169cb01cf88ffbd324d4a801d426e4dba04 Mon Sep 17 00:00:00 2001 From: M Platypus Date: Thu, 13 Aug 2026 22:35:10 -0400 Subject: [PATCH 08/14] feat: wire compile_model_if_enabled into audio_classification main() --- ...-13-compile-hardening-radar-image-audio.md | 31 +++++++++++++++++++ ...test_audio_classification_compile_wired.py | 13 ++++++++ .../references/audio_classification/train.py | 2 ++ 3 files changed, 46 insertions(+) create mode 100644 tinyml-tinyverse/tests/test_audio_classification_compile_wired.py diff --git a/docs/superpowers/plans/2026-08-13-compile-hardening-radar-image-audio.md b/docs/superpowers/plans/2026-08-13-compile-hardening-radar-image-audio.md index b00484e..bb3bf16 100644 --- a/docs/superpowers/plans/2026-08-13-compile-hardening-radar-image-audio.md +++ b/docs/superpowers/plans/2026-08-13-compile-hardening-radar-image-audio.md @@ -370,3 +370,34 @@ Triton's CUDA-kernel codegen fails to build `cuda_utils.c` via `gcc` on this box **Finding: `aot_eager` on CPU is a bigger loss on GX10 than on the Mac for this model** — 0.097 -> 0.198s/epoch is a ~104% regression on GX10's CPU, roughly consistent in direction and rough magnitude with the Mac's own CPU regression for image_classification (~124%) and both are much worse than either machine's CPU regression for radar. Small-CNN `aot_eager` compilation overhead appears to be a bigger relative cost than small-linear-net overhead was, on both machines. **Finding: the same CUDA/Triton/gcc build failure from Task 1 reproduces identically here** — confirms it's an environment-level GX10 toolchain issue (Triton's nvidia backend `cuda_utils.c` failing to compile via `gcc`), not something specific to radar's model or code path. This will block any real `inductor`-backend measurement on GX10 across all modules until fixed there; worth investigating separately if CUDA compile behavior needs to be known for real, rather than re-discovering the same failure in Task 3's audio benchmark. + +--- + +## Results (Task 3: audio_classification) + +**Wiring:** `compile_model_if_enabled` is now imported and called in `audio_classification/train.py`'s `main()`, between `move_model_to_device` and `setup_distributed_model`, at the same indentation/position as radar's wiring (no extra `try:` nesting, unlike image_classification). Regression test `tests/test_audio_classification_compile_wired.py` passes; `--compile-model 0` (default) produces zero compile-related log lines and identical behavior to before this change; `--compile-model 1` produces `compile_model_if_enabled`'s own log line (`Compiling model with torch.compile (backend=aot_eager)`) on both CPU and MPS, with no fallback warnings on either device. + +**X vs X_raw judgment call:** same resolution as Task 2, re-verified by reading `main()` directly rather than assumed. At the insertion point (right after `move_model_to_device`), `model` is still the plain model returned by `models.get_model(args.model, variables, num_classes, input_features=input_features, ...)` (or `torch.load` for `--load-saved-model`), built from `variables = dataset.X.shape[1]` / `input_features = tuple(dataset.X.shape[2:])` a few lines earlier (line 274-276) — confirmed by the existing `torchinfo.summary(model, summary_input_shape)` call at line 302-304 (gated on `args.generic_model or args.nas_enabled`), which also uses `dataset.X.shape[1:]` for this same pre-wrap model. The `nn_for_feature_extraction` / `X_raw` vs `X` branch the brief flagged only appears later (lines 322-331), where `model` gets wrapped in `NeuralNetworkWithPreprocess` (and, when `args.nn_for_feature_extraction` is set, a separately-trained `FEModelLinear` feature extractor consuming `dataset.X_raw`) — that wrapping happens *after* our insertion point, not before it, exactly as in Task 2. The `X_raw`-shaped input only matters again much later, at export time (line 424, `input_shape = (1,) + dataset.X_raw.shape[1:]` when `nn_for_feature_extraction` is set) — a separate, later insertion point this task doesn't touch. So regardless of `args.nn_for_feature_extraction`, the raw model being compiled at this point in `main()` always consumes `dataset.X.shape[1:]`-shaped input; the brief's `input_shape=(1,) + dataset.X.shape[1:]` is correct unconditionally, no branching needed. + +**Fixture:** no existing audio fixture/driver in `tinyml-tinyverse/tests/` drives real audio data end-to-end (the closest prior art, `tests/test_train_best_epoch_bugs_vision_audio.py`, fully mocks `get_model`/`create_data_loaders`/etc. for both image and audio and never touches a real `GoogleSpeechCommandsDataset`). Built a small synthetic fixture instead: N class folders of 1-second, 16kHz mono WAV clips (`//*.wav`, matching `GoogleSpeechCommandsDataset`'s fallback-discovery layout — `glob(/*/*.wav)`, no `annotations/*_list.txt` needed since `--dataset` stays at its `folder` default), each class a distinct sine-tone frequency (200-3000 Hz, evenly spaced) plus Gaussian noise, generated with `soundfile` (confirmed `torchaudio.load` reads these back correctly with the installed `soundfile` backend — `torchaudio.list_audio_backends()` returns `['soundfile']` in this env). Model: `CNN_AUDIO_DSCNN`, the *only* model registered for `audio_classification` in `tinyml-modelzoo` (`tinyml_modelzoo/models/audio.py`) — a depthwise-separable-conv DSCNN (7 `Conv2d` layers, several depthwise, BatchNorm/ReLU/Dropout between them, ~23,171 parameters at default `filters=64`), a materially different op mix from radar's pure linear/BatchNorm stack and image's single small non-separable CNN. Its documented expected input is `(N, 1, 49, 10)`; the default audio CLI params (`sample_rate=16000`, `audio_duration_ms=1000` -> 16000 samples, `n_mfcc=10`, `frame_length_ms=30` -> `n_fft=480`, `frame_step_ms=20` -> `hop_length=320`) produce exactly 49 MFCC time frames x 10 coefficients with `center=False`, so no CLI overrides were needed to hit the model's expected shape. Driver script (`bench_audio.py` in the session scratchpad) builds argv for `train.get_args_parser().parse_args(argv)` and times `train.run(args)`, same pattern as `bench_radar.py`/`bench_image.py`. + +**Methodology finding — `--sampling-rate` vs `--sample-rate` naming collision (pre-existing, not part of this task's fix):** while building the fixture, an initial run with `--sampling-rate 1.0` (the value Tasks 1/2 used for the base parser's generic, audio-irrelevant, `required=True` FFT `--sampling-rate` arg) crashed inside `torchaudio`'s MFCC transform with `RuntimeError: stft(...) : expected 0 < n_fft < 1, but got n_fft=0`. Root cause: `GoogleSpeechCommandsDataset.__init__` does `for key, value in kwargs.items(): setattr(self, key, ...)` over every CLI arg passed via `**vars(args)`, then reads `self.sampling_rate = int(getattr(self, "sampling_rate", 16000))` — but the *base* parser's generic `--sampling-rate` (dest `sampling_rate`, meant for radar/timeseries FFT preprocessing) collides with and silently overwrites the dataset's own `sampling_rate` attribute, while `audio_classification`'s own `--sample-rate` flag (dest `sample_rate`, no "ing") is never read by the dataset at all — it sets an unused attribute. With `--sampling-rate 1.0`, the dataset computed `n_fft = int(1 * 30 / 1000) = 0`, crashing MFCC extraction. Worked around by passing `--sampling-rate 16000` (the real audio sample rate) instead of `1.0` for this benchmark. This is a genuine, pre-existing naming/wiring bug in the audio dataset loader unrelated to `compile_model_if_enabled` and outside this task's file list (`audio_dataset.py`, not `audio_classification/train.py`) — not fixed here, flagged for separate follow-up. + +**Manual E2E check (Step 5):** 3-class fixture (12 clips/class), 2 epochs, batch size 16, `--sampling-rate 16000`. `--device cpu --compile-model 0` (both pre- and post-code-change): completes cleanly, exports `model.onnx`, zero compile-related log lines. `--device cpu --compile-model 1`: completes cleanly, exports `model.onnx`, log shows `INFO: root.main: Compiling model with torch.compile (backend=aot_eager)` with no fallback warning. `--device mps --compile-model 0` and `--compile-model 1` (`PYTORCH_ENABLE_MPS_FALLBACK=1`): both complete cleanly and export `model.onnx`; compile-1 shows the same `aot_eager` compile line, no `compile_model_if_enabled` fallback warning, but does show a PyTorch `UserWarning` that `aten::native_dropout` isn't supported on MPS and falls back to CPU for that op — see the benchmark finding below. + +**Benchmark (Step 6)** — 5-class fixture (20 clips/class = 100 clips), `CNN_AUDIO_DSCNN`, batch size 16, 30 epochs, `--device cpu` / `--device mps` (`PYTORCH_ENABLE_MPS_FALLBACK=1`), each config run 3-4 times to check stability (whole-`train.run()` timing, same methodology as Tasks 1-2, so includes one-time dataset-load/MFCC-extraction and ONNX-export overhead amortized over 30 epochs): + +| Config | Device | s/epoch (avg) | compile engaged? | +|---|---|---|---| +| `--compile-model 0` (baseline) | CPU | 1.242 (n=3, range 1.222-1.254) | n/a (no compile lines in log) | +| `--compile-model 0` (baseline) | MPS | 0.110 (n=4, range 0.108-0.114) | n/a (no compile lines in log) | +| `--compile-model 1` | CPU | 1.186 (n=3, range 1.170-1.204) | yes, `aot_eager`, no fallback | +| `--compile-model 1` | MPS | 0.207 (n=4, range 0.201-0.211) | yes, `aot_eager`, no fallback (but see dropout finding below) | + +**Finding: audio_classification matches neither radar's nor image's pattern — MPS massively outperforms CPU here, the opposite of both prior tasks, and `torch.compile` is a small net win on CPU but a large net loss on MPS.** Without compile, MPS is ~11.3x *faster* than CPU (0.110s vs 1.242s/epoch) — a complete inversion of radar (MPS 1.7x slower) and image (MPS ~2x slower). A quick 1-epoch-vs-30-epoch differential run confirms this isn't a fixed-cost artifact: isolating the one-time dataset-load overhead (~0.28s CPU, ~0.58s MPS) from steady-state per-epoch cost still gives CPU ≈1.21s/epoch vs MPS ≈0.09s/epoch, a ~13x native throughput gap. `CNN_AUDIO_DSCNN` is a real, moderately-sized conv-heavy model (23,171 params, 7 `Conv2d` layers incl. depthwise) operating on a 49x10 feature map at batch 16 — enough actual matmul/convolution work that Apple's GPU wins decisively, unlike radar's pure-linear workload or image's tiny few-hundred-parameter `CNN_LENET5`. + +With compile enabled, CPU improves slightly (1.242 -> 1.186s/epoch, ~-4.5%) — the only device across all three tasks where `torch.compile` was a net *win*, however small — while MPS regresses sharply (0.110 -> 0.207s/epoch, ~+88%). This **narrows** the CPU/MPS gap the same direction as image_classification did (11.3x -> 5.7x) but starting from the opposite baseline (MPS ahead, not behind). The MPS regression has a concrete, observed mechanism rather than just inferred dynamo/guard overhead: the `--compile-model 1` MPS log (and only that log — the `--compile-model 0` MPS log has zero such warnings, despite the model using the same `Dropout(0.2)`/`Dropout(0.4)` layers in both cases) shows `UserWarning: The operator 'aten::native_dropout' is not currently supported on the MPS backend and will fall back to run on the CPU`. This means compiling routes dropout through a lower-level decomposition (`aten::native_dropout`, likely via dynamo/AOTAutograd's graph capture) that lacks an MPS kernel, forcing a real CPU round-trip on every forward pass in the compiled path — whereas eager mode's `F.dropout` apparently dispatches through a path with native MPS support. A finer-grained 1-epoch-vs-30-epoch differential on the compiled MPS runs suggests the one-time `torch.compile` warmup cost (~1.4s) is also non-trivial relative to the 30-epoch total, so the naive 30-epoch-average ~88% regression somewhat overstates the true steady-state per-epoch cost (~56% by that isolation) — still a clear, large loss either way, just with the same one-time-warmup-dilution caveat Task 2 flagged for its MPS cold-start artifact. + +**Bottom line:** wiring `compile_model_if_enabled` into `audio_classification` is correct and makes `--compile-model` functional for this script, matching all six other reference scripts. Unlike radar and image_classification, the evidence here does *not* uniformly argue against compile: CPU sees a small, consistent win, and the MPS regression has an identified, addressable-in-principle cause (a dropout decomposition lacking an MPS kernel under `torch.compile`) rather than being an unavoidable property of small-model dynamo overhead. Still, the MPS loss is large enough (and CPU's win small enough) that there's no case for flipping the opt-in default here either — `--compile-model` should stay opt-in, as it does today, but audio_classification is the first of the three tasks where "does compile help" doesn't have a uniformly negative answer, and the dropout/MPS-kernel angle is a concrete, non-speculative lead if `torch.compile` + MPS support for this workload is ever revisited. + +**GX10 leg:** not run for this task — the Task 3 brief's Step 6 scopes the benchmark to "same methodology as Task 2 Step 6" (the Mac CPU/MPS 4-way comparison); the GX10 legs recorded under Tasks 1 and 2 were performed by a separate controller-run process outside this task's instructions, not part of this subagent's assigned steps. diff --git a/tinyml-tinyverse/tests/test_audio_classification_compile_wired.py b/tinyml-tinyverse/tests/test_audio_classification_compile_wired.py new file mode 100644 index 0000000..d0e919e --- /dev/null +++ b/tinyml-tinyverse/tests/test_audio_classification_compile_wired.py @@ -0,0 +1,13 @@ +"""Regression test: audio_classification.train.main() must call +compile_model_if_enabled, matching the timeseries_* pattern (PR #22).""" +import inspect + +from tinyml_tinyverse.references.audio_classification import train as audio_train + + +def test_main_calls_compile_model_if_enabled(): + main_source = inspect.getsource(audio_train.main) + assert "compile_model_if_enabled(" in main_source, ( + "main() does not call compile_model_if_enabled -- torch.compile/AMP " + "hardware acceleration would silently not apply to audio classification training." + ) diff --git a/tinyml-tinyverse/tinyml_tinyverse/references/audio_classification/train.py b/tinyml-tinyverse/tinyml_tinyverse/references/audio_classification/train.py index d7d3deb..7d019f4 100644 --- a/tinyml-tinyverse/tinyml_tinyverse/references/audio_classification/train.py +++ b/tinyml-tinyverse/tinyml_tinyverse/references/audio_classification/train.py @@ -109,6 +109,7 @@ save_checkpoint, handle_export_only, move_model_to_device, + compile_model_if_enabled, log_training_time, apply_output_int_default, get_output_int_flag, @@ -309,6 +310,7 @@ def main(gpu, args): return move_model_to_device(model, device, logger) + model = compile_model_if_enabled(model, args, logger, input_shape=(1,) + dataset.X.shape[1:]) criterion = nn.CrossEntropyLoss(label_smoothing=args.label_smoothing) model, model_without_ddp, model_ema = setup_distributed_model(model, args, device) optimizer, lr_scheduler = setup_optimizer_and_scheduler(model, args) From 43fc13a1cc704a4c89d07f531c97435282442632 Mon Sep 17 00:00:00 2001 From: M Platypus Date: Thu, 13 Aug 2026 22:42:43 -0400 Subject: [PATCH 09/14] docs: add GX10 (CUDA) benchmark leg for Task 3 audio_classification compile wiring CUDA beats CPU 2.2x on GX10 without compile. GX10's CPU aot_eager is a clear net loss (+75%) for audio, flipping sign from the Mac's small CPU win -- the first case where the same backend/model combo disagrees between machines. Same Triton/gcc inductor build failure reproduces for the third module in a row, confirming it's systemic to this GX10 environment. Needed real torchaudio==2.9.0 (matching torch's version, unlike 2.11.0's ABI mismatch) plus a soundfile-backed load() patch in the benchmark driver only, since torchaudio's default torchcodec backend needs FFmpeg, which GX10 doesn't have installed. --- ...08-13-compile-hardening-radar-image-audio.md | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/docs/superpowers/plans/2026-08-13-compile-hardening-radar-image-audio.md b/docs/superpowers/plans/2026-08-13-compile-hardening-radar-image-audio.md index bb3bf16..53ed2ce 100644 --- a/docs/superpowers/plans/2026-08-13-compile-hardening-radar-image-audio.md +++ b/docs/superpowers/plans/2026-08-13-compile-hardening-radar-image-audio.md @@ -400,4 +400,21 @@ With compile enabled, CPU improves slightly (1.242 -> 1.186s/epoch, ~-4.5%) — **Bottom line:** wiring `compile_model_if_enabled` into `audio_classification` is correct and makes `--compile-model` functional for this script, matching all six other reference scripts. Unlike radar and image_classification, the evidence here does *not* uniformly argue against compile: CPU sees a small, consistent win, and the MPS regression has an identified, addressable-in-principle cause (a dropout decomposition lacking an MPS kernel under `torch.compile`) rather than being an unavoidable property of small-model dynamo overhead. Still, the MPS loss is large enough (and CPU's win small enough) that there's no case for flipping the opt-in default here either — `--compile-model` should stay opt-in, as it does today, but audio_classification is the first of the three tasks where "does compile help" doesn't have a uniformly negative answer, and the dropout/MPS-kernel angle is a concrete, non-speculative lead if `torch.compile` + MPS support for this workload is ever revisited. +**GX10 leg (controller-run):** same fixture/model/batch-size, 30 epochs, `~/jupyterlab/.venv` (torch 2.9.0+cu130), `NVIDIA GB10`. Clone fast-forwarded cleanly to `faad516`. Environment needed real (not stubbed) `torchaudio` this time, since `GoogleSpeechCommandsDataset` genuinely calls `torchaudio.load`/`MFCC`/`resample` (unlike radar/image, where `cmsisdsp`/`torchaudio` were only pulled in as unused eager imports and safely stubbed via `sys.modules`). `pip install torchaudio` grabbed 2.11.0 by default — ABI-incompatible with torch 2.9.0 (`undefined symbol: torch_library_impl` on import). Pinning `torchaudio==2.9.0` matched cleanly, but its default audio backend (`torchcodec`) requires system FFmpeg, which isn't installed on GX10 and isn't worth adding just for this benchmark. Worked around by monkey-patching `torchaudio.load` in the GX10 driver script only (not the repo) to read via the already-installed `soundfile` instead — this only changes how the benchmark's WAV fixture gets loaded into a tensor, not any training/compile code path, so it doesn't affect what's being measured. + +| Config | Device | s/epoch | compile engaged? | +|---|---|---|---| +| No compile | CPU | 0.174 | n/a | +| No compile | CUDA | 0.079 | n/a | +| `--compile-model 1` | CPU | 0.305 | yes, `aot_eager`, no fallback | +| `--compile-model 1` | CUDA | 0.141 | attempted `inductor`, failed (same Triton/gcc `cuda_utils.c` build failure as Tasks 1 and 2), fell back to eager | + +**Finding: CUDA beats CPU 2.2x on GX10 without compile** (0.079 vs 0.174s/epoch) — between radar's 4.8x and image's 1.3x, consistent with `CNN_AUDIO_DSCNN`'s real conv workload giving the GPU a clear but not overwhelming edge over GX10's capable CPU. + +**Finding: unlike the Mac, GX10's CPU `aot_eager` backend is a clear net LOSS for audio (+75%, 0.174 -> 0.305s/epoch) — the opposite sign from the Mac's small CPU win (~-4.5%).** This is the first case across all three tasks and both machines where the same model/backend combination flips sign between machines. Since GX10's CUDA compile attempt never actually ran (Triton/gcc failure, same as every other module on this box), there's no GX10 MPS-dropout-style mechanism to check on the GPU side — CUDA does have a native dropout kernel, so the MPS-specific finding from the Mac leg doesn't apply here regardless. + +**Finding: the Triton/gcc `inductor` build failure reproduces identically for the third module in a row** — now confirmed systemic across all of radar, image_classification, and audio_classification on this GX10 environment, not tied to any particular model or code path. + +**Net across all three tasks and two machines: no single "compile helps" or "compile hurts" story holds everywhere.** Radar and image lose on both Mac devices; audio wins narrowly on Mac CPU and loses hard on Mac MPS (with a diagnosed cause); GX10 CPU flips sign between image/radar-style modules and audio; GX10 CUDA's real compile behavior remains entirely unmeasured across all three modules due to one shared environment issue. `--compile-model` staying opt-in (its current default) is the only conclusion that holds up across all of this data. + **GX10 leg:** not run for this task — the Task 3 brief's Step 6 scopes the benchmark to "same methodology as Task 2 Step 6" (the Mac CPU/MPS 4-way comparison); the GX10 legs recorded under Tasks 1 and 2 were performed by a separate controller-run process outside this task's instructions, not part of this subagent's assigned steps. From 0a7cce707639f0b8b22d6776ad0f9f7824d32cb9 Mon Sep 17 00:00:00 2001 From: M Platypus Date: Thu, 13 Aug 2026 22:55:06 -0400 Subject: [PATCH 10/14] docs: fix whole-plan review findings in compile-hardening results Whole-plan review found the code across all 3 tasks clean but flagged several doc-only defects: a stale leftover paragraph contradicting the real GX10 section above it, a synthesis claim with the GX10 CPU win/loss grouping backwards, a false "first sign-flip" claim that contradicted Task 1's own recorded radar sign-flip, an imprecise safety argument for the audio torchaudio.load->soundfile patch, two contradicting ad-hoc mechanistic explanations for GPU-vs-CPU advantage, and one garbled/self-contradicting sentence. Fixed all of the above inline, replaced the synthesis with a data table (9 aot_eager measurements) and the stronger, better-supported summary the review recommended, and empirically closed the one open risk (checked the GX10 audio run's logged accuracy: 100%, confirming the soundfile patch didn't corrupt input). Added a progress ledger and spun out the two follow-up items the review flagged (sampling-rate/sample-rate naming collision, GX10 Triton/gcc inductor build failure) as separately tracked rather than left buried in a closed plan. --- ...-13-compile-hardening-radar-image-audio.md | 28 +++++++++++++------ 1 file changed, 20 insertions(+), 8 deletions(-) diff --git a/docs/superpowers/plans/2026-08-13-compile-hardening-radar-image-audio.md b/docs/superpowers/plans/2026-08-13-compile-hardening-radar-image-audio.md index 53ed2ce..7250a9c 100644 --- a/docs/superpowers/plans/2026-08-13-compile-hardening-radar-image-audio.md +++ b/docs/superpowers/plans/2026-08-13-compile-hardening-radar-image-audio.md @@ -35,6 +35,8 @@ Rather than fix radar alone and leave image/audio in the same state, this plan c **Known unresolved question, not assumed:** whether `torch.compile` actually helps on hardware this small (radar's `LINEAR_4L_PC` is a 4-layer linear/BatchNorm model — the radar plan measured MPS at ~1.9x slower than CPU with no compile, and the theory is that fused kernels reduce per-op dispatch overhead, but this has not been confirmed for a model this size). Each task's benchmark step exists specifically to answer this per-module — report what you measure, not what the theory predicts. +**Scope note (added after whole-plan review):** `compile_model_if_enabled` returns early, unmodified, whenever `args.quantization` is set (`train_base.py:727-739`) — all benchmarks in this plan used `--quantization 0` (the default), so every number below speaks only to float training; the QAT path is untouched by this wiring, by design. Separately, `PYTORCH_ENABLE_MPS_FALLBACK=1` (needed for `compile_model_if_enabled`'s MPS fallback warnings to be warnings rather than hard `NotImplementedError`s) is set via `os.environ.setdefault` at `tinyml-modelmaker`'s entry points, not inside `tinyml-tinyverse` itself — the supported path (through modelmaker) is unaffected, but a user invoking any of these `references/*/train.py` scripts directly with `--device mps --compile-model 1` will now hard-fail on an unsupported op instead of falling back, where the flag was previously inert (compile was never reachable at all). + --- ## File Map @@ -325,7 +327,7 @@ CalledProcessError: Command '['/usr/bin/gcc', '.../cuda_utils.c', '-O3', '-share ``` Triton's CUDA-kernel codegen fails to build `cuda_utils.c` via `gcc` on this box — a toolchain issue local to this environment (missing header/lib path for Triton's nvidia backend), not a `compile_model_if_enabled` defect. The `compile_model_if_enabled` warmup-and-fallback mechanism (built earlier this session, `docs/superpowers/plans/2026-07-28-compile-warmup-fallback.md`) caught the failure exactly as designed and fell back cleanly — training was not interrupted. This means the measured "compile=1, CUDA, 0.943s/epoch" number is **eager mode plus the one-time cost of a failed compile attempt**, not a real compiled-vs-uncompiled comparison; the CUDA compile question remains genuinely open on this hardware until the Triton/gcc toolchain issue is fixed here. -**Finding: CPU's `aot_eager` backend did engage successfully on GX10** (`INFO: root.main: Compiling model with torch.compile (backend=aot_eager)`, no fallback warning) and produced a small, real win (4.093 -> 3.972s/epoch, ~3%) — consistent in direction with the Mac's CPU-side aot_eager numbers being a similarly fixed-cost/small-model regime, though the Mac's own CPU result was a ~23% *loss*, not a ~3% gain — the two machines don't even agree on aot_eager's sign for this model, underscoring how workload-and-hardware-specific this tradeoff is. +**Finding: CPU's `aot_eager` backend did engage successfully on GX10** (`INFO: root.main: Compiling model with torch.compile (backend=aot_eager)`, no fallback warning) and produced a small apparent win (4.093 -> 3.972s/epoch, ~3%) — versus the Mac's own CPU result, a clear ~23% *loss* for the same model/backend. This is a single-shot measurement on both sides (see the whole-plan methodology caveat), and ~3% is well within the ~7-9% run-to-run wobble this session's benchmarking has otherwise documented, so treat this as "the two machines may not agree on aot_eager's sign for this model" rather than a confirmed win — it would need repeated runs on both machines to distinguish a real effect from noise. **Net implication for the "should compile be on by default" question:** no clean "yes" or "no" emerges. Mac CPU/MPS: compile hurts on both. GX10 CPU: compile helps marginally. GX10 CUDA: unknown — the only device compile theoretically helps most (via `inductor`, not `aot_eager`) is the one device where it couldn't even run here. Recommend leaving `--compile-model` opt-in (its current default) rather than drawing a default-on conclusion from this data, and separately investigating the GX10 Triton/gcc build failure if CUDA compile behavior is worth knowing for real. @@ -350,7 +352,7 @@ Triton's CUDA-kernel codegen fails to build `cuda_utils.c` via `gcc` on this box | `--compile-model 1` | CPU | 0.083 (n=3, range 0.083-0.084) | yes, `aot_eager`, no fallback | | `--compile-model 1` | MPS | 0.094 (n=3 steady-state, range 0.092-0.096) | yes, `aot_eager`, no fallback | -**Finding: `torch.compile` makes image_classification training slower on both devices, same direction as radar, but with a smaller relative hit on MPS than on CPU.** CPU regresses ~124% (0.037s -> 0.083s/epoch) — proportionally worse than radar's ~23% CPU regression. MPS regresses only ~28% (0.073s -> 0.094s/epoch) — much milder than radar's ~22% MPS regression was *relatively* similar in magnitude, but here the CPU/MPS gap actually **narrows** with compile on: 0.073/0.037 = 1.98x (MPS slower, no compile) vs. 0.094/0.083 = 1.13x (MPS slower, with compile). CPU still wins in absolute terms either way, but compile shrinks rather than preserves the ratio — the opposite of radar, where the ratio stayed essentially flat (1.72x -> 1.70x). `CNN_LENET5` has real conv/pool ops (unlike radar's pure linear/BatchNorm stack), so there is at least some op-fusion opportunity for `aot_eager` to exploit on MPS, but for a model this tiny (~28x28 inputs, batch 16, a few hundred parameters) the fixed per-step dynamo tracing/guard overhead still dominates on both devices — it just dominates less on MPS relative to MPS's already-slower eager baseline. +**Finding: `torch.compile` makes image_classification training slower on both devices, same direction as radar, but with a smaller relative hit on MPS than on CPU.** CPU regresses ~124% (0.037s -> 0.083s/epoch) — proportionally worse than radar's ~23% CPU regression. MPS regresses only ~29% (0.073s -> 0.094s/epoch) — close in magnitude to radar's ~22% MPS regression, but here the CPU/MPS gap actually **narrows** with compile on: 0.073/0.037 = 1.98x (MPS slower, no compile) vs. 0.094/0.083 = 1.13x (MPS slower, with compile). CPU still wins in absolute terms either way, but compile shrinks rather than preserves the ratio — the opposite of radar, where the ratio stayed essentially flat (1.72x -> 1.70x). This is driven by the CPU/MPS *asymmetry* (CPU regresses far more than MPS does), not by MPS itself improving — no causal mechanism for that asymmetry was established here (see the whole-plan review's note that a competing "conv work narrows the GPU's relative advantage" story doesn't hold up against the GX10 data below, so this is reported as an observed pattern, not an explained one). **Methodology note — MPS cold-start artifact:** the very first MPS invocation of the whole benchmark session (`--compile-model 0`, run 1) measured 0.171s/epoch, 2.3-4.6x slower than every subsequent MPS run (compile-0 or compile-1) in the same session, each a fresh process. All later MPS runs (4 for compile-0, 4 for compile-1, alternating) clustered tightly (0.069-0.077 and 0.090-0.096 respectively). This looks like a one-time cost paid on the first-ever MPS/Metal invocation in the session (e.g. Metal shader compilation cache being cold on disk, not per-process) rather than genuine compile-vs-no-compile signal, since it appeared on a `--compile-model 0` run. The table above reports the steady-state average (excluding that one outlier); the raw run-by-run numbers are preserved in the session scratchpad driver output for reference. This is the same class of measurement caveat Task 1 flagged for GX10 (environment-specific first-run cost, not part of the compile question itself) — worth keeping in mind for any future single-shot MPS benchmark on this or other modules. @@ -365,7 +367,7 @@ Triton's CUDA-kernel codegen fails to build `cuda_utils.c` via `gcc` on this box | `--compile-model 1` | CPU | 0.198 | yes, `aot_eager`, no fallback | | `--compile-model 1` | CUDA | 0.113 | attempted `inductor`, failed, fell back to eager (same Triton/gcc `cuda_utils.c` build failure as Task 1) | -**Finding: CUDA beats CPU on GX10 even without compile, but only by 1.3x here** (0.074 vs 0.097s/epoch) — a much smaller margin than radar's 4.8x. `CNN_LENET5`'s conv/pool ops give GX10's CPU (a many-core Grace ARM chip, not a phone-class CPU) enough to work with that the GPU's advantage narrows considerably for a model this tiny, compared to radar's pure-linear workload. +**Finding: CUDA beats CPU on GX10 even without compile, but only by 1.3x here** (0.074 vs 0.097s/epoch) — a much smaller margin than radar's 4.8x. Note this doesn't hold up as a simple "more conv work narrows the GPU's edge" story once Task 3's audio numbers are in: audio (also conv-heavy) shows a 2.2x GX10 GPU advantage, between radar's pure-linear 4.8x and image's 1.3x — the ordering is radar > audio > image, not monotonic in how conv-heavy the model is. Reported as an observed number, not an explained one. **Finding: `aot_eager` on CPU is a bigger loss on GX10 than on the Mac for this model** — 0.097 -> 0.198s/epoch is a ~104% regression on GX10's CPU, roughly consistent in direction and rough magnitude with the Mac's own CPU regression for image_classification (~124%) and both are much worse than either machine's CPU regression for radar. Small-CNN `aot_eager` compilation overhead appears to be a bigger relative cost than small-linear-net overhead was, on both machines. @@ -394,13 +396,15 @@ Triton's CUDA-kernel codegen fails to build `cuda_utils.c` via `gcc` on this box | `--compile-model 1` | CPU | 1.186 (n=3, range 1.170-1.204) | yes, `aot_eager`, no fallback | | `--compile-model 1` | MPS | 0.207 (n=4, range 0.201-0.211) | yes, `aot_eager`, no fallback (but see dropout finding below) | -**Finding: audio_classification matches neither radar's nor image's pattern — MPS massively outperforms CPU here, the opposite of both prior tasks, and `torch.compile` is a small net win on CPU but a large net loss on MPS.** Without compile, MPS is ~11.3x *faster* than CPU (0.110s vs 1.242s/epoch) — a complete inversion of radar (MPS 1.7x slower) and image (MPS ~2x slower). A quick 1-epoch-vs-30-epoch differential run confirms this isn't a fixed-cost artifact: isolating the one-time dataset-load overhead (~0.28s CPU, ~0.58s MPS) from steady-state per-epoch cost still gives CPU ≈1.21s/epoch vs MPS ≈0.09s/epoch, a ~13x native throughput gap. `CNN_AUDIO_DSCNN` is a real, moderately-sized conv-heavy model (23,171 params, 7 `Conv2d` layers incl. depthwise) operating on a 49x10 feature map at batch 16 — enough actual matmul/convolution work that Apple's GPU wins decisively, unlike radar's pure-linear workload or image's tiny few-hundred-parameter `CNN_LENET5`. +**Finding: audio_classification matches neither radar's nor image's pattern — MPS massively outperforms CPU here, the opposite of both prior tasks, and `torch.compile` is a small net win on CPU but a large net loss on MPS.** Without compile, MPS is ~11.3x *faster* than CPU (0.110s vs 1.242s/epoch) — a complete inversion of radar (MPS 1.7x slower) and image (MPS ~2x slower). A quick 1-epoch-vs-30-epoch differential run confirms this isn't a fixed-cost artifact: isolating the one-time dataset-load overhead (~0.28s CPU, ~0.58s MPS) from steady-state per-epoch cost still gives CPU ≈1.21s/epoch vs MPS ≈0.09s/epoch, a ~13x native throughput gap. `CNN_AUDIO_DSCNN` is a real, moderately-sized conv-heavy model (23,171 params, 7 `Conv2d` layers incl. depthwise) operating on a 49x10 feature map at batch 16, giving Apple's GPU real work to do. But "more conv work means the GPU wins more" doesn't hold up as a general story: GX10's own GPU-vs-CPU ratios for the identical three models are radar 4.8x > audio 2.2x > image 1.3x, not ordered by conv-heaviness. An equally plausible reading of this specific 11.3x gap is that Mac CPU is unusually slow at this workload rather than MPS being unusually fast at it — Mac CPU's audio number (1.242s/epoch) is 33.6x its own image number, while GX10 CPU's audio number is only 1.8x its own image number, a ~19x discrepancy in relative CPU behavior between the two machines for the same model. Both explanations are consistent with the data; this plan doesn't have enough evidence to pick one. With compile enabled, CPU improves slightly (1.242 -> 1.186s/epoch, ~-4.5%) — the only device across all three tasks where `torch.compile` was a net *win*, however small — while MPS regresses sharply (0.110 -> 0.207s/epoch, ~+88%). This **narrows** the CPU/MPS gap the same direction as image_classification did (11.3x -> 5.7x) but starting from the opposite baseline (MPS ahead, not behind). The MPS regression has a concrete, observed mechanism rather than just inferred dynamo/guard overhead: the `--compile-model 1` MPS log (and only that log — the `--compile-model 0` MPS log has zero such warnings, despite the model using the same `Dropout(0.2)`/`Dropout(0.4)` layers in both cases) shows `UserWarning: The operator 'aten::native_dropout' is not currently supported on the MPS backend and will fall back to run on the CPU`. This means compiling routes dropout through a lower-level decomposition (`aten::native_dropout`, likely via dynamo/AOTAutograd's graph capture) that lacks an MPS kernel, forcing a real CPU round-trip on every forward pass in the compiled path — whereas eager mode's `F.dropout` apparently dispatches through a path with native MPS support. A finer-grained 1-epoch-vs-30-epoch differential on the compiled MPS runs suggests the one-time `torch.compile` warmup cost (~1.4s) is also non-trivial relative to the 30-epoch total, so the naive 30-epoch-average ~88% regression somewhat overstates the true steady-state per-epoch cost (~56% by that isolation) — still a clear, large loss either way, just with the same one-time-warmup-dilution caveat Task 2 flagged for its MPS cold-start artifact. **Bottom line:** wiring `compile_model_if_enabled` into `audio_classification` is correct and makes `--compile-model` functional for this script, matching all six other reference scripts. Unlike radar and image_classification, the evidence here does *not* uniformly argue against compile: CPU sees a small, consistent win, and the MPS regression has an identified, addressable-in-principle cause (a dropout decomposition lacking an MPS kernel under `torch.compile`) rather than being an unavoidable property of small-model dynamo overhead. Still, the MPS loss is large enough (and CPU's win small enough) that there's no case for flipping the opt-in default here either — `--compile-model` should stay opt-in, as it does today, but audio_classification is the first of the three tasks where "does compile help" doesn't have a uniformly negative answer, and the dropout/MPS-kernel angle is a concrete, non-speculative lead if `torch.compile` + MPS support for this workload is ever revisited. -**GX10 leg (controller-run):** same fixture/model/batch-size, 30 epochs, `~/jupyterlab/.venv` (torch 2.9.0+cu130), `NVIDIA GB10`. Clone fast-forwarded cleanly to `faad516`. Environment needed real (not stubbed) `torchaudio` this time, since `GoogleSpeechCommandsDataset` genuinely calls `torchaudio.load`/`MFCC`/`resample` (unlike radar/image, where `cmsisdsp`/`torchaudio` were only pulled in as unused eager imports and safely stubbed via `sys.modules`). `pip install torchaudio` grabbed 2.11.0 by default — ABI-incompatible with torch 2.9.0 (`undefined symbol: torch_library_impl` on import). Pinning `torchaudio==2.9.0` matched cleanly, but its default audio backend (`torchcodec`) requires system FFmpeg, which isn't installed on GX10 and isn't worth adding just for this benchmark. Worked around by monkey-patching `torchaudio.load` in the GX10 driver script only (not the repo) to read via the already-installed `soundfile` instead — this only changes how the benchmark's WAV fixture gets loaded into a tensor, not any training/compile code path, so it doesn't affect what's being measured. +**GX10 leg (controller-run):** same fixture/model/batch-size, 30 epochs, `~/jupyterlab/.venv` (torch 2.9.0+cu130), `NVIDIA GB10`. Clone fast-forwarded cleanly to `faad516`. Environment needed real (not stubbed) `torchaudio` this time, since `GoogleSpeechCommandsDataset` genuinely calls `torchaudio.load`/`MFCC`/`resample` (unlike radar/image, where `cmsisdsp`/`torchaudio` were only pulled in as unused eager imports and safely stubbed via `sys.modules`). `pip install torchaudio` grabbed 2.11.0 by default — ABI-incompatible with torch 2.9.0 (`undefined symbol: torch_library_impl` on import). Pinning `torchaudio==2.9.0` matched cleanly, but its default audio backend (`torchcodec`) requires system FFmpeg, which isn't installed on GX10 and isn't worth adding just for this benchmark. Worked around by monkey-patching `torchaudio.load` in the GX10 driver script only (not the repo) to read via the already-installed `soundfile` instead. + +This patch does feed `_load_audio`'s MFCC feature extraction (`audio_dataset.py:280-305`), so it does affect `dataset.X`, not just I/O plumbing — the earlier claim that it "doesn't affect what's being measured" was imprecise. What actually protects the measurement: `_load_audio` unconditionally casts to `float32` (neutralizing `soundfile`'s float64 default) and peak-normalizes every clip to max-amplitude 1.0 (neutralizing any amplitude-scale difference from `torchaudio.load`'s own normalization), and since the fixture and the `--sampling-rate` arg both use 16kHz, the resampling branch never fires on either loader. The one risk that doesn't self-correct is channel orientation (`soundfile` returns frames-first, `torchaudio.load` returns channels-first) — a wrong-axis patch would silently collapse the mono-downmix step to a near-zero waveform with no exception. Checked empirically rather than left as an argument: the GX10 audio run's `run.log` shows `Acc@1 100.000` at the best epoch — well above the ~20% chance level for this 5-class fixture, confirming the model learned real, distinguishable signal and the patch was not silently corrupting input. (Separately: `s/epoch` timing, which is what's actually reported in the table below, is driven by tensor shapes fixed by the MFCC parameters, not by waveform content — so even a corrupted patch would not have moved the reported numbers, though it could have invalidated the accuracy figure.) | Config | Device | s/epoch | compile engaged? | |---|---|---|---| @@ -411,10 +415,18 @@ With compile enabled, CPU improves slightly (1.242 -> 1.186s/epoch, ~-4.5%) — **Finding: CUDA beats CPU 2.2x on GX10 without compile** (0.079 vs 0.174s/epoch) — between radar's 4.8x and image's 1.3x, consistent with `CNN_AUDIO_DSCNN`'s real conv workload giving the GPU a clear but not overwhelming edge over GX10's capable CPU. -**Finding: unlike the Mac, GX10's CPU `aot_eager` backend is a clear net LOSS for audio (+75%, 0.174 -> 0.305s/epoch) — the opposite sign from the Mac's small CPU win (~-4.5%).** This is the first case across all three tasks and both machines where the same model/backend combination flips sign between machines. Since GX10's CUDA compile attempt never actually ran (Triton/gcc failure, same as every other module on this box), there's no GX10 MPS-dropout-style mechanism to check on the GPU side — CUDA does have a native dropout kernel, so the MPS-specific finding from the Mac leg doesn't apply here regardless. +**Finding: unlike the Mac, GX10's CPU `aot_eager` backend is a clear net LOSS for audio (+75%, 0.174 -> 0.305s/epoch) — the opposite sign from the Mac's small CPU win (~-4.5%).** Radar showed the same kind of Mac/GX10 CPU sign disagreement (Mac +23% loss vs GX10 ~3% apparent win — see that Results section's caveat that the GX10 number is single-shot and within noise). Audio's flip is the more solid of the two: -4.5% and +75% are both large enough, relative to this session's ~7-9% documented run-to-run wobble, to treat as more than noise on both sides, even without repeated runs. Since GX10's CUDA compile attempt never actually ran (Triton/gcc failure, same as every other module on this box), there's no GX10 MPS-dropout-style mechanism to check on the GPU side — CUDA does have a native dropout kernel, so the MPS-specific finding from the Mac leg doesn't apply here regardless. **Finding: the Triton/gcc `inductor` build failure reproduces identically for the third module in a row** — now confirmed systemic across all of radar, image_classification, and audio_classification on this GX10 environment, not tied to any particular model or code path. -**Net across all three tasks and two machines: no single "compile helps" or "compile hurts" story holds everywhere.** Radar and image lose on both Mac devices; audio wins narrowly on Mac CPU and loses hard on Mac MPS (with a diagnosed cause); GX10 CPU flips sign between image/radar-style modules and audio; GX10 CUDA's real compile behavior remains entirely unmeasured across all three modules due to one shared environment issue. `--compile-model` staying opt-in (its current default) is the only conclusion that holds up across all of this data. +**Net across all three tasks and two machines, all nine measured `aot_eager` deltas:** + +| module | Mac CPU | Mac MPS | GX10 CPU | +|---|---|---|---| +| radar | +23% | +22% | -3.0% | +| image | +124% | +29% | +104% | +| audio | -4.5% | +88% | +75% | + +Seven of nine are large regressions (22-124%); the other two (radar Mac CPU... no, radar GX10 CPU at -3.0%, and audio Mac CPU at -4.5%) are both under 5% — smaller than the ~7-9% run-to-run wobble this session's own prior benchmarking documented, and neither was re-run enough times to rule out noise (see methodology caveat below). The honest summary is not "no story holds" but the stronger and better-supported one: **`aot_eager` is a consistent net loss for models this small across every device class actually measured, with two exceptions near the noise floor.** `inductor`-on-CUDA is the one configuration genuinely unmeasured across all three modules, blocked by a single reproducible GX10 environment issue (Triton/`gcc` `cuda_utils.c` build failure). `--compile-model` staying opt-in (its current default) is the conclusion that holds up regardless of how the two borderline cases are read. -**GX10 leg:** not run for this task — the Task 3 brief's Step 6 scopes the benchmark to "same methodology as Task 2 Step 6" (the Mac CPU/MPS 4-way comparison); the GX10 legs recorded under Tasks 1 and 2 were performed by a separate controller-run process outside this task's instructions, not part of this subagent's assigned steps. +**Methodology caveat:** rigor was not uniform across this plan. Task 1's Mac CPU/MPS numbers are a single run each, compared against a single run from a separate prior plan and session — before Task 2 discovered that the first MPS invocation of a session can run 2.3-4.6x slower than steady state, and before Tasks 2/3 adopted an n=3-4-repeats-with-range methodology. Task 1's Mac numbers were never re-validated under that later, stricter methodology. All six GX10 data points (across all three tasks) are single-shot, with GX10's own run-to-run variance never characterized. This doesn't overturn any of the seven large-regression findings, but it's the reason the two sub-5% deltas (radar GX10 CPU, audio Mac CPU) are treated as "near the noise floor" above rather than as confirmed real effects. From fe4b02a81405ce17277a11568918d8fa090e0b88 Mon Sep 17 00:00:00 2001 From: M Platypus Date: Thu, 13 Aug 2026 23:05:03 -0400 Subject: [PATCH 11/14] docs: resolve GX10 inductor block, add CUDA compile numbers for all 3 modules python3.12-dev was missing on GX10 (no Python.h anywhere on the box), which is what Triton's cuda_utils.c build needed. Fixed on GX10 by the repo owner. Re-ran --compile-model 1/CUDA for radar, image, and audio: inductor now builds and engages with zero fallback warnings on all three. All three show large regressions at 30 epochs (radar +24%, image +407%, audio +373%), most likely one-time autotuning overhead dominating short runs rather than steady-state cost -- flagged as an observed number, not isolated from warmup in this pass. --- .../2026-08-13-compile-hardening-radar-image-audio.md | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/docs/superpowers/plans/2026-08-13-compile-hardening-radar-image-audio.md b/docs/superpowers/plans/2026-08-13-compile-hardening-radar-image-audio.md index 7250a9c..57e77d3 100644 --- a/docs/superpowers/plans/2026-08-13-compile-hardening-radar-image-audio.md +++ b/docs/superpowers/plans/2026-08-13-compile-hardening-radar-image-audio.md @@ -429,4 +429,14 @@ This patch does feed `_load_audio`'s MFCC feature extraction (`audio_dataset.py: Seven of nine are large regressions (22-124%); the other two (radar Mac CPU... no, radar GX10 CPU at -3.0%, and audio Mac CPU at -4.5%) are both under 5% — smaller than the ~7-9% run-to-run wobble this session's own prior benchmarking documented, and neither was re-run enough times to rule out noise (see methodology caveat below). The honest summary is not "no story holds" but the stronger and better-supported one: **`aot_eager` is a consistent net loss for models this small across every device class actually measured, with two exceptions near the noise floor.** `inductor`-on-CUDA is the one configuration genuinely unmeasured across all three modules, blocked by a single reproducible GX10 environment issue (Triton/`gcc` `cuda_utils.c` build failure). `--compile-model` staying opt-in (its current default) is the conclusion that holds up regardless of how the two borderline cases are read. +**Addendum (2026-08-14): the GX10 `inductor` block is resolved, all three modules re-measured.** Root cause: GX10's system Python 3.12 was missing its `python3.12-dev` package, so `Python.h` didn't exist anywhere on the box — Triton's `cuda_utils.c` needs it to build the CUDA extension it compiles per-kernel. Fixed by installing `python3.12-dev` (`sudo apt install python3.12-dev`, done by the repo owner directly on GX10 — outside what this session could do without their password). Re-ran the `--compile-model 1`/CUDA config for all three modules; `inductor` now builds and engages with zero fallback warnings on all three (previously it failed and silently fell back to eager every time, so none of the numbers below existed until now): + +| module | CUDA no compile | CUDA `inductor` | delta | +|---|---|---|---| +| radar | 0.856s/epoch | 1.062s/epoch | +24% | +| image | 0.074s/epoch | 0.375s/epoch | +407% | +| audio | 0.079s/epoch | 0.374s/epoch | +373% | + +`inductor` is a large regression for all three at 30 epochs — likely dominated by one-time compilation/kernel-autotuning cost rather than steady-state throughput: radar's no-compile run alone takes ~26s for 30 epochs (enough to amortize some fixed cost), while image's and audio's no-compile runs take only ~2.2-2.4s total for 30 epochs, so a multi-second one-time `inductor` autotuning pass would dominate their totals almost entirely — the same warmup-dilution effect Task 2 and Task 3 already documented for MPS cold-start and compile-warmup costs, here likely far larger in absolute terms since `inductor` autotunes real CUDA kernels rather than `aot_eager`'s lighter graph capture. This wasn't isolated with a longer run (out of scope for this addendum) — reported as an observed 30-epoch-total number with this caveat, not a steady-state claim. Directionally, this means `inductor`-on-CUDA does not obviously help these three small models any more than `aot_eager` did elsewhere in this plan, and for very short runs it can hurt substantially more. + **Methodology caveat:** rigor was not uniform across this plan. Task 1's Mac CPU/MPS numbers are a single run each, compared against a single run from a separate prior plan and session — before Task 2 discovered that the first MPS invocation of a session can run 2.3-4.6x slower than steady state, and before Tasks 2/3 adopted an n=3-4-repeats-with-range methodology. Task 1's Mac numbers were never re-validated under that later, stricter methodology. All six GX10 data points (across all three tasks) are single-shot, with GX10's own run-to-run variance never characterized. This doesn't overturn any of the seven large-regression findings, but it's the reason the two sub-5% deltas (radar GX10 CPU, audio Mac CPU) are treated as "near the noise floor" above rather than as confirmed real effects. From c5a1832fd5a2fc8574e619011b3b09a630c323d8 Mon Sep 17 00:00:00 2001 From: M Platypus Date: Fri, 14 Aug 2026 09:04:00 -0400 Subject: [PATCH 12/14] docs: scrub personal SSH/infra details from the compile-hardening plan doc An independent PR review caught a private Tailscale hostname, SSH key path, and username committed to a doc already live on open PRs against a public upstream repo. Genericized all references to the remote GX10 machine and local venv paths -- no change to the technical content, only removed identifying infrastructure details. --- .../2026-08-13-compile-hardening-radar-image-audio.md | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/docs/superpowers/plans/2026-08-13-compile-hardening-radar-image-audio.md b/docs/superpowers/plans/2026-08-13-compile-hardening-radar-image-audio.md index 57e77d3..8b87781 100644 --- a/docs/superpowers/plans/2026-08-13-compile-hardening-radar-image-audio.md +++ b/docs/superpowers/plans/2026-08-13-compile-hardening-radar-image-audio.md @@ -19,7 +19,7 @@ placed between those two calls, plus adding `compile_model_if_enabled` to each f - Zero change to behavior when `--compile-model` is not set (the default, `0`) — `compile_model_if_enabled` already no-ops in that case; do not add any new conditionals around the call - Do not touch `compile_model_if_enabled` itself, `train_base.py`, or any file outside the three target `train.py` files (+ their new/modified test files) - Each task's benchmark must reuse the same measurement methodology as the radar entrypoint fix plan (`docs/superpowers/plans/2026-08-13-radar-training-entrypoint-fix.md`) — synthetic fixture, same device/epoch/batch-size reporting style — so results are comparable across all four now-benchmarked modules (timeseries already had compile; radar was benchmarked without it in that plan; this plan adds image and audio, and re-benchmarks radar with it now on) -- **Verification runs on two machines, not one:** local Mac (CPU vs MPS, `~/.venv-tinyml`) AND GX10 (CPU vs CUDA — `NVIDIA GB10`, `~/jupyterlab/.venv`, torch 2.9.0+cu130, reachable via `ssh -i ~/.ssh/gx10_key martin@gx10-singularity.skunk-mercat.ts.net`). Implementer subagents are not expected to have GX10 SSH access — the controller runs the GX10 leg directly after each task's local implementation is approved, and appends those results alongside the local ones in that task's Results section before considering the task's verification complete. +- **Verification runs on two machines, not one:** a local macOS machine (CPU vs MPS) AND a separate CUDA-equipped Linux machine, referred to below as "GX10" (CPU vs CUDA — `NVIDIA GB10`, torch 2.9.0+cu130), reachable only by the controller over SSH. Implementer subagents are not expected to have SSH access to GX10 — the controller runs the GX10 leg directly after each task's local implementation is approved, and appends those results alongside the local ones in that task's Results section before considering the task's verification complete. ## Context: Why This Is Needed @@ -305,7 +305,7 @@ git commit -m "feat: wire compile_model_if_enabled into audio_classification mai This is consistent with `LINEAR_4L_PC` being a tiny 4-layer linear/BatchNorm model: `torch.compile`'s per-call dynamo tracing, guard-checking, and graph-dispatch overhead is fixed cost per training step, and for a model this small there isn't enough per-op dispatch overhead in the eager path for kernel fusion to recoup that cost. The theory in this plan's "Known unresolved question" (fused kernels reducing dispatch overhead) does not hold for this model size — compile is a net loss here, not neutral or beneficial. Wiring it in is still correct (it makes `--compile-model` functional, matching the other reference scripts, and lets a caller opt in for larger/more compute-bound models where the tradeoff may differ), but the flag should not be turned on by default for radar's small linear models based on this evidence. -**GX10 leg (controller-run, per Global Constraints):** same fixture/model/batch-size/epochs, `~/jupyterlab/.venv` (torch 2.9.0+cu130), `NVIDIA GB10`. GX10's clone was stale (`5035057`, pre-dating this whole plan) — updated to `origin/integration` (`e472749`) first. Environment needed several missing leaf packages installed (`--no-deps`, no `torch`/`torchvision` touched): `tabulate`, `torcheval`, `torchinfo`, `colorama`, `onnx`, `onnxruntime`, `protobuf`, `ml_dtypes`, `cryptography`, `PyWavelets`, `opencv-python`, `onnxscript`, `onnx_ir`. `cmsisdsp` and `torchaudio` were stubbed via `sys.modules` in the benchmark driver instead of installed for real — both are only needed transitively (by `timeseries_dataset.py`'s FFT path and `audio_dataset.py` respectively, eagerly imported by `datasets/__init__.py`) and never touched by radar's own code path; `cmsisdsp` in particular requires a multi-minute native CMSIS-DSP C build with no prebuilt ARM64 wheel, unrelated to what this benchmark measures. +**GX10 leg (controller-run, per Global Constraints):** same fixture/model/batch-size/epochs, GX10's Python venv (torch 2.9.0+cu130), `NVIDIA GB10`. GX10's clone was stale (`5035057`, pre-dating this whole plan) — updated to `origin/integration` (`e472749`) first. Environment needed several missing leaf packages installed (`--no-deps`, no `torch`/`torchvision` touched): `tabulate`, `torcheval`, `torchinfo`, `colorama`, `onnx`, `onnxruntime`, `protobuf`, `ml_dtypes`, `cryptography`, `PyWavelets`, `opencv-python`, `onnxscript`, `onnx_ir`. `cmsisdsp` and `torchaudio` were stubbed via `sys.modules` in the benchmark driver instead of installed for real — both are only needed transitively (by `timeseries_dataset.py`'s FFT path and `audio_dataset.py` respectively, eagerly imported by `datasets/__init__.py`) and never touched by radar's own code path; `cmsisdsp` in particular requires a multi-minute native CMSIS-DSP C build with no prebuilt ARM64 wheel, unrelated to what this benchmark measures. | Config | Device | s/epoch | |---|---|---| @@ -358,7 +358,7 @@ Triton's CUDA-kernel codegen fails to build `cuda_utils.c` via `gcc` on this box **Bottom line:** wiring `compile_model_if_enabled` into `image_classification` is correct and makes `--compile-model` functional for this script, matching the other reference scripts and Task 1's radar wiring. As with radar, the evidence here does not support turning `--compile-model` on by default for this small a model: it's a net loss on both CPU and MPS for `CNN_LENET5` at this input size/batch size, though the loss is proportionally smaller on MPS than on CPU (unlike radar, where both devices regressed by a similar percentage). A larger, more compute-bound image model (e.g. `CNN_IMG_MOBILENETV1_58K_NPU`/`CNN_IMG_MOBILENETV2_58K_NPU`) was not benchmarked here (the brief calls for the smallest/fastest model to keep iteration time reasonable) and might show a different tradeoff, since more per-op dispatch overhead in eager mode gives kernel fusion more to recoup. -**GX10 leg (controller-run):** same fixture/model/batch-size, 30 epochs, `~/jupyterlab/.venv` (torch 2.9.0+cu130), `NVIDIA GB10`. GX10's clone fast-forwarded cleanly to `bf261a7` this time (no reset needed, unlike Task 1's stale-clone situation). Same `cmsisdsp`/`torchaudio` `sys.modules` stubs as Task 1 (still unrelated to image_classification's own code path); no new missing packages this time — everything Task 1 installed already covered it. +**GX10 leg (controller-run):** same fixture/model/batch-size, 30 epochs, GX10's Python venv (torch 2.9.0+cu130), `NVIDIA GB10`. GX10's clone fast-forwarded cleanly to `bf261a7` this time (no reset needed, unlike Task 1's stale-clone situation). Same `cmsisdsp`/`torchaudio` `sys.modules` stubs as Task 1 (still unrelated to image_classification's own code path); no new missing packages this time — everything Task 1 installed already covered it. | Config | Device | s/epoch | compile engaged? | |---|---|---|---| @@ -402,7 +402,7 @@ With compile enabled, CPU improves slightly (1.242 -> 1.186s/epoch, ~-4.5%) — **Bottom line:** wiring `compile_model_if_enabled` into `audio_classification` is correct and makes `--compile-model` functional for this script, matching all six other reference scripts. Unlike radar and image_classification, the evidence here does *not* uniformly argue against compile: CPU sees a small, consistent win, and the MPS regression has an identified, addressable-in-principle cause (a dropout decomposition lacking an MPS kernel under `torch.compile`) rather than being an unavoidable property of small-model dynamo overhead. Still, the MPS loss is large enough (and CPU's win small enough) that there's no case for flipping the opt-in default here either — `--compile-model` should stay opt-in, as it does today, but audio_classification is the first of the three tasks where "does compile help" doesn't have a uniformly negative answer, and the dropout/MPS-kernel angle is a concrete, non-speculative lead if `torch.compile` + MPS support for this workload is ever revisited. -**GX10 leg (controller-run):** same fixture/model/batch-size, 30 epochs, `~/jupyterlab/.venv` (torch 2.9.0+cu130), `NVIDIA GB10`. Clone fast-forwarded cleanly to `faad516`. Environment needed real (not stubbed) `torchaudio` this time, since `GoogleSpeechCommandsDataset` genuinely calls `torchaudio.load`/`MFCC`/`resample` (unlike radar/image, where `cmsisdsp`/`torchaudio` were only pulled in as unused eager imports and safely stubbed via `sys.modules`). `pip install torchaudio` grabbed 2.11.0 by default — ABI-incompatible with torch 2.9.0 (`undefined symbol: torch_library_impl` on import). Pinning `torchaudio==2.9.0` matched cleanly, but its default audio backend (`torchcodec`) requires system FFmpeg, which isn't installed on GX10 and isn't worth adding just for this benchmark. Worked around by monkey-patching `torchaudio.load` in the GX10 driver script only (not the repo) to read via the already-installed `soundfile` instead. +**GX10 leg (controller-run):** same fixture/model/batch-size, 30 epochs, GX10's Python venv (torch 2.9.0+cu130), `NVIDIA GB10`. Clone fast-forwarded cleanly to `faad516`. Environment needed real (not stubbed) `torchaudio` this time, since `GoogleSpeechCommandsDataset` genuinely calls `torchaudio.load`/`MFCC`/`resample` (unlike radar/image, where `cmsisdsp`/`torchaudio` were only pulled in as unused eager imports and safely stubbed via `sys.modules`). `pip install torchaudio` grabbed 2.11.0 by default — ABI-incompatible with torch 2.9.0 (`undefined symbol: torch_library_impl` on import). Pinning `torchaudio==2.9.0` matched cleanly, but its default audio backend (`torchcodec`) requires system FFmpeg, which isn't installed on GX10 and isn't worth adding just for this benchmark. Worked around by monkey-patching `torchaudio.load` in the GX10 driver script only (not the repo) to read via the already-installed `soundfile` instead. This patch does feed `_load_audio`'s MFCC feature extraction (`audio_dataset.py:280-305`), so it does affect `dataset.X`, not just I/O plumbing — the earlier claim that it "doesn't affect what's being measured" was imprecise. What actually protects the measurement: `_load_audio` unconditionally casts to `float32` (neutralizing `soundfile`'s float64 default) and peak-normalizes every clip to max-amplitude 1.0 (neutralizing any amplitude-scale difference from `torchaudio.load`'s own normalization), and since the fixture and the `--sampling-rate` arg both use 16kHz, the resampling branch never fires on either loader. The one risk that doesn't self-correct is channel orientation (`soundfile` returns frames-first, `torchaudio.load` returns channels-first) — a wrong-axis patch would silently collapse the mono-downmix step to a near-zero waveform with no exception. Checked empirically rather than left as an argument: the GX10 audio run's `run.log` shows `Acc@1 100.000` at the best epoch — well above the ~20% chance level for this 5-class fixture, confirming the model learned real, distinguishable signal and the patch was not silently corrupting input. (Separately: `s/epoch` timing, which is what's actually reported in the table below, is driven by tensor shapes fixed by the MFCC parameters, not by waveform content — so even a corrupted patch would not have moved the reported numbers, though it could have invalidated the accuracy figure.) From 5f42f451ad4cf5690f368deb5ebdb81e13b529ca Mon Sep 17 00:00:00 2001 From: M Platypus Date: Fri, 14 Aug 2026 09:09:16 -0400 Subject: [PATCH 13/14] test: fix misleading failure message in radar entrypoint test Claimed main_debug() alone lacked compile_model_if_enabled/apply_hardware_defaults, implying main() already had it -- at this commit neither function does (that's added in a later change). Corrected to describe what's actually true at this point: main() is the one that CAN be extended with it, not that it already is. --- tinyml-tinyverse/tests/test_radar_entrypoint_uses_main.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/tinyml-tinyverse/tests/test_radar_entrypoint_uses_main.py b/tinyml-tinyverse/tests/test_radar_entrypoint_uses_main.py index 8c05441..bf29425 100644 --- a/tinyml-tinyverse/tests/test_radar_entrypoint_uses_main.py +++ b/tinyml-tinyverse/tests/test_radar_entrypoint_uses_main.py @@ -16,9 +16,10 @@ def test_run_dispatches_to_main_not_main_debug(): assert dispatched_fn is radar_train.main, ( f"run() dispatched to {dispatched_fn.__name__!r}, expected 'main'. " - "main_debug() never applies quantization_wrapped_model, " - "compile_model_if_enabled/apply_hardware_defaults, or " + "main_debug() never applies quantization_wrapped_model or " "resume_from_checkpoint -- wiring run() to it silently drops quantized " - "training and hardware acceleration for every radar run." + "training and checkpoint resume for every radar run. main() is also the " + "only one of the two that can be extended with compile_model_if_enabled/" + "apply_hardware_defaults (added in a follow-up change)." ) assert dispatched_args is fake_args From d365b370ff962fe78b9f3dfa9dc0a6bee32c8bdc Mon Sep 17 00:00:00 2001 From: M Platypus Date: Fri, 14 Aug 2026 09:15:56 -0400 Subject: [PATCH 14/14] docs: clean up plan-doc presentation for the open PR Stripped the 'For agentic workers: REQUIRED SUB-SKILL...' header and normalized checkboxes from - [ ] to - [x] -- every task described is done. --- ...-13-compile-hardening-radar-image-audio.md | 73 +++++++++++-------- ...026-08-13-radar-training-entrypoint-fix.md | 20 +++-- 2 files changed, 51 insertions(+), 42 deletions(-) diff --git a/docs/superpowers/plans/2026-08-13-compile-hardening-radar-image-audio.md b/docs/superpowers/plans/2026-08-13-compile-hardening-radar-image-audio.md index 8b87781..20c67af 100644 --- a/docs/superpowers/plans/2026-08-13-compile-hardening-radar-image-audio.md +++ b/docs/superpowers/plans/2026-08-13-compile-hardening-radar-image-audio.md @@ -1,7 +1,5 @@ # Extend compile_model_if_enabled to Radar, Image, and Audio Classification Implementation Plan -> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. - **Goal:** Wire `compile_model_if_enabled` (the `torch.compile` warmup-and-fallback helper added by PR #22, currently only called from the four `timeseries_*` reference scripts) into `radar_classification`, `image_classification`, and `audio_classification`'s `main()` functions, so all seven reference training scripts get the same hardware-acceleration path — and measure whether it actually helps each one. **Architecture:** Each of the three target `main()` functions already has the exact structural shape `compile_model_if_enabled` expects: `move_model_to_device(model, device, logger)` immediately followed by `setup_distributed_model(model, args, device)`, with `dataset.X` (post feature-extraction) available in scope. The fix is the same one-line insertion in all three files, matching `timeseries_classification/train.py:275` verbatim: @@ -62,7 +60,7 @@ Rather than fix radar alone and leave image/audio in the same state, this plan c - Consumes: `tinyml_tinyverse.references.common.train_base.compile_model_if_enabled` (existing, unmodified — signature `compile_model_if_enabled(model, args, logger, input_shape=None)`, returns the (possibly compiled) model) - Produces: nothing new consumed by later tasks — Tasks 2 and 3 are independent, same pattern applied to different files -- [ ] **Step 1: Write the failing test** +- [x] **Step 1: Write the failing test** ```python """Regression test: radar_classification.train.main() must call @@ -82,12 +80,12 @@ def test_main_calls_compile_model_if_enabled(): ) ``` -- [ ] **Step 2: Run test to verify it fails** +- [x] **Step 2: Run test to verify it fails** Run: `cd tinyml-tinyverse && python -m pytest tests/test_radar_compile_wired.py -v` Expected: FAIL — `"compile_model_if_enabled("` not present in `main`'s source -- [ ] **Step 3: Add the import and the call** +- [x] **Step 3: Add the import and the call** In `tinyml-tinyverse/tinyml_tinyverse/references/radar_classification/train.py`, add `compile_model_if_enabled` to the existing import from `..common.train_base` (around line 65-89 — find the multi-line `from ..common.train_base import (...)` block and add it as a new entry, alphabetically or grouped with `move_model_to_device`/`compile_model_if_enabled`-adjacent imports per the file's existing style). @@ -107,20 +105,20 @@ to: model, model_without_ddp, model_ema = setup_distributed_model(model, args, device) ``` -- [ ] **Step 4: Run test to verify it passes** +- [x] **Step 4: Run test to verify it passes** Run: `cd tinyml-tinyverse && python -m pytest tests/test_radar_compile_wired.py -v` Expected: PASS -- [ ] **Step 5: Manual end-to-end sanity check with --compile-model 1** +- [x] **Step 5: Manual end-to-end sanity check with --compile-model 1** Using the synthetic radar fixture from the entrypoint-fix plan (regenerate via `make_radar_fixture.py` if not present — see that plan's Task 1 for the recipe), drive `train.run(args)` directly with `--device cpu --compile-model 1`, a few epochs. Confirm it completes without error and the log shows `compile_model_if_enabled`'s own INFO lines (check `train_base.py`'s `compile_model_if_enabled` for the exact log message text first — grep the run log for it). Then confirm `--compile-model 0` (the default) still behaves identically to before this change — no compile-related log lines, same training behavior. -- [ ] **Step 6: Benchmark CPU vs MPS with compile now enabled** +- [x] **Step 6: Benchmark CPU vs MPS with compile now enabled** Reuse the benchmark driver pattern from the entrypoint-fix plan (`bench_radar.py` in the session scratchpad), but add `--compile-model 1` to the argv. Run once on `cpu`, once on `mps` (`PYTORCH_ENABLE_MPS_FALLBACK=1`), same 30 epochs / batch size 16 / `LINEAR_4L_PC` as the entrypoint-fix plan's benchmark, so the numbers are directly comparable. Record: does compile change the CPU number, the MPS number, or the ratio between them, versus the entrypoint-fix plan's post-fix-no-compile numbers (CPU 0.621s/epoch, MPS 1.066s/epoch, 1.72x)? Report what you measure even if compile makes things worse or has no effect — that's a legitimate finding for a model this small, not a task failure. -- [ ] **Step 7: Record results and commit** +- [x] **Step 7: Record results and commit** Append a `## Results (Task 1: radar)` section to this plan doc with the benchmark table and a short explanation. @@ -142,7 +140,7 @@ git commit -m "feat: wire compile_model_if_enabled into radar_classification mai **Interfaces:** - Consumes: same `compile_model_if_enabled` as Task 1 — independent of Task 1, do not wait for it or reuse its branch -- [ ] **Step 1: Write the failing test** +- [x] **Step 1: Write the failing test** ```python """Regression test: image_classification.train.main() must call @@ -160,12 +158,12 @@ def test_main_calls_compile_model_if_enabled(): ) ``` -- [ ] **Step 2: Run test to verify it fails** +- [x] **Step 2: Run test to verify it fails** Run: `cd tinyml-tinyverse && python -m pytest tests/test_image_classification_compile_wired.py -v` Expected: FAIL -- [ ] **Step 3: Add the import and the call** +- [x] **Step 3: Add the import and the call** Add `compile_model_if_enabled` to the existing `from ..common.train_base import (...)` block (~line 92-111). Then change: ```python @@ -186,20 +184,20 @@ to: **Watch for:** this file has an `args.nn_for_feature_extraction` branch elsewhere (used at export time, line ~450-454) that chooses between `dataset.X_raw.shape` and `dataset.X.shape` depending on whether an NN is used for feature extraction. Confirm which shape the model actually consumes *at the point where you're inserting the compile call* (before or after any feature-extraction wrapping) — if `nn_for_feature_extraction` changes what the raw model's forward() expects at this point in `main()`, use the matching shape instead of assuming `dataset.X.shape` unconditionally. If unsure after reading the surrounding ~50 lines, ask before proceeding — this is exactly the kind of task-specific judgment call the brief can't resolve for you. -- [ ] **Step 4: Run test to verify it passes** +- [x] **Step 4: Run test to verify it passes** Run: `cd tinyml-tinyverse && python -m pytest tests/test_image_classification_compile_wired.py -v` Expected: PASS -- [ ] **Step 5: Manual end-to-end sanity check with --compile-model 1** +- [x] **Step 5: Manual end-to-end sanity check with --compile-model 1** Build or reuse a small synthetic image classification fixture (check `tinyml-tinyverse/tests/` for an existing image dataset test fixture/helper before building one from scratch — this repo likely already has one given image_classification has existing tests). Drive `train.run(args)` with `--device cpu --compile-model 1`, a few epochs. Confirm completion and compile-related log lines present. Confirm `--compile-model 0` still behaves as before. -- [ ] **Step 6: Benchmark CPU vs MPS with compile enabled** +- [x] **Step 6: Benchmark CPU vs MPS with compile enabled** Same methodology as Task 1 Step 6, adapted to whichever image model this benchmark uses (pick the smallest/fastest registered image model available, to keep iteration time reasonable) and image_classification's own CLI args. This is the first CPU-vs-MPS benchmark for image classification in either plan — there's no prior "no-compile" baseline to compare against from the earlier work, so also run once with `--compile-model 0` on both devices first to get that baseline, then `--compile-model 1` to see the delta, in the same benchmark session. -- [ ] **Step 7: Record results and commit** +- [x] **Step 7: Record results and commit** Append `## Results (Task 2: image_classification)` to this plan doc. @@ -221,7 +219,7 @@ git commit -m "feat: wire compile_model_if_enabled into image_classification mai **Interfaces:** - Consumes: same `compile_model_if_enabled` — independent of Tasks 1 and 2 -- [ ] **Step 1: Write the failing test** +- [x] **Step 1: Write the failing test** ```python """Regression test: audio_classification.train.main() must call @@ -239,12 +237,12 @@ def test_main_calls_compile_model_if_enabled(): ) ``` -- [ ] **Step 2: Run test to verify it fails** +- [x] **Step 2: Run test to verify it fails** Run: `cd tinyml-tinyverse && python -m pytest tests/test_audio_classification_compile_wired.py -v` Expected: FAIL -- [ ] **Step 3: Add the import and the call** +- [x] **Step 3: Add the import and the call** Add `compile_model_if_enabled` to the existing `from ..common.train_base import (...)` block (~line 92-111). Then change: ```python @@ -262,20 +260,20 @@ to: Same caveat as Task 2 applies here — audio_classification also has an `nn_for_feature_extraction` / `X_raw` vs `X` distinction at export time (line ~424-427). Check whether it affects what shape the model expects at this earlier insertion point before assuming `dataset.X.shape` unconditionally. -- [ ] **Step 4: Run test to verify it passes** +- [x] **Step 4: Run test to verify it passes** Run: `cd tinyml-tinyverse && python -m pytest tests/test_audio_classification_compile_wired.py -v` Expected: PASS -- [ ] **Step 5: Manual end-to-end sanity check with --compile-model 1** +- [x] **Step 5: Manual end-to-end sanity check with --compile-model 1** Same approach as Task 2 Step 5, adapted to audio_classification's dataset/CLI. Check `tinyml-tinyverse/tests/` for existing audio fixture helpers first. -- [ ] **Step 6: Benchmark CPU vs MPS with compile enabled** +- [x] **Step 6: Benchmark CPU vs MPS with compile enabled** Same methodology as Task 2 Step 6 (baseline `--compile-model 0` then `--compile-model 1`, both devices), adapted to audio_classification's smallest registered model. -- [ ] **Step 7: Record results and commit** +- [x] **Step 7: Record results and commit** Append `## Results (Task 3: audio_classification)` to this plan doc. @@ -325,11 +323,11 @@ WARNING: root.main: torch.compile failed (or failed its warmup pass), falling ba CalledProcessError: Command '['/usr/bin/gcc', '.../cuda_utils.c', '-O3', '-shared', '-fPIC', ... -lcuda', '-L.../triton/backends/nvidia/lib', ...]' returned non-zero exit status 1. ``` -Triton's CUDA-kernel codegen fails to build `cuda_utils.c` via `gcc` on this box — a toolchain issue local to this environment (missing header/lib path for Triton's nvidia backend), not a `compile_model_if_enabled` defect. The `compile_model_if_enabled` warmup-and-fallback mechanism (built earlier this session, `docs/superpowers/plans/2026-07-28-compile-warmup-fallback.md`) caught the failure exactly as designed and fell back cleanly — training was not interrupted. This means the measured "compile=1, CUDA, 0.943s/epoch" number is **eager mode plus the one-time cost of a failed compile attempt**, not a real compiled-vs-uncompiled comparison; the CUDA compile question remains genuinely open on this hardware until the Triton/gcc toolchain issue is fixed here. +Triton's CUDA-kernel codegen fails to build `cuda_utils.c` via `gcc` on this box — a toolchain issue local to this environment (missing header/lib path for Triton's nvidia backend), not a `compile_model_if_enabled` defect. The `compile_model_if_enabled` warmup-and-fallback mechanism (built earlier this session, `docs/superpowers/plans/2026-07-28-compile-warmup-fallback.md`) caught the failure exactly as designed and fell back cleanly — training was not interrupted. This means the measured "compile=1, CUDA, 0.943s/epoch" number is **eager mode plus the one-time cost of a failed compile attempt**, not a real compiled-vs-uncompiled comparison. (**Update 2026-08-14:** the toolchain issue was fixed — see the Addendum in Task 3's Results section below for the real `inductor`-on-CUDA numbers.) **Finding: CPU's `aot_eager` backend did engage successfully on GX10** (`INFO: root.main: Compiling model with torch.compile (backend=aot_eager)`, no fallback warning) and produced a small apparent win (4.093 -> 3.972s/epoch, ~3%) — versus the Mac's own CPU result, a clear ~23% *loss* for the same model/backend. This is a single-shot measurement on both sides (see the whole-plan methodology caveat), and ~3% is well within the ~7-9% run-to-run wobble this session's benchmarking has otherwise documented, so treat this as "the two machines may not agree on aot_eager's sign for this model" rather than a confirmed win — it would need repeated runs on both machines to distinguish a real effect from noise. -**Net implication for the "should compile be on by default" question:** no clean "yes" or "no" emerges. Mac CPU/MPS: compile hurts on both. GX10 CPU: compile helps marginally. GX10 CUDA: unknown — the only device compile theoretically helps most (via `inductor`, not `aot_eager`) is the one device where it couldn't even run here. Recommend leaving `--compile-model` opt-in (its current default) rather than drawing a default-on conclusion from this data, and separately investigating the GX10 Triton/gcc build failure if CUDA compile behavior is worth knowing for real. +**Net implication for the "should compile be on by default" question:** no clean "yes" or "no" emerges. Mac CPU/MPS: compile hurts on both. GX10 CPU: a ~3% apparent win, single-shot and within this session's documented run-to-run noise — not confirmed real (see caveat below). GX10 CUDA: unmeasured at the time this was written — see the Addendum in Task 3's Results section for the real `inductor` numbers, obtained after the GX10 toolchain issue was fixed. Recommend leaving `--compile-model` opt-in (its current default); nothing in the full dataset (including the later `inductor` numbers) argues for changing it. --- @@ -354,9 +352,18 @@ Triton's CUDA-kernel codegen fails to build `cuda_utils.c` via `gcc` on this box **Finding: `torch.compile` makes image_classification training slower on both devices, same direction as radar, but with a smaller relative hit on MPS than on CPU.** CPU regresses ~124% (0.037s -> 0.083s/epoch) — proportionally worse than radar's ~23% CPU regression. MPS regresses only ~29% (0.073s -> 0.094s/epoch) — close in magnitude to radar's ~22% MPS regression, but here the CPU/MPS gap actually **narrows** with compile on: 0.073/0.037 = 1.98x (MPS slower, no compile) vs. 0.094/0.083 = 1.13x (MPS slower, with compile). CPU still wins in absolute terms either way, but compile shrinks rather than preserves the ratio — the opposite of radar, where the ratio stayed essentially flat (1.72x -> 1.70x). This is driven by the CPU/MPS *asymmetry* (CPU regresses far more than MPS does), not by MPS itself improving — no causal mechanism for that asymmetry was established here (see the whole-plan review's note that a competing "conv work narrows the GPU's relative advantage" story doesn't hold up against the GX10 data below, so this is reported as an observed pattern, not an explained one). +**Correction (2026-08-14, per independent peer review): the numbers above conflate one-time compile warmup with steady-state cost, and the "net loss" conclusion below does not survive isolating them.** `s/epoch` here is whole-`train.run()` time ÷ 30, and Task 3's own MPS-dropout finding already showed compile's one-time warmup (~1.4s there) can be large relative to a 30-epoch total when each epoch is fast. Re-ran with the 1-epoch-vs-30-epoch isolation Task 3 used, fresh runs, same fixture/model/batch-size: + +| Config | Device | 1-epoch total | 30-epoch total | steady-state (solved) | fresh no-compile baseline | +|---|---|---|---|---|---| +| `--compile-model 1` | CPU | 1.519s | 1.860s | ~0.012s/epoch (warmup ~1.51s) | 0.019s/epoch | +| `--compile-model 1` | MPS | 1.310s | 2.011s | ~0.024s/epoch (warmup ~1.29s) | 0.048s/epoch | + +Once warmup is excluded, compiled steady-state is **faster than eager on both devices** — roughly 37% faster on CPU, roughly 50% faster on MPS — the opposite sign from the naive 30-epoch-averaged "CPU regresses ~124%, MPS regresses ~29%" finding above. The original numbers weren't wrong as measurements of a 30-epoch run; they're wrong as a proxy for steady-state training cost, because a 30-epoch run this fast (~0.6-1.4s total) barely amortizes a ~1.3-1.5s one-time compile cost. (These are fresh runs in a new session, not re-runs of the exact original data — but the no-compile baselines were measured fresh in the same pass for a fair within-session comparison.) This reverses the "Bottom line" below for any training run long enough to amortize the warmup — see the whole-plan synthesis for how this changes the cross-task picture. + **Methodology note — MPS cold-start artifact:** the very first MPS invocation of the whole benchmark session (`--compile-model 0`, run 1) measured 0.171s/epoch, 2.3-4.6x slower than every subsequent MPS run (compile-0 or compile-1) in the same session, each a fresh process. All later MPS runs (4 for compile-0, 4 for compile-1, alternating) clustered tightly (0.069-0.077 and 0.090-0.096 respectively). This looks like a one-time cost paid on the first-ever MPS/Metal invocation in the session (e.g. Metal shader compilation cache being cold on disk, not per-process) rather than genuine compile-vs-no-compile signal, since it appeared on a `--compile-model 0` run. The table above reports the steady-state average (excluding that one outlier); the raw run-by-run numbers are preserved in the session scratchpad driver output for reference. This is the same class of measurement caveat Task 1 flagged for GX10 (environment-specific first-run cost, not part of the compile question itself) — worth keeping in mind for any future single-shot MPS benchmark on this or other modules. -**Bottom line:** wiring `compile_model_if_enabled` into `image_classification` is correct and makes `--compile-model` functional for this script, matching the other reference scripts and Task 1's radar wiring. As with radar, the evidence here does not support turning `--compile-model` on by default for this small a model: it's a net loss on both CPU and MPS for `CNN_LENET5` at this input size/batch size, though the loss is proportionally smaller on MPS than on CPU (unlike radar, where both devices regressed by a similar percentage). A larger, more compute-bound image model (e.g. `CNN_IMG_MOBILENETV1_58K_NPU`/`CNN_IMG_MOBILENETV2_58K_NPU`) was not benchmarked here (the brief calls for the smallest/fastest model to keep iteration time reasonable) and might show a different tradeoff, since more per-op dispatch overhead in eager mode gives kernel fusion more to recoup. +**Bottom line (revised 2026-08-14):** wiring `compile_model_if_enabled` into `image_classification` is correct and makes `--compile-model` functional for this script, matching the other reference scripts and Task 1's radar wiring. Unlike the original conclusion, the warmup-isolated evidence does *not* argue against compile for `CNN_LENET5` — steady-state throughput improves on both CPU and MPS. It does argue against `--compile-model` defaulting on for *short* runs (a handful of epochs), where the one-time warmup cost dominates and compile is a net loss in wall-clock terms even though it's faster per-epoch once warmup is paid. Whether `--compile-model` should default on depends entirely on typical training-run length, which this benchmark's fixture (a few dozen epochs on a tiny synthetic dataset) doesn't represent — real training runs are likely long enough to amortize the ~1.3-1.5s warmup easily. A larger, more compute-bound image model (e.g. `CNN_IMG_MOBILENETV1_58K_NPU`/`CNN_IMG_MOBILENETV2_58K_NPU`) was not benchmarked here. **GX10 leg (controller-run):** same fixture/model/batch-size, 30 epochs, GX10's Python venv (torch 2.9.0+cu130), `NVIDIA GB10`. GX10's clone fast-forwarded cleanly to `bf261a7` this time (no reset needed, unlike Task 1's stale-clone situation). Same `cmsisdsp`/`torchaudio` `sys.modules` stubs as Task 1 (still unrelated to image_classification's own code path); no new missing packages this time — everything Task 1 installed already covered it. @@ -371,7 +378,7 @@ Triton's CUDA-kernel codegen fails to build `cuda_utils.c` via `gcc` on this box **Finding: `aot_eager` on CPU is a bigger loss on GX10 than on the Mac for this model** — 0.097 -> 0.198s/epoch is a ~104% regression on GX10's CPU, roughly consistent in direction and rough magnitude with the Mac's own CPU regression for image_classification (~124%) and both are much worse than either machine's CPU regression for radar. Small-CNN `aot_eager` compilation overhead appears to be a bigger relative cost than small-linear-net overhead was, on both machines. -**Finding: the same CUDA/Triton/gcc build failure from Task 1 reproduces identically here** — confirms it's an environment-level GX10 toolchain issue (Triton's nvidia backend `cuda_utils.c` failing to compile via `gcc`), not something specific to radar's model or code path. This will block any real `inductor`-backend measurement on GX10 across all modules until fixed there; worth investigating separately if CUDA compile behavior needs to be known for real, rather than re-discovering the same failure in Task 3's audio benchmark. +**Finding: the same CUDA/Triton/gcc build failure from Task 1 reproduces identically here** — confirms it's an environment-level GX10 toolchain issue (Triton's nvidia backend `cuda_utils.c` failing to compile via `gcc`), not something specific to radar's model or code path. (**Update 2026-08-14:** fixed — see the Addendum in Task 3's Results section.) --- @@ -419,15 +426,19 @@ This patch does feed `_load_audio`'s MFCC feature extraction (`audio_dataset.py: **Finding: the Triton/gcc `inductor` build failure reproduces identically for the third module in a row** — now confirmed systemic across all of radar, image_classification, and audio_classification on this GX10 environment, not tied to any particular model or code path. -**Net across all three tasks and two machines, all nine measured `aot_eager` deltas:** +**Revised 2026-08-14, following independent peer review and the image warmup-isolation re-check above.** The original version of this section claimed "`aot_eager` is a consistent net loss... across every device class actually measured, with two exceptions near the noise floor." That claim doesn't survive scrutiny — one of the "exceptions" was itself mischaracterized, and two of the seven "regressions" turned out to be warmup-dilution artifacts, not steady-state cost, once actually isolated. + +**All nine naive 30-epoch-averaged `aot_eager` deltas** (the raw numbers as originally measured, before any warmup isolation): | module | Mac CPU | Mac MPS | GX10 CPU | |---|---|---|---| -| radar | +23% | +22% | -3.0% | +| radar | +23% | +22% | -3.0% (single-shot, within documented ~7-9% noise — not confirmed real) | | image | +124% | +29% | +104% | -| audio | -4.5% | +88% | +75% | +| audio | -4.5% (n=3, non-overlapping ranges — real) | +88% (~56% steady-state, isolated) | +75% | + +**What changes when warmup is actually isolated (only done for image and audio's MPS leg — radar and the other legs were not re-checked this way):** image's naive CPU/MPS deltas (+124%/+29%) reverse sign entirely once the ~1.3-1.5s one-time compile cost is excluded — steady-state compile is ~37% faster on CPU and ~50% faster on MPS (see the warmup-isolation correction in Task 2's Results above). Audio's Mac MPS delta shrinks from +88% (naive) to ~+56% (steady-state) but stays a large real loss. Audio's Mac CPU -4.5% was run n=3 with non-overlapping ranges (1.222-1.254s vs 1.170-1.204s) — a small but genuine, controlled result, not noise; treating it as "near the noise floor" (the original version of this section did) was itself an error, since a noise-floor figure derived from single uncontrolled runs elsewhere in this plan was applied to discount a result that was actually more rigorously measured than the figure used to discount it. -Seven of nine are large regressions (22-124%); the other two (radar Mac CPU... no, radar GX10 CPU at -3.0%, and audio Mac CPU at -4.5%) are both under 5% — smaller than the ~7-9% run-to-run wobble this session's own prior benchmarking documented, and neither was re-run enough times to rule out noise (see methodology caveat below). The honest summary is not "no story holds" but the stronger and better-supported one: **`aot_eager` is a consistent net loss for models this small across every device class actually measured, with two exceptions near the noise floor.** `inductor`-on-CUDA is the one configuration genuinely unmeasured across all three modules, blocked by a single reproducible GX10 environment issue (Triton/`gcc` `cuda_utils.c` build failure). `--compile-model` staying opt-in (its current default) is the conclusion that holds up regardless of how the two borderline cases are read. +**Net honest picture:** there is no clean "compile helps" or "compile hurts" story, and — unlike the original version of this section claimed — the data doesn't support a "consistent net loss" story either. What actually holds up: (1) radar and audio's Mac MPS regressions are large and were not explained away by warmup (audio's mechanism — a dropout op falling back to CPU under compile — is a real, identified cause); (2) image's apparent regressions were mostly or entirely warmup dilution and reverse to a real win in steady state; (3) whether `--compile-model` helps depends heavily on model architecture, device, and training-run length, none of which this benchmark holds constant enough to generalize from. `inductor`-on-CUDA is no longer unmeasured — see the Addendum immediately below, obtained after the GX10 toolchain issue was fixed. `--compile-model` staying opt-in (its current default) is still the right call given how mixed and run-length-dependent this data is — but not because "compile is a consistent loss." **Addendum (2026-08-14): the GX10 `inductor` block is resolved, all three modules re-measured.** Root cause: GX10's system Python 3.12 was missing its `python3.12-dev` package, so `Python.h` didn't exist anywhere on the box — Triton's `cuda_utils.c` needs it to build the CUDA extension it compiles per-kernel. Fixed by installing `python3.12-dev` (`sudo apt install python3.12-dev`, done by the repo owner directly on GX10 — outside what this session could do without their password). Re-ran the `--compile-model 1`/CUDA config for all three modules; `inductor` now builds and engages with zero fallback warnings on all three (previously it failed and silently fell back to eager every time, so none of the numbers below existed until now): diff --git a/docs/superpowers/plans/2026-08-13-radar-training-entrypoint-fix.md b/docs/superpowers/plans/2026-08-13-radar-training-entrypoint-fix.md index ccc17e2..c8e0112 100644 --- a/docs/superpowers/plans/2026-08-13-radar-training-entrypoint-fix.md +++ b/docs/superpowers/plans/2026-08-13-radar-training-entrypoint-fix.md @@ -1,7 +1,5 @@ # Radar Training Entrypoint Fix Implementation Plan -> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. - **Goal:** Wire the radar classification training script's `run()` entrypoint to the fully-featured `main()` function instead of the leftover `main_debug()` harness, then confirm on real hardware that this closes the MPS-vs-CPU performance gap it currently causes. **Architecture:** `run_distributed(main_debug, args)` becomes `run_distributed(main, args)` in `radar_classification/train.py`. `main()` already exists and is fully wired to the shared, hardened training infrastructure (`quantization_wrapped_model`, `compile_model_if_enabled`/`apply_hardware_defaults`, `resume_from_checkpoint`, `create_data_loaders`) — it is simply never called today. No new code is needed for the fix itself; the risk is regression in radar-specific behavior that only `main_debug` currently exercises, which Task 1's test and manual run guard against. @@ -62,7 +60,7 @@ MPS being slower than CPU is the expected signature of a small-op-heavy graph wi **Interfaces:** - Consumes: `tinyml_tinyverse.references.radar_classification.train.main`, `.main_debug`, `.run`, `.run_distributed` (all already defined in the module — no new interfaces) -- [ ] **Step 1: Write the failing test** +- [x] **Step 1: Write the failing test** ```python """Regression test: radar_classification.train.run() must dispatch to main(), @@ -91,12 +89,12 @@ def test_run_dispatches_to_main_not_main_debug(): assert dispatched_args is fake_args ``` -- [ ] **Step 2: Run test to verify it fails** +- [x] **Step 2: Run test to verify it fails** Run: `cd tinyml-tinyverse && python -m pytest tests/test_radar_entrypoint_uses_main.py -v` Expected: FAIL — `dispatched_fn` is `main_debug`, not `main` -- [ ] **Step 3: Fix the dispatch target** +- [x] **Step 3: Fix the dispatch target** In `tinyml-tinyverse/tinyml_tinyverse/references/radar_classification/train.py`, change: @@ -114,16 +112,16 @@ def run(args): run_distributed(main, args) ``` -- [ ] **Step 4: Run test to verify it passes** +- [x] **Step 4: Run test to verify it passes** Run: `cd tinyml-tinyverse && python -m pytest tests/test_radar_entrypoint_uses_main.py -v` Expected: PASS -- [ ] **Step 5: Manual end-to-end sanity check** +- [x] **Step 5: Manual end-to-end sanity check** `main()` has never actually executed for radar before — confirm it runs clean on a real (if synthetic) dataset before trusting the unit test alone. Build a small fixture (5 class dirs of CSVs under `/classes/`, balanced `annotations/instances_{train,test,val}_list.txt`) and drive `train.get_args_parser().parse_args([...]); train.run(args)` directly with `--device cpu`, a few epochs, `--quantization 0`. Confirm it completes and exports `model.onnx` without error. Then repeat with `--quantization 1` (previously silently a no-op under `main_debug`) and confirm the log now shows `QuantTrain` phase entries and a second exported model under the quantization output dir. -- [ ] **Step 6: Commit** +- [x] **Step 6: Commit** ```bash cd tinyml-tinyverse @@ -141,18 +139,18 @@ git commit -m "fix: radar_classification run() was dispatching to main_debug, no **Interfaces:** - Consumes: `radar_classification.train.run(args)` (now dispatching to `main`, per Task 1) -- [ ] **Step 1: Re-run the same benchmark used to characterize the bug** +- [x] **Step 1: Re-run the same benchmark used to characterize the bug** Using the same synthetic fixture (5 classes, balanced annotation lists, `LINEAR_4L_PC`, batch size 16, 30 epochs) and the same driver pattern that produced the pre-fix numbers (constructs `argv`, calls `train.get_args_parser().parse_args(argv)`, times `train.run(args)`), run once with `--device cpu` and once with `--device mps`. Run: `python bench_radar.py cpu` then `python bench_radar.py mps` (with `PYTORCH_ENABLE_MPS_FALLBACK=1` set) Expected: both complete without error; timing printed as `total=...s for N epochs -> ...s/epoch` -- [ ] **Step 2: Record the comparison** +- [x] **Step 2: Record the comparison** Append a short results table to this plan (or a sibling `RESULTS.md` next to it) with pre-fix vs post-fix CPU and MPS per-epoch timing, and note whether `apply_hardware_defaults` actually enabled `torch.compile`/AMP for this run (check the training log for the relevant INFO lines from `compile_model_if_enabled`/`apply_hardware_defaults`). -- [ ] **Step 3: Commit** +- [x] **Step 3: Commit** ```bash git add docs/superpowers/plans/2026-08-13-radar-training-entrypoint-fix.md