diff --git a/.gitattributes b/.gitattributes new file mode 100644 index 0000000..1bf01c4 --- /dev/null +++ b/.gitattributes @@ -0,0 +1,41 @@ +# Normalize line endings. Without this, every commit from a Windows checkout +# prints "warning: CRLF will be replaced by LF the next time Git touches it", +# and a contributor on another platform sees whole-file diffs that contain no +# actual change. +# +# `text=auto eol=lf` means: store everything Git detects as text with LF in the +# repository, and check it out with LF everywhere, including on Windows. LF is +# the right choice rather than native because the DSP-side C is read by +# hexagon-clang and by this repo's own source-assertion tests, which slice C +# function bodies by byte offset (hexlib/tests/csource.py) -- a mixed-ending +# working tree makes those offsets platform-dependent. +* text=auto eol=lf + +# Explicitly text, so no heuristic has to guess. +*.py text eol=lf +*.c text eol=lf +*.h text eol=lf +*.idl text eol=lf +*.md text eol=lf +*.json text eol=lf +*.toml text eol=lf +*.cfg text eol=lf +*.yml text eol=lf +*.yaml text eol=lf +*.txt text eol=lf +*.sh text eol=lf + +# Binary. Never normalize, never diff as text. +*.npz binary +*.bin binary +*.elf binary +*.so binary +*.a binary +*.png binary +*.zip binary + +# The committed vision-encoder golden vectors. Marked binary explicitly rather +# than relying on the *.npz rule above, because this file is the correctness +# oracle for the whole encoder (docs/research/oracle-provenance.md) and a +# line-ending "fix" applied to it would corrupt it silently. +hexlib/tests/data/qwen35_vision_tiny.npz binary diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 77a971e..d7adee6 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -13,7 +13,12 @@ jobs: - uses: actions/setup-python@v5 with: python-version: "3.11" - - run: pip install -e ".[dev]" || pip install -e . pytest + # No `|| pip install -e . pytest` fallback. That was here to be a safety + # net and could not be one: pip treats a missing extra as a warning and + # exits 0, so with no `dev` extra defined the fallback never ran and + # pytest was never installed. The extra is defined in pyproject.toml now + # and this install is required to be sufficient by itself. + - run: pip install -e ".[dev]" - run: python -m pytest -q -m "not sdk" kernel-contract: diff --git a/.gitignore b/.gitignore index 78a3d0c..5a4d20f 100644 --- a/.gitignore +++ b/.gitignore @@ -1,6 +1,12 @@ # ---- hexlib build and verification output ---- # `hexlib test` writes ELFs, objects, disassembly, and result tables here. -_work/ +# +# NOTE: these two patterns match the transient `.result.md` / +# `.result.json` a run drops in the working directory. They do NOT match +# `kernels/*/RESULT.md`, which is the promoted, reviewed record for a gated +# kernel and IS tracked on purpose. Do not "unify" these into `RESULT.md` or +# `*result*` -- that would untrack every kernel's evidence. +_work*/ *.result.json *.result.md @@ -24,11 +30,33 @@ _work/ pmu_stats*.txt packet_analyze*.json +# QuRT-hosted simulator configs generated by +# hexlib.runtime.build.write_qurt_sim_configs (task 7b). Each is a thin, +# machine-specific reference to an SDK path (the QuRT debugger model DLL, the +# qtimer/l2vic cosim DLLs) -- regenerated on every build, and named exactly +# rather than matched by a blanket *.cfg so a config someone actually means to +# commit elsewhere in the repo is not silently swallowed. +osam.cfg +q6ss.cfg + # FastRPC / qaic generated sources (silicon path). These are generated from # the IDL at build time; the IDL is the source of truth. *_stub.c *_skel.c +# Generated runtime artifacts (qaic output, skel objects, entry points) +hexlib/runtime/gen/ +kernels/*/_build/ +_work/runtime/ + +# Task 10's cross-compiled Android arm64 client (hexlib/runtime/build.py's +# build_device_binary). It is a stripped ELF executable with NO extension -- +# the target is Android/aarch64, not the build host, so `tc.exe()` is never +# applied to it and none of the extension-based rules above (*.elf, *.o, ...) +# match it. Named exactly, the same way osam.cfg/q6ss.cfg are, rather than a +# blanket rule that could also swallow an unrelated file named "hexlib_run". +hexlib_run + # The Hexagon SDK is license-restricted and is NEVER vendored, bundled, or # fetched. It is discovered through HEXAGON_SDK_ROOT. These guard against an # accidental copy landing in the repository. @@ -92,7 +120,34 @@ qdc_credentials* # ---- Agent workspace ---- # Plan-scoped scratch: ledgers, briefs, reports, review packages. Git history # is the durable record. +# +# Because this directory is never published, NO tracked document may cite a +# path inside it as evidence a reader can check. Where an older doc does, the +# durable record is the commit message and docs/STATE.md, not the report. .superpowers/ +.claude/ + +# ---- Project documentation, deliberately untracked ---- +# `docs/` -- STATE.md, the design specs, the plans, the HVX tour, the research +# audits -- is kept on disk and OUT of the repository, and was purged from git +# history rather than merely untracked from here on. +# +# The consequence, stated once so nobody rediscovers it: a tracked file MAY +# still cite a `docs/...` path as provenance for a decision, and those +# citations are now unresolvable to anyone but the author. They are left in +# place because a comment naming where a fact came from is still worth more +# than no attribution -- but NOTHING tracked may DEPEND on a docs path at run +# time. One test did (`test_host_source.py`'s §6.1 coherency-table check read +# the design spec) and passed only on the machine that still had the untracked +# copy on disk; it was fixed when this rule landed, not after CI found it. +docs/ + +# ---- Patch and merge debris ---- +*.orig +*.rej +*.bak +*.patch.tmp +*.log # ---- Editors ---- .vscode/ @@ -116,3 +171,13 @@ $RECYCLE.BIN/ # ---- Windows / MSYS ---- *.stackdump + +# ---- Simulator run artifacts, never committed ---- +# A sim run writes these RELATIVE TO THE LAUNCHING PROCESS'S CWD, not to --usefs +# (see hexlib/runtime/build.py sim_qurt_command). So a suite run from the repo +# root drops them here. Two were committed by accident in f796edd via a blanket +# `git add -A` and removed in the following commit. +hexlib_out.bin +hexlib_rsp.bin +hexlib_in.bin +hexlib_batch.bin diff --git a/ATTRIBUTION.md b/ATTRIBUTION.md index dd5d59a..3594db2 100644 --- a/ATTRIBUTION.md +++ b/ATTRIBUTION.md @@ -34,11 +34,45 @@ llama.cpp or ggml, and none of the copied headers reference ggml (verified by `hexlib/tests/test_vendored_headers.py`, which fails the build if a `ggml` reference appears in any of them, or if the vendored directory is empty). -**Deferred to plan 2 (the tile DSL / v2 spec):** ggml-hexagon's IDL, host driver -(`htp-drv.cpp`), and CMake toolchain file are also planned to be copied under this -same MIT attribution, once the DSL and device runtime work that needs them begins. -Nothing under those categories has been copied yet in this plan; when it is, this -document must be updated alongside it. +## Adapted: ggml-hexagon's FastRPC runtime (MIT) + +**Source:** `ggml/src/ggml-hexagon/` in +[`ggml-org/llama.cpp`](https://github.com/ggml-org/llama.cpp). +**License:** MIT. **Copyright:** (c) 2023-2026 The ggml authors. +**Upstream commit:** `6a32c29a746a2e44de463de647f9f6661eb5086b` (2026-08-06). + +hexlib's silicon-path runtime (`hexlib/runtime/`) is **adapted** from this +backend — rewritten in hexlib's own tree, not copied verbatim. What was adapted, +and from where: + +| hexlib | upstream | what was taken | +|---|---|---| +| `runtime/idl/hexlib_iface.idl` | `htp/htp_iface.idl` | the session lifecycle: `start`, `stop`, `mmap`, `munmap`, `hwinfo` | +| `runtime/host/driver.c` | `htp-drv.cpp` | dlopen/dlsym of `libcdsprpc`, so a missing driver is a message rather than a loader failure | +| `runtime/skel/skel_bufs.c` | `htp/main.c` `reuse_buf`/`mmap_buf`/`prep_tensor` | fd→base mmap caching, and the **(buffer index, offset)** tensor addressing that keeps host addresses off the wire | +| `runtime/skel/skel_vtcm.c` | `htp/main.c` `vtcm_acquire`/`vtcm_alloc` | `HAP_compute_res_*` acquisition with a release callback | +| `runtime/skel/hexlib_dsp.h` | `htp/htp-ops.h` | the batch descriptor SHAPE, and `htp_status`'s "OK is 1, not 0" | +| `runtime/wire.py` | `htp/htp-ops.h` `htp_opbatch_req` | the batch request SHAPE that `wire.py:10` credits: a fixed header, then buffer descriptors, tensor descriptors and ops in one opaque blob. The Python serializer is hexlib's own; only the layout is adapted, and it is the host-side mirror of the `hexlib_dsp.h` row above | +| `runtime/skel/skel.c` | `htp/main.c` session entry points | the `open`/`close`/`start`/`stop`/`mmap`/`munmap`/`hwinfo` lifecycle qaic's skel dispatches to; `invoke` is hexlib's own (a single opaque batch, not a dspqueue packet per op) | +| `runtime/host/session.c` (`hexlib_query_caps`'s `ARCH_VER` query) | `htp-drv.cpp` `htpdrv_get_arch` | the `remote_dsp_capability` / `DSPRPC_GET_DSP_INFO` query shape. Not adapted from it: hexlib queries every capability it needs (`DOMAIN_SUPPORT`, `UNSIGNED_PD_SUPPORT`, `HVX_SUPPORT_128B`, `VTCM_PAGE`, `VTCM_COUNT`, `ARCH_VER`, `HMX_SUPPORT_DEPTH`) through one loop rather than one bespoke function per attribute, and cross-checks the **arch** against the skel's own `hwinfo` reply rather than trusting the driver alone. **Corrected 2026-08-11:** this row previously implied every capability is cross-checked. Only `arch` is. `vtcm_page × vtcm_count` vs the skel's `vtcm_size`, `hvx_support_128b` vs `n_hvx`, and `hmx_support_depth` vs `n_hmx` are queried and never compared — and `n_hvx`/`n_hmx` are host echoes rather than DSP facts anyway, so there is currently nothing on the DSP side to compare them against | +| `runtime/host/session.c` `hexlib_decode_bcd_arch` | `htp-drv.cpp` `htpdrv_get_arch` (the decode, not just the query shape) | the actual formula, copied line-for-line: `val = arch_ver & 0xff; arch = (val >> 4) * 10 + (val & 0x0f)`. **Bug found and fixed while adapting this, not upstream's:** an earlier draft of this file compared the skel's plain-decimal `__HEXAGON_ARCH__` (75) directly against the driver's raw, BCD-packed `ARCH_VER` (0x8c75 = 35957) with no decode at all, which can never agree on any real device and would have refused every session unconditionally; extracting and adapting `htpdrv_get_arch`'s decode is the fix | +| `exec/quant.py` | `ggml/src/ggml-quants.c` `quantize_row_q4_0_ref` and `ggml/src/ggml-common.h` `block_q4_0` | the **q4_0 block format and the quantization arithmetic**, transcribed rather than copied: the 18-byte block (one fp16 scale then 32 nibbles), the `j`/`j+16` nibble pairing, the signed `d = max / -8` scale, and the `min(15, (int8_t)(x * id + 8.5f))` truncating quantize step. hexlib needs it because 75 of the encoder's plan steps take a q4_0 weight and hexlib has no checkpoint loader, so nothing else in the tree can produce one. It is a reference implementation in numpy for tests and the end-to-end run -- not on any hot path -- and it is bound to an independent scalar transcription of the same upstream loop by `tests/test_quant_q4_0.py`, byte for byte | +| `runtime/host/buffers.c` | `htp-drv.cpp` | the sequence, not the code. It performs the same `rpcmem_alloc` / `rpcmem_to_fd` / `fastrpc_mmap` calls that `htp-drv.cpp` wraps, written from the SDK's own documented call order rather than copied — upstream's allocation call sites live in `htp-drv.cpp`'s caller, not in the file the `driver.c` row above already attributes | + +**Deliberately not adapted:** `dspqueue` dispatch (`htp_main_thread`, +`htp_packet_callback`, `process_opbatch`), because it has no simulator path; +`htp_tensor`'s `ne`/`nb` strides, because hexlib uses an enumerated layout; and +the ggml opcode enum. + +**One upstream defect is fixed rather than carried over:** `mmap_buf` returns +silently with `base == 0` when all mmap slots are occupied, after which +`prep_tensor` computes `0 + offset` and the kernel reads or writes a small bogus +address; it also `abort()`s on a failed mapping. hexlib returns +`HEXLIB_DSP_ERR_NO_MMAP_SLOT` / `HEXLIB_DSP_ERR_MMAP_FAILED` and runs no op. + +This is **adapted, not vendored** — unlike `include/hexlib/hvx/`, which is +byte-identical upstream and must never be edited in place. hexlib still has no +build or runtime dependency on llama.cpp or ggml. ## Adapted, not vendored: hexbench (same author) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 0a1e639..8c23c62 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -50,10 +50,20 @@ A kernel clears these one at a time, and is unsupported until it clears all six. documented authorship record, and a comment explaining *why* it is fast. 5. **Adversarial** — the `nearmiss_*.c` variants must fail, and `spec.json` edge cases become named tests. -6. **Silicon** — batched QDC or local device, sim-vs-silicon drift recorded. **Not - implemented in this plan.** Gate 6, the `local` and `qdc` backends, and drift - recording arrive with the silicon-path plan. A kernel that has cleared gates 1-5 - here has not cleared gate 6, and hexlib does not currently have a way to run it. +6. **Silicon** — batched QDC or local device, sim-vs-silicon drift recorded. **The + transport now exists; the per-kernel record does not.** A QDC session with + `--stage-dir` pushes a batch blob and its arena to an SM8650, runs it, and pulls the + arena back — the whole encoder has gone through it in a single FastRPC invoke, and + the one sim-vs-silicon cycle comparison that produced is in `README.md`. What is + still missing is a *per-kernel* drift record and a place to put it, so a kernel that + has cleared gates 1-5 has still not cleared gate 6. Do not mark one as gate-6 clear + on the strength of the whole-encoder run. + + **An HMX kernel cannot clear gates 3 and 5 on the standalone-ELF path at all** — HMX + needs power, acquisition, a lock and possibly a dedicated thread, none of which exist + in a program with no protection domain around it. HMX work belongs on the QuRT-hosted + batch path; see `kernels/hmx_matmul_fp16/README.md`, which is a worked example of a + kernel committed *without* a `RESULT.md` because it does not gate. `hexlib new-kernel ` scaffolds a directory that satisfies the structural half of gates 1-5 (required files, a near-miss stub, a matching `spec.json`); `hexlib diff --git a/README.md b/README.md index 7625130..ee59a37 100644 --- a/README.md +++ b/README.md @@ -1,111 +1,357 @@ # hexlib -A kernel library and programming model for Qualcomm Hexagon NPUs. hexlib gives -developers callable, verified, cycle-measured kernels, and gives kernel authors a -documented way to write and run their own, targeting a single NSP (v75) first. - -**The Hexagon SDK is required to use hexlib at all** — not just to contribute to it. -Every command below that touches a kernel (`hexlib test`, and anything that compiles -or simulates) needs `hexagon-clang` and `hexagon-sim` from the SDK, discovered through -`HEXAGON_SDK_ROOT`. hexlib never vendors, bundles, or fetches the SDK; it is -license-restricted and you obtain it yourself. If you don't have it, you can still -read the docs, write scalar reference implementations, and design `spec.json` files -(see `CONTRIBUTING.md`), but you cannot build or run a kernel. - -## What v1 covers - -This is the **simulation path**. Kernels compile with `hexagon-clang` and run on -`hexagon-sim`; correctness, cycle counts, and HVX/HMX use (proven from the compiled -ELF, not from source text or a self-reported flag) all come from the simulator. -**Device backends (`--device local`, `--device qdc`) and silicon validation (gate 6) -are not implemented in this plan** — they arrive with the silicon-path plan. Nothing -here should be read as a working device pipeline; `hexlib test --device local` and -`--device qdc` currently just print that the backend isn't implemented yet. - -## The model path - -`hexlib plan qwen35 --print` compiles the Qwen3.5-0.8B vision encoder to a VTCM -and DMA plan and prints it: the high-water mark, the predicted DDR traffic, and -the op kinds that still have no kernel. It needs no SDK, no simulator and no -device — the whole graph and scheduling layer is pure host Python, which is -where most of the contribution surface is. See -[`docs/superpowers/specs/2026-08-09-vlm-encoder-design.md`](docs/superpowers/specs/2026-08-09-vlm-encoder-design.md). +**A compiler and kernel library for Qualcomm Hexagon NPUs.** + +hexlib takes a neural network, compiles it to an explicit VTCM and DMA plan, and +executes that plan against hand-written HVX/HMX kernels on a Hexagon NSP — targeting +v75 (Snapdragon 8 Gen 3 / SM8650) first. Every kernel arrives with a correctness +verdict against a scalar reference, a cycle count, ELF-level proof that the vector or +matrix unit was genuinely used, and near-miss variants that must still fail. + +Two things make it unusual. **The scheduling layer is pure host Python** — the graph, +the passes, the allocator and the plan need no SDK, no simulator and no device, which +is where most of the contribution surface lives. And **the verification is adversarial +by construction**: a kernel does not pass because it produced plausible numbers, it +passes because a deliberately-broken variant of it demonstrably fails. + +--- + +## Status + +Honest state, because this project's own worst recurring bug is a claim that outruns +its evidence. + +| | what works | where | +|---|---|---| +| **Kernel pipeline** | ✅ shipped | write a `.c`, run `hexlib test`, get a gate verdict + cycles + ELF proof | +| **Graph → plan compiler** | ✅ shipped | `hexlib plan qwen35 --print`, no SDK needed | +| **Plan executor** | ✅ shipped | whole encoder runs end to end, and now against the **real** Qwen3.5-0.8B checkpoint at 256×256 | +| **12 kernels** | ✅ gated | **every one of the encoder's 259 real-work ops selects a kernel** | +| **Silicon-path runtime** | ✅ runs on hardware | whole plan, **one** FastRPC invoke, on an SM8650 | +| **On-device execution** | ⚠️ tiny config only | the 256×256 encoder has never been run on a device | +| **HMX** | ❌ blocked | a kernel exists and does **not** gate — see below | + +Cycle counts, unless a row says silicon, come from `hexagon-sim` under a pinned bus +model. The simulator is cycle-*approximate* — see +[`docs/hardware/simulator-accuracy.md`](docs/hardware/simulator-accuracy.md) for where +it is most likely to drift. On the one workload measured both ways it runs 6.7% high — +see [Results](#results-pytorch--simulator--silicon) below, which chains PyTorch, the +simulator and the device. + +### The Hexagon SDK is required to build or run a kernel + +Not just to contribute — to use hexlib on a kernel at all. `hexagon-clang` and +`hexagon-sim` are discovered through `HEXAGON_SDK_ROOT`. **hexlib never vendors, +bundles, or fetches the SDK**; it is licence-restricted and you obtain it yourself. + +Without it you can still do a great deal, and it is the most useful work available: +the entire graph and scheduling layer, the op registry, the numpy reference executor, +scalar baselines, `spec.json` contracts and test vectors are all pure Python. See +[`CONTRIBUTING.md`](CONTRIBUTING.md) for the tier system. + +--- ## Quickstart ```bash pip install -e . -hexlib new-kernel my_kernel # scaffolds kernels/my_kernel -hexlib test kernels/my_kernel # builds it, simulates it, prints a gate table + +# No SDK required — compile a model to a VTCM/DMA plan and inspect it +hexlib plan qwen35 --print + +# SDK required — scaffold, then gate, a kernel +hexlib new-kernel my_kernel +hexlib test my_kernel +``` + +`hexlib new-kernel` writes a conforming directory: `kernel.c`, `kernel_api.h`, +`baseline.c` (your scalar reference), `harness.c` (which builds its own inputs and so +cannot be handed a passing answer), a `nearmiss_*.c` stub, and `spec.json`. + +`hexlib test` compiles it, runs it on the simulator, disassembles the ELF to prove HVX +or HMX was used, confirms every near-miss variant is still rejected, and writes a +`RESULT.md` you attach to a PR. + +--- + +## How it works + +``` + model (PyTorch/HF config) + │ + ▼ + graph IR ── op registry (13 kinds, each with a numpy reference) + │ + ├── shapes → fuse → order → liveness → VTCM alloc → DMA + │ │ + ▼ ▼ + Plan ─────────────────────────────────────────► serialized, diffable + │ + ▼ + executor ── replays the plan through a real VTCM byte image + │ + ├──► numpy reference (any op without a kernel) + ├──► standalone ELF path (one simulator launch per op) + └──► DSP skel batch path (one FastRPC invoke per batch) ◄── the silicon path +``` + +Every pass is a pure function, so ~80% of the system is testable with no SDK and no +device. The plan is the contract between the two halves: the compiler decides *where +every byte lives and when it moves*, and the executor is deliberately dumb. + +**Layout is an enumerated value, not `ne`/`nb` strides.** This makes "the kernel got +un-repacked weights" a plan-time error rather than silent numerical corruption, and +strides cannot express a VTCM-resident tile of a DDR tensor — which is the central +object the compiler manipulates. + +Design docs: [encoder](docs/superpowers/specs/2026-08-09-vlm-encoder-design.md) · +[silicon path](docs/superpowers/specs/2026-08-10-silicon-path-runtime-design.md) · +[architecture overview](docs/architecture.md) + +--- + +## Kernels + +Twelve kernels through gates 1–5. Cycles are `kernel_cycles` — the DSP-side count for +the kernel call alone, never whole-program `cycles`, which carries 155k–190k of roughly +constant harness and CRT overhead. Each row's number and near-miss set is the +tool-generated `kernels//RESULT.md`, not a figure retyped here. + +| kernel | cycles | max abs error | notes | +|---|---|---|---| +| `transpose_th_fp16` | **706** | 0 | perm (1,0,2), both directions | +| `scale_fp16` | **886** | 0 | factor 0.125 is a power of two, so no mantissa bit is lost | +| `add_fp16` | **1139** | 0 | 1 ULP class: the hardware's fp16 narrowing is not IEEE round-to-nearest-even | +| `cast_f32_f16` | **1176** | 0.5 | needs a lane deal — the widening conversion interleaves | +| `rope_2d_fp16` | **1212** | 6.10e-05 | six near-misses, the largest set here — pairing, table indexing and sign are each independently wrong-able | +| `rmsnorm_fp16` | **2231** | 1.95e-03 | **31.13×** over a 69443-cycle scalar baseline | +| `transpose_hd_fp16` | **5606** | 0 | perm (0,2,1); movement-only, no arithmetic in the ELF | +| `softmax_fp16` | **11292** | 0 | fp32 exp path deliberately — see the upstream findings below | +| `layernorm_fp16` | 111088 | 4.88e-04 | **a first rung, not a result** — reductions still scalar | +| `matmul_fp16` | 995714 | 9.77e-04 | the encoder's 24 unfused attention matmuls; gated at N=200 | +| `patchify_fp32` | 2018331 | 0 | runs once, at the input; emits merge-block order, not raster | +| `matmul_epilogue_fp16` | 14310406 | 2.44e-04 | fused matmul+bias+activation over q4_0 weights — 75 of 308 steps and 95.5% of all DDR traffic | + +`layernorm_fp16`'s number is deliberately unoptimised: the affine epilogue is +vectorised, both reductions are not. It was left scalar so the reduction has a +*recorded* baseline to beat rather than an assumed one. A rotate-and-add butterfly +already exists in `kernels/rmsnorm_fp16/`. `matmul_epilogue_fp16` is the same story at +the other end of the scale — it is correct and gated, and nothing has been optimised +about it yet. + +Full bake-off records, including the candidates that **lost**, live in each kernel's +`BAKEOFF.md`. + +**A thirteenth kernel is committed and does not gate.** +[`kernels/hmx_matmul_fp16/`](kernels/hmx_matmul_fp16/README.md) has no `RESULT.md`, +deliberately, because the simulator faults before the harness prints a verdict — and +the gate reports that as a failure, not a pass. The reason is now known exactly and it +is structural: **HMX cannot be used inside `hexlib test`'s standalone ELF at all.** The +directory's `README.md` records the four things it needs, the two real defects found on +the way, and what to do next. + +### Target model + +The Qwen3.5-0.8B vision encoder, at 256×256: + ``` +VTCM high water 5,355,648 of 8,388,608 bytes (63.8%) +DDR ↔ VTCM 58,643,456 bytes +Plan steps 308 (396 ops before fusion) +``` + +`matmul_epilogue` alone accounts for 56.0 of those 58.6 MB — 95.5% of all the traffic. -`hexlib new-kernel` writes a conforming, empty kernel directory (`kernel_api.h`, -`baseline.c`, `harness.c`, a `nearmiss_*.c` stub, `spec.json`, `README.md`). Fill in -the contract and the scalar reference, then write the kernel. `hexlib test` builds it -against the SDK, runs it on the simulator, disassembles the ELF to prove HVX/HMX use, -confirms the near-miss variant is still rejected, and writes a result table you attach -to your PR. See `CONTRIBUTING.md` for the full gate sequence. +Every one of the 259 real-work steps now selects a dispatchable kernel; the remaining 49 +are reshapes, which are pure metadata once resident and need none. That claim is +*asked*, not counted — `hexlib/tests/test_encoder_dispatch_coverage.py` puts every step +of the real compiled plan through `select()`, because this project published a wrong +coverage number three times by counting kernel directories instead. (`hexlib plan +--print` still lists all eleven kinds under "no kernel": `OpDef.kernel` is a separate +registry that is still `None` everywhere, and wiring it moves figures several tests pin.) -## Worked example: rmsnorm_fp16 +### Accuracy — measured on the real checkpoint -The one kernel shipped so far. RMSNorm with a per-column gain, row-wise, fp16; -shape `R=8, C=128, eps=1e-5`. Full record in -[`kernels/rmsnorm_fp16/BAKEOFF.md`](kernels/rmsnorm_fp16/BAKEOFF.md) and -[`kernels/rmsnorm_fp16/RESULT.md`](kernels/rmsnorm_fp16/RESULT.md). +The shipped `Qwen/Qwen3.5-0.8B` vision weights at 256×256, against `transformers`, with +fp32 arithmetic on **both** sides so nothing but the weight format differs: -| gate | result | +| | cosine vs `transformers` | |---|---| -| correct | PASS | -| kernel_cycles | 2021 | -| accel (ELF-proven) | hvx, hvx-compute | -| near-miss `nearmiss_mean_not_rms.c` | correctly rejected | -| near-miss `nearmiss_no_eps.c` | correctly rejected | -| **gate** | **PASS** | - -target `v75` · toolchain `19.0.04` - -The winning candidate (an adapted v6 `rmsnorm_gain_fp16`) measured **2021 kernel -cycles** against a **69443**-cycle scalar baseline — **34.36x** — and beat the other -HVX candidate measured for this kernel (an adapted v6 `fp16_rmsnorm`, 10498 cycles) by -5.19x. These are numbers from `hexagon-sim` under a pinned bus model -(`--timing --buspenalty 75 --busratio 2`), not silicon measurements — see -[`docs/hardware/simulator-accuracy.md`](docs/hardware/simulator-accuracy.md) for what -the simulator does and does not guarantee. `kernel_cycles` is the DSP-side cycle count -for the kernel call alone; never compare whole-program `cycles`, which includes -155k-190k cycles of roughly constant harness/CRT overhead. +| hexlib fp32, full-precision weights | **0.9999999999** | +| q8_0 weights | **0.999002** (max_rel 7.98e-02, 1.89× the weight bytes) | +| q4_0 weights | **0.867606** (max_rel 4.32e-01) | +| fp16 *activations*, weights exact | 0.999991 — **15,000× smaller than quantization** | + +**q4_0 is not enough for this encoder, and it is not the kernels' fault.** Mixed +precision was measured and rejected: the error is *diffuse*, so keeping the patch +embedding, the merger and all of attention at 8 bits while the MLPs stay q4_0 still only +reaches 0.913. Smaller blocks were measured and rejected: block=8 spends 6 bits/value on +more scales for 0.937, where q8_0 spends 8.5 on mantissa for 0.999. Fusion and HMX +cannot recover it either — fusion's entire budget is the fp16-activation term, 8.8e-06 +of cosine, and HMX is a speed lever, not an accuracy one. + +**A tiny-config sweep said the opposite of all of this** and nearly sent the work the +wrong way: at 2 layers with random weights it ranked the merger dominant, put `wq`/`wk` +at the noise floor, and made mixed precision look like a 30× win. Two layers is not +enough depth for diffuse error to compound, and random normals have no outliers. **Do +not tune quantization against the tiny config.** + +The tiny config (2 layers, hidden 64, image 32) is still what the committed golden +vectors cover, and still what runs with no torch at test time: 4.47e-08 vs upstream, +4.470e-08 through the plan executor in fp32, 6.747e-05 in fp16. + +*(Corrected 2026-08-11: the tiny-config figures previously sat directly under the "at +256×256" heading with no scale caveat, which read as a claim about the full model. The +full-size reference that was missing then now exists — it is the first row of the table +above.)* + +--- + +## Results: PyTorch → simulator → silicon + +The chain is four links, and each is checked against the one before it rather than +against an assumption. Read down the table: what upstream `transformers` computes, what +hexlib's numpy reference computes from the same weights, what the Hexagon simulator +computes running the real kernels, and what an SM8650 computes running the same blob. + +| link | what is compared | scale | result | +|---|---|---|---| +| **PyTorch → hexlib reference** | upstream `transformers` vs the graph + passes + plan executor, fp32 both sides, real `Qwen/Qwen3.5-0.8B` weights | **256×256, 12 layers** | cosine **0.9999999999** | +| **PyTorch → hexlib reference** | same, against committed golden vectors with no torch at test time | tiny (2 layers) | **4.47e-08** max abs | +| **hexlib reference → simulator, per-op** | every op with a kernel routed through `hexagon-sim`, one launch each, vs the numpy registry over the identical plan and feeds | tiny (49 ops) | max rel **1.1319e-03**, corr **1.000000** | +| **hexlib reference → simulator, one invoke** | the whole plan as a *single* batch blob, one simulator entry | tiny (49 ops) | the same figure | +| **hexlib reference → silicon, one invoke** | the same blob, one FastRPC invoke on an SM8650 | tiny (49 ops) | the same figure — cosine **0.99999967** | + +**Three transports, one answer.** The per-op simulator path, the single-invoke simulator +path and the device agree to the digits printed above; that agreement is the point, not +the individual number. The residual 1.13e-03 is fp16 activations compounding across the +encoder, not a wrong kernel — every intermediate narrows to fp16 and feeds the next op. + +Cycles, on the one workload measured both ways: + +| | cycles_total | note | +|---|---|---| +| simulator, single invoke | 302,087,160 | no `--timing --buspenalty 75 --busratio 2` on the batch path | +| SM8650, single invoke | **283,220,278** | 0 ops not OK | + +The simulator is **6.7% high** here. That is a bare comparison of two numbers, not a +calibrated drift figure — the flags differ, and one workload is not a model. + +**Everything on the device is the tiny config.** The 256×256 encoder compiles to a 46 KB +blob over a 203.7 MB arena and **has never been run on a device**. Nothing above is a +full-model silicon result. The reason is cost, not capability: a full-size *simulator* +run is 259 separate `hexagon-sim` launches, which is hours for a signal a 49-op graph +gives in minutes, and QDC sessions bill for their whole timeout. + +Three things silicon settled that no simulator test could: + +- **`cycles_total` is non-zero in a user-mode unsigned PD.** `SYSCFG.PCYCLEEN` cannot be + set there, and a dead counter would have invalidated every cycle figure this project + has ever reported. It is alive. +- **`arch_ver` is 0x8c75, bit-identical to the simulator**, with `unsigned_pd_support=1` + and `vtcm_total_bytes=8388608`. An assertion that had never actually been checked. +- **A real defect the simulator structurally could not catch.** FastRPC keeps the two + caches coherent only for buffers passed as invoke arguments; hexlib's are mapped out + of band by `fastrpc_mmap` and named by fd, so nothing wrote them back. `--self-test` + returned 3859/4100 values not bit-exact and the encoder's last op read all zero. It is + *ordered* corruption — early writes landed, the final op's 1024 bytes never left the + cache — which is what identified it, because random corruption does not sort itself by + age. Fixed with `qurt_mem_cache_clean`: **invalidate before and flush after**. Flush + alone works for exactly one invoke per session and then silently computes on old data. + +**Device cycle counts from before that fix are still valid** — PCYCLE is a register read. +Device *data* from before it is not. + +Reaching a device at all needs three non-obvious things, none of them in the SDK +signature: the SSH key must be the one **QDC** issued rather than your own, +`session_parameters=[SSHONLY]` is what provisions SSH at all, and what you get back is +an **adb tunnel**, not a shell — nothing runs remotely, everything goes through a local +`adb -P `. Sessions bill for the whole timeout, not for what you use. + +--- + +## How correctness is established + +The gates exist because of specific ways this project has been wrong before, each +recorded in [`CONTRIBUTING.md`](CONTRIBUTING.md): + +- **The harness builds its own inputs** and never reads a file, so it cannot be handed + a passing answer. The runner that *does* read files is a separate binary that prints + no verdict. Both facts are asserted by a test. +- **Acceleration is proven from the compiled ELF**, by disassembly — not from source + text, and not from a self-reported flag. +- **Near-misses must fail.** A dropped tail, a mean instead of an RMS, a forgotten lane + deal: each is committed as a variant that the harness has to reject. One of them + found a real bug in hexlib's own simulator wrapper. +- **A tolerance wide enough for the widest shape can be wider than the bug it is meant + to catch.** `layernorm`'s unbiased-variance near-miss is a 0.065% error at C=768, + where fp16's own precision is ~0.05% — indistinguishable. It was *wrongly accepted* + on the first run. The fix was a shape where the bug is bigger (C=64, 0.79%), not a + tolerance argued down. +- **Absence is never success.** Status codes start at 1, so a zero-filled response + buffer that was never written cannot read as OK. + +--- ## Repository layout ``` -hexlib/ the CLI and verification pipeline (new-kernel, validate, test) -include/hexlib/ DSP-side headers a kernel #includes, including vendored HVX math -kernels/ one self-contained directory per promoted kernel +hexlib/graph/ IR, op registry, and the pass pipeline (pure Python, no SDK) +hexlib/exec/ the plan executor and its three dispatch backends +hexlib/runtime/ the silicon path: wire format, IDL, DSP skel, host, build recipes +hexlib/device/ device backends (QDC job plumbing) +hexlib/tests/ the offline suite — runs without an SDK, except where marked +include/hexlib/ DSP-side headers a kernel includes, incl. vendored HVX math +kernels/ one self-contained directory per gated kernel +docs/hvx/ learning HVX: a function-by-function tour of the vendored headers docs/hardware/ measured hardware notes (HMX int8, simulator accuracy) +docs/research/ audit records — what was read directly vs. inferred ``` ## Documentation -- [`CONTRIBUTING.md`](CONTRIBUTING.md) — the three tiers, the six gates, the - bake-off, and what CI does and does not check. -- [`ROADMAP.md`](ROADMAP.md) — the op backlog, with status and tier, so tier-0 work - (no SDK needed) is always visible. -- [`ATTRIBUTION.md`](ATTRIBUTION.md) — every vendored source, its license, and the - commit it came from. -- [`docs/hvx/`](docs/hvx/README.md) — **learning HVX.** A guided, function-by-function - tour of the vendored headers: the vector types and predicates, alignment handling, - horizontal reductions, transcendentals built from polynomial approximation, division by - Newton–Raphson, and the reduce-then-broadcast pattern that every transformer kernel is a - variation on. Start here if you have never written HVX. -- [`docs/hardware/hmx-int8.md`](docs/hardware/hmx-int8.md) — the measured HMX int8 - MAC sequence. +**Start here** +- [`docs/architecture.md`](docs/architecture.md) — how the pieces fit together +- [`CONTRIBUTING.md`](CONTRIBUTING.md) — the tiers, the six gates, what CI does and does not check +- [`ROADMAP.md`](ROADMAP.md) — the op backlog, so tier-0 work is always visible + +**Learning HVX** +- [`docs/hvx/`](docs/hvx/README.md) — a guided tour of all 22 vendored headers: vector + types and predicates, alignment, horizontal reductions, transcendentals from + polynomial approximation, division by Newton–Raphson, and the reduce-then-broadcast + pattern nearly every transformer kernel is a variation on. **They record what they + could not explain** rather than smoothing it over — the open questions are written + down deliberately. (No count is given here on purpose: a number in a doc goes stale, + and this one had.) +- [`docs/hvx/upstream-findings.md`](docs/hvx/upstream-findings.md) — three real defects + found in upstream llama.cpp while writing that tour, with evidence. hexlib calls none + of them; the worst is a coefficient off by 234,118× inside an fp16 exponential. + +**Hardware reality** - [`docs/hardware/simulator-accuracy.md`](docs/hardware/simulator-accuracy.md) — what - "cycle-approximate" means and where the simulator is most likely to drift from - silicon. + "cycle-approximate" means and where it drifts +- [`docs/hardware/hmx-int8.md`](docs/hardware/hmx-int8.md) — the measured HMX int8 MAC sequence - [`docs/research/oracle-provenance.md`](docs/research/oracle-provenance.md) — what the - committed vision-encoder golden vectors prove, and what they do not. + committed golden vectors prove, and what they do not + +**Project state** +- [`docs/STATE.md`](docs/STATE.md) — the working handoff record: what is decided, what + is proven, what is merely claimed, and every open question + +## Contributing + +Tier-0 work needs no SDK and no hardware: scalar baselines, `spec.json` contracts, test +vectors, near-miss variants, documentation, and anything in the graph or pass pipeline. +[`ROADMAP.md`](ROADMAP.md) keeps that work visible. Read +[`CONTRIBUTING.md`](CONTRIBUTING.md) first — the gates are non-negotiable, and the +reason each one exists is written down. ## License -[MIT](LICENSE). hexlib also vendors MIT-licensed code from llama.cpp's ggml-hexagon -backend, which carries its own attribution requirement — see +[MIT](LICENSE). + +hexlib **vendors** MIT-licensed HVX math headers from llama.cpp's `ggml-hexagon` +backend (byte-identical, never edited in place) and **adapts** its FastRPC runtime +(rewritten in hexlib's own tree). Both carry attribution requirements, and every source +— with its licence and the upstream commit it came from — is recorded in [`ATTRIBUTION.md`](ATTRIBUTION.md). diff --git a/ROADMAP.md b/ROADMAP.md index 389c940..d9f2cef 100644 --- a/ROADMAP.md +++ b/ROADMAP.md @@ -11,20 +11,30 @@ be attempted, not by difficulty. | op | status | tier needed to advance it | subsystem checkpoint | |---|---|---|---| -| `rmsnorm_fp16` | **done** — gates 1-5 cleared, gate 6 (silicon) pending the silicon-path plan | 1 (kernel exists; tier-0 work: additional near-misses, wider shape coverage in `spec.json`) | reduction | +| `rmsnorm_fp16` | **done** — gates 1-5 cleared | 1 (kernel exists; tier-0 work: additional near-misses, wider shape coverage in `spec.json`) | reduction | | `rmsnorm_f32` | not started | 0 (baseline, spec, test vectors) then 1 (kernel) | competitive comparison | -| `softmax_fp16` | not started | 0 then 1 | transcendental | -| `rope_fp16` | not started | 0 then 1 | elementwise with position state | -| `matmul_i8_hmx` | not started | 0 then 1 (needs `caps: ["hmx"]`) | HMX | +| `softmax_fp16` | **done** — gates 1-5 cleared, 11292 cycles, dispatchable | 1 | transcendental | +| `rope_fp16` | **done as `rope_2d_fp16`** — gates 1-5 cleared, 1212 cycles, dispatchable | 1 | elementwise with position state | +| `matmul_i8_hmx` | **blocked, and the reason is known** — see `kernels/hmx_matmul_fp16/README.md` | 1, on the **batch** path, not the gate path | HMX | | `flash_attn_fp16` | not started | 0 then 1 | full composition | +**Gate 6 (silicon) is no longer theoretical for any of these.** A QDC session with +`--stage-dir` pushes a batch blob and its arena, runs it on an SM8650 and pulls the +arena back; the whole encoder has gone through it. What is still missing is *per-kernel* +sim-vs-silicon drift recording, not the transport. See the top-level `README.md`. + ## Notes per op **`rmsnorm_fp16` (done).** The first kernel through all five simulation-path gates. Bake-off in `kernels/rmsnorm_fp16/BAKEOFF.md`: an adapted v6 `rmsnorm_gain_fp16` won -at 2021 kernel cycles (34.36x over a 69443-cycle scalar baseline), beating an adapted -v6 `fp16_rmsnorm` (10498 cycles) by 5.19x. ggml-hexagon's `hvx-norm.h` was not -evaluated against it — see below. +at **2231** kernel cycles (**31.13x** over a 69443-cycle scalar baseline), beating an +adapted v6 `fp16_rmsnorm` (10498 cycles) by **4.71x**. ggml-hexagon's `hvx-norm.h` was +not evaluated against it — see below. + +*(Corrected 2026-08-11: this paragraph read 2021 cycles / 34.36x / 5.19x, which are the +pre-fix numbers. `BAKEOFF.md`'s "Fix round 2" moved the winner 2021 → 2231 when an +out-of-bounds read was fixed, and `kernels/rmsnorm_fp16/RESULT.md` — the tool-generated +record — has said 2231 since. README.md carried the same stale figures.)* **`rmsnorm_f32` (next).** The first direct head-to-head against production ggml-hexagon code. `hvx_fast_rms_norm_mul_f32` (`include/hexlib/hvx/hvx-norm.h`) is @@ -33,20 +43,39 @@ not be baked off fairly against `rmsnorm_fp16`'s fp16 candidates — comparing a implementation to fp16 ones would measure the dtype, not the implementation. An `rmsnorm_f32` kernel on identical fp32 shapes settles that comparison honestly. -**`softmax_fp16`.** First transcendental op on the roadmap — exercises the vendored -`hvx-exp.h` reduction-and-normalize pattern rather than `hvx-norm.h`'s reduce-and-scale -one. +**`softmax_fp16` (done).** First transcendental op on the roadmap — exercises the +vendored `hvx-exp.h` reduction-and-normalize pattern rather than `hvx-norm.h`'s +reduce-and-scale one. **11292 kernel cycles**, max abs error 0, three near-misses (no +max subtraction, fp16 sum, wrong axis). It uses the **fp32** exp path deliberately: the +fp16 one is broken upstream — see `docs/hvx/upstream-findings.md`. -**`rope_fp16`.** First op with per-position state (rotation angle depends on sequence -position, not just on the row/column shape), rather than a pure per-row or per-column -reduction. +**`rope_fp16` (done, as `rope_2d_fp16`).** First op with per-position state (rotation +angle depends on sequence position, not just on the row/column shape), rather than a +pure per-row or per-column reduction. The encoder needs the 2D variant, so that is what +was built. **1212 kernel cycles**, max abs error 6.10e-05, and the **largest near-miss +set in the repository — six**: adjacent pairing, fp16 accumulate, negated term, partial +rotation, swapped cos/sin, and a table indexed by head. Position state is wrong-able in +more independent ways than any other op here, which is the whole reason it is on this +list. -**`matmul_i8_hmx`.** First kernel requiring HMX (`caps: ["hmx"]`), and the first -integer kernel on this roadmap, so it is bit-exact rather than tolerance-compared (see -`CONTRIBUTING.md`). Read `docs/hardware/hmx-int8.md` before attempting this one: the -HMX int8 tile engine's readout applies a non-unit fp8-like scale, so an -exact-int32-out matmul is not directly representable through it, and this needs to be -a *quantizing* kernel design, not a literal int8 GEMM. +**`matmul_i8_hmx` (blocked, upstream of the kernel).** First kernel requiring HMX +(`caps: ["hmx"]`), and the first integer kernel on this roadmap, so it is bit-exact +rather than tolerance-compared (see `CONTRIBUTING.md`). Read +`docs/hardware/hmx-int8.md` before attempting this one: the HMX int8 tile engine's +readout applies a non-unit fp8-like scale, so an exact-int32-out matmul is not directly +representable through it, and this needs to be a *quantizing* kernel design, not a +literal int8 GEMM. + +**Attempt the fp16 tile first, and know what stops it.** `kernels/hmx_matmul_fp16/` is +committed **without a `RESULT.md`** because it does not gate — and the reason is +structural, not a bug in the kernel: **HMX needs power, acquisition, a lock and possibly +a dedicated thread, and a standalone simulator ELF has none of them.** That is how every +other kernel here is gated, so **HMX work belongs on the QuRT-hosted batch path**. +`kernels/hmx_matmul_fp16/README.md` has the four requirements with upstream line +references, the two real defects already found (`HMX_SET_BIAS` takes a 256-byte +per-column scale/bias area, not a 32×32 tile; `mxmem` needs 2048-byte alignment, not +`HEXLIB_ALIGN`'s 128), and the cheapest next move. Do that before writing more HMX +kernel code. **`flash_attn_fp16`.** The full composition — reduction, transcendental, and elementwise state together, on the hardest and highest-value kernel. Deliberately @@ -69,26 +98,50 @@ one moving the most DDR traffic and therefore the one most likely to be memory-b in practice. Kinds tied at zero bytes moved (their inputs are already VTCM-resident; they cost compute cycles, not DMA) are broken by step count, descending. -| op kind | steps | predicted bytes moved | status | related backlog kernel | +**Status column updated 2026-08-13, and the whole table is now green.** `OpDef.kernel` +is still `None` for every kind — wiring that registry to the kernel directories is a +separate change, tracked in `docs/STATE.md`'s open items, because it moves figures +several tests pin. So `hexlib plan --print` still lists all eleven kinds under +"NOT IMPLEMENTED", and that output is now **actively misleading**: `OpDef.kernel` is not +what decides dispatch. `RunnerSpec` is, and every kind below has one. + +| op kind | steps | predicted bytes moved | status | notes | |---|---|---|---|---| -| `matmul_epilogue` | 75 | 55,999,488 | `kernel: None` | fused matmul+bias, closest to `matmul_i8_hmx` | -| `patchify` | 1 | 1,572,864 | `kernel: None` | runs once, at the input -- now includes the image's own DMA-in | -| `add` | 25 | 786,432 | `kernel: None` | residual add, elementwise | -| `layernorm` | 25 | 153,600 | `kernel: None` | reduction, adjacent to `rmsnorm_fp16`/`rmsnorm_f32` | -| `rope_2d` | 24 | 131,072 | `kernel: None` | 2D variant of `rope_fp16` | -| `transpose` | 60 | 0 | `kernel: None` | layout op, no DDR traffic once resident | -| `reshape` | 49 | 0 | `kernel: None` | layout op, no DDR traffic once resident | -| `matmul` | 24 | 0 | `kernel: None` | unfused QK^T / attn·V, compute-bound not DMA-bound | -| `scale` | 12 | 0 | `kernel: None` | elementwise | -| `softmax` | 12 | 0 | `kernel: None` | same subsystem checkpoint as `softmax_fp16` | -| `cast` | 1 | 0 | `kernel: None` | runs once, at the input | +| `matmul_epilogue` | 75 | 55,999,488 | ✅ `matmul_epilogue_fp16`, gated, dispatchable | fused matmul+bias+activation over q4_0 weights. 95.5% of all DDR traffic; the highest-value thing to *optimise*, now that it exists | +| `patchify` | 1 | 1,572,864 | ✅ `patchify_fp32`, gated, dispatchable | runs once, at the input. Emits merge-block order, not raster | +| `add` | 25 | 786,432 | ✅ `add_fp16`, gated, dispatchable | residual add, elementwise | +| `layernorm` | 25 | 153,600 | ✅ `layernorm_fp16`, gated, dispatchable | reduction. Gated long before it was dispatchable — see below | +| `rope_2d` | 24 | 131,072 | ✅ `rope_2d_fp16`, gated, dispatchable | 2D variant; six near-misses | +| `transpose` | 60 | 0 | ✅ **two** kernels, both dispatchable | `transpose_th_fp16` for perm (1,0,2), `transpose_hd_fp16` for perm (0,2,1). One kind, two `RunnerSpec` variants — which is why `SPECS` is keyed by variant, not by kind | +| `reshape` | 49 | 0 | ✅ needs no kernel | pure metadata once resident | +| `matmul` | 24 | 0 | ✅ `matmul_fp16`, gated, dispatchable | unfused QK^T / attn·V, compute-bound not DMA-bound. HVX today; HMX is the speed lever it has not had yet | +| `scale` | 12 | 0 | ✅ `scale_fp16`, gated, dispatchable | elementwise | +| `softmax` | 12 | 0 | ✅ `softmax_fp16`, gated, dispatchable | uses the fp32 exp path — see `docs/hvx/upstream-findings.md` | +| `cast` | 1 | 0 | ✅ `cast_f32_f16`, gated, dispatchable | runs once, at the input | + +**All 259 of the ops that need a kernel are covered and dispatchable**; the other 49 of +the 308 steps are reshapes needing none. + +**That number is asked, not counted, and the distinction is the whole point.** +`hexlib/tests/test_encoder_dispatch_coverage.py` puts every step of the real compiled +plan through `select()` and reports which kinds have no reachable kernel. It exists +because "has a gated kernel" is **not** "can be dispatched" — a `RunnerSpec` is what +makes an op reachable on the DSP batch path, and this project published a wrong coverage +number **three separate times** by counting kernel directories instead: `layernorm` was +gated and unreachable, `softmax` and `rope_2d` repeated it the same week, and `matmul` +and `matmul_epilogue` repeated it again. Counting is what goes wrong, so the test does +not count. A second test pins the plan's own size (308 steps, 49 reshapes, 24 `matmul`, +75 `matmul_epilogue`), so a shrunken encoder cannot make the first one pass vacuously. Total across all eleven kinds: `predicted_bytes_moved = 58,643,456` at 256x256, against the measured `vtcm_high_water = 5,355,648` of an 8,388,608-byte budget (63.8%). Both figures now include the image's DMA-in and the encoder output's DMA-out (previously missing entirely -- see the whole-branch fix report), and `vtcm_high_water` now includes the const/weight-streaming region (previously computed and budget-checked, -but never folded into `plan.vtcm` or the reported high-water number). See -`.superpowers/sdd/2026-08-09-vlm-encoder-m1-pass-pipeline/task-9-report.md` for the -original plan and `final-fix-report.md` for the cross-cutting fixes that produced the -numbers above. +but never folded into `plan.vtcm` or the reported high-water number). + +Reproduce both numbers yourself with `hexlib plan qwen35 --print`, which needs no SDK. +The reasoning behind each fix is in the commit that made it — `git log --grep=vtcm` and +`git log --grep=traffic` — and the durable summary is in `docs/STATE.md`. (Earlier +revisions of this section cited report files under `.superpowers/`, which is a +git-ignored agent scratch directory and therefore not something a reader can open.) diff --git a/conftest.py b/conftest.py new file mode 100644 index 0000000..b6787e4 --- /dev/null +++ b/conftest.py @@ -0,0 +1,68 @@ +# conftest.py -- repo root +"""KEEP THE ON-DEVICE TEST OUT OF THE OFFLINE SUITE AS A MECHANISM, NOT A +CONVENTION. + +`hexlib/device/qdc/test_on_device.py` runs ON THE PHONE, inside the QDC +artifact zip, under the farm's own pytest -- never here. Beside it in that zip +sits a flat `utils.py` which it imports as a TOP-LEVEL module (`from utils +import sh, write_qdc_log`), because on the farm the artifact is extracted into +one flat directory with no package around it. In this repo the same file lives +inside the `hexlib.device.qdc` package, so pytest imports it as +`hexlib.device.qdc.test_on_device`, `utils` is not a top-level module, and the +import raises ModuleNotFoundError AT COLLECTION TIME -- which pytest reports as +`Interrupted: 1 error during collection`, exit code 2, and ZERO tests run. + +That is not a hypothetical. `.github/workflows/ci.yml` runs `pytest -q -m "not +sdk"` with NO PATH, so collection starts at the repo root and walks into +`hexlib/device`. Reproduced on this machine before this file existed: + + python -m pytest -q -m "not sdk" + -> ModuleNotFoundError: No module named 'utils' + -> Interrupted: 1 error during collection (exit 2, zero tests run) + +It was latent only because this repo has no git remote yet, so CI has never +actually run. + +WHY `collect_ignore` HERE AND NOT `testpaths` IN pyproject.toml. `testpaths` +applies ONLY when no path is given on the command line. That does cover CI's +bare invocation, which is the case that was broken -- but it covers nothing +else: `pytest .`, `pytest hexlib`, or `pytest hexlib/device` would each walk +back into the on-device file and break collection again, and each is a +plausible thing for a human or a future workflow step to type. `collect_ignore` +in the ROOT conftest.py is consulted during directory collection no matter how +collection was started, so it covers the bare invocation AND every path form +above with one mechanism. `hexlib/tests/test_qdc_on_device_is_excluded.py` +pins both properties by actually running pytest, so this reasoning is checked +rather than merely asserted here. + +WHAT THIS DELIBERATELY DOES NOT DO. It does not make the on-device file +importable, and it must not: the file is correct as written -- flat `import +utils` is what works on the farm. It also does not stop +`pytest hexlib/device/qdc/test_on_device.py` if someone names the file +directly; that is an explicit request, and it will fail loudly on the import +rather than silently pass, which is the right outcome. +""" + +# Paths are relative to this file's directory (the repo root). The whole +# directory, not just the one file: a second on-device test added next to +# test_on_device.py must be excluded for the same reason, without anyone +# having to remember to come back here. +collect_ignore = ["hexlib/device"] + +# The SAME defect by a second route. `hexlib test --device qdc --out DIR` +# stages the artifact into `DIR/qdc_job_stage/`, and that staging tree contains +# a COPY of `test_on_device.py` -- so after any qdc run inside the repo, +# collection walks into it and dies on the flat `import utils` exactly as +# above. Worse than the original: with two `--out` directories there are two +# copies with one basename, and pytest refuses both with "import file mismatch" +# before running a single test. +# +# Reproduced on 2026-08-12 with `_work_qdc` and `_work_qdc2` present: +# python -m pytest -q -m "not sdk" +# -> ERROR _work_qdc2/qdc_job_stage/test_on_device.py +# -> Interrupted: 1 error during collection (5 deselected, 0 run) +# +# A glob, not a fixed name, because `--out` is the caller's to choose and +# `_work` is only the default; anything a run drops beside it is build output, +# never a test this suite should collect. +collect_ignore_glob = ["_work*"] diff --git a/hexlib/cli.py b/hexlib/cli.py index eca3fd3..3fc701b 100644 --- a/hexlib/cli.py +++ b/hexlib/cli.py @@ -8,7 +8,10 @@ import argparse import os +import re import sys +import xml.etree.ElementTree as ET +from typing import NamedTuple from hexlib import kerneldir as kd from hexlib.graph.plan import V75_VTCM_TOTAL_BYTES @@ -17,6 +20,845 @@ DEVICES = ("sim", "local", "qdc") +# Above this many requested minutes, `--device qdc` refuses to submit without +# an explicit `--yes`. Not a limit on the job itself (job.py's own is +# 1..240) -- only on doing so without a human confirming it. 15 minutes is +# chosen to sit below the step-6 example the silicon-path plan itself uses +# (`--timeout-min 20 --yes`, which is deliberately ABOVE this threshold and +# therefore carries `--yes`), and above job.py's own docstring example of "a +# single, cheap, short-timeout dry run." +_QDC_YES_THRESHOLD_MIN = 15 + +# The two measurement strings a genuinely successful device run must have +# produced -- read directly out of hexlib/runtime/host/main.c (run_self_test's +# own printf calls), never guessed. A results.xml that parses clean with +# zero failures is NOT enough on its own: this project's own named failure +# mode is a job that completed having run (or measured) nothing at all, and +# a clean JUnit report with no measurements behind it is exactly that shape +# of success again, one level up. See _qdc_submit's post-parse check below. +_SELFTEST_PASS_MARKER = "hexlib: --self-test: PASS" +_CYCLES_TOTAL_MARKER = "cycles_total=" + +# `cycles_total=0` CONTAINS `cycles_total=`. That is the whole reason this +# regex exists. The presence check below (`marker not in combined`) was the +# only thing standing between "the logs carry a real measurement" and "a +# success value constructible with zero measurements in it" -- and a literal +# ZERO measurement satisfied it. Verified against fabricated local log files: +# `hexlib: --self-test: cycles_total=0` plus a clean `` printed "measurement lines present" and exited 0. +# +# Zero is not a pedantic edge case here, it is the EXPECTED shape of the +# failure this project is most exposed to. The DSP reads PCYCLE inside a +# user-mode unsigned PD, where SYSCFG.PCYCLEEN cannot be set from +# (skel_dispatch.c's own note, and include/hexlib/hexlib_harness.h's), so +# `cycles_total=0` is precisely what the first silicon job would print if the +# counter never advances there -- the single most important thing that job can +# tell us, and the one thing this check used to swallow. +# +# BUILT FROM THE MARKER RATHER THAN RESPELLING IT, so the presence check and +# the value check cannot drift apart. `(\S*)` deliberately captures whatever +# follows, valid or not, so a malformed value is DISTINGUISHABLE from an +# absent line instead of both silently reading as "no match". +_CYCLES_TOTAL_RE = re.compile(re.escape(_CYCLES_TOTAL_MARKER) + r"(\S*)") +_CYCLES_TOTAL_DIGITS = re.compile(r"[0-9]+") + + +def _qdc_cycles_total_verdict(combined: str) -> tuple[bool, str]: + """`(ok, detail)` for the `cycles_total=` measurement in the fetched logs. + + `ok` is True only if at least one `cycles_total=` line carries a value + that parses as a non-negative decimal integer AND is strictly greater + than zero. `detail` always says which of the four states was found, so a + caller's message names the real problem rather than "missing": + + - no `cycles_total=` line anywhere; + - a line whose value is not a decimal integer at all (truncated log, + interleaved output, a format change nobody updated this for); + - every line reporting exactly 0 -- a measurement that measured + nothing, which is the state this function was added for; + - at least one positive value: the only pass. + + AT LEAST ONE, not all: a single job's logs legitimately contain several + `cycles_total=` lines (`--self-test` prints one, `--coherency-check` + prints another), and there is no requirement that every mode a job ran + produced a nonzero count -- only that the run genuinely measured + something. If PCYCLE is dead in the unsigned PD, EVERY line reads 0 and + no `max` over them can rescue it, so taking the maximum cannot hide the + failure mode this exists to catch.""" + found = _CYCLES_TOTAL_RE.findall(combined) + if not found: + return False, f"no `{_CYCLES_TOTAL_MARKER}` line anywhere in the fetched logs" + + values: list[int] = [] + malformed: list[str] = [] + for raw in found: + if _CYCLES_TOTAL_DIGITS.fullmatch(raw): + values.append(int(raw, 10)) + else: + malformed.append(raw) + + positive = [v for v in values if v > 0] + if positive: + return True, f"{_CYCLES_TOTAL_MARKER}{max(positive)}" + if values: + return False, ( + f"every `{_CYCLES_TOTAL_MARKER}` line reports 0 " + f"({len(values)} such line(s)) -- the DSP measured NOTHING, which " + "is what PCYCLE returns when SYSCFG.PCYCLEEN is clear, and a " + "user-mode unsigned PD cannot set it" + ) + return False, ( + f"`{_CYCLES_TOTAL_MARKER}` is present but its value is not a decimal " + f"integer: {malformed[0]!r}" + ) + +# The operator's own account budget, in minutes -- read from the environment, +# exactly the way job.py reads QDC_API_KEY, because it is personal and this +# module has no way to learn it without a network call (which no CLI code +# path may ever make from inside a test, and which this function does not +# make at all, from anywhere). Unset means "unknown," printed as such, never +# guessed at. +_QDC_BUDGET_ENV = "QDC_BUDGET_MIN" + +# Only a plain non-negative decimal count of minutes. `QDC_BUDGET_MIN=abc` +# used to be echoed verbatim as "remaining budget: abc minutes" -- a number +# that is not a number, printed as though it were one, and compared against +# nothing at all. +_QDC_BUDGET_RE = re.compile(r"[0-9]+") + + +class _QdcBudgetError(Exception): + """`QDC_BUDGET_MIN` is set to something that is not a count of minutes. + Refused rather than ignored: a budget guard that silently disables itself + on a typo is worse than no guard, because the operator believes it is + watching.""" + + +def _qdc_remaining_budget_min() -> int | None: + """The operator's own recorded remaining minutes, or None if UNSET. + + Read from the environment, exactly the way job.py reads QDC_API_KEY, + because it is personal and this module has no way to learn it without a + network call (which no CLI code path may ever make from inside a test, + and which nothing here makes at all, from anywhere). Never queries QDC: + there is no such API on this account (job.py's own module docstring: + `get_job_status` returns `state=None`, `get_jobs_list` lags over 30 + minutes), so the only honest source is whatever the operator has + recorded for themselves. + + WHAT UNSET MEANS -- A DECISION, NOT AN OVERSIGHT. Unset means UNKNOWN, + and unknown means this CLI performs no budget comparison and submits + anyway (loudly saying so). It does NOT mean unlimited, and it must not be + read as an assurance that the job fits. The alternative -- refusing to + submit at all without the variable -- was considered and rejected for two + reasons: (1) the documented stage-3 entry point (`docs/STATE.md`, the + plan's step 6) does not set it, so making it mandatory would mean no job + can ever be submitted the way this project's own instructions say to; and + (2) a mandatory number nobody can verify invites `QDC_BUDGET_MIN=99999` + under time pressure, which yields a guard that is present, green, and + meaningless -- strictly worse than a stated "unknown, not checked". The + `--timeout-min`/`--yes` confirmation threshold still applies either way, + and it is the guard that does not depend on the operator having recorded + anything. + + Raises `_QdcBudgetError` if the variable is set but is not a non-negative + integer. + """ + raw = os.environ.get(_QDC_BUDGET_ENV) + if raw is None: + return None + text = raw.strip() + if not _QDC_BUDGET_RE.fullmatch(text): + raise _QdcBudgetError( + f"{_QDC_BUDGET_ENV}={raw!r} is not a non-negative whole number of " + "minutes. It is the only thing standing between a --timeout-min " + "and a budget it does not fit in, so a value that cannot be " + "compared is refused rather than printed and ignored. Unset it " + "to submit with no budget check at all (which will say so)." + ) + return int(text, 10) + + +def _qdc_budget_refusal(timeout_min: object) -> str | None: + """The budget COMPARISON ALONE -- the reason to refuse, or None. Prints + nothing. + + Split out of `_qdc_budget_guard` so that `_qdc_submit`, which is the + function that actually spends the minutes, can re-run the comparison + without printing the budget line a second time. The printing is a + courtesy to whoever typed the command; the comparison is the guard, and + only the guard has to be duplicated at the point of spending. Same shape + and same reason as `_qdc_kernel_refusal`. + + A non-int `timeout_min` is refused rather than compared. `_cmd_test_qdc` + rejects `None` before it gets here, but a second caller invoking + `_qdc_submit` directly has no such guarantee, and `None > budget` raises + on some paths and passes on others -- neither of which may decide whether + money is spent. + """ + if not isinstance(timeout_min, int) or isinstance(timeout_min, bool): + return ( + f"--timeout-min must be a whole number of minutes, got " + f"{timeout_min!r} -- a value that cannot be compared to a budget " + "is refused rather than submitted with no comparison at all" + ) + try: + budget = _qdc_remaining_budget_min() + except _QdcBudgetError as e: + return str(e) + if budget is None: + return None + if timeout_min > budget: + return ( + f"--timeout-min {timeout_min} exceeds the {budget} " + f"minute(s) recorded in {_QDC_BUDGET_ENV} -- refusing to submit a " + "job whose own timeout is larger than the budget it has to spend " + "from. Device minutes are non-renewable. Lower --timeout-min, or " + f"correct {_QDC_BUDGET_ENV} if it is stale." + ) + return None + + +def _qdc_confirm_refusal(timeout_min: object, yes: object) -> str | None: + """The `--yes` threshold ALONE -- the reason to refuse, or None. Prints + nothing, for the same reason as `_qdc_budget_refusal`. + + This is the guard that does not depend on the operator having recorded a + budget anywhere, so it is the one that must hold on every path. + """ + if not isinstance(timeout_min, int) or isinstance(timeout_min, bool): + return None # _qdc_budget_refusal owns that message + if timeout_min > _QDC_YES_THRESHOLD_MIN and not yes: + return ( + f"--timeout-min {timeout_min} is above the " + f"{_QDC_YES_THRESHOLD_MIN}-minute confirmation threshold -- pass " + "--yes to submit anyway. This does not limit the job itself, " + "only submitting one this size without a human confirming it." + ) + return None + + +def _qdc_budget_guard(timeout_min: int) -> int: + """Print the (locally recorded, never queried) remaining budget and + COMPARE it to `timeout_min`. Returns 0 to proceed, 2 to refuse. + + THE COMPARISON IS THE POINT. This function used to only print, which + meant `QDC_BUDGET_MIN=3 hexlib test scale_fp16 --device qdc --timeout-min + 240 --yes` printed `remaining budget: 3 minutes` and then submitted a + 240-minute job -- an 80x overspend of non-renewable minutes passing every + guard, with the figure that would have caught it on screen. + + `timeout_min > budget` is refused, not warned about: `--timeout-min` is + the ceiling QDC itself will enforce on the job, so a job whose ceiling + exceeds the stated remaining budget can, on its own, exhaust the account. + Equality is allowed (spending the last minutes deliberately is a real + thing to want); exceeding is not. The comparison itself lives in + `_qdc_budget_refusal` so `_qdc_submit` can repeat it silently. + """ + try: + budget = _qdc_remaining_budget_min() + except _QdcBudgetError: + pass # the refusal below carries the message + else: + if budget is None: + print( + f"remaining budget: unknown ({_QDC_BUDGET_ENV} is not set) -- " + "NO BUDGET CHECK WAS PERFORMED. Nothing here queries QDC for a " + "remaining-minutes figure (there is no reliable API for it on " + f"this account), and unset means unknown, NOT unlimited: set " + f"{_QDC_BUDGET_ENV} to have --timeout-min actually checked " + "against it." + ) + else: + print(f"remaining budget: {budget} minutes (from {_QDC_BUDGET_ENV})") + + refusal = _qdc_budget_refusal(timeout_min) + if refusal is not None: + print(f"error: {refusal}", file=sys.stderr) + return 2 + return 0 + + +# Stage 3 runs exactly one kernel, and it is not a parameter of anything. +# device/qdc/test_on_device.py hard-codes `./hexlib_run --self-test`, and +# main.c's run_self_test() hard-codes build_scale_batch() -- so the kernel the +# device actually exercises is fixed in C, three layers below the CLI. +_QDC_SUPPORTED_KERNEL = "scale_fp16" + + +def _qdc_kernel_refusal(kernel: object) -> str | None: + """None if `kernel` names the one kernel `--device qdc` can genuinely + run; otherwise the operator-facing reason it is refused. + + WHY REFUSE RATHER THAN IGNORE. `--device qdc` accepted the `` + argument and read it nowhere: `hexlib test add_fp16 --device qdc + --timeout-min 20 --yes` spent 20 non-renewable minutes running + `scale_fp16` and returned green, and the operator's notebook then said + "add_fp16 validated on silicon". Making the argument actually work means + parameterising main.c's batch builder and the staged on-device script -- + a real change, deliberately not made here. Refusing is the honest + intermediate state: the command either does what it was asked or says it + cannot. + + Accepts a bare name (`scale_fp16`, what docs/STATE.md's own entry point + uses) or a path to the kernel directory (`kernels/scale_fp16`, what + `--device sim` takes), since one CLI takes both. + """ + if not isinstance(kernel, str) or not kernel.strip(): + return ( + "--device qdc needs the argument to name " + f"{_QDC_SUPPORTED_KERNEL} explicitly; got {kernel!r}. It is not " + "optional and it is not ignored -- see _qdc_kernel_refusal." + ) + name = os.path.basename(os.path.normpath(kernel.strip())) + if name != _QDC_SUPPORTED_KERNEL: + return ( + f"--device qdc cannot run {name!r}: stage 3 is " + f"{_QDC_SUPPORTED_KERNEL}-only today. The staged on-device script " + "(hexlib/device/qdc/test_on_device.py) hard-codes `./hexlib_run " + "--self-test`, which runs main.c's fixed build_scale_batch(), so " + f"submitting this would spend real device minutes measuring " + f"{_QDC_SUPPORTED_KERNEL} and report the result under {name!r}. " + f"Run `hexlib test {_QDC_SUPPORTED_KERNEL} --device qdc ...`, or " + "use --device sim for any other kernel." + ) + return None + + +# Extra wall-clock slack, beyond the job's own `--timeout-min`, that `wait()` +# will keep polling for. Covers everything that happens outside the timeout +# QDC enforces on the run itself: queueing for a free SM8650, provisioning, +# and the farm collecting and publishing TestLogs/ afterwards. A POLICY +# CHOICE, NOT A MEASUREMENT -- stage 3 has never run on this account, so +# there is no observed queue time to derive it from; 15 minutes is chosen to +# be comfortably longer than any single step above plausibly takes. +# +# Waiting longer is FREE. Minutes are spent by the job, bounded by its own +# timeout; the CLI blocking on `get_job_log_files` costs nothing. Waiting too +# LITTLE is what costs: `wait()`'s old hard-coded 1800 s cap against a +# `submit()` that accepts 240 minutes meant a legitimate 35-minute job under +# `--timeout-min 60` was abandoned at 30 minutes, the minutes already spent, +# and the results.xml that appeared five minutes later never fetched. +_QDC_WAIT_GRACE_S = 900 + + +def _qdc_wait_cap_s(timeout_min: int) -> int: + """How long to poll for results.xml, DERIVED FROM THE JOB'S OWN TIMEOUT + rather than a constant that can silently be smaller than it.""" + return timeout_min * 60 + _QDC_WAIT_GRACE_S + + +def _qdc_fetch_logs(job_id: int, log_dir: str) -> tuple[list[str], str | None]: + """`(paths, error_text)` -- downloads whatever logs QDC has and NEVER + raises. Used on the giving-up path as well as the happy one: a timeout + that discards the evidence is worse than one that waits, and the fetch + failing is not a reason to also throw away the fact that the job ran.""" + from hexlib.device.qdc import job + + try: + return job.fetch(job_id, log_dir), None + except Exception as e: # noqa: BLE001 -- see docstring + return [], f"{type(e).__name__}: {e}" + + +def _qdc_submit(args) -> int: + """The real work, reached only once every guard in `_cmd_test_qdc` has + already passed: build the device artifacts, stage them together with the + on-device pytest, and submit to QDC. Isolated into its own function so + tests can monkeypatch it directly and verify the guards run in the right + order and print the right things WITHOUT ever touching the SDK, a + credential, or the network -- none of which any test may require or + contact. + """ + from hexlib.device.qdc import artifact, job + from hexlib.runtime import build as runtime_build + + # RE-CHECKED HERE, not only in _cmd_test_qdc. This is the function that + # spends the minutes, and it is called directly (by tests today, and by + # any second caller tomorrow) without going through _cmd_test_qdc's + # guards at all -- `getattr` with no default so an args object carrying no + # `kernel` attribute is refused rather than silently submitting for + # whatever main.c happens to hard-code. + refusal = _qdc_kernel_refusal(getattr(args, "kernel", None)) + if refusal is not None: + print(f"error: {refusal}", file=sys.stderr) + return 2 + + # AND SO ARE THE TWO GUARDS THAT COST MONEY, for the reason stated above -- + # which previously applied only to the CHEAPEST of the guards. The kernel + # check was re-run here while `_qdc_budget_guard` and the `--yes` threshold + # were left one frame up in `_cmd_test_qdc`, so `cli._qdc_submit(args)` with + # `timeout_min=240` under `QDC_BUDGET_MIN=3` built, staged and submitted a + # 240-minute job with no comparison, no confirmation, and nothing printed: + # exactly the 80x overspend `_qdc_budget_guard`'s docstring says it exists + # to prevent, past a guard that was present but not on this path. + # + # The `_refusal` (silent) forms are used, not `_qdc_budget_guard`, so the + # normal route through `_cmd_test_qdc` prints the budget line ONCE. Money + # guards duplicated; the courtesy print not. + for check in (_qdc_budget_refusal(getattr(args, "timeout_min", None)), + _qdc_confirm_refusal(getattr(args, "timeout_min", None), + getattr(args, "yes", False))): + if check is not None: + print(f"error: {check}", file=sys.stderr) + return 2 + + build_dir = os.path.join(args.out, "qdc_build") + try: + hexlib_run = runtime_build.build_device_binary(build_dir) + except runtime_build.RuntimeBuildError as e: + print(f"error: building the device artifacts failed: {e}", file=sys.stderr) + return 1 + # NAME IT THE WAY THE LINKER DID. `device_skel_so_name()` derives it + # from the IDL stem because FastRPC dlopens the name in qaic's + # generated URI, not one we pick -- see its docstring for the device + # failure (rc -2147482618) that this bond exists to prevent. + from hexlib.runtime.build import device_skel_so_name + skel_so = os.path.join(build_dir, device_skel_so_name()) + + here = os.path.dirname(__file__) + test_script = os.path.join(here, "device", "qdc", "test_on_device.py") + utils_py = os.path.join(here, "device", "qdc", "utils.py") + # conftest.py is what copies the junit report into QDC's collected log + # directory. Without it the report is written to the runner's working + # directory, never collected, and `job.wait()` polls its whole cap for a + # results.xml that exists on the device and will never be listed -- + # observed on job 756124. See device/qdc/conftest.py. + conftest_py = os.path.join(here, "device", "qdc", "conftest.py") + + out_base = os.path.join(args.out, "qdc_job") + try: + # binaries -> bin/, python -> tests/. See artifact.stage: a flat zip + # is accepted and never runs. + zip_path = artifact.stage( + [hexlib_run, skel_so], test_script, out_base, + support_files=[utils_py, conftest_py], + ) + except artifact.StagingError as e: + print(f"error: staging the QDC artifact failed: {e}", file=sys.stderr) + return 1 + + try: + job_id = job.submit(zip_path, timeout_min=args.timeout_min) + except job.QdcError as e: + print(f"error: {e}", file=sys.stderr) + return 1 + print(f"submitted job {job_id} (timeout {args.timeout_min} min)") + + log_dir = os.path.join(args.out, "qdc_logs") + cap_s = _qdc_wait_cap_s(args.timeout_min) + + # cap_s IS PASSED EXPLICITLY. Omitting it took job.wait's 1800 s default, + # which is SMALLER than the timeout submit() accepts (240 min) -- see + # _QDC_WAIT_GRACE_S for the full account of what that cost. + if not job.wait(job_id, cap_s=cap_s): + # FETCH THE EVIDENCE BEFORE GIVING UP. The minutes are already spent; + # whatever the farm has published so far (logcat, a partial + # TestLogs/, the caps log) is the only thing the operator can + # diagnose from, and abandoning it leaves them with nothing but "it + # timed out". This deliberately does NOT then evaluate those logs as a + # result: a job that never produced results.xml within the cap is a + # failure regardless of what else it printed. + paths, fetch_error = _qdc_fetch_logs(job_id, log_dir) + print( + f"error: job {job_id} produced no results.xml within the " + f"{cap_s}s wait cap (derived from --timeout-min " + f"{args.timeout_min}) -- a job with no results is a failure, " + "never a pass", + file=sys.stderr, + ) + if fetch_error is not None: + print( + f"error: job {job_id}: fetching the logs that DO exist also " + f"failed ({fetch_error}) -- check the job by hand in the QDC " + "console before spending more minutes", + file=sys.stderr, + ) + else: + print( + f"fetched {len(paths)} log file(s) that existed at the cap to " + f"{log_dir} -- the job may still finish; its results.xml was " + "not there yet", + file=sys.stderr, + ) + return 1 + + paths, fetch_error = _qdc_fetch_logs(job_id, log_dir) + if fetch_error is not None: + print( + f"error: job {job_id}: results.xml appeared but fetching the log " + f"files failed ({fetch_error}) -- an unverifiable job is a " + "failure, never a pass", + file=sys.stderr, + ) + return 1 + print(f"fetched {len(paths)} log file(s) to {log_dir}") + + return _qdc_check_results(job_id, paths) + + +class _QdcResultsError(Exception): + """Raised by `_qdc_parse_results_xml` for any results.xml that must not + be treated as a pass -- unparseable, a shape this project does not + produce, MISSING any of the attributes a JUnit report always carries + (`_REQUIRED_SUITE_ATTRS`; a missing one is never a zero), or carrying a + negative count. Caught by `_qdc_check_results`, never allowed to + propagate past `_qdc_submit`.""" + + +class _JUnitCounts(NamedTuple): + """Every count `_qdc_check_results` needs, all four of them REQUIRED. + + `skipped` is here because it was for a while parsed NOWHERE, which made a + collected-but-never-run test indistinguishable from a passing one: + `` plus a good + self-test log exited 0 and printed "5 test(s), 0 failures, 0 errors" with + no mention of the skips. On a device a skip overwhelmingly means the test + could not run at all, which is this project's own named failure mode + (absence read as success) wearing a different attribute name.""" + + tests: int + failures: int + errors: int + skipped: int + + +# EVERY ONE OF THESE IS REQUIRED ON EVERY , AND A MISSING ONE IS A +# MALFORMED REPORT, NOT A ZERO. `suite.get("failures", "0")` read a report +# with no `failures` attribute at all as a clean pass -- verified: +# `` plus a good log exited 0 and reported +# "0 failures, 0 errors". That directly contradicted `_QdcResultsError`'s own +# docstring ("missing the attributes a JUnit report always carries"), and it +# is the same absence-read-as-success shape the rest of this file exists to +# refuse: the counts that decide the verdict must be PRESENT, never defaulted. +# +# All four really are always emitted by the only producer this parser is +# pinned to -- pytest's own `--junitxml` (device/qdc/artifact.py's pytest.ini) +# writes `errors`, `failures`, `skipped`, `tests`, `time`, `timestamp`, +# `hostname` and `name` on every `` it emits. Confirmed by +# generating one locally with this repo's own pytest, not assumed from +# memory. Refusing a report that lacks any of them therefore cannot reject a +# report our own device job produced; it rejects a truncated or foreign one, +# which is the safe direction to fail in (the caller reports a parse failure +# as a failure, never a pass). +_REQUIRED_SUITE_ATTRS = ("tests", "failures", "errors", "skipped") + + +def _qdc_parse_results_xml(path: str) -> _JUnitCounts: + """Parse a JUnit-style results.xml and return the `_JUnitCounts` summed + across every `` element. Raises `_QdcResultsError` on anything + that is not a genuinely parseable report with real counts on it -- a + truncated or non-XML file, a ``/`` tree with no + testsuite elements at all, a `` missing any of + `_REQUIRED_SUITE_ATTRS`, a negative count, or a shape this function does + not recognize -- so the caller never has to guess whether "zero" means + "ran zero tests" or "could not even find the count". + + ONLY ONE SHAPE IS ACCEPTED, PINNED TO WHAT THIS PROJECT ACTUALLY + PRODUCES, NOT GUESSED AT AS A GENERAL JUNIT PARSER. The on-device job's + own pytest.ini (device/qdc/artifact.py's `_PYTEST_INI`) sets + `--junitxml=TestLogs/results.xml`, and pytest's `--junitxml` always + emits exactly one ``, either as the document root or as the + sole immediate child of a `` wrapper -- it never nests one + `` inside another. An earlier version of this function + accepted ANY shape by summing `root.findall(".//testsuite")` -- every + `` at any depth -- which would silently DOUBLE-COUNT a report + whose parent `` totals already include a nested child's + counts. Rather than guess how such a report should be summed, this + refuses it outright: a parse failure here blocks a false pass (the + caller reports it as a failure, never a pass -- see + `_qdc_check_results`), which is the safe direction to fail in. + """ + try: + root = ET.parse(path).getroot() + except ET.ParseError as e: + raise _QdcResultsError(f"{path} did not parse as XML: {e}") from e + + if root.tag == "testsuite": + suites = [root] + elif root.tag == "testsuites": + suites = root.findall("testsuite") # DIRECT children only. + else: + raise _QdcResultsError( + f"{path} root is <{root.tag}>, not or " + "-- not a JUnit report shape this project recognizes" + ) + if not suites: + raise _QdcResultsError( + f"{path} contains no element -- not a JUnit report " + "this project recognizes" + ) + + # Refuse a nested inside another ANYWHERE in the + # tree, rather than silently summing it -- pytest's own --junitxml never + # produces this shape (see the docstring above), and summing it would + # double-count a parent's already-rolled-up totals. + for suite in suites: + if suite.findall(".//testsuite"): + raise _QdcResultsError( + f"{path} has a nested inside another " + "-- not the flat shape pytest's --junitxml produces, and " + "summing nested totals would double-count them; refusing " + "rather than guessing how to sum it" + ) + + totals = dict.fromkeys(_REQUIRED_SUITE_ATTRS, 0) + for suite in suites: + for attr in _REQUIRED_SUITE_ATTRS: + raw = suite.get(attr) + if raw is None: + raise _QdcResultsError( + f"{path} has a with no {attr!r} attribute -- " + "pytest's own --junitxml always writes all of " + f"{', '.join(_REQUIRED_SUITE_ATTRS)}, so an absent one is a " + "malformed or truncated report and must NEVER be read as " + "zero (that is how a report carrying no failure count at " + "all used to be reported as '0 failures, 0 errors')" + ) + try: + value = int(raw) + except ValueError as e: + raise _QdcResultsError( + f"{path} has a non-integer {attr!r} attribute: {e}" + ) from e + if value < 0: + raise _QdcResultsError( + f"{path} has {attr}={raw!r}, a NEGATIVE count -- pytest " + "cannot produce that, so the report is malformed; a " + "negative count must not be summed into a total that then " + "compares as 'no failures'" + ) + totals[attr] += value + return _JUnitCounts(**totals) + + +def _qdc_check_results(job_id: int, paths: list[str]) -> int: + """The parse `job.wait`/`job.fetch` never do. A device job whose + TestLogs/results.xml merely *exists* proves nothing on its own -- this + project's own named failure mode is a job that completed having run (or + measured) NOTHING and still reported passing. Every one of the following + must hold before this returns 0, each with its own distinct message so a + real failure is diagnosable from which check tripped: + + - a results.xml was actually fetched, and it PARSES as XML, WITH all + four of `_REQUIRED_SUITE_ATTRS` actually present on every + `` (a missing `failures`/`errors` is a malformed report, + never a zero); + - it reports `tests > 0` -- zero tests is a failure, never a pass, and + so is a negative count; + - `failures == 0` and `errors == 0`; + - `skipped == 0`. A SKIP IS NOT A PASS. On this device path a skip + cannot mean "not applicable here": every test in + device/qdc/test_on_device.py is unconditional, so a skip means the + farm's pytest collected a test and then did not run it -- the + device could not be reached, a `skipif` was added upstream, or + collection half-failed. Reported (and refused) with its own message + rather than folded into the failure count, because "5 skipped" and + "5 failed" call for completely different next actions; + - the fetched logs actually CONTAIN the measurement lines + `hexlib_run` itself prints on a genuine pass (`cycles_total=` and + the `--self-test` PASS line, both read directly out of main.c) -- + a clean JUnit report with none of hexlib's own evidence behind it + is exactly the "success constructible with zero measurements in + it" shape this check exists to rule out; + - and the `cycles_total=` line's VALUE is a real integer greater + than zero. The bullet above was for a while the whole of this + check, and `cycles_total=0` satisfied it -- the substring is + there, so a run in which the DSP measured literally nothing + printed "measurement lines present" and exited 0. See + `_qdc_cycles_total_verdict` for why zero is the expected shape of + the failure rather than a pedantic edge case. + """ + # SELECTED BY THE SAME SUFFIX TEST `job.wait()` USES, not by basename, and + # AMBIGUITY IS A FAILURE. `job.fetch` mirrors QDC's directory layout beneath + # `log_dir` precisely because `TestLogs/results.xml` and `logs/results.xml` + # used to collide on disk -- so from that fix onwards TWO files named + # results.xml can exist, and `os.path.basename(p) == "results.xml"` took + # whichever QDC happened to list first. A framework placeholder + # (``) listed ahead of the real + # report is then parsed as the verdict, and since `write_qdc_log` runs + # before the asserts the combined log still carries the PASS and + # `cycles_total=` lines: a failed device job reported as a pass, which is + # this project's own named Critical, on the money path. + # + # `job._results_filename` requires the full `TestLogs/results.xml` suffix, + # which is what actually distinguishes the report from the placeholder. Two + # matches means the layout is not what this reader was written against, and + # a verdict read from a guess is worse than no verdict. + from hexlib.device.qdc import job as _qdc_job + + matches = [p for p in paths if _qdc_job._results_filename(p)] + if len(matches) > 1: + print( + f"error: job {job_id}: {len(matches)} files match " + f"{_qdc_job.RESULTS_MARKER} ({', '.join(sorted(matches))}) -- " + "refusing to guess which one is the verdict. A job whose report " + "cannot be identified unambiguously is a failure, never a pass.", + file=sys.stderr, + ) + return 1 + results_path = matches[0] if matches else None + if results_path is None: + print( + f"error: job {job_id}: no {_qdc_job.RESULTS_MARKER} was among the " + "fetched log files -- a job with no results is a failure, never a " + "pass", + file=sys.stderr, + ) + return 1 + + try: + counts = _qdc_parse_results_xml(results_path) + except _QdcResultsError as e: + print( + f"error: job {job_id}: could not parse results.xml as a JUnit " + f"report -- a truncated or unparseable results file is a " + f"failure, never a pass: {e}", + file=sys.stderr, + ) + return 1 + + tests, failures, errors, skipped = counts + + # `<= 0`, NOT `== 0`. A negative total is already refused by the parser, + # so this is belt-and-braces rather than the only guard -- but `tests == + # 0` was verified to let `` + # through, printing "-1 test(s)" and exiting 0, and a comparison that only + # catches the exact value it was written for is not a bound. + if tests <= 0: + print( + f"error: job {job_id}: results.xml reports {tests} tests -- a job " + "that ran no tests is a failure, never a pass", + file=sys.stderr, + ) + return 1 + + if failures != 0 or errors != 0: + print( + f"error: job {job_id}: results.xml reports {failures} failure(s) " + f"and {errors} error(s) across {tests} test(s) " + f"({skipped} skipped)", + file=sys.stderr, + ) + return 1 + + # A SKIP IS NOT A PASS -- see this function's docstring. + if skipped != 0: + print( + f"error: job {job_id}: results.xml reports {skipped} skipped " + f"test(s) out of {tests} -- every test in " + "hexlib/device/qdc/test_on_device.py is unconditional, so a skip " + "on device means a test was collected and never actually run. " + "Collected-but-not-run is not passed: this is the project's own " + "named failure mode (absence read as success) spelled `skipped=`", + file=sys.stderr, + ) + return 1 + + combined = "" + for p in paths: + try: + with open(p, encoding="utf-8", errors="replace") as f: + combined += f.read() + except OSError: + continue + + missing = [ + marker + for marker in (_CYCLES_TOTAL_MARKER, _SELFTEST_PASS_MARKER) + if marker not in combined + ] + if missing: + print( + f"error: job {job_id}: results.xml reports {tests} test(s) with " + "no failures, but the fetched logs are missing the expected " + f"measurement line(s): {', '.join(missing)!r} -- a pass with no " + "measurements behind it is the exact failure mode this check " + "exists to rule out", + file=sys.stderr, + ) + return 1 + + # PRESENCE IS NOT MEASUREMENT. The check above only proved the substring + # `cycles_total=` appears; this one reads the number after it. + cycles_ok, cycles_detail = _qdc_cycles_total_verdict(combined) + if not cycles_ok: + print( + f"error: job {job_id}: results.xml reports {tests} test(s) with " + f"no failures, but the DSP's own cycle measurement is not usable: " + f"{cycles_detail} -- a pass whose only measurement is zero is " + "still a pass with no measurements behind it", + file=sys.stderr, + ) + return 1 + + # The skip count is printed on the PASS line too, not only when it is + # nonzero: a success line that silently omits a count it checked leaves a + # reader unable to tell "0 skipped" from "skips were never looked at", + # which is exactly the state this line was in before. + print( + f"job {job_id}: {tests} test(s), 0 failures, 0 errors, " + f"{skipped} skipped, measurement lines present ({cycles_detail})" + ) + return 0 + + +def _cmd_test_qdc(args) -> int: + """`--device qdc`: refuses a kernel stage 3 cannot actually run, refuses + without an explicit `--timeout-min`, refuses a `--timeout-min` outside + job.py's own 1..240, always prints the (locally known, never queried) + remaining budget and refuses a job that does not fit in it, and requires + `--yes` above `_QDC_YES_THRESHOLD_MIN` -- every one of these guards runs + before `_qdc_submit` ever touches the SDK, a credential, or the network. + + THE KERNEL CHECK IS FIRST, deliberately. "This command cannot run the + thing you asked for" is more useful than "you forgot --timeout-min" when + both are true, and it is the cheapest of the five. + """ + refusal = _qdc_kernel_refusal(getattr(args, "kernel", None)) + if refusal is not None: + print(f"error: {refusal}", file=sys.stderr) + return 2 + + if args.timeout_min is None: + print( + "error: --device qdc requires --timeout-min (1..240) -- a " + "runaway job spends real, non-renewable minutes, and there is " + "no default that could be right for every account.", + file=sys.stderr, + ) + return 2 + + # RANGE-CHECKED HERE, NOT ONLY IN job.submit. job.py enforces 1..240 too + # (it is the authority, and these bounds are imported from it rather than + # respelled), but it does so AFTER _qdc_submit has run a full SDK build of + # hexlib_run + the skel .so and staged a zip -- minutes of local work + # thrown away to reject an argument that was wrong before any of it + # started. A lazy import: job.py pulls in nothing but the stdlib at module + # scope, and never the vendor SDK. + from hexlib.device.qdc import job as qdc_job + + if not qdc_job.MIN_TIMEOUT_MIN <= args.timeout_min <= qdc_job.MAX_TIMEOUT_MIN: + print( + f"error: --timeout-min must be {qdc_job.MIN_TIMEOUT_MIN}.." + f"{qdc_job.MAX_TIMEOUT_MIN}, got {args.timeout_min} -- QDC itself " + "refuses anything else, and finding that out only after a full " + "device build has been run and staged wastes the build.", + file=sys.stderr, + ) + return 2 + + budget_rc = _qdc_budget_guard(args.timeout_min) + if budget_rc != 0: + return budget_rc + + confirm = _qdc_confirm_refusal(args.timeout_min, args.yes) + if confirm is not None: + print(f"error: {confirm}", file=sys.stderr) + return 2 + + return _qdc_submit(args) + def _cmd_new_kernel(args) -> int: try: @@ -41,13 +883,43 @@ def _cmd_validate(args) -> int: def _cmd_test(args) -> int: - if args.device != "sim": + # `--timeout-min` and `--yes` exist ONLY for `--device qdc`; both were + # accepted and silently ignored for sim/local, so `hexlib test k --device + # sim --timeout-min 20 --yes` looked like it had asked for something and + # had it granted. Refused rather than warned about: the two flags are the + # spend controls for the one backend that spends anything, and a spend + # control that is accepted where it does nothing teaches an operator that + # passing it is harmless. + if args.device != "qdc": + ignored = [ + flag + for flag, given in ( + ("--timeout-min", args.timeout_min is not None), + ("--yes", bool(args.yes)), + ) + if given + ] + if ignored: + print( + f"error: {' and '.join(ignored)} appl" + f"{'y' if len(ignored) > 1 else 'ies'} only to --device qdc, " + f"not --device {args.device} -- nothing here spends device " + "minutes, so there is nothing to time out or to confirm. " + "Refused rather than ignored.", + file=sys.stderr, + ) + return 2 + + if args.device == "local": print( - f"error: --device {args.device} is not implemented in the simulation " - "path. The local and qdc backends arrive with the silicon-path plan.", + "error: --device local is not implemented, no device available -- " + "the shape exists so a contributor with a phone can wire it up; " + "this codebase does not pretend it works without one.", file=sys.stderr, ) return 2 + if args.device == "qdc": + return _cmd_test_qdc(args) result = verify(args.kernel, args.out) if not is_ok(result): @@ -137,6 +1009,16 @@ def main(argv: list[str] | None = None) -> int: t.add_argument("kernel") t.add_argument("--device", choices=DEVICES, default="sim") t.add_argument("--out", default="_work") + t.add_argument( + "--timeout-min", type=int, default=None, + help="required for --device qdc (1..240); no default, a runaway job " + "spends real money", + ) + t.add_argument( + "--yes", action="store_true", + help="confirm a --device qdc submission above the confirmation " + "threshold (see --timeout-min)", + ) t.set_defaults(func=_cmd_test) pl = sub.add_parser("plan", help="compile a model's encoder to a VTCM/DMA plan") diff --git a/hexlib/device/__init__.py b/hexlib/device/__init__.py new file mode 100644 index 0000000..b9f91c2 --- /dev/null +++ b/hexlib/device/__init__.py @@ -0,0 +1,8 @@ +# hexlib/device/__init__.py +"""Stage 3: getting hexlib_run and libhexlib_iface_skel.so onto a real phone. + +Everything under here that talks to Qualcomm Device Cloud is exercised +offline, against a fake client, in hexlib/tests/test_qdc.py. See +hexlib/device/qdc/job.py for why completion is detected from log files +rather than by polling job status. +""" diff --git a/hexlib/device/qdc/__init__.py b/hexlib/device/qdc/__init__.py new file mode 100644 index 0000000..618de7e --- /dev/null +++ b/hexlib/device/qdc/__init__.py @@ -0,0 +1,9 @@ +# hexlib/device/qdc/__init__.py +"""Qualcomm Device Cloud plumbing: stage an artifact, submit it, and detect +completion without ever trusting job status or the jobs list. + +qualcomm_device_cloud_sdk is imported lazily, inside functions, in job.py -- +never at module import time -- so `import hexlib.device.qdc` and the offline +tests in hexlib/tests/test_qdc.py work on a machine where that package is +not installed. It is only required once code actually talks to the network. +""" diff --git a/hexlib/device/qdc/artifact.py b/hexlib/device/qdc/artifact.py new file mode 100644 index 0000000..0f0a033 --- /dev/null +++ b/hexlib/device/qdc/artifact.py @@ -0,0 +1,151 @@ +# hexlib/device/qdc/artifact.py +"""Stage the stage-2 binaries and the on-device pytest into a zip QDC can run. + +The zip is uploaded as a flat TestScript (see job.py's _real_upload_artifact +for why that artifact type and not TestPackage): hexlib_run, libhexlib_iface_skel.so, and the +on-device test script sit next to a pytest.ini and requirements.txt, matching +what TestFramework.APPIUM finds once QDC extracts it at /qdc/appium. There is +no subdirectory nesting here on purpose -- the on-farm scripts invoke a plain +`adb`, and a path that only exists relative to some assumed staging root is +exactly the kind of thing that fails silently on real hardware and nowhere +else. + +StagingError is raised the moment any declared input is missing OR EMPTY, and +again if -- somehow -- something staged does not make it into the zip. A job +that runs against a binary that silently wasn't there is how you burn device +minutes for nothing. + +WHY EMPTINESS IS CHECKED AND NOT JUST EXISTENCE. `os.path.isfile` was the +whole test, and `stage` was verified to accept four 0-byte files and produce a +perfectly submittable zip. A link or a copy that fails part-way leaves exactly +that: a `libhexlib_iface_skel.so` of length zero, present, named correctly, and +completely unrunnable -- discovered on the device, after the minutes are spent, +as a dlopen failure with no obvious cause. Size zero is the one truncation +that is unambiguous and free to detect here; deeper validation (ELF magic, +machine type) deliberately is NOT done in this function, because `binaries` +also legitimately carries a `.py` file (see cli.py's own call, which stages +`utils.py` through this list) and a check that has to special-case its inputs +by extension is a check that will be wrong about the next input added. +""" +from __future__ import annotations + +import os +import shutil +import zipfile + +# --junitxml=results.xml, FLAT. The framework publishes pytest's report itself +# as `//TestLogs/results.xml` -- that path is QDC's, not ours, and +# it is what job.RESULTS_MARKER matches. Verified against two working jobs on +# this account (744001 hexbench, 743551 llama.cpp), both of which set exactly +# this and both of which have a TestLogs/results.xml. +# +# Writing `--junitxml=TestLogs/results.xml` ourselves, as this used to, is +# actively harmful: conftest.py also pushes a copy into QDC_logs, and two +# collected files whose names both end in `TestLogs/results.xml` is the +# "TWO MATCHES IS A FAILURE" case cli._qdc_check_results deliberately refuses. +_PYTEST_INI = "[pytest]\naddopts = --junitxml=results.xml\n" +# Appium-Python-Client is here because the job runs under +# TestFramework.APPIUM and conftest.py opens a session with it; the version is +# the one llama.cpp's own QDC runner pins against this same account. +_REQUIREMENTS = "pytest\nAppium-Python-Client==5.2.4\n" + + +class StagingError(Exception): + """A declared binary or test script does not exist, is empty, or did not + survive into the zip. Never produce an artifact that is missing what it + claims to carry.""" + + +def _require_real_file(path: str, what: str) -> None: + """Present AND non-empty. See the module docstring for why the second + half is not pedantry.""" + if not os.path.isfile(path): + raise StagingError(f"{what} not found: {path}") + if os.path.getsize(path) == 0: + raise StagingError( + f"{what} is 0 bytes, refusing to stage it: {path} -- an empty " + "artifact is what a failed link or a truncated copy leaves " + "behind, and it would be discovered on the device after the " + "minutes are spent" + ) + + +def stage( + binaries: list[str], + test_script: str | None, + out_base: str, + support_files: list[str] | None = None, +) -> str: + """Stage `binaries` under `bin/`, `test_script` and `support_files` under + `tests/`, generate `pytest.ini` and `requirements.txt` at the root, zip it + all to `.zip`, and return that path. + + THE LAYOUT IS COPIED FROM A JOB THAT WORKS, not chosen. Job 744001 + (hexbench, this account, this device) has pytest report + `rootdir: /qdc/appium`, `configfile: pytest.ini`, collect + `tests/test_capprobe.py`, and push `/qdc/appium/bin/capprobe`. Job 743551 + (llama.cpp) has the same shape. + + A FLAT ZIP DOES NOT RUN. hexlib's first three jobs (756124, 756159, + 756206) staged everything at the root, were accepted, dispatched, and + reached Completed having produced NO `//` tree at all -- no + TestLogs, no install.txt, no screen recording -- which is what a job whose + test stage never started looks like. Nothing inside the tests could have + mattered while that was true. + + Raises StagingError if any input is missing or empty, or if the zip that + would result is missing anything that was staged. + """ + support_files = list(support_files or []) + for b in binaries: + _require_real_file(b, "binary") + for s in support_files: + _require_real_file(s, "support file") + if test_script is not None: + _require_real_file(test_script, "test script") + + stage_dir = out_base + "_stage" + if os.path.exists(stage_dir): + shutil.rmtree(stage_dir) + os.makedirs(stage_dir, exist_ok=True) + + # (absolute path on disk, name inside the zip) -- the second is what the + # runner sees, and it is the whole point of this function. + staged: list[tuple[str, str]] = [] + + def _place(src: str, subdir: str) -> None: + rel = os.path.join(subdir, os.path.basename(src)) if subdir else os.path.basename(src) + dest = os.path.join(stage_dir, rel) + os.makedirs(os.path.dirname(dest), exist_ok=True) + shutil.copy2(src, dest) + staged.append((dest, rel.replace(os.sep, "/"))) + + for src in binaries: + _place(src, "bin") + for src in support_files: + _place(src, "tests") + if test_script is not None: + _place(test_script, "tests") + + for name, body in (("pytest.ini", _PYTEST_INI), + ("requirements.txt", _REQUIREMENTS)): + path = os.path.join(stage_dir, name) + with open(path, "w") as f: + f.write(body) + staged.append((path, name)) + + zip_path = out_base + ".zip" + zip_dir = os.path.dirname(zip_path) + if zip_dir: + os.makedirs(zip_dir, exist_ok=True) + + with zipfile.ZipFile(zip_path, "w", zipfile.ZIP_DEFLATED) as zf: + for path, arcname in staged: + zf.write(path, arcname) + + names = set(zipfile.ZipFile(zip_path).namelist()) + missing = [arc for _, arc in staged if arc not in names] + if missing: + raise StagingError(f"declared file(s) missing from zip: {missing}") + + return zip_path diff --git a/hexlib/device/qdc/conftest.py b/hexlib/device/qdc/conftest.py new file mode 100644 index 0000000..93f95ce --- /dev/null +++ b/hexlib/device/qdc/conftest.py @@ -0,0 +1,144 @@ +"""Runs ON THE DEVICE, beside `test_on_device.py`, under the farm's own pytest. + +WHY THIS FILE EXISTS. `pytest.ini` (written by `artifact.py`) points +`--junitxml` at the RELATIVE path `TestLogs/results.xml`, which lands in +whatever directory QDC's runner happens to invoke pytest from. QDC does not +collect that. It collects `/data/local/tmp/QDC_logs` -- which is precisely +what `utils.write_qdc_log` writes to, and which NOTHING called with the +report until this file existed. + +That gap is not theoretical: job 756124 (2026-08-12, the first hexlib job QDC +ever accepted) reached state Completed and returned exactly one log file -- +a stale `LauncherUI` log from an unrelated job four days earlier. No +`TestLogs/results.xml`, so `wait()` polled for its whole cap and returned +False on a job that may well have run correctly. The report was written; it +just never left the device. + +`llama.cpp`'s own QDC runner (`scripts/snapdragon/qdc/tests/conftest.py`) +solves the same problem the same way, and is the working reference this was +matched to. It does the copy in `pytest_sessionfinish`; this uses +`pytest_unconfigure`, which is strictly later -- the junitxml plugin writes +the file during its OWN `pytest_sessionfinish`, and two hookimpls for one hook +have no ordering guarantee worth betting a device job on. + +THE SUBDIRECTORY IS LOAD-BEARING. `job.wait()` matches a log whose name ENDS +WITH `TestLogs/results.xml`, and QDC lists collected logs as +`/`. Writing a flat `results.xml` would be listed as +`756124/results.xml`, which does not match that suffix, and `wait()` would +miss a report that had arrived. The nesting here and `RESULTS_MARKER` in +job.py are one decision recorded in two places. + +FAIL CLOSED. If the junitxml is missing or unreadable, this writes a file at +the same path SAYING SO rather than writing nothing. Writing nothing is +indistinguishable from the job never finishing, costs the full wait cap, and +tells the operator nothing; a file that exists and does not parse is caught +immediately by `cli._qdc_check_results` and names its own cause. +""" +import os +import traceback + +import pytest + +from utils import QDC_LOG_DIR, write_qdc_log + + +@pytest.fixture(scope="session", autouse=True) +def driver(): + """Open the Appium session QDC's APPIUM framework expects. + + hexlib's tests drive the phone through `adb` and never touch this object. + It exists because the framework is `TestFramework.APPIUM` and llama.cpp's + working runner on this same account opens exactly this session; a package + that never establishes one is the most plausible remaining reason two + hexlib jobs reached Completed having emitted nothing of their own. + + NON-FATAL BY DESIGN, and that is the important part. This fixture is + session-scoped and autouse, so if it raised it would error EVERY test -- + turning a run that would otherwise have worked into a total failure, to + obtain an object none of these tests use. + + That is not a hypothetical trade-off. Job 744001 (hexbench, this account, + this device) collected and ran `tests/test_capprobe.py` against the phone + purely over adb, with no Appium session anywhere in its stdout. So the + session is plausibly unnecessary here; it is attempted because llama.cpp's + runner does establish one and a missing session is the other candidate + explanation for a test stage that never starts. Attempt it, keep it if it + works, and never let its absence be the reason a job reports nothing. + + Imported inside the fixture so that collecting this file does not require + the Appium client to be installed -- the report-copying hook below is the + part that must work regardless. + """ + try: + from appium import webdriver + from appium.options.common import AppiumOptions + + options = AppiumOptions() + options.set_capability("automationName", "UiAutomator2") + options.set_capability("platformName", "Android") + options.set_capability("deviceName", os.getenv("ANDROID_DEVICE_VERSION")) + return webdriver.Remote( + command_executor="http://127.0.0.1:4723/wd/hub", options=options + ) + except Exception: + # Recorded, not raised: a reader of the collected logs needs to know + # whether a session existed when interpreting whatever the job did. + try: + write_qdc_log( + "hexlib_appium_session.txt", + "no Appium session was established; tests ran over adb " + f"alone:\n{traceback.format_exc()}", + ) + except Exception: + pass + return None + +# FLAT, and deliberately NOT `TestLogs/results.xml`. QDC publishes pytest's +# own junitxml as `//TestLogs/results.xml` -- that is the file +# job.wait() matches. This copy lands under `UserCollectedLogs/QDC_logs/` +# (exactly where job 744001's own `results.xml` is), so naming it +# `TestLogs/results.xml` would produce a SECOND collected name ending in that +# suffix, and cli._qdc_check_results treats two matches as a failure rather +# than picking one. This copy exists to survive the framework not publishing +# its own, not to compete with it. +_RESULTS_NAME = "results.xml" + + +def _copy_report(config): + xml_path = getattr(config.option, "xmlpath", None) + if not xml_path: + return ( + "" + ) + if not os.path.exists(xml_path): + return ( + f"" + ) + try: + with open(xml_path, encoding="utf-8") as f: + return f.read() + except Exception: + return f"" + + +def pytest_unconfigure(config): + """Copy the JUnit report into QDC's collected log directory. + + Never raises: an exception here would be reported as an error in the + runner's own teardown, on a path whose entire job is to make the real + result visible. Any failure is written to a second log instead. + """ + try: + write_qdc_log(_RESULTS_NAME, _copy_report(config)) + except Exception: + try: + write_qdc_log( + "hexlib_conftest_error.txt", + "copying the junit report into " + f"{QDC_LOG_DIR} raised:\n{traceback.format_exc()}", + ) + except Exception: + pass diff --git a/hexlib/device/qdc/job.py b/hexlib/device/qdc/job.py new file mode 100644 index 0000000..68610f2 --- /dev/null +++ b/hexlib/device/qdc/job.py @@ -0,0 +1,466 @@ +# hexlib/device/qdc/job.py +"""Submit a staged artifact to Qualcomm Device Cloud and detect completion. + +THREE MEASURED FACTS, do not re-derive or contradict them: + + 1. DEVICE = "SM8650", TARGET_ID = 3625030. That device's CDSP reads + ARCH_VER == 0x8c75, bit-identical to the simulator -- no microarch + confound between stage 1 and stage 3. + 2. NEVER poll get_job_status or get_jobs_list for completion. + get_job_status returns state=None on this account. get_jobs_list + lagged more than 30 minutes on both jobs observed. Completion is + detected by the *appearance* of TestLogs/results.xml among a job's + log files. + + WHAT wait() IS AND IS NOT. It is a COMPLETION DETECTOR, not a verdict. + `wait() -> True` means one thing and only one thing: a log file whose + name ends in TestLogs/results.xml showed up. It does not open that file, + so it cannot distinguish a real report from a zero-byte placeholder, and + an earlier version of this docstring claiming "wait() exists to make + [a false pass] impossible" was overclaiming: it makes DECLARING + COMPLETION EARLY impossible, which is a different (and narrower) thing. + A job that ran zero tests once reported passing on this account, and the + check that actually rules that out lives in + `hexlib/cli.py::_qdc_check_results` -- it parses the report, requires + tests > 0 with no failures/errors/skips, and requires hexlib's own + measurement lines in the fetched logs. Any new caller of wait() needs + that check too; wait()'s True is its precondition, never its conclusion. + 3. Artifact is a zip, TestFramework.APPIUM, entry_script=None, extracted + at /qdc/appium, logs collected from /data/local/tmp/QDC_logs. On-farm + scripts have a plain `adb`. + +CREDENTIALS. + - QDC_API_KEY (required, or ~/.qdc_api_key as a fallback): the operator's + own API key. Never logged, never printed (including in an exception + message), never committed, and never held on anyone's behalf. + - QDC_BASE_URL (optional override): defaults to Qualcomm's own public + endpoint, qualcomm_device_cloud_sdk.api.qdc_api.API_BASE_URL. Set this + only for a private or government tenant with a different endpoint -- + the default is correct for the account this module was built against. + +AUTH HEADER: this module authenticates through the SDK's own +get_public_api_client_using_api_key(), not a hand-rolled client, so the +header scheme (a raw Authorization header plus X-QCOM-TokenType: apikey -- +NOT an OAuth-style token prefix) comes from Qualcomm's code. See _client() +below. + +TIMEOUTS. submit() takes timeout_min as a required keyword argument with no +default, refuses values outside 1..240, and never guesses on the caller's +behalf. A runaway job spends real money; a missing timeout must be a +TypeError raised by Python itself before this module runs a single line. + +qualcomm_device_cloud_sdk IS IMPORTED LAZILY. Every function in this module +that needs it imports it inside its own body, not at module scope, so that +importing hexlib.device.qdc.job -- and running every test in +hexlib/tests/test_qdc.py -- works whether or not that package is installed. +The real SDK is only required once code actually reaches the network, which +none of the offline tests do. +""" +from __future__ import annotations + +import os +import pathlib +import time +import types + +DEVICE = "SM8650" +TARGET_ID = 3625030 +POLL_S = 30 +RESULTS_MARKER = "TestLogs/results.xml" + +# PUBLIC on purpose: hexlib/cli.py range-checks `--timeout-min` against these +# before it runs a full SDK build, and it must not respell the bounds. This +# module stays the single authority for them. +MIN_TIMEOUT_MIN = 1 +MAX_TIMEOUT_MIN = 240 + +_API_KEY_ENV = "QDC_API_KEY" +_BASE_URL_ENV = "QDC_BASE_URL" + +# Labels the SDK's own client constructor requires. None of these are +# secrets -- they identify the caller to QDC's own logging, nothing more. +_APP_NAME = "hexlib" +_CLIENT_TYPE = "Python" + + +class QdcError(Exception): + """Anything that must stop a submission before it spends device + minutes: a bad timeout, a missing credential, a missing artifact, or + the API itself refusing the request.""" + + +def _key_file() -> pathlib.Path | None: + p = pathlib.Path.home() / ".qdc_api_key" + return p if p.is_file() else None + + +def _api_key() -> str: + """The key is the operator's own, from the environment or + ~/.qdc_api_key. Never committed, never in CI, never held on anyone's + behalf, and never included in any error this function raises.""" + key = os.environ.get(_API_KEY_ENV) + if not key: + f = _key_file() + key = f.read_text().strip() if f else None + if not key: + raise QdcError( + f"no QDC credential: set {_API_KEY_ENV} or create ~/.qdc_api_key. " + "Credentials are personal and are never stored in this repository." + ) + return key + + +def _on_behalf_of() -> str: + """A caller label for QDC's own logging, not a credential. Best-effort + from the environment; falls back to a fixed string rather than failing + a submission over a cosmetic field.""" + return ( + os.environ.get("QDC_OPERATOR") + or os.environ.get("USERNAME") + or os.environ.get("USER") + or "hexlib" + ) + + +def _base_url_override() -> str | None: + """QDC_BASE_URL is optional: unset, the SDK's own public endpoint + (qualcomm_device_cloud_sdk.api.qdc_api.API_BASE_URL) is used, via the + SDK's own client constructor. Set only for a private or government + tenant with a different endpoint.""" + return os.environ.get(_BASE_URL_ENV) + + +def _client(): + """Build the QDC client the way the SDK itself does, so the auth header + scheme, the default base URL, and any header Qualcomm adds later all + come from their code, not a hand-rolled guess. + + Imports the real package lazily so that nothing above this line, and no + test in hexlib/tests/test_qdc.py, requires it to be installed. + """ + from qualcomm_device_cloud_sdk.api import qdc_api as _vendor + + key = _api_key() + override = _base_url_override() + if not override: + return _vendor.get_public_api_client_using_api_key( + app_name_header=_APP_NAME, + on_behalf_of_header=_on_behalf_of(), + client_type_header=_CLIENT_TYPE, + api_key_header=key, + ) + + # QDC_BASE_URL was set: same header recipe as + # get_public_api_client_using_api_key (copied from + # qualcomm_device_cloud_sdk/api/qdc_api.py since that function does not + # take a base_url argument), pointed at a different endpoint. This path + # is exercised even less than the default one -- confirm it against the + # tenant's own documentation before trusting it with a real submission. + from qualcomm_device_cloud_sdk import Client + + return Client( + base_url=override, + headers={ + "Authorization": key, + "X-QCOM-TokenType": "apikey", + "X-QCOM-AppName": _APP_NAME, + "X-QCOM-ClientType": _CLIENT_TYPE, + "X-QCOM-OnBehalfOf": _on_behalf_of(), + }, + ) + + +# --- qdc_api: a thin, monkeypatchable facade over the SDK's own qdc_api --- +# +# Tests patch attributes directly onto this namespace (e.g. +# `job.qdc_api.get_job_log_files = fake`) without the real +# qualcomm_device_cloud_sdk package ever being imported. Each wrapper below +# imports the real SDK lazily, inside itself, for the same reason _client() +# does: importing this module must never require the package to exist. +# +# EVERY WRAPPER BELOW DELEGATES TO qualcomm_device_cloud_sdk.api.qdc_api, +# Qualcomm's own high-level module (get_job_log_files, get_job_status, +# upload_file, submit_job, download_job_log_files, get_jobs_list) rather +# than hand-rolling calls to the low-level generated endpoints. That module +# already builds the right request bodies and already raises on a non-200 +# response (via its own try_call helper) -- there is no reason to duplicate +# that logic here and every reason not to, since duplicating it is exactly +# how field names or status-code handling drift out of sync with what +# Qualcomm ships. +# +# NOTE ON PRODUCTION READINESS: get_job_log_files and get_job_status go +# through the same vendor code Qualcomm's own client uses, so their shapes +# are as trustworthy as the SDK itself. upload_file/submit_job have never +# been exercised against a live account from this codebase -- every +# historical submission went through the QDC web console instead. Confirm +# end to end on a single, cheap, short-timeout dry run before trusting this +# path with anything that matters. get_jobs_list is wired up here only so a +# test can prove it is never called by wait() -- see the guard test in +# hexlib/tests/test_qdc.py and fact 2 in this module's docstring. + + +def _real_get_job_log_files(client, job_id): + from qualcomm_device_cloud_sdk.api import qdc_api as _vendor + + return _vendor.get_job_log_files(client, job_id) or [] + + +def _real_get_job_status(client, job_id): + from qualcomm_device_cloud_sdk.api import qdc_api as _vendor + + return _vendor.get_job_status(client, job_id) + + +def _real_get_jobs_list(client, page_number=0, page_size=20): + # Wired up for completeness and for the guard test only. wait() must + # never call this -- get_jobs_list lagged more than 30 minutes on both + # jobs observed on this account. + from qualcomm_device_cloud_sdk.api import qdc_api as _vendor + + return _vendor.get_jobs_list(client, page_number, page_size) + + +def _real_upload_artifact(client, zip_path: str) -> str: + from qualcomm_device_cloud_sdk.api import qdc_api as _vendor + from qualcomm_device_cloud_sdk.models.artifact_type import ArtifactType + + # TESTSCRIPT, NOT TESTPACKAGE, and this was measured rather than reasoned. + # Uploading the same zip as TESTPACKAGE authenticates and uploads fine and + # is then refused at submission with HTTP 400 "Appium tests requires 1 test + # script and at most 1 test package" -- the package is the OPTIONAL half and + # the script is the required one, so one package and zero scripts is not a + # valid Appium job. llama.cpp's own QDC runner + # (scripts/snapdragon/qdc/run_qdc_jobs.py) uploads its Appium zip as + # TESTSCRIPT against this same API, which is the working reference this was + # matched to. Costs no device minutes to get wrong -- the job is rejected + # before dispatch -- but it does cost a round trip. + uuid = _vendor.upload_file(client, zip_path, ArtifactType.TESTSCRIPT) + if not uuid: + raise QdcError("QDC accepted the artifact upload but returned no uuid") + return uuid + + +def _real_submit_job(client, artifact_uuid: str, timeout_min: int): + from qualcomm_device_cloud_sdk.api import qdc_api as _vendor + from qualcomm_device_cloud_sdk.models.job_type import JobType + from qualcomm_device_cloud_sdk.models.job_mode import JobMode + from qualcomm_device_cloud_sdk.models.test_framework import TestFramework + + return _vendor.submit_job( + client, + target_id=TARGET_ID, + job_name="hexlib", + external_job_id=None, + job_type=JobType.AUTOMATED, + job_mode=JobMode.APPLICATION, + timeout=timeout_min, + test_framework=TestFramework.APPIUM, + entry_script=None, + job_artifacts=[artifact_uuid], + monkey_events=None, + monkey_session_timeout=None, + ) + + +def _real_download_log_file(client, filename: str, local_path: str) -> bool: + from qualcomm_device_cloud_sdk.api import qdc_api as _vendor + + return bool(_vendor.download_job_log_files(client, filename, local_path)) + + +qdc_api = types.SimpleNamespace( + get_job_log_files=_real_get_job_log_files, + get_job_status=_real_get_job_status, + get_jobs_list=_real_get_jobs_list, + upload_artifact=_real_upload_artifact, + submit_job=_real_submit_job, + download_log_file=_real_download_log_file, +) + + +def submit(zip_path: str, *, timeout_min: int) -> int: + """Submit `zip_path` (from artifact.stage) as a job on TARGET_ID and + return the job id. timeout_min is required -- there is no default -- + and must be in 1..240; a runaway job spends real money.""" + if not MIN_TIMEOUT_MIN <= timeout_min <= MAX_TIMEOUT_MIN: + raise QdcError( + f"timeout_min must be {MIN_TIMEOUT_MIN}..{MAX_TIMEOUT_MIN}, " + f"got {timeout_min}" + ) + if not os.path.isfile(zip_path): + raise QdcError(f"artifact not found: {zip_path}") + + client = _client() + artifact_uuid = qdc_api.upload_artifact(client, zip_path) + job_id = qdc_api.submit_job(client, artifact_uuid, timeout_min) + if job_id is None: + raise QdcError("QDC accepted the submission but returned no job_id") + return job_id + + +def _results_filename(name: object) -> bool: + """True if `name` IS the results file, by path suffix -- not merely a name + that CONTAINS the marker somewhere. `RESULTS_MARKER in name` also matched + `TestLogs/results.xml.part` and `TestLogs/results.xml.tmp`, i.e. exactly + the half-written intermediate whose appearance is the one thing a + completion detector must not fire on. Separators are normalized because + QDC's own filenames use forward slashes and nothing guarantees a future + field will.""" + if not isinstance(name, str) or not name: + return False + return name.replace("\\", "/").endswith(RESULTS_MARKER) + + +def _has_results(files) -> bool: + """PURELY A FILENAME TEST -- it never opens anything. A zero-byte + results.xml satisfies it. That is deliberate (this module cannot read a + file it has not downloaded yet) and it is why `wait()` is not a verdict; + see `wait()` and fact 2 in the module docstring.""" + return any(_results_filename(getattr(f, "filename", None)) for f in files) + + +def wait(job_id: int, cap_s: int = 1800) -> bool: + """Block until a log file named TestLogs/results.xml APPEARS among + job_id's log files, or until cap_s seconds have passed. + + WHAT True GUARANTEES, EXACTLY: that a file with that name now exists in + QDC's log listing for this job. NOTHING MORE. This function does not + download it, does not open it, does not parse it, and cannot tell a real + JUnit report from a zero-byte file with the right name -- `_has_results` + is a filename test (see its own docstring). True therefore means + "finished, probably" and is a PRECONDITION for judging the job, never the + judgement. + + WHERE THE REAL CHECK LIVES: `hexlib/cli.py::_qdc_check_results`, called by + `_qdc_submit` after `fetch()`. It parses the report and requires + tests > 0, no failures, no errors, no skips, and hexlib's own measurement + lines (`hexlib: --self-test: PASS`, a positive `cycles_total=`) in the + fetched logs. A caller that treats this function's True as a pass + reintroduces this account's own history -- a job that ran zero tests and + reported passing -- with the check one level further away. + + THE CAP IS THE CALLER'S TO CHOOSE, and the default is not a safe one for + every job: 1800 s is SMALLER than the 240 minutes `submit()` accepts, so + passing a job's own timeout through is the caller's responsibility (cli.py + derives it in `_qdc_wait_cap_s`). Returns False at the cap rather than + hanging forever; False must never be confused with success, and it also + does not mean the job failed -- it means this function stopped watching, + which is why cli.py fetches whatever logs exist before reporting it. + + Never touches get_job_status or the jobs list: see the module docstring. + """ + client = _client() + deadline = time.monotonic() + cap_s + errors = 0 + while True: + # A TRANSIENT API ERROR IS NOT A VERDICT. This call used to be + # unguarded, so one HTTP 502 anywhere in ~60 polls raised straight out + # of wait(), through _qdc_submit, and killed the CLI -- discarding a + # job whose minutes were already spent, before anything was fetched or + # checked. Observed on job 756206 (2026-08-12), and a second 502 hit + # get_job_log_upload_status for 756159 the same afternoon, so this is + # a property of the service rather than one bad moment. llama.cpp's + # runner retries these calls for the same reason. + # + # Swallowed only until the cap, never forever, and counted so that + # "the API was down the whole time" cannot masquerade as the ordinary + # "no results appeared" answer -- those are different findings and the + # caller is told which one it got. + try: + files = qdc_api.get_job_log_files(client, job_id) + if _has_results(files): + return True + except Exception as e: # noqa: BLE001 - any transport failure retries + errors += 1 + print( + f"qdc: polling job {job_id} log files failed " + f"({errors} time(s)), retrying: {e}" + ) + if time.monotonic() >= deadline: + if errors: + print( + f"qdc: gave up on job {job_id} after {cap_s}s with " + f"{errors} failed poll(s) -- if that is every poll, this " + f"is an API outage, not a job that produced nothing" + ) + return False + time.sleep(POLL_S) + + +def _local_log_path(dest: str, name: str) -> str | None: + """Where QDC's log file `name` should land under `dest`, PRESERVING QDC's + own directory structure, or None if `name` cannot be mapped safely. + + `os.path.join(dest, os.path.basename(name))` flattened it, and QDC's + listings are not flat: `TestLogs/results.xml` and `logs/results.xml` both + became `dest/results.xml`, so one silently overwrote the other and the + returned `paths` list held two entries pointing at one file -- with + cli.py's results check then reading whichever download happened to finish + last. Flattening is not a cosmetic problem here: the one file this whole + path is built to read is identified by that basename. + + Returns None for anything that would escape `dest` -- an absolute path, a + drive letter, or a `..` component. QDC's own names have never looked like + that, which is exactly why nothing would notice if one did. + + THE CHECK MUST NOT DEPEND ON THE HOST OS, and it did. This used + `os.path.isabs` and `os.path.splitdrive`, both of which are the RUNNING + platform's notion of a path. So `C:/Windows/System32/evil.txt` was refused + on Windows and ACCEPTED on Linux -- where it is not absolute at all, merely + a relative path whose first component happens to be named `C:` -- and the + file landed in `dest/C:/Windows/System32/evil.txt`. The mirror case is + `/etc/passwd`, absolute on POSIX and not on Windows. + + That asymmetry matters because the two ends run on different systems: the + developer machine here is Windows, CI is Linux, and the name being screened + arrives from a remote service. A traversal guard that only holds on the + platform you happened to test on is the "claim of protection that protects + nothing" pattern, in a place where the input is not ours. + + Both flavours are now consulted, so a name is refused if EITHER OS would + read it as absolute or drive-qualified, regardless of where this runs. + Found by CI on the first push after CI itself was repaired. + """ + unified = name.replace("\\", "/").strip() + if not unified: + return None + parts = [p for p in unified.split("/") if p not in ("", ".")] + if not parts or any(p == ".." for p in parts): + return None + for flavour in (pathlib.PureWindowsPath, pathlib.PurePosixPath): + p = flavour(unified) + if p.is_absolute() or p.anchor or p.drive: + return None + # A bare drive-relative name (`C:foo`) has no anchor under PureWindowsPath + # but still names a drive, and a component containing `:` cannot be a + # legitimate QDC log path segment on any platform we target. + if any(":" in p for p in parts): + return None + return os.path.join(dest, *parts) + + +def fetch(job_id: int, dest: str) -> list[str]: + """Download every log file QDC has for job_id into dest, MIRRORING QDC's + own directory layout beneath it, and return the local paths written. + + Only meaningful after wait() has returned True -- and note that wait() + returning True proves only that a file with the right NAME appeared, so + even a full fetch is not a verdict: `hexlib/cli.py::_qdc_check_results` + is what judges the job. + """ + client = _client() + files = qdc_api.get_job_log_files(client, job_id) + os.makedirs(dest, exist_ok=True) + + paths = [] + for f in files: + name = getattr(f, "filename", None) + if not name or not isinstance(name, str): + continue + local = _local_log_path(dest, name) + if local is None: + continue + os.makedirs(os.path.dirname(local), exist_ok=True) + if qdc_api.download_log_file(client, name, local): + paths.append(local) + return paths diff --git a/hexlib/device/qdc/test_on_device.py b/hexlib/device/qdc/test_on_device.py new file mode 100644 index 0000000..3338db1 --- /dev/null +++ b/hexlib/device/qdc/test_on_device.py @@ -0,0 +1,342 @@ +# hexlib/device/qdc/test_on_device.py +"""Runs ON THE DEVICE, under the farm's own pytest (see `artifact.py`'s +`requirements.txt`, which asks QDC's runner to `pip install pytest` before +running this file). NOT PART OF HEXLIB'S OWN SUITE -- see +`hexlib/tests/test_qdc_on_device_is_excluded.py` for the mechanism that keeps +`pytest hexlib/tests` from ever collecting this file, and the test that +proves it. + +FAIL CLOSED, LOUDLY. A device-farm job on this project's own QDC account once +COMPLETED HAVING RUN ZERO TESTS AND REPORTED PASSING. So, throughout this +file: every expected line is asserted PRESENT, never merely "the bad thing is +absent" (an empty log satisfies "absent" trivially); the binary's OWN exit +code is checked (via the `; echo RC=$?` convention `utils.sh` documents), not +just whatever text happened to reach stdout; and a missing, empty, or +unparseable log is written and then failed on, never silently skipped. + +EVERY STRING ASSERTED BELOW WAS READ DIRECTLY OUT OF +`hexlib/runtime/host/main.c`, not guessed or copied from an earlier draft of +this task. Two of the checks below (the unmapped-fd refusal and the +cache-coherency discriminator) need a small addition to `main.c` that does +not exist yet -- see the comment on each for exactly what and why, and +`.superpowers/sdd/2026-08-10-silicon-path-runtime/task-12-report.md` for the +full account. Those two are still written here, in full, on purpose: the +alternative -- leaving the requirement out because today's binary cannot +satisfy it -- is exactly the kind of silent gap this project's own history +(the false-pass job) says not to leave. +""" +import re + +from utils import push, sh, write_qdc_log + +DEV = "/data/local/tmp/hexlib" + +# A LITERAL, DELIBERATELY, AND BONDED BY A TEST. This module runs on the QDC +# runner where hexlib is not installed -- it imports `utils` as a flat module, +# not `hexlib.runtime.build` -- so it cannot call `device_skel_so_name()` and +# has to spell the name out. `hexlib/tests/test_device_skel_so_name.py` reads +# this literal back and asserts it equals what the linker produces, because two +# independently written copies of a filename are exactly the drift that cost a +# device session: the skel was linked `libhexlib_skel.so`, FastRPC dlopens the +# name in qaic's generated URI, and `hexlib_iface_open` failed with +# rc -2147482618 until the file was pushed under both names by hand. +SKEL_SO = "libhexlib_iface_skel.so" + +# `hexlib_run` prints `hexlib: : cycles_total=%llu` (main.c's +# run_self_test and run_coherency_check). Built as a regex, not a substring, +# because the SUBSTRING IS SATISFIED BY `cycles_total=0` -- a run in which the +# DSP's PCYCLE counter never advanced at all. That is not a pedantic edge +# case: the skel reads the counter inside a user-mode unsigned PD, where +# SYSCFG.PCYCLEEN cannot be set (skel_dispatch.c's hexlib_read_pcycle, and +# include/hexlib/hexlib_harness.h, which sets that bit explicitly for the +# standalone-ELF runtime because the register reads 0 without it). Zero is +# precisely what this job would print if the counter is dead on this silicon, +# and it is the single most important thing this job can report. +_CYCLES_RE = re.compile(r"cycles_total=(\d+)") + + +def assert_cycles_total_is_a_real_measurement(out, what): + """Assert `out` carries at least one `cycles_total=` line whose value is a + decimal integer greater than zero, and return that value. + + PRESENT AND POSITIVE, as two separate failures with two separate + messages -- "absent" and "zero" are different findings and must not be + reported as each other. Absence means `hexlib_run` stopped printing it (or + never got a response); zero means it printed a measurement of nothing, + which on this platform points straight at PCYCLEEN in the unsigned PD.""" + values = [int(m) for m in _CYCLES_RE.findall(out)] + assert values, ( + f"{what}: no `cycles_total=` line at all. hexlib_run prints one after " + f"the PASS line (main.c's run_self_test) -- its ABSENCE is a failure, " + f"never a success:\n{out}" + ) + best = max(values) + assert best > 0, ( + f"{what}: cycles_total={best} -- the DSP measured NOTHING. The kernel " + f"call is bracketed by PCYCLE on the DSP (skel_dispatch.c), so a real " + f"call cannot take zero cycles; a zero here means the counter did not " + f"advance, which is what happens when SYSCFG.PCYCLEEN is clear -- and " + f"a user-mode unsigned PD cannot set it. THIS IS THE MOST IMPORTANT " + f"THING THIS JOB CAN REPORT: every cycle figure in stage 1 was " + f"measured the same way, and if the counter is dead here then none of " + f"them transfer to silicon:\n{out}" + ) + return best + + +def test_binaries_are_present_and_executable(): + # PUSHED, not copied. pytest runs on the QDC RUNNER and the binaries are + # AArch64/Hexagon, so `cp` moved them between two host directories and + # succeeded -- the artifact "landed" and only failed later, as an exec + # format error. See utils.py's module docstring. + sh(f"mkdir -p {DEV}") + push("hexlib_run", DEV) + push(SKEL_SO, DEV) + sh(f"chmod 755 {DEV}/hexlib_run") + out = sh(f"ls -l {DEV}") + assert "hexlib_run" in out, f"hexlib_run did not land in {DEV}:\n{out}" + assert SKEL_SO in out, f"{SKEL_SO} did not land in {DEV}:\n{out}" + + +def test_capabilities_report_a_v75_cdsp_with_unsigned_pd(): + """Every substring below is `print_caps()`'s OWN output format + (`hexlib/runtime/host/main.c`), not the brief's earlier, wrong guesses + (`ARCH_VER`, `UNSIGNED_PD_SUPPORT = 1`) -- the real lines are lowercase + and shaped `domain = CDSP (3)`, `unsigned_pd_support = 1`, + `arch_ver = 35957 (0x8c75)`. CDSP is domain 3, measured; ADSP + (domain 0) is a v73 part with `UNSIGNED_PD_SUPPORT = 0` and must never be + the thing this printed. + + THE EXIT CODE IS NOW CHECKED, AND UNTIL 2026-08-11 IT WAS NOT. This was + the ONLY test in this file that ran `hexlib_run` without the `; echo RC=$?` + convention, which made this file's own stated discipline -- "the binary's + OWN exit code is checked" (module docstring) -- inapplicable to the one + mode most likely to fail first on unfamiliar silicon. It was + unenforceable, not merely unenforced: `print_caps()` returned `void` and + `main()` returned `HEXLIB_EXIT_OK` unconditionally, so on a device whose + image has no `libcdsprpc.so` for this ABI, `--caps` printed "could not + load the FastRPC driver" and exited 0. `print_caps()` now returns + HEXLIB_EXIT_SESSION_FAILED (2) on either failure branch, and this asserts + RC=0 -- so a driver that will not load fails HERE, loudly, instead of + being read as a pass by this test and by every `set -e` wrapper around + it.""" + out = sh(f"cd {DEV} && ADSP_LIBRARY_PATH={DEV} ./hexlib_run --caps; echo RC=$?") + write_qdc_log("hexlib_caps.log", out) + assert "RC=0" in out, ( + f"hexlib_run --caps exited nonzero -- the FastRPC driver did not load " + f"or the capability query failed (both are exit 2, " + f"HEXLIB_EXIT_SESSION_FAILED):\n{out}" + ) + assert "CDSP (3)" in out, f"expected domain CDSP (3), got:\n{out}" + assert "arch_ver" in out, f"no arch_ver line at all:\n{out}" + assert "35957" in out and "0x8c75" in out, f"unexpected arch, expected 35957 (0x8c75):\n{out}" + assert "unsigned_pd_support = 1" in out, ( + f"expected unsigned_pd_support = 1 on CDSP, got:\n{out}" + ) + + +def test_scale_fp16_runs_on_the_dsp_and_is_correct(): + """`run_self_test()` in main.c prints exactly one line on success: + `hexlib: --self-test: PASS (4100 values, bit-exact)` -- there is no + `SELFTEST`/`status=1`/`cycles=` text anywhere in that function; the + brief's earlier draft invented all three. Checked directly against the + source before writing this assertion. + + GAP CLOSED 2026-08-11 -- AND THE STALE DOCSTRING IS WHY THE ASSERTION WAS + MISSING. This said: "KNOWN GAP: `run_self_test()` computes + `hexlib_batch_rsp_hdr.cycles_total` ... but never prints it, so this file + cannot read a silicon cycle count off `hexlib_run`'s own stdout today." + That was superseded by an earlier commit -- `main.c`'s `run_self_test` + prints `hexlib: --self-test: cycles_total=%llu` right after the PASS line + -- but the docstring stayed, and because it said the line could not be + read, nothing here read it. The consequence was concrete: NOTHING ON + DEVICE asserted the `cycles_total=` line at all, so deleting those two + `printf` lines from main.c would have kept all five of this file's tests + green while flipping `hexlib/cli.py`'s post-job check to exit 1 -- the + device job passing and the gate above it failing, off the same run. + + The assertion below is now the load-bearing one for stage 3's most + important open question: whether PCYCLE advances at all in a user-mode + unsigned PD. See `assert_cycles_total_is_a_real_measurement`.""" + out = sh(f"cd {DEV} && ADSP_LIBRARY_PATH={DEV} ./hexlib_run --self-test; echo RC=$?") + write_qdc_log("hexlib_selftest.log", out) + assert "RC=0" in out, f"hexlib_run --self-test exited nonzero:\n{out}" + assert "hexlib: --self-test: PASS" in out, ( + f"the PASS line must be PRESENT -- absence is failure, not success:\n{out}" + ) + assert "bit-exact)" in out, f"PASS line present but not the bit-exact qualifier:\n{out}" + assert_cycles_total_is_a_real_measurement(out, "hexlib_run --self-test") + # A weaker, supplementary check ONLY -- the two asserts above already + # require the specific success line to be present; this just also rules + # out a run that printed both the PASS line AND a mismatch report, which + # would be self-contradictory output worth catching on its own. + assert "mismatch" not in out.lower() + + +def test_the_dsp_refuses_an_unmapped_fd_on_silicon_too(): + """The same discriminator that `hexlib/tests/test_dsp_sim.py` proved by + mutation on the simulator (see `docs/STATE.md`'s Stage 1 entry), exercised + on real hardware instead of `hexagon-sim`. It should pass trivially here + -- but if it does NOT, the simulator was hiding something, and that is + the single most important thing this job can report; hence this is + asserted explicitly rather than left implicit in a passing self-test. + + SUPERSEDED 2026-08-11 -- THE FLAG NOW EXISTS (cd2582b). Kept because it records why it was added. Formerly: `hexlib_run`'s + `main()` only ever inspects `argv[1]` (`--caps` / `--self-test` / + `--batch`) -- there is no `--unmapped` flag today, unlike + `hexlib/runtime/simhost/simhost.c`'s, which deliberately skips + `hexlib_iface_mmap` for exactly this test. `--self-test --unmapped` + therefore runs the ORDINARY self-test right now, ignoring the extra + argument, and this test will fail (not vacuously pass) until main.c + grows the small addition described in the task-12 report: build the + second self-test buffer's fd via the driver-level rpcmem/fastrpc_mmap + calls `hexlib_host.h` already exposes, WITHOUT the + `hexlib_iface_mmap` registration call `hexlib_alloc` normally makes, so + `hexlib_bufs_map`'s table lookup (skel_bufs.c) genuinely has nothing to + find. That addition is deliberately NOT made here -- it would mean + editing `hexlib/runtime/*`, out of this task's scope -- so this test + documents the requirement and fails loudly rather than being silently + dropped. The strings below are what `main.c`'s EXISTING status-handling + code already prints once that one addition lands: `hexlib_dispatch_batch` + (skel_dispatch.c) writes `HEXLIB_DSP_ERR_UNMAPPED` (7) as the batch's + top-level status, and `run_self_test()`'s existing + `status != HEXLIB_DSP_OK` branch already prints + `"hexlib: --self-test: batch status %u, not HEXLIB_DSP_OK"` and returns + `HEXLIB_EXIT_OP_FAILED` (4) -- no NEW print statement is needed, only the + skipped registration call. + """ + out = sh(f"cd {DEV} && ADSP_LIBRARY_PATH={DEV} ./hexlib_run --self-test --unmapped; echo RC=$?") + write_qdc_log("hexlib_unmapped.log", out) + assert "RC=4" in out, ( + f"expected HEXLIB_EXIT_OP_FAILED (4) once --unmapped exists; got:\n{out}" + ) + assert "batch status 7, not HEXLIB_DSP_OK" in out, ( + f"expected HEXLIB_DSP_ERR_UNMAPPED (7) reported by the DSP, got:\n{out}" + ) + + +def test_cache_coherency_is_independent_of_marshalling_and_of_any_kernel(): + """Design doc §6.1: `buffers.c` maps with `FASTRPC_MAP_FD`, `remote.h` + documents that flag as putting cache maintenance on US, `rpcmem` + allocates CACHED memory by default, and the SDK has no CPU-side flush or + invalidate call at all. A coherency miss on real hardware therefore + presents EXACTLY like a marshalling bug -- wrong values out of a call + that otherwise looks correct -- and the device path is the only place + marshalling is exercised at all, so the two confound each other precisely + where there is no cheaper way to tell them apart. This check exists so a + failure says WHICH of the two it is on the FIRST job, not the third. + + SUPERSEDED 2026-08-11 -- THE FLAG NOW EXISTS (cd2582b) AND THIS DESIGN WAS INCOMPLETE; see the correction at the end. Formerly: Unlike the unmapped-fd + flag above, `--coherency-check` does not exist ANYWHERE today, not even + in shape (there is no simulator equivalent to mirror, because host and + DSP share one address space there and a cache-coherency question does + not exist to ask). This is this file's OWN proposed design for the + smallest addition that would make the check expressible with capabilities + `hexlib_run` already has -- described in full in the task-12 report -- + and it is NOT implemented, on purpose (implementing it means editing + `hexlib/runtime/*`, out of this task's scope). The design, so the strings + below are not arbitrary: + + 1. Build the same two self-test buffers `run_self_test()` already builds + (`x`, `y`), but before `hexlib_invoke`, the CPU writes a known + NON-ZERO sentinel (e.g. every fp16 lane set to 1.0) into `y`'s rpcmem + -- something `run_self_test()` does not do today, since it never + reads `y` until after invoke. + 2. The batch's `scale` op uses `factor = 0.0`, not `0.125`. `x * 0.0` is + bit-exact zero in fp16 for any finite, non-NaN `x` -- there is no + numerically ambiguous case, so a wrong result here cannot be blamed on + kernel arithmetic. + 3. After invoke, if the response is valid AND its status is + `HEXLIB_DSP_OK` (proving the batch genuinely ran -- both marshalling + and dispatch already succeeded), re-read `y`. If it is bit-exact + all-zero, the DSP's write reached the CPU: coherent. If it still reads + the sentinel, the DSP wrote zero but the CPU observed its OWN stale + cache line instead -- conclusively a coherency miss, not a marshalling + bug (marshalling already succeeded, per the status check) and not a + kernel bug (the arithmetic is exact and trivial). + 4. Print a line whose presence states the verdict, e.g. + `"COHERENCY sentinel_overwritten"` (coherent -- the DSP's write was + observed) vs. `"COHERENCY sentinel_unchanged"` (a coherency miss). + + Until that lands, this test's own `sh()` call fails the moment the flag + is rejected -- which is the honest state of this discriminator today: not + yet expressible, not silently skipped. + + FIRST CORRECTION (design doc 6.1, 2026-08-11): THE SENTINEL ALONE DOES + NOT DISCRIMINATE. If dispatch silently no-ops and still returns + HEXLIB_DSP_OK, the observable is identical to a coherency miss: status OK, + sentinel intact. + + SECOND CORRECTION (design doc 6.1, same day) -- AND THE FIRST + CORRECTION'S OWN FIX WAS ALSO WRONG. It said `cycles_total` separates the + two, and printed this table, WHICH IS NOW RETRACTED: + cycles 0, sentinel intact -> kernel never ran: a dispatch bug + cycles >0, sentinel intact -> ran, write never reached host: COHERENCY + cycles >0, sentinel overwritten -> healthy, for this direction + It does not separate them, for three reasons: + 1. ROW 1 IS UNREACHABLE. main.c reads the sentinel only after BOTH the + batch status and the op's own result->status are HEXLIB_DSP_OK, which + in skel_dispatch.c happens only if `k->fn(&a)` was called and + returned OK. PCYCLE brackets exactly that call, so cycles > 0 for any + real call. Dispatch REFUSING the batch is distinguishable -- by exit + 4 and by NO COHERENCY line at all -- but not by a cycle count. + 2. THE NAMED DEFECT IS INVISIBLE TO IT. "Dispatch no-ops and returns OK" + means a generated entry or kernel that returns OK WITHOUT WRITING + `y`. PCYCLE still brackets a real returning call, so cycles > 0 and + the sentinel is intact -- row 2 fires and prints + `sentinel_unchanged`, misattributing a dispatch bug to coherency. + 3. IF PCYCLE READS 0 IN THE UNSIGNED PD, EVERY ROW INVERTS: a genuine + coherency miss would read cycles 0 + sentinel intact, which row 1 + called a dispatch bug. + + WHAT THE OUTPUT ACTUALLY MEANS (the honest table; see design 6.1 and + run_coherency_check()'s own header comment in main.c): + no COHERENCY line, exit 4 -> dispatch refused the batch. The + one genuinely diagnostic outcome. + cycles_total=0, any COHERENCY line -> THE COUNTER IS DEAD, not "the + kernel never ran" (see 1). Settle + this before reading anything else. + cycles >0, sentinel_overwritten -> the DSP's write reached the host, + for scale_fp16's write pattern and + this buffer size only. + cycles >0, sentinel_unchanged -> NOT DISCRIMINATED: a coherency + miss, OR a kernel/entry that + returned OK without writing `y`, + OR an fd mapped elsewhere. Rule + the no-op out with the ordinary + --self-test (factor 0.125, whose + bit-exact values a no-op cannot + produce) BEFORE spending a job on + cache flags. + cycles >0, buffer_garbled -> partial or misdirected write. Its + own outcome, never folded above. + + WHAT WOULD DISCRIMINATE, deliberately not built: a skel-side echo or + memset op with its own kind id, so the write is performed by the skel and + not by any generated kernel. Deferred in 6.1; reason 2 above is what that + deferral costs. + + Two limits stated rather than implied: riding on `scale_fp16` is NOT + kernel-independent (needs that skel-side echo op), and this covers only + DSP-write -> host-read. The host-write -> DSP-read direction, which + every input buffer and the batch blob depend on, is UNTESTED. + + """ + out = sh( + f"cd {DEV} && ADSP_LIBRARY_PATH={DEV} ./hexlib_run --self-test " + f"--coherency-check; echo RC=$?" + ) + write_qdc_log("hexlib_coherency.log", out) + assert "RC=0" in out, f"hexlib_run --self-test --coherency-check failed:\n{out}" + assert "COHERENCY sentinel_overwritten" in out, ( + "the CPU must observe the DSP's own write, not a stale sentinel -- " + f"a coherency miss looks exactly like a marshalling bug, and this line's " + f"absence is that miss:\n{out}" + ) + # NECESSARY, NOT SUFFICIENT -- see the second correction above. A positive + # cycles_total does NOT prove this was a coherency verdict rather than a + # dispatch one; it rules out exactly one thing, that the DSP measured + # nothing at all, which would make every other line here unreadable. + assert_cycles_total_is_a_real_measurement( + out, "hexlib_run --self-test --coherency-check" + ) diff --git a/hexlib/device/qdc/utils.py b/hexlib/device/qdc/utils.py new file mode 100644 index 0000000..8f6e6cc --- /dev/null +++ b/hexlib/device/qdc/utils.py @@ -0,0 +1,185 @@ +# hexlib/device/qdc/utils.py +"""Helpers for `test_on_device.py`, staged into the SAME flat TestPackage +directory (see `artifact.py`'s own "no subdirectory nesting" reasoning) and +imported there as a bare top-level module -- `from utils import sh, +write_qdc_log` -- because there is no `hexlib` package once QDC's runner +extracts the zip; only whatever files were zipped up sit next to each other. + +THIS FILE RUNS ON THE DEVICE, alongside `test_on_device.py`, under the farm's +own pytest -- never under `hexlib/tests`. See that file's own module +docstring for the exclusion this project relies on, and note that this module +would be just as uncollectable there: it assumes a POSIX shell and +`/data/local/tmp` exist, neither of which is true of the machine running our +own offline suite. + +`sh()` WRAPS EVERY COMMAND IN `adb shell`, AND THAT IS A CORRECTION. + +This file used to argue the opposite at length: that QDC provisions a Python +ON the device and runs the TestPackage there, so on-device paths could be +named directly with no `adb` prefix. That reasoning ended with "if that +assumption is ever wrong, the fix belongs in `sh()`, in one place." It was +wrong, and this is that one place. + +WHAT SETTLED IT. llama.cpp's own QDC runner -- the working reference on this +same account and framework -- reaches the device exclusively through `adb`: +`run_adb_command` is `adb shell "; echo __RC__:$?"`, its `write_qdc_log` +does `adb push` into `/data/local/tmp/QDC_logs`, and its `SCRIPTS_DIR` +(`/qdc/appium`) is a HOST path holding the extracted zip. pytest runs on the +QDC RUNNER, not on the phone. + +WHAT THE OLD ASSUMPTION COST. Jobs 756124 and 756159 (2026-08-12) both +reached Completed and returned no logs of their own. Under the old model +`write_qdc_log` wrote to `/data/local/tmp/QDC_logs` ON THE RUNNER, a +directory QDC never collects because it collects that path from the DEVICE -- +so a report could be written perfectly and still be invisible. Worse, +`test_binaries_are_present_and_executable` ran a plain `cp`, which on a Linux +runner SUCCEEDS at copying an AArch64 binary into a host directory, and the +failure only surfaces later as an exec-format error on a file that "landed" +correctly. + +So: `sh()` runs on the device, `push()` moves staged files there, and +`write_qdc_log` pushes the report to where QDC actually collects from. +""" +from __future__ import annotations + +import os +import subprocess +import tempfile + +QDC_LOG_DIR = "/data/local/tmp/QDC_logs" + +# Where the staged BINARIES are ON THE RUNNER. This module lives in `tests/` +# and the binaries in `bin/`, both under QDC's extraction point (`/qdc/appium` +# in every observed job), so the binaries are a sibling directory up one +# level -- derived from this file's own location rather than hardcoding +# `/qdc/appium`, which is documented but not promised. +# +# Falls back to this module's own directory when there is no `bin/` sibling, +# which is how it looks in the repo, so importing this file outside an +# extracted artifact still gives a sane value instead of a path that cannot +# exist. +_TESTS_DIR = os.path.dirname(os.path.abspath(__file__)) +_BIN_DIR = os.path.join(os.path.dirname(_TESTS_DIR), "bin") +STAGE_DIR = _BIN_DIR if os.path.isdir(_BIN_DIR) else _TESTS_DIR + + +class ShError(Exception): + """`sh()` raised because the command exited nonzero. See `sh()`'s own + docstring for why a command that embeds `echo RC=$?` never reaches this + at all -- that is ordinary shell semantics, not special-cased here.""" + + +def sh(cmd: str) -> str: + """Run `cmd` through a POSIX shell and return its combined stdout+stderr. + + Raises `ShError` if the shell's own exit status is nonzero. A caller that + wants to inspect a COMMAND's failure explicitly (`--self-test --unmapped` + is *expected* to make `hexlib_run` exit nonzero; that expected failure is + the whole point of the test) appends `; echo RC=$?` to `cmd` itself: the + shell's own exit status then becomes `echo`'s -- always 0 -- regardless of + what the real command did, and the caller reads the real code back out of + the text this function returns. Nothing here special-cases that string; + it is a consequence of how `;`-joined shell commands report their exit + status, not a convention this function has to know about. + + FAIL CLOSED: a command that exits nonzero WITHOUT that trick is a genuine, + unexpected failure -- of the command, of `cd`, of a path that does not + exist -- and must stop the test right there rather than let a later + assertion run against output that was never produced for the reason the + test assumed. + """ + proc = subprocess.run( + ["adb", "shell", cmd], + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, + text=True, + ) + out = proc.stdout or "" + if proc.returncode != 0: + raise ShError( + f"`adb shell` exited {proc.returncode}, and the command did not " + f"embed its own `echo RC=$?` to report that itself: {cmd!r}\n{out}" + ) + return out + + +def push(src_name: str, dest_dir: str) -> str: + """`adb push` a file staged beside this module into `dest_dir` on the + device, and return the resulting device path. + + A PUSH, NOT A COPY. `cp` was what this used to be, back when the test was + believed to run on the phone; on the QDC runner that copies an AArch64 + binary from one host directory to another, reports success, and defers + the failure to an exec-format error nobody would connect back to it. + """ + src = os.path.join(STAGE_DIR, src_name) + if not os.path.isfile(src): + raise ShError( + f"{src_name} is not beside this module ({STAGE_DIR}); the artifact " + f"did not stage what it claimed to. Contents: " + f"{sorted(os.listdir(STAGE_DIR))}" + ) + proc = subprocess.run( + ["adb", "push", src, dest_dir], + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, + text=True, + ) + if proc.returncode != 0: + raise ShError( + f"adb push {src!r} -> {dest_dir!r} exited {proc.returncode}:\n" + f"{proc.stdout or ''}" + ) + return dest_dir.rstrip("/") + "/" + src_name + + +def write_qdc_log(name: str, text: str) -> str: + """Push `text` to `{QDC_LOG_DIR}/{name}` ON THE DEVICE and return that + device path. + + A PUSH, not a local write. QDC collects that directory FROM THE PHONE; + writing it on the runner (which is what this did) produces a report that + is real, correct, and never collected -- see the module docstring for the + two jobs that cost. + + Always writes, even if `text` is empty -- an empty or missing log must + never be silently indistinguishable from "nothing worth logging"; that + exact confusion is what let a device-farm job on this account once + complete having run zero tests and report passing. This function's job is + only to guarantee the write happens at a real, returned path; it does not + itself judge whether `text` is meaningful -- the caller's own assertions, + run BEFORE this is called, are what a reader should trust for that. + """ + # POSIX join, never os.path.join: this path is on the DEVICE, and a + # Windows runner would otherwise build `TestLogs\results.xml`. + device_path = QDC_LOG_DIR + "/" + name.replace("\\", "/").lstrip("/") + + # mkdir -p THE PARENT, on the device. `name` legitimately carries a + # subdirectory -- conftest.py writes `TestLogs/results.xml`, because + # job.wait() matches that suffix and QDC lists collected logs as + # `/`, so a flat name would never be recognised as the + # report at all. + parent = device_path.rsplit("/", 1)[0] + subprocess.run( + ["adb", "shell", f"mkdir -p {parent}"], + stdout=subprocess.PIPE, stderr=subprocess.STDOUT, text=True, + ) + + with tempfile.NamedTemporaryFile( + mode="w", suffix=".log", delete=False, encoding="utf-8" + ) as f: + f.write(text) + tmp_path = f.name + try: + proc = subprocess.run( + ["adb", "push", tmp_path, device_path], + stdout=subprocess.PIPE, stderr=subprocess.STDOUT, text=True, + ) + if proc.returncode != 0: + raise ShError( + f"adb push of the log to {device_path!r} exited " + f"{proc.returncode}:\n{proc.stdout or ''}" + ) + finally: + os.unlink(tmp_path) + return device_path diff --git a/hexlib/exec/dsp.py b/hexlib/exec/dsp.py new file mode 100644 index 0000000..c1ad25d --- /dev/null +++ b/hexlib/exec/dsp.py @@ -0,0 +1,715 @@ +"""A DSP-runtime backend for the simulator: the skel's batch path, in-process. + +THIS IS NOT FASTRPC, AND THE MODULE IS NAMED TO NOT IMPLY IT IS ONE. See +`docs/superpowers/specs/2026-08-10-silicon-path-runtime-design.md` sections 0.1 +and 0.1.1: the simulator omits the qaic-generated stub entirely (it would define +the exact same symbol names as the skel's own implementation, so linking both +into one module is a duplicate-symbol error, not merely redundant), so +`simhost.c`'s calls to `hexlib_iface_open/_start/_mmap/_invoke/_stop/_close` +bind DIRECTLY to `skel.c`, as plain intra-module C function calls. No argument +marshalling happens anywhere on this path. An earlier draft of the design spec +claimed the opposite and was corrected in place; do not reintroduce the error +here by naming this class or module after the transport it does not exercise. + +WHAT A PASS THROUGH THIS BACKEND PROVES, AND WHAT IT DOES NOT (spec `0.1.1`): + +| exercised here | NOT exercised here | +|--------------------------------------------------------------|--------------------------| +| batch blob parsing and validation (`hexlib_dispatch_batch`) | qaic argument marshalling| +| the fd->address table and the pointer-free invariant (`skel_bufs.c`) | real ION allocation | +| the kernel dispatch table (`hexlib_kernel_table`) | `fastrpc_mmap` | +| kernel numerical correctness, through real argument unpacking | unsigned PD loading | +| PCYCLE measurement, read from the response header | the aarch64 host binary | +| VTCM acquisition, under the QuRT-hosted `.so` | | + +What IS proven is still most of hexlib's own risk: the batch format, the buffer +table, the dispatch table and the generated kernel adapters are all ours. +Marshalling is vendor code neither written nor fixable here, and is stage 3's +concern (a real device), not this one's. + +THE DISCRIMINATOR THAT MAKES A SIMULATOR PASS MEAN ANYTHING. Under the +simulator, host and DSP share ONE address space, and the SDK's own simulator +`HAP_mmap` is `return (void*)(uintptr_t)fd;` while `rpcmem_to_fd` is +`return (int)(uintptr_t)po;` -- so the whole pointer -> fd -> map -> base chain +is an IDENTITY FUNCTION here. A skel that leaned on the host's pointer instead +of resolving an fd through its own mmap table would compute the RIGHT ANSWER on +the simulator and fail instantly on silicon, and no comparison of VALUES could +ever catch that. Only a TABLE LOOKUP can: `hexlib_bufs_map` refuses any fd +`hexlib_bufs_register` never populated, whatever that fd numerically equals. +`run_unmapped` below drives exactly that path (`simhost.c`'s `--unmapped` mode, +which deliberately skips registering the buffer) and is the load-bearing test +in `hexlib/tests/test_dsp_sim.py`. + +ALIGNMENT. Kernels such as `scale_fp16` cast their buffer pointers straight to +`HVX_Vector *` and dereference them with an ALIGNED vector load/store -- see +`kernels/scale_fp16/kernel_api.h`'s own alignment note. The file-based runner +path (`hexlib/exec/hexagon.py`) gets this for free because its two buffers are +separate, independently 128-byte-aligned C arrays. Here both input and output +share ONE rpcmem buffer (there is exactly one fd), so every tensor's start +offset within it is rounded up to the next 128-byte boundary explicitly +(`_align_up`) -- getting this wrong would not be caught by any test that only +checks a status code, and would show up as silently wrong values or a fault. + +FAIL CLOSED, matching `hexlib/sim.py`'s own rule (its docstring explains why: a +device-farm job once ran zero tests and reported passing). A simulator run that +produces no parseable `SIMHOST hwinfo`/`SIMHOST invoke` line raises, and never +returns a default result -- there is no code path here that can manufacture a +`SimHostResult` without both having been read off the process's own stdout. +`HEXLIB_DSP_OK` is 1, never 0, so a response nothing wrote cannot read as +success either (`wire.unpack_response` rejects it directly). + +ONE NON-OBVIOUS BUILD DETAIL. Under the QuRT-hosted packaging, `sim_qurt_command` +launches a real QuRT kernel whose relative `fopen` READS resolve through +`--usefs`, but whose WRITES land in the launching process's OWN working +directory instead. `run_sim` below sets the subprocess `cwd` to the work +directory for exactly this reason (`_cwd`, matching +`hexlib/tests/test_runtime_sim_build.py`'s own helper) -- getting this wrong +does not fail loudly, it silently reads a stale `hexlib_out.bin` (or none) from +wherever the caller's own process happened to be running, which presents as a +wrong kernel rather than a wrong working directory. +""" +from __future__ import annotations + +import contextlib +import os +import re +import struct +from dataclasses import dataclass +from typing import Any, Mapping, Sequence + +import numpy as np + +from hexlib import toolchain as tc +from hexlib.exec.runner import ( + RawTensor, + RunnerSpec, + SPECS, + WIRE_DTYPE, + WIRE_RAW, + select, +) +from hexlib.runtime import build as rb +from hexlib.runtime import wire +from hexlib.runtime.genentry import KIND_ID + +BATCH_NAME = "hexlib_batch.bin" +IN_NAME = "hexlib_in.bin" +RSP_NAME = "hexlib_rsp.bin" +OUT_NAME = "hexlib_out.bin" + +# Matches simhost.c's own `#define MAX_BLOB (16 * 1024 * 1024)` -- the fixed +# size of the ONE rpcmem buffer it always allocates, regardless of what any +# particular batch actually needs. +MAX_BLOB = 16 * 1024 * 1024 + +# HVX vector width for the fp16 lane count kernels like scale_fp16 assume -- +# see the module docstring's "ALIGNMENT" paragraph. +HVX_ALIGN = 128 + +_HWINFO_RE = re.compile(r"SIMHOST hwinfo arch=(\d+) threads=(\d+) vtcm=(\d+)") +_INVOKE_RE = re.compile( + r"SIMHOST invoke rc=(-?\d+) rsp_len=(\d+) status=(\d+) n_ops=(\d+) cycles=(\d+)" +) + + +class DspSimError(Exception): + pass + + +@dataclass(frozen=True) +class RunnerStats: + """Deliberately NOT `hexlib.exec.hexagon.RunnerStats` (off limits to + modify, and shaped around file-based per-call bookkeeping this backend + does not do). `cycles` is read from the batch response header -- PCYCLE + around the kernel call only, never the simulator's whole-program count.""" + + calls: int = 0 + cycles: int = 0 + + +@dataclass(frozen=True) +class SimHostResult: + """One `hexagon-sim` launch's outcome, parsed from its `SIMHOST` lines. + + `status`/`cycles` come from the `SIMHOST invoke` line (the batch response + header: `hexlib_batch_rsp_hdr.status`/`.cycles_total`). `arch`/`vtcm` come + from the `SIMHOST hwinfo` line. `exit_code` is the simulator PROCESS's own + exit code (0 only when the batch's own status was `HEXLIB_DSP_OK` -- see + `simhost.c`'s `main`'s return statement), not a field taken off the wire. + """ + + status: int + cycles: int + arch: int + vtcm: int + stdout: str + exit_code: int | None + + +@contextlib.contextmanager +def _cwd(path: str): + """`tc.run` takes no `cwd` kwarg. See `sim_qurt_command`'s own docstring + and `hexlib/tests/test_runtime_sim_build.py`'s identically-named helper: + under the QuRT-hosted launch, `fopen(..., "wb")` writes land in the + LAUNCHING process's cwd, never in `--usefs`'s directory, so a caller that + does not pin its own cwd here would write `hexlib_rsp.bin`/ + `hexlib_out.bin` into wherever pytest happened to be invoked from -- + which, run from the repo root, previously and accidentally committed two + stray artifacts.""" + prev = os.getcwd() + os.chdir(path) + try: + yield + finally: + os.chdir(prev) + + +def run_sim(work_dir: str, extra_args: Sequence[str] = (), + sdk_root: str | None = None) -> SimHostResult: + """Launch the QuRT-hosted `.so` `build_sim_so` already wrote into + `work_dir` (by its fixed name, `hexlib_sim.so`) under `hexagon-sim`, and + parse its `SIMHOST` lines. + + FAIL CLOSED. Neither `hwinfo` nor `invoke` info is ever synthesised: a run + that does not print a parseable line for each raises `DspSimError`. This + mirrors `hexlib/sim.py`'s own rule for exactly the reason its docstring + gives -- a farm job once ran zero tests and reported passing, and a + zero-filled `SimHostResult` returned here on a parse miss would be that + same failure in a new shape. + """ + root = sdk_root or tc.default_sdk_root() + so_path = os.path.join(work_dir, "hexlib_sim.so") + cmd = rb.sim_qurt_command(work_dir, so_path, sdk_root=root, extra_args=tuple(extra_args)) + bin_dir = tc.find_toolchain_bin(root) + env = tc.toolchain_env(bin_dir) + + # MUST run with cwd == work_dir -- see the module docstring's "ONE + # NON-OBVIOUS BUILD DETAIL" and `_cwd`'s own docstring. + with _cwd(work_dir): + rc, out, err, timed_out = tc.run(cmd, env, timeout=tc.SIM_TIMEOUT_MAX_S) + combined = out + err + + if timed_out: + raise DspSimError(f"hexagon-sim timed out after {tc.SIM_TIMEOUT_MAX_S}s:\n{combined}") + + m_hw = _HWINFO_RE.search(combined) + if not m_hw: + raise DspSimError( + "no SIMHOST hwinfo line recovered -- start did not succeed, or " + "printed nothing parseable. Fail-closed: this never returns a " + f"default result.\n{combined}" + ) + m_inv = _INVOKE_RE.search(combined) + if not m_inv: + raise DspSimError( + "no SIMHOST invoke line recovered -- nothing was computed. " + f"Fail-closed: this never returns a default result.\n{combined}" + ) + + return SimHostResult( + status=int(m_inv.group(3)), + cycles=int(m_inv.group(5)), + arch=int(m_hw.group(1)), + vtcm=int(m_hw.group(3)), + stdout=combined, + exit_code=rc, + ) + + +def _align_up(n: int, align: int = HVX_ALIGN) -> int: + return (n + align - 1) // align * align + + +def _ne(shape: Sequence[int]) -> tuple[int, int, int, int]: + if len(shape) > 4: + raise DspSimError(f"shape {tuple(shape)} has more than 4 dims; hexlib_tensor.ne is fixed at 4") + padded = list(int(s) for s in shape) + [1] * (4 - len(shape)) + return tuple(padded) # type: ignore[return-value] + + +def _out_shape(spec: RunnerSpec, arrays: tuple[np.ndarray, ...], + attrs: Mapping[str, Any]) -> tuple[int, ...]: + """Mirrors `hexlib.exec.hexagon._out_shape` (that module is off limits to + modify or import a private helper from). Elementwise ops keep the first + input's shape; a permutation reorders it; an explicit `shape` attr wins. + + AND FOR EVERYTHING ELSE, THE OP REGISTRY'S OWN `infer` IS ASKED. The three + rules above are the whole of what this used to do, and they cover every + elementwise op, `transpose` and `reshape` -- but `patchify` turns + fp32 (3,2,256,256) into (256,1536) by a function of `patch`, `merge`, + `grid_h` and `grid_w`, which none of them can express. The fallthrough + returned the INPUT shape, so the output buffer was sized 3*2*256*256 instead + of 256*1536 and the batch was built with a tensor of the wrong length. + + `infer` is asked rather than a fourth rule being written here because it is + already the authority -- it is what the graph builder used to declare the + tensor this op writes into, so re-deriving it in the backend is exactly the + two-copies-of-one-rule problem that a shape mismatch makes silent. If it + raises, the raise is the answer: an op whose output shape the registry + refuses to compute is not one this backend should be guessing for. + """ + shape = tuple(arrays[0].shape) + perm = attrs.get("perm") + if perm is not None: + return tuple(shape[i] for i in perm) + declared = attrs.get("shape") + if declared is not None: + return tuple(declared) + + import hexlib.graph.opdefs # noqa: F401 -- registers the op definitions + from hexlib.graph.ir import Tensor + from hexlib.graph.ops import REGISTRY + + try: + opdef = REGISTRY.get(spec.kind) + except (KeyError, ValueError): + return shape + tensors = [ + # `infer` reads `.shape`, `.dtype` and `.name` only. The names are + # positional because this backend has no graph to take them from, and + # they appear only in the registry's own error messages. + Tensor(name=f"in{i}", dtype=dt, shape=tuple(a.shape)) + for i, (a, dt) in enumerate(zip(arrays, spec.inputs)) + ] + inferred = opdef.infer(tensors, dict(attrs)) + if not inferred: + return shape + first = inferred[0] + # `infer` returns ((shape, dtype), ...) per its OpDef docstring, but some + # defs return Tensor objects. Accept either rather than depending on which, + # since getting it wrong here is a wrong buffer size and not an exception. + if isinstance(first, Tensor): + return tuple(first.shape) + return tuple(first[0]) + + +def _encode_params(spec: RunnerSpec, arrays: tuple[np.ndarray, ...], + attrs: Mapping[str, Any]) -> tuple[int, ...]: + """Only the ATTR-sourced scalars go into `a->params`, bit for bit and in + order -- `numel:`/`dim:` scalars are derived on the DSP from the tensor's + own `ne[]` (see `genentry.py`'s `_scalar_expr`), never carried here. A + float attr is packed as its raw IEEE-754 bit pattern reinterpreted as + int32, because `hexlib_args.params` is `const void *` and the generated + entry casts it to `(const float *)` before indexing -- packing the value + as a plain `int` would send the wrong bits. + """ + params: list[int] = [] + for sc in spec.scalars: + if not sc.source.startswith("attr:"): + continue + value = sc.value(arrays, attrs) + if sc.ctype == "int": + params.append(int(value)) + else: + params.append(struct.unpack(" tuple[int, int, int]: + """(aligned end-of-input offset, output nbytes, total buffer size) for a + single-input/single-output op of `n` elements -- the shape every kind in + this backend's required tests actually drives (`scale`). Kept separate + from the general `run()` layout logic so `build_batch`/`run_unmapped`, which + build a batch WITHOUT any real array data, do not need one.""" + in_dtype = WIRE_DTYPE[spec.inputs[0]] + out_dtype = WIRE_DTYPE[spec.out_dtype] + nbytes_in = n * in_dtype.itemsize + in_end = _align_up(nbytes_in) + nbytes_out = n * out_dtype.itemsize + return in_end, nbytes_out, in_end + nbytes_out + + +class DspSimBackend: + """Drives a real kernel through the DSP skel's batch path, on the + simulator. See the module docstring for exactly what that does and does + not prove -- in short, our own code (batch parsing, the buffer table, the + dispatch table, the kernel adapter, PCYCLE) and NOT qaic marshalling, + which does not run on this path at all. + + Builds the skel archive, the QuRT-hosted `.so`, and the sim configs ONCE + per instance (construction is the slow part); each `run`/`run_raw`/ + `run_unmapped`/`hwinfo` call is one `hexagon-sim` launch reusing them. + """ + + def __init__(self, kernels: list[str], work_dir: str, sdk_root: str | None = None): + self.work_dir = work_dir + self.sdk_root = sdk_root or tc.default_sdk_root() + os.makedirs(work_dir, exist_ok=True) + rb.build_skel_lib(kernels, work_dir, sdk_root=self.sdk_root) + self.so_path = rb.build_sim_so(work_dir, sdk_root=self.sdk_root) + rb.write_qurt_sim_configs(work_dir, sdk_root=self.sdk_root) + + def _clear_outputs(self) -> None: + """Delete the previous call's artifacts before this one runs. + + Mirrors `hexlib/exec/hexagon.py`'s own removal of `hexlib_out.bin` and + its reason: A STALE OUTPUT WOULD BE READ AS THIS CALL'S RESULT if the + run failed to write one. `simhost.c` writes `hexlib_out.bin` only when + the batch status is OK and `hexlib_rsp.bin` only once the invoke + returned, so a launch that prints its `SIMHOST invoke ... status=1` + line and then dies (or a `hexagon-sim` that never gets that far at all) + leaves whatever the LAST call wrote sitting in the work directory, at + the same names and -- for a same-shaped op -- at the same offsets. + Deleting them here rather than merely not trusting them is deliberate: + it means any future code path that reads these files inherits the + guarantee instead of having to re-derive it. + + Every call shape goes through `_write_call`, so this runs for `run`, + `run_unmapped`, `run_raw` and `hwinfo` alike. + """ + for name in (OUT_NAME, RSP_NAME): + path = os.path.join(self.work_dir, name) + if os.path.exists(path): + os.remove(path) + + def _write_call(self, blob: bytes, payload: bytes) -> None: + self._clear_outputs() + with open(os.path.join(self.work_dir, BATCH_NAME), "wb") as f: + f.write(blob) + with open(os.path.join(self.work_dir, IN_NAME), "wb") as f: + f.write(payload) + + def _read_response(self) -> wire.BatchResponse: + rsp_path = os.path.join(self.work_dir, RSP_NAME) + if not os.path.isfile(rsp_path): + raise DspSimError(f"sim produced no {RSP_NAME}; nothing was computed") + with open(rsp_path, "rb") as f: + raw = f.read() + return wire.unpack_response(raw) + + # -- the acceptance path ------------------------------------------------- + + def run(self, kind: str, arrays: Sequence[np.ndarray], + attrs: Mapping[str, Any]) -> tuple[np.ndarray, RunnerStats]: + """Invoke `kind` on real array data, through the skel's batch path. + + Builds one buffer holding every input then the output, each tensor's + start 128-byte aligned (see the module docstring's "ALIGNMENT"), a + single op naming them by index, and reads the result back out of + `hexlib_out.bin` at the output tensor's own offset. + + WHAT IS REFUSED HERE, AND WHY IT CANNOT BE REFUSED ANYWHERE ELSE. See + `spec.check_requires` below: an op kind is not always one kernel, and + the DSP has no field to check the difference against. Everything this + method refuses has the same failure mode if it is not refused -- a + correctly-shaped, OK-status wrong answer -- which is why each check is + an error and never a fallback. + """ + # THE VARIANT IS CHOSEN HERE, FROM THE ATTRS, because the wire cannot + # choose it later. `transpose` is two kernels -- perm(1,0,2) moves whole + # aligned vectors, perm(0,2,1) shares no contiguous run and needed a + # different implementation -- and the DSP is handed an id and a buffer + # list with no perm to branch on. `select` refuses an op no variant + # accepts, and refuses ambiguity rather than taking the first match. + spec_name, spec = select(kind, attrs) + + # AN OP KIND IS NOT ALWAYS ONE KERNEL, and the check has to be here. + # `hexlib/exec/hexagon.py` has always done this, for the reason its own + # comment gives: the failure mode otherwise is a correctly-shaped wrong + # answer. This path skipped it, and the DSP cannot make up the + # difference -- `hexlib_args` carries no field for a perm at all (see + # genentry.py's `_requires_check`), and for the one key it DOES carry, + # `dtype`, the generated check compares a value THIS serializer derived + # from `spec.out_dtype` against the same spec's own requirement, so it + # passes by construction whatever the caller asked for. Only a check + # against the CALLER'S OWN attrs can fail, and this is it. + spec.check_requires(attrs) + + # `zip` stops at the shorter sequence, so an op mis-wired with an extra + # input array silently dropped it: `add` with three inputs computed + # a+b, ignored c, and returned the right shape with no error. + # `RunnerSpec.payload` raises on the other transport for exactly this. + if len(arrays) != len(spec.inputs): + raise DspSimError( + f"{kind} takes {len(spec.inputs)} inputs, got {len(arrays)}; " + "an extra array would be dropped and a missing one would leave " + "the op naming a tensor that was never packed" + ) + + # A RAW input passes through untouched. `np.ascontiguousarray(rawtensor)` + # would build a 0-d object array and every downstream `.nbytes`/`.shape` + # would then describe a Python pointer rather than the weight. RawTensor + # carries `.shape`, `.nbytes` and `.size`, so everything below that reads + # those three works on either kind without knowing which it has. + arrays = tuple( + a if dt in WIRE_RAW + else np.ascontiguousarray(a, dtype=WIRE_DTYPE[dt]) + for a, dt in zip(arrays, spec.inputs) + ) + for i, (a, dt) in enumerate(zip(arrays, spec.inputs)): + if dt in WIRE_RAW and not isinstance(a, RawTensor): + raise DspSimError( + f"{kind} input {i} is declared {dt!r}, which is staged as " + f"raw quantized bytes, so it must be a RawTensor and not a " + f"{type(a).__name__} -- see hexlib/exec/runner.py's RawTensor" + ) + out_shape = _out_shape(spec, arrays, attrs) + out_dtype = WIRE_DTYPE[spec.out_dtype] + out_nbytes = int(np.prod(out_shape)) * out_dtype.itemsize if out_shape else out_dtype.itemsize + + # FROM THE SPEC, NOT THE CONSTANT "row_major". Both tensor loops below + # used to hard-code it, which meant the DSP's layout guard (added with + # `genentry._layout_check`) could never see anything but row_major from + # this serializer -- the same self-comparison the output dtype has, and + # for the same structural reason. A spec declaring a `q4_0_repacked` + # weight now actually says so on the wire. + buf_layouts = spec.buf_layouts() + + payload = bytearray() + tensors = [] + offset = 0 + for i, (a, dt) in enumerate(zip(arrays, spec.inputs)): + # `nbytes` is the BYTE count and `ne` is the ELEMENT shape, and for a + # q4_0 weight those are not related by an item size: (768, 768) + # elements occupy 331776 bytes of 18-byte blocks. `wire.py` checks + # only that the buffer holds `offset + nbytes`, never that + # `nbytes == prod(ne) * itemsize`, so both fields stay true and a + # kernel reading `a->ne` gets the shape it needs to index blocks with. + nbytes = a.nbytes + tensors.append(wire.TensorDesc( + bi=0, offset=offset, nbytes=nbytes, dtype=dt, + layout=buf_layouts[i], ne=_ne(tuple(a.shape)), + )) + payload += a.data if dt in WIRE_RAW else a.tobytes() + offset += nbytes + aligned = _align_up(offset) + if aligned != offset: + payload += b"\x00" * (aligned - offset) + offset = aligned + + out_offset = offset + # The output tensor's dtype describes the BYTES this buffer will hold, + # so it comes from the spec, not from any attr -- the buffer was sized + # from the same place two lines up. That is precisely why a + # `("dtype", ...)` requirement cannot be validated on the DSP through + # this serializer (it would be comparing the spec with itself), and why + # `check_requires` above is the check that actually decides it. + tensors.append(wire.TensorDesc( + bi=0, offset=out_offset, nbytes=out_nbytes, dtype=spec.out_dtype, + layout=buf_layouts[-1], ne=_ne(out_shape), + )) + payload += b"\x00" * out_nbytes + + bufs = [wire.BufDesc(fd=0, size=len(payload))] + src = tuple(range(len(arrays))) + dst = (len(arrays),) + params = _encode_params(spec, arrays, attrs) + # THE VARIANT'S id, not the op kind's. For a kind with one kernel these + # are the same string; for `transpose` they are not, and sending + # KIND_ID["transpose"] for a perm(0,2,1) op would dispatch it to the + # perm(1,0,2) kernel -- right shape, right status, wrong answer. This is + # the one place the host's variant decision becomes a number on the wire. + kind_id = KIND_ID[spec_name] + ops = [wire.OpDesc(kind=kind_id, params=params, src=src, dst=dst)] + blob = wire.pack_batch(bufs, tensors, ops) + + self._write_call(blob, bytes(payload)) + res = run_sim(self.work_dir, sdk_root=self.sdk_root) + if res.status != wire.STATUS["OK"]: + name = wire.STATUS_NAME.get(res.status, res.status) + raise DspSimError(f"{kind}: DSP invoke returned status {name} ({res.status})") + # `simhost.c` returns 0 if and only if the batch status was + # HEXLIB_DSP_OK, so an OK status line together with a nonzero exit means + # the process died AFTER printing it -- before, for instance, writing + # the output file. This was captured in the result and never read. + if res.exit_code not in (0, None): + raise DspSimError( + f"{kind}: the batch reported OK but the simulator process " + f"exited {res.exit_code}, so it did not finish. Anything it had " + f"written by then is a partial result, not an answer.\n" + f"{res.stdout}" + ) + + # WHAT CAME BACK MUST ANSWER WHAT WAS ASKED. `skel_dispatch.c` fills + # `results[i].kind` from the op's own kind BEFORE the table lookup, so + # this compares the id that reached the DSP with the id packed here. It + # is the only host-side check that can see the host and the DSP + # disagreeing about kind ids -- a drift that otherwise dispatches an op + # to another kernel with a matching buffer count and reports OK. The + # per-op status is a separate field from the batch header's status + # (`wire.BatchResponse.ok` requires both) and was never read either. + rsp = self._read_response() + if rsp.n_ops != 1 or len(rsp.results) != 1: + raise DspSimError( + f"{kind}: the batch carried 1 op but the response reports " + f"n_ops={rsp.n_ops} with {len(rsp.results)} results" + ) + result = rsp.results[0] + if result.kind != kind_id: + raise DspSimError( + f"{kind}: asked for kind {kind_id} but the response answers for " + f"kind {result.kind}. The host and the DSP disagree about kind " + f"ids, so this op was served by another kernel." + ) + if not result.ok: + name = wire.STATUS_NAME.get(result.status, result.status) + raise DspSimError( + f"{kind}: the batch status was OK but the op itself returned " + f"{name} ({result.status})" + ) + + out_path = os.path.join(self.work_dir, OUT_NAME) + if not os.path.isfile(out_path): + raise DspSimError( + f"{kind}: sim reported OK but wrote no {OUT_NAME}; nothing was computed" + ) + with open(out_path, "rb") as f: + raw = f.read() + y_raw = raw[out_offset: out_offset + out_nbytes] + y = np.frombuffer(y_raw, dtype=out_dtype).reshape(out_shape) + + return y, RunnerStats(calls=1, cycles=res.cycles) + + # -- the discriminator ---------------------------------------------------- + + def run_unmapped(self, kind: str, n: int, factor: float) -> wire.BatchResponse: + """Drive `simhost.c`'s `--unmapped` mode: the payload buffer's fd is + patched into the batch as usual, but `hexlib_iface_mmap` is + deliberately never called, so the skel must refuse with + `HEXLIB_DSP_ERR_UNMAPPED` rather than resolve an address it was never + given. See the module docstring's own section on why this is the one + test that makes a simulator pass mean anything on silicon. + """ + spec = SPECS[kind] + _, _, total = _scale_layout(spec, n) + blob = self.build_batch(kind, n, factor) + self._write_call(blob, b"\x00" * total) + run_sim(self.work_dir, extra_args=("--unmapped",), sdk_root=self.sdk_root) + return self._read_response() + + # -- DSP-side validation, bypassing the host-side serializer --------------- + + def run_raw(self, blob: bytes) -> wire.BatchResponse: + """Send `blob` verbatim as `hexlib_batch.bin`, bypassing + `wire.pack_batch`'s own host-side checks entirely -- so a bad magic, a + truncated blob or an unknown op kind exercises the DSP's OWN + validation in `hexlib_dispatch_batch`, not the Python serializer's.""" + return self.run_raw_verbose(blob)[0] + + def run_raw_verbose(self, blob: bytes) -> tuple[wire.BatchResponse, str]: + """`run_raw`, plus the simulator's own stdout. + + THE RESPONSE ALONE CANNOT DISTINGUISH SOME MALFORMED BLOBS from the + host mishandling them, which is why this exists. simhost.c patches the + real fd into the buffer table before invoking, and refuses to patch a + table that does not lie inside the blob (see its comment). With + `off_bufs = 0xFFFFFF00` and a 32-bit `size_t`, the unbounded loop + computed `g_batch - 256`, corrupted whatever static preceded it, and + invoked anyway -- and the skel returned the SAME ERR_TRUNCATED it + returns when the table is correctly left alone. A test asserting only + on `status` was green either way. + + The host's own `SIMHOST note=...` line is the only place that + difference is observable, so a caller that needs to tell the two apart + reads it here rather than inferring it from a status that does not + carry it. + """ + self._write_call(blob, b"") + sim = run_sim(self.work_dir, sdk_root=self.sdk_root) + return self._read_response(), sim.stdout + + def build_batch(self, kind: str, n: int, factor: float, + kind_override: int | None = None) -> bytes: + """A scale-shaped batch: one input tensor of `n` elements, one + same-length output tensor, both slices of a single buffer (`fd=0`, + the placeholder `simhost.c` patches to the real fd). `kind_override` + lets a caller build an otherwise-well-formed batch naming an + unregistered op kind, to drive `HEXLIB_DSP_ERR_NO_KERNEL` without + touching `wire.pack_batch`'s own validation (which does not check + kind against the dispatch table at all -- that check is the DSP's). + """ + spec = SPECS[kind] + in_dtype = WIRE_DTYPE[spec.inputs[0]] + in_end, nbytes_out, total = _scale_layout(spec, n) + nbytes_in = n * in_dtype.itemsize + + bufs = [wire.BufDesc(fd=0, size=total)] + tensors = [ + wire.TensorDesc(bi=0, offset=0, nbytes=nbytes_in, dtype=spec.inputs[0], + layout="row_major", ne=(n, 1, 1, 1)), + wire.TensorDesc(bi=0, offset=in_end, nbytes=nbytes_out, dtype=spec.out_dtype, + layout="row_major", ne=(n, 1, 1, 1)), + ] + factor_bits = struct.unpack(" SimHostResult: + """An empty-but-well-formed batch (0 bufs/tensors/ops) is enough to + drive open -> start -> hwinfo -> invoke -> stop -> close, with no + rpcmem/fd plumbing needed -- the same shape + `test_sim_run_reaches_start_and_reports_real_vtcm` in + `test_runtime_sim_build.py` already proved reaches a successful + `start()`.""" + self._write_call(wire.pack_batch([], [], []), b"") + return run_sim(self.work_dir, sdk_root=self.sdk_root) + + +# --------------------------------------------------------------------------- +# Wiring the DSP into the plan executor +# --------------------------------------------------------------------------- + + +def interpreter_backends(sim: "DspSimBackend", kinds=None) -> dict: + """One `hexlib.exec.interpreter` Backend per op kind that has a kernel. + + This is what makes a WHOLE-ENCODER run on the DSP possible rather than a + per-op demonstration. `interpreter.run(model, feeds, backends=...)` takes a + mapping from op kind to a callable and falls back to the op registry's numpy + reference for anything absent — so the encoder runs end to end from the first + kernel onwards, and each new kernel replaces exactly one entry. A kind with + no entry is a reference computation, never a silently skipped step. + + ONE SIMULATOR LAUNCH PER OP. `hexagon-sim` is started, run and torn down for + every call, so a 259-step encoder is 259 launches. That is minutes at a tiny + config and hours at 256x256; it is a correctness path, not a benchmark, and + the plan-walking driver that would fix it (one ELF, one invoke, the whole + batch) is separate work. + + THE VARIANT IS RESOLVED PER CALL, from the op's own attrs, which is why this + cannot be a dict of pre-bound kernels: `transpose` is two kernels and which + one an op needs is in its `perm`. `select` refuses an op no variant accepts, + so a perm nothing implements is an error here rather than a wrong answer. + + RAW INPUTS ARE QUANTIZED ON THE WAY IN. The interpreter works in + `COMPUTE_DTYPE` and the op registry maps q4_0 to float32 (see + `graph/eager.py`), so a `matmul_epilogue` weight arrives here as a dense fp32 + array while the spec declares `q4_0`. It is quantized with + `hexlib.exec.quant` and handed over as a `RawTensor`. + + WHAT THAT MEANS FOR ANY ACCURACY CLAIM, and it is not a detail: the DSP then + computes with a 4-BIT weight while the reference has the fp32 one, and 4-bit + quantization error is ~1/16 of each block's range — orders of magnitude + larger than any kernel bug worth hunting. A comparison against the fp32 + reference measures the FORMAT, not the kernel. To measure the kernel, build + the reference from `quant.dequantize_q4_0` of the same bytes; then the only + remaining difference is arithmetic. `quant.dequantize_q4_0` exists for that. + """ + from hexlib.exec.runner import SPECS, WIRE_RAW, RawTensor, select + + available = sorted({spec.kind for spec in SPECS.values()}) + wanted = available if kinds is None else [k for k in kinds if k in available] + + def make(kind: str): + def backend(arrays, attrs): + from hexlib.exec.quant import quantize_q4_0 + + _, spec = select(kind, dict(attrs)) + prepared = [] + for a, dt in zip(arrays, spec.inputs): + if dt in WIRE_RAW: + arr = np.asarray(a) + prepared.append( + RawTensor(dt, tuple(arr.shape), quantize_q4_0(arr)) + ) + else: + prepared.append(a) + out, _ = sim.run(kind, prepared, dict(attrs)) + return (out,) + + return backend + + return {kind: make(kind) for kind in wanted} diff --git a/hexlib/exec/quant.py b/hexlib/exec/quant.py new file mode 100644 index 0000000..c293490 --- /dev/null +++ b/hexlib/exec/quant.py @@ -0,0 +1,220 @@ +# hexlib/exec/quant.py +"""Reference q4_0 quantization, in numpy, matching llama.cpp block for block. + +WHY THIS EXISTS AND WHY IT IS "REFERENCE". 75 of the encoder's 259 real-work plan +steps are `matmul_epilogue` with a q4_0 weight, and hexlib has no checkpoint +loader: nothing in the tree can produce a q4_0 buffer. `RunnerSpec`'s own error +message says "hexlib does not quantize here", and that remains true of the +serializer — a `RawTensor` is bytes somebody else already quantized. This module +is that somebody, for tests and for the end-to-end encoder run. It is not on any +hot path and makes no attempt to be fast. + +THE FORMAT, from `ggml-common.h:194-199` and `quantize_row_q4_0_ref` +(`ggml-quants.c:113-146`), MIT — see ATTRIBUTION.md. Read in place from +`../llama.cpp`; nothing is imported from or built against it. + + #define QK4_0 32 + typedef struct { + ggml_half d; // fp16 scale + uint8_t qs[QK4_0 / 2]; // 16 bytes, two 4-bit quants each + } block_q4_0; // 18 bytes total + + d = max / -8 where `max` is the SIGNED element of largest magnitude + id = d ? 1/d : 0 + for j in 0..15: + xi0 = min(15, (int8_t)(x[j] * id + 8.5f)) + xi1 = min(15, (int8_t)(x[j + 16] * id + 8.5f)) + qs[j] = xi0 | (xi1 << 4) + +THREE THINGS THAT ARE EASY TO GET WRONG, each producing a plausible wrong answer +rather than an error: + +1. THE NIBBLE PAIRING IS j WITH j+16, NOT j WITH j+1. The LOW nibble of byte j + holds element j and the HIGH nibble holds element j + 16 — the two halves of + the block, not adjacent elements. Pairing adjacently round-trips to a + permutation of the right values, which has the right norm and the right + histogram and is wrong everywhere. + +2. `d` IS SIGNED, and dividing by -8 rather than by 8 is deliberate. `max` keeps + the sign of the largest-magnitude element, so for a block whose extreme value + is positive `d` is negative. Using `amax / 8` instead flips the sign of every + dequantized value in that block. + +3. THE CAST TRUNCATES TOWARD ZERO and the clamp is one-sided. `(int8_t)(x + 8.5f)` + is C truncation, not rounding, and only the upper end is clamped (to 15). The + `+ 8.5` is what makes truncation behave as round-half-up over the expected + range. `np.trunc` is used here rather than `np.round` for exactly this reason; + `np.round` is banker's rounding and disagrees on every exact .5. +""" +from __future__ import annotations + +import numpy as np + +from hexlib.graph import ir + +# Named from the same place `hexlib.graph.ir` takes them, rather than respelled: +# ir is the authority on the block size and the byte count, and a second copy of +# either is a wrong answer and not a crash. +QK4_0 = ir.Q4_0_BLOCK +BLOCK_BYTES = ir.Q4_0_BLOCK_BYTES + + +def quantize_q4_0(x: np.ndarray) -> bytes: + """`x` (any shape, last axis a multiple of 32) -> packed q4_0 blocks. + + Blocks run along the LAST axis, in C order, which is what makes the byte + stream match `ir.nbytes(x.shape, "q4_0")` and what a row-major q4_0 weight + means: row 0's blocks, then row 1's. + """ + a = np.ascontiguousarray(x, dtype=np.float32) + if a.ndim == 0 or a.shape[-1] % QK4_0 != 0: + raise ValueError( + f"q4_0 needs a last axis that is a multiple of {QK4_0}; got shape " + f"{tuple(a.shape)}" + ) + blocks = a.reshape(-1, QK4_0) + n = blocks.shape[0] + + # `max` is the SIGNED element of largest magnitude, so argmax over |v| and + # then take the value at that index -- not amax, whose sign is always +. + idx = np.abs(blocks).argmax(axis=1) + mx = blocks[np.arange(n), idx].astype(np.float32) + + d = (mx / np.float32(-8.0)).astype(np.float32) + # THE SCALE IS ROUND-TRIPPED THROUGH fp16 BEFORE IT IS USED. It is stored as + # fp16 in the block, so the dequantizer will see the narrowed value; scaling + # by the fp32 original here would quantize against a scale that does not + # exist on the wire and make this function's own round trip look better than + # the kernel's can be. + d16 = d.astype(np.float16) + d_used = d16.astype(np.float32) + # `np.divide(where=...)` and not `np.where(cond, 1/d, 0)`: the latter + # evaluates BOTH branches, so an all-zero block (d == 0, which llama.cpp + # guards with `id = d ? 1/d : 0`) raises a divide-by-zero warning and puts an + # inf in the array before the select discards it. Correct either way here, + # but a warning that is always emitted is a warning nobody reads. + inv = np.zeros_like(d_used) + np.divide(np.float32(1.0), d_used, out=inv, where=(d_used != 0.0)) + + scaled = blocks * inv[:, None] + # Truncation toward zero, one-sided clamp at 15 -- see the module docstring. + q = np.trunc(scaled + np.float32(8.5)).astype(np.int32) + q = np.clip(q, 0, 15).astype(np.uint8) + + lo = q[:, : QK4_0 // 2] # elements 0..15 -> low nibbles + hi = q[:, QK4_0 // 2 :] # elements 16..31 -> high nibbles + qs = (lo | (hi << 4)).astype(np.uint8) + + out = np.empty((n, BLOCK_BYTES), dtype=np.uint8) + out[:, :2] = d16.view(np.uint8).reshape(n, 2) + out[:, 2:] = qs + return out.tobytes() + + +def dequantize_q4_0(data: bytes, shape: tuple[int, ...]) -> np.ndarray: + """The inverse, to fp32: `v[j] = (nibble - 8) * d`. + + Provided so a test can check the round trip and so a reference matmul can be + computed from the SAME bytes the kernel is given -- which is the only way to + separate "the kernel's arithmetic is wrong" from "the kernel decoded the + blocks differently". + """ + want = ir.nbytes(tuple(shape), "q4_0") + if len(data) != want: + raise ValueError( + f"a q4_0 tensor of shape {tuple(shape)} is {want} bytes; got {len(data)}" + ) + raw = np.frombuffer(data, dtype=np.uint8).reshape(-1, BLOCK_BYTES) + d = raw[:, :2].copy().view(np.float16).reshape(-1).astype(np.float32) + qs = raw[:, 2:] + + lo = (qs & 0x0F).astype(np.int32) + hi = (qs >> 4).astype(np.int32) + q = np.concatenate([lo, hi], axis=1) # back to element order 0..31 + v = (q - 8).astype(np.float32) * d[:, None] + return v.reshape(shape) + + +# --------------------------------------------------------------------------- +# q8_0. TRANSCRIBED FROM `quantize_row_q8_0_ref` (ggml-quants.c:276-299) AND +# `dequantize_row_q8_0` (:553-567), NOT ADAPTED FROM q4_0 ABOVE. +# +# #define QK8_0 32 +# typedef struct { +# ggml_half d; // delta +# int8_t qs[QK8_0]; // quants +# } block_q8_0; // 34 bytes +# +# amax = max|x[j]| +# d = amax / 127 +# id = d ? 1/d : 0 +# qs[j] = roundf(x[j] * id) +# +# THREE THINGS DIFFER FROM q4_0 AND EVERY ONE OF THEM PRODUCES A PLAUSIBLE WRONG +# ANSWER IF CARRIED OVER: +# +# 1. `d` IS POSITIVE. q4_0 uses the SIGNED largest-magnitude element over -8, +# so a block whose extreme is positive stores a negative scale. q8_0 uses +# `amax`, which has no sign, over +127. Reusing q4_0's sign trick here +# negates every value in the block -- the same failure q4_0's own +# `test_blocks_whose_extreme_value_is_POSITIVE_get_a_negative_scale` +# exists to catch, in the opposite direction. +# 2. THE ROUNDING IS `roundf`, which is round-half-AWAY-FROM-ZERO. It is not +# q4_0's `trunc(x + 8.5)` (that trick exists only because q4_0 stores an +# unsigned nibble biased by 8) and it is NOT `np.round`, which is banker's +# rounding and disagrees on every exact .5. np.floor(|x| + 0.5) with the +# sign reapplied is what matches. +# 3. THERE IS NO BIAS AND NO NIBBLE PACKING. `qs` is a plain int8 per element, +# in element order. No `- 8`, no low/high nibble split, so none of q4_0's +# j-with-j+16 pairing applies. +QK8_0 = ir.Q8_0_BLOCK +Q8_0_BLOCK_BYTES = ir.Q8_0_BLOCK_BYTES + + +def quantize_q8_0(x: np.ndarray) -> bytes: + """`x` (any shape, last axis a multiple of 32) -> packed q8_0 blocks.""" + a = np.ascontiguousarray(x, dtype=np.float32) + if a.ndim == 0 or a.shape[-1] % QK8_0 != 0: + raise ValueError( + f"q8_0 needs a last axis that is a multiple of {QK8_0}; got shape " + f"{tuple(a.shape)}" + ) + blocks = a.reshape(-1, QK8_0) + n = blocks.shape[0] + + # UNSIGNED, unlike q4_0's signed `mx`. See note 1 above. + amax = np.abs(blocks).max(axis=1).astype(np.float32) + d = (amax / np.float32(127.0)).astype(np.float32) + # Narrowed before use, for the same reason q4_0 narrows: `d` is stored as + # fp16, so quantizing against the fp32 value would make this function's own + # round trip look better than the kernel's can be. + d16 = d.astype(np.float16) + d_used = d16.astype(np.float32) + inv = np.zeros_like(d_used) + np.divide(np.float32(1.0), d_used, out=inv, where=(d_used != 0.0)) + + scaled = blocks * inv[:, None] + # `roundf`, not np.round. See note 2 above. + q = np.sign(scaled) * np.floor(np.abs(scaled) + np.float32(0.5)) + # int8 saturation: 127/-128. amax/127 makes |scaled| <= 127 before fp16 + # narrowing, but narrowing `d` DOWNWARD makes it slightly larger, so the + # extreme element can land on 128 and wrap to -128 without this. + q = np.clip(q, -128, 127).astype(np.int8) + + out = np.empty((n, Q8_0_BLOCK_BYTES), dtype=np.uint8) + out[:, :2] = d16.view(np.uint8).reshape(n, 2) + out[:, 2:] = q.view(np.uint8) + return out.tobytes() + + +def dequantize_q8_0(data: bytes, shape: tuple[int, ...]) -> np.ndarray: + """The inverse, to fp32: `v[j] = qs[j] * d`. No bias term, unlike q4_0.""" + want = ir.nbytes(tuple(shape), "q8_0") + if len(data) != want: + raise ValueError( + f"a q8_0 tensor of shape {tuple(shape)} is {want} bytes; got {len(data)}" + ) + raw = np.frombuffer(data, dtype=np.uint8).reshape(-1, Q8_0_BLOCK_BYTES) + d = raw[:, :2].copy().view(np.float16).reshape(-1).astype(np.float32) + q = raw[:, 2:].copy().view(np.int8).astype(np.float32) + return (q * d[:, None]).reshape(shape) diff --git a/hexlib/exec/runner.py b/hexlib/exec/runner.py index 71d020f..e22ce44 100644 --- a/hexlib/exec/runner.py +++ b/hexlib/exec/runner.py @@ -40,22 +40,151 @@ "int32": np.dtype("ne` would inherit +# it. +WIRE_RAW: frozenset[str] = frozenset({"q4_0", "q8_0"}) + _STRUCT_CODE = {"int": "i", "float": "f"} +# THE LAYOUT NAMES ARE IMPORTED, NOT RESPELLED, unlike WIRE_DTYPE above. That +# table is deliberately independent because it maps names to numpy dtypes and +# only agrees with wire.py's by coincidence of naming; this one is the same set +# of names for the same field, and a second copy of it is precisely the drift +# that left main.c emitting layout 0 while pack_batch could emit something else. +# wire.py imports nothing but the stdlib, so this cannot cycle. +from hexlib.runtime.wire import LAYOUT_ID as WIRE_LAYOUT # noqa: E402 + + +@dataclass(frozen=True) +class RawTensor: + """An already-quantized input: opaque bytes plus its LOGICAL element shape. + + A separate type rather than a numpy array, for two reasons. `np.ndarray` does + not accept arbitrary attributes, so the logical shape cannot simply be + attached to a uint8 array -- and more importantly, a bare uint8 array would + make the dangerous mistake silent. Handing `payload` an fp32 weight array and + staging its bytes as though they were q4_0 blocks yields either a buffer of + the wrong size or, when the sizes happen to line up, a correctly-shaped + answer computed from noise; numpy never objects, because `.tobytes()` on any + array is always willing. Requiring a distinct type at the call site means the + caller has to have known what they were passing. + + `shape` is the ELEMENT shape -- (768, 768) for a q4_0 weight, not the 331776 + bytes it occupies. That is what goes in `ne` on the wire; `nbytes` carries the + byte count separately, and `wire.py` never conflates them. + """ + + dtype: str + shape: tuple[int, ...] + data: bytes + + def __post_init__(self) -> None: + from hexlib.graph import ir + + if self.dtype not in WIRE_RAW: + raise ValueError( + f"RawTensor is for block-quantized storage only; {self.dtype!r} " + f"is not in WIRE_RAW ({sorted(WIRE_RAW)})" + ) + # ir.nbytes is the authority on q4_0's 18-bytes-per-32-elements block and + # already refuses a last dimension that is not a multiple of 32. Asking it + # here rather than recomputing means the two cannot disagree, and it is + # what makes a truncated or mis-shaped weight a construction-time error + # instead of a wrong answer. + want = ir.nbytes(tuple(self.shape), self.dtype) + if len(self.data) != want: + raise ValueError( + f"a {self.dtype} tensor of logical shape {tuple(self.shape)} is " + f"{want} bytes of blocks, but {len(self.data)} were given" + ) + + @property + def nbytes(self) -> int: + return len(self.data) + + @property + def size(self) -> int: + """Element count, so `numel:` scalars read the LOGICAL count.""" + n = 1 + for d in self.shape: + n *= int(d) + return n + + +def raw_bytes(kind: str, idx: int, dtype: str, value) -> bytes: + """The already-quantized bytes of one raw input, verbatim.""" + if not isinstance(value, RawTensor): + raise ValueError( + f"{kind}: input {idx} is declared {dtype!r}, which is staged as raw " + f"quantized bytes, so it must be a RawTensor and not a " + f"{type(value).__name__}. hexlib does not quantize here -- pass the " + f"already-quantized bytes with their logical shape." + ) + if value.dtype != dtype: + raise ValueError( + f"{kind}: input {idx} is declared {dtype!r} but the RawTensor says " + f"{value.dtype!r}" + ) + return value.data + @dataclass(frozen=True) class Scalar: """One value in the header. - `source` is either 'attr:' (read from the op's attrs), 'numel:' - (the element count of input i), or 'dim::' (one dimension of input - i). Those three cover every kernel in the encoder without letting a spec - smuggle in arbitrary host-side computation, which would put logic somewhere - no kernel test looks. + `source` is one of: + 'attr:' read from the op's attrs + 'numel:' the element count of input i + 'dim::' one dimension of input i + 'rows::' input i's element count divided by that axis + + These four cover every kernel in the encoder without letting a spec smuggle + in arbitrary host-side computation, which would put logic somewhere no kernel + test looks. + + `rows:` EXISTS FOR LAST-AXIS REDUCTIONS OVER A RANK-3 TENSOR. `softmax` is + called on fp16 (12, 256, 256) with axis -1, and its kernel takes (R, C) -- + 3072 rows of 256. R is the PRODUCT of two axes, which no single `dim:` can + give, and `numel:` alone cannot either. The axis is named rather than assumed + to be the last one, because `ne` is padded to four with ones and "the last + axis" of a rank-3 tensor is then ambiguous between index 2 and index 3. + + `codes` TURNS A NON-NUMERIC ATTR INTO AN int PARAM, which `matmul_epilogue` + needs: fusion folds `gelu_tanh` and `gelu_erf` into its `act` attr (see + `graph/fuse.py`'s `FUSABLE_ACTS`), so the attr is a STRING and the wire + carries only ints and floats. `int("gelu_tanh")` raises, and the alternative + -- three separate kernels selected by `requires` -- would triple a 75-op + kernel to spare one switch. + + It is a tuple of pairs rather than a dict because `Scalar` is frozen and + hashable. THE MAPPING BELONGS BESIDE THE KERNEL THAT DECODES IT: these + numbers are a contract with one kernel's `kernel_api.h`, not a project-wide + enum, and an unmapped value is refused rather than defaulted -- silently + sending 0 for an unknown activation means running the wrong epilogue and + getting a correctly-shaped wrong answer. """ source: str ctype: str = "int" + codes: tuple[tuple[str, int], ...] = () def value(self, arrays: tuple[np.ndarray, ...], attrs: Mapping[str, Any]) -> Any: kind, _, rest = self.source.partition(":") @@ -64,15 +193,36 @@ def value(self, arrays: tuple[np.ndarray, ...], attrs: Mapping[str, Any]) -> Any raise KeyError( f"runner scalar wants attr {rest!r}; op attrs are {sorted(attrs)}" ) - return attrs[rest] + value = attrs[rest] + if self.codes: + table = dict(self.codes) + if value not in table: + raise ValueError( + f"runner scalar {self.source!r}: {value!r} is not one of " + f"{sorted(table)}. Refused rather than defaulted -- " + f"sending a code this kernel does not implement runs the " + f"wrong branch and returns a correctly-shaped wrong answer." + ) + return table[value] + return value if kind == "numel": return int(arrays[int(rest)].size) if kind == "dim": idx, _, axis = rest.partition(":") return int(arrays[int(idx)].shape[int(axis)]) + if kind == "rows": + idx, _, axis = rest.partition(":") + a = arrays[int(idx)] + extent = int(a.shape[int(axis)]) + if extent <= 0: + raise ValueError( + f"runner scalar {self.source!r}: axis {axis} of input {idx} " + f"has extent {extent}, so rows cannot be computed" + ) + return int(a.size) // extent raise ValueError( f"unknown runner scalar source {self.source!r}; expected attr:, " - "numel: or dim:" + "numel:, dim: or rows:" ) @@ -90,12 +240,43 @@ class RunnerSpec: # passes, so nothing but the values would catch it. Checked before the # kernel is invoked, and a mismatch is an error rather than a fallback. requires: tuple[tuple[str, Any], ...] = () + # The wire layout each buffer must be declared as, inputs then output, or () + # for "row_major throughout" -- which every kernel shipped so far is. Named + # per buffer rather than per kernel because the matmul this leads to takes a + # `q4_0_repacked` weight beside a row-major activation, and `LAYOUT_ID` + # already carries that value. See `genentry._layout_check` for what enforces + # it on the DSP and why an unchecked enum is a comment, not a mechanism. + layouts: tuple[str, ...] = () # Output shape comes from the graph, not from the kernel: the op's `infer` # already declared it and the executor checks it. A kernel that returned a # different length fails the byte-count check in the backend. out_shape_from: str = "declared" notes: str = "" + def buf_layouts(self) -> tuple[str, ...]: + """The layout of every buffer, inputs then output, always fully spelled. + + `layouts=()` means row_major throughout, which is what every kernel + shipped so far is -- so the default keeps the declaration short without + making "unspecified" a third possibility anything downstream has to + handle. Callers get one buffer per buffer, in the order + `skel_dispatch.c` walks src then dst. + """ + if not self.layouts: + return ("row_major",) * (len(self.inputs) + 1) + return self.layouts + + def accepts(self, attrs: Mapping[str, Any]) -> bool: + """True if this variant's `requires` are all satisfied. + + The predicate form of `check_requires`, for `select` to choose between + variants of one op kind. Kept as the same loop over the same tuple so the + two cannot disagree about what "satisfied" means -- a spec that `accepts` + an op but whose `check_requires` then raises would make dispatch depend + on which one a caller happened to ask. + """ + return all(attrs.get(key) == want for key, want in self.requires) + def check_requires(self, attrs: Mapping[str, Any]) -> None: for key, want in self.requires: got = attrs.get(key) @@ -107,15 +288,47 @@ def check_requires(self, attrs: Mapping[str, Any]) -> None: ) def __post_init__(self) -> None: - for dtype in self.inputs + (self.out_dtype,): - if dtype not in WIRE_DTYPE: + for dtype in self.inputs: + if dtype not in WIRE_DTYPE and dtype not in WIRE_RAW: raise ValueError( - f"{self.kind}: {dtype!r} has no wire form; block-quantized " - "inputs are staged as raw bytes and need their own path" + f"{self.kind}: {dtype!r} is neither a dense wire dtype " + f"({sorted(WIRE_DTYPE)}) nor a raw block-quantized one " + f"({sorted(WIRE_RAW)})" ) + # THE OUTPUT MAY NOT BE RAW, and that is a real restriction rather than an + # oversight. `decode` reads the result back through `np.frombuffer` with a + # numpy dtype, and no kernel in this encoder writes a quantized result -- + # the weights arrive quantized and everything computed is fp16. Allowing it + # would mean a `decode` that cannot decode. + if self.out_dtype not in WIRE_DTYPE: + raise ValueError( + f"{self.kind}: out_dtype {self.out_dtype!r} is not a dense wire " + f"dtype. A kernel may READ block-quantized bytes (see WIRE_RAW) " + f"but not write them: the result has to be decodable." + ) for s in self.scalars: if s.ctype not in _STRUCT_CODE: raise ValueError(f"{self.kind}: unknown scalar ctype {s.ctype!r}") + # Refused at construction, not at generate time: an unknown layout name + # would reach `genentry._layout_check` as a KeyError from a dict lookup + # inside an f-string, which says nothing about which spec is wrong. A + # short `layouts` is the worse error of the two -- it silently leaves the + # output buffer, or an input, with no guard at all. + if self.layouts: + want = len(self.inputs) + 1 + if len(self.layouts) != want: + raise ValueError( + f"{self.kind}: layouts has {len(self.layouts)} entries but " + f"this kernel has {len(self.inputs)} input(s) plus one " + f"output = {want}. Every buffer must be named, in src-then-" + f"dst order, or leave layouts=() for row_major throughout." + ) + for layout in self.layouts: + if layout not in WIRE_LAYOUT: + raise ValueError( + f"{self.kind}: {layout!r} is not a layout on the wire; " + f"known layouts are {sorted(WIRE_LAYOUT)}" + ) def header( self, arrays: tuple[np.ndarray, ...], attrs: Mapping[str, Any] @@ -133,8 +346,11 @@ def payload(self, arrays: tuple[np.ndarray, ...]) -> bytes: f"{self.kind} takes {len(self.inputs)} inputs, got {len(arrays)}" ) out = bytearray() - for array, dtype in zip(arrays, self.inputs): - out += np.ascontiguousarray(array, dtype=WIRE_DTYPE[dtype]).tobytes() + for i, (array, dtype) in enumerate(zip(arrays, self.inputs)): + if dtype in WIRE_RAW: + out += raw_bytes(self.kind, i, dtype, array) + else: + out += np.ascontiguousarray(array, dtype=WIRE_DTYPE[dtype]).tobytes() return bytes(out) def encode( @@ -213,11 +429,273 @@ def decode(self, raw: bytes, shape: tuple[int, ...]) -> np.ndarray: "The remaining 12 are perm (0,2,1), which transposes the INNERMOST " "two axes -- no contiguous run survives, so it is a genuinely " "different kernel. `requires` refuses them rather than returning a " - "correctly-shaped wrong answer." + "correctly-shaped wrong answer -- and `transpose_hd` below is now " + "that kernel, selected by `select()` off the same `perm` attr." + ), + ), + "transpose_hd": RunnerSpec( + kind="transpose", + kernel_dir="kernels/transpose_hd_fp16", + inputs=("fp16",), + out_dtype="fp16", + scalars=( + Scalar("dim:0:0", "int"), # B + Scalar("dim:0:1", "int"), # T + Scalar("dim:0:2", "int"), # D + ), + requires=(("perm", (0, 2, 1)),), + notes=( + "THE OTHER 12 TRANSPOSES, and the first case of two kernels serving " + "one op kind -- which is why `SPECS` is keyed by variant and not by " + "kind. fp16 [12,256,64]->[12,64,256], the QK^T operand layout move.\n" + "perm(0,2,1) transposes the INNERMOST two axes, so unlike its sibling " + "no run is contiguous on both operands and no whole-vector " + "permutation exists. The kernel gathers a strided column scalar-wise " + "and commits each contiguous output row with one HVX vector store; " + "the store side is vectorised and the load side is not, which its " + "header says rather than claiming more.\n" + "`requires` is the disjoint half of `transpose`'s: between them the " + "two cover all 60 ops and no op matches both, which `select` " + "verifies by refusing ambiguity." + ), + ), + "patchify": RunnerSpec( + kind="patchify", + kernel_dir="kernels/patchify_fp32", + inputs=("fp32",), + out_dtype="fp32", + scalars=( + Scalar("dim:0:0", "int"), # C = 3 channels + Scalar("dim:0:1", "int"), # T = 2 temporal_patch + Scalar("dim:0:2", "int"), # H = 256 + Scalar("dim:0:3", "int"), # W = 256 + Scalar("attr:patch", "int"), # 16 + Scalar("attr:merge", "int"), # 2 + Scalar("attr:grid_h", "int"), # 16 + Scalar("attr:grid_w", "int"), # 16 + ), + notes=( + "1 op, and the encoder's FIRST -- fp32 (3,2,256,256) -> (256,1536). The " + "only rank-4 input in the graph, so the first spec to use `dim:0:3`, " + "and the only op still in fp32 on both sides (the `cast` right after it " + "is where fp16 begins).\n" + "MOST SCALARS OF ANY KERNEL HERE: eight, four from the input's own " + "extents and four from attrs. `patch`, `merge`, `grid_h` and `grid_w` " + "cannot be derived from the shapes -- (256,1536) is consistent with " + "several (patch, grid) factorisations -- so they cross as params.\n" + "`merge` CHANGES THE ANSWER and is not metadata. The registry " + "(opdefs/structural.py:200-206) reshapes the patch grid into merge " + "blocks and transposes (2,5,3,6,0,1,4,7), so token order is " + "(bh, bw, mh, mw): consecutive runs of merge*merge = 4 rows must " + "already BE the 2x2 spatial block the downstream merger folds " + "together, because that merger is a pure reshape. Per-patch FEATURE " + "order (c, t, ph, pw) is untouched by merge. Ignoring the reordering " + "gives a correctly-shaped wrong answer that every shape check accepts, " + "which is one of the three near-misses the harness rejects.\n" + "2018331 cycles -- by far the most expensive op in the encoder per " + "invocation, though it runs once. It moves 1.5 MB with no arithmetic, " + "and only the innermost W run is contiguous on both sides." + ), + ), + "matmul": RunnerSpec( + kind="matmul", + kernel_dir="kernels/matmul_fp16", + inputs=("fp16", "fp16"), + out_dtype="fp16", + scalars=( + Scalar("dim:0:0", "int"), # Bn + Scalar("dim:0:1", "int"), # M + Scalar("dim:0:2", "int"), # K + Scalar("dim:1:2", "int"), # N, input 1's last axis + ), + notes=( + "24 ops: the encoder's attention matmuls, QK^T (12) and AV (12). " + "No bias and no activation -- those are matmul_epilogue, a " + "different op kind and a different kernel. Accumulation is fp32 " + "with a single narrow to fp16 per row, which is what every " + "near-miss in the gate is judged against." + ), + ), + "matmul_epilogue": RunnerSpec( + kind="matmul_epilogue", + kernel_dir="kernels/matmul_epilogue_fp16", + inputs=("fp16", "q4_0", "fp32"), + out_dtype="fp16", + scalars=( + Scalar("dim:0:0", "int"), # M, activation rows + Scalar("dim:0:1", "int"), # K, the reduction axis + Scalar("dim:1:1", "int"), # N, the weight's free axis + Scalar( + "attr:act", "int", + codes=(("none", 0), ("gelu_tanh", 1), ("gelu_erf", 2)), + ), + ), + notes=( + "75 ops -- the single largest kind in the encoder, and 55.9 MB of " + "its 58.6 MB of traffic. Weights cross the wire as q4_0 (WIRE_RAW) " + "in row-major order with blocks along N, NOT ggml-hexagon's " + "576-byte HMX tile order. The act codes match MM_ACT_* in the " + "kernel's own kernel_api.h; `codes` is what lets a string attr " + "cross a wire that carries only numbers." + ), + ), + "softmax": RunnerSpec( + kind="softmax", + kernel_dir="kernels/softmax_fp16", + inputs=("fp16",), + out_dtype="fp16", + scalars=( + Scalar("rows:0:2", "int"), # R = numel / C = 12*256 = 3072 + Scalar("dim:0:2", "int"), # C = 256, the reduced axis + ), + requires=(("axis", -1),), + notes=( + "12 ops, all one signature: fp16 (12,256,256) with axis -1, i.e. 3072 " + "independent rows of 256. Attention softmax, over the QK^T scores.\n" + "FIRST USE OF `rows:`, and the reason it exists: the kernel takes " + "(R, C) and R is the PRODUCT of the two leading axes, which no single " + "`dim:` can express. The axis is named rather than taken to be the " + "last, because `ne` is padded to four with ones and 'the last axis' " + "of a rank-3 tensor is then ambiguous between index 2 and 3.\n" + "`requires` pins axis=-1. softmax is a general op kind and softmax " + "along any other axis is a different kernel -- and with the last two " + "dims both 256, a wrong-axis result has the SAME SHAPE and byte count, " + "so nothing downstream could catch it. The kernel's own harness is " + "6x256 for exactly that reason.\n" + "Uses hvx_vec_exp_f32, NOT hvx_vec_exp2_f16, whose E5 coefficient is " + "wrong upstream (0x5082 for 0x090c, 262% error at frac 0.7). 11292 " + "cycles at the gate shape; max and sum reductions are vectorised, " + "with a scalar tail for C % 64 that C=256 never reaches." + ), + ), + "rope_2d": RunnerSpec( + kind="rope_2d", + kernel_dir="kernels/rope_2d_fp16", + inputs=("fp16", "fp32", "fp32"), + out_dtype="fp16", + scalars=( + Scalar("dim:0:0", "int"), # T = 256 tokens + Scalar("dim:0:1", "int"), # H = 12 heads + Scalar("dim:0:2", "int"), # D = 64 head_dim + ), + notes=( + "24 ops, all one signature: fp16 (256,12,64) against fp32 (256,64) " + "cos and sin tables. Second op here with three inputs and mixed input " + "dtypes, after layernorm.\n" + "THE PAIRING IS SPLIT-HALF -- i with i + D/2, GPT-NeoX style -- read " + "off hexlib/graph/opdefs/structural.py:253, where the registry builds " + "concat(-x[..., half:], x[..., :half]). NOT adjacent pairs. A wrong " + "pairing is a correctly-shaped wrong answer, so it was checked against " + "two independent implementations as well: forge2's verified reference " + "for this exact shape, and llama.cpp's hvx_rope_neox_f32_aa (which " + "HTP_ROPE_TYPE_VISION routes to). All three agree sign for sign.\n" + "No `requires`: the tables carry the position encoding, so there is no " + "attr that could select a different kernel. Note the tables are " + "indexed by token and head_dim but NOT by head -- the same rotation " + "applies to every head at a given token, and a near-miss that indexes " + "them by head is one of the six the harness rejects.\n" + "1212 cycles, fully vectorised with no scalar remainder: split-half " + "makes both halves contiguous runs, so the rotation needs no " + "deinterleave, and unlike layernorm this op has no reduction at all. " + "D=64 is the only head_dim the encoder uses; other D fall back to a " + "correct scalar loop." + ), + ), + "layernorm": RunnerSpec( + kind="layernorm", + kernel_dir="kernels/layernorm_fp16", + inputs=("fp16", "fp32", "fp32"), + out_dtype="fp16", + scalars=( + Scalar("dim:0:0", "int"), # R + Scalar("dim:0:1", "int"), # C + Scalar("attr:eps", "float"), + ), + notes=( + "All 25 layernorm ops in the encoder share ONE signature -- checked " + "against the built graph, not assumed: x=(256,768) fp16, weight and " + "bias both (768,) fp32, eps=1e-06. Because the input is rank 2, R and " + "C are each a single dimension, so no new Scalar source was needed.\n" + "THE KERNEL EXISTED AND GATED GREEN FOR A DAY WITHOUT THIS SPEC, and " + "the spec -- not a runner.c -- is what makes an op dispatchable on the " + "DSP batch path. Until this landed, KIND_ID['layernorm'] = 3 was " + "reachable on the wire and answered ERR_NO_KERNEL, while STATE.md " + "counted its 25 ops as covered.\n" + "First kernel here with THREE inputs, and the first with MIXED input " + "dtypes: fp16 data against fp32 affine parameters. That makes it the " + "first real exercise of the generated entry's per-input dtype check, " + "which previously only ever saw buffers of one type.\n" + "Its 111088 cycles are a first rung, not a result -- both reductions " + "are still scalar. Note the gate measured R=4; the encoder needs " + "R=256, which the kernel takes as a parameter and no harness has run." ), ), } +def variants_for(kind: str) -> tuple[str, ...]: + """Every SPECS key implementing this op kind, in declaration order.""" + return tuple(name for name, s in SPECS.items() if s.kind == kind) + + +def select(kind: str, attrs: Mapping[str, Any]) -> tuple[str, RunnerSpec]: + """The variant of `kind` this op belongs to: `(spec_name, spec)`. + + ONE OP KIND IS NOT ONE KERNEL, and `transpose` is where that stopped being a + hypothetical. The encoder's 60 transposes are two different permutations: + 48 at perm(1,0,2), which `transpose_th_fp16` does by moving whole aligned + 128-byte vectors, and 12 at perm(0,2,1), which shares no contiguous run + between its operands and needed a completely different kernel. Neither can + serve the other's ops -- the result would be correctly shaped and silently + wrong -- so `SPECS` is keyed by VARIANT and `RunnerSpec.kind` says which op + kind each variant implements. + + THE WIRE CARRIES NO ATTRS, which is why the variant has to be resolved HERE, + on the host, and why each variant then needs its own entry in `KIND_ID`. The + DSP is handed an id and a buffer list; it has no perm, no axis and no + activation to branch on, and inventing a field for them would mean the DSP + re-deciding something the host already knew. `check_requires` remains the + guard for the case where a caller reaches a specific spec directly. + + Refuses ambiguity rather than taking the first match: two variants that both + accept an op means the `requires` sets are not actually disjoint, and picking + one by dict order would be a coin flip whose outcome is a wrong answer. + """ + cands = [(n, s) for n, s in SPECS.items() if s.kind == kind] + if not cands: + raise KeyError( + f"no kernel implements {kind!r}; known kinds are " + f"{sorted({s.kind for s in SPECS.values()})}" + ) + ok = [(n, s) for n, s in cands if s.accepts(attrs)] + if len(ok) == 1: + return ok[0] + if not ok: + detail = "; ".join( + f"{n} requires {dict(s.requires)}" for n, s in cands + ) + raise ValueError( + f"no {kind!r} kernel accepts this op's attrs " + f"{ {k: attrs.get(k) for _, s in cands for k, _ in s.requires} }. " + f"Candidates: {detail}. Dispatching it to any of them would produce " + f"a correctly-shaped wrong answer, so it is refused." + ) + raise ValueError( + f"{len(ok)} {kind!r} kernels all accept this op ({[n for n, _ in ok]}); " + f"their `requires` sets are not disjoint. Resolving that by dict order " + f"would make which kernel runs an accident." + ) + + def spec_for(kind: str) -> RunnerSpec | None: - return SPECS.get(kind) + """The sole variant of `kind`, or None. + + For the standalone-ELF path (`hexlib/exec/hexagon.py`), which predates + variants and drives one kernel per kind with no attrs to select on. Returns + None rather than guessing when a kind has several variants -- that caller + has no attrs, so it genuinely cannot choose, and a guess would be silent. + """ + names = variants_for(kind) + if len(names) != 1: + return None + return SPECS[names[0]] diff --git a/hexlib/exec/wholeplan.py b/hexlib/exec/wholeplan.py new file mode 100644 index 0000000..9f923f6 --- /dev/null +++ b/hexlib/exec/wholeplan.py @@ -0,0 +1,409 @@ +"""ONE batch blob carrying EVERY op of a compiled plan -- a single entry point +into the DSP, and a single exit. + +WHY THIS EXISTS. `hexlib/exec/dsp.py`'s `DspSimBackend` packs one op per batch +and launches the simulator once per op. That is a correctness path and it works, +but it costs a process start, a QuRT boot and a tear-down per op -- measured at +~7 s of pure overhead on this machine, before any compute. Across the encoder's +259 real-work ops that is half an hour of transport alone, and on a DEVICE the +equivalent per-op round trip is the reason nothing can hold a plan. + +The wire was built for this from the start and nothing here extends it: +`wire.pack_batch(bufs, tensors, ops)` already takes a LIST of ops, and +`skel_dispatch.c:171` already loops `for (i = 0; i < hdr.n_ops; i++)` filling a +per-op status and cycle count. What was missing was a host-side builder that +emits the whole plan at once. This is that builder, and it is host-only: no C +changed, no IDL changed, no new kernel. + +ONE BUFFER, NOT SEVERAL. Every tensor lands in a single arena at `bi = 0`. +`MAX_BUFS` (8) is NOT an arena limit -- it is the size of the DSP's per-op +`hexlib_args.buf[]` array, which `pack_batch` already checks as +`len(src) + len(dst)`. A `BufDesc` is a mapped fd, so one fd for the whole plan +is both legal and what a device wants: one `rpcmem_alloc`, one `fastrpc_mmap`. + +THE ARENA'S THREE REGIONS, and the reason they are not one: + + consts -- every weight and bias, each at its own offset. Written once, + never aliased, read throughout. At 256x256 this is 56.9 MB and it + dominates the allocation. + activations -- at the PLAN'S OWN VTCM slot offsets, which alias: 88 slots at + the tiny config share far less space than their total size, + because `Slot.first_use`/`last_use` say when each is dead. Reusing + the plan's offsets rather than inventing new ones means this path + executes the allocator's decisions instead of second-guessing + them -- and an aliasing bug shows up as a wrong answer, which is + exactly what `test_aliased_slots_actually_corrupt_a_value` proves + a name-keyed dict cannot catch. + io -- the graph's declared inputs and outputs, each at its own offset, + never aliased. The host writes the input and reads the output, so + these must survive the whole run whatever liveness says. + +RESHAPE EMITS NO OP. 49 of the plan's 308 steps are reshapes, and in row-major a +reshape is a pure reinterpretation: same bytes, different `ne`. So the output +tensor is placed AT ITS INPUT'S OFFSET and no op is emitted -- zero copies, zero +kernels, and `ne` differs between the two descriptors, which is the whole +content of the operation. Doing so BREAKS the plan's VTCM slot offsets -- +see `_refuse_unsafe_aliasing` at the bottom of this file, which is why +`alias_activations` defaults to False and refuses when asked. +""" +from __future__ import annotations + +from dataclasses import dataclass, field +from typing import Any, Mapping + +import numpy as np + +from hexlib.exec.dsp import _align_up, _encode_params, _ne +from hexlib.graph.ir import nbytes as ir_nbytes +from hexlib.exec.runner import WIRE_DTYPE, WIRE_RAW, RawTensor, select +from hexlib.runtime.genentry import KIND_ID +from hexlib.runtime import wire + + +class WholePlanError(Exception): + pass + + +@dataclass(frozen=True) +class Placement: + """Where one tensor lives in the arena, and how it is described on the wire.""" + name: str + offset: int + nbytes: int + dtype: str + layout: str + ne: tuple[int, int, int, int] + region: str # "const" | "act" | "io" | "reshape-alias" + + +@dataclass +class WholePlanBatch: + blob: bytes + payload: bytearray + placements: dict[str, Placement] + index: dict[str, int] # tensor name -> wire tensor index + n_ops: int + arena_bytes: int + outputs: tuple[str, ...] + skipped_reshapes: int + op_names: tuple[str, ...] = field(default=()) + + +def build_whole_plan_batch(compiled, feeds: Mapping[str, Any], + alias_activations: bool = False) -> WholePlanBatch: + """Pack `compiled`'s entire plan into one batch, with `feeds` staged in. + + `feeds` supplies every graph input and every const, exactly as + `interpreter.run` takes them. A const declared `q4_0` is quantized here if it + arrives as a dense array, the same way `dsp.interpreter_backends` does it, so + the caller passes the same feeds to both paths and the comparison is of + arithmetic rather than of weight values. + """ + from hexlib.exec.quant import quantize_q4_0 + + graph, plan = compiled.graph, compiled.plan + tensors = graph.tensors + + # --- 1. Which region does each tensor belong to? ----------------------- + io_names = set(graph.inputs) | set(graph.outputs) + const_names = {n for n, t in tensors.items() if t.const} + + # A reshape's output is its input reinterpreted. Resolve chains (a reshape + # feeding a reshape) to the ultimate storage owner before anything is placed. + reshape_src: dict[str, str] = {} + for step in plan.steps: + op = getattr(step, "op", None) + if op is not None and op.kind == "reshape": + reshape_src[op.outputs[0]] = op.inputs[0] + + def storage_owner(name: str) -> str: + seen = {name} + while name in reshape_src: + name = reshape_src[name] + if name in seen: + raise WholePlanError(f"reshape chain cycles at {name!r}") + seen.add(name) + return name + + # An io tensor must never be aliased away by a reshape -- the host has to be + # able to read the declared output at a stable address. + aliased_io = {n for n in reshape_src if n in io_names} + if aliased_io: + raise WholePlanError( + f"these declared graph inputs/outputs are reshape outputs and would " + f"be aliased onto another tensor's storage: {sorted(aliased_io)}. " + f"Give them their own placement before using this path." + ) + + slot_offset = {s.tensor: s.offset for s in plan.vtcm} + + # --- 1b. LAYOUT IS A PROPERTY OF THE OP'S BUFFER SLOT, NOT OF THE TENSOR. + # `spec.buf_layouts()` returns one layout per buffer in src-then-dst order, + # so the same q4_0 weight is `row_major` for a kernel that reads plain + # blocks and `q4_0_repacked` for one that wants tiles. Deriving it from the + # DTYPE instead -- which this file did first -- sends `q4_0_repacked` for + # every quantized weight and the DSP's `_layout_check` rejects the op with + # ERR_REQUIRES, correctly. A tensor consumed by two ops that disagree is a + # real defect (it would need repacking between them), so it is refused here + # rather than resolved by picking one. + layout_of: dict[str, str] = {} + dtype_of: dict[str, str] = {} + + def claim_layout(name: str, layout: str, where: str) -> None: + prev = layout_of.setdefault(name, layout) + if prev != layout: + raise WholePlanError( + f"tensor {name!r} is used as {prev!r} and as {layout!r} (at " + f"{where}); one arena cannot hold both without a repack step" + ) + + # THE SPEC'S DTYPE, NOT THE GRAPH'S, decides the bytes in the arena. + # `dsp.py`'s per-op path coerces each array to `spec.inputs[i]` on the way + # out (`np.ascontiguousarray(a, dtype=WIRE_DTYPE[dt])`) and declares that + # dtype on the wire. A whole-plan arena has no such per-call moment, so the + # coercion has to happen once, at staging. `pos_embed` is the case that + # forced this: the graph declares it fp32, `add` declares both inputs fp16, + # and staging the graph's dtype makes the DSP reject the op with + # ERR_REQUIRES -- correctly, because a kernel reading fp32 bytes as fp16 + # would otherwise return a correctly-shaped wrong answer. + def claim_dtype(name: str, dtype: str, where: str) -> None: + prev = dtype_of.setdefault(name, dtype) + if prev != dtype: + raise WholePlanError( + f"tensor {name!r} is read as {prev!r} and as {dtype!r} (at " + f"{where}); one arena slot cannot hold both, and an implicit " + f"conversion here would be invisible to every downstream check" + ) + + op_specs: dict[int, tuple[str, Any]] = {} + for step in plan.steps: + op = getattr(step, "op", None) + if op is None or op.kind == "reshape": + continue + spec_name, spec = select(op.kind, dict(op.attrs)) + spec.check_requires(dict(op.attrs)) + op_specs[op.id] = (spec_name, spec) + buf_layouts = spec.buf_layouts() + for i, n in enumerate(op.inputs): + claim_layout(n, buf_layouts[i], f"{op.id}:{spec_name} src{i}") + claim_dtype(n, spec.inputs[i], f"{op.id}:{spec_name} src{i}") + for n in op.outputs: + claim_layout(n, buf_layouts[-1], f"{op.id}:{spec_name} dst") + claim_dtype(n, spec.out_dtype, f"{op.id}:{spec_name} dst") + + # --- 2. Lay the arena out --------------------------------------------- + placements: dict[str, Placement] = {} + cursor = 0 + + def wire_dtype(name: str) -> str: + return dtype_of.get(name, tensors[name].dtype) + + def wire_nbytes(name: str) -> int: + return ir_nbytes(tensors[name].shape, wire_dtype(name)) + + def place(name: str, offset: int, region: str) -> None: + t = tensors[name] + placements[name] = Placement( + name=name, offset=offset, nbytes=wire_nbytes(name), + dtype=wire_dtype(name), layout=layout_of.get(name, "row_major"), + ne=_ne(t.shape), region=region, + ) + + for name in sorted(const_names): + place(name, cursor, "const") + cursor = _align_up(cursor + wire_nbytes(name)) + const_end = cursor + + act_names = [ + n for n in tensors + if n not in const_names and n not in io_names and n not in reshape_src + ] + retyped = [n for n in act_names if wire_dtype(n) != tensors[n].dtype] + if retyped: + raise WholePlanError( + f"these activations would be staged in a dtype other than the one " + f"the plan sized their VTCM slot with, so every offset after them " + f"is wrong: {sorted(retyped)[:8]}" + ) + missing_slot = [n for n in act_names if n not in slot_offset] + if missing_slot: + raise WholePlanError( + f"{len(missing_slot)} activation tensors have no VTCM slot in the " + f"plan, so this builder has no offset for them: " + f"{sorted(missing_slot)[:8]}" + ) + if alias_activations: + # WHY THIS IS GUARDED AND OFF BY DEFAULT. Reusing the plan's VTCM slot + # offsets as flat-arena addresses is only safe if the allocator's + # disjointness guarantee still holds, and RESHAPE ELISION BREAKS IT. + # The plan gives a reshape output its own slot; this builder puts it on + # its input's storage instead, so the input's slot stays live longer + # than the allocator was told and whatever legitimately owns that + # address in the meantime is overwritten. Measured at the tiny config: + # correlation with the reference fell from 1.000000 to 0.277. + # + # The allocator is NOT at fault -- checked directly, zero of its 88 + # slots overlap in live range. Making this mode correct means unifying + # reshape chains into one tensor BEFORE allocation, which is a plan-pass + # change, not a change here. Until then the check below refuses rather + # than silently returning a corrupted arena, because a wrong answer from + # this path looks exactly like a kernel bug. + _refuse_unsafe_aliasing(plan, reshape_src, slot_offset, tensors, + act_names, wire_nbytes) + act_span = max( + (slot_offset[n] + wire_nbytes(n) for n in act_names), default=0 + ) + for name in act_names: + place(name, const_end + slot_offset[name], "act") + cursor = _align_up(const_end + act_span) + else: + # NO ALIASING: every activation gets its own address. Costs memory and + # exercises none of the allocator, but it is the control case -- if the + # answer is right here and wrong with aliasing on, the defect is in the + # liveness the slots encode and not in the kernels or the wire. + cursor = const_end + for name in sorted(act_names): + place(name, cursor, "act") + cursor = _align_up(cursor + wire_nbytes(name)) + + for name in sorted(io_names): + place(name, cursor, "io") + cursor = _align_up(cursor + wire_nbytes(name)) + + # Reshape outputs borrow their owner's offset but keep their OWN ne. + for name in reshape_src: + owner = storage_owner(name) + if owner not in placements: + raise WholePlanError(f"reshape {name!r} resolves to unplaced {owner!r}") + t = tensors[name] + base = placements[owner] + if wire_nbytes(name) != base.nbytes: + raise WholePlanError( + f"reshape {name!r} is {wire_nbytes(name)} bytes but its owner " + f"{owner!r} is {base.nbytes}; a reshape must preserve byte count" + ) + placements[name] = Placement( + name=name, offset=base.offset, nbytes=wire_nbytes(name), + dtype=wire_dtype(name), layout=layout_of.get(name, "row_major"), + ne=_ne(t.shape), region="reshape-alias", + ) + + arena_bytes = _align_up(cursor) + + # --- 3. Stage the feeds ----------------------------------------------- + payload = bytearray(arena_bytes) + for name in sorted(const_names) + sorted(graph.inputs): + if name not in feeds: + raise WholePlanError(f"no feed supplied for {name!r}") + p = placements[name] + value = feeds[name] + if p.dtype in WIRE_RAW: + raw = value if isinstance(value, RawTensor) else RawTensor( + p.dtype, tuple(np.asarray(value).shape), + quantize_q4_0(np.asarray(value)), + ) + data = raw.data + else: + data = np.ascontiguousarray( + np.asarray(value), dtype=WIRE_DTYPE[p.dtype] + ).tobytes() + if len(data) != p.nbytes: + raise WholePlanError( + f"feed {name!r} staged {len(data)} bytes, but the graph declares " + f"{p.nbytes} -- a dtype or shape disagreement, not a rounding one" + ) + payload[p.offset:p.offset + p.nbytes] = data + + # --- 4. Wire descriptors ---------------------------------------------- + order = sorted(placements) + index = {name: i for i, name in enumerate(order)} + wire_tensors = [ + wire.TensorDesc( + bi=0, offset=placements[n].offset, nbytes=placements[n].nbytes, + dtype=placements[n].dtype, layout=placements[n].layout, + ne=placements[n].ne, + ) + for n in order + ] + + ops: list[wire.OpDesc] = [] + op_names: list[str] = [] + skipped = 0 + for step in plan.steps: + op = getattr(step, "op", None) + if op is None: + continue + if op.kind == "reshape": + skipped += 1 + continue + spec_name, spec = op_specs[op.id] + stand_ins = tuple(_ShapeOnly(tensors[n].shape) for n in op.inputs) + ops.append(wire.OpDesc( + kind=KIND_ID[spec_name], + params=_encode_params(spec, stand_ins, dict(op.attrs)), + src=tuple(index[n] for n in op.inputs), + dst=tuple(index[n] for n in op.outputs), + )) + op_names.append(f"{op.id}:{spec_name}") + + bufs = [wire.BufDesc(fd=0, size=arena_bytes)] + blob = wire.pack_batch(bufs, wire_tensors, ops) + + return WholePlanBatch( + blob=blob, payload=payload, placements=placements, index=index, + n_ops=len(ops), arena_bytes=arena_bytes, + outputs=tuple(graph.outputs), skipped_reshapes=skipped, + op_names=tuple(op_names), + ) + + +@dataclass(frozen=True) +class _ShapeOnly: + """`_encode_params` only ever reads `.shape` for attr-sourced scalars, and + `numel:`/`dim:` are derived on the DSP from `ne[]`. Passing real arrays here + would mean materializing every intermediate on the host, which is precisely + what this path exists to avoid.""" + shape: tuple[int, ...] + + +def _refuse_unsafe_aliasing(plan, reshape_src, slot_offset, tensors, + act_names, wire_nbytes) -> None: + """Raise unless every pair of activations sharing an address is disjoint in + time ONCE reshape aliasing is accounted for. + + The plan's own slots satisfy this by construction. What this checks is the + property AFTER this builder has moved reshape outputs onto their inputs, + which is the step that can violate it. + """ + live = {s.tensor: [s.first_use, s.last_use] for s in plan.vtcm} + + # A reshape output's storage is its input's, so the input must be treated as + # live for the union of both ranges. + for out, inp in reshape_src.items(): + if out in live and inp in live: + live[inp][0] = min(live[inp][0], live[out][0]) + live[inp][1] = max(live[inp][1], live[out][1]) + + placed = [(n, slot_offset[n], wire_nbytes(n)) for n in act_names + if n in slot_offset and n in live] + clashes = [] + for i in range(len(placed)): + ni, oi, si = placed[i] + for j in range(i + 1, len(placed)): + nj, oj, sj = placed[j] + if oi < oj + sj and oj < oi + si: # byte ranges overlap + a, b = live[ni], live[nj] + if a[0] <= b[1] and b[0] <= a[1]: # and so do live ranges + clashes.append((ni, a, nj, b)) + if clashes: + detail = "; ".join( + f"{n1}[{a[0]},{a[1]}] vs {n2}[{b[0]},{b[1]}]" + for n1, a, n2, b in clashes[:5] + ) + raise WholePlanError( + f"{len(clashes)} activation pairs share an address while both are " + f"live, once reshape aliasing is folded in: {detail}. This arena " + f"would compute a wrong answer that looks like a kernel bug. Use " + f"alias_activations=False, or unify reshape chains before the VTCM " + f"pass so the allocator sees one tensor instead of two." + ) diff --git a/hexlib/graph/ir.py b/hexlib/graph/ir.py index c258ac4..98c61da 100644 --- a/hexlib/graph/ir.py +++ b/hexlib/graph/ir.py @@ -12,9 +12,10 @@ from types import MappingProxyType from typing import Any, Mapping -DTYPES = frozenset({"fp32", "fp16", "int32", "q4_0"}) +DTYPES = frozenset({"fp32", "fp16", "int32", "q4_0", "q8_0"}) -# Dense dtypes only. q4_0 is block-structured and handled separately in nbytes(). +# Dense dtypes only. q4_0 and q8_0 are block-structured and handled separately +# in nbytes(). _DENSE_BYTES = {"fp32": 4, "fp16": 2, "int32": 4} # Q4_0: 32 four-bit values (16 bytes) + one fp16 scale (2 bytes) = 18 bytes. @@ -22,6 +23,23 @@ Q4_0_BLOCK = 32 Q4_0_BLOCK_BYTES = 18 +# `block_q8_0` (ggml-common.h:250-256): `ggml_half d; int8_t qs[32];` = 34 bytes. +# WHY A SECOND BLOCK-QUANTIZED WEIGHT FORMAT EXISTS AT ALL. Measured against +# transformers on the shipped Qwen3.5-0.8B vision weights at 256x256, with fp32 +# arithmetic on both sides so nothing but the weight format differs: +# +# q4_0 encoder output cosine 0.867606 +# q8_0 encoder output cosine 0.999002 +# +# for 1.89x the weight bytes (55.5 MB -> 105 MB). Four bits is not enough for +# this encoder, and the reason is that the error is DIFFUSE rather than +# concentrated -- keeping the merger, the patch embedding and all of attention +# at higher precision while leaving the MLPs at q4_0 still only reaches 0.913, +# because 12 layers of small errors compound. Mixed precision was measured and +# rejected on that evidence, not assumed. +Q8_0_BLOCK = 32 +Q8_0_BLOCK_BYTES = 34 + _ATTR_SCALARS = (bool, int, float, str, type(None)) @@ -43,13 +61,17 @@ def nbytes(shape: tuple[int, ...], dtype: str) -> int: if dtype not in DTYPES: raise ValueError(f"unknown dtype {dtype!r}; expected one of {sorted(DTYPES)}") numel = math.prod(shape) if shape else 1 - if dtype == "q4_0": - if not shape or shape[-1] % Q4_0_BLOCK != 0: + if dtype in ("q4_0", "q8_0"): + block, block_bytes = ( + (Q4_0_BLOCK, Q4_0_BLOCK_BYTES) if dtype == "q4_0" + else (Q8_0_BLOCK, Q8_0_BLOCK_BYTES) + ) + if not shape or shape[-1] % block != 0: raise ValueError( - f"q4_0 tensor of shape {shape} has last dim {shape[-1] if shape else 0}, which is not a " - f"multiple of the {Q4_0_BLOCK}-element block size" + f"{dtype} tensor of shape {shape} has last dim {shape[-1] if shape else 0}, which is not a " + f"multiple of the {block}-element block size" ) - return numel // Q4_0_BLOCK * Q4_0_BLOCK_BYTES + return numel // block * block_bytes return numel * _DENSE_BYTES[dtype] diff --git a/hexlib/hostload.py b/hexlib/hostload.py new file mode 100644 index 0000000..b6ea069 --- /dev/null +++ b/hexlib/hostload.py @@ -0,0 +1,200 @@ +"""Tell a starved simulator apart from a kernel that never finishes. + +WHY THIS EXISTS. `hexlib test` measures WALL time and kills a stage at +SIM_TIMEOUT_MAX_S. Wall time is not a property of the kernel -- it is a +property of the kernel AND whatever else the host happened to be running -- +so a timeout on its own says nothing about which of the two caused it. The +gate used to resolve that ambiguity by guessing, and the guess was wrong +twice in one session on matmul_fp16: + + * nearmiss_fp16_accumulate.c genuinely needed 1195s against a 900s ceiling + and was reported as "the kernel may not terminate". It terminates. + * nearmiss_wrong_batch_stride.c had ALREADY passed in 825s, then blew past + 1800s on a re-run because a game launched 5 minutes into that stage. Same + byte-identical ELF, same deterministic input, so the simulated work was + identical; only the host had changed. + +Both were reported with the same words, and neither set of words was true. +An hour went into re-running a gate to discover the second one. + +THE MEASUREMENT. Simulated cycles would be the ideal evidence, but the +harness prints HEXLIB_KCYCLES at the END of a run, so a stage that times out +has no cycle count -- and that is exactly the stage in question. What IS +available while the process is alive is its CPU time. hexagon-sim is +single-threaded, so on an unloaded host it accrues very close to one +CPU-second per wall-second. A process that got 0.4 was starved by something +else on the machine; a process that got 0.98 was given everything it asked +for and still did not finish, which IS evidence about the kernel. + +WHY NOT SYSTEM-WIDE LOAD. "Is the machine busy" answers a different question +than "was THIS process starved", and answers it worse: it needs the host's +core count to interpret, and it convicts an idle-but-slow run of contention +that never touched it. Per-process CPU share needs no such correction. + +psutil IS OPTIONAL. It is not in this project's dependencies and is not +being added for a diagnostic: an import failure degrades to "unmeasured", +which reports the ambiguity honestly instead of inventing a cause. Same +reasoning as hexlib/device/qdc/job.py's lazy SDK import. +""" +from __future__ import annotations + +import os +import threading +import time +from dataclasses import dataclass + +try: # optional, see module docstring + import psutil +except Exception: # pragma: no cover - depends on the environment + psutil = None + +# The simulator executable, matched case-insensitively as a prefix so +# "hexagon-sim" and "hexagon-sim.exe" both hit. +_SIM_PROC_HINT = "hexagon-sim" + +# Below this share of one CPU, a single-threaded process was competing for the +# machine rather than using it. Ordinary scheduling noise on an idle host does +# not push a busy single-threaded process under ~0.9; the contended stage that +# motivated this file would have scored far below 0.75. +STARVED_BELOW = 0.75 + +_SAMPLE_INTERVAL_S = 2.0 + + +@dataclass(frozen=True) +class LoadStats: + """What a run cost in wall time, and how much CPU it was actually given. + + `cpu_s` is None when nothing could be measured (psutil missing, or the + process began and ended between two samples). None means UNKNOWN and must + never be read as either "starved" or "not starved". + """ + + wall_s: float + cpu_s: float | None + samples: int + + @property + def cpu_share(self) -> float | None: + """CPU-seconds per wall-second. ~1.0 for an unstarved single thread.""" + if self.cpu_s is None or self.wall_s <= 0: + return None + return self.cpu_s / self.wall_s + + @property + def starved(self) -> bool | None: + """True/False when measured, None when unknown. Tri-state on purpose.""" + share = self.cpu_share + return None if share is None else share < STARVED_BELOW + + +class SimLoadMonitor: + """Sample the simulator child's CPU time for the life of a `with` block. + + Deliberately samples a CHILD DISCOVERED BY PID rather than taking a pid as + an argument: that keeps toolchain.run's signature and its carefully + documented decoding behaviour untouched, and keeps every existing caller + and the tests that monkeypatch it working unchanged. + + A monitor that measures nothing is not an error. `stats()` reports + cpu_s=None and the caller says so out loud. + """ + + def __init__( + self, + proc_hint: str = _SIM_PROC_HINT, + interval_s: float = _SAMPLE_INTERVAL_S, + ) -> None: + self._hint = proc_hint.lower() + self._interval = interval_s + self._stop = threading.Event() + self._thread: threading.Thread | None = None + self._cpu_s: float | None = None + self._samples = 0 + self._t0 = 0.0 + self._t1: float | None = None + + def __enter__(self) -> "SimLoadMonitor": + self._t0 = time.time() + if psutil is not None: + self._thread = threading.Thread(target=self._loop, daemon=True) + self._thread.start() + return self + + def __exit__(self, *exc: object) -> None: + self._t1 = time.time() + self._stop.set() + if self._thread is not None: + # The sampler only reads counters, so it cannot block on anything + # slow; a short join keeps a wedged thread from holding up a gate. + self._thread.join(timeout=self._interval * 2) + + def _matching_child(self): + me = psutil.Process(os.getpid()) + for child in me.children(recursive=True): + try: + if child.name().lower().startswith(self._hint): + return child + except psutil.Error: + continue + return None + + def _loop(self) -> None: + while not self._stop.is_set(): + try: + child = self._matching_child() + if child is not None: + t = child.cpu_times() + # Cumulative and monotonic, so the last reading before the + # process dies is the total. max() guards against reading a + # DIFFERENT, younger hexagon-sim after a restart. + total = float(t.user) + float(t.system) + self._cpu_s = total if self._cpu_s is None else max(self._cpu_s, total) + self._samples += 1 + except Exception: + # Sampling is diagnostic. It must never be the reason a gate + # fails, so every error here degrades to "unmeasured". + pass + self._stop.wait(self._interval) + + def stats(self) -> LoadStats: + end = self._t1 if self._t1 is not None else time.time() + return LoadStats( + wall_s=end - self._t0, cpu_s=self._cpu_s, samples=self._samples + ) + + +def timeout_diagnosis(timeout_s: float, stats: LoadStats) -> str: + """Explain a timeout in terms of what was measured, never by guessing. + + Pure: takes the numbers, returns the sentence. The three branches are the + three things that can actually be true, and the unmeasured branch says so + rather than defaulting to blaming the kernel. + """ + share = stats.cpu_share + if share is None: + return ( + "the simulator's CPU share could not be measured, so this timeout " + "does NOT distinguish a kernel that never terminates from a host " + "too busy to finish one that does. Re-run on an idle host before " + "treating it as a kernel defect." + ) + if share < STARVED_BELOW: + return ( + f"the simulator was given only {share:.0%} of one CPU " + f"({stats.cpu_s:.0f} CPU-seconds over {stats.wall_s:.0f} wall-" + f"seconds) — the HOST was contended. This is not evidence that " + f"the kernel fails to terminate; wall-clock budgets measure the " + f"machine as much as the code. Re-run on an idle host." + ) + return ( + f"the simulator was given {share:.0%} of one CPU " + f"({stats.cpu_s:.0f} CPU-seconds over {stats.wall_s:.0f} wall-seconds), " + f"so it was not DESCHEDULED. That is weaker than 'the host was idle': a " + f"process keeps accruing a full CPU-second per wall-second while losing " + f"badly to memory-bandwidth contention, an SMT sibling, or cache " + f"pressure. Measured 2026-08-13: a byte-identical near-miss ELF ran " + f"1218s once and exceeded 1800s twice at ~99% share. So this points at " + f"the kernel or the shape WITHOUT ruling out a loaded host -- check what " + f"else was running before concluding the code is at fault." + ) diff --git a/hexlib/models/loader.py b/hexlib/models/loader.py new file mode 100644 index 0000000..d1a1614 --- /dev/null +++ b/hexlib/models/loader.py @@ -0,0 +1,227 @@ +"""Checkpoint weights -> hexlib graph consts, for the Qwen3.5 vision tower. + +THIS MAPPING WAS WRITTEN FOR A TEST AND BELONGS IN THE LIBRARY. It lived in +`hexlib/tests/test_vision_oracle.py` as `_feeds_from_golden`, keyed to that +file's TINY_CFG and to an npz of random parameters. Every part of it -- the +Conv3d flatten, the fused-qkv split, each transpose, the two folded tables -- +is exactly what a real checkpoint needs too, and a second copy written for the +checkpoint would be a second chance to get a transpose wrong. + +WHAT IS NOT A COPY. `pos_embed` and `rope_cos`/`rope_sin` are DERIVED, not read. +Upstream computes the positional embedding at run time as a bilinear resample of +a learned grid, and the rotary tables from position ids; hexlib folds both to +constants because the resolution is fixed at compile time. A loader that matched +parameter names and copied tensors would produce a graph that is complete, +shape-correct, and wrong in three of its 203 consts. + +THE TRANSPOSES ARE THE DANGEROUS PART. torch `nn.Linear` stores [out, in] and +hexlib wants [in, out]; the fused qkv is [3H, H] and splits into three [H, H] +blocks in q, k, v order (modeling_qwen3_5.py:929 reshapes to (seq, 3, heads, -1) +and unbinds along the `3` axis). At hidden_size 768 every one of those is +square, so a forgotten transpose passes every shape check this file makes and +produces a correctly-shaped wrong answer. +""" +from __future__ import annotations + +import numpy as np + +from hexlib.models.vit import VitConfig + + +def vision_bilinear_indices_and_weights( + grid: int, num_grid_per_side: int, merge: int +) -> tuple[np.ndarray, np.ndarray]: + """Numpy port of `get_vision_bilinear_indices_and_weights`. + + transformers/vision_utils.py:147-217 (single image, grid_thw = [[1, grid, + grid]], so the `t` loop there runs once and `.repeat(t)` is a no-op). + Read, not reinvented: the four-corner bilinear indices/weights over the + learned `side x side` grid, then reordered into merge-block token order + (the same order `patchify` and `get_vision_position_ids` use) by + `vision_utils.py:207-209`. + """ + side = num_grid_per_side + h = w = grid + + h_grid = np.linspace(0, side - 1, h) + w_grid = np.linspace(0, side - 1, w) + h_floor = h_grid.astype(np.int64) + w_floor = w_grid.astype(np.int64) + h_ceil = np.minimum(h_floor + 1, side - 1) + w_ceil = np.minimum(w_floor + 1, side - 1) + h_frac = h_grid - h_floor + w_frac = w_grid - w_floor + + h_floor_offset = h_floor * side + h_ceil_offset = h_ceil * side + + corner_indices = [ + (h_floor_offset[:, None] + w_floor[None, :]).flatten(), + (h_floor_offset[:, None] + w_ceil[None, :]).flatten(), + (h_ceil_offset[:, None] + w_floor[None, :]).flatten(), + (h_ceil_offset[:, None] + w_ceil[None, :]).flatten(), + ] + corner_weights = [ + ((1 - h_frac)[:, None] * (1 - w_frac)[None, :]).flatten(), + ((1 - h_frac)[:, None] * w_frac[None, :]).flatten(), + (h_frac[:, None] * (1 - w_frac)[None, :]).flatten(), + (h_frac[:, None] * w_frac[None, :]).flatten(), + ] + + h_idx = np.arange(h).reshape(h // merge, merge) + w_idx = np.arange(w).reshape(w // merge, merge) + base = h_idx[:, :, None, None] * w + w_idx[None, None, :, :] + reorder = base.transpose(0, 2, 1, 3).flatten() + + indices = np.stack([c[reorder] for c in corner_indices]) + weights = np.stack([c[reorder] for c in corner_weights]) + return indices, weights + + +def vision_position_ids(grid: int, merge: int) -> np.ndarray: + """Numpy port of `get_vision_position_ids`, transformers/vision_utils.py:76-81. + + Single image, grid_thw = [[1, grid, grid]] -- the `t` loop runs once and + `.repeat(t)` is a no-op. + """ + h = w = grid + hpos_ids = np.broadcast_to(np.arange(h)[:, None], (h, w)) + hpos_ids = hpos_ids.reshape(h // merge, merge, w // merge, merge).transpose(0, 2, 1, 3).flatten() + + wpos_ids = np.broadcast_to(np.arange(w)[None, :], (h, w)) + wpos_ids = wpos_ids.reshape(h // merge, merge, w // merge, merge).transpose(0, 2, 1, 3).flatten() + + return np.stack([hpos_ids, wpos_ids], axis=-1) + + +def rope_tables(cfg: VitConfig) -> tuple[np.ndarray, np.ndarray]: + """rope_cos, rope_sin -- modeling_qwen3_5.py:84-92 (inv_freq, rotary forward) + and :1096-1108 (position ids -> freqs -> cos/sin), reproduced on the host. + + `Qwen3_5VisionRotaryEmbedding(head_dim // 2)`: inv_freq has + `dim = head_dim // 2` elements halved again by the `arange(0, dim, 2)` + step, so `inv_freq` has `head_dim // 4` entries. Each token's (row, col) + position id is multiplied by inv_freq and flattened row-major (row block, + then col block) to `[n, head_dim // 2]`, then duplicated by + `cat((freqs, freqs), dim=-1)` to `[n, head_dim]` before cos/sin. + """ + d = cfg.head_dim + dim = d // 2 + inv_freq = 1.0 / (cfg.rope_theta ** (np.arange(0, dim, 2, dtype=np.float64) / dim)) + + position_ids = vision_position_ids(cfg.grid, cfg.spatial_merge_size).astype(np.float64) + freqs = position_ids[:, :, None] * inv_freq[None, None, :] # [n, 2, dim//2] + n = position_ids.shape[0] + flat = freqs.reshape(n, -1) # [n, head_dim//2] + emb = np.concatenate([flat, flat], axis=-1) # [n, head_dim] + return np.cos(emb).astype(np.float32), np.sin(emb).astype(np.float32) + + +def pos_embed_table(cfg: VitConfig, table: np.ndarray) -> np.ndarray: + """pos_embed -- modeling_qwen3_5.py:1090, :1100 and + `get_vision_bilinear_indices_and_weights` (transformers/vision_utils.py:147-217), + reproduced on the host: bilinearly resample the learned + `num_grid_per_side x num_grid_per_side` grid onto this config's patch grid, + in merge-block token order. + + `num_grid_per_side` is derived from `table.shape[0]` (the golden's own + `pos_embed.weight` row count), not hardcoded, exactly as + `self.num_grid_per_side = int(config.num_position_embeddings**0.5)` + (modeling_qwen3_5.py:1039) computes it from the config. A hardcoded + constant here would let a future regeneration with a different learned + grid size leave `side` silently wrong while `table`'s shape still passes + every shape check -- shape-correct but numerically wrong, the exact + failure class this oracle otherwise eliminates. + """ + side = int(table.shape[0] ** 0.5) + if side * side != table.shape[0]: + raise ValueError( + f"pos_embed table has {table.shape[0]} rows, which is not a perfect " + "square; num_grid_per_side is undefined" + ) + indices, weights = vision_bilinear_indices_and_weights(cfg.grid, side, cfg.spatial_merge_size) + out = np.zeros((indices.shape[1], table.shape[1]), dtype=np.float64) + for i in range(4): + out += table[indices[i]].astype(np.float64) * weights[i][:, None] + return out.astype(np.float32) + + +def feeds_from_params(graph, cfg: VitConfig, get) -> dict[str, np.ndarray]: + """Map the upstream parameter names onto hexlib's tensor names. + + Returns a feed for the graph's image input and for every const tensor. + hexlib's graph starts from the IMAGE and patchifies it itself, so the feed + is z["image"] -- not z["patches"]. That is deliberate: it puts patchify + inside the differential rather than beside it. + + Every Linear weight is TRANSPOSED (torch stores [out, in]; hexlib stores + [k, n]). The fused qkv weight [3H, H] is split into three [H, H] blocks in + q, k, v order and transposed -- modeling_qwen3_5.py:929 reshapes the + linear's output to (seq, 3, heads, -1) and unbinds along the `3` axis, so + contiguous output-row blocks [0:H], [H:2H], [2H:3H] are q, k, v + respectively; the same split applies to the weight's rows and the bias. + pos_embed and rope_cos/sin are computed here, on the host, exactly as + modeling_qwen3_5.py:1090-1108 does -- that is the point of folding them. + """ + p = get + H = cfg.hidden_size + + feeds: dict[str, np.ndarray] = {} + + # Patch embedding: Conv3d weight [embed, C, T, ph, pw] flattens (row-major) + # to [embed, C*T*ph*pw] -- exactly patchify's feature order -- then + # transposes to hexlib's [feat, embed]. + conv_w = p("patch_embed.proj.weight") + feeds["w_patch_embed"] = conv_w.reshape(conv_w.shape[0], -1).T + feeds["b_patch_embed"] = p("patch_embed.proj.bias") + + feeds["pos_embed"] = pos_embed_table(cfg, p("pos_embed.weight")) + cos, sin = rope_tables(cfg) + feeds["rope_cos"] = cos + feeds["rope_sin"] = sin + + for layer in range(cfg.depth): + b = f"blocks.{layer}" + blk = f"blk{layer}" + + qkv_w = p(f"{b}.attn.qkv.weight") # [3H, H] + qkv_b = p(f"{b}.attn.qkv.bias") # [3H] + for i, which in enumerate(("q", "k", "v")): + feeds[f"{blk}.w{which}"] = qkv_w[i * H:(i + 1) * H, :].T + feeds[f"{blk}.b{which}"] = qkv_b[i * H:(i + 1) * H] + + feeds[f"{blk}.wo"] = p(f"{b}.attn.proj.weight").T + feeds[f"{blk}.bo"] = p(f"{b}.attn.proj.bias") + + feeds[f"{blk}.w_fc1"] = p(f"{b}.mlp.linear_fc1.weight").T + feeds[f"{blk}.b_fc1"] = p(f"{b}.mlp.linear_fc1.bias") + feeds[f"{blk}.w_fc2"] = p(f"{b}.mlp.linear_fc2.weight").T + feeds[f"{blk}.b_fc2"] = p(f"{b}.mlp.linear_fc2.bias") + + feeds[f"{blk}.ln1_w"] = p(f"{b}.norm1.weight") + feeds[f"{blk}.ln1_b"] = p(f"{b}.norm1.bias") + feeds[f"{blk}.ln2_w"] = p(f"{b}.norm2.weight") + feeds[f"{blk}.ln2_b"] = p(f"{b}.norm2.bias") + + feeds["m_ln_w"] = p("merger.norm.weight") + feeds["m_ln_b"] = p("merger.norm.bias") + feeds["w_m1"] = p("merger.linear_fc1.weight").T + feeds["b_m1"] = p("merger.linear_fc1.bias") + feeds["w_m2"] = p("merger.linear_fc2.weight").T + feeds["b_m2"] = p("merger.linear_fc2.bias") + + const_names = {t.name for t in graph.tensors.values() if t.const} + missing = const_names - set(feeds) + assert not missing, f"feeds_from_params supplied no feed for: {sorted(missing)}" + extra = set(feeds) - const_names + assert not extra, f"feeds_from_params supplied a feed for non-const tensors: {sorted(extra)}" + + for name, array in feeds.items(): + want_shape = graph.tensor(name).shape + got_shape = tuple(np.asarray(array).shape) + assert got_shape == want_shape, ( + f"feed {name!r} has shape {got_shape}, but the graph declares {want_shape} " + "-- a shape mismatch here is almost always a forgotten transpose" + ) + + return feeds diff --git a/hexlib/runtime/__init__.py b/hexlib/runtime/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/hexlib/runtime/build.py b/hexlib/runtime/build.py new file mode 100644 index 0000000..fc66bef --- /dev/null +++ b/hexlib/runtime/build.py @@ -0,0 +1,821 @@ +# hexlib/runtime/build.py +"""Build the runtime: qaic, the DSP skel, the simulator artifact, the device binary. + +`hexlib/build.py` is untouched — it builds standalone kernel ELFs and its +contract is depended on by the whole existing gate. This is a second builder for +a second kind of artifact, sharing only `toolchain.py`. + +THE LINK RECIPES ARE NOT RECONSTRUCTED. They were recovered by actually +building SDK reference examples on this toolchain and reading back the exact +commands the SDK's own build system used, never guessed: + +- The (now-retired) standalone-qexe recipe was recovered from the SDK + calculator example's own `calculator_q_link.txt` at v75, where it printed + `Sum = 32640 / Pass: 2 Fail: 0` at rev_id 0x00008c75. +- The QuRT-hosted `.so` recipe below (`build_sim_so`, `SIM_SO_LINK_FLAGS`) was + recovered from the SDK's own `libs/run_main_on_hexagon` example's + `test_main_so_link.txt`, `run_main_on_hexagon_sim_link.txt`, and + `sim_cmd_line.txt`, produced by `make hexagon BUILD=Debug DSP_ARCH=v75` in + that example directory (never inside this repo, never checked in — see + `.superpowers/sdd/2026-08-10-silicon-path-runtime/ + investigation-sim-vtcm-and-marshalling.md`). That run demonstrated a real + 8 MiB VTCM acquisition succeeding (`rc=0`, `ptr=0xd9000000`) under a real + QuRT kernel on `hexagon-sim`; the standalone qexe cannot do this (VTCM's + manager object needs real QuRT thread/clock primitives no standalone qexe + can provide — see the same investigation). + +WHY THE STANDALONE QEXE RECIPE IS GONE. `build_sim_qexe`/`SIM_LINK_FLAGS`/ +`SIM_LINK_EXTRAS` used to live here (task 7). They still link and still reach +`hexlib_iface_open`, but `hexlib_iface_start` can NEVER succeed through that +path — VTCM acquisition needs real QuRT, which a standalone +`--force-dynamic` qexe structurally cannot host. Keeping a build path around +that is permanently incapable of the one thing this runtime exists to prove +is worse than removing it: a future reader would have to rediscover, by +hitting the same wall again, that it is a dead end. The QuRT-hosted `.so` +below is the only path that reaches a successful `start()` in simulation. +""" +from __future__ import annotations + +import os +from dataclasses import dataclass + +from hexlib import toolchain as tc + + +class RuntimeBuildError(Exception): + def __init__(self, message: str, output: str = "") -> None: + super().__init__(message) + self.output = output + + +@dataclass(frozen=True) +class QaicOutput: + header: str + stub: str + skel: str + + +def qaic_path(sdk_root: str) -> str: + """qaic lives in a per-platform directory inside the SDK.""" + if os.name == "nt": + return os.path.join(sdk_root, "ipc", "fastrpc", "qaic", "bin", "qaic.exe") + return os.path.join(sdk_root, "ipc", "fastrpc", "qaic", "Ubuntu", "qaic") + + +def qaic_include_dirs(sdk_root: str) -> list[str]: + """AEEStdDef.idl and remote.idl live here.""" + return [os.path.join(sdk_root, "incs"), os.path.join(sdk_root, "incs", "stddef")] + + +HEXLIB_IDL_STEM = "hexlib_iface" + + +def device_skel_so_name(idl_stem: str = HEXLIB_IDL_STEM) -> str: + """The filename FastRPC will `dlopen`, derived rather than chosen. + + THIS WAS A REAL DEFECT AND ONLY SILICON COULD FIND IT. The skel was linked + as `libhexlib_skel.so`, but the URI qaic generates into `hexlib_iface.h` + (`hexlib_iface_URI`, used by `session.c:163`) names the library after the + IDL: `hexlib_iface.idl` -> `libhexlib_iface_skel.so`. On the device + `hexlib_iface_open` failed with rc -2147482618 until the file was pushed + under BOTH names by hand. + + Nothing offline could see it. Stage 1's simulator build LINKS the skel + directly rather than loading it by name, so the filename never participates + -- which is exactly the class of thing this project has learned to bind + structurally instead of asserting. Hence a derivation from the IDL stem, + used by the linker, the artifact packer and the device test alike, so the + three cannot drift apart again. + """ + return f"lib{idl_stem}_skel.so" + + +def run_qaic(idl: str, out_dir: str, sdk_root: str | None = None) -> QaicOutput: + root = sdk_root or tc.default_sdk_root() + if not os.path.isfile(idl): + raise RuntimeBuildError(f"IDL not found: {idl}") + qaic = qaic_path(root) + if not os.path.isfile(qaic): + raise RuntimeBuildError(f"qaic not found: {qaic}") + + os.makedirs(out_dir, exist_ok=True) + cmd = [qaic, "-mdll", "-o", out_dir] + for d in qaic_include_dirs(root): + cmd += ["-I", d] + cmd.append(idl) + + rc, out, err, timed_out = tc.run(cmd, os.environ.copy(), timeout=60) + if timed_out or rc != 0: + raise RuntimeBuildError(f"qaic failed on {idl}", (out + err).strip()) + + stem = os.path.splitext(os.path.basename(idl))[0] + res = QaicOutput( + header=os.path.join(out_dir, f"{stem}.h"), + stub=os.path.join(out_dir, f"{stem}_stub.c"), + skel=os.path.join(out_dir, f"{stem}_skel.c"), + ) + # FAIL CLOSED: qaic exiting 0 without writing the files is a failure, not a + # build we then link and get confusing errors from. Covered by + # test_qaic_exit_zero_without_files_still_raises, which monkeypatches + # tc.run to succeed while writing nothing -- do not delete this as + # "redundant" with the happy-path test; that one can't fail this check. + for f in (res.header, res.stub, res.skel): + if not os.path.isfile(f): + raise RuntimeBuildError( + f"qaic exited 0 but did not produce {f}", (out + err).strip() + ) + return res + + +# ============================================================================ +# The historical standalone-qexe recipe (task 7's SIM_LINK_FLAGS/ +# SIM_LINK_EXTRAS/build_sim_qexe) lived here and is gone -- see the module +# docstring's "WHY THE STANDALONE QEXE RECIPE IS GONE" for why. The recipe +# below replaces it: a QuRT-hosted shared object (see "The simulator +# artifact" further down). +# ============================================================================ + + +def runtime_include_dirs(sdk_root: str, gen_dir: str) -> list[str]: + repo = os.path.dirname(os.path.dirname(os.path.dirname(__file__))) + return [ + gen_dir, + os.path.join(repo, "hexlib", "runtime", "skel"), + os.path.join(repo, "include"), + os.path.join(sdk_root, "incs"), + os.path.join(sdk_root, "incs", "stddef"), + os.path.join(sdk_root, "ipc", "fastrpc", "rpcmem", "inc"), + # `tc.sdk_include_dirs` already appends + # rtos/qurt/compute/include/qurt (needed by skel_vtcm.c's + # `#include "qurt_thread.h"`) and its posix/ sibling. + ] + tc.sdk_include_dirs(sdk_root) + + +def build_skel_lib(kernels: list[str], out_dir: str, + sdk_root: str | None = None) -> str: + """qaic, generate entries, compile skel + kernels, archive. + + `kernels` is a REQUEST, not the full set that gets compiled: genentry's + dispatch table always references every kernel that has a RunnerSpec AND a + directory on disk, regardless of what this function was asked to build, so + every one of those must be compiled into the archive or the later link + fails on an undefined symbol for whichever one is missing. + """ + from hexlib.build import compile_command + from hexlib.exec.runner import SPECS + from hexlib.runtime import genentry + + root = sdk_root or tc.default_sdk_root() + bin_dir = tc.find_toolchain_bin(root) + version = tc.toolchain_version(bin_dir) + if version != tc.TOOLCHAIN_VERSION: + raise RuntimeBuildError( + f"toolchain is {version}, expected {tc.TOOLCHAIN_VERSION} — cycle " + "numbers are not comparable across toolchain versions" + ) + env = tc.toolchain_env(bin_dir) + compiler = os.path.join(bin_dir, tc.exe(tc.COMPILER)) + ar = os.path.join(bin_dir, tc.exe("hexagon-ar")) + + repo = os.path.dirname(os.path.dirname(os.path.dirname(__file__))) + gen = os.path.join(out_dir, "gen") + os.makedirs(gen, exist_ok=True) + + idl = os.path.join(repo, "hexlib", "runtime", "idl", "hexlib_iface.idl") + qa = run_qaic(idl, gen, root) + + # The REPO root: spec.kernel_dir already carries the "kernels/" prefix. + entries = genentry.generate(repo, gen) + + used_kernels = sorted({ + os.path.basename(spec.kernel_dir) + for spec in SPECS.values() + if os.path.isdir(os.path.join(repo, spec.kernel_dir)) + }) + all_kernels = sorted(set(kernels) | set(used_kernels)) + + skel_dir = os.path.join(repo, "hexlib", "runtime", "skel") + # Each entry below is (source path, object basename, this file's OWN extra + # include dirs). Two things go wrong if a single shared include list and a + # single `basename(s).replace(".c", ".o")` are used for all of these, both + # confirmed by actually hitting them: + # 1. Every kernel directory has a "kernel_api.h" with a DIFFERENT + # declared function inside, so a generated _entry.c file must see + # ONLY its own kernel's directory on the include path -- putting + # every kernel dir on one shared list let scale_fp16_entry.c's + # `#include "kernel_api.h"` resolve to add_fp16/kernel_api.h instead + # (whichever kernel sorts first), failing with "call to undeclared + # function 'scale_fp16'". + # 2. Every kernel's implementation file is literally named "kernel.c", + # so compiling them all to a name derived from their own basename + # collided on disk: each subsequent kernel.o silently overwrote the + # previous one, and the archive ended up with only the + # alphabetically-last kernel's code (transpose_th_fp16) -- `nm` on + # the resulting archive showed the other three kernels' *_entry.o + # referencing their own kernel function as undefined. + sim_shims = os.path.join(repo, "hexlib", "runtime", "simhost", "sim_shims.c") + base = [ + (qa.skel, "hexlib_iface_skel.o", []), + (os.path.join(skel_dir, "skel.c"), "skel.o", []), + (os.path.join(skel_dir, "skel_bufs.c"), "skel_bufs.o", []), + (os.path.join(skel_dir, "skel_vtcm.c"), "skel_vtcm.o", []), + (os.path.join(skel_dir, "skel_dispatch.c"), "skel_dispatch.o", []), + (sim_shims, "sim_shims.o", []), + ] + for e in entries: + stem = os.path.basename(e)[:-len(".c")] + if stem.endswith("_entry"): + k = stem[: -len("_entry")] + base.append((e, f"{stem}.o", [os.path.join(repo, "kernels", k)])) + else: + base.append((e, f"{stem}.o", [])) # hexlib_kernel_table.c + for k in all_kernels: + kdir = os.path.join(repo, "kernels", k) + base.append((os.path.join(kdir, "kernel.c"), f"{k}_kernel.o", [kdir])) + hand = os.path.join(kdir, "dsp_entry.c") + if os.path.isfile(hand): + base.append((hand, f"{k}_dsp_entry.o", [kdir])) + + common_includes = runtime_include_dirs(root, gen) + + objs = [] + for s, obj_name, extra in base: + o = os.path.join(out_dir, obj_name) + cmd = compile_command( + compiler, [s], o, ["hvx"], common_includes + extra, compile_only=True + ) + # -fpic: this archive is only ever linked into build_sim_so's shared + # object below (a real device skel is ALSO always built as a shared + # object loaded by qaic's own dlopen machinery — this is not a + # simulator-only concession, it is the same code shape a device skel + # needs). PIC objects link into a --force-dynamic executable too, so + # this does not foreclose reusing the archive for a non-PIC link + # later if one is ever needed. + cmd.insert(1, "-fpic") + rc, out, err, to = tc.run(cmd, env, timeout=tc.SIM_TIMEOUT_S) + if to or rc != 0: + raise RuntimeBuildError(f"compile failed: {s}", (out + err).strip()) + objs.append(o) + + lib = os.path.join(out_dir, "libhexlib_skel.a") + rc, out, err, to = tc.run([ar, "rcs", lib] + objs, env, timeout=60) + if to or rc != 0 or not os.path.isfile(lib): + raise RuntimeBuildError("archiving libhexlib_skel.a failed", + (out + err).strip()) + return lib + + +# ============================================================================ +# The simulator artifact: a QuRT-hosted shared object, dlopen'd by the SDK's +# OWN prebuilt `run_main_on_hexagon_sim` under a real booted QuRT kernel. +# +# WHY A SHARED OBJECT, NOT A STANDALONE QEXE. Task 7's standalone +# `--force-dynamic` qexe (gone now, see the module docstring) never links in +# the VTCM manager's weak symbols with real definitions, and forcing that +# object in demands real QuRT thread/clock primitives a standalone qexe +# cannot provide (investigation-sim-vtcm-and-marshalling.md, Q1). The SDK's +# OWN way to run arbitrary code under a real QuRT kernel on `hexagon-sim` is +# `libs/run_main_on_hexagon`: a prebuilt host executable +# (`run_main_on_hexagon_sim`, already shipped for this exact toolchain/arch +# combination at +# `$SDK/libs/run_main_on_hexagon/ship/hexagon_toolv19_v75/run_main_on_hexagon_sim`) +# that links real `libqurt.a` + `rtld.a` + `test_util.a` + `atomic.a` +# `--whole-archive` (confirmed by reading its own recovered link line), boots +# a real QuRT kernel under `hexagon-sim` via `runelf.pbn` + `osam.cfg`, then +# `dlopen()`s a user-supplied `.so` and calls its `main()`. `test_main.so` is +# the SDK's own reference payload for this; `build_sim_so` below produces +# hexlib's own payload the identical way, recovered from that same example's +# `test_main_so_link.txt` (`-fpic -shared -Wl,-Bsymbolic -lc`, nothing else -- +# rtld/test_util/atomic/libqurt are NOT relinked into the .so, because they +# are already inside the host process that dlopen's it, and its own -E/ +# --export-dynamic-equivalent link makes their symbols visible to the .so at +# dlopen time). This is what makes VTCM acquisition, real: HAP_compute_res.h's +# weak `compute_resource_query_VTCM` pointer, unresolved (null) in a +# standalone qexe, resolves for real here against test_util.a's +# sysmon_vtcm_mgr_client.o -- and its hard qurt_thread_get_id/ +# qurt_sysclock_get_hw_ticks/etc. dependencies resolve against the real +# libqurt.a already linked into the host process. Demonstrated end to end in +# the investigation: rc=0, an 8 MiB query, and a real acquired pointer +# (0xd9000000) inside the QuRT kernel's own reported TCM_PHYSPOOL range. +# +# ========================================================================== +# WHAT A SIMULATOR RUN OF THIS .SO DOES NOT PROVE -- READ THIS FIRST. +# +# simhost.c (compiled into this .so) still calls hexlib_iface_open/_start/ +# _mmap/_invoke/_stop/_close as PLAIN C FUNCTIONS, bound directly to skel.c's +# definitions -- both are compiled into the SAME .so, so this is still an +# ordinary intra-module call, not a qaic-marshalled one. The qaic-generated +# stub (hexlib_iface_stub.c) is DELIBERATELY NOT ONE OF THE SOURCES LINKED +# HERE, for the identical reason as before: it defines the exact same +# function names as skel.c's DSP-side implementation (confirmed by running +# qaic and reading both generated files back), so linking both into one +# module is a duplicate-symbol error, not merely redundant. Packaging the +# host as a shared object rather than a standalone executable does not +# change this -- it changes HOW VTCM's own weak symbols get resolved (now +# dynamically, against the host process, at dlopen time), not whether the +# qaic stub is linked (it still is not). +# +# CONSEQUENCE: a simulator run through this .so exercises hexlib's OWN code +# -- batch parsing, the buffer table, the kernel dispatch table, kernel +# correctness, PCYCLE accounting, and now VTCM acquisition -- but it does NOT +# exercise qaic's argument marshaling/demarshaling at all. That is a real gap +# against this project's own design spec, which describes the simulator path +# as exercising "a qaic stub/skel invoke": what actually happens is a plain +# function call, and the marshaling layer is completely bypassed. Marshaling +# is only exercised on a real device, where the stub and skel genuinely live +# in separate processes and the call cannot avoid the wire. +# ========================================================================== + +SIM_SO_LINK_FLAGS = [ + "-G0", + "-Wl,--defsym=ISDB_TRUSTED_FLAG=2", + "-Wl,--defsym=ISDB_SECURE_FLAG=2", + "-Wl,--no-threads", + "-fpic", + "-shared", + "-Wl,-Bsymbolic", + "-lc", +] + +# SIM_V_ARCH: the SDK's OWN per-arch simulator revision string, recovered +# verbatim from build/make.d.ext/hexagon/defines_hexagon_1_9.min -- NOT a +# mechanical "na_1" suffix rule (v68 and v81 use different suffixes +# entirely), so this is a lookup, not a format string. +_SIM_V_ARCH = { + "v68": "v68n_1024", + "v69": "v69na", + "v73": "v73na_1", + "v75": "v75na_1", + "v79": "v79na_1", + "v81": "v81qa_1", +} + + +def sim_v_arch(arch: str = tc.DSP_ARCH) -> str: + return _SIM_V_ARCH.get(arch, arch) + + +def _run_main_on_hexagon_dir(sdk_root: str, arch: str = tc.DSP_ARCH) -> str: + return os.path.join( + sdk_root, "libs", "run_main_on_hexagon", "ship", + f"hexagon_toolv19_{arch}", + ) + + +def run_main_on_hexagon_sim_path(sdk_root: str, arch: str = tc.DSP_ARCH) -> str: + """The SDK's OWN prebuilt QuRT-hosted launcher. Referenced by path from + the SDK, never copied into this repo (the SDK is license-restricted).""" + return os.path.join(_run_main_on_hexagon_dir(sdk_root, arch), + "run_main_on_hexagon_sim") + + +def runelf_pbn_path(sdk_root: str, arch: str = tc.DSP_ARCH) -> str: + return os.path.join(sdk_root, "rtos", "qurt", f"compute{arch}", + "sdksim_bin", "runelf.pbn") + + +def build_sim_so(out_dir: str, sdk_root: str | None = None) -> str: + """Link the simulator host + skel into a QuRT-hosted shared object. + + Recovered from the SDK's own `libs/run_main_on_hexagon` example's + `test_main_so_link.txt` (see the module-level comment above this + function for the full recipe provenance). + """ + root = sdk_root or tc.default_sdk_root() + bin_dir = tc.find_toolchain_bin(root) + env = tc.toolchain_env(bin_dir) + compiler = os.path.join(bin_dir, tc.exe(tc.COMPILER)) + gen = os.path.join(out_dir, "gen") + + lib = os.path.join(out_dir, "libhexlib_skel.a") + if not os.path.isfile(lib): + raise RuntimeBuildError(f"build_skel_lib must run first: {lib} missing") + + repo = os.path.dirname(os.path.dirname(os.path.dirname(__file__))) + simhost_c = os.path.join(repo, "hexlib", "runtime", "simhost", "simhost.c") + + so = os.path.join(out_dir, "hexlib_sim.so") + cmd = [compiler] + tc.cflags_for_caps(["hvx"]) + SIM_SO_LINK_FLAGS + for d in runtime_include_dirs(root, gen): + cmd.append(f"-I{d}") + cmd += [ + "-Wl,-Map=" + so + ".map", + "-Wl,-soname=" + os.path.basename(so), + "-o", so, + "-Wl,--start-group", simhost_c, lib, "-Wl,--end-group", + ] + + rc, out, err, to = tc.run(cmd, env, timeout=tc.SIM_TIMEOUT_S) + if to or rc != 0 or not os.path.isfile(so): + raise RuntimeBuildError("linking hexlib_sim.so failed", (out + err).strip()) + return so + + +def write_qurt_sim_configs(out_dir: str, sdk_root: str | None = None, + tools_root: str | None = None, + arch: str = tc.DSP_ARCH) -> tuple[str, str]: + """Write osam.cfg and q6ss.cfg into `out_dir` (never the source tree). + + Both filenames are matched BY NAME in `.gitignore` (`osam.cfg`, `q6ss.cfg` + -- not a blanket `*.cfg`, so a config someone actually means to commit + elsewhere is not silently swallowed). That rule is what keeps these out + of the repo; `out_dir` living inside a pytest `tmp_path` today is + incidental, not the actual protection -- a caller is free to pass a work + dir INSIDE the repo, and `.gitignore` is what stops the resulting files + from being committed, not where the caller happened to point `out_dir`. + + NOT VENDORING: each file is one or two lines naming an existing SDK + artifact BY PATH (the QuRT debugger model, and two cosim timer/interrupt + controller shims); no SDK file's bytes are copied. Recovered verbatim + from rtos/qurt/qurt_libs_priv.min's own $(OBJ_DIR)/osam.cfg and + $(OBJ_DIR)/q6ss.cfg rules (Windows branch: this project only targets + Windows per toolchain.py/CLAUDE.md conventions already established + elsewhere in this codebase). + """ + root = sdk_root or tc.default_sdk_root() + bin_dir = tc.find_toolchain_bin(root) + tools = tools_root or os.path.dirname(os.path.dirname(bin_dir)) + os.makedirs(out_dir, exist_ok=True) + + is_arm64 = os.environ.get("PROCESSOR_ARCHITEW6432") == "ARM64" + debugger_dir = "Win_arm64" if is_arm64 else "Win" + qurt_model = os.path.join(root, "rtos", "qurt", f"compute{arch}", + "debugger", debugger_dir, "qurt_model.dll") + if not os.path.isfile(qurt_model): + raise RuntimeBuildError(f"QuRT debugger model not found: {qurt_model}") + osam_cfg = os.path.join(out_dir, "osam.cfg") + with open(osam_cfg, "w") as f: + f.write(qurt_model + "\n") + + iss_dir = os.path.join(tools, "Tools", "lib", "iss") + qtimer = os.path.join(iss_dir, "qtimer.dll") + l2vic = os.path.join(iss_dir, "l2vic.dll") + for f in (qtimer, l2vic): + if not os.path.isfile(f): + raise RuntimeBuildError(f"simulator cosim module not found: {f}") + q6ss_cfg = os.path.join(out_dir, "q6ss.cfg") + with open(q6ss_cfg, "w") as f: + f.write(f"{qtimer} --csr_base=0xFC900000 --irq_p=3 --freq=19200000 --cnttid=1\n") + f.write(f"{l2vic} 32 0xFC910000\n") + + return osam_cfg, q6ss_cfg + + +def sim_qurt_command(out_dir: str, so_path: str, sdk_root: str | None = None, + extra_args: tuple[str, ...] = (), + arch: str = tc.DSP_ARCH) -> list[str]: + """Assemble the full hexagon-sim invocation that boots a real QuRT + kernel, then dlopen's `so_path` (which must already sit inside + `out_dir`) and calls its main(argc, argv) with `extra_args`. + + Recovered from the SDK's own `libs/run_main_on_hexagon` example's + `sim_cmd_line.txt` at v75/toolv19 -- the exact shape its own + QURT_QEXE_EXEC/QEXE_EXEC make rules produce (rtos/qurt/ + qurt_libs_priv.min), not reconstructed. + + Pure with respect to the filesystem except for locating hexagon-sim and + the two prebuilt SDK artifacts by path -- assertable without running it, + like sim.py's own sim_command(). + + CALLER MUST SET THE SUBPROCESS cwd TO `out_dir` -- CONFIRMED EMPIRICALLY, + NOT A GUESS. Under this QuRT-hosted launch, relative fopen() READS inside + the dlopen'd .so (hexlib_batch.bin, hexlib_in.bin) resolve through + `--usefs out_dir` correctly, but relative fopen(..., "wb") WRITES + (hexlib_rsp.bin, hexlib_out.bin) land in the LAUNCHING PROCESS'S OWN + working directory instead -- QuRT's own POSIX filesystem layer does not + consult --usefs the same way for file creation. This function cannot fix + that from here (it only assembles argv; it does not spawn the process), + so whatever runs this command (e.g. `subprocess.run(cmd, cwd=out_dir, + ...)`, or `tc.run` -- which does not itself take a cwd -- called from a + caller that has already os.chdir'd or otherwise pinned its own cwd to + `out_dir`) MUST set the subprocess's cwd to this same `out_dir`, or the + response/output files will not be where the batch/input files were + written. See hexlib/runtime/simhost/simhost.c's own header comment for + the full empirical writeup. + """ + root = sdk_root or tc.default_sdk_root() + bin_dir = tc.find_toolchain_bin(root) + sim_exe = os.path.join(bin_dir, tc.exe("hexagon-sim")) + + osam_cfg = os.path.join(out_dir, "osam.cfg") + q6ss_cfg = os.path.join(out_dir, "q6ss.cfg") + run_main = run_main_on_hexagon_sim_path(root, arch) + runelf = runelf_pbn_path(root, arch) + + cmd = [ + sim_exe, f"-m{sim_v_arch(arch)}", "--simulated_returnval", + "--usefs", out_dir, + "--pmu_statsfile", os.path.join(out_dir, "pmu_stats.txt"), + "--cosim_file", q6ss_cfg, + "--l2tcm_base", "0xd800", + "--rtos", osam_cfg, + runelf, "--", + run_main, "--", + os.path.basename(so_path), + ] + cmd += list(extra_args) + return cmd + + +# ============================================================================ +# STAGE 2 GATE — the aarch64 CPU side and the device skel .so. Built, never +# run: no phone is available, so this proves only "it builds and is the right +# machine type" (aarch64 for hexlib_run, Hexagon for libhexlib_skel.so). +# Stage 3 is the first thing that ever executes either artifact. +# +# THE STUB/SKEL SPLIT INVERTS FROM THE SIMULATOR, AND THIS IS THE CRUX OF THIS +# TASK. build_sim_so (above) deliberately never links hexlib_iface_stub.c, +# because it defines the exact same symbol names as skel.c and both would +# have to live in one process/module. On a device they are two SEPARATE +# binaries, so the arrangement inverts: +# - hexlib_run (aarch64) links the qaic-generated STUB +# (hexlib_iface_stub.c) plus hexlib/runtime/host/*.c, against -ldl -llog. +# Calling hexlib_iface_invoke() from it marshals arguments into a +# remote_arg[] and calls remote_handle64_invoke() for real (see +# host/main.c's own header comment). +# - libhexlib_skel.so (Hexagon, -shared -fPIC) contains the qaic SKEL +# (hexlib_iface_skel.c) plus skel.c/skel_bufs.c/skel_vtcm.c/ +# skel_dispatch.c, the generated per-kernel entry points, and the +# kernels themselves -- loaded by the FastRPC framework on the DSP, +# which calls INTO it, never the reverse. +# Get this backwards (stub in the skel .so, or skel code in the aarch64 +# binary) and the result is either a duplicate-symbol link error or a binary +# that can never marshal at all. THE DEVICE PATH IS THE ONLY PATH IN THIS +# PROJECT THAT WILL EVER EXERCISE QAIC'S REAL MARSHALLING — every simulator +# run before this task (build_sim_so, above) calls skel.c as plain C +# functions in one address space; see host/main.c's own file header for the +# same point made from the other side of the wire. +# ============================================================================ + + +def ndk_bin_dir(sdk_root: str) -> str: + """The NDK's prebuilt toolchain `bin` directory for the CURRENT host OS. + VERIFIED against the actual installed SDK (not assumed): only a + `windows-x86_64` prebuilt tree exists there, so that is the only + non-Windows-host branch this repo could ever actually exercise, but the + `linux-x86_64` name is the NDK's own documented convention, kept for a + contributor on a different host.""" + host_tag = "windows-x86_64" if os.name == "nt" else "linux-x86_64" + return os.path.join(tc.ndk_root(sdk_root), "toolchains", "llvm", "prebuilt", + host_tag, "bin") + + +def ndk_clang(sdk_root: str | None = None) -> str: + """Path to the NDK's aarch64 Android clang driver, pinned to tc.ANDROID_API. + + ON WINDOWS THIS MUST BE THE `.cmd` FORM, NOT THE BARE NAME — VERIFIED, NOT + GUESSED. The bare `aarch64-linux-android-clang` next to it is a Bourne + shell script (confirmed with `file`), which a plain `subprocess.run([...])` + call (no shell) cannot execute on Windows at all; `subprocess.run` against + the `.cmd` wrapper was confirmed to actually run and print a real clang + version banner. `tc.run` (toolchain.py) never sets `shell=True`, so the + bare name would fail with "cannot execute" on every Windows caller. + """ + root = sdk_root or tc.default_sdk_root() + bin_dir = ndk_bin_dir(root) + name = f"aarch64-linux-android{tc.ANDROID_API}-clang" + if os.name == "nt": + name += ".cmd" + return os.path.join(bin_dir, name) + + +# Recovered from the SDK's OWN shared-library link recipe for this exact +# toolchain version -- $SDK/build/make.d.ext/hexagon/defines_hexagon_1_9.min's +# `DLL_LD_FLAGS` (1_9 is the "hexagon_toolv19" family TOOLCHAIN_VERSION 19.0.04 +# belongs to) -- NOT SIM_SO_LINK_FLAGS above, which is a DIFFERENT recipe for a +# different situation. SIM_SO_LINK_FLAGS builds a .so meant to be dlopen'd into +# an ALREADY-RUNNING QuRT host process (run_main_on_hexagon_sim) that already +# has its own allocator; DLL_LD_FLAGS is the SDK's general-purpose "this is a +# Hexagon shared library" recipe, and it carries five `--wrap=` flags +# SIM_SO_LINK_FLAGS does not: `malloc`/`calloc`/`free`/`realloc`/`memalign`, +# the PD heap-interposition every real Hexagon DLL gets so its allocations are +# routed through the loading process's own signed/unsigned-PD heap manager +# rather than a bare libc allocator. A real FastRPC-loaded skel needs that; +# the sim .so does not (it never leaves the one host process it was dlopen'd +# into). `-Wl,--no-undefined`/`-z defs` is deliberately absent, exactly as in +# the SDK's own recipe: symbols like HAP_mmap2 and the HAP_compute_res_* +# family are resolved dynamically, at dlopen time, against the framework +# already running in the DSP process the skel loads into -- never statically +# linked here, on device OR on the simulator (see sim_shims.c's own header for +# the simulator side of that same fact). +DEVICE_SKEL_LINK_FLAGS = [ + "-G0", + "-Wl,--defsym=ISDB_TRUSTED_FLAG=2", + "-Wl,--defsym=ISDB_SECURE_FLAG=2", + "-Wl,--no-threads", + "-fpic", + "-shared", + "-Wl,-Bsymbolic", + "-Wl,--wrap=malloc", + "-Wl,--wrap=calloc", + "-Wl,--wrap=free", + "-Wl,--wrap=realloc", + "-Wl,--wrap=memalign", + "-lc", +] + + +def _build_device_skel_so(out_dir: str, root: str, gen: str, qa: QaicOutput) -> str: + """Compile the skel + kernels + generated entries into a real Hexagon + SHARED OBJECT (named by `device_skel_so_name()`, i.e. + `libhexlib_iface_skel.so`), the device counterpart of + `build_skel_lib`'s `.a` above. Deliberately NOT a thin wrapper around + `build_skel_lib` -- the object sets genuinely differ (see below), and + `build_skel_lib`'s own `-fpic` insertion is asserted, by literal source + text, by `test_runtime_sim_build.py:: + test_build_skel_lib_compiles_position_independent_code`; reshaping that + function to share code with this one is out of scope for this task and + risks that assertion for no benefit, since the compiled objects are not + even byte-identical between the two paths (see next paragraph). + + `sim_shims.c` IS DELIBERATELY NOT COMPILED IN HERE. It exists only to + backfill `HAP_mmap2`/`HAP_munmap2` on top of `test_util.a`'s + simulator-only, int-length `HAP_mmap` (see `simhost/sim_shims.c`'s own + header) -- this build never links `test_util.a` at all, and a real + device's FastRPC framework resolves `HAP_mmap2` for real, dynamically, + inside the signed/unsigned PD process this .so is loaded into. + """ + from hexlib.build import compile_command + from hexlib.exec.runner import SPECS + from hexlib.runtime import genentry + + bin_dir = tc.find_toolchain_bin(root) + version = tc.toolchain_version(bin_dir) + if version != tc.TOOLCHAIN_VERSION: + raise RuntimeBuildError( + f"toolchain is {version}, expected {tc.TOOLCHAIN_VERSION} — cycle " + "numbers are not comparable across toolchain versions" + ) + env = tc.toolchain_env(bin_dir) + compiler = os.path.join(bin_dir, tc.exe(tc.COMPILER)) + + repo = os.path.dirname(os.path.dirname(os.path.dirname(__file__))) + entries = genentry.generate(repo, gen) + + used_kernels = sorted({ + os.path.basename(spec.kernel_dir) + for spec in SPECS.values() + if os.path.isdir(os.path.join(repo, spec.kernel_dir)) + }) + + skel_dir = os.path.join(repo, "hexlib", "runtime", "skel") + # "dev_"-prefixed object basenames: this compiles into the SAME out_dir + # build_skel_lib/build_sim_so may also use for a sim artifact, and their + # object names (skel.o, skel_bufs.o, ...) would otherwise collide on disk + # with these PIC-but-differently-sourced objects (no sim_shims.o here at + # all -- seeded from a genuinely different source-file set, not just a + # different flag). + base = [ + (qa.skel, "dev_hexlib_iface_skel.o", []), + (os.path.join(skel_dir, "skel.c"), "dev_skel.o", []), + (os.path.join(skel_dir, "skel_bufs.c"), "dev_skel_bufs.o", []), + (os.path.join(skel_dir, "skel_vtcm.c"), "dev_skel_vtcm.o", []), + (os.path.join(skel_dir, "skel_dispatch.c"), "dev_skel_dispatch.o", []), + ] + for e in entries: + stem = os.path.basename(e)[:-len(".c")] + if stem.endswith("_entry"): + k = stem[: -len("_entry")] + base.append((e, f"dev_{stem}.o", [os.path.join(repo, "kernels", k)])) + else: + base.append((e, f"dev_{stem}.o", [])) # hexlib_kernel_table.c + for k in used_kernels: + kdir = os.path.join(repo, "kernels", k) + base.append((os.path.join(kdir, "kernel.c"), f"dev_{k}_kernel.o", [kdir])) + hand = os.path.join(kdir, "dsp_entry.c") + if os.path.isfile(hand): + base.append((hand, f"dev_{k}_dsp_entry.o", [kdir])) + + common_includes = runtime_include_dirs(root, gen) + + objs = [] + for s, obj_name, extra in base: + o = os.path.join(out_dir, obj_name) + cmd = compile_command( + compiler, [s], o, ["hvx"], common_includes + extra, compile_only=True + ) + cmd.insert(1, "-fpic") # required for a -shared link, same as build_skel_lib. + rc, out, err, to = tc.run(cmd, env, timeout=tc.SIM_TIMEOUT_S) + if to or rc != 0: + raise RuntimeBuildError(f"device skel compile failed: {s}", (out + err).strip()) + objs.append(o) + + # LIB_HEXAGON, from the SAME defines_hexagon_1_9.min recipe DEVICE_SKEL_LINK_FLAGS + # is recovered from: "$(HEXAGON_LIB_DIR)/$(V_ARCH)/G0/libhexagon.a", and its + # own comment notes the linker only pulls symbols from it if something else + # in the link needs them -- so including it unconditionally is what the + # SDK's own recipe does, not an addition of convenience. + tools_root = os.path.dirname(os.path.dirname(bin_dir)) + lib_hexagon = os.path.join(tools_root, "Tools", "target", "hexagon", "lib", + tc.DSP_ARCH, "G0", "libhexagon.a") + if not os.path.isfile(lib_hexagon): + raise RuntimeBuildError(f"libhexagon.a not found: {lib_hexagon}") + + so = os.path.join(out_dir, device_skel_so_name()) + cmd = [compiler] + tc.cflags_for_caps(["hvx"]) + DEVICE_SKEL_LINK_FLAGS + cmd += [ + "-Wl,-Map=" + so + ".map", + "-Wl,-soname=" + os.path.basename(so), + "-o", so, + "-Wl,--start-group", + ] + objs + [lib_hexagon, "-Wl,--end-group"] + + rc, out, err, to = tc.run(cmd, env, timeout=tc.SIM_TIMEOUT_S) + if to or rc != 0 or not os.path.isfile(so): + raise RuntimeBuildError(f"linking {os.path.basename(so)} failed", + (out + err).strip()) + return so + + +def build_device_binary(out_dir: str, sdk_root: str | None = None) -> str: + """Cross-compile `hexlib_run` for Android aarch64, and (as a side effect) + `libhexlib_skel.so` for the Hexagon device, both into `out_dir`. Returns + the path to `hexlib_run`. + + NEITHER ARTIFACT IS EVER RUN HERE — no device is available (see the + module-level "STAGE 2 GATE" comment above). This function's entire job is + "it builds, and it is the right machine type"; stage 3 is the first thing + that ever executes either one. + """ + root = sdk_root or tc.default_sdk_root() + os.makedirs(out_dir, exist_ok=True) + gen = os.path.join(out_dir, "gen") + os.makedirs(gen, exist_ok=True) + + repo = os.path.dirname(os.path.dirname(os.path.dirname(__file__))) + idl = os.path.join(repo, "hexlib", "runtime", "idl", "hexlib_iface.idl") + qa = run_qaic(idl, gen, root) + + # The Hexagon side, built first: a failure here (e.g. a bad SDK path) + # surfaces before any aarch64 work is wasted. + _build_device_skel_so(out_dir, root, gen, qa) + + # The aarch64 side: the qaic STUB (never the skel) plus every host/*.c + # file (Task 9). See the module-level comment for why this is the + # opposite arrangement from the Hexagon side. + clang = ndk_clang(root) + if not os.path.isfile(clang): + raise RuntimeBuildError(f"NDK clang not found: {clang}") + + host_dir = os.path.join(repo, "hexlib", "runtime", "host") + skel_dir = os.path.join(repo, "hexlib", "runtime", "skel") + sources = [ + qa.stub, + os.path.join(host_dir, "driver.c"), + os.path.join(host_dir, "session.c"), + os.path.join(host_dir, "buffers.c"), + os.path.join(host_dir, "main.c"), + ] + # `$SDK/incs` (remote.h, AEEStdDef.h -- needed by the qaic-generated + # header and stub), `$SDK/incs/stddef`, `$SDK/ipc/fastrpc/rpcmem/inc` + # (rpcmem.h, buffers.c), `gen` (hexlib_iface.h), `host_dir` + # (hexlib_host.h), and `skel_dir` (hexlib_dsp.h -- session.c/buffers.c + # both include it for the wire structs, purely as a header; no skel .c + # file is compiled into this binary). + includes = [ + gen, + host_dir, + skel_dir, + os.path.join(root, "incs"), + os.path.join(root, "incs", "stddef"), + os.path.join(root, "ipc", "fastrpc", "rpcmem", "inc"), + ] + + # NO libcdsprpc.so IMPORT LIBRARY HERE -- DELIBERATELY. The qaic-generated + # stub (hexlib_iface_stub.c, never hand-edited) calls + # remote_handle64_open/_invoke/_close directly, as ordinary strong + # `extern` functions declared in (confirmed: no `weak` + # attribute there, __QAIC_REMOTE(ff) defaults to identity). The first + # version of this function linked against the SDK's own aarch64 import + # stub (ipc/fastrpc/remote/ship/android_aarch64/libcdsprpc.so) to satisfy + # that -- it linked, but it reintroduced the exact failure mode + # driver.c's own "WHY DLOPEN AND NOT A LINK-TIME DEPENDENCY" comment + # exists to avoid: a device missing libcdsprpc.so would fail to even + # start hexlib_run (a dynamic-linker load error, before main() runs), + # never reaching hexlib_drv_init()'s readable message at all. Fixed at + # the source instead: driver.c now DEFINES remote_handle64_open/_invoke/ + # _close itself, as thin forwarders to the hexlib_remote_handle64_* + # function pointers it already dlsym's -- see driver.c's own comment on + # them. That satisfies the stub's link-time reference without ever + # linking libcdsprpc.so at build time, so the driver stays exclusively + # dlopen'd, exactly as designed. + exe = os.path.join(out_dir, "hexlib_run") + # -Wall -Werror for the SAME reason tc.HVX_CFLAGS carries them (see the long + # comment there): every caller of tc.run checks only `rc != 0`, so a warning + # is emitted and discarded. This side of the wire assembles the batch blob + # and the buffer table by hand in host/*.c, which is exactly the kind of + # code where a pointer/qualifier diagnostic is the only automatic notice + # that two things were swapped. MEASURED before enabling: this link + # produces ZERO warnings under -Wall on the pinned NDK (r25c, API 33), so + # nothing is being grandfathered in. NOT tc.HVX_CFLAGS itself -- that list + # is Hexagon-specific (-mv75/-mhvx) and means nothing to an aarch64 clang. + cmd = [clang, "-O2", "-Wall", "-Werror"] + for d in includes: + cmd.append(f"-I{d}") + cmd += sources + cmd += ["-o", exe, "-ldl", "-llog"] + + rc, out, err, to = tc.run(cmd, os.environ.copy(), timeout=tc.SIM_TIMEOUT_S) + if to or rc != 0 or not os.path.isfile(exe): + raise RuntimeBuildError("linking hexlib_run failed", (out + err).strip()) + return exe diff --git a/hexlib/runtime/genentry.py b/hexlib/runtime/genentry.py new file mode 100644 index 0000000..4df71c9 --- /dev/null +++ b/hexlib/runtime/genentry.py @@ -0,0 +1,455 @@ +# hexlib/runtime/genentry.py +"""Emit each kernel's DSP entry point, and the dispatch table, from RunnerSpec. + +WHY GENERATED WHEN harness.c IS HAND-WRITTEN. The harness DECIDES CORRECTNESS, so +a generated one would be a harness nobody read. An argument unpacker decides +nothing — it only marshals — and `RunnerSpec` already declares the mapping for the +file-based path. Generating from the same declaration is what stops the two +transports disagreeing about argument order, which is a class of bug that +produces a plausible wrong answer rather than a failure. + +THE ESCAPE HATCH IS REAL. A kernel directory containing its own `dsp_entry.c` +keeps it; nothing is generated for that kernel. So a mapping the declaration +cannot express is written by hand, visibly, rather than by bending the +declaration. + +KIND IDS ARE EXPLICIT AND ORDERED. They cross the wire, so they must not depend +on dict iteration order: a silent renumber sends every op to the wrong kernel. +Appending is safe; reordering is not. + +`KIND_ID` BELOW IS THE ONE SOURCE OF TRUTH FOR THOSE IDS, AND THE COPIES ARE +TEST-BOUND. It is hand-maintained (nothing derives it from the op registry, +whatever an earlier draft of the design spec claimed), and there are two other +places the same numbers appear: the generated DSP dispatch table, which +`emit_table` below writes straight from this dict, and one `#define +HEXLIB_KIND_SCALE 9u` in `hexlib/runtime/host/main.c`, which is a hand-copy +because the host binary has no generated header to read. Three tests hold that +together, because a wrong id is not a compile error and not a crash: +`test_host_source.py::test_the_hosts_scale_kind_id_is_the_same_number_the_dsp_ +dispatches_on` binds main.c's `#define` to this dict (mutating either fails), +`test_runtime_genentry.py::test_the_shipped_kind_ids_never_move` freezes the 11 +shipped values against a renumber, and +`::test_every_kind_a_COMPILED_PLAN_can_contain_has_a_wire_id` checks the table +covers every kind a plan can actually contain (the registry's 13 minus +`fuse.FUSABLE_ACTS`, which fusion absorbs into `matmul_epilogue`). + +DTYPES ARE CHECKED PER BUFFER, ALWAYS -- NOT ONLY WHEN `requires` MENTIONS THEM. +`hexlib_args` carries `dtype[HEXLIB_MAX_BUFS]`, filled from the same `DTYPE_ID` +table `hexlib/runtime/wire.py` serializes with, and each buffer is about to be +cast to the C type this spec DECLARES for it. So each cast is guarded by the +matching check: a batch declaring a `scale` input as fp32 would otherwise be read +through `const hexlib_hf *` at half stride -- half the tensor, HEXLIB_DSP_OK, a +plausible wrong answer. That hazard is named in `emit_entry`'s own docstring and +was previously guarded on this side only for whichever single buffer a `requires` +entry happened to mention. + +WHAT `requires` CAN AND CANNOT VERIFY HERE. A `("dtype", ...)` requirement lands +on the per-buffer check just described -- and, because the only serializer there +is (`hexlib/exec/dsp.py`) fills that field from `spec.out_dtype`, the check +compares the spec with itself on that path and passes by construction. It is +genuinely reachable from a hand-built batch (main.c, `run_raw`, a future +planner), which is why it is emitted; what it CANNOT do is police the attr a +caller asked for. That is `RunnerSpec.check_requires`'s job, on the host, and +both transports now call it. `hexlib_args` carries no field at all for a +permutation, a shape, or any other op-level attribute, so a `("perm", ...)` +requirement cannot be checked here in any form: writing an `if` that always +passes would be a check that protects nothing, so none is emitted -- only a +comment saying so. +""" +from __future__ import annotations + +import os +from typing import Sequence + +from hexlib.exec.runner import RunnerSpec, Scalar, WIRE_RAW +from hexlib.runtime.wire import DTYPE_ID, LAYOUT_ID + +KIND_ID: dict[str, int] = { + "add": 1, + "cast": 2, + "layernorm": 3, + "matmul": 4, + "matmul_epilogue": 5, + "patchify": 6, + "reshape": 7, + "rope_2d": 8, + "scale": 9, + "softmax": 10, + "transpose": 11, + # KIND_ID IS KEYED BY KERNEL VARIANT, NOT BY OP KIND, from here on. The first + # eleven happen to coincide because each of those op kinds had at most one + # kernel; `transpose_hd` is the first that does not. The encoder's 60 + # transposes are two different permutations needing two different kernels + # (see runner.SPECS), and THE WIRE CARRIES NO ATTRS -- the DSP gets an id and + # a buffer list, with no perm to branch on. So the variant has to be resolved + # on the host, by `runner.select`, and then named on the wire by its own id. + # Appending is safe; reordering is not. + "transpose_hd": 12, +} + +# hexlib_args C types. Keyed by the same wire-dtype strings as +# `hexlib.exec.runner.WIRE_DTYPE` AND `hexlib.runtime.wire.DTYPE_ID` -- all +# three spell int32 "int32". `DTYPE_ID` spelled it "i32" until this was fixed, +# which meant a spec declaring an int32 input was accepted by `RunnerSpec`, +# refused by `pack_batch` as an unknown dtype, and a KeyError here at generate +# time. test_runtime_wire.py binds the three key sets so they cannot drift again. +_CTYPE = {"fp16": "hexlib_hf", "fp32": "float", "int32": "int"} + +# BLOCK-QUANTIZED BUFFERS ARE HANDED OVER AS BYTES, deliberately. A q4_0 weight +# is a stream of 18-byte blocks (an fp16 scale then 32 4-bit values) and there is +# no C scalar type for one element of it -- so the entry does not invent one. The +# kernel receives `const unsigned char *` and the block layout is its business, +# which is also why `hexlib_dsp.h` enumerates `q4_0_repacked` as a LAYOUT: the +# guard `_layout_check` emits is what stops an un-repacked weight being read as a +# repacked one. Casting these to `hexlib_hf *` instead would compile fine and +# read the fp16 scale bytes as data. +_RAW_CTYPE = "unsigned char" + +# C types for values packed into the `a->params` blob, matching +# `Scalar.ctype` / `RunnerSpec._STRUCT_CODE` ('i' -> int, 'f' -> float). Both +# are 4 bytes, so indexing by element (not byte) is safe even when scalars of +# different ctypes are packed back to back in the same blob. +_PARAM_CTYPE = {"int": "int", "float": "float"} + + +class GenError(Exception): + pass + + +def _comment(text: str) -> str: + """One block comment, wrapped, at the entry body's indent. Generated code is + still read by people -- a 400-column comment line is not.""" + import textwrap + + lines = textwrap.wrap(" ".join(text.split()), width=72) + if len(lines) == 1: + return f" /* {lines[0]} */" + body = "\n".join(f" * {ln}" for ln in lines[1:]) + return f" /* {lines[0]}\n{body} */" + + +def _scalar_expr(sc: Scalar, spec: RunnerSpec, param_index: int) -> str: + """The C expression for one scalar -- the DSP-side derivation.""" + src = sc.source + if src.startswith("attr:"): + ctype = _PARAM_CTYPE[sc.ctype] + return f"((const {ctype} *) a->params)[{param_index}]" + if src.startswith("numel:"): + i = int(src.split(":", 1)[1]) + # From the tensor's OWN extent, not from a number the host asserted. + return f"(int) (a->ne[{i}][0] * a->ne[{i}][1] * a->ne[{i}][2] * a->ne[{i}][3])" + if src.startswith("dim:"): + _, i, axis = src.split(":") + return f"(int) a->ne[{i}][{axis}]" + if src.startswith("rows:"): + # numel / ne[axis], both from the tensor's OWN extents. `ne` is padded to + # four with ones, so this is exact for any rank -- and the axis is named + # rather than inferred because "the last axis" of a rank-3 tensor is + # ambiguous once the padding is there. A zero extent would divide by + # zero, so the guard is emitted alongside rather than assumed away: a + # batch declaring ne[axis] = 0 is malformed, and the DSP says so. + _, i, axis = src.split(":") + return ( + f"(a->ne[{i}][{axis}] ? (int) ((a->ne[{i}][0] * a->ne[{i}][1] * " + f"a->ne[{i}][2] * a->ne[{i}][3]) / a->ne[{i}][{axis}]) : 0)" + ) + raise GenError(f"unknown scalar source {src!r} in spec for {spec.kind}") + + +def _dtype_check(idx: int, dtype: str, role: str) -> str: + """The guard for ONE buffer, emitted for every buffer the entry casts. + + `a->dtype[idx]` is filled by `skel_dispatch.c` from the tensor's own dtype + field, which the host serialized through the same `DTYPE_ID` table imported + here -- so this compares the dtype the BATCH declared with the dtype this + entry is about to cast the pointer to. Without it, a batch declaring an + fp32 buffer where the kernel wants fp16 is read (or written) at half + stride, over half the tensor, and returns HEXLIB_DSP_OK. + """ + if dtype in WIRE_RAW: + detail = ( + f"{role} buf[{idx}] is handed over as {_RAW_CTYPE} * -- a stream of " + f"block-quantized {dtype} data -- so the batch must have declared it " + f"{dtype} ({DTYPE_ID[dtype]} in hexlib.runtime.wire.DTYPE_ID). A " + f"dense buffer arriving here would be read as blocks: its values " + f"decoded as 4-bit fields against scales that are really data." + ) + else: + detail = ( + f"{role} buf[{idx}] is cast to {_CTYPE[dtype]} *, so the batch must " + f"have declared it {dtype} ({DTYPE_ID[dtype]} in " + f"hexlib.runtime.wire.DTYPE_ID). Casting a wider or narrower dtype " + f"would silently halve or double every stride." + ) + return ( + _comment(detail) + + f"\n if (a->dtype[{idx}] != {DTYPE_ID[dtype]}u) " + f"return HEXLIB_DSP_ERR_REQUIRES;" + ) + + +def _layout_check(idx: int, layout: str, role: str) -> str: + """The layout guard for ONE buffer, emitted beside its dtype guard. + + `a->layout[idx]` is filled by `skel_dispatch.c` from the tensor's own + layout field, serialized through the same `LAYOUT_ID` table imported here. + Nothing checked it before: `main.c` wrote a bare literal `0` with a comment + for the binding, `grep -c layout hexlib/tests/test_host_source.py` was 0, + and `--self-test` printed `PASS (4100 values, bit-exact)` regardless of + what the batch declared. + + THIS IS THE CHECK THAT MAKES THE ENUM WORTH HAVING. `hexlib_dsp.h`'s own + header says the layout is enumerated rather than ne/nb strides so that + "un-repacked weights are a plan-time error rather than silent corruption" -- + and `LAYOUT_ID` already carries `q4_0_repacked`, the matmul weight layout. + Without a guard here that sentence describes an intention, not a mechanism: + a q4_0-repacked weight buffer handed to a row-major kernel is read as + row-major fp16 and returns HEXLIB_DSP_OK with a plausible wrong answer, the + same failure mode the per-buffer dtype check exists to stop. + """ + return ( + _comment( + f"{role} buf[{idx}] is addressed as {layout}, so the batch must " + f"have declared it {layout} ({LAYOUT_ID[layout]} in " + f"hexlib.runtime.wire.LAYOUT_ID). A differently-laid-out buffer of " + f"the same dtype and byte count passes every other check here." + ) + + f"\n if (a->layout[{idx}] != {LAYOUT_ID[layout]}u) " + f"return HEXLIB_DSP_ERR_REQUIRES;" + ) + + +def _requires_check(key: str, want, spec: RunnerSpec, out_idx: int) -> str: + """One `requires` clause as C, or an honest comment if it cannot be one. + + Only `("dtype", )` maps onto a field `hexlib_args` actually + carries, and it is already covered: `_dtype_check` emits a guard for EVERY + buffer from the spec's own declared dtypes, so the clause for `out_idx` is + the same condition this would emit. Rather than emit it twice, this points + at it. + + WHICH BUFFER A REQUIREMENT IS ABOUT CANNOT BE SAID. This used to assume + `out_idx` for every key -- correct for `cast`, whose requirement is about + its output, and silently wrong for any future dtype requirement about an + INPUT, which would have inspected the output's dtype instead. `requires` + has no place to name a buffer, so the ambiguous case is refused at generate + time instead of guessed at: a dtype requirement that is not the spec's own + declared output dtype is either about an input (unexpressible) or a + contradiction (it would refuse every batch the host serializer can build, + since that stamps the output's dtype from `spec.out_dtype`). + + Everything else (`perm`, and anything not representable in + `ne`/`dtype`/`layout`) has no wire representation at all, so it is + documented as unverified rather than given a check that cannot fail. + """ + if key == "dtype": + if want != spec.out_dtype: + raise GenError( + f"{spec.kind}: requires ('dtype', {want!r}) but the spec's " + f"out_dtype is {spec.out_dtype!r}. `requires` cannot say WHICH " + f"buffer a dtype requirement is about, and assuming the output " + f"would emit a check that either inspects the wrong buffer or " + f"refuses every batch the host can build. Declare the dtype on " + f"the buffer itself (inputs=/out_dtype=) instead." + ) + return _comment( + f"requires {key} == {want!r}: checked above, by the " + f"a->dtype[{out_idx}] guard emitted for this kernel's declared " + f"output dtype -- the same condition, from the same DTYPE_ID table. " + f"NOTE it cannot fail through hexlib/exec/dsp.py, which fills that " + f"field from spec.out_dtype: only a hand-built batch can violate it. " + f"The CALLER'S attr is policed on the host, in " + f"RunnerSpec.check_requires." + ) + # HONEST GAP: hexlib_args has no field for this key. buf/ne/dtype/layout are + # all per-buffer tensor properties; `perm` (and anything else outside that + # set) is an op-level attribute the wire format never carries down to the + # kernel entry. So this cannot be verified on the DSP today -- enforcement + # lives only in RunnerSpec.check_requires, on the host, before the op is + # ever encoded. HEXLIB_DSP_ERR_REQUIRES is the status this would return if + # the wire format grows a field for it; until then, no check is emitted, + # because a check that cannot fail is not a check. + return ( + f" /* requires {key} == {want!r}: NOT VERIFIED ON THE DSP -- " + f"hexlib_args carries no field for {key!r}. Enforced only on the host " + f"today (RunnerSpec.check_requires). Would return HEXLIB_DSP_ERR_REQUIRES " + f"if the wire format ever carries this. */" + ) + + +def emit_entry(name: str, spec: RunnerSpec) -> str: + """The C adapter from hexlib_args to the kernel's real signature. + + NOTE `spec.kernel_dir` is a PATH ("kernels/scale_fp16"), so the function name + is its basename. And `spec.inputs` is a tuple of DTYPES, not names -- so each + input is cast to its own type, which matters for `cast` (fp32 in, fp16 out): + casting an fp32 buffer to hexlib_hf* would halve every stride silently. + """ + fn = os.path.basename(spec.kernel_dir) # "scale_fp16" + n_in = len(spec.inputs) + n_buf = n_in + 1 # inputs + one output + out_idx = n_in + + args: list[str] = [] + for i, in_dtype in enumerate(spec.inputs): + ctype = _RAW_CTYPE if in_dtype in WIRE_RAW else _CTYPE[in_dtype] + args.append(f"(const {ctype} *) a->buf[{i}]") + args.append(f"({_CTYPE[spec.out_dtype]} *) a->buf[{out_idx}]") + + param_index = 0 + for sc in spec.scalars: + expr = _scalar_expr(sc, spec, param_index) + if sc.source.startswith("attr:"): + param_index += 1 + args.append(expr) + + checks = [ + f" if (a->n_buf != {n_buf}) return HEXLIB_DSP_ERR_INVAL_PARAMS;", + ] + for i in range(n_buf): + checks.append(f" if (!a->buf[{i}]) return HEXLIB_DSP_ERR_INVAL_PARAMS;") + + # EVERY buffer's declared dtype, not just whichever one `requires` mentions + # -- see `_dtype_check` and the module docstring. After the count and null + # checks, because `a->dtype[i]` means nothing for a buffer the batch did not + # supply. + for i, in_dtype in enumerate(spec.inputs): + checks.append(_dtype_check(i, in_dtype, "input")) + checks.append(_dtype_check(out_idx, spec.out_dtype, "output")) + + # AND EVERY BUFFER'S DECLARED LAYOUT, for the same reason and on the same + # terms -- see `_layout_check`. `spec.buf_layouts()` defaults every buffer to + # row_major, so this is a no-op for every kernel shipped today and becomes + # load-bearing the moment a q4_0_repacked weight appears. + for i, layout in enumerate(spec.buf_layouts()): + checks.append(_layout_check(i, layout, "input" if i < len(spec.inputs) + else "output")) + + # `requires` is enforced HERE as well as on the host where it is genuinely + # checkable -- see `_requires_check` for exactly which keys that is, and the + # module docstring for why the rest are documented rather than faked. + for key, want in spec.requires: + checks.append(_requires_check(key, want, spec, out_idx)) + + body = ",\n ".join(args) + return f'''/* GENERATED by hexlib/runtime/genentry.py -- do not edit. + * Source of truth: hexlib/exec/runner.py SPECS[{name!r}]. + * A hand-written dsp_entry.c in this kernel's own directory takes precedence + * over this generated file. + */ +#include "hexlib_dsp.h" +#include "kernel_api.h" + +int {fn}_entry(const hexlib_args *a) {{ +{chr(10).join(checks)} + {fn}({body}); + return HEXLIB_DSP_OK; +}} +''' + + +def emit_table(specs: dict[str, RunnerSpec]) -> str: + rows = [] + externs = [] + for name in sorted(specs): + fn = os.path.basename(specs[name].kernel_dir) + externs.append(f"extern int {fn}_entry(const hexlib_args *);") + rows.append(f' {{ {KIND_ID[name]}u, "{name}", {fn}_entry }},') + return f'''/* GENERATED by hexlib/runtime/genentry.py -- do not edit. + * + * Generated rather than hand-maintained so that adding a kernel touches no + * shared file and parallel contributions cannot conflict in a central registry. + */ +#include "hexlib_dsp.h" + +{chr(10).join(externs)} + +const struct hexlib_kernel_entry hexlib_kernel_table[] = {{ +{chr(10).join(rows)} +}}; + +const uint32_t hexlib_kernel_table_len = + sizeof(hexlib_kernel_table) / sizeof(hexlib_kernel_table[0]); +''' + + +def generate(repo_root: str, out_dir: str, + expect: Sequence[str] | None = None) -> list[str]: + """Emit entries for every kernel that does not hand-write its own. + + `spec.kernel_dir` is already repo-relative ("kernels/scale_fp16"), so it is + joined to the REPO root, not to a kernels root -- joining it to `.../kernels` + would produce `kernels/kernels/scale_fp16` and silently find nothing, which + would emit an empty dispatch table rather than an error. + + `expect` is the set of op kinds whose kernel directory MUST be present, + defaulting to every kind with a `RunnerSpec`. A tree missing any of them is + an incomplete checkout, not a smaller build -- see the partial-table comment + below. Pass a narrower tuple only from a caller that genuinely holds a + subset and says so (the unit tests in test_runtime_genentry.py, which build + one-kernel trees in tmp dirs). + """ + from hexlib.exec.runner import SPECS + + if expect is None: + expect = tuple(SPECS) + else: + unknown = sorted(set(expect) - set(SPECS)) + if unknown: + raise GenError( + f"expect names {unknown}, which has no RunnerSpec; `expect` " + f"narrows a claim about what is on disk, it cannot invent a " + f"kernel. Known kinds: {sorted(SPECS)}" + ) + + os.makedirs(out_dir, exist_ok=True) + written: list[str] = [] + used: dict[str, RunnerSpec] = {} + for name, spec in SPECS.items(): + kdir = os.path.join(repo_root, spec.kernel_dir) + if not os.path.isdir(kdir): + continue + used[name] = spec + + # AN EMPTY TABLE IS AN ERROR, NOT AN EMPTY SUCCESS. It would link cleanly and + # then answer every op with ERR_NO_KERNEL at run time, which reads as "the + # kernel is broken" rather than "the generator was pointed at the wrong + # directory". This project has been bitten four times by absence reported as + # success; a wrong `repo_root` is exactly that shape. + if not used: + raise GenError( + f"no kernel directory found under {repo_root!r} for any of " + f"{sorted(SPECS)} — kernel_dir is repo-relative " + f"('kernels/scale_fp16'), so pass the REPO root" + ) + + # AND A PARTIAL TABLE IS THE SAME BUG ONE STEP DOWN. Finding SOME kernels + # used to be enough: the table came out missing those rows, no error was + # raised, and the affected ops answered ERR_NO_KERNEL at run time -- which + # reads as a broken kernel rather than an incomplete tree. Checked BEFORE + # anything is written, so a refused run leaves no half-generated table for a + # build to pick up. + missing = sorted(set(expect) - set(used)) + if missing: + raise GenError( + f"kernel directory missing under {repo_root!r} for {missing} " + f"(expected {sorted(expect)}, found {sorted(used)}). Generating " + f"anyway would emit a dispatch table without those rows, which " + f"links cleanly and then answers ERR_NO_KERNEL at run time." + ) + + for name, spec in used.items(): + kdir = os.path.join(repo_root, spec.kernel_dir) + fn = os.path.basename(spec.kernel_dir) + if os.path.isfile(os.path.join(kdir, "dsp_entry.c")): + continue # hand-written wins + path = os.path.join(out_dir, f"{fn}_entry.c") + with open(path, "w", encoding="utf-8") as f: + f.write(emit_entry(name, spec)) + written.append(path) + + path = os.path.join(out_dir, "hexlib_kernel_table.c") + with open(path, "w", encoding="utf-8") as f: + f.write(emit_table(used)) + written.append(path) + return written diff --git a/hexlib/runtime/host/buffers.c b/hexlib/runtime/host/buffers.c new file mode 100644 index 0000000..3be74d6 --- /dev/null +++ b/hexlib/runtime/host/buffers.c @@ -0,0 +1,137 @@ +/* hexlib/runtime/host/buffers.c -- rpcmem allocation, fastrpc_mmap, and the + * DSP-side registration handshake. + * + * THE HOST NEVER PUTS AN ADDRESS ON THE WIRE. `hexlib_buf_desc.base` (see + * hexlib_dsp.h) is DSP-side scratch: the skel resolves its own address for a + * registered fd via HAP_mmap (skel_bufs.c) when a batch actually references + * it. There is no field on the wire for a host address at all, so + * hexlib_buf_to_desc writes `base = 0` explicitly, at the one place + * a hexlib_buf_desc is ever filled in from this side -- the same invariant + * hexlib.runtime.wire.py enforces on the Python side and hexlib_dsp.h states + * for the DSP side. + */ +#include "hexlib_host.h" + +#include /* INT_MAX -- see hexlib_alloc's size guard. */ +#include +#include +#include +#include +#include +#include + +#include "hexlib_dsp.h" /* struct hexlib_buf_desc */ +#include "hexlib_iface.h" /* hexlib_iface_mmap / hexlib_iface_munmap */ + +int hexlib_alloc(hexlib_ctx *ctx, hexlib_buf **out, size_t size) { + *out = NULL; + + /* SIZE IS NARROWED TWICE BELOW, AND NEITHER CAST CAN REPORT A LOSS. + * `rpcmem_alloc` takes an `int` and `hexlib_iface_mmap` takes a `uint32` + * (that is qaic's own generated signature, from the IDL -- not something + * this file chose), so a `size_t` of 2 GiB or more truncates on the way to + * one or both: `(int) size` can even go NEGATIVE, and the two casts can + * disagree, which would register a mapping of one length for a buffer + * allocated at another. + * + * UNREACHABLE TODAY, AND GUARDED ANYWAY. Every call site passes a plan- + * computed tensor size; the largest thing the encoder moves is far below + * 2 GiB, and if a truncated pair ever did get through, the DSP side fails + * CLOSED rather than reading out of bounds (skel_bufs.c's `b->size > + * m->size` check answers HEXLIB_DSP_ERR_INVAL_PARAMS). So this is a guard + * and a comment, deliberately NOT a widening of the IDL or of + * hexlib_alloc's own signature -- changing the wire for a case that + * cannot arise would be the larger risk. + * + * INT_MAX, not UINT32_MAX: the narrower of the two casts is the binding + * one, and this must fail before either happens rather than after one of + * them has already silently succeeded. + * + * NOTE THE DUPLICATE: main.c's `alloc_maybe_unmapped` is a deliberate + * local copy of this allocation sequence (for --unmapped) and carries the + * identical pair of narrowings. Its only call site passes a fixed 8200 + * bytes, so it is not reachable there either. */ + if (size == 0 || size > (size_t) INT_MAX) { + fprintf(stderr, + "hexlib: hexlib_alloc: %zu bytes is out of range -- rpcmem_alloc " + "takes an int and hexlib_iface_mmap takes a uint32, so a size " + "at or above 2 GiB would be truncated by one or both with no " + "way to report it\n", size); + return -1; + } + + void *ptr = hexlib_rpcmem_alloc(RPCMEM_HEAP_ID_SYSTEM, RPCMEM_DEFAULT_FLAGS, + (int) size); + if (ptr == NULL) { + fprintf(stderr, "hexlib: rpcmem_alloc(%zu bytes) failed\n", size); + return -1; + } + + int fd = hexlib_rpcmem_to_fd(ptr); + if (fd < 0) { + fprintf(stderr, "hexlib: rpcmem_to_fd failed\n"); + hexlib_rpcmem_free(ptr); + return -1; + } + + /* Maps the buffer into the CDSP's address space on the CPU-driver side. + * `addr` is the CPU virtual address purely so the driver can track which + * local mapping this fd corresponds to for cache maintenance -- it is + * NEVER the address the DSP will use, and never crosses the wire. */ + int rc = hexlib_fastrpc_mmap(ctx->domain, fd, ptr, 0, size, FASTRPC_MAP_FD); + if (rc != 0) { + fprintf(stderr, + "hexlib: fastrpc_mmap(fd=%d, size=%zu) failed (rc %d)\n", + fd, size, rc); + hexlib_rpcmem_free(ptr); + return -1; + } + + /* Registers the SAME fd with the DSP-side skel (hexlib_iface_mmap -> + * hexlib_bufs_register -> HAP_mmap in skel_bufs.c). This is a second, + * independent map of one fd: the fastrpc_mmap above lets the CPU driver + * account for the buffer, this one is what the skel's buffer table looks + * up by fd at invoke time. Neither one hands the other side an address. */ + int arc = hexlib_iface_mmap(ctx->handle, (uint32_t) fd, (uint32_t) size); + if (arc != AEE_SUCCESS) { + fprintf(stderr, "hexlib: hexlib_iface_mmap(fd=%d) failed (rc %d)\n", fd, arc); + hexlib_fastrpc_munmap(ctx->domain, fd, ptr, size); + hexlib_rpcmem_free(ptr); + return -1; + } + + hexlib_buf *buf = (hexlib_buf *) calloc(1, sizeof(*buf)); + if (buf == NULL) { + hexlib_iface_munmap(ctx->handle, (uint32_t) fd); + hexlib_fastrpc_munmap(ctx->domain, fd, ptr, size); + hexlib_rpcmem_free(ptr); + return -1; + } + buf->ptr = ptr; + buf->fd = fd; + buf->size = size; + *out = buf; + return 0; +} + +void hexlib_free(hexlib_ctx *ctx, hexlib_buf *buf) { + if (buf == NULL) { + return; + } + hexlib_iface_munmap(ctx->handle, (uint32_t) buf->fd); + hexlib_fastrpc_munmap(ctx->domain, buf->fd, buf->ptr, buf->size); + hexlib_rpcmem_free(buf->ptr); + free(buf); +} + +/* THE HOST NEVER PUTS AN ADDRESS ON THE WIRE -- see the file header. `d->base` + * is set to 0 unconditionally, first, before any other field: there is no + * value derived from `buf->ptr` (the CPU-side virtual address) that could + * ever legally end up here. */ +void hexlib_buf_to_desc(const hexlib_buf *buf, struct hexlib_buf_desc *d) { + memset(d, 0, sizeof(*d)); + d->base = 0; /* DSP-side scratch. Never a host address. */ + d->size = (uint64_t) buf->size; + d->fd = (uint32_t) buf->fd; + d->flags = 0; +} diff --git a/hexlib/runtime/host/driver.c b/hexlib/runtime/host/driver.c new file mode 100644 index 0000000..d35d30b --- /dev/null +++ b/hexlib/runtime/host/driver.c @@ -0,0 +1,179 @@ +/* hexlib/runtime/host/driver.c -- dlopen the FastRPC driver, by symbol name. + * + * Adapted from llama.cpp ggml-hexagon's htp-drv.cpp `htpdrv_init()` (MIT); see + * ATTRIBUTION.md. Rewritten in C -- upstream is C++ and carries dspqueue + * plumbing hexlib does not use (see runtime/idl/hexlib_iface.idl for why: + * dspqueue has no simulator path). What is kept is the shape that mattered: + * `libcdsprpc.so` is dlopen'd, never linked, and every symbol this file needs + * is resolved by NAME through dlsym and checked before use. + * + * WHY DLOPEN AND NOT A LINK-TIME DEPENDENCY. `libcdsprpc.so` exists only on a + * device that actually has the FastRPC driver installed for the compute DSP. + * Linking it directly turns "this device doesn't have it" into an + * unresolved-symbol failure at process load, before main() ever runs, with no + * message a user can act on. dlopen makes that failure a readable string + * instead -- this is the exact property that made the same author's + * capability probe work on the first real-silicon attempt: a missing driver + * said so, rather than refusing to start. + * + * A MISSING SYMBOL IS A NAMED ERROR, NEVER A NULL CALL. Calling through a NULL + * function pointer segfaults with no indication of which symbol was missing. + * HEXLIB_DLSYM logs the exact symbol name and dlerror()'s text, then fails + * hexlib_drv_init() outright for anything required -- it does not leave a + * NULL pointer behind for some later call site to trip over. + */ +#include "hexlib_host.h" + +#include +#include +#include + +hexlib_rpcmem_alloc_fn hexlib_rpcmem_alloc = NULL; +hexlib_rpcmem_alloc2_fn hexlib_rpcmem_alloc2 = NULL; +hexlib_rpcmem_free_fn hexlib_rpcmem_free = NULL; +hexlib_rpcmem_to_fd_fn hexlib_rpcmem_to_fd = NULL; +hexlib_fastrpc_mmap_fn hexlib_fastrpc_mmap = NULL; +hexlib_fastrpc_munmap_fn hexlib_fastrpc_munmap = NULL; +hexlib_remote_handle64_open_fn hexlib_remote_handle64_open = NULL; +hexlib_remote_handle64_invoke_fn hexlib_remote_handle64_invoke = NULL; +hexlib_remote_handle64_close_fn hexlib_remote_handle64_close = NULL; +hexlib_remote_handle_control_fn hexlib_remote_handle_control = NULL; +hexlib_remote_session_control_fn hexlib_remote_session_control = NULL; + +static void *g_driver_handle = NULL; +static int g_initialized = 0; + +/* Resolve `symbol` into `pfn` (whose type is already declared, so __typeof__ + * gives back the right function-pointer type for the cast dlsym's void* + * return needs). `required` false means "log and keep going" -- used only for + * rpcmem_alloc2, which some driver builds lack (see htp-drv.cpp's own + * treatment of it); every other symbol here is required, and its absence + * fails hexlib_drv_init() rather than being silently tolerated. */ +#define HEXLIB_DLSYM(pfn, symbol, required) \ + do { \ + (pfn) = (__typeof__(pfn)) dlsym(handle, #symbol); \ + if ((pfn) == NULL) { \ + if (required) { \ + fprintf(stderr, "hexlib: dlsym(%s) failed: %s\n", #symbol, \ + dlerror()); \ + return -1; \ + } \ + fprintf(stderr, \ + "hexlib: %s not present on this driver (optional), " \ + "continuing without it\n", #symbol); \ + } \ + } while (0) + +int hexlib_drv_init(void) { + if (g_initialized) { + return 0; + } + + /* Two candidate paths, tried in order: the normal dynamic-linker-visible + * name, then the vendor partition path some builds only expose the + * driver under. The first that dlopen()s successfully wins. */ + static const char *const candidates[] = { + "libcdsprpc.so", + "/vendor/lib64/libcdsprpc.so", + }; + + void *handle = NULL; + for (size_t i = 0; i < sizeof(candidates) / sizeof(candidates[0]); i++) { + handle = dlopen(candidates[i], RTLD_NOW); + if (handle != NULL) { + break; + } + fprintf(stderr, "hexlib: dlopen(%s) failed: %s\n", candidates[i], dlerror()); + } + if (handle == NULL) { + fprintf(stderr, + "hexlib: could not load the FastRPC driver from any candidate " + "path -- is this a compute-DSP-capable device?\n"); + return -1; + } + + HEXLIB_DLSYM(hexlib_rpcmem_alloc, rpcmem_alloc, 1); + HEXLIB_DLSYM(hexlib_rpcmem_alloc2, rpcmem_alloc2, 0); + HEXLIB_DLSYM(hexlib_rpcmem_free, rpcmem_free, 1); + HEXLIB_DLSYM(hexlib_rpcmem_to_fd, rpcmem_to_fd, 1); + HEXLIB_DLSYM(hexlib_fastrpc_mmap, fastrpc_mmap, 1); + HEXLIB_DLSYM(hexlib_fastrpc_munmap, fastrpc_munmap, 1); + HEXLIB_DLSYM(hexlib_remote_handle64_open, remote_handle64_open, 1); + HEXLIB_DLSYM(hexlib_remote_handle64_invoke, remote_handle64_invoke, 1); + HEXLIB_DLSYM(hexlib_remote_handle64_close, remote_handle64_close, 1); + HEXLIB_DLSYM(hexlib_remote_handle_control, remote_handle_control, 1); + HEXLIB_DLSYM(hexlib_remote_session_control, remote_session_control, 1); + + g_driver_handle = handle; + g_initialized = 1; + return 0; +} + +/* ========================================================================== + * remote_handle64_open/_invoke/_close -- STRONG, GLOBALLY-NAMED FORWARDERS. + * + * WHY THESE EXIST AT ALL. The qaic-generated stub (hexlib_iface_stub.c, + * never hand-edited -- see main.c's own header comment) calls + * `remote_handle64_open`/`_invoke`/`_close` directly, as ordinary strong + * `extern` functions declared in (confirmed by reading it: no + * `weak` attribute, `__QAIC_REMOTE(ff)` defaults to identity, so the + * generated stub really does call these three names literally). Without a + * definition for them somewhere in this binary, `hexlib_run` cannot link at + * all -- confirmed the hard way (task 10): the first build attempt failed + * with "undefined symbol: remote_handle64_open/_invoke/_close". + * + * THE WRONG FIX, TRIED FIRST AND REVERTED: link directly against the SDK's + * `libcdsprpc.so` import stub. That satisfies the linker, but it reintroduces + * exactly the failure mode this file's own "WHY DLOPEN AND NOT A LINK-TIME + * DEPENDENCY" comment above exists to avoid -- a device missing + * `libcdsprpc.so` would fail to even start `hexlib_run` (a dynamic-linker + * load error, before `main` runs), never reaching the readable message the + * init routine above prints at all. + * + * THE ACTUAL FIX, matching llama.cpp ggml-hexagon's own `htp-drv.cpp` + * (`remote_handle64_open`/`_invoke`/`_close`, right next to the dlopen logic + * these are adapted from): define these three names ourselves, as thin + * one-line forwarders to the `hexlib_remote_handle64_*` function pointers + * the init routine above already dlsym's. This satisfies the stub's + * link-time reference WITHOUT linking `libcdsprpc.so` at build time -- + * `libcdsprpc.so` stays exclusively `dlopen`'d, so a device that lacks it + * still gets that routine's own readable stderr message, never a + * process-load failure. + * + * ONLY THESE THREE. `remote_handle_control`/`remote_session_control` are + * called only through `hexlib_remote_handle_control`/ + * `hexlib_remote_session_control` indirection inside session.c -- never as + * bare `extern` references from generated code -- so a forwarder for either + * would be dead code with no caller. + * + * NO NULL CHECK HERE, AND NONE IS NEEDED: THIS IS NOT AN OVERSIGHT. + * `hexlib_open` in session.c always calls the init routine above first and + * returns its error before ever reaching the qaic-generated `_open` call -- + * the only path that can call into the stub, and therefore into these + * forwarders. So by the time any of the three below runs, + * `hexlib_remote_handle64_open`/`_invoke`/`_close` are already non-NULL, or + * this code is unreachable. A defensive check here would be dead code + * guarding against a state the caller has already made impossible. + * + * PLACED AFTER THE INIT ROUTINE ABOVE, DELIBERATELY, NOT MERELY FOR + * READING ORDER: hexlib/tests/test_host_source.py locates that routine's + * body by regex, scanning for its own name followed by a `{` with no `;`/`{` + * in between -- which would also match INTO one of these forwarders' bodies + * if this comment block (mentioning that routine's name several times, + * parenthesised, in prose) sat between its signature and one of these three + * function definitions. Keeping this block textually AFTER that routine's + * closing brace means the test's leftmost search always finds the real + * definition first, so it does not matter what prose reappears afterward. + * ========================================================================*/ + +int remote_handle64_open(const char *name, remote_handle64 *ph) { + return hexlib_remote_handle64_open(name, ph); +} + +int remote_handle64_invoke(remote_handle64 h, uint32_t dwScalars, remote_arg *pra) { + return hexlib_remote_handle64_invoke(h, dwScalars, pra); +} + +int remote_handle64_close(remote_handle64 h) { + return hexlib_remote_handle64_close(h); +} diff --git a/hexlib/runtime/host/hexlib_host.h b/hexlib/runtime/host/hexlib_host.h new file mode 100644 index 0000000..4a44697 --- /dev/null +++ b/hexlib/runtime/host/hexlib_host.h @@ -0,0 +1,150 @@ +/* hexlib/runtime/host/hexlib_host.h -- the CPU-side (aarch64, Android) API. + * + * This is the ONLY path that ever exercises qaic's real argument marshalling + * (see main.c): on the simulator (Task 7/8) the qaic stub was deliberately + * never linked, because skel.c's function names collide with it in one + * address space. On a device there are two separate binaries -- this one + * links the qaic-generated STUB (hexlib_iface_stub.c) and calls + * hexlib_iface_open/_start/_mmap/_munmap/_hwinfo/_invoke/_stop/_close exactly + * like any other function; the marshalling into remote_handle64_invoke() + * happens inside those generated wrappers, invisibly to this file. + * + * THE SDK IS NEVER VENDORED. and the rest of the Hexagon SDK are + * found via HEXAGON_SDK_ROOT at build time (Task 10), never copied into this + * repository. + */ +#ifndef HEXLIB_HOST_H +#define HEXLIB_HOST_H + +#include +#include + +#include + +#ifdef __cplusplus +extern "C" { +#endif + +/* ========================================================================== + * driver.c -- dlopen'd libcdsprpc.so, resolved by symbol name. + * + * NONE of these are linked at build time: libcdsprpc.so exists only on a + * device with the FastRPC driver installed, and even then only for the + * domains that device supports. hexlib_drv_init() must be called (and must + * return 0) before any of the function pointers below are valid. + * ========================================================================*/ + +typedef void *(*hexlib_rpcmem_alloc_fn)(int heapid, uint32_t flags, int size); +typedef void *(*hexlib_rpcmem_alloc2_fn)(int heapid, uint32_t flags, size_t size); +typedef void (*hexlib_rpcmem_free_fn)(void *po); +typedef int (*hexlib_rpcmem_to_fd_fn)(void *po); +typedef int (*hexlib_fastrpc_mmap_fn)(int domain, int fd, void *addr, int offset, + size_t length, enum fastrpc_map_flags flags); +typedef int (*hexlib_fastrpc_munmap_fn)(int domain, int fd, void *addr, size_t length); +typedef int (*hexlib_remote_handle64_open_fn)(const char *name, remote_handle64 *ph); +typedef int (*hexlib_remote_handle64_invoke_fn)(remote_handle64 h, uint32_t dwScalars, + remote_arg *pra); +typedef int (*hexlib_remote_handle64_close_fn)(remote_handle64 h); +typedef int (*hexlib_remote_handle_control_fn)(uint32_t req, void *data, uint32_t datalen); +typedef int (*hexlib_remote_session_control_fn)(uint32_t req, void *data, uint32_t datalen); + +extern hexlib_rpcmem_alloc_fn hexlib_rpcmem_alloc; +extern hexlib_rpcmem_alloc2_fn hexlib_rpcmem_alloc2; /* may stay NULL */ +extern hexlib_rpcmem_free_fn hexlib_rpcmem_free; +extern hexlib_rpcmem_to_fd_fn hexlib_rpcmem_to_fd; +extern hexlib_fastrpc_mmap_fn hexlib_fastrpc_mmap; +extern hexlib_fastrpc_munmap_fn hexlib_fastrpc_munmap; +extern hexlib_remote_handle64_open_fn hexlib_remote_handle64_open; +extern hexlib_remote_handle64_invoke_fn hexlib_remote_handle64_invoke; +extern hexlib_remote_handle64_close_fn hexlib_remote_handle64_close; +extern hexlib_remote_handle_control_fn hexlib_remote_handle_control; +extern hexlib_remote_session_control_fn hexlib_remote_session_control; + +/* Loads libcdsprpc.so and resolves every symbol above by name. Returns 0 on + * success. Idempotent -- a second call is a no-op that also returns 0. A + * missing REQUIRED symbol fails the whole call (named in stderr, via + * dlerror()) rather than leaving some pointers NULL for a later call site to + * crash on. */ +int hexlib_drv_init(void); + +/* ========================================================================== + * session.c -- CDSP only, unsigned PD, arch cross-checked against the skel. + * ========================================================================*/ + +typedef struct hexlib_ctx { + int domain; /* Always CDSP_DOMAIN_ID; see hexlib_open(). */ + remote_handle64 handle; + uint32_t arch; + uint32_t n_threads; + uint32_t n_hvx; + uint32_t n_hmx; + uint64_t vtcm_size; /* ACQUIRED size, not the part's total. */ + int started; +} hexlib_ctx; + +/* One DSPRPC_GET_DSP_INFO query per field, each attribute named from + * 's own `enum remote_dsp_attributes` -- see session.c. */ +struct hexlib_caps { + uint32_t domain_support; + uint32_t unsigned_pd_support; + uint32_t hvx_support_128b; + uint32_t vtcm_page; + uint32_t vtcm_count; + uint32_t arch_ver; + uint32_t hmx_support_depth; /* 0 is NOT evidence HMX is absent. */ +}; + +/* Requires hexlib_drv_init() to have already succeeded. */ +int hexlib_query_caps(int domain, struct hexlib_caps *out); + +/* Opens a session on `domain`, which MUST be CDSP_DOMAIN_ID -- ADSP is a v73 + * part with a different UNSIGNED_PD_SUPPORT and must never be substituted + * silently. Requests an unsigned PD, opens the qaic handle, starts the + * session, reads hwinfo back, and cross-checks the arch the driver reports + * against the arch the skel reports; disagreement is refused, not logged and + * ignored. On success, *out is a session ready for hexlib_alloc/hexlib_invoke. */ +int hexlib_open(hexlib_ctx **out, int domain); +int hexlib_close(hexlib_ctx *ctx); + +int hexlib_hwinfo(hexlib_ctx *ctx, uint32_t *arch, uint32_t *n_threads, + uint32_t *n_hvx, uint32_t *n_hmx, uint64_t *vtcm_size); + +/* Runs one batch. `batch`/`batch_len` is a wire-format blob (hexlib_dsp.h / + * hexlib.runtime.wire.py); `rsp`/`rsp_cap` is the caller's response buffer. + * `*rsp_len` is set to the number of bytes that are actually meaningful, + * computed from hexlib_batch_rsp_hdr.n_ops -- qaic's `rout sequence` + * carries no out-length of its own (see hexlib_iface.h / ATTRIBUTION.md), so + * there is nowhere else this number could come from. Returns 0 only if the + * RPC itself succeeded; the caller must still check the response's own magic + * and status (see main.c) before trusting the bytes as a result. */ +int hexlib_invoke(hexlib_ctx *ctx, const void *batch, size_t batch_len, + void *rsp, size_t rsp_cap, size_t *rsp_len); + +/* ========================================================================== + * buffers.c -- rpcmem + fastrpc_mmap. The host never puts an address on the + * wire; see hexlib_buf_to_desc() below and hexlib_dsp.h's own comment. + * ========================================================================*/ + +typedef struct hexlib_buf { + void *ptr; /* CPU-side virtual address -- for THIS process only. */ + int fd; + size_t size; +} hexlib_buf; + +int hexlib_alloc(hexlib_ctx *ctx, hexlib_buf **out, size_t size); +void hexlib_free(hexlib_ctx *ctx, hexlib_buf *buf); + +/* Forward-declared, not included: hexlib_buf_desc lives in + * runtime/skel/hexlib_dsp.h, a DSP-side header this one does not otherwise + * need. Only a pointer to it crosses this interface. */ +struct hexlib_buf_desc; + +/* Fills `d->base = 0` always -- the DSP resolves its own address for `fd` + * (skel_bufs.c); there is no field on the wire for a host address at all. */ +void hexlib_buf_to_desc(const hexlib_buf *buf, struct hexlib_buf_desc *d); + +#ifdef __cplusplus +} +#endif + +#endif /* HEXLIB_HOST_H */ diff --git a/hexlib/runtime/host/main.c b/hexlib/runtime/host/main.c new file mode 100644 index 0000000..b1c1a92 --- /dev/null +++ b/hexlib/runtime/host/main.c @@ -0,0 +1,1157 @@ +/* hexlib/runtime/host/main.c -- hexlib_run: the CPU-side FastRPC client. + * + * ON A DEVICE, THE QAIC STUB IS LINKED -- THE OPPOSITE OF THE SIMULATOR + * ARRANGEMENT. Through Task 8, hexlib_iface_open/_start/_mmap/_invoke/_stop/ + * _close were called as plain C functions bound directly to skel.c's + * definitions in one Hexagon ELF (see runtime/build.py's build_sim_qexe): + * the qaic-generated stub was deliberately never linked there, because it + * defines those exact same names and both live in one address space. Here + * the skel is a separate Hexagon .so the FastRPC framework loads on the + * CDSP, and this aarch64 binary links hexlib_iface_stub.c instead -- so + * calling hexlib_iface_invoke() from run_self_test() below is the FIRST + * thing in this project ever to exercise qaic's real argument marshalling + * into a remote_arg[] and a genuine remote_handle64_invoke() call. Every + * simulator run before this task tested hexlib's own code (batch parsing, + * dispatch, kernels) with the marshalling layer completely bypassed; this is + * the one binary that finally puts it in the loop. + * + * ABSENCE OF A RESPONSE IS A FAILURE, NEVER A SUCCESS. HEXLIB_DSP_OK is 1, + * never 0 (hexlib_dsp.h), specifically so a zero-filled buffer that nothing + * ever wrote cannot read as success. Every path below that reads a response + * checks its magic FIRST, before its status: absent, truncated, or + * wrong-magic all fail with a distinct exit code and, on the --batch path, + * write no output file at all. This project has already shipped a device-farm + * job that ran no tests and reported passing off an empty result; the same + * shape of bug here would be a "successful" run with a garbage or all-zero + * output file. + */ +#include "hexlib_host.h" +#include "hexlib_dsp.h" +#include "hexlib_iface.h" /* hexlib_iface_mmap/_munmap -- needed ONLY for + * --unmapped's deliberately-skipped registration + * call; see alloc_maybe_unmapped() below. Every + * ordinary allocation still goes through + * hexlib_alloc() (buffers.c), which already + * pulls this header in the same way. */ + +#include +#include /* RPCMEM_HEAP_ID_SYSTEM / RPCMEM_DEFAULT_FLAGS, + * for the same reason as above. */ +#include /* uint16_t -- --coherency-check classifies fp16 + * lanes by raw bit pattern, not by __fp16 + * arithmetic; see hexlib_classify_coherency_lane() + * below and run_coherency_check()'s own header. */ +#include +#include +#include + +/* The wire "kind" for the scale op. Must match + * hexlib.runtime.genentry.KIND_ID["scale"] == 9 -- there is no shared C + * header for these ids (genentry.py emits the DSP-side dispatch table + * straight from that Python dict; nothing generates a host-side mirror of + * it), so this one constant is pinned here, by name and by comment, rather + * than left to drift silently. A wrong value here is not silent, though: it + * would make hexlib_dispatch_batch() return HEXLIB_DSP_ERR_NO_KERNEL, which + * --self-test below reports as a failure, never a pass. */ +#define HEXLIB_KIND_SCALE 9u + +#define SELF_TEST_N 4100 /* 64*64 + 4: exercises the scalar tail. */ +#define SELF_TEST_FACTOR 0.125f /* A power of two: exact in fp16. */ + +/* --coherency-check's two constants -- see run_coherency_check()'s own + * header comment for why each one is what it is. */ +#define COHERENCY_SENTINEL 1.0f /* Any nonzero, finite fp16 value works; + * the expected result is bit-exact zero + * IN MAGNITUDE, so this can never be + * confused with it. See + * run_coherency_check()'s own header on + * why "zero" must be checked by + * magnitude (masking off the sign bit, + * 0x7FFF), not bit-exact equality + * against +0.0. */ +#define COHERENCY_FACTOR 0.0f /* x * 0.0 is zero in fp16 for any finite, + * non-NaN x -- no numerically ambiguous + * case, so a wrong result here cannot be + * blamed on kernel arithmetic. NOT + * necessarily +0.0, though: IEEE-754 + * negative-zero rules mean x * 0.0 is + * -0.0 (sign bit set, 0x8000) whenever x + * is negative, which the self-test's own + * input (`x[i] = ((i % 17) - 8) * 0.5f`, + * negative for many i) genuinely is. A + * bit-exact compare against `(__fp16) + * 0.0f` would then read a HEALTHY result + * as "sentinel unchanged" and misreport a + * coherency miss that never happened. */ + +enum { + HEXLIB_EXIT_OK = 0, + HEXLIB_EXIT_USAGE = 1, + HEXLIB_EXIT_SESSION_FAILED = 2, + HEXLIB_EXIT_NO_RESPONSE = 3, /* absent / truncated / wrong-magic */ + HEXLIB_EXIT_OP_FAILED = 4, + HEXLIB_EXIT_MISMATCH = 5, + HEXLIB_EXIT_COHERENCY_MISS = 6, /* --coherency-check: status OK, op OK, + * but the sentinel survived. "MISS" is + * the name, not the diagnosis: this is + * equally consistent with a kernel or + * generated entry that returned OK + * without writing `y`. cycles_total + * does NOT tell the two apart -- an + * earlier version of this comment said + * it did. See run_coherency_check()'s + * own header for why, and for what + * would. */ + HEXLIB_EXIT_COHERENCY_GARBLED = 7, /* --coherency-check: status OK, op OK, + * but the output buffer is neither the + * expected zero result NOR the intact + * sentinel -- a THIRD outcome (garbled + * or partially-written buffer) that + * must never be folded into a + * coherency-miss claim; see + * run_coherency_check()'s own header. */ +}; + +static void usage(const char *argv0) { + fprintf(stderr, + "usage: %s --caps\n" + " %s --self-test [--unmapped | --coherency-check]\n" + " %s --batch --in --out \n", + argv0, argv0, argv0); +} + +/* Checks the ONE thing that makes a response trustworthy at all: the magic. + * A NULL/too-short/wrong-magic buffer is refused before its status field is + * even read -- there is no status to trust in a response that was never + * written, or that belongs to some other protocol entirely. */ +static int response_is_valid(const uint8_t *rsp, size_t rsp_len, uint32_t *status_out) { + if (rsp == NULL || rsp_len < sizeof(struct hexlib_batch_rsp_hdr)) { + return 0; + } + struct hexlib_batch_rsp_hdr hdr; + memcpy(&hdr, rsp, sizeof(hdr)); + if (hdr.magic != HEXLIB_BATCH_MAGIC) { + return 0; + } + *status_out = hdr.status; + return 1; +} + +/* RETURNS AN EXIT CODE -- IT USED TO RETURN void, AND THAT WAS THE BUG. + * Both failure branches below printed to stderr and returned, and main() + * returned HEXLIB_EXIT_OK regardless. On a device whose image has no + * `libcdsprpc.so` for this ABI, `./hexlib_run --caps; echo RC=$?` printed + * "could not load the FastRPC driver" and `RC=0` -- so any `set -e` wrapper, + * shell step, or CI stage read a total driver-load failure as a pass. `--caps` + * is also the FIRST thing run on unfamiliar silicon and the mode most likely + * to fail there, which made it the worst possible place for the exit code to + * be a constant. HEXLIB_EXIT_SESSION_FAILED (2) is the right code for both: + * neither is a usage error, and both are exactly "the DSP side could not be + * reached", which is what that code already means everywhere else here. */ +static int print_caps(void) { + if (hexlib_drv_init() != 0) { + fprintf(stderr, "hexlib: --caps: could not load the FastRPC driver\n"); + return HEXLIB_EXIT_SESSION_FAILED; + } + struct hexlib_caps caps; + if (hexlib_query_caps(CDSP_DOMAIN_ID, &caps) != 0) { + fprintf(stderr, "hexlib: --caps: capability query failed\n"); + return HEXLIB_EXIT_SESSION_FAILED; + } + printf("domain = CDSP (%d)\n", CDSP_DOMAIN_ID); + printf("domain_support = %u\n", caps.domain_support); + printf("unsigned_pd_support = %u\n", caps.unsigned_pd_support); + printf("hvx_support_128b = %u\n", caps.hvx_support_128b); + printf("vtcm_page = %u\n", caps.vtcm_page); + printf("vtcm_count = %u\n", caps.vtcm_count); + printf("vtcm_total_bytes = %llu\n", + (unsigned long long) caps.vtcm_page * (unsigned long long) caps.vtcm_count); + printf("arch_ver = %u (0x%04x)\n", caps.arch_ver, caps.arch_ver); + /* HMX_SUPPORT_DEPTH reads 0 on the measured target. That is NOT evidence + * HMX is absent -- see the task's own measured-device-facts record -- so + * this prints the raw number and says so, rather than translating it + * into a yes/no HMX verdict this query cannot actually support. */ + printf("hmx_support_depth = %u (0 is not evidence HMX is absent -- " + "settle by direct test, not by this query)\n", + caps.hmx_support_depth); + return HEXLIB_EXIT_OK; +} + +/* Build a one-op scale_fp16 batch: two buffers (x, y), one tensor per + * buffer, one `scale` op. Field-for-field, this is hexlib_batch_hdr / + * hexlib_buf_desc / hexlib_tensor / hexlib_op_desc from hexlib_dsp.h, which + * on a little-endian aarch64 host has the IDENTICAL in-memory layout as + * hexlib.runtime.wire.py's struct-packed format (verified: every field in + * every one of those four C structs is naturally aligned already, so there + * is no padding a Python `struct.pack("<...")` format string would not also + * produce). So this function fills the real C structs and memcpy()s them + * into the blob -- no hand-rolled byte packing, and nothing here can drift + * from hexlib_dsp.h the way independently-maintained packing code could. */ +static uint8_t *build_scale_batch(int fd_x, int fd_y, size_t nbytes, float factor, + size_t *out_len) { + size_t total = sizeof(struct hexlib_batch_hdr) + + 2 * sizeof(struct hexlib_buf_desc) + + 2 * sizeof(struct hexlib_tensor) + + 1 * sizeof(struct hexlib_op_desc); + uint8_t *blob = (uint8_t *) calloc(1, total); + if (blob == NULL) { + return NULL; + } + + uint32_t off_bufs = (uint32_t) sizeof(struct hexlib_batch_hdr); + uint32_t off_tensors = off_bufs + 2 * (uint32_t) sizeof(struct hexlib_buf_desc); + uint32_t off_ops = off_tensors + 2 * (uint32_t) sizeof(struct hexlib_tensor); + + struct hexlib_batch_hdr hdr; + memset(&hdr, 0, sizeof(hdr)); + hdr.magic = HEXLIB_BATCH_MAGIC; + hdr.version = HEXLIB_BATCH_VERSION; + hdr.total_size = (uint32_t) total; + hdr.n_bufs = 2; + hdr.n_tensors = 2; + hdr.n_ops = 1; + hdr.off_bufs = off_bufs; + hdr.off_tensors = off_tensors; + hdr.off_ops = off_ops; + hdr.flags = 0; + memcpy(blob, &hdr, sizeof(hdr)); + + struct hexlib_buf_desc bufs[2]; + memset(bufs, 0, sizeof(bufs)); + bufs[0].base = 0; /* Host never writes an address -- see buffers.c. */ + bufs[0].size = (uint64_t) nbytes; + bufs[0].fd = (uint32_t) fd_x; + bufs[0].flags = 0; + bufs[1].base = 0; + bufs[1].size = (uint64_t) nbytes; + bufs[1].fd = (uint32_t) fd_y; + bufs[1].flags = 0; + memcpy(blob + off_bufs, bufs, sizeof(bufs)); + + struct hexlib_tensor tens[2]; + memset(tens, 0, sizeof(tens)); + for (int i = 0; i < 2; i++) { + tens[i].bi = (uint32_t) i; + tens[i].offset = 0; + tens[i].nbytes = (uint32_t) nbytes; + tens[i].dtype = 1; /* hexlib.runtime.wire.DTYPE_ID["fp16"] */ + tens[i].layout = HEXLIB_LAYOUT_ROW_MAJOR; + tens[i].ne[0] = SELF_TEST_N; + tens[i].ne[1] = 1; + tens[i].ne[2] = 1; + tens[i].ne[3] = 1; + tens[i].data = 0; /* DSP-side scratch. Host writes 0. */ + } + memcpy(blob + off_tensors, tens, sizeof(tens)); + + struct hexlib_op_desc op; + memset(&op, 0, sizeof(op)); + op.kind = HEXLIB_KIND_SCALE; + op.flags = 0; + /* `factor` is a float attr, so its wire slot carries the caller's IEEE-754 + * bit pattern (0.125f from run_self_test, 0.0f from + * run_coherency_check), not the integer 0 that `(int32_t) factor` would + * silently produce -- genentry.py's generated entry reads it back as + * `((const float *) a->params)[0]`, a raw reinterpretation, not a + * numeric conversion. */ + union { float f; int32_t i; } factor_bits; + factor_bits.f = factor; + op.params[0] = factor_bits.i; + for (int i = 1; i < HEXLIB_MAX_PARAMS; i++) { + op.params[i] = 0; + } + for (int i = 0; i < HEXLIB_MAX_SRC; i++) { + op.src[i] = 0xFFFF; + } + for (int i = 0; i < HEXLIB_MAX_DST; i++) { + op.dst[i] = 0xFFFF; + } + op.src[0] = 0; /* tensor 0: x */ + op.dst[0] = 1; /* tensor 1: y */ + memcpy(blob + off_ops, &op, sizeof(op)); + + *out_len = total; + return blob; +} + +/* ========================================================================== + * --unmapped -- THE LOAD-BEARING CHECK. + * + * On the simulator, host and DSP share one address space: `HAP_mmap` is + * `return (void*)(uintptr_t)fd;` and `rpcmem_to_fd` is + * `return (int)(uintptr_t)po;` there, so the whole pointer -> fd -> map -> + * base chain is an IDENTITY FUNCTION and the "mapped" address is always the + * real host pointer. No comparison of VALUES can tell a correct DSP + * implementation apart from one that simply read the host's own address -- + * which would work perfectly on the simulator and fail instantly on real + * hardware. The ONLY thing that discriminates is a table lookup: + * `hexlib_bufs_map` (skel_bufs.c) consults a table that only + * `hexlib_bufs_register` populates, and that only happens in response to a + * genuine `hexlib_iface_mmap` call. So this mode allocates rpcmem and gets + * an fd exactly as `hexlib_alloc()` (buffers.c) does, but DELIBERATELY SKIPS + * the `hexlib_iface_mmap` registration call -- mirroring + * `hexlib/runtime/simhost/simhost.c`'s own `--unmapped`, which withholds the + * identical call for the identical reason (see that file's header comment). + * `hexlib_dispatch_batch` (skel_dispatch.c) must then refuse with + * `HEXLIB_DSP_ERR_UNMAPPED` (7) as the batch's TOP-LEVEL status -- no new + * print statement is needed for that refusal to be visible: run_self_test's + * own `status != HEXLIB_DSP_OK` branch already reports it and returns + * HEXLIB_EXIT_OP_FAILED (4). + * + * buffers.c is out of scope for this change (only this file may move) and + * `hexlib_alloc()` has no knob for skipping its registration call, so this + * is a small, local duplicate of its allocation sequence -- not an edit to + * it. The CPU-side rpcmem_alloc/rpcmem_to_fd/fastrpc_mmap sequence still + * runs in full ("the host allocates its rpcmem buffer and gets its fd as + * usual"); only the DSP-side registration is withheld. + * ========================================================================*/ +static int alloc_maybe_unmapped(hexlib_ctx *ctx, hexlib_buf **out, size_t size, + int skip_dsp_register) { + *out = NULL; + + void *ptr = hexlib_rpcmem_alloc(RPCMEM_HEAP_ID_SYSTEM, RPCMEM_DEFAULT_FLAGS, + (int) size); + if (ptr == NULL) { + fprintf(stderr, "hexlib: rpcmem_alloc(%zu bytes) failed\n", size); + return -1; + } + + int fd = hexlib_rpcmem_to_fd(ptr); + if (fd < 0) { + fprintf(stderr, "hexlib: rpcmem_to_fd failed\n"); + hexlib_rpcmem_free(ptr); + return -1; + } + + int rc = hexlib_fastrpc_mmap(ctx->domain, fd, ptr, 0, size, FASTRPC_MAP_FD); + if (rc != 0) { + fprintf(stderr, + "hexlib: fastrpc_mmap(fd=%d, size=%zu) failed (rc %d)\n", + fd, size, rc); + hexlib_rpcmem_free(ptr); + return -1; + } + + if (skip_dsp_register) { + /* DELIBERATELY NOT REGISTERED WITH THE SKEL. hexlib_bufs_map()'s + * table lookup (skel_bufs.c) has nothing to find for this fd, so a + * batch that references it must be refused with + * HEXLIB_DSP_ERR_UNMAPPED (7) -- never silently succeed by reading a + * host address, which is the one failure mode the simulator's + * identity-mapped HAP_mmap/rpcmem_to_fd cannot rule out. See the + * file header comment above this function. */ + printf("hexlib: --unmapped: fd %d deliberately not registered with the skel\n", fd); + } else { + /* Registers the SAME fd with the DSP-side skel (hexlib_iface_mmap -> + * hexlib_bufs_register -> HAP_mmap in skel_bufs.c) -- the ordinary + * path, identical to hexlib_alloc()'s own second mapping call. */ + int arc = hexlib_iface_mmap(ctx->handle, (uint32_t) fd, (uint32_t) size); + if (arc != AEE_SUCCESS) { + fprintf(stderr, "hexlib: hexlib_iface_mmap(fd=%d) failed (rc %d)\n", fd, arc); + hexlib_fastrpc_munmap(ctx->domain, fd, ptr, size); + hexlib_rpcmem_free(ptr); + return -1; + } + } + + hexlib_buf *buf = (hexlib_buf *) calloc(1, sizeof(*buf)); + if (buf == NULL) { + if (!skip_dsp_register) { + hexlib_iface_munmap(ctx->handle, (uint32_t) fd); + } + hexlib_fastrpc_munmap(ctx->domain, fd, ptr, size); + hexlib_rpcmem_free(ptr); + return -1; + } + buf->ptr = ptr; + buf->fd = fd; + buf->size = size; + *out = buf; + return 0; +} + +/* Mirror of hexlib_free() (buffers.c), for a buffer allocated by + * alloc_maybe_unmapped() above. `was_unmapped` must match the + * `skip_dsp_register` the buffer was allocated with -- calling + * hexlib_iface_munmap() on an fd that was never registered would just be + * one more no-op RPC, but skipping this parameter entirely and always + * calling it would silently paper over a mismatch between allocation and + * teardown, which is exactly the kind of asymmetry this file's callers must + * get right by construction rather than by accident. */ +static void free_maybe_unmapped(hexlib_ctx *ctx, hexlib_buf *buf, int was_unmapped) { + if (buf == NULL) { + return; + } + if (!was_unmapped) { + hexlib_iface_munmap(ctx->handle, (uint32_t) buf->fd); + } + hexlib_fastrpc_munmap(ctx->domain, buf->fd, buf->ptr, buf->size); + hexlib_rpcmem_free(buf->ptr); + free(buf); +} + +/* `unmapped`: when true, both self-test buffers are allocated via + * alloc_maybe_unmapped() with DSP-side registration withheld -- see that + * function's header comment. The rest of this function is otherwise + * unchanged; the DSP is expected to refuse with HEXLIB_DSP_ERR_UNMAPPED (7), + * which the existing `status != HEXLIB_DSP_OK` branch below already reports + * and turns into HEXLIB_EXIT_OP_FAILED (4). */ +static int run_self_test(int unmapped) { + hexlib_ctx *ctx = NULL; + if (hexlib_open(&ctx, CDSP_DOMAIN_ID) != 0) { + fprintf(stderr, "hexlib: --self-test: could not open a CDSP session\n"); + return HEXLIB_EXIT_SESSION_FAILED; + } + + size_t nbytes = (size_t) SELF_TEST_N * sizeof(__fp16); + hexlib_buf *bx = NULL, *by = NULL; + int alloc_failed = unmapped + ? (alloc_maybe_unmapped(ctx, &bx, nbytes, 1) != 0 || + alloc_maybe_unmapped(ctx, &by, nbytes, 1) != 0) + : (hexlib_alloc(ctx, &bx, nbytes) != 0 || + hexlib_alloc(ctx, &by, nbytes) != 0); + if (alloc_failed) { + fprintf(stderr, "hexlib: --self-test: buffer allocation failed\n"); + if (unmapped) { + free_maybe_unmapped(ctx, bx, 1); + free_maybe_unmapped(ctx, by, 1); + } else { + hexlib_free(ctx, bx); + hexlib_free(ctx, by); + } + hexlib_close(ctx); + return HEXLIB_EXIT_SESSION_FAILED; + } + + /* Deliberately not a single repeated value: exercises the full range + * scale_fp16 handles, body and scalar tail alike. Scaling by a power of + * two (0.125f = 2^-3) only shifts the exponent field -- no mantissa bit + * is lost -- so the expected result is BIT-EXACT, not approximate. Any + * difference at all is therefore a marshalling bug, never a precision + * one; see kernels/scale_fp16/kernel_api.h. */ + __fp16 *x = (__fp16 *) bx->ptr; + for (int i = 0; i < SELF_TEST_N; i++) { + x[i] = (__fp16) ((float) ((i % 17) - 8) * 0.5f); + } + + size_t batch_len = 0; + uint8_t *batch = build_scale_batch(bx->fd, by->fd, nbytes, SELF_TEST_FACTOR, &batch_len); + if (batch == NULL) { + fprintf(stderr, "hexlib: --self-test: out of memory building the batch\n"); + if (unmapped) { + free_maybe_unmapped(ctx, bx, 1); + free_maybe_unmapped(ctx, by, 1); + } else { + hexlib_free(ctx, bx); + hexlib_free(ctx, by); + } + hexlib_close(ctx); + return HEXLIB_EXIT_SESSION_FAILED; + } + + size_t rsp_cap = sizeof(struct hexlib_batch_rsp_hdr) + sizeof(struct hexlib_op_result); + uint8_t *rsp = (uint8_t *) calloc(1, rsp_cap); + size_t rsp_len = 0; + int rc = hexlib_invoke(ctx, batch, batch_len, rsp, rsp_cap, &rsp_len); + + uint32_t status = 0; + int exit_code = HEXLIB_EXIT_OK; + if (rc != 0 || !response_is_valid(rsp, rsp_len, &status)) { + fprintf(stderr, + "hexlib: --self-test: no valid response from the DSP (rc=%d) " + "-- absence of a response is a failure, never a pass\n", rc); + exit_code = HEXLIB_EXIT_NO_RESPONSE; + } else if (status != HEXLIB_DSP_OK) { + fprintf(stderr, "hexlib: --self-test: batch status %u, not HEXLIB_DSP_OK\n", + status); + exit_code = HEXLIB_EXIT_OP_FAILED; + } else { + const struct hexlib_op_result *result = + (const struct hexlib_op_result *) (rsp + sizeof(struct hexlib_batch_rsp_hdr)); + if (rsp_len < sizeof(struct hexlib_batch_rsp_hdr) + sizeof(*result) || + result->status != HEXLIB_DSP_OK) { + fprintf(stderr, "hexlib: --self-test: op result missing or not OK\n"); + exit_code = HEXLIB_EXIT_OP_FAILED; + } else { + const __fp16 *y = (const __fp16 *) by->ptr; + int mismatches = 0; + for (int i = 0; i < SELF_TEST_N; i++) { + __fp16 expect = (__fp16) ((float) x[i] * SELF_TEST_FACTOR); + if (memcmp(&expect, &y[i], sizeof(__fp16)) != 0) { + if (mismatches < 5) { + fprintf(stderr, "hexlib: --self-test: mismatch at index %d\n", i); + } + mismatches++; + } + } + if (mismatches != 0) { + fprintf(stderr, "hexlib: --self-test: %d/%d values not bit-exact\n", + mismatches, SELF_TEST_N); + exit_code = HEXLIB_EXIT_MISMATCH; + } else { + printf("hexlib: --self-test: PASS (%d values, bit-exact)\n", SELF_TEST_N); + /* The response header's own PCYCLE-measured total (see + * skel_dispatch.c) -- the only DSP-measured cycle count this + * binary can report at all. WHAT IT IS FOR: proving the DSP + * measured ANYTHING. A zero here means the counter did not + * advance, which is what PCYCLE does when SYSCFG.PCYCLEEN is + * clear and a user-mode unsigned PD cannot set it -- the one + * thing about stage 3 no simulator run can answer. It is NOT + * a discriminator between a coherency miss and a dispatch + * no-op; --coherency-check's header explains why not. + * Previously validated by response_is_valid() above and read + * fresh here rather than threaded through as an extra + * out-parameter. */ + struct hexlib_batch_rsp_hdr full_hdr; + memcpy(&full_hdr, rsp, sizeof(full_hdr)); + printf("hexlib: --self-test: cycles_total=%llu\n", + (unsigned long long) full_hdr.cycles_total); + } + } + } + + free(rsp); + free(batch); + if (unmapped) { + free_maybe_unmapped(ctx, bx, 1); + free_maybe_unmapped(ctx, by, 1); + } else { + hexlib_free(ctx, bx); + hexlib_free(ctx, by); + } + hexlib_close(ctx); + return exit_code; +} + +/* ========================================================================== + * --coherency-check -- distinguishes a cache-coherency miss from a + * marshalling/dispatch bug. Design doc §6.1 (corrected 2026-08-11). + * + * WHY THIS EXISTS. `buffers.c` maps rpcmem with FASTRPC_MAP_FD, which + * documents as putting cache maintenance on US; `rpcmem` + * allocates CACHED memory by default; and there is no CPU-side flush or + * invalidate call anywhere in the SDK. So a DSP write that never becomes + * visible to the CPU is a real possibility on real hardware, and it + * presents EXACTLY like a marshalling bug: status OK, wrong bytes. This is + * the one place marshalling is exercised at all (see main.c's own file + * header), so the two failure modes would otherwise confound each other + * with no cheaper way to tell them apart. + * + * THIS CHECK DOES NOT ACTUALLY DISCRIMINATE, AND THE PREVIOUS VERSION OF + * THIS COMMENT CLAIMED IT DID. Corrected 2026-08-11 (second correction). + * Read this whole block before believing any verdict this function prints. + * + * The claim that was here was: `cycles_total == 0` means "the kernel never + * ran, a dispatch bug" and `cycles_total > 0` with the sentinel intact means + * "it ran and the write did not reach the host, COHERENCY". Both halves are + * wrong, for two independent reasons, and the table they formed had one + * unreachable row and one row carrying two different defects under one name. + * + * 1. THE `cycles 0` ROW IS UNREACHABLE FROM HERE. Everything below runs + * only after `status == HEXLIB_DSP_OK` AND `result->status == + * HEXLIB_DSP_OK`. In skel_dispatch.c those two are OK only if `k->fn(&a)` + * was genuinely called and genuinely returned OK -- every other path + * writes a specific non-OK status instead. PCYCLE brackets exactly that + * call, so `t1 - t0 > 0` for any real call and this branch cannot be + * reached with cycles_total == 0 while the counter works. The state row 1 + * was reaching for -- dispatch refused the batch -- IS distinguishable, + * but by the exit code and the ABSENCE of any COHERENCY line (the + * `status != HEXLIB_DSP_OK` branch above, HEXLIB_EXIT_OP_FAILED), never + * by a cycle count printed here. + * + * 2. THE DEFECT THE CHECK NAMES IS THE ONE IT CANNOT SEE. "Dispatch + * silently no-ops and still returns HEXLIB_DSP_OK" means a generated + * entry (genentry.py) or a kernel that returns OK WITHOUT WRITING `y`. + * PCYCLE still brackets a real, returning call, so cycles_total > 0 -- + * and the sentinel is intact, because nothing wrote over it. That is + * bit-for-bit the same observable as a genuine coherency miss, and this + * function prints `COHERENCY sentinel_unchanged` and exits 6 for it, + * MISATTRIBUTING A DISPATCH BUG TO COHERENCY. That is precisely the + * confusion §6.1 was added to prevent, reintroduced one level down. + * + * 3. IF PCYCLE READS 0 IN THE UNSIGNED PD, EVERY ROW INVERTS. SYSCFG.PCYCLEEN + * gates whether the counter advances at all and a user-mode unsigned PD + * cannot set it (see skel_dispatch.c's hexlib_read_pcycle and + * include/hexlib/hexlib_harness.h, which sets the bit explicitly for the + * standalone runtime). If HAP_perf_get_pcycles() returns 0 there, a + * genuine coherency miss reads `cycles 0` + sentinel intact -- and the + * old table called that "a dispatch bug". + * + * WHAT THE OBSERVABLES ACTUALLY MEAN. This is the honest table; it names what + * each output is CONSISTENT WITH, not what it proves. + * + * cycles_total | read-back of `y` | printed | exit + * -------------+----------------------+-------------------------+----- + * 0 | any | (any COHERENCY line) | 0/6/7 + * The COUNTER IS DEAD -- not "the kernel never ran". Reaching here at + * all proves k->fn was called and returned OK (see 1). Expected reading + * if PCYCLEEN is clear in the unsigned PD. Settle this before reading + * any row below: with a dead counter no row below means anything. + * -------------+----------------------+-------------------------+----- + * > 0 | every lane magnitude | sentinel_overwritten | 0 + * | zero (+0.0 or -0.0) | | + * The DSP's write reached the host. Healthy -- for scale_fp16's one + * write pattern, this one buffer size, and the DSP-write -> host-read + * direction only. Nothing more. + * -------------+----------------------+-------------------------+----- + * > 0 | every lane bit-exact | sentinel_unchanged | 6 + * | the sentinel | | + * NOT DISCRIMINATED. Consistent with a genuine coherency miss, AND with + * a kernel or generated entry that returned HEXLIB_DSP_OK without + * writing `y`, AND with an fd mapped to a buffer other than the one + * this side reads. Exit 6 says "one of these", never "coherency". DO + * NOT spend a follow-up job on uncached rpcmem off this row alone: + * rule the no-op out first (e.g. by checking the same batch's ordinary + * --self-test, whose factor=0.125 result a no-op cannot produce). + * -------------+----------------------+-------------------------+----- + * > 0 | some lanes neither | buffer_garbled | 7 + * A partial write, or a write that landed somewhere else. Kept as its + * own outcome precisely so it is never folded into the row above. + * + * WHAT WOULD ACTUALLY DISCRIMINATE, and is deliberately not built here: a + * skel-side echo or memset op with its own kind id, whose write is performed + * by the skel itself rather than by any generated kernel. Then "the write did + * not arrive" cannot be a kernel no-op, because no kernel is involved. §6.1 + * notes and defers it; this comment exists so nobody reads the table above as + * a substitute for it. + * + * This function still prints BOTH the cycles_total line and the COHERENCY + * verdict line unconditionally (once the batch status and op status are + * both confirmed OK), so every row above is at least VISIBLE from stdout + * alone -- never just "the bad thing is absent" (see this file's + * project-wide discipline on that, stated in the header above main()). + * Visible is not the same as discriminated, which is the whole point of the + * three paragraphs above. + * + * "SENTINEL INTACT" IS NOT A BIT-COMPARE AGAINST +0.0, AND IT IS A REAL + * CHECK OF THE SENTINEL'S BYTES, NOT JUST "NOT EXACTLY ZERO". Two defects + * were found here and both are fixed the same way: by classifying every + * lane of `y` on read-back, rather than testing a single condition. The + * classification itself lives in hexlib_classify_coherency_lane() below, + * a small pure function kept SEPARATE from this one on purpose -- see its + * own header comment for why. + * + * 1. "The expected result is zero" was checked as `memcmp` against + * `(__fp16) 0.0f`. But COHERENCY_FACTOR is 0.0f and the self-test's own + * input is negative for many lanes (`x[i] = ((i % 17) - 8) * 0.5f`), and + * IEEE-754 makes `x * 0.0f` equal to -0.0 (0x8000) whenever `x` is + * negative -- there is no -ffast-math here (toolchain.py) to paper over + * that. A bit-exact compare against +0.0 therefore read HEALTHY + * hardware as "sentinel unchanged" and reported a coherency miss that + * never happened. Fixed by comparing MAGNITUDE (masking off the sign + * bit, 0x7FFF), which is true of -0.0 and +0.0 alike and is the only + * thing "the write reached the host and reads as zero" actually claims. + * 2. The code never verified the surviving bytes were genuinely the + * SENTINEL before calling them "unchanged" -- a garbled or + * partially-written buffer (neither the expected zero nor the intact + * sentinel) would fall through to the same "sentinel_unchanged" / + * COHERENCY_MISS verdict as a real miss, misattributing a THIRD, worse + * failure mode to this one specific diagnosis. Fixed by requiring an + * exact bit-compare against the sentinel's own bits before calling + * anything "unchanged"; a buffer that is neither all-zero-magnitude nor + * all-sentinel prints its own distinct verdict (`buffer_garbled`, + * HEXLIB_EXIT_COHERENCY_GARBLED) instead of being folded into either. + * + * BOTH DEFECTS WERE FIXED ONCE BEFORE BY SOURCE ALONE -- THIS TIME THE FIX + * IS PROVEN BEHAVIOURALLY. A source assertion (test_host_source.py) can only + * confirm that a magnitude check and a sentinel check EXIST; it cannot + * confirm they classify -0.0 (0x8000) as zero rather than as an unchanged + * sentinel, which is the exact case that produced the original false + * coherency-miss report. hexlib_classify_coherency_lane() is compiled and + * RUN against real bit patterns -- including 0x8000 -- by + * hexlib/tests/test_coherency_lane_classification.py, which is to this fix + * what test_session_arch_decode.py is to the arch-decode fix above it in + * this project's own history: the same defect class (a comparison whose + * OPERANDS were wrong, not merely a comparison whose existence a source + * assertion could confirm), closed the same way. + * + * WHAT THIS DOES NOT PROVE -- DO NOT READ MORE INTO A PASS THAN THIS. + * This exercises only the DSP-write -> host-read direction (the DSP writes + * `y`, the CPU reads it back afterwards). A host-write -> DSP-read miss (the + * CPU writes `x`, the DSP reads something stale from ITS cache) is a + * different direction through the same cache hierarchy and is NOT covered + * here at all. Nor is this kernel-independent: it says something about + * scale_fp16's one write pattern and this one buffer size, not about every + * kernel or every buffer size hexlib might ever dispatch. + * ========================================================================*/ + +/* One fp16 lane's read-back classification: ZERO (the expected result, by + * MAGNITUDE), SENTINEL (bit-exact the value written before invoke, i.e. + * genuinely unchanged), or OTHER (neither -- a garbled or partially-written + * lane, a third outcome that must never be folded into either of the first + * two; see run_coherency_check()'s own header comment above). + * + * TAKES RAW uint16_t BITS, NOT __fp16 VALUES -- ON PURPOSE, NOT MERELY FOR + * CONVENIENCE. Every question this function answers is a question about + * which BITS are set, never about floating-point arithmetic: fp16's only + * two zero bit patterns are 0x0000 (+0.0) and 0x8000 (-0.0), so masking off + * the sign bit (bit 15) and comparing the rest to zero is bit-for-bit + * equivalent to `fabsf((float) v) == 0.0f` for every fp16 value, with no + * float-to-int rounding step in between to second-guess. Expressing the + * check this way -- rather than through __fp16/fabsf() -- means this exact + * function can be extracted and compiled on ANY host C compiler, including + * one with no __fp16 support at all, which is precisely the machine + * hexlib/tests/test_coherency_lane_classification.py's behavioural test + * runs on (see that file's own module docstring). Kept as its own pure + * function (no I/O, no globals, no side effects) for the same reason + * hexlib_decode_bcd_arch() is (session.c) -- so it can be extracted and + * unit-tested directly against the one value that broke this check for + * real (0x8000) rather than only asserted by source pattern. */ +enum hexlib_coherency_lane { + HEXLIB_LANE_ZERO = 0, /* magnitude zero: +0.0 (0x0000) or -0.0 (0x8000) */ + HEXLIB_LANE_SENTINEL = 1, /* bit-exact the sentinel written before invoke */ + HEXLIB_LANE_OTHER = 2, /* neither -- garbled or partially written */ +}; + +static enum hexlib_coherency_lane +hexlib_classify_coherency_lane(uint16_t bits, uint16_t sentinel_bits) { + if ((uint16_t) (bits & 0x7FFFu) == 0) { + return HEXLIB_LANE_ZERO; + } + if (bits == sentinel_bits) { + return HEXLIB_LANE_SENTINEL; + } + return HEXLIB_LANE_OTHER; +} + +static int run_coherency_check(void) { + hexlib_ctx *ctx = NULL; + if (hexlib_open(&ctx, CDSP_DOMAIN_ID) != 0) { + fprintf(stderr, "hexlib: --coherency-check: could not open a CDSP session\n"); + return HEXLIB_EXIT_SESSION_FAILED; + } + + size_t nbytes = (size_t) SELF_TEST_N * sizeof(__fp16); + hexlib_buf *bx = NULL, *by = NULL; + if (hexlib_alloc(ctx, &bx, nbytes) != 0 || hexlib_alloc(ctx, &by, nbytes) != 0) { + fprintf(stderr, "hexlib: --coherency-check: buffer allocation failed\n"); + hexlib_free(ctx, bx); + hexlib_free(ctx, by); + hexlib_close(ctx); + return HEXLIB_EXIT_SESSION_FAILED; + } + + /* `x` need not be anything special -- factor=0.0 makes the correct + * result bit-exact zero regardless of its contents, for any finite, + * non-NaN input. Reused shape from run_self_test purely for a + * reasonable non-degenerate input. */ + __fp16 *x = (__fp16 *) bx->ptr; + for (int i = 0; i < SELF_TEST_N; i++) { + x[i] = (__fp16) ((float) ((i % 17) - 8) * 0.5f); + } + + /* THE SENTINEL. Written into the OUTPUT buffer, before invoke, so that + * only the DSP's own write to `y` -- or the CPU's failure to observe it + * -- can change what this side reads back. */ + __fp16 *y = (__fp16 *) by->ptr; + for (int i = 0; i < SELF_TEST_N; i++) { + y[i] = (__fp16) COHERENCY_SENTINEL; + } + + size_t batch_len = 0; + uint8_t *batch = build_scale_batch(bx->fd, by->fd, nbytes, COHERENCY_FACTOR, &batch_len); + if (batch == NULL) { + fprintf(stderr, "hexlib: --coherency-check: out of memory building the batch\n"); + hexlib_free(ctx, bx); + hexlib_free(ctx, by); + hexlib_close(ctx); + return HEXLIB_EXIT_SESSION_FAILED; + } + + size_t rsp_cap = sizeof(struct hexlib_batch_rsp_hdr) + sizeof(struct hexlib_op_result); + uint8_t *rsp = (uint8_t *) calloc(1, rsp_cap); + size_t rsp_len = 0; + int rc = hexlib_invoke(ctx, batch, batch_len, rsp, rsp_cap, &rsp_len); + + uint32_t status = 0; + int exit_code = HEXLIB_EXIT_OK; + if (rc != 0 || !response_is_valid(rsp, rsp_len, &status)) { + fprintf(stderr, + "hexlib: --coherency-check: no valid response from the DSP (rc=%d) " + "-- absence of a response is a failure, never a pass\n", rc); + exit_code = HEXLIB_EXIT_NO_RESPONSE; + } else if (status != HEXLIB_DSP_OK) { + fprintf(stderr, "hexlib: --coherency-check: batch status %u, not HEXLIB_DSP_OK\n", + status); + exit_code = HEXLIB_EXIT_OP_FAILED; + } else { + const struct hexlib_op_result *result = + (const struct hexlib_op_result *) (rsp + sizeof(struct hexlib_batch_rsp_hdr)); + if (rsp_len < sizeof(struct hexlib_batch_rsp_hdr) + sizeof(*result) || + result->status != HEXLIB_DSP_OK) { + fprintf(stderr, "hexlib: --coherency-check: op result missing or not OK\n"); + exit_code = HEXLIB_EXIT_OP_FAILED; + } else { + /* STATUS OK, PROVEN: marshalling and dispatch both genuinely + * succeeded (both the batch-level status and this op's own + * status say so). Only now is reading the sentinel back + * meaningful at all -- see this function's own header comment. */ + struct hexlib_batch_rsp_hdr full_hdr; + memcpy(&full_hdr, rsp, sizeof(full_hdr)); + + /* Classify every lane through hexlib_classify_coherency_lane() + * -- see that function's own header comment for why both halves + * of this matter and why it operates on raw bits. `all_zero` and + * `all_sentinel` are true only if EVERY lane classified the same + * way; a lane that is neither (HEXLIB_LANE_OTHER) clears both, + * so a garbled or partially-written buffer falls through to its + * own distinct verdict below rather than being reported as + * either a clean pass or a coherency miss. */ + const __fp16 *yr = (const __fp16 *) by->ptr; + __fp16 sentinel = (__fp16) COHERENCY_SENTINEL; + uint16_t sentinel_bits; + memcpy(&sentinel_bits, &sentinel, sizeof(sentinel_bits)); + int all_zero = 1; + int all_sentinel = 1; + for (int i = 0; i < SELF_TEST_N; i++) { + uint16_t bits; + memcpy(&bits, &yr[i], sizeof(bits)); + enum hexlib_coherency_lane lane = + hexlib_classify_coherency_lane(bits, sentinel_bits); + if (lane != HEXLIB_LANE_ZERO) { + all_zero = 0; + } + if (lane != HEXLIB_LANE_SENTINEL) { + all_sentinel = 0; + } + } + + /* Every line, always. cycles_total is printed unconditionally + * because a zero here is the one thing that would invalidate + * every other row of this function's table at once (PCYCLEEN in + * the unsigned PD -- see this function's header), NOT because it + * separates a coherency miss from a dispatch no-op. It does not; + * that claim was wrong and is corrected in the header above. */ + printf("hexlib: --coherency-check: cycles_total=%llu\n", + (unsigned long long) full_hdr.cycles_total); + if (all_zero) { + printf("COHERENCY sentinel_overwritten\n"); + } else if (all_sentinel) { + printf("COHERENCY sentinel_unchanged\n"); + exit_code = HEXLIB_EXIT_COHERENCY_MISS; + } else { + printf("COHERENCY buffer_garbled\n"); + fprintf(stderr, + "hexlib: --coherency-check: output buffer is neither " + "the expected zero result nor the intact sentinel -- " + "a garbled or partially-written buffer, not " + "classifiable as a coherency miss or a clean pass\n"); + exit_code = HEXLIB_EXIT_COHERENCY_GARBLED; + } + } + } + + free(rsp); + free(batch); + hexlib_free(ctx, bx); + hexlib_free(ctx, by); + hexlib_close(ctx); + return exit_code; +} + +static uint8_t *read_file(const char *path, size_t *len_out) { + FILE *f = fopen(path, "rb"); + if (f == NULL) { + return NULL; + } + if (fseek(f, 0, SEEK_END) != 0) { + fclose(f); + return NULL; + } + long n = ftell(f); + if (n < 0 || fseek(f, 0, SEEK_SET) != 0) { + fclose(f); + return NULL; + } + uint8_t *buf = (uint8_t *) malloc((size_t) n > 0 ? (size_t) n : 1); + if (buf == NULL) { + fclose(f); + return NULL; + } + size_t got = fread(buf, 1, (size_t) n, f); + fclose(f); + if (got != (size_t) n) { + free(buf); + return NULL; + } + *len_out = (size_t) n; + return buf; +} + +/* The general path: `--batch ` is a wire-format template built ahead of + * time (buffer sizes and every tensor/op already filled in; each + * hexlib_buf_desc's `fd` is a placeholder this function overwrites once it + * has actually allocated rpcmem for it -- `base` in the template is already + * required to be 0, same as everywhere else on this side of the wire). + * + * CONVENTION, NOT PROTOCOL: this CLI treats every buffer the template + * declares except the last as an input, filled in order from `--in`, and the + * last as the output, written to `--out`. The wire format itself has no + * concept of "input" vs "output" buffer -- that only exists in how an op's + * src/dst reference tensors -- so this is a one-shot-CLI simplification, not + * something skel_dispatch.c or wire.py know about. */ +static int run_batch_file(const char *batch_path, const char *in_path, + const char *out_path) { + size_t tmpl_len = 0; + uint8_t *tmpl = read_file(batch_path, &tmpl_len); + if (tmpl == NULL || tmpl_len < sizeof(struct hexlib_batch_hdr)) { + fprintf(stderr, "hexlib: --batch: could not read %s\n", batch_path); + free(tmpl); + return HEXLIB_EXIT_USAGE; + } + + struct hexlib_batch_hdr hdr; + memcpy(&hdr, tmpl, sizeof(hdr)); + if (hdr.magic != HEXLIB_BATCH_MAGIC) { + fprintf(stderr, "hexlib: --batch: %s is not a hexlib batch (bad magic)\n", + batch_path); + free(tmpl); + return HEXLIB_EXIT_USAGE; + } + if (hdr.n_bufs == 0 || hdr.n_bufs > HEXLIB_MAX_BUFS || + (uint64_t) hdr.off_bufs + (uint64_t) hdr.n_bufs * sizeof(struct hexlib_buf_desc) > tmpl_len) { + fprintf(stderr, "hexlib: --batch: malformed buffer table in %s\n", batch_path); + free(tmpl); + return HEXLIB_EXIT_USAGE; + } + + hexlib_ctx *ctx = NULL; + if (hexlib_open(&ctx, CDSP_DOMAIN_ID) != 0) { + fprintf(stderr, "hexlib: --batch: could not open a CDSP session\n"); + free(tmpl); + return HEXLIB_EXIT_SESSION_FAILED; + } + + hexlib_buf **bufs = (hexlib_buf **) calloc(hdr.n_bufs, sizeof(hexlib_buf *)); + struct hexlib_buf_desc *descs = (struct hexlib_buf_desc *) (tmpl + hdr.off_bufs); + + size_t in_len = 0; + uint8_t *in_data = read_file(in_path, &in_len); + if (in_data == NULL) { + fprintf(stderr, "hexlib: --batch: could not read %s\n", in_path); + free(bufs); + free(tmpl); + hexlib_close(ctx); + return HEXLIB_EXIT_USAGE; + } + + int ok = 1; + size_t in_off = 0; + for (uint32_t i = 0; i < hdr.n_bufs && ok; i++) { + size_t sz = (size_t) descs[i].size; + if (hexlib_alloc(ctx, &bufs[i], sz) != 0) { + fprintf(stderr, "hexlib: --batch: failed to allocate buffer %u (%zu bytes)\n", + i, sz); + ok = 0; + break; + } + if (i + 1 < hdr.n_bufs) { /* an input, per the convention above */ + if (in_off + sz > in_len) { + fprintf(stderr, + "hexlib: --batch: %s is shorter than the inputs the " + "batch template declares\n", in_path); + ok = 0; + break; + } + memcpy(bufs[i]->ptr, in_data + in_off, sz); + in_off += sz; + } else { + /* THE LAST BUFFER IS FILLED TOO IF `--in` CARRIES ITS BYTES. + * + * A WHOLE-PLAN BATCH IS ONE IN-OUT ARENA. hexlib/exec/wholeplan.py + * packs every op of a compiled plan into a single blob over a + * single buffer holding the weights, the activations and the + * graph's input and output all at once -- so that buffer must be + * written by the host BEFORE the invoke (weights, image) and read + * back after it (the encoder's output). With n_bufs == 1 the + * convention above would have left it never filled, and the DSP + * would have multiplied by whatever rpcmem happened to hand back. + * + * NOT a second buffer, deliberately: `simhost.c` patches the SAME + * fd into every buf_desc (one rpcmem allocation for the whole + * batch), so on the simulator two buffers share one address space + * while on a device they do not. The blob that runs on QDC has to + * be the blob the simulator already validated, byte for byte, or + * the rehearsal proves nothing. + * + * The size is checked exactly rather than "as much as is left", so + * a truncated `--in` is a refusal and not a partly-staged arena. */ + size_t remaining = in_len - in_off; + if (remaining == sz) { + memcpy(bufs[i]->ptr, in_data + in_off, sz); + in_off += sz; + } else if (remaining != 0) { + fprintf(stderr, + "hexlib: --batch: %s has %zu bytes left for the final " + "buffer, which is %zu -- pass either nothing for it " + "(output-only) or exactly its size (in-out arena)\n", + in_path, remaining, sz); + ok = 0; + break; + } + } + /* Patch the real fd into the working copy of the buffer table. + * `base` stays 0 -- hexlib_buf_to_desc() never sets anything else. */ + struct hexlib_buf_desc d; + hexlib_buf_to_desc(bufs[i], &d); + memcpy(&descs[i], &d, sizeof(d)); + } + /* NOTHING IN `--in` MAY GO UNUSED. A file longer than the buffers it is + * staging means the caller and the template disagree about the layout, and + * every byte that did land went to an offset derived from that same + * disagreement. Previously this was silent, so a stale or wrongly-built + * `--in` produced a full run and a plausible output file. */ + if (ok && in_off != in_len) { + fprintf(stderr, + "hexlib: --batch: %s is %zu bytes but the batch template " + "consumed only %zu -- the two disagree about the buffer " + "layout\n", in_path, in_len, in_off); + ok = 0; + } + free(in_data); + + int exit_code = HEXLIB_EXIT_OK; + uint8_t *rsp = NULL; + + if (!ok) { + exit_code = HEXLIB_EXIT_USAGE; + } else { + size_t rsp_cap = sizeof(struct hexlib_batch_rsp_hdr) + + (size_t) hdr.n_ops * sizeof(struct hexlib_op_result); + rsp = (uint8_t *) calloc(1, rsp_cap); + size_t rsp_len = 0; + int rc = hexlib_invoke(ctx, tmpl, tmpl_len, rsp, rsp_cap, &rsp_len); + + uint32_t status = 0; + if (rc != 0 || !response_is_valid(rsp, rsp_len, &status)) { + /* NO OUTPUT FILE IS WRITTEN ON THIS PATH. See the file header -- + * an absent, truncated, or wrong-magic response must never be + * mistaken for a result worth saving. */ + fprintf(stderr, + "hexlib: --batch: no valid response from the DSP -- " + "writing no output file\n"); + exit_code = HEXLIB_EXIT_NO_RESPONSE; + } else if (status != HEXLIB_DSP_OK) { + fprintf(stderr, + "hexlib: --batch: batch status %u, not HEXLIB_DSP_OK -- " + "writing no output file\n", status); + exit_code = HEXLIB_EXIT_OP_FAILED; + } else { + /* THE PER-OP RESULTS, PRINTED. They already came back in `rsp` and + * nothing ever looked at them, so when the 49-op encoder returned + * an all-zero final output the only way to find out whether the + * last op had even run was to diff the arena byte by byte on the + * host. It had run; the answer was in the DSP's cache. One summary + * line plus every non-OK op means the next such run says so itself. + * + * cycles_total is DSP-measured and brackets the whole batch, so a + * zero here is the same alarm it is in --self-test: PCYCLE dead in + * the unsigned PD, and every cycle figure meaningless. */ + struct hexlib_batch_rsp_hdr rh; + memcpy(&rh, rsp, sizeof(rh)); + size_t have = (rsp_len - sizeof(rh)) / sizeof(struct hexlib_op_result); + const struct hexlib_op_result *ops = + (const struct hexlib_op_result *) (rsp + sizeof(rh)); + uint32_t n_bad = 0; + for (size_t i = 0; i < have; i++) { + if (ops[i].status != HEXLIB_DSP_OK) { + fprintf(stderr, "hexlib: --batch: op %zu (kind %u) %s\n", + i, (unsigned int) ops[i].kind, + hexlib_dsp_status_name((int) ops[i].status)); + n_bad++; + } + } + printf("hexlib: --batch: %u ops reported, %u not OK, " + "cycles_total=%llu\n", + (unsigned int) rh.n_ops, (unsigned int) n_bad, + (unsigned long long) rh.cycles_total); + if (rh.n_ops != hdr.n_ops) { + /* The batch status was OK, so this cannot be a failed op -- it + * means the DSP stopped early for a reason that did not + * propagate, which would otherwise look like a clean run over + * a plan that was never finished. */ + fprintf(stderr, + "hexlib: --batch: the template carries %u ops but only " + "%u were reported -- the batch did not run to the end\n", + (unsigned int) hdr.n_ops, (unsigned int) rh.n_ops); + exit_code = HEXLIB_EXIT_OP_FAILED; + } + + /* ONLY NOW, after the magic AND the status are both confirmed + * good, does anything get written to disk. */ + hexlib_buf *out_buf = bufs[hdr.n_bufs - 1]; + FILE *f = fopen(out_path, "wb"); + if (f == NULL || fwrite(out_buf->ptr, 1, out_buf->size, f) != out_buf->size) { + fprintf(stderr, "hexlib: --batch: could not write %s\n", out_path); + exit_code = HEXLIB_EXIT_USAGE; + } else { + printf("hexlib: --batch: wrote %zu bytes to %s\n", out_buf->size, out_path); + } + if (f != NULL) { + fclose(f); + } + } + } + + free(rsp); + if (bufs != NULL) { + for (uint32_t i = 0; i < hdr.n_bufs; i++) { + if (bufs[i] != NULL) { + hexlib_free(ctx, bufs[i]); + } + } + free(bufs); + } + free(tmpl); + hexlib_close(ctx); + return exit_code; +} + +int main(int argc, char **argv) { + if (argc >= 2 && strcmp(argv[1], "--caps") == 0) { + /* NOT `print_caps(); return HEXLIB_EXIT_OK;` -- see print_caps()'s own + * header comment. That is what made a driver-load failure exit 0. */ + return print_caps(); + } + if (argc >= 2 && strcmp(argv[1], "--self-test") == 0) { + /* Two independent modifiers, either optional, checked past argv[1]: + * `--unmapped` (run_self_test's own unmapped-buffer path) and + * `--coherency-check` (a distinct function, since it needs a + * pre-written sentinel and a different scale factor). If both are + * given, --coherency-check wins and --unmapped is ignored -- that + * combination is not part of this project's on-device test plan and + * is left unspecified rather than given a third code path. */ + int unmapped = 0, coherency = 0; + for (int i = 2; i < argc; i++) { + if (strcmp(argv[i], "--unmapped") == 0) { + unmapped = 1; + } else if (strcmp(argv[i], "--coherency-check") == 0) { + coherency = 1; + } + } + if (coherency) { + return run_coherency_check(); + } + return run_self_test(unmapped); + } + if (argc >= 2 && strcmp(argv[1], "--batch") == 0) { + const char *batch_path = NULL, *in_path = NULL, *out_path = NULL; + for (int i = 1; i + 1 < argc; i += 2) { + if (strcmp(argv[i], "--batch") == 0) { + batch_path = argv[i + 1]; + } else if (strcmp(argv[i], "--in") == 0) { + in_path = argv[i + 1]; + } else if (strcmp(argv[i], "--out") == 0) { + out_path = argv[i + 1]; + } + } + if (batch_path == NULL || in_path == NULL || out_path == NULL) { + usage(argv[0]); + return HEXLIB_EXIT_USAGE; + } + return run_batch_file(batch_path, in_path, out_path); + } + + usage(argv[0]); + return HEXLIB_EXIT_USAGE; +} diff --git a/hexlib/runtime/host/session.c b/hexlib/runtime/host/session.c new file mode 100644 index 0000000..2094332 --- /dev/null +++ b/hexlib/runtime/host/session.c @@ -0,0 +1,321 @@ +/* hexlib/runtime/host/session.c -- open a session on the CDSP, unsigned PD, + * arch cross-checked rather than assumed. + * + * CDSP ONLY, NEVER ADSP. Measured on the target device (SM8650): CDSP is + * domain 3 and reports UNSIGNED_PD_SUPPORT = 1. ADSP is a v73 part on the + * same SoC and reports UNSIGNED_PD_SUPPORT = 0 -- opening it instead would + * not fail, it would silently produce a measurement from different hardware. + * hexlib_open (below) refuses any domain that is not CDSP_DOMAIN_ID outright. + * + * NO LITERAL REQUEST IDS. A wrong request-id constant does not fail loudly -- + * it queries something else on the DSP, or comes back with an error that + * reads exactly like "unsupported", which is indistinguishable from the + * answer a genuinely unsupported query would give. (This project hardcoded + * DSPRPC_GET_DSP_INFO as 11 once, by counting an enum in a doc comment; the + * real value, from 's own `enum handle_control_req_id`, is 2.) + * Every request id and every capability attribute in this file is therefore + * spelled by name from -- `enum handle_control_req_id`, `enum + * remote_dsp_attributes`, `enum session_control_req_id` -- never typed as a + * bare number. + * + * THE ARCH IS QUERIED, NEVER ASSUMED, ON BOTH SIDES OF THE WIRE, AND THE TWO + * ARE CROSS-CHECKED. `hexlib_query_caps` asks the DRIVER what silicon this + * is (ARCH_VER via DSPRPC_GET_DSP_INFO); `hexlib_iface_hwinfo` asks the SKEL + * what it was compiled for (__HEXAGON_ARCH__, baked in at Task 7 build time). + * The two must agree -- disagreement means the wrong skel .so is loaded for + * this part, a version-skew bug, not a hardware fact -- so hexlib_open + * fails rather than proceeding on a mismatched measurement. + * + * THE TWO SIDES ARE IN DIFFERENT ENCODINGS -- DECODE, NEVER COMPARE RAW. + * `arch` (skel hwinfo) is plain decimal: __HEXAGON_ARCH__ is 75 on the + * measured target (see skel.c, test_dsp_sim.py). `caps.arch_ver` (driver + * ARCH_VER) is NOT the same number in the same base: on the identical part + * it reads 35957 = 0x8c75 (see device/qdc/test_on_device.py, job.py's own + * measured-facts header). The low byte packs the arch as two BCD digits -- + * 0x75 means digits 7 and 5, i.e. 75, not the integer 0x75 = 117 and + * certainly not 35957. Comparing the raw values, as an earlier draft of + * this file did, is unconditionally false for every real device: no session + * could ever open. `hexlib_decode_bcd_arch` below does the same decode + * llama.cpp's `htpdrv_get_arch` does (ggml-hexagon/htp-drv.cpp:412-413, MIT; + * see ATTRIBUTION.md) -- `val = arch_ver & 0xff; arch = (val >> 4) * 10 + + * (val & 0x0f)` -- and hexlib_open compares ITS output against `arch`, never + * `caps.arch_ver` directly. + */ +#include "hexlib_host.h" + +#include +#include +#include +#include +#include + +#include "hexlib_dsp.h" /* struct hexlib_batch_rsp_hdr / hexlib_op_result */ +#include "hexlib_iface.h" /* qaic-generated from runtime/idl/hexlib_iface.idl */ + +int hexlib_query_caps(int domain, struct hexlib_caps *out) { + memset(out, 0, sizeof(*out)); + + struct { + enum remote_dsp_attributes attr; + uint32_t *dst; + } queries[] = { + { DOMAIN_SUPPORT, &out->domain_support }, + { UNSIGNED_PD_SUPPORT, &out->unsigned_pd_support }, + { HVX_SUPPORT_128B, &out->hvx_support_128b }, + { VTCM_PAGE, &out->vtcm_page }, + { VTCM_COUNT, &out->vtcm_count }, + { ARCH_VER, &out->arch_ver }, + { HMX_SUPPORT_DEPTH, &out->hmx_support_depth }, + }; + + for (size_t i = 0; i < sizeof(queries) / sizeof(queries[0]); i++) { + struct remote_dsp_capability cap; + memset(&cap, 0, sizeof(cap)); + cap.domain = (uint32_t) domain; + cap.attribute_ID = (uint32_t) queries[i].attr; + + /* DSPRPC_GET_DSP_INFO, from 's own `enum + * handle_control_req_id` -- see the file header. */ + int rc = hexlib_remote_handle_control(DSPRPC_GET_DSP_INFO, &cap, sizeof(cap)); + if (rc != 0) { + fprintf(stderr, + "hexlib: DSPRPC_GET_DSP_INFO attribute %u failed (rc %d)\n", + (unsigned) queries[i].attr, rc); + return -1; + } + *queries[i].dst = cap.capability; + } + return 0; +} + +/* Must run BEFORE the handle is opened -- once the PD exists, it is already + * signed or unsigned. Request id from 's `enum + * session_control_req_id`, never a literal. */ +static int enable_unsigned_pd(int domain) { + struct remote_rpc_control_unsigned_module req; + memset(&req, 0, sizeof(req)); + req.domain = domain; + req.enable = 1; /* Measured UNSIGNED_PD_SUPPORT = 1 on CDSP; see caller. */ + + int rc = hexlib_remote_session_control(DSPRPC_CONTROL_UNSIGNED_MODULE, + &req, sizeof(req)); + if (rc != 0) { + fprintf(stderr, + "hexlib: DSPRPC_CONTROL_UNSIGNED_MODULE failed (rc %d)\n", rc); + } + return rc; +} + +/* Pure BCD-nibble decode of the driver's ARCH_VER capability -- byte-for-byte + * ported from llama.cpp's htpdrv_get_arch (ggml-hexagon/htp-drv.cpp:412-413, + * MIT; see ATTRIBUTION.md). ARCH_VER's low byte packs the arch as two BCD + * digits (0x8c75 -> low byte 0x75 -> nibbles 7 and 5 -> 75), not the plain + * integer __HEXAGON_ARCH__ encodes -- see this file's own header comment for + * why comparing the raw values can never agree. Kept as its own pure + * function (no I/O, no globals, no side effects) so it can be extracted and + * unit-tested directly against the one measured value this project has on + * record (0x8c75 -> 75) rather than only asserted by source pattern -- see + * hexlib/tests/test_session_arch_decode.py. */ +static uint32_t hexlib_decode_bcd_arch(uint32_t arch_ver) { + uint32_t val = arch_ver & 0xff; + return (val >> 4) * 10 + (val & 0x0f); +} + +int hexlib_open(hexlib_ctx **out, int domain) { + *out = NULL; + + /* Refuse anything that is not CDSP outright. ADSP is a real, openable + * domain on the same device -- opening it would not fail, it would + * silently measure different hardware. See the file header. */ + if (domain != CDSP_DOMAIN_ID) { + fprintf(stderr, + "hexlib: refusing domain %d -- only CDSP_DOMAIN_ID (%d) is " + "supported; ADSP is a v73 part and must never be substituted\n", + domain, CDSP_DOMAIN_ID); + return -1; + } + + if (hexlib_drv_init() != 0) { + return -1; + } + + struct hexlib_caps caps; + if (hexlib_query_caps(domain, &caps) != 0) { + return -1; + } + if (!caps.unsigned_pd_support) { + fprintf(stderr, + "hexlib: CDSP reports UNSIGNED_PD_SUPPORT = 0 on this device " + "(the measured target reports 1) -- refusing rather than " + "silently taking a signed-PD path that has never been " + "measured\n"); + return -1; + } + + if (enable_unsigned_pd(domain) != 0) { + return -1; + } + + /* hexlib_iface_URI is qaic-generated (hexlib_iface.h); CDSP_DOMAIN is + * 's own domain-suffix macro (&_dom=cdsp). Adjacent + * string-literal concatenation -- neither half is retyped by hand, so + * this is built, not a domain suffix written out by hand again here. */ + static const char uri[] = hexlib_iface_URI CDSP_DOMAIN; + + hexlib_ctx *ctx = (hexlib_ctx *) calloc(1, sizeof(*ctx)); + if (ctx == NULL) { + return -1; + } + ctx->domain = domain; + + int rc = hexlib_iface_open(uri, &ctx->handle); + if (rc != 0) { + fprintf(stderr, "hexlib: hexlib_iface_open(%s) failed (rc %d)\n", uri, rc); + free(ctx); + return -1; + } + + rc = hexlib_iface_start(ctx->handle, /* sess_id */ 0, /* n_hvx */ 0, + /* n_hmx */ 0, /* max_vmem: unbounded for now */ 0); + if (rc != AEE_SUCCESS) { + /* DECODED, NOT PRINTED RAW. `skel.c:58-72` tags a real DSP status into + * the AEE return with HEXLIB_AEE_FROM_STATUS precisely so a VTCM + * contention failure can be told apart from a signing failure or a + * missing skel -- and until now NOTHING ON THE HOST READ IT. The macros + * appeared only in the header and in a test probe, so the detail was on + * the wire and every cause printed the same bare negative number, which + * is the operator confusion the tag exists to remove. */ + if (HEXLIB_AEE_IS_STATUS(rc)) { + fprintf(stderr, + "hexlib: hexlib_iface_start failed (rc %d) -- the DSP " + "reported status %d: %s\n", + rc, HEXLIB_AEE_STATUS(rc), + hexlib_dsp_status_name(HEXLIB_AEE_STATUS(rc))); + } else { + fprintf(stderr, + "hexlib: hexlib_iface_start failed (rc %d) -- no DSP status " + "tag, so this came from qaic or the RPC layer, not from the " + "skel's own code\n", rc); + } + hexlib_iface_close(ctx->handle); + free(ctx); + return -1; + } + + uint32_t arch = 0, n_threads = 0, n_hvx = 0, n_hmx = 0; + /* `uint64`, not `uint64_t`: hexlib_iface_hwinfo's qaic-generated + * prototype (AEEStdDef.h's `unsigned __int64`) is a distinct type from + * 's uint64_t on an LP64 target even though both are 64 bits, + * and passing the wrong one is a real pointer-type mismatch, not + * pedantry -- confirmed by `-fsyntax-only` against the real generated + * header (see the task report). */ + uint64 vtcm_size = 0; + rc = hexlib_iface_hwinfo(ctx->handle, &arch, &n_threads, &n_hvx, &n_hmx, &vtcm_size); + if (rc != AEE_SUCCESS) { + fprintf(stderr, "hexlib: hexlib_iface_hwinfo failed (rc %d)\n", rc); + hexlib_iface_stop(ctx->handle); + hexlib_iface_close(ctx->handle); + free(ctx); + return -1; + } + + /* CROSS-CHECK: the arch the DRIVER reports (queried above, from the CDSP + * firmware itself) against the arch the SKEL reports (what THIS .so was + * compiled for). See the file header -- disagreement is a version-skew + * bug and must fail, not merely log. + * + * THE DRIVER'S VALUE IS DECODED FIRST -- see hexlib_decode_bcd_arch() + * and this file's own header comment. Comparing `arch` against + * `caps.arch_ver` directly (its raw, BCD-packed encoding) would be + * unconditionally false on every real device -- e.g. 75 != 35957 -- and + * every session would refuse before measuring anything. */ + uint32_t driver_arch = hexlib_decode_bcd_arch(caps.arch_ver); + if (arch != driver_arch) { + fprintf(stderr, + "hexlib: arch mismatch -- driver ARCH_VER raw=%u (0x%04x) " + "decodes to %u, skel hwinfo reports %u; refusing to run a " + "mismatched binary\n", + caps.arch_ver, caps.arch_ver, driver_arch, arch); + hexlib_iface_stop(ctx->handle); + hexlib_iface_close(ctx->handle); + free(ctx); + return -1; + } + + ctx->arch = arch; + ctx->n_threads = n_threads; + ctx->n_hvx = n_hvx; + ctx->n_hmx = n_hmx; + ctx->vtcm_size = vtcm_size; + ctx->started = 1; + + *out = ctx; + return 0; +} + +int hexlib_close(hexlib_ctx *ctx) { + if (ctx == NULL) { + return 0; + } + int rc = AEE_SUCCESS; + if (ctx->started) { + int src = hexlib_iface_stop(ctx->handle); + if (src != AEE_SUCCESS) { + rc = src; + } + ctx->started = 0; + } + int crc = hexlib_iface_close(ctx->handle); + if (crc != 0 && rc == AEE_SUCCESS) { + rc = crc; + } + free(ctx); + return rc == AEE_SUCCESS ? 0 : -1; +} + +int hexlib_hwinfo(hexlib_ctx *ctx, uint32_t *arch, uint32_t *n_threads, + uint32_t *n_hvx, uint32_t *n_hmx, uint64_t *vtcm_size) { + *arch = ctx->arch; + *n_threads = ctx->n_threads; + *n_hvx = ctx->n_hvx; + *n_hmx = ctx->n_hmx; + *vtcm_size = ctx->vtcm_size; + return 0; +} + +int hexlib_invoke(hexlib_ctx *ctx, const void *batch, size_t batch_len, + void *rsp, size_t rsp_cap, size_t *rsp_len) { + *rsp_len = 0; + if (!ctx->started) { + return -1; + } + + /* THE FIRST CODE IN THIS PROJECT TO EXERCISE QAIC'S REAL ARGUMENT + * MARSHALLING. Every simulator run through Task 8 called skel.c's + * hexlib_iface_invoke as a plain C function in the same address space + * (see runtime/build.py's build_sim_so); the qaic stub was never + * linked there. Here it is: this call goes through the generated + * hexlib_iface_stub.c, which marshals `batch`/`result` into a + * remote_arg[] and calls remote_handle64_invoke() for real. */ + int rc = hexlib_iface_invoke(ctx->handle, (const unsigned char *) batch, + (int) batch_len, (unsigned char *) rsp, + (int) rsp_cap); + if (rc != AEE_SUCCESS) { + return -1; + } + + /* NO resultLenOut. `rout sequence result` marshals only a + * CAPACITY (see hexlib_iface.h / ATTRIBUTION.md) -- there is no wire + * channel back to the host for "how many bytes are meaningful". The + * response is self-describing instead: hexlib_batch_rsp_hdr.n_ops says + * how many hexlib_op_result entries follow. Compute the real length from + * THAT, never from anything qaic handed back (it handed back nothing). */ + if (rsp_cap < sizeof(struct hexlib_batch_rsp_hdr)) { + return -1; + } + struct hexlib_batch_rsp_hdr hdr; + memcpy(&hdr, rsp, sizeof(hdr)); + size_t need = sizeof(hdr) + (size_t) hdr.n_ops * sizeof(struct hexlib_op_result); + *rsp_len = need <= rsp_cap ? need : rsp_cap; + return 0; +} diff --git a/hexlib/runtime/idl/hexlib_iface.idl b/hexlib/runtime/idl/hexlib_iface.idl new file mode 100644 index 0000000..5e889cb --- /dev/null +++ b/hexlib/runtime/idl/hexlib_iface.idl @@ -0,0 +1,69 @@ +//============================================================================ +/// hexlib_iface.idl -- the FastRPC interface for running hexlib kernels on a +/// Hexagon DSP. +/// +/// SYNCHRONOUS, NOT DSPQUEUE, AND THAT IS THE WHOLE DESIGN. llama.cpp's +/// htp_iface carries no ops at all: `start` takes a queue identifier and every +/// op travels as a dspqueue packet. dspqueue has no simulator path -- of all SDK +/// examples, the six that build a simulator-runnable target are every one of +/// them plain synchronous FastRPC -- so adopting it would mean the first time +/// this interface ever ran would be on non-renewable device minutes. +/// +/// dspqueue amortizes per-call latency over thousands of small dispatches. +/// hexlib has a compiler that knows all 308 steps before the first one runs, so +/// one call carries a whole batch and there is nothing to amortize. `invoke` +/// takes opaque bytes precisely so the same descriptor can move to a queue +/// packet later without touching this file. +/// +/// Session lifecycle adapted from llama.cpp ggml-hexagon's htp_iface.idl (MIT); +/// see ATTRIBUTION.md. +//============================================================================ + +#ifndef HEXLIB_IDL +#define HEXLIB_IDL + +#include "AEEStdDef.idl" +#include "remote.idl" + +interface hexlib_iface : remote_handle64 { + + /// Acquire VTCM and open a session. n_hvx/n_hmx are unit counts; max_vmem + /// caps how much host memory may stay mapped at once. + AEEResult start(in uint32 sess_id, in uint32 n_hvx, in uint32 n_hmx, + in uint64 max_vmem); + + AEEResult stop(); + + /// Register a host buffer by fd. The DSP maps it ITSELF -- the host never + /// sends an address. + AEEResult mmap(in uint32 fd, in uint32 size); + AEEResult munmap(in uint32 fd); + + /// ONLY vtcm_size IS A DSP FACT. Corrected 2026-08-11; this said "what the + /// DSP says about itself, never what a marketing name implies", which is + /// true of exactly one of the five out-parameters: + /// arch -- COMPILE-TIME __HEXAGON_ARCH__, i.e. what this skel was + /// built for. Still useful: the host cross-checks it against + /// the driver's ARCH_VER so a wrong-arch skel is a visible + /// disagreement rather than a mystery. But it is not a query. + /// n_threads -- HARDCODED 1. + /// n_hvx -- ECHO of what the host passed to start(). + /// n_hmx -- ECHO of what the host passed to start(). + /// vtcm_size -- the ACQUIRED size, genuinely queried, not the part's total. + /// The echoes are circular by construction: hexlib_open passes 0 for both and + /// then records what comes back as a capability. Read the real values from + /// the DRIVER side instead -- hexlib_caps already carries hvx_support_128b + /// and hmx_support_depth from remote_dsp_capability. + /// They are not fixed rather than reworded because start()'s n_hmx drives + /// whether the skel requests HMX from the compute-res manager, and changing + /// what the host sends there changes acquisition behaviour on hardware that + /// has never run this code. See design doc SS4. + AEEResult hwinfo(rout uint32 arch, rout uint32 n_threads, rout uint32 n_hvx, + rout uint32 n_hmx, rout uint64 vtcm_size); + + /// Run a batch. `batch` is hexlib_batch_hdr and friends; `result` is + /// hexlib_batch_rsp_hdr and friends. + AEEResult invoke(in sequence batch, rout sequence result); +}; + +#endif /* HEXLIB_IDL */ diff --git a/hexlib/runtime/simhost/sim_shims.c b/hexlib/runtime/simhost/sim_shims.c new file mode 100644 index 0000000..42dc565 --- /dev/null +++ b/hexlib/runtime/simhost/sim_shims.c @@ -0,0 +1,71 @@ +/* hexlib/runtime/simhost/sim_shims.c -- symbols this SDK's local test library + * does not provide, needed only to link and run under the simulator. + * + * TASK 7B UPDATE: this file is now compiled into the QuRT-hosted shared + * object build_sim_so produces (dlopen'd by the SDK's own prebuilt + * run_main_on_hexagon_sim under a real QuRT kernel), not into task 7's + * retired standalone qexe. Nothing in this file changed -- it still wraps + * test_util.a's int-length HAP_mmap/HAP_munmap, and test_util.a is still + * confirmed absent of a real HAP_mmap2/HAP_munmap2 for any Hexagon target + * (see below). Under the new packaging, `HAP_mmap` itself is resolved + * DYNAMICALLY at dlopen time against the host process (run_main_on_hexagon_sim + * already links test_util.a `--whole-archive` and exports its symbols), + * rather than being statically linked directly into the same standalone qexe + * as this file -- a different resolution mechanism, but the same concrete + * function, with the same fd-as-address behavior documented below. + * + * HAP_mmap2/HAP_munmap2 (declared in HAP_mem.h for every Hexagon target) are + * what skel_bufs.c calls, deliberately preferred over the older int-length + * HAP_mmap/HAP_munmap for size_t safety on large buffers (see + * task-4-report.md). On real silicon they are backed by QuRT. This SDK's + * utils/sim_utils/src/test_utils.c -- built into test_util.a, the local-test + * transport this .so ultimately depends on (via the host process it is + * dlopen'd into) instead of a real device driver -- predates the "2" + * variants and defines only the int-length pair. `nm` across every prebuilt + * .a in this SDK confirms HAP_mmap2/HAP_munmap2 are defined nowhere for a + * Hexagon target build (checked again for task 7b, including real + * `libqurt.a` -- also absent there). + * + * These thin wrappers exist ONLY so the simulator artifact links and runs; + * they do not change skel_bufs.c's own size_t-safe call, and they are never + * linked into anything that runs on silicon (a real device skel links + * against the real QuRT-backed HAP_mmap2, not this file). hexlib_sim.so's + * own buffers are bounded by MAX_BLOB (16 MiB, see simhost.c), well inside + * `int` range, so the narrowing here is safe for what this harness actually + * exercises. + * + * WHAT THIS SHIM MAKES UNTESTABLE UNDER THE SIMULATOR -- read this before + * trusting a simulator pass on the buffer-mapping path. The HAP_mmap this + * wraps is test_util.c's, and its ENTIRE body (utils/sim_utils/src/ + * test_utils.c:59) is `return (void *)(uintptr_t)fd;`. It never fails for any + * nonzero fd, and the "address" it returns IS the fd's bit pattern, not a + * real mapping of anything. + * + * 1. hexlib_bufs_register's own failure path (HEXLIB_DSP_ERR_MMAP_FAILED, + * in skel_bufs.c, taken when HAP_mmap2 returns 0 or -1) is UNREACHABLE + * under the simulator for any realistic nonzero fd. A simulator pass + * proves nothing about that path; only silicon, where HAP_mmap2 talks to + * a real mapper that can genuinely fail, can exercise it. + * + * 2. Because the fd doubles as the "address" here, the simulator cannot + * tell "resolved the fd through hexlib_bufs_register's table" apart from + * "happened to use the fd as an address" by comparing VALUES alone -- + * both would produce the same base pointer for a registered fd. What + * still discriminates the two is the table LOOKUP itself, not the + * value: hexlib_bufs_map (skel_bufs.c) consults ctx->mmap, and that + * table is populated ONLY by hexlib_bufs_register. A never-registered fd + * has no entry regardless of what HAP_mmap would have returned for it, + * so hexlib_bufs_map still refuses it (HEXLIB_DSP_ERR_UNMAPPED). That is + * why the --unmapped test (see simhost.c) remains a real discriminator + * even with this fd-as-address stand-in underneath it: it is testing + * whether the lookup happened at all, not what value came out of it. + */ +#include "HAP_mem.h" + +void *HAP_mmap2(void *addr, size_t len, int prot, int flags, int fd, long offset) { + return HAP_mmap(addr, (int) len, prot, flags, fd, offset); +} + +int HAP_munmap2(void *addr, size_t len) { + return HAP_munmap(addr, (int) len); +} diff --git a/hexlib/runtime/simhost/simhost.c b/hexlib/runtime/simhost/simhost.c new file mode 100644 index 0000000..8e68263 --- /dev/null +++ b/hexlib/runtime/simhost/simhost.c @@ -0,0 +1,283 @@ +/* hexlib/runtime/simhost/simhost.c -- the host side, for the simulator. + * + * TASK 7B UPDATE: this file is now compiled into a QuRT-hosted SHARED OBJECT + * (build_sim_so, hexlib/runtime/build.py), dlopen'd by the SDK's own prebuilt + * `run_main_on_hexagon_sim` under a real booted QuRT kernel, instead of a + * standalone `--force-dynamic` qexe (task 7's build_sim_qexe, retired -- see + * build.py's module docstring for why). This file's OWN code did not change; + * only how it gets packaged and launched did. It still has a plain `main()` + * that `run_main_on_hexagon`'s own dsp-side driver calls after dlopen. + * + * ============================================================================ + * WHAT A SIMULATOR RUN OF THIS FILE DOES NOT PROVE -- READ THIS FIRST. + * + * This file calls hexlib_iface_open/_start/_mmap/_invoke/_stop/_close as + * PLAIN C FUNCTIONS, bound by the linker DIRECTLY to skel.c's definitions -- + * simhost.o and the skel archive are both compiled into the SAME .so, so + * this is still an ordinary intra-module call, not a qaic-marshalled one. + * The qaic-generated stub (hexlib_iface_stub.c) -- the code that would + * actually marshal these calls into a `remote_arg` scalar/buffer list and + * drive them through `remote_handle64_open`/`_invoke` -- is DELIBERATELY NOT + * LINKED INTO THIS .SO AT ALL. It defines the exact same function names as + * skel.c's DSP-side implementation (confirmed by running qaic and reading + * both generated files back), so linking both into one module is a + * duplicate-symbol error, not merely redundant. The SDK's own calculator + * example makes the identical choice: `calculator_q_C_SRCS` in + * examples/calculator/hexagon.min never includes calculator_stub.c either. + * Packaging this file as a shared object rather than a standalone executable + * does NOT change this -- it changes how VTCM's own weak symbols get + * resolved (dynamically, against the host process, at dlopen time -- see + * build_sim_so's comment in build.py), not whether the qaic stub is linked + * (it still is not). + * + * CONSEQUENCE: a simulator run through this file exercises hexlib's OWN + * code -- batch parsing (hexlib_dispatch_batch), the buffer table + * (hexlib_bufs_register/_map), the kernel dispatch table + * (hexlib_kernel_table), kernel correctness, PCYCLE accounting, and now (as + * of task 7b) real VTCM acquisition -- but it does NOT exercise qaic's + * argument marshaling/demarshaling at all. That is a real gap against this + * project's own design spec, which describes the simulator path as + * exercising "a qaic stub/skel invoke": what actually happens here is a + * plain function call, and the marshaling layer is completely bypassed. + * Marshaling is only exercised on a real device, where the stub and skel + * genuinely live in separate processes and the call cannot avoid the wire. + * ============================================================================ + * + * WHY THIS EXISTS. On a device the host is an aarch64 Android binary. On the + * simulator there is no aarch64, so the "host" is Hexagon code in the same + * module as the skel. Originally (task 7) that module was a monolithic + * standalone qexe, the SDK's own BUILD_QEXES pattern (examples/calculator's + * calculator_q). As of task 7b it is a shared object instead, because a + * standalone qexe can never satisfy VTCM's real client/server manager (real + * QuRT thread/clock primitives it structurally cannot host) -- see + * .superpowers/sdd/2026-08-10-silicon-path-runtime/ + * investigation-sim-vtcm-and-marshalling.md. + * + * WHY THIS FILE CALLS hexlib_iface_open/start/mmap/invoke/stop/close DIRECTLY, + * NOT THROUGH THE QAIC-GENERATED STUB. Reading calculator_q's own link line and + * its generated calculator_stub.c/calculator_skel.c settled it: the generated + * STUB (hexlib_iface_stub.c) defines the SAME function names + * (hexlib_iface_open, _start, _mmap, _invoke, ...) as the DEVELOPER'S skel-side + * implementation in skel.c -- on a device these live in two different ELFs + * (host APK vs. DSP .so) so the names never collide, but statically linking + * both into ONE module would be a duplicate-symbol error. calculator's own + * hexagon.min never compiles calculator_stub.c into calculator_q either: only + * the generated *_skel.c (an unused, harmless archive member here) and the + * developer's *_imp.c (which implements calculator_open/_close/_sum/_max + * directly) go into calculator_q's link. calculator_test.c's calls to + * calculator_open/_sum resolve straight to that developer implementation -- + * there is no marshaling, no remote_handle64_open/_invoke, on this path at all + * (confirmed by `hexagon-nm` on rtld.a/test_util.a/atomic.a: none of them + * define remote_handle64_open/_close/_invoke). This file follows the same + * shape: it calls hexlib_iface_open/etc. as plain C functions, which the + * linker binds directly to skel.c's definitions. + * + * FILE I/O UNDER THE QURT-HOSTED PACKAGING -- READS AND WRITES ARE NOT + * SYMMETRIC, CONFIRMED EMPIRICALLY (task 7b). Under task 7's standalone qexe, + * relative fopen() paths resolved through hexagon-sim's own `--usefs ` + * angel-mode redirection for both reads and writes, so hexlib_in.bin/ + * hexlib_batch.bin/hexlib_out.bin/hexlib_rsp.bin all lived in one place. Under + * this file's new QuRT-hosted packaging, relative fopen() READS of + * hexlib_batch.bin/hexlib_in.bin still resolve through --usefs correctly + * (verified: a batch that exists ONLY under --usefs's directory, nowhere + * else, is read and executed correctly). But relative fopen(..., "wb") WRITES + * of hexlib_rsp.bin/hexlib_out.bin land in the REAL launching process's own + * working directory instead -- QuRT's own POSIX filesystem layer, not the + * simulator's angel-mode redirection, appears to own file creation once a + * real QuRT kernel is booted, and it does not consult --usefs the same way. + * CONSEQUENCE FOR CALLERS (Task 8): a harness that wants the response/output + * files to land next to the batch/input files it wrote MUST launch the + * hexagon-sim subprocess with its OWN working directory set to the same + * directory passed as --usefs (e.g. Python's subprocess `cwd=` kwarg) -- + * this file cannot fix this from its own side, because it does not know at + * compile time what directory a future caller will use as --usefs. + * + * THE ONE THING TO BE CAREFUL ABOUT. Host and DSP are one address space here. + * This file must never hand the skel a pointer; it registers an fd with + * rpcmem_alloc()+rpcmem_to_fd() and sends offsets, exactly as the device host + * does. `--unmapped` exercises the negative case, which is the test that makes + * a simulator pass transferable: it deliberately skips hexlib_iface_mmap, so + * the skel must refuse (HEXLIB_DSP_ERR_UNMAPPED), not silently read the host's + * address the way a shared-address-space bug would let it. THIS BEHAVIOR MUST + * NOT CHANGE: `--unmapped` is load-bearing for Task 8's own discriminator + * test, which proves a skel that leaned on the shared address space would + * pass a request whose buffer was never mapped -- do not "fix" this into + * mapping anyway. + * + * hexlib_iface_invoke HAS NO "resultLenOut" PARAMETER (see skel.c's own header + * comment: `rout sequence result` marshals only a capacity). The + * response is self-describing -- hexlib_batch_rsp_hdr.n_ops says how many + * hexlib_op_result entries follow -- so that is what this file uses to decide + * how many bytes of the response buffer are meaningful. + */ +#include +#include +#include + +#include "hexlib_dsp.h" +#include "hexlib_iface.h" +#include "rpcmem.h" + +#define MAX_BLOB (16 * 1024 * 1024) + +static unsigned char g_batch[65536]; +static unsigned char g_rsp[65536]; + +static long read_file(const char *path, void *dst, long cap) { + FILE *f = fopen(path, "rb"); + if (!f) return -1; + long n = (long) fread(dst, 1, (size_t) cap, f); + fclose(f); + return n; +} + +int main(int argc, char **argv) { + int want_unmapped = 0; + for (int i = 1; i < argc; i++) { + if (strcmp(argv[i], "--unmapped") == 0) want_unmapped = 1; + } + + remote_handle64 h = 0; + int rc = hexlib_iface_open(hexlib_iface_URI, &h); + if (rc != 0) { + printf("SIMHOST error=open rc=%d\n", rc); + return 2; + } + + rc = hexlib_iface_start(h, 1, 1, 1, (uint64) MAX_BLOB); + if (rc != 0) { + printf("SIMHOST error=start rc=%d\n", rc); + return 3; + } + + uint32 arch = 0, nthr = 0, nhvx = 0, nhmx = 0; + uint64 vtcm = 0; + rc = hexlib_iface_hwinfo(h, &arch, &nthr, &nhvx, &nhmx, &vtcm); + if (rc != 0) { + printf("SIMHOST error=hwinfo rc=%d\n", rc); + return 4; + } + printf("SIMHOST hwinfo arch=%u threads=%u vtcm=%llu\n", + (unsigned int) arch, (unsigned int) nthr, (unsigned long long) vtcm); + + long blen = read_file("hexlib_batch.bin", g_batch, (long) sizeof(g_batch)); + if (blen <= 0) { + printf("SIMHOST error=no_batch\n"); + return 5; + } + + /* The payload buffer. rpcmem gives an fd, which is the ONLY thing the skel + * is told; it maps that fd itself and computes every address. */ + void *data = rpcmem_alloc(RPCMEM_HEAP_ID_SYSTEM, RPCMEM_DEFAULT_FLAGS, MAX_BLOB); + if (!data) { + printf("SIMHOST error=rpcmem_alloc\n"); + return 6; + } + long dlen = read_file("hexlib_in.bin", data, MAX_BLOB); + if (dlen < 0) { + printf("SIMHOST error=no_input\n"); + rpcmem_free(data); + return 7; + } + int fd = rpcmem_to_fd(data); + + if (!want_unmapped) { + rc = hexlib_iface_mmap(h, (uint32) fd, (uint32) MAX_BLOB); + if (rc != 0) { + printf("SIMHOST error=mmap rc=%d\n", rc); + rpcmem_free(data); + return 8; + } + } else { + /* DELIBERATELY NOT MAPPED. The skel must refuse. If it returns a + * result anyway, it read the host's address -- which works here and + * would fail on silicon. This is the discriminator. */ + printf("SIMHOST note=fd_deliberately_unmapped\n"); + } + + /* The batch was built by the host with fd 0 as a placeholder; patch in the + * real fd. Offsets are unchanged -- they are all this side ever sends. + * + * BOUNDED HERE, AND NOT BY BORROWING skel_dispatch.c's CHECK. That check + * is correct and is widened to 64-bit arithmetic, but it runs on the far + * side of hexlib_iface_invoke -- and this loop writes 24 bytes per + * iteration into a fixed 64 KiB static array before then, straight from + * fields the batch file supplied. `DspSimBackend.run_raw` exists precisely + * to feed the DSP arbitrary blobs "so a bad magic, a truncated blob or an + * unknown op kind exercises the DSP's OWN validation", so those fields are + * untrusted BY DESIGN on this path: n_bufs=1 with off_bufs=0xFFFFFF00 was + * a ~4 GiB out-of-range write that faults hexagon-sim, and + * n_bufs=0x01000000 with off_bufs=40 walked 384 MiB forward in 24-byte + * steps over g_rsp and the skel's own static bufs[]/tens[]. Either way the + * caller saw an opaque simulator crash instead of the DSP's rejection + * status, which is the one thing that call was written to observe. + * + * A blob that does not fit is therefore NOT PATCHED AND STILL SENT: the + * skel answers with its own status, unchanged. Refusing to send it here + * would substitute the host's verdict for the DSP's and defeat the same + * purpose from the other direction. main.c:911-916 already validates these + * fields on the device path; the asymmetry was the defect. + * + * Widened to uint64_t so a large n_bufs cannot wrap the comparison back + * into passing, and blen is checked against the header size first so a + * short read cannot leave `hdr` holding whatever was in g_batch before. */ + struct hexlib_batch_hdr hdr; + memset(&hdr, 0, sizeof(hdr)); + int patch_bufs = 0; + if (blen >= (long) sizeof(hdr)) { + memcpy(&hdr, g_batch, sizeof(hdr)); + uint64_t need = (uint64_t) hdr.off_bufs + + (uint64_t) hdr.n_bufs * sizeof(struct hexlib_buf_desc); + patch_bufs = (need <= (uint64_t) blen); + } + if (patch_bufs) { + for (uint32_t i = 0; i < hdr.n_bufs; i++) { + struct hexlib_buf_desc b; + size_t off = (size_t) hdr.off_bufs + (size_t) i * sizeof(b); + memcpy(&b, g_batch + off, sizeof(b)); + b.fd = (uint32_t) fd; + b.base = 0; /* never an address, on any path */ + memcpy(g_batch + off, &b, sizeof(b)); + } + } else { + /* Printed, not silent: an unpatched batch is a legitimate thing to + * send here, but it is never what a normal run wants, so a normal run + * showing this line is a bug in the caller and must be visible. */ + printf("SIMHOST note=bufs_out_of_range_not_patched " + "n_bufs=%u off_bufs=%u blen=%ld\n", + (unsigned int) hdr.n_bufs, (unsigned int) hdr.off_bufs, blen); + } + + rc = hexlib_iface_invoke(h, g_batch, (int) blen, g_rsp, (int) sizeof(g_rsp)); + if (rc != 0) { + printf("SIMHOST error=invoke rc=%d\n", rc); + hexlib_iface_stop(h); + hexlib_iface_close(h); + rpcmem_free(data); + return 9; + } + + struct hexlib_batch_rsp_hdr rh; + memcpy(&rh, g_rsp, sizeof(rh)); + uint64_t want = (uint64_t) sizeof(rh) + + (uint64_t) rh.n_ops * (uint64_t) sizeof(struct hexlib_op_result); + uint32_t rsp_len = (want > sizeof(g_rsp)) ? (uint32_t) sizeof(g_rsp) : (uint32_t) want; + + printf("SIMHOST invoke rc=%d rsp_len=%u status=%u n_ops=%u cycles=%llu\n", + rc, (unsigned int) rsp_len, (unsigned int) rh.status, + (unsigned int) rh.n_ops, (unsigned long long) rh.cycles_total); + + FILE *rf = fopen("hexlib_rsp.bin", "wb"); + if (rf) { fwrite(g_rsp, 1, rsp_len, rf); fclose(rf); } + + if (rh.status == HEXLIB_DSP_OK) { + FILE *of = fopen("hexlib_out.bin", "wb"); + if (of) { fwrite(data, 1, (size_t) dlen, of); fclose(of); } + } + + hexlib_iface_stop(h); + hexlib_iface_close(h); + rpcmem_free(data); + printf("SIMHOST done\n"); + return rh.status == HEXLIB_DSP_OK ? 0 : 1; +} diff --git a/hexlib/runtime/skel/hexlib_dsp.h b/hexlib/runtime/skel/hexlib_dsp.h new file mode 100644 index 0000000..ff5789f --- /dev/null +++ b/hexlib/runtime/skel/hexlib_dsp.h @@ -0,0 +1,210 @@ +/* hexlib/runtime/skel/hexlib_dsp.h -- the DSP-side contract. + * + * THE HOST WRITES FDS AND OFFSETS. THE DSP WRITES ADDRESSES. `hexlib_buf_desc.base` + * and `hexlib_tensor.data` are scratch fields this side fills in; a host that put + * an address in either would produce code that works on the simulator -- where + * host and DSP share one address space -- and fails instantly on silicon. There is + * no field on the wire for an address, which is what makes a simulator result + * transferable. + * + * Adapted in shape from llama.cpp ggml-hexagon's htp-ops.h (MIT); see + * ATTRIBUTION.md. Two things are deliberately NOT adapted: ne/nb strides (hexlib + * uses an enumerated layout, so un-repacked weights are a plan-time error rather + * than silent corruption, and strides cannot describe a VTCM-resident tile of a + * DDR tensor) and the ggml opcode enum. + */ +#ifndef HEXLIB_DSP_H +#define HEXLIB_DSP_H + +#include +#include + +#define HEXLIB_BATCH_MAGIC 0x424C5848u /* 'HXLB' little-endian */ +#define HEXLIB_BATCH_VERSION 1 + +#define HEXLIB_MAX_BUFS 8 +#define HEXLIB_MAX_SRC 6 +#define HEXLIB_MAX_DST 4 +#define HEXLIB_MAX_PARAMS 16 +#define HEXLIB_MAX_TENSORS 512 + +/* OK IS 1, NOT 0. A response buffer that is never written is all zeros, and zero + * must not read as success -- this project has been bitten four times by absence + * reported as success, including a device-farm job that ran no tests and passed. */ +enum hexlib_dsp_status { + HEXLIB_DSP_OK = 1, + HEXLIB_DSP_ERR_INTERNAL = 2, + HEXLIB_DSP_ERR_BAD_MAGIC = 3, + HEXLIB_DSP_ERR_BAD_VERSION = 4, + HEXLIB_DSP_ERR_TRUNCATED = 5, + HEXLIB_DSP_ERR_INVAL_PARAMS = 6, + HEXLIB_DSP_ERR_UNMAPPED = 7, + HEXLIB_DSP_ERR_NO_MMAP_SLOT = 8, + HEXLIB_DSP_ERR_MMAP_FAILED = 9, + HEXLIB_DSP_ERR_NO_KERNEL = 10, + HEXLIB_DSP_ERR_VTCM_TOO_SMALL = 11, + HEXLIB_DSP_ERR_VTCM_RECLAIMED = 12, + HEXLIB_DSP_ERR_REQUIRES = 13, + HEXLIB_DSP_ERR_NOT_STARTED = 14, + /* A qurt_mem_cache_clean() call failed. Its own status, not folded into + * ERR_INTERNAL, because the consequence is specific and misleading: the + * ops all ran and the data the host reads back may be stale rather than + * wrong. Telling those apart from the host is impossible without this. */ + HEXLIB_DSP_ERR_CACHE = 15, +}; + +/* The status as text, for the one place a human reads it: the host's error + * output. `switch` rather than a string array indexed by the value, so the + * compiler warns on a status added to the enum and not to this, and so an + * out-of-range value cannot index past the end. + * + * WHY THIS EXISTS AT ALL. `HEXLIB_AEE_FROM_STATUS` below tags a real DSP status + * into an AEEResult so that a VTCM-contention failure can be told apart from a + * signing failure or a missing skel -- and for a while nothing on the host + * decoded it. `HEXLIB_AEE_IS_STATUS`/`HEXLIB_AEE_STATUS` appeared only in this + * header and in a test probe, so the detail crossed the wire and every cause + * printed the same bare negative number, which is exactly the operator + * confusion the tag was added to remove. `session.c` decodes it now. + * + * The names match `hexlib.runtime.wire.STATUS` string for string, minus the + * `HEXLIB_DSP_` prefix, and a compiled test binds the two tables. */ +static inline const char *hexlib_dsp_status_name(int s) { + switch (s) { + case HEXLIB_DSP_OK: return "OK"; + case HEXLIB_DSP_ERR_INTERNAL: return "ERR_INTERNAL"; + case HEXLIB_DSP_ERR_BAD_MAGIC: return "ERR_BAD_MAGIC"; + case HEXLIB_DSP_ERR_BAD_VERSION: return "ERR_BAD_VERSION"; + case HEXLIB_DSP_ERR_TRUNCATED: return "ERR_TRUNCATED"; + case HEXLIB_DSP_ERR_INVAL_PARAMS: return "ERR_INVAL_PARAMS"; + case HEXLIB_DSP_ERR_UNMAPPED: return "ERR_UNMAPPED"; + case HEXLIB_DSP_ERR_NO_MMAP_SLOT: return "ERR_NO_MMAP_SLOT"; + case HEXLIB_DSP_ERR_MMAP_FAILED: return "ERR_MMAP_FAILED"; + case HEXLIB_DSP_ERR_NO_KERNEL: return "ERR_NO_KERNEL"; + case HEXLIB_DSP_ERR_VTCM_TOO_SMALL: return "ERR_VTCM_TOO_SMALL"; + case HEXLIB_DSP_ERR_VTCM_RECLAIMED: return "ERR_VTCM_RECLAIMED"; + case HEXLIB_DSP_ERR_REQUIRES: return "ERR_REQUIRES"; + case HEXLIB_DSP_ERR_CACHE: return "ERR_CACHE"; + case HEXLIB_DSP_ERR_NOT_STARTED: return "ERR_NOT_STARTED"; + default: return "UNKNOWN"; + } +} + +/* CARRY A STATUS OUT THROUGH AN AEEResult, for the one call that has no response + * buffer to put it in. `invoke` returns its status inside the response blob, but + * `start` fails before any blob exists, so a bare AEE_EFAILED there flattened + * every VTCM outcome into the single result the host already prints for a dozen + * unrelated causes -- leaving a device operator unable to tell VTCM contention + * from a signing failure, a URI error, or a missing skel. + * + * 0x8FA0xxxx sits in the AEE "reserved for OEM/vendor" high half, so it is + * nonzero (every `if (rc != AEE_SUCCESS)` still fails) and does not collide with + * AEE_EFAILED or the AEE_E* range. Decode with HEXLIB_AEE_STATUS; test with + * HEXLIB_AEE_IS_STATUS first, because a failure from qaic or the RPC layer + * itself will not carry this tag. */ +#define HEXLIB_AEE_STATUS_TAG 0x8FA00000u +#define HEXLIB_AEE_FROM_STATUS(s) ((int) (HEXLIB_AEE_STATUS_TAG | ((unsigned) (s) & 0xFFu))) +#define HEXLIB_AEE_IS_STATUS(r) (((unsigned) (r) & 0xFFFFFF00u) == HEXLIB_AEE_STATUS_TAG) +#define HEXLIB_AEE_STATUS(r) ((int) ((unsigned) (r) & 0xFFu)) + +struct hexlib_batch_hdr { + uint32_t magic; + uint32_t version; + uint32_t total_size; + uint32_t n_bufs; + uint32_t n_tensors; + uint32_t n_ops; + uint32_t off_bufs; + uint32_t off_tensors; + uint32_t off_ops; + uint32_t flags; +}; + +struct hexlib_buf_desc { + uint64_t base; /* DSP-SIDE SCRATCH. Host writes 0. */ + uint64_t size; + uint32_t fd; + uint32_t flags; +}; + +/* THE ENUMERATED LAYOUT, spelled once. These MUST equal + * hexlib.runtime.wire.LAYOUT_ID, which is what the host serializes through, and + * hexlib/tests/test_wire_struct_layout.py compares the two tables so a value + * added on one side cannot drift from the other. + * + * Named rather than left as bare integers because `tens[i].layout = 0` in + * main.c's hand-built batch was a literal 0 with only a comment tying it to + * `LAYOUT_ID["row_major"]` -- and unlike the `dtype` literal beside it, which + * genentry's emitted `a->dtype[..] != 1u` guard catches loudly at run time, + * nothing checked layout at all. Inserting a layout ahead of row_major would + * have left `pack_batch` emitting 1 while main.c kept emitting 0, and + * `--self-test` would still print `PASS (4100 values, bit-exact)` while + * declaring the buffer as something else entirely. Latent only because no + * kernel reads `a->layout` YET: `q4_0_repacked` is the matmul weight layout, + * and the whole reason this field is an enum rather than ne/nb strides is so + * that un-repacked weights are a plan-time error and not silent corruption. */ +#define HEXLIB_LAYOUT_ROW_MAJOR 0u +#define HEXLIB_LAYOUT_TILED_32X32 1u +#define HEXLIB_LAYOUT_Q4_0_REPACKED 2u + +struct hexlib_tensor { + uint32_t bi; + uint32_t offset; + uint32_t nbytes; + uint32_t dtype; + uint32_t layout; + uint32_t ne[4]; + uint32_t data; /* DSP-SIDE SCRATCH. Host writes 0. */ + uint32_t pad; +}; + +struct hexlib_op_desc { + uint32_t kind; + uint32_t flags; + int32_t params[HEXLIB_MAX_PARAMS]; + uint16_t src[HEXLIB_MAX_SRC]; + uint16_t dst[HEXLIB_MAX_DST]; +}; + +struct hexlib_batch_rsp_hdr { + uint32_t magic; + uint32_t version; + uint32_t status; + uint32_t n_ops; + uint64_t cycles_total; + uint32_t arch; + uint32_t pad; +}; + +struct hexlib_op_result { + uint32_t kind; + uint32_t status; + uint64_t cycles; +}; + +/* The kernel ABI. `int`, not `void`: a kernel that cannot serve a request says so + * rather than producing a plausible wrong answer. */ +typedef struct { + void *buf[HEXLIB_MAX_BUFS]; + uint32_t ne[HEXLIB_MAX_BUFS][4]; + uint32_t dtype[HEXLIB_MAX_BUFS]; + uint32_t layout[HEXLIB_MAX_BUFS]; + uint32_t n_buf; + uint8_t *vtcm; + size_t vtcm_size; + const void *params; + uint32_t n_threads; +} hexlib_args; + +typedef int (*hexlib_kernel_fn)(const hexlib_args *); + +struct hexlib_kernel_entry { + uint32_t kind; + const char *name; + hexlib_kernel_fn fn; +}; + +/* Generated by hexlib/runtime/genentry.py. */ +extern const struct hexlib_kernel_entry hexlib_kernel_table[]; +extern const uint32_t hexlib_kernel_table_len; + +#endif /* HEXLIB_DSP_H */ diff --git a/hexlib/runtime/skel/skel.c b/hexlib/runtime/skel/skel.c new file mode 100644 index 0000000..4b0aff2 --- /dev/null +++ b/hexlib/runtime/skel/skel.c @@ -0,0 +1,197 @@ +/* hexlib/runtime/skel/skel.c -- the FastRPC entry points qaic's skel calls. + * + * Session lifecycle adapted from llama.cpp ggml-hexagon htp/main.c (MIT); see + * ATTRIBUTION.md. `start` deliberately takes no dsp_queue_id: dispatch is a + * synchronous invoke, because dspqueue has no simulator path. + * + * THESE PROTOTYPES ARE THE REAL ONES QAIC GENERATES from + * runtime/idl/hexlib_iface.idl -- verified by actually running qaic (not + * derived by hand) into a scratch directory and reading hexlib_iface.h back. + * Two things that are easy to get wrong by guessing: + * - `open`/`close` (qaic's implicit pair for a `: remote_handle64` interface) + * return plain `int`, not `AEEResult`. They are the same underlying type + * (AEEResult is `typedef int AEEResult`), but the header spells it `int`. + * - `invoke` has NO "resultLenOut" parameter. `rout sequence result` + * marshals only a capacity in `resultLen`; there is no channel back to the + * host for "how many bytes are actually meaningful" -- the transport + * doesn't carry one. The response is therefore self-describing: + * `hexlib_batch_rsp_hdr.n_ops` is what the host reads to know how many + * `hexlib_op_result` entries follow, not any out-parameter here. + */ +#include "hexlib_iface.h" +#include "skel_internal.h" + +#include + +#include "HAP_farf.h" + +static struct hexlib_ctx g_ctx; + +int hexlib_iface_open(const char *uri, remote_handle64 *handle) { + (void) uri; + memset(&g_ctx, 0, sizeof(g_ctx)); + for (uint32_t i = 0; i < HEXLIB_MAX_MMAPS; i++) { + g_ctx.mmap[i].fd = -1; + } + *handle = (remote_handle64) &g_ctx; + return AEE_SUCCESS; +} + +int hexlib_iface_close(remote_handle64 handle) { + struct hexlib_ctx *ctx = (struct hexlib_ctx *) handle; + hexlib_vtcm_release(ctx); + hexlib_vtcm_free(ctx); + ctx->started = 0; + return AEE_SUCCESS; +} + +AEEResult hexlib_iface_start(remote_handle64 handle, uint32 sess_id, uint32 n_hvx, + uint32 n_hmx, uint64 max_vmem) { + struct hexlib_ctx *ctx = (struct hexlib_ctx *) handle; + ctx->sess_id = sess_id; + ctx->n_hvx = n_hvx; + ctx->n_hmx = n_hmx; + ctx->max_vmem = max_vmem; + + int rc = hexlib_vtcm_alloc(ctx); + if (rc != HEXLIB_DSP_OK) { + /* CARRY THE SPECIFIC STATUS ACROSS THE RPC BOUNDARY. This returned a + * bare AEE_EFAILED, which flattened every VTCM failure into the one + * outcome the host already reports for a dozen unrelated causes, so on + * a device the operator could not tell VTCM contention from a signing + * failure, a URI error, or a missing skel -- the same undiagnosable + * session-open dead end as the arch-decode bug, from a different cause. + * + * The FARF above is not enough on its own: it lands in the DSP log, + * which an operator running a device-farm job may not be able to + * retrieve. The return value always comes back. + * + * HEXLIB_AEE_FROM_STATUS keeps this a nonzero failure for every caller + * that only tests success, while making the reason recoverable for one + * that looks. If FastRPC ever normalises the value we lose only the + * detail, never the failure. */ + ctx->start_status = rc; + FARF(ERROR, "hexlib: start failed, VTCM rc %d", rc); + return HEXLIB_AEE_FROM_STATUS(rc); + } + ctx->start_status = HEXLIB_DSP_OK; + ctx->started = 1; + FARF(HIGH, "hexlib: session %u started, VTCM %u bytes", + sess_id, (uint32_t) ctx->vtcm_size); + return AEE_SUCCESS; +} + +AEEResult hexlib_iface_stop(remote_handle64 handle) { + struct hexlib_ctx *ctx = (struct hexlib_ctx *) handle; + hexlib_vtcm_release(ctx); + hexlib_vtcm_free(ctx); + ctx->started = 0; + return AEE_SUCCESS; +} + +AEEResult hexlib_iface_mmap(remote_handle64 handle, uint32 fd, uint32 size) { + struct hexlib_ctx *ctx = (struct hexlib_ctx *) handle; + int rc = hexlib_bufs_register(ctx, fd, size); + return rc == HEXLIB_DSP_OK ? AEE_SUCCESS : AEE_EFAILED; +} + +AEEResult hexlib_iface_munmap(remote_handle64 handle, uint32 fd) { + struct hexlib_ctx *ctx = (struct hexlib_ctx *) handle; + int rc = hexlib_bufs_unregister(ctx, fd); + return rc == HEXLIB_DSP_OK ? AEE_SUCCESS : AEE_EFAILED; +} + +AEEResult hexlib_iface_hwinfo(remote_handle64 handle, uint32 *arch, + uint32 *n_threads, uint32 *n_hvx, uint32 *n_hmx, + uint64 *vtcm_size) { + struct hexlib_ctx *ctx = (struct hexlib_ctx *) handle; + /* __HEXAGON_ARCH__ is what THIS BINARY was built for; the host cross-checks + * it against what the driver reports the part to be, so a skel built for + * the wrong arch is a visible disagreement rather than a mystery. Verified + * (not assumed) to expand to 75 when compiled -mv75 on the 19.0.04 + * toolchain -- see task-6-report.md for how. */ + *arch = __HEXAGON_ARCH__; + /* HARDCODED, NOT QUERIED. There is no runtime thread count here; the skel + * runs single-threaded. Reported so the wire struct has the field, not + * because the DSP was asked. */ + *n_threads = 1; + /* ECHOES, AND CIRCULAR BY CONSTRUCTION. These are whatever the host passed + * to start(), returned unchanged. hexlib_open (session.c) passes 0 for both + * and then stores what comes back as ctx->n_hvx/n_hmx -- so the host reads + * back its own zero and records it as a DSP capability. The real values are + * on the DRIVER side: hexlib_caps already carries hvx_support_128b and + * hmx_support_depth from remote_dsp_capability. + * + * NOT "fixed" by forwarding the driver's values, deliberately: start()'s + * n_hmx is what decides whether hexlib_vtcm_alloc requests HMX from the + * compute-res manager, so changing what the host sends changes acquisition + * behaviour on hardware that has never run this code. The simulator already + * passes 1, 1 where the device host passes 0, 0, so the two take DIFFERENT + * branches there -- pinned by test_vtcm_contention.py so it stays a + * recorded decision instead of a surprise. Revisit when an HMX kernel + * lands, which is the same trigger skel_vtcm.c names. */ + *n_hvx = ctx->n_hvx; + *n_hmx = ctx->n_hmx; + /* The ACQUIRED size, never the part's total: vtcm_size test guards this. */ + *vtcm_size = (uint64) ctx->vtcm_size; + return AEE_SUCCESS; +} + +AEEResult hexlib_iface_invoke(remote_handle64 handle, const unsigned char *batch, + int batchLen, unsigned char *result, int resultLen) { + struct hexlib_ctx *ctx = (struct hexlib_ctx *) handle; + + /* Nothing readable can be written into a buffer smaller than the response + * header itself. The only channel left to say so is the RPC return code. */ + if (resultLen < 0 || (uint32_t) resultLen < sizeof(struct hexlib_batch_rsp_hdr)) { + FARF(ERROR, "hexlib: invoke result buffer too small (%d)", resultLen); + return AEE_EFAILED; + } + + /* BOTH LENGTHS ARE SIGNED ON THE WIRE (qaic spells `sequence` as a + * pointer plus an `int`), so both need this and only `resultLen` had it. + * A negative `batchLen` cast to uint32_t becomes an enormous length, and + * hexlib_dispatch_batch's first size test is `len < sizeof(struct + * hexlib_batch_hdr)` -- which a huge value PASSES. It then memcpy()s the + * full 40-byte header out of `batch` (skel_dispatch.c) BEFORE + * `hdr.total_size != len` can reject anything: an out-of-bounds read of a + * buffer the host may have made much shorter than that. Rejecting it here, + * before the cast, is the only place the sign is still visible. + * + * Written into the response and returned AEE_SUCCESS, not returned as an + * RPC error: the result buffer was just proven big enough for a header + * (above), so the host can be told exactly what was wrong instead of + * having a marshalled response discarded. Same reasoning as the + * invoke-before-start refusal below. */ + if (batchLen < 0) { + FARF(ERROR, "hexlib: invoke batch length is negative (%d)", batchLen); + hexlib_write_rsp_hdr(result, HEXLIB_DSP_ERR_INVAL_PARAMS, 0, 0); + return AEE_SUCCESS; + } + + if (!ctx->started) { + /* No op runs -- not even the truncation path inside + * hexlib_dispatch_batch. The response gets the REAL status + * (HEXLIB_DSP_ERR_NOT_STARTED), not a generic one reached by feeding + * hexlib_dispatch_batch a batch length of zero: a host that reads + * HEXLIB_DSP_ERR_NOT_STARTED off the wire knows exactly what to fix + * (call start() first), rather than seeing HEXLIB_DSP_ERR_TRUNCATED + * and wondering whether its own encoder is broken. */ + FARF(ERROR, "hexlib: invoke before start (%d)", HEXLIB_DSP_ERR_NOT_STARTED); + hexlib_write_rsp_hdr(result, HEXLIB_DSP_ERR_NOT_STARTED, 0, 0); + return AEE_SUCCESS; + } + + uint32_t rsp_len = 0; + int rc = hexlib_dispatch_batch(ctx, batch, (uint32_t) batchLen, result, + (uint32_t) resultLen, &rsp_len); + /* A non-OK batch still returns AEE_SUCCESS with a populated response: the + * host reads hexlib_batch_rsp_hdr.status off the wire, and an RPC-level + * error would discard the response qaic already marshaled back to it. + * `rsp_len` has no wire home either -- see the file header -- so it is + * only useful to a caller of hexlib_dispatch_batch directly (e.g. a test), + * not to this FastRPC entry point. */ + (void) rc; + (void) rsp_len; + return AEE_SUCCESS; +} diff --git a/hexlib/runtime/skel/skel_bufs.c b/hexlib/runtime/skel/skel_bufs.c new file mode 100644 index 0000000..56f0ebc --- /dev/null +++ b/hexlib/runtime/skel/skel_bufs.c @@ -0,0 +1,263 @@ +/* hexlib/runtime/skel/skel_bufs.c -- fd to mapped address, and nothing else. + * + * THE INVARIANT THIS FILE EXISTS TO HOLD: the host writes fds and offsets, this + * side writes addresses. `hexlib_buf_desc.base` arrives as zero and is + * OVERWRITTEN before it is read, from a table only hexlib_bufs_register + * populates. So a host address cannot reach a kernel even if one were somehow + * placed on the wire. + * + * WHY THAT MATTERS MORE THAN IT LOOKS. Under the simulator qexe the host side and + * this side are one process in one address space. A skel that used the host's + * pointer would work perfectly on the simulator and fail instantly on silicon -- + * a gate certifying nothing. The unmapped-fd test is what catches it: on the + * simulator, an implementation leaning on the shared address space returns the + * RIGHT ANSWER to a request whose buffer was never mapped, so the test fails + * exactly when the invariant is broken. + * + * Adapted from llama.cpp ggml-hexagon htp/main.c reuse_buf/mmap_buf/prep_tensor + * (MIT); see ATTRIBUTION.md. Upstream's silent `base == 0` fallthrough and its + * hard process abort on a failed mapping are both replaced by error returns. + */ +#include "skel_internal.h" + +#include "HAP_farf.h" +#include "HAP_mem.h" + +/* QuRT ONLY, AND THE GUARD IS NARROW ON PURPOSE. `qurt_memory.h` exists in the + * Hexagon SDK's RTOS tree and nowhere else, and `hexlib/tests/ + * test_genentry_entry_probe.py` compiles this file with the HOST compiler to + * drive the dispatcher behaviourally. Both the device skel and the QuRT-hosted + * simulator .so define `__hexagon__`, so the real calls are taken everywhere + * they can possibly matter; the host fallback exists solely so that probe can + * link, and it is NOT a portability layer. + * + * The danger of a fallback like this is that it silently disables the very + * thing it stands in for. That is bound to the artifact rather than trusted: + * `test_device_cache_maintenance.py` asserts `qurt_mem_cache_clean` is an + * UNDEFINED symbol in the linked libhexlib_iface_skel.so, which is false the + * moment this stub is what got compiled. */ +#if defined(__hexagon__) +#include "qurt_memory.h" +#else +typedef unsigned long qurt_addr_t; +typedef unsigned long qurt_size_t; +typedef int qurt_mem_cache_op_t; +typedef int qurt_mem_cache_type_t; +#define QURT_MEM_CACHE_FLUSH 0 +#define QURT_MEM_CACHE_INVALIDATE 1 +#define QURT_MEM_DCACHE 0 +static int qurt_mem_cache_clean(qurt_addr_t a, qurt_size_t n, + qurt_mem_cache_op_t op, qurt_mem_cache_type_t t) { + (void) a; (void) n; (void) op; (void) t; + return 0; /* host probe only -- see the comment above */ +} +#endif + +/* THE LOOKUP THE WHOLE FILE EXISTS TO GATE. Matches by fd, never by whatever + * `base` the host sent. Occupied slots have a nonzero size; fd alone is not + * enough, since an unregistered slot's fd field is reset to -1, not left + * stale. */ +static struct hexlib_mmap *find_by_fd(struct hexlib_ctx *ctx, uint32_t fd) { + for (uint32_t i = 0; i < HEXLIB_MAX_MMAPS; i++) { + struct hexlib_mmap *m = &ctx->mmap[i]; + if (m->size && m->fd == (int32_t) fd) { + return m; + } + } + return 0; +} + +int hexlib_bufs_register(struct hexlib_ctx *ctx, uint32_t fd, uint32_t size) { + if (find_by_fd(ctx, fd)) { + return HEXLIB_DSP_OK; /* already mapped; idempotent */ + } + for (uint32_t i = 0; i < HEXLIB_MAX_MMAPS; i++) { + struct hexlib_mmap *m = &ctx->mmap[i]; + if (m->size) { + continue; + } + /* HAP_mmap's `len` is `int`; HAP_mmap2's is `size_t`. Cast explicitly per + * branch rather than relying on the implicit uint32_t->int narrowing the + * older API's signature otherwise forces on us. See HAP_mem.h. */ +#if __HVX_ARCH__ > 73 + void *va = HAP_mmap2(0, (size_t) size, HAP_PROT_READ | HAP_PROT_WRITE, 0, (int) fd, 0); +#else + void *va = HAP_mmap(0, (int) size, HAP_PROT_READ | HAP_PROT_WRITE, 0, (int) fd, 0); +#endif + if (va == (void *) -1 || va == 0) { + FARF(ERROR, "hexlib: mmap failed fd %u size %u", fd, size); + return HEXLIB_DSP_ERR_MMAP_FAILED; + } + m->base = (uint64_t) va; + m->size = size; + m->fd = (int32_t) fd; + FARF(HIGH, "hexlib: mmap fd %u base %p size %u", fd, va, size); + return HEXLIB_DSP_OK; + } + /* Upstream returns silently here and lets the caller compute 0 + offset. + * That is a write to a low address, not a diagnosable failure. */ + FARF(ERROR, "hexlib: no free mmap slot for fd %u (max %u)", fd, HEXLIB_MAX_MMAPS); + return HEXLIB_DSP_ERR_NO_MMAP_SLOT; +} + +int hexlib_bufs_unregister(struct hexlib_ctx *ctx, uint32_t fd) { + struct hexlib_mmap *m = find_by_fd(ctx, fd); + if (!m) { + return HEXLIB_DSP_ERR_UNMAPPED; + } + uint64_t base = m->base; + uint64_t size = m->size; + /* Free the slot regardless of the unmap outcome below: whatever happens at + * the OS level, this fd must stop being something hexlib_bufs_map() can + * hand back on a future lookup. */ + m->base = 0; + m->size = 0; + m->fd = -1; +#if __HVX_ARCH__ > 73 + int rc = HAP_munmap2((void *) base, (size_t) size); +#else + int rc = HAP_munmap((void *) base, (int) size); +#endif + if (rc != 0) { + /* No dedicated "unmap failed" status exists on the wire (see + * hexlib_dsp.h, not modified by this file); MMAP_FAILED is the closest + * available fit for "a HAP_mem mapping call did not do what we asked". + * Checked rather than ignored: a failed unmap does not threaten the + * pointer invariant (the slot above is already cleared either way), + * but silently discarding an OS-level failure here is exactly the + * kind of thing this file exists to stop doing. */ + FARF(ERROR, "hexlib: munmap failed for fd %u base %p size %u rc %d", + fd, (void *) base, (uint32_t) size, rc); + return HEXLIB_DSP_ERR_MMAP_FAILED; + } + return HEXLIB_DSP_OK; +} + +int hexlib_bufs_map(struct hexlib_ctx *ctx, struct hexlib_buf_desc *bufs, uint32_t n) { + for (uint32_t i = 0; i < n; i++) { + struct hexlib_buf_desc *b = bufs + i; + + /* CLEAR FIRST. Whatever the host put here is destroyed before it can be + * read. This single line is the invariant. */ + b->base = 0; + + struct hexlib_mmap *m = find_by_fd(ctx, b->fd); + if (!m) { + /* NOT mapped on demand. The host must have called mmap(). A buffer + * appearing for the first time inside invoke() is a host bug, and + * mapping it here would hide it -- and on the simulator, would let + * the shared address space paper over the invariant entirely. */ + FARF(ERROR, "hexlib: buffer %u fd %u was never mapped", i, b->fd); + return HEXLIB_DSP_ERR_UNMAPPED; + } + if (b->size > m->size) { + FARF(ERROR, "hexlib: buffer %u claims %u bytes, mapping has %u", + i, (uint32_t) b->size, (uint32_t) m->size); + return HEXLIB_DSP_ERR_INVAL_PARAMS; + } + b->base = m->base; + } + return HEXLIB_DSP_OK; +} + +int hexlib_tensors_resolve(struct hexlib_ctx *ctx, struct hexlib_buf_desc *bufs, + uint32_t n_bufs, struct hexlib_tensor *tens, + uint32_t n_tens) { + (void) ctx; + for (uint32_t i = 0; i < n_tens; i++) { + struct hexlib_tensor *t = tens + i; + if (t->bi >= n_bufs) { + FARF(ERROR, "hexlib: tensor %u names buffer %u of %u", i, t->bi, n_bufs); + return HEXLIB_DSP_ERR_INVAL_PARAMS; + } + struct hexlib_buf_desc *b = bufs + t->bi; + if (!b->base) { + return HEXLIB_DSP_ERR_UNMAPPED; + } + /* Bounds-checked HERE as well as on the host. The host is the thing + * being served, not the thing being trusted. */ + if ((uint64_t) t->offset + (uint64_t) t->nbytes > b->size) { + FARF(ERROR, "hexlib: tensor %u offset %u + %u exceeds buffer %u size %u", + i, t->offset, t->nbytes, t->bi, (uint32_t) b->size); + return HEXLIB_DSP_ERR_TRUNCATED; + } + t->data = (uint32_t) (b->base + t->offset); + } + return HEXLIB_DSP_OK; +} + +/* --------------------------------------------------------------------------- + * CACHE MAINTENANCE, WHICH FASTRPC DOES NOT DO FOR THESE BUFFERS. + * + * FastRPC keeps the two caches coherent for anything passed AS AN INVOKE + * ARGUMENT -- it knows the direction of each `rin`/`rout` parameter and cleans + * or invalidates accordingly. Our data buffers are not arguments. They are + * mapped once, out of band, through `fastrpc_mmap` and named on the wire only + * by fd, precisely so no address ever crosses between the two processors. That + * design decision stands. Its consequence is that FastRPC has no idea these + * pages were touched, so NOTHING happens unless we do it here. + * + * WHAT THAT COST, MEASURED ON SM8650 BEFORE THIS EXISTED. All three of these + * are the same bug wearing different clothes: + * + * * `hexlib_run --self-test`: 3859 of 4100 fp16 values not bit-exact, where + * the simulator gives exactly 0 error. Not garbage -- a PARTIAL write. The + * 241 correct values were the lines that happened to get evicted. + * * `--coherency-check`: the sentinel came back unchanged, exit 6. The write + * never reached DDR at all. + * * The whole 49-op encoder through `--batch`: 179,124 bytes of the arena + * changed, but `merger.out` -- written by the LAST op, 1024 bytes -- came + * back all zero. That one is decisive because it is ORDERED. Early writes + * landed because subsequent work evicted them; the final write was still + * sitting in the cache when the invoke returned. + * + * BOTH DIRECTIONS ARE NEEDED AND THEY ARE NOT THE SAME OPERATION. + * `invalidate` before the batch: the host has just written weights and inputs + * into these pages, and any line this DSP still holds from a PREVIOUS invoke is + * stale. `flush` after: our writes must reach memory before the host reads + * them. Doing only the flush works for exactly one invoke per session and then + * silently reads old data, which is a worse bug than the one being fixed + * because it needs two runs to show up. + * + * WHOLE BUFFERS, NOT WRITTEN RANGES. The op descriptors say which tensors an op + * writes, so a narrower flush is computable -- and it would be a per-op loop + * over sub-ranges, each rounded out to a cache line, with the line-straddling + * case at every boundary. `qurt_memory.h` warns that the operation takes the + * whole line and therefore "the contents of the adjoining buffer can be flushed + * and invalidated if it falls in any of the cache line", so partial ranges make + * neighbours another correctness question. Once per batch over whole buffers + * has no such case. If this ever shows up in a profile, narrow it THEN, with a + * measurement to point at. + */ +static int cache_op_all(struct hexlib_ctx *ctx, qurt_mem_cache_op_t op, + const char *what) { + int worst = HEXLIB_DSP_OK; + for (uint32_t i = 0; i < HEXLIB_MAX_MMAPS; i++) { + struct hexlib_mmap *m = &ctx->mmap[i]; + if (!m->size) { + continue; + } + int rc = qurt_mem_cache_clean((qurt_addr_t) m->base, + (qurt_size_t) m->size, + op, QURT_MEM_DCACHE); + if (rc != 0) { + /* LOUD, AND IT CHANGES THE BATCH STATUS. A cache operation that + * quietly failed would put us back in exactly the state this code + * exists to leave: a run that reports OK and returns data the host + * cannot see. */ + FARF(ERROR, "hexlib: %s failed for fd %d base %p size %u rc %d", + what, (int) m->fd, (void *) (uintptr_t) m->base, + (uint32_t) m->size, rc); + worst = HEXLIB_DSP_ERR_CACHE; + } + } + return worst; +} + +int hexlib_bufs_invalidate(struct hexlib_ctx *ctx) { + return cache_op_all(ctx, QURT_MEM_CACHE_INVALIDATE, "cache invalidate"); +} + +int hexlib_bufs_flush(struct hexlib_ctx *ctx) { + return cache_op_all(ctx, QURT_MEM_CACHE_FLUSH, "cache flush"); +} diff --git a/hexlib/runtime/skel/skel_dispatch.c b/hexlib/runtime/skel/skel_dispatch.c new file mode 100644 index 0000000..f43827d --- /dev/null +++ b/hexlib/runtime/skel/skel_dispatch.c @@ -0,0 +1,302 @@ +/* hexlib/runtime/skel/skel_dispatch.c -- validate a batch, then walk it. + * + * THE RESPONSE HEADER GOES DOWN FIRST, with a non-OK status. Every early return + * therefore leaves a readable failure, and there is no path on which the host + * reads a zero-filled buffer and has to guess. Status OK is 1, so an unwritten + * buffer cannot read as success. + * + * PCYCLE BRACKETS THE KERNEL CALL AND NOTHING ELSE -- not tensor resolution, not + * mapping, not the response write. Harness overhead is roughly constant, so + * including it manufactures ratios out of nothing; this is the same counter and + * the same placement `hexlib.sim` reports, which is what makes sim-vs-silicon + * comparison mean anything. + * + * EVERY OFFSET IS VALIDATED IN 64-BIT ARITHMETIC BEFORE A BYTE IS READ. `len` and + * `rsp_cap` are uint32_t, and so are the wire offsets and counts, so a host- + * supplied count multiplied by a struct size can wrap a 32-bit accumulator back + * into range and turn a bounds check into a lie -- in particular there is no + * upper bound on `n_ops` on the wire, so an unwidened `off_ops + n_ops * + * sizeof(op_desc)` could overflow back under `len` for a large enough n_ops. + * Every offset+size computation below widens to uint64_t first for exactly that + * reason: a host bug must not become an out-of-bounds read or write on the DSP. + */ +#include "skel_internal.h" + +#include + +#include "HAP_farf.h" +#include "HAP_perf.h" + +/* THE SDK'S OWN READ, NOT A HAND-ROLLED ONE -- AND THE DIFFERENCE IS NOT + * COSMETIC. This was `__asm__ __volatile__("%0 = c15:14")`, issued directly. + * That instruction only advances if SYSCFG.PCYCLEEN is set, and A USER-MODE + * UNSIGNED PD CANNOT SET THAT BIT: this project's own + * include/hexlib/hexlib_harness.h sets it explicitly (`hexlib_enable_pcycle`, + * "bit 5 = PCYCLEEN") precisely because the standalone-ELF runtime it belongs + * to runs where that is permitted. The skel does not. So the hand-rolled read + * had no handling of the one precondition it depends on, in the one PD where + * that precondition may not hold -- and the SIMULATOR CANNOT TELL US, because + * there the bit is effectively always on and this path measures a plausible + * four-figure number either way. + * + * HAP_perf_get_pcycles() ($HEXAGON_SDK_ROOT/incs/HAP_perf.h) issues the + * IDENTICAL `C15:14` read -- so this is not a change of mechanism and the + * measured number is expected to be unchanged (it was: 1287 cycles on the + * simulator before and after). What changes is whose claim it is. If + * Qualcomm's own documented perf API returns 0 in an unsigned PD, that is a + * platform fact about the PD, discoverable from the SDK and reportable as + * such; if our own inline asm returned 0 it would be indistinguishable from + * our bug. Reading 0 on silicon remains POSSIBLE -- nothing here prevents it, + * and no simulator run can rule it out -- which is exactly why main.c prints + * cycles_total, cli.py now requires it to be > 0, and the on-device test + * asserts it. See docs §6.1 and this file's PCYCLE note above. + * + * KEPT AS A NAMED WRAPPER rather than calling HAP_perf_get_pcycles() at the + * two sites: the bracketing test (test_skel_dispatch_source.py) locates the + * before/after pair by this name and checks that ONLY `k->fn(&a)` sits between + * them, and one name is also the one place to state the above. */ +static inline uint64_t hexlib_read_pcycle(void) { + return (uint64_t) HAP_perf_get_pcycles(); +} + +/* Shared with skel.c (see skel_internal.h): both callers write the same header + * shape. `arch` is never a caller-supplied value -- it is always what THIS + * BINARY was built for, via __HEXAGON_ARCH__, so a skel built for the wrong + * part shows up as a visible disagreement rather than a silently accepted + * parameter. */ +void hexlib_write_rsp_hdr(uint8_t *rsp, uint32_t status, uint32_t n_ops, + uint64_t cycles) { + struct hexlib_batch_rsp_hdr h; + memset(&h, 0, sizeof(h)); + h.magic = HEXLIB_BATCH_MAGIC; + h.version = HEXLIB_BATCH_VERSION; + h.status = status; + h.n_ops = n_ops; + h.cycles_total = cycles; + h.arch = __HEXAGON_ARCH__; + memcpy(rsp, &h, sizeof(h)); +} + +int hexlib_dispatch_batch(struct hexlib_ctx *ctx, const uint8_t *batch, uint32_t len, + uint8_t *rsp, uint32_t rsp_cap, uint32_t *rsp_len) { + *rsp_len = 0; + if (rsp_cap < sizeof(struct hexlib_batch_rsp_hdr)) { + /* Cannot write anything readable into a buffer this small. There is + * nothing left to say except at the RPC-return-code level, which is + * the caller's (skel.c's) job. */ + return HEXLIB_DSP_ERR_TRUNCATED; + } + /* FAILURE IS THE DEFAULT. Every return below either overwrites this with a + * specific status (OK included) or leaves this one in place. */ + hexlib_write_rsp_hdr(rsp, HEXLIB_DSP_ERR_INTERNAL, 0, 0); + *rsp_len = sizeof(struct hexlib_batch_rsp_hdr); + + if (len < sizeof(struct hexlib_batch_hdr)) { + hexlib_write_rsp_hdr(rsp, HEXLIB_DSP_ERR_TRUNCATED, 0, 0); + return HEXLIB_DSP_ERR_TRUNCATED; + } + struct hexlib_batch_hdr hdr; + memcpy(&hdr, batch, sizeof(hdr)); + + if (hdr.magic != HEXLIB_BATCH_MAGIC) { + hexlib_write_rsp_hdr(rsp, HEXLIB_DSP_ERR_BAD_MAGIC, 0, 0); + return HEXLIB_DSP_ERR_BAD_MAGIC; + } + if (hdr.version != HEXLIB_BATCH_VERSION) { + hexlib_write_rsp_hdr(rsp, HEXLIB_DSP_ERR_BAD_VERSION, 0, 0); + return HEXLIB_DSP_ERR_BAD_VERSION; + } + if (hdr.total_size != len) { + FARF(ERROR, "hexlib: header says %u bytes, got %u", hdr.total_size, len); + hexlib_write_rsp_hdr(rsp, HEXLIB_DSP_ERR_TRUNCATED, 0, 0); + return HEXLIB_DSP_ERR_TRUNCATED; + } + if (hdr.n_bufs > HEXLIB_MAX_BUFS || hdr.n_tensors > HEXLIB_MAX_TENSORS) { + hexlib_write_rsp_hdr(rsp, HEXLIB_DSP_ERR_INVAL_PARAMS, 0, 0); + return HEXLIB_DSP_ERR_INVAL_PARAMS; + } + /* Every section must lie inside the blob, and the response must have room + * for every op's result -- checked before a byte of either is read. Widened + * to uint64_t (see file header) so a huge host-supplied n_ops cannot wrap + * this very check back into passing. */ + if ((uint64_t) hdr.off_bufs + (uint64_t) hdr.n_bufs * sizeof(struct hexlib_buf_desc) > (uint64_t) len || + (uint64_t) hdr.off_tensors + (uint64_t) hdr.n_tensors * sizeof(struct hexlib_tensor) > (uint64_t) len || + (uint64_t) hdr.off_ops + (uint64_t) hdr.n_ops * sizeof(struct hexlib_op_desc) > (uint64_t) len) { + hexlib_write_rsp_hdr(rsp, HEXLIB_DSP_ERR_TRUNCATED, 0, 0); + return HEXLIB_DSP_ERR_TRUNCATED; + } + if ((uint64_t) sizeof(struct hexlib_batch_rsp_hdr) + + (uint64_t) hdr.n_ops * sizeof(struct hexlib_op_result) > (uint64_t) rsp_cap) { + hexlib_write_rsp_hdr(rsp, HEXLIB_DSP_ERR_TRUNCATED, 0, 0); + return HEXLIB_DSP_ERR_TRUNCATED; + } + + /* Working copies: base and data are filled in HERE, never taken from the + * host's bytes. The input blob is const for exactly that reason. + * + * STATIC, NOT STACK OR HEAP: there is no allocator on the DSP side, and 512 + * tensors (~22 KB) is more than is safe to put on a QuRT thread's stack. + * The trade is that hexlib_dispatch_batch is NOT reentrant -- two concurrent + * invokes on the same skel instance would corrupt each other's working + * copy. FastRPC already serializes calls to a single handle, and skel.c + * hands out exactly one handle (g_ctx), so this holds today; it would need + * revisiting if that ever changed (e.g. multiple sessions on one skel). */ + static struct hexlib_buf_desc bufs[HEXLIB_MAX_BUFS]; + static struct hexlib_tensor tens[HEXLIB_MAX_TENSORS]; + memcpy(bufs, batch + hdr.off_bufs, hdr.n_bufs * sizeof(bufs[0])); + memcpy(tens, batch + hdr.off_tensors, hdr.n_tensors * sizeof(tens[0])); + + int rc = hexlib_bufs_map(ctx, bufs, hdr.n_bufs); + if (rc != HEXLIB_DSP_OK) { + hexlib_write_rsp_hdr(rsp, (uint32_t) rc, 0, 0); + return rc; + } + rc = hexlib_tensors_resolve(ctx, bufs, hdr.n_bufs, tens, hdr.n_tensors); + if (rc != HEXLIB_DSP_OK) { + hexlib_write_rsp_hdr(rsp, (uint32_t) rc, 0, 0); + return rc; + } + rc = hexlib_vtcm_acquire(ctx); + if (rc != HEXLIB_DSP_OK) { + hexlib_write_rsp_hdr(rsp, (uint32_t) rc, 0, 0); + return rc; + } + + /* BEFORE ANY KERNEL READS ANYTHING. The host has just written weights and + * inputs into these pages from the applications processor; any line THIS + * DSP still holds from a previous invoke on the same session is stale. + * Skipping this works for exactly one invoke and then silently computes on + * old data -- a worse failure than the one the flush below fixes, because + * it takes two runs to appear. */ + int cache_rc = hexlib_bufs_invalidate(ctx); + if (cache_rc != HEXLIB_DSP_OK) { + hexlib_write_rsp_hdr(rsp, (uint32_t) cache_rc, 0, 0); + return cache_rc; + } + + struct hexlib_op_result *results = + (struct hexlib_op_result *) (rsp + sizeof(struct hexlib_batch_rsp_hdr)); + uint64_t total = 0; + uint32_t done = 0; + int batch_status = HEXLIB_DSP_OK; + + for (uint32_t i = 0; i < hdr.n_ops; i++) { + struct hexlib_op_desc op; + memcpy(&op, batch + hdr.off_ops + (uint64_t) i * sizeof(op), sizeof(op)); + + /* Never left at whatever was in the response buffer before: filled in + * before the kernel lookup even runs, same "failure is the default" + * discipline as the batch-level header above, just per-op. */ + results[i].kind = op.kind; + results[i].status = HEXLIB_DSP_ERR_INTERNAL; + results[i].cycles = 0; + done = i + 1; + + const struct hexlib_kernel_entry *k = 0; + for (uint32_t j = 0; j < hexlib_kernel_table_len; j++) { + if (hexlib_kernel_table[j].kind == op.kind) { + k = &hexlib_kernel_table[j]; + break; + } + } + if (!k) { + FARF(ERROR, "hexlib: no kernel for kind %u", op.kind); + results[i].status = HEXLIB_DSP_ERR_NO_KERNEL; + batch_status = HEXLIB_DSP_ERR_NO_KERNEL; + break; + } + + hexlib_args a; + memset(&a, 0, sizeof(a)); + uint32_t nb = 0; + int op_invalid = 0; + for (uint32_t s = 0; s < HEXLIB_MAX_SRC && !op_invalid; s++) { + if (op.src[s] == 0xFFFF) continue; + if (op.src[s] >= hdr.n_tensors || nb >= HEXLIB_MAX_BUFS) { + op_invalid = 1; + break; + } + struct hexlib_tensor *t = &tens[op.src[s]]; + a.buf[nb] = (void *) (uintptr_t) t->data; + a.dtype[nb] = t->dtype; + a.layout[nb] = t->layout; + for (int e = 0; e < 4; e++) a.ne[nb][e] = t->ne[e]; + nb++; + } + for (uint32_t o = 0; o < HEXLIB_MAX_DST && !op_invalid; o++) { + if (op.dst[o] == 0xFFFF) continue; + if (op.dst[o] >= hdr.n_tensors || nb >= HEXLIB_MAX_BUFS) { + op_invalid = 1; + break; + } + struct hexlib_tensor *t = &tens[op.dst[o]]; + a.buf[nb] = (void *) (uintptr_t) t->data; + a.dtype[nb] = t->dtype; + a.layout[nb] = t->layout; + for (int e = 0; e < 4; e++) a.ne[nb][e] = t->ne[e]; + nb++; + } + if (op_invalid) { + results[i].status = HEXLIB_DSP_ERR_INVAL_PARAMS; + batch_status = HEXLIB_DSP_ERR_INVAL_PARAMS; + break; + } + a.n_buf = nb; + a.vtcm = ctx->vtcm_base; + a.vtcm_size = ctx->vtcm_size; + a.params = op.params; + a.n_threads = 1; + + uint64_t t0 = hexlib_read_pcycle(); + int krc = k->fn(&a); + uint64_t t1 = hexlib_read_pcycle(); + + results[i].status = (uint32_t) krc; + results[i].cycles = t1 - t0; + total += (t1 - t0); + + if (krc != HEXLIB_DSP_OK) { + batch_status = krc; + break; + } + /* A competing session asked for VTCM back. Stop cleanly at an op + * boundary and ACTUALLY GIVE IT BACK: the release callback in + * skel_vtcm.c only records the request (it must not release memory a + * batch in flight is still using), so the OS-level release happens + * here, once we are between ops and genuinely done with it -- not + * merely stopping while still holding the reservation the competing + * session is waiting on. */ + if (ctx->vtcm_needs_release) { + FARF(HIGH, "hexlib: VTCM reclaim requested after op %u of %u", + i + 1, hdr.n_ops); + hexlib_vtcm_release(ctx); + batch_status = HEXLIB_DSP_ERR_VTCM_RECLAIMED; + break; + } + } + + /* BEFORE THE HOST READS ANYTHING. Our writes are in this DSP's data cache + * and FastRPC will not write them back for us -- these buffers are mapped + * out of band by fd, not passed as invoke arguments, so it does not know + * they were touched. Measured on SM8650 without this: the 49-op encoder + * returned 179,124 changed bytes and an all-zero final output, because the + * last op's 1024 bytes never got evicted. See skel_bufs.c. + * + * RUNS EVEN WHEN THE BATCH FAILED, and deliberately: an op that failed + * halfway still wrote whatever it wrote, and leaving those lines in cache + * makes the wreckage unreadable to anyone debugging it from the host. + * + * DOES NOT OVERWRITE A REAL FAILURE. If the batch already has a status, that + * is what the caller needs to see; a cache failure on top of it is logged by + * cache_op_all and only becomes the reported status when nothing else went + * wrong. */ + cache_rc = hexlib_bufs_flush(ctx); + if (cache_rc != HEXLIB_DSP_OK && batch_status == HEXLIB_DSP_OK) { + batch_status = cache_rc; + } + + hexlib_write_rsp_hdr(rsp, (uint32_t) batch_status, done, total); + *rsp_len = sizeof(struct hexlib_batch_rsp_hdr) + + done * sizeof(struct hexlib_op_result); + return batch_status; +} diff --git a/hexlib/runtime/skel/skel_internal.h b/hexlib/runtime/skel/skel_internal.h new file mode 100644 index 0000000..55553fa --- /dev/null +++ b/hexlib/runtime/skel/skel_internal.h @@ -0,0 +1,81 @@ +/* hexlib/runtime/skel/skel_internal.h -- skel-private state. */ +#ifndef HEXLIB_SKEL_INTERNAL_H +#define HEXLIB_SKEL_INTERNAL_H + +#include "hexlib_dsp.h" + +#define HEXLIB_MAX_MMAPS 32 + +struct hexlib_mmap { + uint64_t base; + uint64_t size; + int32_t fd; +}; + +struct hexlib_ctx { + struct hexlib_mmap mmap[HEXLIB_MAX_MMAPS]; + uint64_t max_vmem; + + /* The last hexlib_iface_start() status. Recorded because start()'s + * AEEResult may be normalised by the RPC layer, and because a later call + * that finds !started can then say WHY rather than only that it must not + * proceed. HEXLIB_DSP_OK once a session is up. */ + int start_status; + + uint8_t *vtcm_base; + size_t vtcm_size; + uint32_t vtcm_rctx; + int vtcm_valid; + + /* VOLATILE BECAUSE IT IS WRITTEN BY A DIFFERENT THREAD. `release_callback` + * in skel_vtcm.c sets this to 1 from HAP_compute_res's own QuRT thread, + * and hexlib_dispatch_batch's per-op loop reads it. It was a plain `int`. + * That worked only by accident: the opaque `k->fn(&a)` call in the loop + * body is a call through a function pointer the compiler cannot see into, + * so it must assume the callee may have written any escaped object and + * reloads this from memory each iteration. An inlined or + * constant-propagated kernel removes that barrier, and the compiler is + * then entitled to hoist the load out of the loop entirely -- at which + * point a reclaim request arriving mid-batch is never noticed, the + * competing session waits on VTCM this one will not give back, and + * nothing about the source looks different. + * + * `volatile`, not an atomic: this is a one-way 0 -> 1 flag with a single + * writer and a single reader, and the only requirement is that the reader + * actually re-reads memory. There is no read-modify-write to make atomic + * and no other object whose ordering relative to this one matters -- the + * release itself happens on the dispatch thread, after the flag is + * observed (skel_dispatch.c), never in the callback. */ + volatile int vtcm_needs_release; + + uint32_t sess_id; + uint32_t n_hvx; + uint32_t n_hmx; + int started; +}; + +int hexlib_bufs_register(struct hexlib_ctx *ctx, uint32_t fd, uint32_t size); +int hexlib_bufs_unregister(struct hexlib_ctx *ctx, uint32_t fd); +int hexlib_bufs_map(struct hexlib_ctx *ctx, struct hexlib_buf_desc *bufs, uint32_t n); +/* Cache maintenance over every mapped buffer. See skel_bufs.c's block comment + * for why FastRPC does not do this for us and what its absence measured. */ +int hexlib_bufs_invalidate(struct hexlib_ctx *ctx); +int hexlib_bufs_flush(struct hexlib_ctx *ctx); +int hexlib_tensors_resolve(struct hexlib_ctx *ctx, struct hexlib_buf_desc *bufs, + uint32_t n_bufs, struct hexlib_tensor *tens, uint32_t n_tens); + +int hexlib_vtcm_alloc(struct hexlib_ctx *ctx); +void hexlib_vtcm_free(struct hexlib_ctx *ctx); +int hexlib_vtcm_acquire(struct hexlib_ctx *ctx); +void hexlib_vtcm_release(struct hexlib_ctx *ctx); + +int hexlib_dispatch_batch(struct hexlib_ctx *ctx, const uint8_t *batch, uint32_t len, + uint8_t *rsp, uint32_t rsp_cap, uint32_t *rsp_len); + +/* Defined in skel_dispatch.c, shared with skel.c: both hexlib_dispatch_batch's + * own failure returns AND hexlib_iface_invoke's invoke-before-start refusal + * must write the exact same response header shape, with the arch this binary + * was built for -- never a value either caller passes in. */ +void hexlib_write_rsp_hdr(uint8_t *rsp, uint32_t status, uint32_t n_ops, uint64_t cycles); + +#endif /* HEXLIB_SKEL_INTERNAL_H */ diff --git a/hexlib/runtime/skel/skel_vtcm.c b/hexlib/runtime/skel/skel_vtcm.c new file mode 100644 index 0000000..a46fffe --- /dev/null +++ b/hexlib/runtime/skel/skel_vtcm.c @@ -0,0 +1,181 @@ +/* hexlib/runtime/skel/skel_vtcm.c -- acquire VTCM once per session. + * + * THE SIZE COMES FROM THE RUNTIME, NEVER A HARDCODED BYTE COUNT OR A FIXED + * SILICON ADDRESS. VTCM is acquired at session start and what we get is what we + * may use; the M1 allocator's budget must be this number. + * + * AND IT CAN BE TAKEN AWAY. llama.cpp registers a release callback and + * deliberately drops its own priority so that it RECEIVES a reclaim request from + * competing sessions -- a QNN-HTP or another GGML-HTP session can take VTCM + * mid-run. So VTCM is not a static budget the compiler owns. Making the + * allocator resilient to that is M2's problem; noticing it and failing loudly is + * this file's. + * + * Adapted from llama.cpp ggml-hexagon htp/main.c vtcm_acquire/vtcm_alloc (MIT); + * see ATTRIBUTION.md. Upstream aborts the process on failure; we return a + * status instead. + */ +#include "skel_internal.h" + +#include "HAP_compute_res.h" +#include "HAP_farf.h" +#include "qurt_thread.h" + +static int release_callback(unsigned int rctx, void *state) { + struct hexlib_ctx *ctx = (struct hexlib_ctx *) state; + (void) rctx; + /* Do not release here -- the batch in flight is still using it. Record it, + * and let the dispatcher finish the current op and report. */ + ctx->vtcm_needs_release = 1; + return 0; +} + +int hexlib_vtcm_alloc(struct hexlib_ctx *ctx) { + /* BOTH SIZES, AND THE SECOND ONE IS THE POINT. The signature is + * (application_id, total_block_size, total_block_layout, avail_block_size, + * avail_block_layout) -- HAP_compute_res.h:1087-1106. `total` is the whole + * partition assigned to this application type (8388608 on v75); `avail` is + * the SDK's own words "largest contiguous memory chunk available". An + * earlier version of this function passed 0 for avail and asked for total + * as an absolute requirement, which is the bug below. */ + unsigned int vtcm_total = 0; + unsigned int vtcm_avail = 0; + if (HAP_compute_res_query_VTCM(0, &vtcm_total, 0, &vtcm_avail, 0) != 0 || + vtcm_total == 0) { + FARF(ERROR, "hexlib: HAP_compute_res_query_VTCM failed"); + return HEXLIB_DSP_ERR_INTERNAL; + } + /* Nothing at all is a real failure and a distinct one: the partition + * exists but every byte of it is held by someone else. */ + if (vtcm_avail == 0) { + FARF(ERROR, "hexlib: VTCM fully contended -- total %u, available 0", + vtcm_total); + return HEXLIB_DSP_ERR_VTCM_TOO_SMALL; + } + + compute_res_attr_t attr; + HAP_compute_res_attr_init(&attr); + HAP_compute_res_attr_set_serialize(&attr, 0); + HAP_compute_res_attr_set_cache_mode(&attr, 1); + /* min_page_size = 0: best-fit page layout (fewest page mappings). The SDK + * only accepts specific page-size values here (4 KB..16 MB); the queried + * size is not guaranteed to be one of them, so passing it as the page size + * (as an earlier draft of this file did) risks the manager rejecting a + * legitimate request. + * + * min_vtcm_size = vtcm_avail, AND THIS IS A BUG FIX, NOT A TUNING CHOICE. + * It was 0, and HAP_compute_res.h:544-546 defines 0 as "the size is an + * absolute requirement" -- so this asked for the part's ENTIRE VTCM and + * refused anything less. On a shared CDSP that means one other client + * holding a single 4 KB page makes HAP_compute_res_acquire below return 0 + * after burning its full one-second timeout, hexlib_iface_start fails, and + * every mode exits at session open. The SIMULATOR CANNOT SHOW THIS, + * because nothing else there holds VTCM -- which is exactly why stage 1 + * was green with this live. + * + * The floor is the SDK's own `avail`, not a constant, which keeps this + * file's governing rule intact (the size comes from the runtime, never a + * hardcoded byte count): ask for the whole partition, accept down to what + * the manager just said is actually free. Asking for `avail` directly + * instead would cap us at a value that can go stale between query and + * acquire, and would give up headroom that may have been freed in between. + * + * WHAT THIS DELIBERATELY DOES NOT DO: check the result against the plan's + * high water. The DSP does not know the plan's high water -- see §8 of the + * design doc, which used to claim this check existed. So a session can now + * start with less VTCM than a given plan needs, and the honest division of + * labour is that `hwinfo` reports the acquired size, the host records it, + * and M2 compares. The exposure today is nil in practice: `hexlib_args` + * carries vtcm/vtcm_size to every kernel, but no kernel on this branch + * uses either. Revisit the moment one does. */ + HAP_compute_res_attr_set_vtcm_param_v2(&attr, vtcm_total, 0, vtcm_avail); + HAP_compute_res_attr_set_release_callback(&attr, release_callback, (void *) ctx); + /* CONDITIONAL ON THE SESSION ACTUALLY ASKING FOR HMX. `ctx->n_hmx` is set + * by hexlib_iface_start() (skel.c) before this function ever runs; no + * kernel on this branch requests HMX, so hexlib_open (session.c) always + * passes n_hmx = 0. Requesting HMX unconditionally here, regardless of + * that, risked the CDSP refusing the whole compute-res reservation for an + * HMX-availability reason that HAP_compute_res_acquire's single status + * code cannot distinguish from a VTCM-size failure -- the operator would + * see a VTCM error for what was actually an HMX one. REVISIT THIS when an + * HMX kernel first lands: this parameter is not requested at all today. */ + if (ctx->n_hmx > 0) { + HAP_compute_res_attr_set_hmx_param(&attr, 1); + } + + uint32_t rctx = HAP_compute_res_acquire(&attr, 1000000); + if (!rctx) { + /* Both numbers, so a device log distinguishes "the partition is busy" + * from "the manager refused a request it should have satisfied". */ + FARF(ERROR, "hexlib: HAP_compute_res_acquire failed -- wanted %u, " + "floor %u, total %u", vtcm_total, vtcm_avail, vtcm_total); + return HEXLIB_DSP_ERR_VTCM_TOO_SMALL; + } + + void *ptr = 0; + unsigned int got = 0; + if (HAP_compute_res_attr_get_vtcm_ptr_v2(&attr, &ptr, &got) != 0 || !ptr) { + HAP_compute_res_release(rctx); + FARF(ERROR, "hexlib: could not get VTCM pointer"); + return HEXLIB_DSP_ERR_VTCM_TOO_SMALL; + } + + ctx->vtcm_base = (uint8_t *) ptr; + ctx->vtcm_size = got; + ctx->vtcm_rctx = rctx; + ctx->vtcm_valid = 0; + ctx->vtcm_needs_release = 0; + + /* THREE NUMBERS, ON PURPOSE. `got` alone cannot tell the operator whether a + * short session is contention or a manager quirk; got-vs-total-vs-floor + * can, and this is the only place any of it is observable on a device. */ + FARF(HIGH, "hexlib: VTCM %u bytes at %p (total %u, available %u)", + got, ptr, vtcm_total, vtcm_avail); + if (got < vtcm_total) { + FARF(HIGH, "hexlib: VTCM is CONTENDED -- got %u of %u bytes. The session " + "is usable; whether it is large enough for a given plan is " + "not checked here (see design doc SS8)", got, vtcm_total); + } + return HEXLIB_DSP_OK; +} + +int hexlib_vtcm_acquire(struct hexlib_ctx *ctx) { + if (ctx->vtcm_valid) { + return HEXLIB_DSP_OK; + } + if (HAP_compute_res_acquire_cached(ctx->vtcm_rctx, 1000000u) != 0) { + FARF(ERROR, "hexlib: failed to acquire cached VTCM"); + return HEXLIB_DSP_ERR_VTCM_TOO_SMALL; + } + ctx->vtcm_needs_release = 0; + ctx->vtcm_valid = 1; + /* Drop priority to the QuRT default. In QuRT, 1 is the highest thread + * priority and 254 the lowest of the user-assignable range (0 and 255 are + * reserved for the kernel; see qurt_thread.h), so this is the lowest + * priority we can hold -- a competing session at any elevated priority + * will reach us through the release callback instead of silently winning + * arbitration. There is no compute-res-specific "default priority" + * constant in the SDK; QURT_THREAD_ATTR_PRIORITY_DEFAULT is the nearest + * verified named constant (HAP_compute_res_update_priority's own doc + * states its priority argument is "in terms of QuRT thread priority"), so + * it is used here instead of a guessed magic number. */ + HAP_compute_res_update_priority(ctx->vtcm_rctx, QURT_THREAD_ATTR_PRIORITY_DEFAULT); + return HEXLIB_DSP_OK; +} + +void hexlib_vtcm_release(struct hexlib_ctx *ctx) { + if (ctx->vtcm_valid) { + ctx->vtcm_valid = 0; + ctx->vtcm_needs_release = 0; + HAP_compute_res_release_cached(ctx->vtcm_rctx); + } +} + +void hexlib_vtcm_free(struct hexlib_ctx *ctx) { + if (ctx->vtcm_rctx) { + HAP_compute_res_release(ctx->vtcm_rctx); + ctx->vtcm_rctx = 0; + ctx->vtcm_base = 0; + ctx->vtcm_size = 0; + } +} diff --git a/hexlib/runtime/wire.py b/hexlib/runtime/wire.py new file mode 100644 index 0000000..a908c71 --- /dev/null +++ b/hexlib/runtime/wire.py @@ -0,0 +1,234 @@ +# hexlib/runtime/wire.py +"""The batch blob the host sends and the response it gets back. + +WHY A BATCH AND NOT ONE OP PER CALL. A batch descriptor -- buffers, tensors and +ops together, tensors addressed as (buffer index, offset) -- is a serialized +`Compiled`. A single-op descriptor is a function call. Only the first can carry a +plan, and carrying a plan on the DSP is M2, which this work exists to unblock. A +one-op batch is the trivial instance, so nothing is made harder by starting here. + +Shape adapted from llama.cpp ggml-hexagon's htp_opbatch_req (MIT). See +ATTRIBUTION.md. + +THERE IS NO FIELD FOR AN ADDRESS. `BufDesc` has no `base` and `TensorDesc` has no +`data`; the serializer writes zeros into those wire slots and the DSP fills them. +Under the simulator the host and the DSP share one address space, so an +implementation that leaned on a host pointer would pass locally and fail on +silicon. Removing the field is what stops that being expressible. +""" +from __future__ import annotations + +import struct +from dataclasses import dataclass, field + +BATCH_MAGIC = 0x424C5848 # 'HXLB' +BATCH_VERSION = 1 + +MAX_BUFS = 8 +MAX_SRC = 6 +MAX_DST = 4 +MAX_PARAMS = 16 +MAX_TENSORS = 512 + +STATUS = { + "OK": 1, + "ERR_INTERNAL": 2, + "ERR_BAD_MAGIC": 3, + "ERR_BAD_VERSION": 4, + "ERR_TRUNCATED": 5, + "ERR_INVAL_PARAMS": 6, + "ERR_UNMAPPED": 7, + "ERR_NO_MMAP_SLOT": 8, + "ERR_MMAP_FAILED": 9, + "ERR_NO_KERNEL": 10, + "ERR_VTCM_TOO_SMALL": 11, + "ERR_VTCM_RECLAIMED": 12, + "ERR_REQUIRES": 13, + "ERR_NOT_STARTED": 14, + "ERR_CACHE": 15, +} +STATUS_NAME = {v: k for k, v in STATUS.items()} + +# Keyed by the SAME strings as `hexlib.exec.runner.WIRE_DTYPE` and +# `hexlib.runtime.genentry._CTYPE` -- "int32", not "i32". The ids are the ABI +# (they are what `hexlib_tensor.dtype` carries); the keys are host-side names, +# so this spelling fix changed no byte on the wire. It was a live break, not a +# tidy-up: the first spec declaring an int32 input would have been refused by +# `pack_batch` as an unknown dtype AND have KeyError'd genentry's `requires` +# codegen, while `RunnerSpec` accepted it happily. +# `test_runtime_wire.py::test_the_dtype_table_uses_the_same_SPELLING_as_the_ +# runner_and_the_generator` binds the three tables so they cannot drift again. +DTYPE_ID = {"fp32": 0, "fp16": 1, "q4_0": 2, "int32": 3, "q8_0": 4} +LAYOUT_ID = {"row_major": 0, "tiled_32x32": 1, "q4_0_repacked": 2} + +_HDR = "<10I" +_BUF = " bool: + return self.status == STATUS["OK"] + + +@dataclass(frozen=True) +class BatchResponse: + status: int + n_ops: int + cycles_total: int + arch: int + results: tuple[OpResult, ...] = field(default=()) + + @property + def ok(self) -> bool: + return self.status == STATUS["OK"] and all(r.ok for r in self.results) + + +def pack_batch(bufs, tensors, ops) -> bytes: + """Serialize a batch. Refuses malformed input HERE, on the host, where the + error message can be read -- rather than shipping it to a DSP that can only + answer with a status code.""" + if len(bufs) > MAX_BUFS: + raise WireError(f"{len(bufs)} buffers exceeds HEXLIB_MAX_BUFS ({MAX_BUFS})") + if len(tensors) > MAX_TENSORS: + raise WireError(f"{len(tensors)} tensors exceeds {MAX_TENSORS}") + + for i, t in enumerate(tensors): + if not 0 <= t.bi < len(bufs): + raise WireError( + f"tensor {i} names buffer index {t.bi}, but the batch declares " + f"{len(bufs)} buffers" + ) + end = t.offset + t.nbytes + if end > bufs[t.bi].size: + raise WireError( + f"tensor {i} runs past the end of buffer {t.bi}: " + f"offset {t.offset} + {t.nbytes} bytes = {end} > {bufs[t.bi].size}" + ) + if t.dtype not in DTYPE_ID: + raise WireError(f"tensor {i} has unknown dtype {t.dtype!r}") + if t.layout not in LAYOUT_ID: + raise WireError(f"tensor {i} has unknown layout {t.layout!r}") + + for i, op in enumerate(ops): + if len(op.params) > MAX_PARAMS: + raise WireError(f"op {i} has {len(op.params)} params, max {MAX_PARAMS}") + if len(op.src) > MAX_SRC or len(op.dst) > MAX_DST: + raise WireError(f"op {i} exceeds MAX_SRC/MAX_DST") + # THE SUM, WHICH THE TWO CHECKS ABOVE DO NOT COVER. MAX_SRC + MAX_DST is + # 10 and MAX_BUFS is 8: the DSP walks src then dst into ONE `a->buf[]` + # array (`skel_dispatch.c`) and abandons the op at + # `nb >= HEXLIB_MAX_BUFS`, which surfaces as a bare batch status + # INVAL_PARAMS with no way to tell it from any other cause. Named here + # with the actual counts instead. + if len(op.src) + len(op.dst) > MAX_BUFS: + raise WireError( + f"op {i} names {len(op.src)} sources + {len(op.dst)} " + f"destinations = {len(op.src) + len(op.dst)} buffers, but " + f"hexlib_args.buf[] holds HEXLIB_MAX_BUFS ({MAX_BUFS}); the DSP " + f"would abandon the op and report only INVAL_PARAMS" + ) + for j in tuple(op.src) + tuple(op.dst): + if not 0 <= j < len(tensors): + raise WireError(f"op {i} names tensor {j}, out of range") + + off_bufs = HDR_SIZE + off_tensors = off_bufs + BUF_SIZE * len(bufs) + off_ops = off_tensors + TENSOR_SIZE * len(tensors) + total = off_ops + OP_SIZE * len(ops) + + out = bytearray() + out += struct.pack( + _HDR, BATCH_MAGIC, BATCH_VERSION, total, len(bufs), len(tensors), + len(ops), off_bufs, off_tensors, off_ops, 0, + ) + for b in bufs: + # base = 0: the DSP resolves it from its own mmap table, by fd. + out += struct.pack(_BUF, 0, b.size, b.fd, b.flags) + for t in tensors: + out += struct.pack( + _TENSOR, t.bi, t.offset, t.nbytes, DTYPE_ID[t.dtype], + LAYOUT_ID[t.layout], *t.ne, 0, 0, # data = 0, pad + ) + for op in ops: + params = tuple(op.params) + (0,) * (MAX_PARAMS - len(op.params)) + src = tuple(op.src) + (0xFFFF,) * (MAX_SRC - len(op.src)) + dst = tuple(op.dst) + (0xFFFF,) * (MAX_DST - len(op.dst)) + out += struct.pack(_OP, op.kind, op.flags, *params, *src, *dst) + + assert len(out) == total, "header's total_size must equal the blob length" + return bytes(out) + + +def unpack_response(raw: bytes) -> BatchResponse: + if len(raw) < RSP_HDR_SIZE: + raise WireError(f"response truncated: {len(raw)} < {RSP_HDR_SIZE} bytes") + magic, version, status, n_ops, cycles, arch, _ = struct.unpack_from(_RSP_HDR, raw) + if magic != BATCH_MAGIC: + raise WireError( + f"response magic 0x{magic:08x} != 0x{BATCH_MAGIC:08x} — the DSP wrote " + "nothing, or wrote something else" + ) + if version != BATCH_VERSION: + raise WireError(f"response version {version} != {BATCH_VERSION}") + if status not in STATUS_NAME: + raise WireError(f"response status {status} is not a known status") + need = RSP_HDR_SIZE + RESULT_SIZE * n_ops + if len(raw) < need: + raise WireError( + f"response truncated: claims {n_ops} results ({need} bytes), " + f"carries {len(raw)}" + ) + results = tuple( + OpResult(*struct.unpack_from(_RESULT, raw, RSP_HDR_SIZE + i * RESULT_SIZE)) + for i in range(n_ops) + ) + return BatchResponse(status, n_ops, cycles, arch, results) diff --git a/hexlib/sim.py b/hexlib/sim.py index 59bdb5c..821e0ce 100644 --- a/hexlib/sim.py +++ b/hexlib/sim.py @@ -15,6 +15,7 @@ import re from dataclasses import dataclass +from hexlib import hostload from hexlib import toolchain as tc from hexlib.build import BuildOutput @@ -106,15 +107,18 @@ def run_sim( sim_exe = os.path.join(build_out.bin_dir, tc.exe("hexagon-sim")) cmd = sim_command(sim_exe, build_out.elf, caps) - rc, out, err, timed_out = tc.run( - cmd, env, timeout=timeout or tc.SIM_TIMEOUT_MAX_S - ) + budget = timeout or tc.SIM_TIMEOUT_MAX_S + # Sampled for the life of the run so a timeout can be ATTRIBUTED rather + # than guessed at. See hexlib/hostload.py for what goes wrong without it. + with hostload.SimLoadMonitor() as monitor: + rc, out, err, timed_out = tc.run(cmd, env, timeout=budget) + load = monitor.stats() combined = out + err if timed_out: raise SimError( - f"simulator timed out after {timeout or tc.SIM_TIMEOUT_MAX_S}s — " - "the kernel may not terminate", + f"simulator timed out after {budget}s — " + f"{hostload.timeout_diagnosis(budget, load)}", combined, ) diff --git a/hexlib/tests/csource.py b/hexlib/tests/csource.py new file mode 100644 index 0000000..7aa9caa --- /dev/null +++ b/hexlib/tests/csource.py @@ -0,0 +1,424 @@ +# hexlib/tests/csource.py +"""Shared, comment-aware C source slicing for the source-assertion test +files: test_host_source.py, test_skel_bufs_source.py, +test_skel_vtcm_source.py, test_skel_dispatch_source.py, test_kernels.py, +test_session_arch_decode.py, test_coherency_lane_classification.py, +test_runtime_wire.py, test_wire_struct_layout.py, test_vtcm_contention.py and +test_device_cycles_assertion.py. Those are all of them -- there is no surviving +private copy of this slicer anywhere in hexlib/tests, and adding one is the +thing this module exists to stop. + +WHY THIS EXISTS. Four test files independently grew their own copy (or a +near-copy, in test_skel_vtcm_source.py's `_block_after_call`) of a +brace-counting function-body slicer built on the regex +`name\\s*\\([^;{]*\\)\\s*\\{`. That regex is COMMENT-BLIND: it is matched +directly against the raw source text, so if the function's own NAME happens +to appear in a comment -- in prose, e.g. "... fails hexlib_drv_init() +rather than being silently tolerated ..." -- ahead of the real definition, +and that comment sits close enough to some unrelated `{` with nothing but +whitespace in between, the match lands on the WRONG brace and the slicer +returns the WRONG function's body (or a fragment of neither). A test built +on top of that slice would then pass or fail based on code it never +intended to inspect -- an assertion that looks like it verifies a guarantee +while actually checking something else. This bit for real once (see +driver.c's own "PLACED AFTER THE INIT ROUTINE ABOVE, DELIBERATELY" comment, +which documents a comment block being physically MOVED to dodge exactly this +regex, rather than the regex being fixed). This module is the fix: extraction +is comment-aware, so future comments can be placed for readability, not to +avoid confusing a test. + +THE CONSOLIDATION CLAIM, STATED HONESTLY. An earlier version of this +docstring said the consolidation was complete. It was not: a private copy +with a weaker `_strip_comments` (comment text DELETED rather than blanked, +so every offset shifted) survived in test_skel_dispatch_source.py, and a +fourth, weaker still (no comment handling at all, `src.index("void " + name ++ "(")`) survived in test_kernels.py. Both have since been migrated here. +The list at the top of this docstring is the enumeration that replaces the +claim: if a new source-assertion test appears and is not on it, the claim is +false again. + +HOW. `code_only` produces a same-LENGTH copy of the source with every +`/* ... */` and `// ...` comment blanked out AND the INTERIOR of every string +and character literal blanked out (replaced by spaces, newlines kept so line +numbers do not shift). All matching -- finding a function's signature, counting +brace depth, finding the next `{` from some offset, or locating a call site -- +is done against this blanked copy. Because blanking preserves length exactly, +an offset computed against the blanked copy is valid against the ORIGINAL +source too, so a slice taken at those offsets is valid against either text. + +COMMENT-AWARE BOUNDARIES ARE ONLY HALF THE JOB -- THE RETURNED TEXT MATTERS +JUST AS MUCH. This module originally returned a slice of the ORIGINAL, +comment-BEARING source, and its docstrings presented that as a feature ("the +caller sees real code -- including any comment that is genuinely inside the +extracted function's own body"). It was a hole, and it was the exact hole +this module was written to close, one level up: consumers run their PAYLOAD +checks (`x in body`, `body.index(...)`, `re.search(..., body)`) against +whatever is returned, so a mutation could delete a guard and leave the +deleted code behind AS A COMMENT and every assertion about it would still +pass. That was proven, not theorised: replacing skel_bufs.c's unmapped-fd +`return HEXLIB_DSP_ERR_UNMAPPED;` with a `continue` that trusts the host's +fd -- precisely the shared-address-space bug the staged gate exists to catch +-- left test_skel_bufs_source.py reporting 8 passed. + +CLOSING THE COMMENT VEHICLE LEFT THE LITERAL VEHICLE WIDE OPEN, AND THAT WAS +THIS MODULE'S FAULT. The version of this file that fixed the comment hole +deliberately returned string and character literals UNTOUCHED, and its own +"STRING LITERAL CAVEAT" reasoned only about a `%p` inside a format string and +about a `/*` inside a literal being misread as a comment start. It never +reckoned with the literal being the same hole in a different vehicle. A +merge-gate review demonstrated three distinct mechanisms, each reproduced +here against the then-current code: + + (A) POSITIVE ASSERTIONS DEFEATED BY A LOG STRING. `assert "TOKEN" in body` + is satisfied by `FARF(HIGH, "TOKEN")` with the real code deleted. This + is not hypothetical and never was: three of the test files listed above + recount, in their OWN docstrings, earlier rounds of exactly this + FARF-string vector being exploited (test_skel_vtcm_source.py's + HAP_compute_res_query_VTCM error string, test_skel_bufs_source.py's + `nbytes`, test_host_source.py's `"hexlib: dlopen(%s) failed"`). Fixing + those three call sites one at a time, while leaving the SLICER handing + literal text to every payload check, was treating instances of a defect + whose cause was here. Reproduced: skel_bufs.c's `b->base = 0;` replaced + by a FARF printing that same text, with the real `b->base = m->base;` + deleted -- so the DSP hands kernels whatever address the host wrote -- + left test_skel_bufs_source.py at 8 passed. + + (B) NEGATIVE ASSERTIONS DEFEATED BY TRUNCATING THE SLICE. Brace depth was + counted over text in which literals survived, so a `}` inside a literal + decremented the count. `FARF(HIGH, "pcycle }")` inside a function made + `function_body` return everything up to that `}` and nothing after it, + so every `X not in body` check was answered by a fragment. Reproduced: + skel_dispatch.c's `hexlib_read_pcycle` reverted to + `__asm__("%0 = c15:14")` -- unreadable in a user-mode unsigned PD, which + is the one thing that wrapper exists to avoid -- hidden after such a + FARF, left test_skel_dispatch_source.py at 12 passed. + + (C) SCOPING DESTROYED BY A LITERAL `{`. The mirror image: an extra `{` + inside a literal made `block_from`'s depth count swallow the function + tail, so an `if`-block "scoped" check saw the rest of the function. + +So `code_only` now blanks the INTERIOR of every string and character literal +as well, and every slicer counts braces and parens over that text. The +literals' own delimiters are kept, and blanking is still exactly +length-preserving, which is what keeps every offset valid against the original +file -- the property the whole module is built on. + +BOUNDARY FINDING STILL NEEDS THE LITERALS WHOLE, WHICH IS WHY THIS IS ONE +TOKENIZER PASS AND NOT TWO REGEX PASSES. A `//` or `/*` inside a string +literal must not be mistaken for a comment start, so the literal has to be +recognized and consumed as a single token BEFORE anything inside it is +considered -- exactly as before. What changed is only what is written back out +for the token once it has been recognized. + +THE ESCAPE HATCH, AND THE THREE PLACES IT IS LEGITIMATE. +`code_only_keeping_strings` blanks comments and leaves literals intact. Use it +only where the CLAIM ITSELF IS ABOUT LITERAL TEXT, and say so at the call +site. In this repo that is: a printed line whose exact wording another test +asserts on (main.c's `PASS (%d values, bit-exact)`), a command-line flag +string main() must recognize, and a path or macro spelling that must or must +not appear (`"libcdsprpc.so"`, `"&_dom=cdsp"`). Everything else -- every +"this guard must be here", every "this constant must not be here" -- goes +through `code_only`, because for those a literal is evidence of nothing. +Boundary finding and brace counting use the fully-blanked text either way, so +asking to keep literals never re-opens (B) or (C); it re-opens only (A), and +only for the one check that asked. + +`#include "hdr.h"` IS NOT A STRING LITERAL AND IS NOT BLANKED. In C a +header-name is its own token class -- no escape processing, no concatenation +-- and it cannot be written by a mutation trying to hide code, because it has +to name a file that exists for the translation unit to compile at all. So +`#include "HAP_perf.h"` survives `code_only` and +test_skel_dispatch_source.py's check that the SDK header is really included +(not merely referred to in prose) keeps working without an escape hatch. +`#include ` was never affected. + +FUNCTION SCOPE IS NOT THE SAME AS REACHABILITY, AND `calls()` IS WHAT THIS +MODULE OFFERS ABOUT THAT. Every negative check built on these slicers is +scoped to one function or one block, so a forbidden construct can be moved one +call level away and the check sees nothing -- no comment and no literal +required. Proven: moving `__asm__("%0 = c15:14")` out of `hexlib_read_pcycle` +into a new `hexlib_raw_pcycle()` helper it calls left +test_skel_dispatch_source.py at 12 passed. `calls()` (see its own docstring) +lets a test state the exhaustive set of callees a block has, so a new helper +is a failure by construction; a whole-FILE negative is the other half, for a +construct that must not exist anywhere at all. + +WHAT THIS STILL DOES NOT HANDLE. Escaped quotes are handled (a +backslash-escaped quote inside a literal does not end it), but a +backslash-newline line continuation inside a literal is not, and a +malformed/unterminated literal will make the regex consume everything up to +the next quote of the same kind, wherever that is. Adjacent literals that C +would concatenate are blanked individually, which is the same answer. +Both remaining gaps are exotic enough, and absent from this project's +straight-line C, that handling them is not worth the complexity here. This is +a test helper for known, checked-in source files, not a general C +preprocessor. + +NOT A GENERAL C PARSER. No handling of trigraphs, raw string edge cases, +`#if 0`-disabled code (see skel_bufs.c's own `#if __HVX_ARCH__ > 73` -- +brace-depth counting still works there because a whole preprocessor +`#if`/`#else`/`#endif` block in this codebase's style always has matching +braces on both sides), or anything else beyond what this project's own +conventionally-formatted C actually does. Good enough for that; nothing more +is claimed. +""" +import re + +# Matches, in priority order at any given position: a `#include "hdr.h"` +# header-name (NOT a string literal -- see the docstring), a block comment, a +# line comment, a double-quoted string literal, or a single-quoted character +# literal. `re.sub` scans left to right for the next position at which ANY +# alternative matches, so a `"` or `'` that starts a real literal is matched +# as a literal rather than having some `//`/`/*` inside it mistaken for a +# comment -- the literal is consumed as one token, so nothing inside it is +# considered separately. That is what makes it safe to blank a literal's +# INTERIOR: the decision to blank is made about a token already known to be a +# literal, not about the characters inside one. +_TOKEN = re.compile( + r'^[ \t]*\#[ \t]*include[ \t]*"[^"\n]*"' + r"|/\*.*?\*/" + r"|//[^\n]*" + r'|"(?:\\.|[^"\\])*"' + r"|'(?:\\.|[^'\\])*'", + re.DOTALL | re.MULTILINE, +) + + +def _spaces(text): + """Same-length whitespace, newlines kept so line numbers do not shift.""" + return "".join("\n" if ch == "\n" else " " for ch in text) + + +def _blank(text, blank_strings): + if text.lstrip().startswith("#"): + return text # `#include "hdr.h"` -- a header-name, not a literal + if text[0] in "\"'": + if not blank_strings: + return text + # Keep the delimiters (so the token is still visibly a literal, and a + # check that a literal EXISTS at all still works) and blank the + # interior. Length is preserved either way. + return text[0] + _spaces(text[1:-1]) + text[-1] + return _spaces(text) # a comment + + +def code_only(text): + """Return a same-length copy of `text` with every `/* ... */` and `// ...` + comment blanked out AND the interior of every string and character literal + blanked out (newlines preserved throughout, so line numbers do not shift). + `#include "hdr.h"` header-names are left alone -- see the module docstring. + + THIS IS THE ONE TO USE. It produces text that a payload check ("this call + must be here", "this constant must not be here") can safely be run + against, because nothing in it came from a comment and nothing in it came + from a log message, a format string or a CLI-flag string. Both of those + were proven vehicles for satisfying an assertion while deleting the code it + was about; see the module docstring for the mutations. + + Because the result is the same length as `text`, an offset found in the + result is valid as an offset into `text` too -- that is the whole point: + boundaries found here can be used to slice either version. + + Idempotent, and interchangeable in either order with + `code_only_keeping_strings`.""" + return _TOKEN.sub(lambda m: _blank(m.group(0), True), text) + + +def code_only_keeping_strings(text): + """`code_only`, but string and character literals are left INTACT. The + deliberate escape hatch, for the few checks whose subject IS literal text: + a printed line whose exact wording is asserted elsewhere, a command-line + flag string, a path or macro spelling that must (or must not) appear. + + Say why at the call site. For anything else this is the wrong function: a + token inside a format string is not evidence that the code it names is + still there, which is the whole finding this module was rewritten for. + + Same same-length guarantee. Note that the slicers below always count + braces and parens over `code_only` text regardless, so passing a + literal-bearing fixture cannot move a boundary.""" + return _TOKEN.sub(lambda m: _blank(m.group(0), False), text) + + +# Keywords and type specifiers that are followed by `(` in this project's C +# without being a call. `sizeof(T)`, `if (`, `while (`, `for (`, `switch (`, +# `return (x)`, and a cast's own type name in `(uint64_t) (uintptr_t) p`. +_NOT_CALLEES = frozenset(""" + if else for while do switch case return goto sizeof + void char short int long float double signed unsigned _Bool + struct union enum const volatile static inline extern register typedef + defined +""".split()) + +_CALLEE = re.compile(r"\b([A-Za-z_]\w*)\s*\(") + + +def calls(fragment): + """The set of identifiers that appear as the CALLEE of a call in + `fragment`, over `code_only` text. `k->fn(&a)` contributes `fn`. + + THIS EXISTS TO CLOSE THE "MOVE IT ONE CALL LEVEL AWAY" ESCAPE. A + function-scoped negative check -- "this block must NOT call X" -- is + satisfied by a mutation that puts X in a new one-line helper and calls the + helper instead. Nothing about the block's own text changed except the name, + so no `X not in block` check can see it. That was demonstrated on + skel_dispatch.c: moving `__asm__("%0 = c15:14")` into a new + `hexlib_raw_pcycle()` left test_skel_dispatch_source.py at 12 passed, with + no comment or literal trick involved at all. + + Two answers to that, and both are used in this repo. Where the forbidden + construct must not exist ANYWHERE, pair the scoped negative with a + whole-FILE one (the raw register read). Where it must not be REACHED from + one specific block, assert the exhaustive set of things that block calls -- + an unexpected callee is then a failure by construction, whatever it is + named, because a helper that hides the construct still has to be called + from somewhere. + + Not a call graph: this is one level, textual, and deliberately so. A + two-level indirection would defeat it, and the whole-file pairing is what + covers that case.""" + return { + m.group(1) + for m in _CALLEE.finditer(code_only(fragment)) + if m.group(1) not in _NOT_CALLEES + } + + +def _view(src, strip, keep_strings): + """The text a slicer RETURNS a slice of, given its two flags.""" + if not strip: + return src # exactly as given, comments and literals included + if keep_strings: + return code_only_keeping_strings(src) + return code_only(src) + + +def function_body(src, name, strip=True, keep_strings=False): + """Slice one C function's definition -- from its own opening brace through + the matching closing brace -- out of `src`, by simple brace-depth + counting. Good enough for this project's straight-line C; not a general + C parser. + + Comment- AND literal-aware in BOTH directions. The signature search and the + brace-depth count run against `code_only(src)`, so a comment that merely + mentions `name` in prose cannot derail the match onto the wrong function, + and neither a comment nor a string literal can supply a stray `{`/`}` that + truncates the body or swallows the next function. And with `strip=True` + (the default) the text RETURNED is `code_only` text too, so a payload check + the caller runs against it cannot be satisfied by a comment inside the body + or by a log message -- see the module docstring for the proven mutations + behind each half. + + `keep_strings=True` returns the comment-blanked but literal-BEARING slice, + for a caller whose claim is genuinely about literal text; boundaries are + unaffected. `strip=False` returns the slice of `src` exactly as given. + Either way, say why at the call site.""" + matching = code_only(src) + m = re.search(rf"\b{re.escape(name)}\s*\([^;{{]*\)\s*\{{", matching) + assert m, f"could not find the definition of {name}() in the source" + out = _view(src, strip, keep_strings) + start = m.end() - 1 # position of the opening brace + depth = 0 + for i in range(start, len(matching)): + if matching[i] == "{": + depth += 1 + elif matching[i] == "}": + depth -= 1 + if depth == 0: + return out[start : i + 1] + raise AssertionError(f"unbalanced braces while slicing {name}()") + + +def block_from(text, pos, strip=True, keep_strings=False): + """From `pos`, find the next `{` and return the brace-matched block it + opens (inclusive). Generalizes `function_body`'s closing half to an + arbitrary starting offset, so one specific `if (...) { ... }` can be + isolated instead of just checking "somewhere in the rest of the + function" -- which a later, unrelated `return` statement could satisfy + by accident. + + Comment- and literal-aware for the same reasons as `function_body`, in both + directions: the brace search and depth count run against + `code_only(text)`, so neither a comment nor a string literal between `pos` + and the real block (or inside it) can supply a spurious `{`/`}` and throw + off the match, and with `strip=True` (the default) the text returned is + blanked so a payload check against it cannot be satisfied by a comment or a + log message inside the block. `pos` is an offset into `text` and is valid + against every version, since blanking preserves length. + + See `function_body` for `strip` and `keep_strings`.""" + matching = code_only(text) + out = _view(text, strip, keep_strings) + brace = matching.index("{", pos) + depth = 0 + for i in range(brace, len(matching)): + if matching[i] == "{": + depth += 1 + elif matching[i] == "}": + depth -= 1 + if depth == 0: + return out[brace : i + 1] + raise AssertionError("unbalanced braces while slicing a block") + + +def block_after_call(body, call_name, strip=True, keep_strings=False): + """Within a function body, find a call to `call_name` and return the + text of the nearest brace-delimited block that checks its result -- + either the call sits inside an `if` condition (`if (call(...) != 0) { + ... }`), or an `if` immediately follows the call as a separate statement + (`x = call(...); if (!x) { ... }`). Both shapes occur in this project's + skel_vtcm.c. + + Comment- and literal-aware for the same reasons as + `function_body`/`block_from`: locating the call, walking its own parens, + checking for a preceding `if (`, finding the block, and counting its brace + depth are ALL done against `code_only(body)`, so neither a comment nor a + string literal mentioning `call_name`, or containing a stray paren or + brace, can be mistaken for the real call site or its guard, or truncate the + block. With `strip=True` (the default) the returned block is blanked too, + so the `return ` a caller then looks for in it can be neither a + commented-out one nor one named in a FARF. + + Asserts an `if (` appears between the call and the block, so a stray + block that has nothing to do with checking the call's result cannot be + picked up by accident. + + See `function_body` for `strip` and `keep_strings`.""" + matching = code_only(body) + out = _view(body, strip, keep_strings) + m = re.search(rf"\b{re.escape(call_name)}\s*\(", matching) + assert m, f"no call to {call_name}() found in this function" + call_start = m.start() + + # Walk the call's own parens to find where its argument list ends -- + # none of this file's calls nest parens, but do it properly anyway. + depth = 0 + call_end = None + for i in range(m.end() - 1, len(matching)): + if matching[i] == "(": + depth += 1 + elif matching[i] == ")": + depth -= 1 + if depth == 0: + call_end = i + 1 + break + assert call_end is not None, f"unbalanced parens in the call to {call_name}()" + + brace_pos = matching.find("{", call_end) + assert brace_pos != -1, f"no block follows the call to {call_name}()" + + window = matching[max(0, call_start - 80) : brace_pos] + assert "if" in window and "(" in window, ( + f"{call_name}()'s result does not appear to be checked by an `if` " + f"before the block that follows it" + ) + + depth = 0 + for i in range(brace_pos, len(matching)): + if matching[i] == "{": + depth += 1 + elif matching[i] == "}": + depth -= 1 + if depth == 0: + return out[brace_pos : i + 1] + raise AssertionError(f"unbalanced braces in the block following {call_name}()") diff --git a/hexlib/tests/test_cli_device_flag.py b/hexlib/tests/test_cli_device_flag.py new file mode 100644 index 0000000..e2de7c4 --- /dev/null +++ b/hexlib/tests/test_cli_device_flag.py @@ -0,0 +1,378 @@ +# hexlib/tests/test_cli_device_flag.py +"""`hexlib test --device sim|local|qdc`. + +Every guard here is checked to run BEFORE `cli._qdc_submit` -- the function +that actually touches the SDK, a credential, or the network -- ever gets +called. `_qdc_submit` itself is monkeypatched in every test that reaches it, +so nothing here builds a real device artifact, reads a credential, or makes +a network call; the tests that exercise the guards past the missing-timeout +check are, structurally, offline tests of argument handling and print +ordering, nothing more. + +THE KERNEL ARGUMENT IS `scale_fp16` THROUGHOUT, and until 2026-08-11 it was +`some/kernel` -- which passed, because `--device qdc` read the argument +NOWHERE. `hexlib test add_fp16 --device qdc --timeout-min 20 --yes` would +spend 20 non-renewable minutes running `scale_fp16` (test_on_device.py +hard-codes `./hexlib_run --self-test`, main.c hard-codes build_scale_batch) +and return green. Every test below that expects submission to PROCEED must +therefore name a kernel stage 3 can genuinely run, and the refusal itself is +pinned by the tests near the end of this file. +""" +import os + +import pytest + +from hexlib import cli +from hexlib.result import Measurements, Ok + + +def _measurements() -> Measurements: + return Measurements( + kernel_cycles=886, + toolchain_version="test", + sdk_version="test", + host="test", + timestamp="2026-08-11T00:00:00", + ) + + +def test_device_defaults_to_sim(monkeypatch, tmp_path): + """No --device at all must reach the SAME code path as --device sim -- + verified by watching `verify` actually get called with this kernel's own + path, not by inference from an error message. A cli.py that quietly + changed the default to `local` or `qdc` would still print SOME output; + only checking that `verify` itself ran, with the right argument, catches + that.""" + calls = {} + + def fake_verify(kernel, out): + calls["kernel"] = kernel + calls["out"] = out + return Ok(_measurements()) + + monkeypatch.setattr(cli, "verify", fake_verify) + kernel_dir = str(tmp_path / "k") + os.makedirs(kernel_dir) + rc = cli.main(["test", kernel_dir, "--out", str(tmp_path / "out")]) + assert rc == 0 + assert calls == {"kernel": kernel_dir, "out": str(tmp_path / "out")} + + +def test_device_sim_is_the_same_path_as_no_flag_at_all(monkeypatch, tmp_path): + """The mirror of the test above, with --device sim spelled out + explicitly -- both must land on the identical `verify` call.""" + calls = [] + + def fake_verify(kernel, out): + calls.append((kernel, out)) + return Ok(_measurements()) + + monkeypatch.setattr(cli, "verify", fake_verify) + kernel_dir = str(tmp_path / "k") + os.makedirs(kernel_dir) + cli.main(["test", kernel_dir, "--device", "sim", "--out", str(tmp_path / "out")]) + assert calls == [(kernel_dir, str(tmp_path / "out"))] + + +def test_device_local_says_it_is_not_implemented_rather_than_failing_obscurely(capsys): + rc = cli.main(["test", "some/kernel", "--device", "local"]) + assert rc != 0 + err = capsys.readouterr().err.lower() + assert "not implemented" in err + assert "no device available" in err + + +def test_device_local_never_reaches_verify_or_qdc(monkeypatch, capsys): + """A --device local that fell through to the sim path (or the qdc path) + would still print SOME error, which could look like "the right shape" to + a weaker test. This one proves it took neither: `verify` and + `cli._qdc_submit` are both wired to raise if reached at all.""" + def boom_verify(kernel, out): + raise AssertionError("verify must not run for --device local") + + def boom_submit(args): + raise AssertionError("_qdc_submit must not run for --device local") + + monkeypatch.setattr(cli, "verify", boom_verify) + monkeypatch.setattr(cli, "_qdc_submit", boom_submit) + rc = cli.main(["test", "some/kernel", "--device", "local"]) + assert rc != 0 + + +def test_qdc_refuses_without_a_timeout(monkeypatch, capsys): + def boom_submit(args): + raise AssertionError("_qdc_submit must not run without --timeout-min") + + monkeypatch.setattr(cli, "_qdc_submit", boom_submit) + rc = cli.main(["test", "scale_fp16", "--device", "qdc"]) + assert rc != 0 + err = capsys.readouterr().err + assert "--timeout-min" in err + assert "timeout" in err.lower() + + +def test_qdc_prints_the_budget_before_submitting(monkeypatch, capsys): + """The budget line must be PRESENT in stdout, and it must appear before + `_qdc_submit` is ever reached -- checked by making the fake submit + itself assert the budget line already printed, not merely by checking + final output order after the fact (which could pass even if a future + edit moved the print to happen lazily, inside submit, or not at all, + as long as it eventually landed in stdout somewhere).""" + order = [] + + def fake_submit(args): + order.append("submit") + # By the time submit runs, the budget line must already be in stdout. + out = capsys.readouterr().out + assert "remaining budget" in out, ( + "the budget must print BEFORE submission is attempted, not after" + ) + return 0 + + monkeypatch.setattr(cli, "_qdc_submit", fake_submit) + monkeypatch.delenv(cli._QDC_BUDGET_ENV, raising=False) + rc = cli.main(["test", "scale_fp16", "--device", "qdc", "--timeout-min", "5"]) + assert rc == 0 + assert order == ["submit"] + + +def test_qdc_prints_the_actual_env_budget_when_set(monkeypatch, capsys): + monkeypatch.setattr(cli, "_qdc_submit", lambda args: 0) + monkeypatch.setenv(cli._QDC_BUDGET_ENV, "42") + cli.main(["test", "scale_fp16", "--device", "qdc", "--timeout-min", "5"]) + out = capsys.readouterr().out + assert "42" in out + assert "remaining budget" in out + + +def test_qdc_requires_yes_above_the_threshold(monkeypatch, capsys): + def boom_submit(args): + raise AssertionError("_qdc_submit must not run above the threshold without --yes") + + monkeypatch.setattr(cli, "_qdc_submit", boom_submit) + above = cli._QDC_YES_THRESHOLD_MIN + 1 + rc = cli.main(["test", "scale_fp16", "--device", "qdc", "--timeout-min", str(above)]) + assert rc != 0 + err = capsys.readouterr().err.lower() + assert "--yes" in err + assert "threshold" in err + + +def test_qdc_proceeds_above_the_threshold_with_yes(monkeypatch, capsys): + calls = [] + monkeypatch.setattr(cli, "_qdc_submit", lambda args: calls.append(args) or 0) + above = cli._QDC_YES_THRESHOLD_MIN + 1 + rc = cli.main([ + "test", "scale_fp16", "--device", "qdc", + "--timeout-min", str(above), "--yes", + ]) + assert rc == 0 + assert len(calls) == 1 + + +def test_qdc_proceeds_at_or_below_the_threshold_without_yes(monkeypatch): + calls = [] + monkeypatch.setattr(cli, "_qdc_submit", lambda args: calls.append(args) or 0) + rc = cli.main([ + "test", "scale_fp16", "--device", "qdc", + "--timeout-min", str(cli._QDC_YES_THRESHOLD_MIN), + ]) + assert rc == 0 + assert len(calls) == 1 + + +# ============================================================================== +# THE ARGUMENT IS READ, NOT IGNORED. +# +# `_cmd_test_qdc` and `_qdc_submit` both used to ignore `args.kernel` entirely +# -- `hexlib/tests/test_cli_qdc_results.py` even built an `argparse.Namespace` +# with NO `kernel` attribute at all and `_qdc_submit` ran fine. So +# `hexlib test add_fp16 --device qdc --timeout-min 20 --yes` spent 20 +# non-renewable minutes, measured `scale_fp16` (the staged script hard-codes +# `./hexlib_run --self-test`; main.c's run_self_test hard-codes +# build_scale_batch), exited 0, and told the operator add_fp16 was validated on +# silicon. Making the argument genuinely work means parameterising main.c -- a +# real change, not made. Refusing is the honest state. +# ============================================================================== + + +def _boom_submit(args): + raise AssertionError("_qdc_submit must not run for an unsupported kernel") + + +@pytest.mark.parametrize("kernel", ["add_fp16", "kernels/add_fp16", + "rmsnorm_fp16", "layernorm_fp16"]) +def test_qdc_refuses_any_kernel_but_scale_fp16(monkeypatch, capsys, kernel): + monkeypatch.setattr(cli, "_qdc_submit", _boom_submit) + rc = cli.main([ + "test", kernel, "--device", "qdc", "--timeout-min", "5", + ]) + assert rc != 0 + err = capsys.readouterr().err + assert "scale_fp16-only" in err + assert os.path.basename(kernel) in err + + +def test_qdc_refuses_an_unsupported_kernel_before_anything_else(monkeypatch, capsys): + """The kernel refusal comes FIRST -- before the missing-timeout check and + before the budget line. "This command cannot run what you asked for" is + the more useful message when more than one guard would fire, and it costs + nothing to check.""" + monkeypatch.setattr(cli, "_qdc_submit", _boom_submit) + rc = cli.main(["test", "add_fp16", "--device", "qdc"]) # no --timeout-min + assert rc != 0 + out, err = capsys.readouterr() + assert "scale_fp16-only" in err + assert "remaining budget" not in out + + +def test_qdc_accepts_the_kernel_directory_path_form_too(monkeypatch): + """`--device sim` takes `kernels/scale_fp16`; docs/STATE.md's stage-3 entry + point spells it `scale_fp16`. One CLI, both forms.""" + calls = [] + monkeypatch.setattr(cli, "_qdc_submit", lambda args: calls.append(args) or 0) + rc = cli.main([ + "test", os.path.join("kernels", "scale_fp16"), "--device", "qdc", + "--timeout-min", "5", + ]) + assert rc == 0 + assert len(calls) == 1 + + +# ============================================================================== +# QDC_BUDGET_MIN IS COMPARED TO --timeout-min, not merely printed. +# +# VERIFIED BEFORE THE FIX: `QDC_BUDGET_MIN=3 hexlib test k --device qdc +# --timeout-min 240 --yes` printed `remaining budget: 3 minutes (from +# QDC_BUDGET_MIN)` and then submitted a 240-minute job -- an 80x overspend of +# non-renewable minutes passing every guard, with the number that should have +# stopped it on screen. `QDC_BUDGET_MIN=abc` printed "remaining budget: abc +# minutes". +# ============================================================================== + + +def test_qdc_refuses_a_timeout_larger_than_the_recorded_budget(monkeypatch, capsys): + monkeypatch.setattr(cli, "_qdc_submit", _boom_submit) + monkeypatch.setenv(cli._QDC_BUDGET_ENV, "3") + rc = cli.main([ + "test", "scale_fp16", "--device", "qdc", "--timeout-min", "240", "--yes", + ]) + assert rc != 0 + err = capsys.readouterr().err + assert "240" in err and "3" in err + assert "budget" in err.lower() + + +def test_qdc_allows_a_timeout_exactly_equal_to_the_budget(monkeypatch): + """Spending the last recorded minutes deliberately is a real thing to + want; only EXCEEDING the budget is refused.""" + calls = [] + monkeypatch.setattr(cli, "_qdc_submit", lambda args: calls.append(args) or 0) + monkeypatch.setenv(cli._QDC_BUDGET_ENV, "5") + rc = cli.main(["test", "scale_fp16", "--device", "qdc", "--timeout-min", "5"]) + assert rc == 0 + assert len(calls) == 1 + + +@pytest.mark.parametrize("bad", ["abc", "", " ", "5.5", "-5", "5 minutes", "1e3"]) +def test_qdc_refuses_a_malformed_budget_rather_than_printing_it( + monkeypatch, capsys, bad +): + """A budget that cannot be compared is refused, not echoed. A guard that + silently disables itself on a typo is worse than no guard, because the + operator believes it is watching.""" + monkeypatch.setattr(cli, "_qdc_submit", _boom_submit) + monkeypatch.setenv(cli._QDC_BUDGET_ENV, bad) + rc = cli.main(["test", "scale_fp16", "--device", "qdc", "--timeout-min", "5"]) + assert rc != 0 + err = capsys.readouterr().err + assert cli._QDC_BUDGET_ENV in err + + +def test_an_unset_budget_means_unknown_and_says_no_check_was_made(monkeypatch, capsys): + """THE STATED DECISION, pinned so it cannot drift into an accident: unset + means UNKNOWN, submission proceeds, and the line says outright that no + budget check happened. It must not read as "unlimited" or as an assurance + that the job fits.""" + calls = [] + monkeypatch.setattr(cli, "_qdc_submit", lambda args: calls.append(args) or 0) + monkeypatch.delenv(cli._QDC_BUDGET_ENV, raising=False) + rc = cli.main(["test", "scale_fp16", "--device", "qdc", "--timeout-min", "5"]) + assert rc == 0 + assert len(calls) == 1 + out = capsys.readouterr().out.lower() + assert "unknown" in out + assert "no budget check" in out + assert "not unlimited" in out + + +def test_the_budget_is_read_from_the_environment_and_never_queried(monkeypatch): + """Nothing in the budget path may reach QDC: there is no reliable + remaining-minutes API on this account, and a test must never make a + network call. Proven by making the whole QDC client constructor explode + if touched.""" + from hexlib.device.qdc import job + + def boom(): + raise AssertionError("the budget path must never build a QDC client") + + monkeypatch.setattr(job, "_client", boom) + monkeypatch.setenv(cli._QDC_BUDGET_ENV, "17") + assert cli._qdc_remaining_budget_min() == 17 + assert cli._qdc_budget_guard(5) == 0 + + +# ============================================================================== +# --timeout-min's 1..240 RANGE IS ENFORCED BEFORE THE BUILD. +# +# job.submit raises on the same range, but only after `_qdc_submit` has run a +# full SDK cross-compile of hexlib_run + libhexlib_skel.so and staged a zip. +# The bounds come from job.py, never respelled here. +# ============================================================================== + + +@pytest.mark.parametrize("bad", [0, -1, 241, 100000]) +def test_qdc_refuses_an_out_of_range_timeout_before_building_anything( + monkeypatch, capsys, bad +): + monkeypatch.setattr(cli, "_qdc_submit", _boom_submit) + rc = cli.main([ + "test", "scale_fp16", "--device", "qdc", "--timeout-min", str(bad), "--yes", + ]) + assert rc != 0 + err = capsys.readouterr().err + assert "--timeout-min" in err + assert "1..240" in err or ("1" in err and "240" in err) + + +def test_the_cli_range_is_job_pys_own_range_not_a_second_copy(): + from hexlib.device.qdc import job + + assert (job.MIN_TIMEOUT_MIN, job.MAX_TIMEOUT_MIN) == (1, 240) + + +# ============================================================================== +# --timeout-min / --yes ARE REFUSED FOR sim AND local, not silently ignored. +# They are the spend controls for the one backend that spends anything. +# ============================================================================== + + +@pytest.mark.parametrize("device", ["sim", "local"]) +@pytest.mark.parametrize( + "extra", [["--timeout-min", "20"], ["--yes"], ["--timeout-min", "20", "--yes"]] +) +def test_timeout_and_yes_are_refused_for_non_qdc_devices( + monkeypatch, capsys, device, extra +): + def boom_verify(kernel, out): + raise AssertionError("verify must not run when the flags are refused") + + monkeypatch.setattr(cli, "verify", boom_verify) + monkeypatch.setattr(cli, "_qdc_submit", _boom_submit) + rc = cli.main(["test", "kernels/scale_fp16", "--device", device] + extra) + assert rc == 2 + err = capsys.readouterr().err + assert "--device qdc" in err + for flag in ("--timeout-min", "--yes"): + if flag in extra: + assert flag in err diff --git a/hexlib/tests/test_cli_qdc_results.py b/hexlib/tests/test_cli_qdc_results.py new file mode 100644 index 0000000..a5bf5bc --- /dev/null +++ b/hexlib/tests/test_cli_qdc_results.py @@ -0,0 +1,1010 @@ +# hexlib/tests/test_cli_qdc_results.py +"""Critical 3: `hexlib._qdc_submit` must never return 0 for a job whose +results.xml reports zero tests, any failures/errors, is unparseable, is +missing entirely, or -- even when it parses clean -- whose fetched logs +never actually show the measurement lines a genuine device run prints. + +Before this fix, `_qdc_submit` returned 0 the moment `job.fetch()` finished +downloading files: nothing in the tree parsed `results.xml` at all (no +`ElementTree`, no `failures=` anywhere), so five failed on-device assertions +-- or a results.xml containing only a collection error -- produced a green +CLI. This is the project's own named failure mode (a device-farm job that +ran zero tests and reported passing), one level up from where it was fixed +in job.py's own `wait()`. + +Every test here monkeypatches `hexlib.runtime.build.build_device_binary`, +`hexlib.device.qdc.artifact.stage`, and `hexlib.device.qdc.job.submit` / +`.wait` / `.fetch` directly -- the same modules `_qdc_submit` imports +lazily -- exactly the way `hexlib/tests/test_qdc.py` monkeypatches the SDK +boundary. No credential, no network, no real device artifact anywhere in +this file. +""" +import argparse +import os + +import pytest + +from hexlib import cli +from hexlib.device.qdc import artifact, job +from hexlib.runtime import build as runtime_build + +# The exact strings hexlib_run's own run_self_test() prints on a genuine +# pass (hexlib/runtime/host/main.c) -- pinned as constants here too so a +# typo in one fabricated log can't accidentally satisfy the other. +PASS_LINE = "hexlib: --self-test: PASS (4100 values, bit-exact)" +CYCLES_LINE = "hexlib: --self-test: cycles_total=886" +GOOD_LOG = f"{PASS_LINE}\n{CYCLES_LINE}\n" + + +def _args(tmp_path, kernel="scale_fp16", timeout_min=5, yes=None): + """A REALISTIC Namespace, `kernel` included. It used to be built with no + `kernel` attribute at all and `_qdc_submit` ran fine -- which was itself + the evidence that `--device qdc` ignored the argument and would spend real + minutes measuring scale_fp16 no matter which kernel was asked for. + `_qdc_submit` now refuses an args object with no kernel on it, so leaving + it out here would fail loudly instead of passing silently. + + `yes` defaults to WHAT THIS TIMEOUT ACTUALLY REQUIRES, because + `_qdc_submit` now re-checks the confirmation threshold and the budget as + well as the kernel -- it is the function that spends the minutes, and it is + reachable without going through `_cmd_test_qdc`. Every test here means to + submit a legitimate job, so a `timeout_min` above the threshold implies + `--yes`; hard-coding `yes=False` for all of them made three tests assert on + a job the CLI would have refused at the front door. Pass `yes` explicitly + to test the guard itself. + """ + if yes is None: + yes = timeout_min > cli._QDC_YES_THRESHOLD_MIN + return argparse.Namespace( + out=str(tmp_path / "out"), timeout_min=timeout_min, yes=yes, kernel=kernel + ) + + +def _stub_build_submit_and_wait(monkeypatch): + """Stub out everything before job.fetch(): building a real device binary + needs the Hexagon SDK, staging needs real files, and submit/wait need a + real QDC account -- none of which is what this file exists to check.""" + + def fake_build_device_binary(build_dir, sdk_root=None): + os.makedirs(build_dir, exist_ok=True) + exe = os.path.join(build_dir, "hexlib_run") + open(exe, "wb").close() + open(os.path.join(build_dir, "libhexlib_skel.so"), "wb").close() + return exe + + def fake_stage(binaries, test_script, out_base, support_files=None): + zip_path = out_base + ".zip" + os.makedirs(os.path.dirname(zip_path), exist_ok=True) + open(zip_path, "wb").close() + return zip_path + + monkeypatch.setattr(runtime_build, "build_device_binary", fake_build_device_binary) + monkeypatch.setattr(artifact, "stage", fake_stage) + monkeypatch.setattr(job, "submit", lambda zip_path, *, timeout_min: 999) + monkeypatch.setattr(job, "wait", lambda job_id, **kw: True) + + +def _write_log(log_dir, rel, text): + """One fetched file at `rel`, a QDC-style relative path with forward + slashes, mirrored beneath `log_dir` the way `job.fetch` mirrors it.""" + p = os.path.join(log_dir, *rel.split("/")) + os.makedirs(os.path.dirname(p), exist_ok=True) + with open(p, "w", encoding="utf-8") as f: + f.write(text) + return p + + +def _fake_fetch(tmp_path, *, results_xml, extra_logs=None, decoy_results_xml=None): + """Write fabricated fetched log files under out/qdc_logs and return the + list of local paths -- the same shape `job.fetch`'s real return value + has (job.py's own `fetch()` returns exactly this: local paths it wrote + from QDC's log files). + + THE REPORT GOES AT `TestLogs/results.xml`, NOT AT THE TOP LEVEL. It used + to be written flat, which quietly made every test here unable to see the + defect that `job.fetch` mirrors QDC's directory layout -- so two files + named results.xml can now exist, and the reader used to pick by basename. + `job.RESULTS_MARKER` is the real remote name and is used here rather than + respelled, so the fixture cannot drift away from what `wait()` looks for. + + `decoy_results_xml` writes a SECOND results.xml at `logs/results.xml`, + which is the shape of the framework placeholder, and puts it FIRST in the + returned list -- QDC's listing order is not ours to choose. + """ + log_dir = os.path.join(str(tmp_path / "out"), "qdc_logs") + os.makedirs(log_dir, exist_ok=True) + paths = [] + if decoy_results_xml is not None: + paths.append(_write_log(log_dir, "logs/results.xml", decoy_results_xml)) + if results_xml is not None: + paths.append(_write_log(log_dir, job.RESULTS_MARKER, results_xml)) + for name, text in (extra_logs or {}).items(): + paths.append(_write_log(log_dir, name, text)) + return paths + + +def _boom_build(monkeypatch): + """Make the FIRST thing after the guards explode. `_qdc_submit`'s guards all + run before `build_device_binary`, so if a guard refuses we get its exit code + and if it does not we get this -- no test here can pass by accident because + something further down happened to fail.""" + + def boom(build_dir, sdk_root=None): + raise AssertionError( + "reached the device build: _qdc_submit accepted a job its own " + "guards should have refused" + ) + + monkeypatch.setattr(runtime_build, "build_device_binary", boom) + + +def test_qdc_submit_refuses_an_over_budget_job_on_its_own(monkeypatch, tmp_path, capsys): + """THE MONEY GUARD, CHECKED WHERE THE MONEY IS SPENT. + + `_qdc_submit` re-checked `_qdc_kernel_refusal` -- the cheapest of the + guards -- with an explicit rationale: it is the function that spends the + minutes and it is called directly, by tests today and any second caller + tomorrow. The budget comparison and the `--yes` threshold were left one + frame up in `_cmd_test_qdc`, so the rationale was written and then applied + to the one guard whose absence costs nothing. + + This is the 80x overspend `_qdc_budget_guard`'s docstring says it exists to + prevent, reached past a guard that is present: a 240-minute job under a + 3-minute recorded budget, submitted with `--yes` so the confirmation + threshold cannot be what stops it. + """ + monkeypatch.setenv(cli._QDC_BUDGET_ENV, "3") + _boom_build(monkeypatch) + rc = cli._qdc_submit(_args(tmp_path, timeout_min=240, yes=True)) + assert rc == 2 + err = capsys.readouterr().err.lower() + assert "240" in err and "budget" in err + + +def test_qdc_submit_refuses_above_the_confirmation_threshold_on_its_own( + monkeypatch, tmp_path, capsys +): + """The other half, and the half that does not depend on the operator + having recorded a budget anywhere -- so it is the guard that must hold on + every path. Budget deliberately unset: nothing but the threshold can + refuse this.""" + monkeypatch.delenv(cli._QDC_BUDGET_ENV, raising=False) + _boom_build(monkeypatch) + rc = cli._qdc_submit(_args(tmp_path, timeout_min=240, yes=False)) + assert rc == 2 + err = capsys.readouterr().err.lower() + assert "--yes" in err + + +def test_qdc_submit_refuses_a_timeout_it_cannot_compare_to_a_budget( + monkeypatch, tmp_path, capsys +): + """`_cmd_test_qdc` rejects `timeout_min=None` before `_qdc_submit` sees it. + A direct caller carries no such guarantee, and `None > budget` raises on + one path and passes on another -- neither may decide whether money is + spent, so it is refused.""" + monkeypatch.setenv(cli._QDC_BUDGET_ENV, "60") + _boom_build(monkeypatch) + rc = cli._qdc_submit(_args(tmp_path, timeout_min=None, yes=True)) + assert rc == 2 + err = capsys.readouterr().err.lower() + assert "timeout-min" in err + + +def test_qdc_submit_prints_the_budget_line_exactly_once_via_the_command( + monkeypatch, tmp_path, capsys +): + """The reason the duplicated guards use the SILENT `_refusal` forms. + + `_cmd_test_qdc` prints the recorded budget as a courtesy, then calls + `_qdc_submit`, which now repeats the comparison. Repeating + `_qdc_budget_guard` instead would print the figure twice on the normal + route and read as two different checks disagreeing about nothing. Money + guards are duplicated; the courtesy print is not. + """ + monkeypatch.setenv(cli._QDC_BUDGET_ENV, "60") + _boom_build(monkeypatch) + # Reaching the build is the SUCCESS condition here: it means both frames' + # guards passed, which is the only situation in which the line could be + # printed twice. + with pytest.raises(AssertionError, match="reached the device build"): + cli._cmd_test_qdc(_args(tmp_path, timeout_min=5)) + out = capsys.readouterr().out + assert out.count("remaining budget:") == 1, ( + f"the budget line was printed {out.count('remaining budget:')} times" + ) + + +def test_a_good_run_with_measurements_present_exits_zero(monkeypatch, tmp_path): + _stub_build_submit_and_wait(monkeypatch) + paths = _fake_fetch( + tmp_path, + results_xml='', + extra_logs={"hexlib_selftest.log": GOOD_LOG}, + ) + monkeypatch.setattr(job, "fetch", lambda job_id, dest: paths) + rc = cli._qdc_submit(_args(tmp_path)) + assert rc == 0 + + +def test_zero_tests_is_a_failure_never_a_pass(monkeypatch, tmp_path, capsys): + """THE ORIGINAL DEFECT'S OWN SHAPE: a results.xml that parses clean but + reports it ran nothing at all must not be a pass.""" + _stub_build_submit_and_wait(monkeypatch) + paths = _fake_fetch( + tmp_path, + results_xml='', + extra_logs={"hexlib_selftest.log": GOOD_LOG}, + ) + monkeypatch.setattr(job, "fetch", lambda job_id, dest: paths) + rc = cli._qdc_submit(_args(tmp_path)) + assert rc != 0 + err = capsys.readouterr().err.lower() + assert "0 test" in err + + +def test_any_failures_is_a_failure(monkeypatch, tmp_path, capsys): + _stub_build_submit_and_wait(monkeypatch) + paths = _fake_fetch( + tmp_path, + results_xml='', + extra_logs={"hexlib_selftest.log": GOOD_LOG}, + ) + monkeypatch.setattr(job, "fetch", lambda job_id, dest: paths) + rc = cli._qdc_submit(_args(tmp_path)) + assert rc != 0 + err = capsys.readouterr().err.lower() + assert "failure" in err + + +def test_any_errors_is_a_failure(monkeypatch, tmp_path, capsys): + _stub_build_submit_and_wait(monkeypatch) + paths = _fake_fetch( + tmp_path, + results_xml='', + extra_logs={"hexlib_selftest.log": GOOD_LOG}, + ) + monkeypatch.setattr(job, "fetch", lambda job_id, dest: paths) + rc = cli._qdc_submit(_args(tmp_path)) + assert rc != 0 + err = capsys.readouterr().err.lower() + assert "error" in err + + +def test_an_unparseable_results_xml_is_a_failure(monkeypatch, tmp_path, capsys): + """A truncated or non-XML results.xml -- e.g. a collection error that + never produced a real report -- must not be silently skipped.""" + _stub_build_submit_and_wait(monkeypatch) + paths = _fake_fetch( + tmp_path, + results_xml="this is not xml at all <<< not even close", + extra_logs={"hexlib_selftest.log": GOOD_LOG}, + ) + monkeypatch.setattr(job, "fetch", lambda job_id, dest: paths) + rc = cli._qdc_submit(_args(tmp_path)) + assert rc != 0 + err = capsys.readouterr().err.lower() + assert "pars" in err or "xml" in err + + +def test_missing_results_xml_entirely_is_a_failure(monkeypatch, tmp_path, capsys): + _stub_build_submit_and_wait(monkeypatch) + paths = _fake_fetch( + tmp_path, results_xml=None, extra_logs={"hexlib_selftest.log": GOOD_LOG} + ) + monkeypatch.setattr(job, "fetch", lambda job_id, dest: paths) + rc = cli._qdc_submit(_args(tmp_path)) + assert rc != 0 + err = capsys.readouterr().err.lower() + assert "results.xml" in err + + +def test_a_clean_decoy_results_xml_cannot_mask_the_real_failing_report( + monkeypatch, tmp_path, capsys +): + """THE BASENAME SELECTION, WITH BOTH FILES PRESENT AND THE DECOY FIRST. + + `898089c` made `job.fetch` mirror QDC's directory layout precisely because + `TestLogs/results.xml` and `logs/results.xml` used to overwrite each other + on disk. From that commit onwards both exist -- and the verdict reader + still did `os.path.basename(p) == "results.xml"`, taking whichever QDC + listed first. + + So: `logs/results.xml` is a framework placeholder reporting one passing + test, listed FIRST; `TestLogs/results.xml` is the real report with three + failures. The self-test log carries a genuine PASS line and a positive + `cycles_total=`, because `write_qdc_log` runs before the asserts do -- so + every other check in `_qdc_check_results` is satisfied and the ONLY thing + standing between this job and `exit 0` is which file gets parsed. + + A failed device job reported as a pass is this project's own named + Critical, and this is it on the money path. Both a nonzero exit and the + reason are asserted: passing for the wrong reason would be worth nothing + here. + """ + _stub_build_submit_and_wait(monkeypatch) + paths = _fake_fetch( + tmp_path, + results_xml='', + decoy_results_xml='', + extra_logs={"hexlib_selftest.log": GOOD_LOG}, + ) + # The decoy is first in QDC's listing order -- that ordering is the defect's + # trigger and is not ours to choose, so it is pinned rather than assumed. + assert paths[0].endswith(os.path.join("logs", "results.xml")) + monkeypatch.setattr(job, "fetch", lambda job_id, dest: paths) + + rc = cli._qdc_submit(_args(tmp_path)) + assert rc != 0, ( + "a job whose real report has 3 failures exited 0 because a clean " + "placeholder results.xml was parsed instead" + ) + err = capsys.readouterr().err.lower() + assert "results.xml" in err + + +def test_two_matching_results_files_are_refused_rather_than_guessed_between( + monkeypatch, tmp_path, capsys +): + """Ambiguity is a failure, even when BOTH candidates look clean. + + The check above passes as soon as the reader stops preferring the decoy -- + including if it simply preferred the other one. That is still a guess, and + the next layout change flips it back. So when two files match the marker, + the reader must refuse and say so, not parse either. + + Both reports here are clean, so nothing except the ambiguity itself can + make this fail. + """ + _stub_build_submit_and_wait(monkeypatch) + clean = '' + log_dir = os.path.join(str(tmp_path / "out"), "qdc_logs") + os.makedirs(log_dir, exist_ok=True) + paths = [ + _write_log(log_dir, job.RESULTS_MARKER, clean), + _write_log(log_dir, "run2/" + job.RESULTS_MARKER, clean), + _write_log(log_dir, "hexlib_selftest.log", GOOD_LOG), + ] + monkeypatch.setattr(job, "fetch", lambda job_id, dest: paths) + + rc = cli._qdc_submit(_args(tmp_path)) + assert rc != 0, ( + "two files matched the results marker and one of them was parsed " + "anyway -- a verdict read from a guess" + ) + err = capsys.readouterr().err.lower() + assert "2 files match" in err or "refusing to guess" in err + + +def test_a_clean_result_missing_the_measurement_lines_is_still_a_failure( + monkeypatch, tmp_path, capsys +): + """A results.xml that parses clean with zero failures is not enough on + its own -- if the fetched logs never actually show `cycles_total=` or + the `--self-test` PASS line, that is 'a success value constructible + with zero measurements inside it', which is exactly what this whole + check exists to make impossible.""" + _stub_build_submit_and_wait(monkeypatch) + paths = _fake_fetch( + tmp_path, + results_xml='', + extra_logs={"hexlib_selftest.log": "nothing useful in this log\n"}, + ) + monkeypatch.setattr(job, "fetch", lambda job_id, dest: paths) + rc = cli._qdc_submit(_args(tmp_path)) + assert rc != 0 + err = capsys.readouterr().err.lower() + assert "cycles_total" in err or "measurement" in err + + +def test_a_clean_result_missing_only_the_pass_line_is_still_a_failure( + monkeypatch, tmp_path, capsys +): + _stub_build_submit_and_wait(monkeypatch) + paths = _fake_fetch( + tmp_path, + results_xml='', + extra_logs={"hexlib_selftest.log": f"{CYCLES_LINE}\n"}, + ) + monkeypatch.setattr(job, "fetch", lambda job_id, dest: paths) + rc = cli._qdc_submit(_args(tmp_path)) + assert rc != 0 + + +def test_a_clean_result_missing_only_cycles_total_is_still_a_failure( + monkeypatch, tmp_path, capsys +): + _stub_build_submit_and_wait(monkeypatch) + paths = _fake_fetch( + tmp_path, + results_xml='', + extra_logs={"hexlib_selftest.log": f"{PASS_LINE}\n"}, + ) + monkeypatch.setattr(job, "fetch", lambda job_id, dest: paths) + rc = cli._qdc_submit(_args(tmp_path)) + assert rc != 0 + + +def test_a_testsuite_nested_inside_another_testsuite_is_refused_not_summed( + monkeypatch, tmp_path, capsys +): + """Critical 3's own aftermath: `.findall(".//testsuite")` sums every + `` at ANY depth, so a report with one `` nested + inside another -- whose parent's own `tests`/`failures`/`errors` + attributes already include the child's counts -- would be double-counted + if summed naively. pytest's own `--junitxml` (device/qdc/artifact.py's + pytest.ini) never produces this shape, so this must be REFUSED as an + unparseable report (never silently summed into a false pass or a + misleading count) -- a parse failure here blocks a false pass, which is + the safe direction to fail in.""" + _stub_build_submit_and_wait(monkeypatch) + xml = ( + "" + '' + '' + "" + "" + ) + paths = _fake_fetch( + tmp_path, results_xml=xml, extra_logs={"hexlib_selftest.log": GOOD_LOG} + ) + monkeypatch.setattr(job, "fetch", lambda job_id, dest: paths) + rc = cli._qdc_submit(_args(tmp_path)) + assert rc != 0 + err = capsys.readouterr().err.lower() + assert "nested" in err or "pars" in err + + +def test_testsuites_wrapper_with_multiple_suites_is_summed(monkeypatch, tmp_path): + """pytest's junit-xml can emit a root wrapping one or more + children -- the counts must be summed across all of them, + not read only off whichever element happens to be the root.""" + _stub_build_submit_and_wait(monkeypatch) + xml = ( + '' + '' + '' + "" + ) + paths = _fake_fetch( + tmp_path, results_xml=xml, extra_logs={"hexlib_selftest.log": GOOD_LOG} + ) + monkeypatch.setattr(job, "fetch", lambda job_id, dest: paths) + rc = cli._qdc_submit(_args(tmp_path)) + assert rc == 0 + + +# ============================================================================== +# Direct unit tests of `_qdc_parse_results_xml` itself -- pinning the exact +# shape it accepts (a single , or a wrapping +# children directly, matching pytest's own --junitxml output; +# see device/qdc/artifact.py's pytest.ini) and confirming it REFUSES anything +# else -- most importantly a genuinely nested +# document, which an earlier +# `.findall(".//testsuite")` would have summed and silently double-counted. +# ============================================================================== + + +def test_parse_bare_testsuite_root(tmp_path): + p = tmp_path / "results.xml" + p.write_text('') + assert cli._qdc_parse_results_xml(str(p)) == (4, 1, 0, 2) + + +def test_parse_testsuites_wrapper_sums_direct_children_only(tmp_path): + p = tmp_path / "results.xml" + p.write_text( + "" + '' + '' + "" + ) + assert cli._qdc_parse_results_xml(str(p)) == (5, 1, 1, 1) + + +def test_parse_refuses_a_testsuite_nested_inside_a_testsuite(tmp_path): + """THE DOUBLE-COUNT GUARD, tested directly against the parser (see the + end-to-end version above via `_qdc_submit`). A parent 's own + tests="5" already includes whatever its nested child's tests="2" + contributed -- summing both, as `.findall(".//testsuite")` would, reports + 7 when only 5 tests genuinely ran. This must be refused outright rather + than guessed at.""" + p = tmp_path / "results.xml" + p.write_text( + "" + '' + '' + "" + "" + ) + with pytest.raises(cli._QdcResultsError, match="nested"): + cli._qdc_parse_results_xml(str(p)) + + +def test_parse_refuses_a_testsuite_nested_directly_under_bare_testsuite_root(tmp_path): + """The same nested shape, but with the outer element as the document + root itself (no wrapper) -- must be refused the same way, + not accepted just because the root tag matched the simple case.""" + p = tmp_path / "results.xml" + p.write_text( + '' + '' + "" + ) + with pytest.raises(cli._QdcResultsError, match="nested"): + cli._qdc_parse_results_xml(str(p)) + + +def test_parse_refuses_an_unrecognized_root_tag(tmp_path): + p = tmp_path / "results.xml" + p.write_text('') + with pytest.raises(cli._QdcResultsError, match="testsuite"): + cli._qdc_parse_results_xml(str(p)) + + +# ============================================================================== +# `cycles_total=0` -- A MEASUREMENT THAT MEASURED NOTHING. +# +# The measurement-lines check was `_CYCLES_TOTAL_MARKER not in combined`, a +# pure substring test. `cycles_total=0` contains `cycles_total=`, so it +# passed: a run in which the DSP's PCYCLE counter never advanced at all +# printed "measurement lines present" and exited 0. That was reproduced +# against fabricated local logs before this fix -- a log carrying +# `hexlib: --self-test: cycles_total=0` and a clean +# `` produced +# "job 1: 5 test(s), 0 failures, 0 errors, measurement lines present" and +# return code 0. +# +# It is not a hypothetical shape. The skel reads PCYCLE inside a user-mode +# unsigned PD, where SYSCFG.PCYCLEEN cannot be set (skel_dispatch.c's +# hexlib_read_pcycle, and include/hexlib/hexlib_harness.h, which sets that bit +# explicitly for the standalone runtime because the register reads 0 without +# it). So zero is exactly what the FIRST silicon job would print if the +# counter is dead there -- the single most important thing that job can +# report, and the one thing this check used to swallow. +# ============================================================================== + +ZERO_CYCLES_LINE = "hexlib: --self-test: cycles_total=0" + + +def test_a_zero_cycle_count_is_a_failure_never_a_pass(monkeypatch, tmp_path, capsys): + """THE DEFECT ITSELF. The PASS line is present, results.xml is clean, and + `cycles_total=` is literally in the logs -- and this must still fail, + because the value is 0 and a run that measured nothing did not measure + anything.""" + _stub_build_submit_and_wait(monkeypatch) + paths = _fake_fetch( + tmp_path, + results_xml='', + extra_logs={"hexlib_selftest.log": f"{PASS_LINE}\n{ZERO_CYCLES_LINE}\n"}, + ) + monkeypatch.setattr(job, "fetch", lambda job_id, dest: paths) + rc = cli._qdc_submit(_args(tmp_path)) + assert rc != 0, ( + "cycles_total=0 satisfied the old substring check and exited 0 -- a " + "success value with a literal zero measurement inside it" + ) + err = capsys.readouterr().err.lower() + assert "cycles_total" in err + assert "0" in err + + +def test_a_malformed_cycle_count_is_a_failure(monkeypatch, tmp_path, capsys): + """A `cycles_total=` whose value is not a decimal integer at all -- a + truncated log, interleaved output, or a format change nobody updated this + for. Must fail, and must say it is malformed rather than reporting it as + absent (it is not absent) or as zero (it is not zero).""" + _stub_build_submit_and_wait(monkeypatch) + paths = _fake_fetch( + tmp_path, + results_xml='', + extra_logs={ + "hexlib_selftest.log": f"{PASS_LINE}\nhexlib: cycles_total= 0` test for the WRONG stated reason, and it must certainly not be + accepted.""" + ok, detail = cli._qdc_cycles_total_verdict("hexlib: cycles_total=-5\n") + assert ok is False + assert "integer" in detail + + +def test_cycles_verdict_malformed(): + ok, detail = cli._qdc_cycles_total_verdict("hexlib: cycles_total=abc\n") + assert ok is False + assert "integer" in detail + + +def test_cycles_verdict_positive(): + ok, detail = cli._qdc_cycles_total_verdict("hexlib: cycles_total=1287\n") + assert ok is True + assert "1287" in detail + + +def test_cycles_verdict_takes_the_largest_positive_value(): + ok, detail = cli._qdc_cycles_total_verdict( + "cycles_total=0\ncycles_total=42\ncycles_total=1287\n" + ) + assert ok is True + assert "1287" in detail + + +# ============================================================================== +# `skipped` -- COLLECTED BUT NEVER RUN IS NOT PASSED. +# +# `skipped` was parsed NOWHERE. VERIFIED before this fix: +# `` plus a good +# self-test log printed "job 999: 5 test(s), 0 failures, 0 errors, measurement +# lines present" and EXITED 0 -- five tests collected, none run, reported as a +# clean pass, with the skip count not even mentioned in the output. +# +# It is reachable the moment anyone `skipif`s the two aspirational +# discriminators in device/qdc/test_on_device.py (the unmapped-fd and the +# coherency check), and commit 6ac7c3e ("stop skipping silently") is the record +# of that being a live temptation. On device a skip cannot mean "not applicable +# here": every test in that file is unconditional, so a skip means the farm +# collected a test and then did not run it. +# ============================================================================== + + +def test_every_test_skipped_is_a_failure_never_a_pass(monkeypatch, tmp_path, capsys): + """THE DEFECT ITSELF: a report whose every test was skipped, with a + perfectly good measurement log beside it.""" + _stub_build_submit_and_wait(monkeypatch) + paths = _fake_fetch( + tmp_path, + results_xml='', + extra_logs={"hexlib_selftest.log": GOOD_LOG}, + ) + monkeypatch.setattr(job, "fetch", lambda job_id, dest: paths) + rc = cli._qdc_submit(_args(tmp_path)) + assert rc != 0, ( + "5 collected, 5 skipped, 0 run exited 0 before this fix -- a skip is " + "not a pass, and on device it means the test could not run at all" + ) + err = capsys.readouterr().err.lower() + assert "skip" in err + assert "5" in err + + +def test_even_one_skipped_test_is_a_failure(monkeypatch, tmp_path, capsys): + """Not a threshold. One test silently not running is one discriminator + silently not applied.""" + _stub_build_submit_and_wait(monkeypatch) + paths = _fake_fetch( + tmp_path, + results_xml='', + extra_logs={"hexlib_selftest.log": GOOD_LOG}, + ) + monkeypatch.setattr(job, "fetch", lambda job_id, dest: paths) + rc = cli._qdc_submit(_args(tmp_path)) + assert rc != 0 + assert "skip" in capsys.readouterr().err.lower() + + +def test_the_skip_count_is_reported_on_the_pass_line_too(monkeypatch, tmp_path, capsys): + """A success line that omits a count it checked leaves a reader unable to + tell "0 skipped" from "skips were never looked at" -- which is exactly the + state this line was in.""" + _stub_build_submit_and_wait(monkeypatch) + paths = _fake_fetch( + tmp_path, + results_xml='', + extra_logs={"hexlib_selftest.log": GOOD_LOG}, + ) + monkeypatch.setattr(job, "fetch", lambda job_id, dest: paths) + assert cli._qdc_submit(_args(tmp_path)) == 0 + assert "0 skipped" in capsys.readouterr().out + + +# ============================================================================== +# A MISSING COUNT ATTRIBUTE IS A MALFORMED REPORT, NEVER A ZERO. +# +# `suite.get("failures", "0")` / `suite.get("errors", "0")` defaulted the two +# attributes that decide the verdict. VERIFIED before this fix: +# `` plus a good log EXITED 0 and printed +# "5 test(s), 0 failures, 0 errors". That directly contradicted +# `_QdcResultsError`'s own docstring, which says the exception exists for a +# report "missing the attributes a JUnit report always carries". +# ============================================================================== + + +@pytest.mark.parametrize("xml", [ + '', + '', # no failures + '', # no errors + '', # no skipped + '', # no tests +]) +def test_a_results_xml_missing_any_count_attribute_is_a_failure( + monkeypatch, tmp_path, capsys, xml +): + _stub_build_submit_and_wait(monkeypatch) + paths = _fake_fetch( + tmp_path, results_xml=xml, extra_logs={"hexlib_selftest.log": GOOD_LOG} + ) + monkeypatch.setattr(job, "fetch", lambda job_id, dest: paths) + rc = cli._qdc_submit(_args(tmp_path)) + assert rc != 0, f"{xml} must not be read as a clean report" + assert "attribute" in capsys.readouterr().err.lower() + + +@pytest.mark.parametrize("attr", ["tests", "failures", "errors", "skipped"]) +def test_parse_refuses_a_suite_missing_any_required_attribute(tmp_path, attr): + attrs = {"tests": "5", "failures": "0", "errors": "0", "skipped": "0"} + del attrs[attr] + body = " ".join(f'{k}="{v}"' for k, v in attrs.items()) + p = tmp_path / "results.xml" + p.write_text(f"") + with pytest.raises(cli._QdcResultsError, match=attr): + cli._qdc_parse_results_xml(str(p)) + + +def test_parse_refuses_a_missing_attribute_on_a_later_suite_too(tmp_path): + """Summing across a wrapper must not let a well-formed first + suite cover for a malformed second one.""" + p = tmp_path / "results.xml" + p.write_text( + "" + '' + '' + "" + ) + with pytest.raises(cli._QdcResultsError, match="failures"): + cli._qdc_parse_results_xml(str(p)) + + +# ============================================================================== +# NEGATIVE AND ZERO COUNTS. +# +# `if tests == 0` was the whole bound. VERIFIED before this fix: +# `` plus a good log EXITED 0 and +# printed "-1 test(s)". +# ============================================================================== + + +@pytest.mark.parametrize("xml", [ + '', + '', + '', + '', +]) +def test_a_negative_count_anywhere_is_a_failure(monkeypatch, tmp_path, capsys, xml): + _stub_build_submit_and_wait(monkeypatch) + paths = _fake_fetch( + tmp_path, results_xml=xml, extra_logs={"hexlib_selftest.log": GOOD_LOG} + ) + monkeypatch.setattr(job, "fetch", lambda job_id, dest: paths) + assert cli._qdc_submit(_args(tmp_path)) != 0, f"{xml} must not read as a pass" + + +def test_parse_refuses_a_negative_count(tmp_path): + p = tmp_path / "results.xml" + p.write_text('') + with pytest.raises(cli._QdcResultsError, match="NEGATIVE"): + cli._qdc_parse_results_xml(str(p)) + + +# ============================================================================== +# THE PARSER STRICTNESS IS CHECKED AGAINST A REPORT PYTEST ACTUALLY WROTE. +# +# Requiring all four attributes is only safe if the producer really emits all +# four. Rather than assert that from memory, this runs pytest's own --junitxml +# and reads the result back: if a future pytest stops emitting one of them, +# this fails HERE (with a report in hand) instead of the device gate refusing +# every genuine results.xml after the minutes are spent. +# ============================================================================== + + +def test_a_real_pytest_junitxml_carries_all_four_counts_and_parses(tmp_path): + import subprocess + import sys + import xml.etree.ElementTree as ET + + probe = tmp_path / "test_probe.py" + probe.write_text( + "import pytest\n" + "def test_pass(): pass\n" + '@pytest.mark.skip(reason="probe")\n' + "def test_skipped(): pass\n" + ) + xml_path = tmp_path / "results.xml" + subprocess.run( + [sys.executable, "-m", "pytest", "-q", "-p", "no:cacheprovider", + f"--junitxml={xml_path}", str(probe)], + cwd=str(tmp_path), capture_output=True, text=True, + ) + assert xml_path.is_file(), "pytest wrote no junitxml at all" + + root = ET.parse(str(xml_path)).getroot() + suite = root if root.tag == "testsuite" else root.find("testsuite") + for attr in cli._REQUIRED_SUITE_ATTRS: + assert suite.get(attr) is not None, ( + f"pytest's own --junitxml did not emit {attr!r} -- cli.py requires " + "it, so this is the check that must fail, not the device gate " + "after the minutes are spent" + ) + + counts = cli._qdc_parse_results_xml(str(xml_path)) + assert counts.tests == 2 + assert counts.skipped == 1 + assert (counts.failures, counts.errors) == (0, 0) + + +# ============================================================================== +# THE WAIT CAP COMES FROM THE JOB'S OWN TIMEOUT, AND A TIMEOUT FETCHES LOGS. +# +# `job.wait(job_id)` was called with no cap_s, taking job.py's 1800 s default +# while `submit()` accepts 240 minutes. Scenario: `--timeout-min 60` on a job +# that legitimately finishes at 35 minutes -- at 30 minutes the CLI printed +# "produced no results.xml within the wait cap", exited 1, and downloaded ZERO +# log files. The minutes were spent, the results.xml that appeared five minutes +# later was never fetched, and the operator had nothing to diagnose from. +# ============================================================================== + + +def test_the_wait_cap_is_derived_from_the_jobs_own_timeout(monkeypatch, tmp_path): + seen = {} + + def recording_wait(job_id, **kw): + seen.update(kw) + return True + + _stub_build_submit_and_wait(monkeypatch) + monkeypatch.setattr(job, "wait", recording_wait) + paths = _fake_fetch( + tmp_path, + results_xml='', + extra_logs={"hexlib_selftest.log": GOOD_LOG}, + ) + monkeypatch.setattr(job, "fetch", lambda job_id, dest: paths) + assert cli._qdc_submit(_args(tmp_path, timeout_min=60)) == 0 + assert seen.get("cap_s", 0) >= 60 * 60, ( + "the wait cap must be at least the job's own timeout -- 1800s against " + f"a 60-minute job abandons it half way; got {seen}" + ) + + +def test_the_wait_cap_covers_the_largest_timeout_submit_accepts(): + from hexlib.device.qdc import job as jobmod + + assert cli._qdc_wait_cap_s(jobmod.MAX_TIMEOUT_MIN) > jobmod.MAX_TIMEOUT_MIN * 60 + assert cli._qdc_wait_cap_s(1) > 60 + + +def test_a_wait_timeout_still_fetches_whatever_logs_exist(monkeypatch, tmp_path, capsys): + """A timeout that discards the evidence is worse than one that waits. The + minutes are already spent; the partial logs are all the operator has.""" + _stub_build_submit_and_wait(monkeypatch) + monkeypatch.setattr(job, "wait", lambda job_id, **kw: False) + fetched = [] + paths = _fake_fetch(tmp_path, results_xml=None, + extra_logs={"logcat.txt": "some device noise\n"}) + + def recording_fetch(job_id, dest): + fetched.append(dest) + return paths + + monkeypatch.setattr(job, "fetch", recording_fetch) + rc = cli._qdc_submit(_args(tmp_path, timeout_min=60)) + assert rc != 0, "no results.xml within the cap is a failure, never a pass" + assert fetched, ( + "the timeout branch downloaded ZERO log files before giving up -- the " + "minutes are spent and this is the only evidence there is" + ) + err = capsys.readouterr().err + assert "results.xml" in err + assert "1 log file" in err + + +def test_a_wait_timeout_whose_log_fetch_also_fails_still_fails_cleanly( + monkeypatch, tmp_path, capsys +): + """Best-effort means best-effort: the fetch blowing up on the giving-up + path must not turn an exit-1 into a traceback.""" + _stub_build_submit_and_wait(monkeypatch) + monkeypatch.setattr(job, "wait", lambda job_id, **kw: False) + + def boom_fetch(job_id, dest): + raise job.QdcError("QDC refused the log listing") + + monkeypatch.setattr(job, "fetch", boom_fetch) + rc = cli._qdc_submit(_args(tmp_path, timeout_min=20)) + assert rc == 1 + err = capsys.readouterr().err + assert "results.xml" in err + assert "QDC refused the log listing" in err + + +def test_a_fetch_failure_on_the_happy_path_is_a_failure_never_a_pass( + monkeypatch, tmp_path, capsys +): + _stub_build_submit_and_wait(monkeypatch) + + def boom_fetch(job_id, dest): + raise job.QdcError("connection reset while downloading") + + monkeypatch.setattr(job, "fetch", boom_fetch) + rc = cli._qdc_submit(_args(tmp_path)) + assert rc != 0 + assert "fetch" in capsys.readouterr().err.lower() + + +# ============================================================================== +# `_qdc_submit` READS args.kernel. It is the function that spends the minutes, +# and it is reachable without going through `_cmd_test_qdc`'s guards at all -- +# this file's own `_args` used to prove that by omitting `kernel` entirely. +# ============================================================================== + + +@pytest.mark.parametrize("kernel", ["add_fp16", "kernels/rmsnorm_fp16", "", None]) +def test_qdc_submit_refuses_a_kernel_stage_three_cannot_run( + monkeypatch, tmp_path, capsys, kernel +): + def boom_build(build_dir, sdk_root=None): + raise AssertionError("the device build must not start for a bad kernel") + + monkeypatch.setattr(runtime_build, "build_device_binary", boom_build) + rc = cli._qdc_submit(_args(tmp_path, kernel=kernel)) + assert rc == 2 + assert "scale_fp16" in capsys.readouterr().err + + +def test_qdc_submit_refuses_an_args_object_with_no_kernel_attribute( + monkeypatch, tmp_path, capsys +): + """The exact shape this file used to pass in. A missing attribute is a + refusal, not a default.""" + def boom_build(build_dir, sdk_root=None): + raise AssertionError("the device build must not start for a bad kernel") + + monkeypatch.setattr(runtime_build, "build_device_binary", boom_build) + args = argparse.Namespace(out=str(tmp_path / "out"), timeout_min=5, yes=False) + assert cli._qdc_submit(args) == 2 + assert "scale_fp16" in capsys.readouterr().err diff --git a/hexlib/tests/test_coherency_lane_classification.py b/hexlib/tests/test_coherency_lane_classification.py new file mode 100644 index 0000000..ea6589e --- /dev/null +++ b/hexlib/tests/test_coherency_lane_classification.py @@ -0,0 +1,290 @@ +# hexlib/tests/test_coherency_lane_classification.py +"""BEHAVIOURAL test for the -0.0 / sentinel classification in main.c's +--coherency-check (hexlib_classify_coherency_lane()). + +WHY A SOURCE ASSERTION IS NOT ENOUGH -- THE SAME DEFECT CLASS AS THE ARCH +DECODE. test_host_source.py can only check that a sign-bit mask (`& 0x7FFF`) +and a bit-exact sentinel compare (`bits == sentinel_bits`) EXIST in +hexlib_classify_coherency_lane() -- it cannot see whether they actually +classify -0.0 (bit pattern 0x8000) as "the kernel wrote zero", which is +exactly the case that produced this project's real false coherency-miss +report: `x * 0.0f` is -0.0 for every negative lane of the self-test's own +input, and a bit-exact compare against `(__fp16) 0.0f` (0x0000) read that +HEALTHY result as "sentinel unchanged". A test that only checks "a magnitude +check and a sentinel check exist" cannot catch a magnitude check that is +subtly wrong for the one input that matters; a test that compiles and RUNS +the actual classification against 0x8000 can. This is the identical shape to +test_session_arch_decode.py's own reason for existing -- see that file's +module docstring -- so this one follows the same recipe. + +WHAT THIS DOES. `hexlib_classify_coherency_lane()` -- and the +`enum hexlib_coherency_lane` it returns -- are extracted straight out of +main.c with `csource.function_body`/`csource.block_from`, the SAME +comment-aware slicers every other source-assertion test in this project +uses, dropped into a tiny standalone .c file next to a one-line harness that +exports it under a stable name, compiled with a host C compiler into a +shared library, and called through ctypes with the bit patterns that matter: +0x0000 (+0.0), 0x8000 (-0.0, the exact value that broke this check for +real), an arbitrary sentinel pattern, and a value that is neither. + +OVER RAW uint16_t BITS, NEVER __fp16 -- BOTH BECAUSE THAT IS WHAT THE REAL +FUNCTION NOW TAKES, AND BECAUSE THE HOST COMPILER HERE HAS NO __fp16 AT ALL. +`__fp16` is an ARM/AArch64 storage type; on this project's Windows dev +machine, the only host C compiler on PATH is a plain x86_64 mingw gcc, which +rejects `__fp16` outright (confirmed: `unknown type name '__fp16'`) even +though the real device build (NDK clang, aarch64) accepts it without issue. +Rather than skip this gap because of that mismatch, main.c's +hexlib_classify_coherency_lane() was written to operate on raw uint16_t bit +patterns in the first place -- see its own header comment in main.c for why +that is bit-for-bit equivalent to the fabsf()-based check it replaced, not a +weaker stand-in for it. That is what makes this test possible on any host +compiler at all, never a reason to water down what it checks. + +THE DECISIVE PROPERTY. If hexlib_classify_coherency_lane() is ever removed +or renamed, `_function_body` raises before this test ever gets to compile or +call anything. If it is present but its zero check regresses to a bit-exact +compare against +0.0 alone (the original bug, reintroduced), the compiled +call on 0x8000 returns SENTINEL or OTHER, never ZERO -- +`test_negative_zero_bits_classify_as_the_expected_zero_result` below fails. +`test_mutation_verify_the_original_bit_exact_compare_misclassifies_negative_zero` +goes one step further and proves this directly: it takes the SAME extracted +source, mechanically reverts the mask to the original `bits == 0x0000` +compare, compiles THAT, and confirms 0x8000 is misclassified under it -- +concrete, run evidence that this test suite would have caught the original +defect, not just an assertion that it currently doesn't reproduce it. + +THE COMPILER-INDEPENDENT GUARD BELOW WAS ONCE FOOLABLE BY A COMMENT. Same +history as test_session_arch_decode.py's -- see that file's docstring. +`csource.function_body` used to return the raw, comment-BEARING body, so +reverting hexlib_classify_coherency_lane() to `bits == 0x0000u` and dropping +HEXLIB_LANE_OTHER, with the old code left in a comment inside the body, passed +`test_the_classification_function_the_behavioural_test_depends_on_still_exists` +-- the one test in this file that runs when there is no host `cc`, and +therefore the only guard at all on such a machine. `function_body` now returns +comment-blanked text; that exact revert now fails it. Verified by mutation. +""" +import ctypes +import pathlib +import re +import shutil +import subprocess +import sys + +import pytest + +from hexlib.tests.csource import block_from as _block_from +from hexlib.tests.csource import function_body as _function_body + +MAIN_C = pathlib.Path("hexlib/runtime/host/main.c") + +HOST_CC = shutil.which("cc") or shutil.which("gcc") or shutil.which("clang") +needs_cc = pytest.mark.skipif( + HOST_CC is None, + reason=( + "no host C compiler found (tried: gcc, cc, clang); the BEHAVIOURAL " + "-0.0/sentinel coherency-classification test is skipped and only " + "the weaker source assertion in test_host_source.py " + "(test_coherency_check_treats_negative_zero_as_the_expected_zero_" + "result / test_coherency_check_verifies_the_surviving_bytes_are_" + "really_the_sentinel) covers this. Install a host C compiler to " + "restore it." + ), +) + + +@pytest.fixture(scope="module") +def main_source(): + return MAIN_C.read_text() + + +def test_the_classification_function_the_behavioural_test_depends_on_still_exists( + main_source, +): + """COMPILER-INDEPENDENT -- runs even when needs_cc above would skip + everything else, so a rename or removal of the function this file's + behavioural tests extract is never invisible on a machine with no host C + compiler. Pairs the exact NAME the fixtures below extract + (hexlib_classify_coherency_lane, enum hexlib_coherency_lane) with what + main.c actually contains, so the behavioural test and the source it + depends on cannot silently drift apart -- see this module's own + docstring and test_session_arch_decode.py's identical guard for the arch + decode.""" + body = _function_body(main_source, "hexlib_classify_coherency_lane") + assert "HEXLIB_LANE_ZERO" in body + assert "HEXLIB_LANE_SENTINEL" in body + assert "HEXLIB_LANE_OTHER" in body + enum_start = main_source.index("enum hexlib_coherency_lane {") + enum_block = _block_from(main_source, enum_start) + assert "HEXLIB_LANE_ZERO" in enum_block + assert "HEXLIB_LANE_SENTINEL" in enum_block + assert "HEXLIB_LANE_OTHER" in enum_block + + +@pytest.fixture(scope="module") +def classify_fn_source(main_source): + """The real hexlib_classify_coherency_lane() BODY, sliced straight out + of main.c and never retyped here -- see test_session_arch_decode.py's + identical fixture for the same rationale.""" + return _function_body(main_source, "hexlib_classify_coherency_lane") + + +@pytest.fixture(scope="module") +def enum_def(main_source): + """The real `enum hexlib_coherency_lane { ... }` definition, sliced out + with `csource.block_from` (comment-aware, same as `function_body`) so the + standalone harness below returns the SAME symbolic values main.c does, + never hand-retyped ones that could silently drift from a renumbering.""" + start = main_source.index("enum hexlib_coherency_lane {") + block = _block_from(main_source, start) + return "enum hexlib_coherency_lane " + block + ";" + + +def _enum_value(enum_source, name): + m = re.search(rf"\b{re.escape(name)}\s*=\s*(\d+)", enum_source) + assert m, f"could not find {name} in the extracted enum definition" + return int(m.group(1)) + + +def _compile_harness(tmp_path, name, enum_source, fn_body): + """Wrap `fn_body` (the extracted or mutated function body) in the real + enum definition and a stable-named uint16_t-in/int-out harness, compile + it into a shared library with the host C compiler, and return a ctypes + callable. Shared by both the real-function test below and the + mutation-verification test -- so both go through the exact same + compile-and-call path, and only the function body under test differs.""" + c_path = tmp_path / f"{name}.c" + c_path.write_text( + "#include \n" + f"{enum_source}\n" + "static enum hexlib_coherency_lane\n" + "hexlib_classify_coherency_lane(uint16_t bits, uint16_t sentinel_bits)\n" + f"{fn_body}\n" + "#if defined(_WIN32)\n" + "__declspec(dllexport)\n" + "#endif\n" + "int harness_classify(uint16_t bits, uint16_t sentinel_bits) {\n" + " return (int) hexlib_classify_coherency_lane(bits, sentinel_bits);\n" + "}\n" + ) + lib_path = tmp_path / (f"{name}.dll" if sys.platform == "win32" else f"{name}.so") + cmd = [HOST_CC, "-shared", "-fPIC", "-o", str(lib_path), str(c_path)] + result = subprocess.run(cmd, capture_output=True, text=True) + assert result.returncode == 0, ( + f"compiling {name}.c (extracted hexlib_classify_coherency_lane) " + f"failed:\n{result.stdout}\n{result.stderr}" + ) + lib = ctypes.CDLL(str(lib_path)) + lib.harness_classify.restype = ctypes.c_int + lib.harness_classify.argtypes = [ctypes.c_uint16, ctypes.c_uint16] + return lib.harness_classify + + +@pytest.fixture +def compiled_classify(classify_fn_source, enum_def, tmp_path): + return _compile_harness(tmp_path, "classify", enum_def, classify_fn_source) + + +@pytest.fixture(scope="module") +def lane_values(enum_def): + """The real ZERO/SENTINEL/OTHER integer values, read out of the + extracted enum text itself -- never hand-typed as 0/1/2, so a + renumbering in main.c is reflected here automatically instead of + silently comparing against stale constants.""" + return { + "ZERO": _enum_value(enum_def, "HEXLIB_LANE_ZERO"), + "SENTINEL": _enum_value(enum_def, "HEXLIB_LANE_SENTINEL"), + "OTHER": _enum_value(enum_def, "HEXLIB_LANE_OTHER"), + } + + +# The four bit patterns this test drives the classifier with. 0x3C00 is fp16 +# 1.0 (sign 0, exponent 01111, mantissa 0) -- used as an arbitrary, clearly +# nonzero sentinel pattern, matching main.c's own COHERENCY_SENTINEL (1.0f). +# 0x4000 is fp16 2.0 -- nonzero, and not equal to the sentinel used here, +# i.e. neither classification. +_SENTINEL_BITS = 0x3C00 +_POS_ZERO_BITS = 0x0000 +_NEG_ZERO_BITS = 0x8000 # THE case: `x * 0.0f` for negative x. +_OTHER_BITS = 0x4000 + + +@needs_cc +def test_negative_zero_bits_classify_as_the_expected_zero_result( + compiled_classify, lane_values +): + """THE DECISIVE CASE. -0.0 (0x8000, exactly what `x * 0.0f` produces for + every negative lane of the self-test's own input) must classify as ZERO + -- "the kernel wrote its result" -- not as SENTINEL ("the write never + reached the host") and not as OTHER. This is the exact input that + produced this project's real false coherency-miss report; see main.c's + own header comment on run_coherency_check() and COHERENCY_FACTOR.""" + assert compiled_classify(_NEG_ZERO_BITS, _SENTINEL_BITS) == lane_values["ZERO"], ( + "-0.0 (0x8000) must classify as ZERO -- a bit-exact compare against " + "+0.0 alone would misclassify this as SENTINEL and report a " + "coherency miss that never happened" + ) + + +@needs_cc +def test_positive_zero_bits_classify_as_zero(compiled_classify, lane_values): + assert compiled_classify(_POS_ZERO_BITS, _SENTINEL_BITS) == lane_values["ZERO"] + + +@needs_cc +def test_the_sentinel_pattern_classifies_as_not_written(compiled_classify, lane_values): + """A lane that is bit-exact the sentinel that was written before invoke + must classify as SENTINEL -- "the write never reached the host" (or the + kernel never ran; cycles_total is what tells those two apart, not this + function).""" + assert ( + compiled_classify(_SENTINEL_BITS, _SENTINEL_BITS) == lane_values["SENTINEL"] + ) + + +@needs_cc +def test_a_value_that_is_neither_zero_nor_sentinel_is_the_third_outcome( + compiled_classify, lane_values +): + """A garbled or partially-written lane -- neither the expected zero + result nor the intact sentinel -- must be its own, third outcome, never + folded into a coherency-miss (SENTINEL) or a clean-pass (ZERO) claim.""" + result = compiled_classify(_OTHER_BITS, _SENTINEL_BITS) + assert result == lane_values["OTHER"] + assert result != lane_values["ZERO"] + assert result != lane_values["SENTINEL"] + + +@needs_cc +def test_mutation_verify_the_original_bit_exact_compare_misclassifies_negative_zero( + classify_fn_source, enum_def, lane_values, tmp_path +): + """MUTATION-VERIFY. Takes the SAME extracted function body and + mechanically reverts the sign-bit mask to the ORIGINAL, buggy bit-exact + compare against +0.0 this project shipped once (`bits == 0x0000` in + place of `(bits & 0x7FFFu) == 0`), compiles THAT, and confirms -0.0 + (0x8000) is misclassified under it -- concrete, run evidence that the + tests above would have caught the original defect, not merely an + assertion that they currently don't reproduce it.""" + target = "(uint16_t) (bits & 0x7FFFu) == 0" + assert target in classify_fn_source, ( + "mutation target text not found in the extracted function body -- " + "did the real sign-bit-mask check change shape? update this " + "mutation to match, don't just delete it" + ) + mutated = classify_fn_source.replace(target, "bits == 0x0000u") + assert mutated != classify_fn_source + + buggy = _compile_harness(tmp_path, "classify_buggy", enum_def, mutated) + result = buggy(_NEG_ZERO_BITS, _SENTINEL_BITS) + assert result != lane_values["ZERO"], ( + "MUTATION CHECK FAILED TO REPRODUCE THE ORIGINAL BUG: reverting to " + "a bit-exact compare against +0.0 should misclassify -0.0 as NOT " + "zero (the exact false coherency-miss this project shipped once), " + "but the mutated function still returned ZERO -- something about " + "this mutation no longer matches the real defect shape" + ) + # And confirm the FIXED function (compiled the ordinary way) does not + # share that failure -- the mutation is a genuine regression relative to + # the real, current source, not a mutation of something already broken. + fixed = _compile_harness(tmp_path, "classify_fixed_for_mutation_check", enum_def, classify_fn_source) + assert fixed(_NEG_ZERO_BITS, _SENTINEL_BITS) == lane_values["ZERO"] diff --git a/hexlib/tests/test_csource.py b/hexlib/tests/test_csource.py new file mode 100644 index 0000000..03bcd47 --- /dev/null +++ b/hexlib/tests/test_csource.py @@ -0,0 +1,555 @@ +# hexlib/tests/test_csource.py +"""Tests for the shared comment-aware C source slicer (csource.py) itself. + +THE PROPERTY BEING PROVEN: a comment that mentions a function's (or a call's) +name in prose, or that merely contains a stray brace/paren, must not be able +to move any of `function_body`/`block_from`/`block_after_call`'s boundaries +off the real code. Each test below is built so that the OLD, comment-blind +regex (`name\\s*\\([^;{]*\\)\\s*\\{`, matched and brace-counted directly +against the raw source) provably gets it wrong on that exact input -- either +landing on the wrong block, or raising "unbalanced braces" outright -- which +is the failure mode this module exists to close. See the task report for the +manual before/after run that confirms this (reverting `csource.py` to the +comment-blind implementation makes these fail). + +THE SECOND PROPERTY, ADDED AFTER A MUTATION FOUND THE GAP: the text these +slicers RETURN must be comment-blanked too, not just their boundaries +comment-aware. The original implementation returned a slice of the raw, +comment-bearing source, so any payload check a consumer ran against it +(`x in body`, `re.search(..., body)`) could be satisfied by a comment INSIDE +the extracted body -- which is exactly how a proven mutation of skel_bufs.c +kept test_skel_bufs_source.py at 8 passed while deleting the branch's central +invariant. `test_*_returns_comment_blanked_text*` below pin that half: each +one fails against the old, non-stripping implementation. + +THE THIRD PROPERTY, AND WHY THIS FILE'S OWN FIRST TEST USED TO ENCODE THE BUG. +Everything above was about COMMENTS, and this file had ZERO coverage of string +literals -- all twelve tests used comments only. Worse, its first test asserted +`'"not a // comment or /* one */ either"' in stripped`: it pinned literal +PRESERVATION as the property to protect. Preservation is right for one narrow +purpose (a literal must be recognized and consumed as one token, so a `//` +inside it is not misread as a comment start) and wrong for the two purposes +that actually matter to every consumer -- brace counting, and the payload text +handed back. A merge-gate review showed literals were the comment hole in a +different vehicle, in three distinct mechanisms, each reproduced against the +then-current code and each pinned by its own test below: + + (A) a positive check satisfied by a FARF/printf format string while the real + code is deleted -- `test_*_blanks_a_log_lines_payload*`; + (B) a `}` inside a literal truncating a brace-depth slice, so every negative + check after it is answered by a fragment -- `test_*_not_truncated_by_a_ + closing_brace_in_a_literal`; + (C) a `{` inside a literal extending a block past its real end, destroying + the scoping the whole module exists to provide -- `test_block_from_is_ + not_extended_by_an_opening_brace_in_a_literal`. + +Each of those fails against the literal-preserving implementation. The narrow +purpose preservation was right for is pinned separately, in +`test_a_comment_lookalike_inside_a_literal_is_not_a_comment_start`: the +literal's PAYLOAD is blanked, and the code after it still survives, which is +only possible if the `//` inside it was never treated as a comment start. +""" +import pathlib +import re + +from hexlib.tests.csource import ( + block_after_call, + block_from, + code_only, + code_only_keeping_strings, + function_body, +) + + +def test_code_only_blanks_comments_preserving_length(): + src = ( + '/* block\n comment */int x = 1; // trailing\n' + 'int y = 2;\n' + ) + stripped = code_only(src) + assert len(stripped) == len(src) + assert "block" not in stripped + assert "trailing" not in stripped + assert "int x = 1;" in stripped + assert "int y = 2;" in stripped + + +def test_a_comment_lookalike_inside_a_literal_is_not_a_comment_start(): + """THE NARROW PROPERTY LITERAL PRESERVATION WAS ACTUALLY FOR, kept, while + the payload is blanked. This is what this file's first test used to get + backwards: it asserted the whole literal SURVIVED. + + A `//` inside a string must not be mistaken for a comment start -- if it + were, everything after it on that line (here, the statement terminator and + the following line's code) would be blanked away too. So the literal has to + be recognized and consumed as ONE token. What is written back for that + token is a separate question, and the answer is: delimiters kept, interior + blanked. Both halves are asserted here, and the second half fails against + the implementation that returned literals untouched.""" + src = ( + 'const char *s = "not a // comment or /* one */ either";\n' + 'int after = 1;\n' + ) + out = code_only(src) + assert len(out) == len(src) + # Consumed as one token: the code after the literal is still there, which + # could not be true if the `//` inside it had started a comment. + assert "int after = 1;" in out + assert 'const char *s =' in out + # ... and it is still visibly a literal, so "a string is present here" is + # still answerable -- only its contents are gone. + assert out.count('"') == 2 + # The payload, on the other hand, must not be readable as code. + assert "comment" not in out + assert "either" not in out + + +def test_code_only_keeping_strings_is_the_deliberate_escape_hatch(): + """The escape hatch really does return the literal, for the few checks + whose subject IS literal text -- and it still blanks comments, so it is + never a way back to raw source.""" + src = '/* a note */ printf("PASS (%d values)"); // trailing\n' + out = code_only_keeping_strings(src) + assert len(out) == len(src) + assert '"PASS (%d values)"' in out + assert "a note" not in out + assert "trailing" not in out + # And the two views are interchangeable in either order, so a fixture built + # with one can be re-blanked by a slicer using the other. + assert code_only(out) == code_only(src) + assert code_only_keeping_strings(code_only(src)) == code_only(src) + + +def test_an_include_header_name_is_not_a_string_literal_and_survives(): + """`#include "HAP_perf.h"` is a header-name token, not a string literal: + no escapes, no concatenation, and it cannot be used to hide code because it + has to name a file that exists for the translation unit to compile. So it + survives `code_only`, and test_skel_dispatch_source.py's check that the SDK + header is really INCLUDED (rather than named in a comment) needs no escape + hatch.""" + src = ( + '#include "HAP_perf.h"\n' + "#include \n" + ' # include "skel_internal.h"\n' + 'const char *s = "HAP_perf.h";\n' + ) + out = code_only(src) + assert len(out) == len(src) + assert '#include "HAP_perf.h"' in out + assert "#include " in out + assert '# include "skel_internal.h"' in out + # But a plain literal that merely SPELLS a header name is still blanked -- + # the exemption is for the directive, not for the text. + assert out.count("HAP_perf.h") == 1 + + +def test_function_body_is_not_derailed_by_a_comment_naming_it_first(): + """A comment mentions `foo()` (with empty, immediately-closed parens) in + prose BEFORE foo's real definition, and an unrelated function `bar`'s + signature+body sits between the comment and foo's real definition, with + only whitespace between `bar`'s own closing `)` and its opening `{`. + + Under the old comment-blind regex, greedily consuming `[^;{]*` from the + comment's "foo(" runs straight through the rest of the comment and stops + at the first `{` it hits at all -- which is `bar`'s, not foo's -- then + backtracks onto `bar(void)`'s `)` immediately followed by `{` + (`bar(void) {`), matching THERE instead: the slicer would then return + `bar`'s body (`{ return 1; }`) for a query about `foo`, silently.""" + src = ( + "/* note: foo() should always return 0 -- see below */\n" + "int bar(void) { return 1; }\n" + "\n" + "int foo(void) {\n" + " return 0;\n" + "}\n" + ) + body = function_body(src, "foo") + assert "return 0;" in body + assert "return 1;" not in body + + +def test_function_body_is_not_derailed_by_a_comment_mentioning_a_sibling(): + """The reverse shape of the bug that actually broke two tests in + test_host_source.py: a comment INSIDE one real function mentions a + DIFFERENT real function's name in prose, ahead of that other function's + own definition (this is the same shape as driver.c's HEXLIB_DLSYM + macro-comment relative to hexlib_drv_init -- that file dodges it only by + luck, via a backslash line-continuation the comment-blind regex cannot + step over; this case has no such luck: `if (1) {` sits between the + mention and the nearest disallowed character with nothing but whitespace + in between, so the old regex has a reachable, wrong `{` to seize on).""" + src = ( + "int helper(void) {\n" + " /* eventually calls target() to finish up */\n" + " if (1) { return 1; }\n" + "}\n" + "\n" + "int target(void) {\n" + " return 42;\n" + "}\n" + ) + body = function_body(src, "target") + assert "return 42;" in body + assert "return 1;" not in body + + +def test_block_from_is_not_derailed_by_a_stray_brace_in_a_comment(): + """Between `pos` (right after the `if`'s condition) and the real guarded + block, a comment contains a brace character that has nothing to do with + real control flow. The old comment-blind `text.index("{", pos)` would + seize on THAT brace as if it opened the guarded block, then either + mis-slice or -- as here, because a second, real `{` still follows before + the matching `}` -- raise "unbalanced braces" outright.""" + text = ( + "if (x) /* a comment with a stray { brace in it */ {\n" + " return 1;\n" + "}\n" + ) + pos = text.index(")") + block = block_from(text, pos) + assert block.strip().startswith("{") + assert "return 1;" in block + # exactly one open/close pair in the returned slice -- the real block, + # not a fragment that swallowed the comment's own stray brace as an + # extra nesting level + assert block.count("{") == 1 and block.count("}") == 1 + + +def test_block_after_call_is_not_derailed_by_a_comment_between_call_and_guard(): + """A comment between the call and its `if`-guard mentions a DIFFERENT + call's name and contains its own parenthesized text -- shaped so the old + comment-blind implementation's own paren-walk (which starts scanning + right at the real call's own `(`, not at the comment) is not directly + fooled, but the `if (` window-check and the block-brace search underneath + it are exercised against live-looking but decorative comment text, which + must not change what block gets returned.""" + body = ( + "int rc = real_call(a, b);\n" + "/* note: other_call() is unrelated and has its own { guard } shape */\n" + "if (rc != 0) {\n" + " return HEXLIB_DSP_ERR_INTERNAL;\n" + "}\n" + ) + block = block_after_call(body, "real_call") + assert "HEXLIB_DSP_ERR_INTERNAL" in block + assert block.count("{") == 1 and block.count("}") == 1 + + +def test_function_body_still_finds_the_real_definition_with_no_comments(): + """Sanity: comment-awareness must not break the ordinary, comment-free + case it is layered on top of.""" + src = "int plain(void) {\n return 7;\n}\n" + assert "return 7;" in function_body(src, "plain") + + +# ============================================================================== +# The returned text, not just the boundaries. A guard deleted and left behind +# as a comment must not satisfy a payload check run against the slice -- the +# exact hole a mutation of skel_bufs.c walked through. +# ============================================================================== + + +def test_function_body_returns_comment_blanked_text_by_default(): + """The mutation shape, in miniature: the real `return -1;` is gone and + survives only as a comment inside the body. `"return -1;" in body` must + NOT be true, and the code that IS still there must still be visible.""" + src = ( + "int guard(int x) {\n" + " if (x < 0) {\n" + " /* return -1; */\n" + " x = 0;\n" + " }\n" + " return x;\n" + "}\n" + ) + body = function_body(src, "guard") + assert "return -1;" not in body, ( + "a commented-out return must not satisfy a payload check on the body" + ) + assert "x = 0;" in body + assert "return x;" in body + # length is preserved, so offsets taken from the slice still line up with + # the same slice of the original text + assert len(body) == len(function_body(src, "guard", strip=False)) + + +def test_function_body_strip_false_still_returns_the_original_text(): + """The escape hatch is real, and explicit: `strip=False` gives back the + comment-bearing slice for a caller that means to inspect comments.""" + src = "int guard(void) {\n /* return -1; */\n return 0;\n}\n" + raw = function_body(src, "guard", strip=False) + assert "/* return -1; */" in raw + + +def test_block_from_returns_comment_blanked_text_by_default(): + """Same property for `block_from`: the guarded block's payload must be + code. A commented-out `return -1;` inside the block is not a refusal.""" + text = "if (x) {\n /* return -1; */\n log_it();\n}\n" + block = block_from(text, text.index(")")) + assert "return -1;" not in block + assert "log_it();" in block + + +def test_block_after_call_returns_comment_blanked_text_by_default(): + """Same property for `block_after_call`: the status the caller looks for + in the checked block must be returned, not merely mentioned in prose.""" + body = ( + "int rc = real_call(a, b);\n" + "if (rc != 0) {\n" + " /* return HEXLIB_DSP_ERR_INTERNAL; */\n" + " rc = 0;\n" + "}\n" + ) + block = block_after_call(body, "real_call") + assert not re.search(r"return\s+HEXLIB_DSP_ERR_\w+\s*;", block), ( + "a commented-out error return must not count as propagating a status" + ) + assert "rc = 0;" in block + + +# ============================================================================== +# MECHANISM (A): the payload of a log line is not code. `assert "TOKEN" in +# body` must not be satisfiable by `FARF(HIGH, "TOKEN")` with the real code +# deleted. Reproduced on skel_bufs.c (`b->base = 0;` demoted to a FARF printing +# that text, `b->base = m->base;` deleted) and on skel_dispatch.c (the +# total_size guard logging HEXLIB_DSP_ERR_TRUNCATED and falling through). +# ============================================================================== + + +def test_function_body_blanks_a_log_lines_payload_by_default(): + src = ( + "int guard(struct buf *b) {\n" + ' FARF(HIGH, "b->base = 0; return HEXLIB_DSP_ERR_UNMAPPED;");\n' + " return 0;\n" + "}\n" + ) + body = function_body(src, "guard") + assert "HEXLIB_DSP_ERR_UNMAPPED" not in body, ( + "a status constant named only inside a format string must not satisfy " + "a payload check on the body" + ) + assert not re.search(r"return\s+HEXLIB_DSP_ERR_\w+\s*;", body) + assert not re.search(r"b->base\s*=\s*0\s*;", body), ( + "an assignment spelled out inside a log message is not an assignment" + ) + # The call itself is still visible -- only its payload is gone, so a check + # that the LOGGING happens is still possible. + assert "FARF(HIGH," in body + assert "return 0;" in body + assert len(body) == len(function_body(src, "guard", strip=False)) + + +def test_block_from_blanks_a_log_lines_payload_by_default(): + text = ( + "if (hdr.total_size != len) {\n" + ' FARF(ERROR, "hexlib: HEXLIB_DSP_ERR_TRUNCATED size mismatch");\n' + "}\n" + ) + block = block_from(text, text.index(")")) + assert "HEXLIB_DSP_ERR_TRUNCATED" not in block, ( + "the guard block must not be able to report a status by logging its " + "name -- that is the fall-through mutation this pins" + ) + assert "FARF(ERROR," in block + + +def test_block_after_call_blanks_a_log_lines_payload_by_default(): + body = ( + "int rc = real_call(a, b);\n" + "if (rc != 0) {\n" + ' FARF(ERROR, "returning return HEXLIB_DSP_ERR_INTERNAL; now");\n' + " rc = 0;\n" + "}\n" + ) + block = block_after_call(body, "real_call") + assert not re.search(r"return\s+HEXLIB_DSP_ERR_\w+\s*;", block), ( + "a status named in a log message must not count as propagating it" + ) + assert "rc = 0;" in block + + +# ============================================================================== +# MECHANISM (B): a `}` inside a literal must not truncate a brace-depth slice. +# Reproduced on skel_dispatch.c: `hexlib_read_pcycle` reverted to +# `__asm__("%0 = c15:14")` hidden behind `FARF(HIGH, "pcycle }")`, which made +# every "this must NOT appear here" check in the wrapper look at a fragment +# ending at the literal's brace. 12 passed. +# ============================================================================== + + +def test_function_body_is_not_truncated_by_a_closing_brace_in_a_literal(): + src = ( + "static uint64_t read_pcycle(void) {\n" + " uint64_t v = 0;\n" + ' FARF(HIGH, "hexlib: pcycle }");\n' + ' __asm__ __volatile__("%0 = c15:14" : "=r"(v));\n' + " return v;\n" + "}\n" + ) + body = function_body(src, "read_pcycle") + assert "__asm__" in body, ( + "code after a literal containing `}` must still be inside the body -- " + "otherwise every negative check on this function is answered by a " + "fragment that stops at the literal" + ) + assert "return v;" in body + assert body.rstrip().endswith("}") + assert body.count("{") == 1 and body.count("}") == 1, ( + "the literal's brace must have been blanked, not counted" + ) + + +def test_a_brace_in_a_char_literal_does_not_truncate_a_body_either(): + """The single-quoted form of the same thing -- and the one shape that can + turn up in real code without anybody trying (`if (c == '}')`).""" + src = ( + "int f(char c) {\n" + " if (c == '}') return 1;\n" + " return 0;\n" + "}\n" + ) + body = function_body(src, "f") + assert "return 0;" in body + assert body.count("}") == 1 + + +def test_block_from_is_not_truncated_by_a_closing_brace_in_a_literal(): + text = ( + "if (ctx->vtcm_needs_release) {\n" + ' FARF(HIGH, "reclaim }");\n' + " hexlib_vtcm_release(ctx);\n" + "}\n" + ) + block = block_from(text, text.index(")")) + assert "hexlib_vtcm_release(ctx);" in block + assert block.count("{") == 1 and block.count("}") == 1 + + +# ============================================================================== +# MECHANISM (C): a `{` inside a literal must not extend a block past its real +# end. The mirror image of (B), and the one that destroys scoping: an +# `if`-block check would see the whole rest of the function. +# ============================================================================== + + +def test_block_from_is_not_extended_by_an_opening_brace_in_a_literal(): + text = ( + "if (x) {\n" + ' FARF(HIGH, "entering {");\n' + " inside_the_block();\n" + "}\n" + "after_the_block();\n" + "if (y) {\n" + " return HEXLIB_DSP_ERR_INTERNAL;\n" + "}\n" + ) + block = block_from(text, text.index(")")) + assert "inside_the_block();" in block + assert "after_the_block();" not in block, ( + "a `{` inside a literal must not make the block swallow the code " + "after it -- that is scoping destroyed, and the whole reason these " + "checks are block-scoped rather than function-wide" + ) + assert "HEXLIB_DSP_ERR_INTERNAL" not in block + assert block.count("{") == 1 and block.count("}") == 1 + + +def test_block_after_call_is_not_derailed_by_a_literal_naming_the_call(): + """A literal that mentions the call by name, with its own parens and its + own `{`, sits BEFORE the real call. With literals preserved, the call-site + search lands inside the format string, the paren-walk closes on the + literal's own `)`, and the block search picks up the literal's `{` -- so + the block returned has nothing to do with checking the call's result.""" + body = ( + ' FARF(ERROR, "real_call() failed { ");\n' + " int rc = real_call(a, b);\n" + " if (rc != 0) {\n" + " return HEXLIB_DSP_ERR_INTERNAL;\n" + " }\n" + ) + block = block_after_call(body, "real_call") + assert "HEXLIB_DSP_ERR_INTERNAL" in block + assert block.count("{") == 1 and block.count("}") == 1 + + +# ============================================================================== +# The escape hatch, end to end: `keep_strings=True` returns literal text but +# must NOT move a boundary, because boundaries are always found in the fully +# blanked view. +# ============================================================================== + + +def test_keep_strings_returns_literals_without_moving_any_boundary(): + src = ( + "static void usage(void) {\n" + ' printf(" --unmapped }\\n");\n' + " printf(\" --coherency-check {\\n\");\n" + " return;\n" + "}\n" + ) + body = function_body(src, "usage", keep_strings=True) + assert "--unmapped" in body and "--coherency-check" in body, ( + "the escape hatch must actually hand back the literal text" + ) + assert "return;" in body, ( + "and the literals' own braces must still not truncate or extend the " + "slice -- boundaries come from the fully blanked view either way" + ) + # Same boundaries as the default view, so an offset found in one is valid + # in the other. This is what lets a consumer locate a flag string in the + # literal-bearing view and brace-slice the block in the blanked one. + assert len(body) == len(function_body(src, "usage")) + assert len(body) == len(function_body(src, "usage", strip=False)) + + +def test_no_test_file_carries_its_own_private_copy_of_the_slicer(): + """THE CONSOLIDATION CLAIM, MADE SELF-ENFORCING RATHER THAN PROMISED. + csource.py's docstring asserted the consolidation was complete while two + private copies were still live -- a claim about the codebase written in + prose, which is exactly the kind of thing that rots silently. This checks + it instead. + + A private copy is a `def` of one of these names in any hexlib/tests module + other than csource.py itself. An `import ... as _function_body` alias is + not a copy and is the intended usage, so only `def` is matched. + `strip_comments` stays on this list although `csource` no longer exports it: + it was the name of the comments-only transformation, and a test file + growing its own `_strip_comments` again is the same regression whether or + not the shared module still has that name. + `_macro_body` in test_host_source.py is deliberately excluded: it slices a + backslash-continued `#define`, which brace counting cannot do, and its own + docstring says why it is a narrowly-scoped sibling rather than a fourth + slicer.""" + shared = ("strip_comments", "code_only", "code_only_keeping_strings", + "calls", "function_body", "block_from", "block_after_call") + here = pathlib.Path(__file__).parent + offenders = [] + for path in sorted(here.glob("test_*.py")): + text = path.read_text(encoding="utf-8") + for name in shared: + if re.search(rf"^\s*def\s+_?{name}\s*\(", text, re.M): + offenders.append(f"{path.name} defines its own {name}()") + assert not offenders, ( + "private copies of the shared C slicer are back -- import them from " + "hexlib.tests.csource instead, and see that module's docstring for why " + "a near-copy is worse than no copy: " + + "; ".join(offenders) + ) + + +def test_code_only_is_the_whole_file_form_of_the_same_guarantee(): + """`code_only` is what a whole-file fixture goes through before any + payload check runs against it -- same blanking, same length, so a + constant or a call named only in a comment (or in a log message) cannot + satisfy (or trip) a file-wide check.""" + src = '/* calls HAP_mmap() here */\nint f(void) { return 0; }\n' + assert "HAP_mmap" not in code_only(src) + assert len(code_only(src)) == len(src) + assert "int f(void) { return 0; }" in code_only(src) + + logged = 'int f(void) { FARF(ERROR, "HAP_mmap failed"); return 0; }\n' + assert "HAP_mmap" not in code_only(logged), ( + "a whole-file fixture must not be able to satisfy a presence check " + "with a log message either" + ) + assert len(code_only(logged)) == len(logged) diff --git a/hexlib/tests/test_device_cache_maintenance.py b/hexlib/tests/test_device_cache_maintenance.py new file mode 100644 index 0000000..f873610 --- /dev/null +++ b/hexlib/tests/test_device_cache_maintenance.py @@ -0,0 +1,119 @@ +# hexlib/tests/test_device_cache_maintenance.py +"""The DSP must write its cache back, and the check has to read the ARTIFACT. + +WHAT THIS EXISTS FOR, MEASURED ON SM8650 (Pineapple, SM8650, 2026-08-13). +FastRPC keeps the two caches coherent for anything passed as an invoke +ARGUMENT. hexlib's data buffers are not arguments -- they are mapped out of +band through `fastrpc_mmap` and named on the wire only by fd, so that no +address ever crosses between the processors. FastRPC therefore does not know +they were written, and before `hexlib_bufs_flush` existed, nothing wrote them +back. Three symptoms, one cause: + + * `hexlib_run --self-test`: 3859 of 4100 fp16 values not bit-exact, against + a simulator that gives exactly 0 error. AFTER the flush: `PASS (4100 + values, bit-exact)`. + * `--coherency-check`: `COHERENCY sentinel_unchanged`, exit 6. + AFTER: `sentinel_overwritten`, RC=0. + * the 49-op encoder through `--batch`: 179,124 arena bytes changed but + `merger.out` -- the LAST op's 1024 bytes -- came back all zero. + AFTER: max relative error 1.1319e-03, correlation 1.000000, which is the + SAME figure the simulator produces. + +WHY THESE TESTS READ THE LINKED .so AND NOT THE SOURCE. `skel_bufs.c` carries a +host-compiler fallback that #defines the QuRT cache API away, because +`test_genentry_entry_probe.py` compiles that file with gcc and there is no +`qurt_memory.h` off-target. A fallback whose whole job is to do nothing is +exactly the thing that could silently become what ships. A source grep cannot +tell which branch a real build took. The linked artifact can: if the Hexagon +build compiled the stub, `qurt_mem_cache_clean` would be DEFINED (or absent) +rather than left UNDEFINED for QuRT to bind at load. +""" +import os +import pathlib +import subprocess + +import pytest + +from hexlib import toolchain as tc +from hexlib.runtime import build as rb +from hexlib.runtime import wire + +HAS_SDK = os.path.isdir(tc.default_sdk_root()) +sdk = pytest.mark.skipif(not HAS_SDK, reason="Hexagon SDK not present") + +REPO = pathlib.Path(__file__).resolve().parents[2] +SKEL = REPO / "hexlib" / "runtime" / "skel" + + +def test_a_cache_failure_has_its_own_status_on_both_sides(): + """Not folded into ERR_INTERNAL, because the consequence is specific and + misleading: every op ran, and what the host reads back may be STALE rather + than wrong. Nothing on the host can tell those apart without a code.""" + assert wire.STATUS["ERR_CACHE"] == 15 + header = (SKEL / "hexlib_dsp.h").read_text(encoding="utf-8") + assert "HEXLIB_DSP_ERR_CACHE = 15," in header + assert 'case HEXLIB_DSP_ERR_CACHE: return "ERR_CACHE";' in header + + +def test_both_cache_directions_are_called_and_not_just_one(): + """Flushing without invalidating works for exactly ONE invoke per session + and then silently computes on data this DSP cached during the previous one. + That is a worse bug than the one the flush fixes, because it needs two runs + to appear -- so the presence of BOTH calls is pinned, not just the flush.""" + src = (SKEL / "skel_dispatch.c").read_text(encoding="utf-8") + live = "\n".join( + ln for ln in src.splitlines() + if not ln.strip().startswith("*") and not ln.strip().startswith("/*") + ) + assert "hexlib_bufs_invalidate(ctx)" in live, ( + "nothing invalidates before the ops read host-written data" + ) + assert "hexlib_bufs_flush(ctx)" in live, ( + "nothing flushes after the ops write their results" + ) + + +def test_the_flush_runs_even_when_the_batch_failed(): + """An op that died halfway still wrote whatever it wrote. Leaving those + lines in cache makes the wreckage invisible to anyone debugging from the + host, which is the position this whole file exists to get out of.""" + src = (SKEL / "skel_dispatch.c").read_text(encoding="utf-8") + flush = src.index("cache_rc = hexlib_bufs_flush(ctx);") + tail = src[flush:] + assert "batch_status == HEXLIB_DSP_OK" in tail, ( + "the flush must not overwrite a real failure status" + ) + # It must not be inside the `if (batch_status == OK)` guard itself. + before = src[:flush].rstrip().splitlines()[-1].strip() + assert not before.startswith("if "), ( + "the flush is guarded by the batch status, so a failed batch leaves " + "its writes in cache" + ) + + +@sdk +def test_the_linked_skel_really_calls_qurts_cache_api(tmp_path): + """THE ONE THAT CANNOT BE SATISFIED BY THE HOST STUB. + + `qurt_mem_cache_clean` must appear as an UNDEFINED symbol in the linked + device .so -- QuRT binds it at load time inside the PD. If the Hexagon + build had compiled `skel_bufs.c`'s host fallback instead, the symbol would + be locally defined and this fails. That is the whole point: a `#if + defined(__hexagon__)` guard is a source-level claim, and this is the + artifact-level check of it.""" + rb.build_device_binary(str(tmp_path)) + so = os.path.join(str(tmp_path), rb.device_skel_so_name()) + nm = os.path.join( + tc.find_toolchain_bin(tc.default_sdk_root()), + "hexagon-nm" + (".exe" if os.name == "nt" else ""), + ) + out = subprocess.run([nm, "-u", so], capture_output=True, text=True).stdout + assert any("qurt_mem_cache_clean" in ln for ln in out.splitlines()), ( + "qurt_mem_cache_clean is not an undefined symbol in the linked skel -- " + "the host fallback in skel_bufs.c was compiled into the DEVICE build, " + "so nothing writes the DSP's cache back and the host reads stale data" + ) + + defined = subprocess.run([nm, so], capture_output=True, text=True).stdout + for sym in ("hexlib_bufs_flush", "hexlib_bufs_invalidate"): + assert any(sym in ln for ln in defined.splitlines()), f"{sym} not linked" diff --git a/hexlib/tests/test_device_cycles_assertion.py b/hexlib/tests/test_device_cycles_assertion.py new file mode 100644 index 0000000..2f17f76 --- /dev/null +++ b/hexlib/tests/test_device_cycles_assertion.py @@ -0,0 +1,207 @@ +# hexlib/tests/test_device_cycles_assertion.py +"""The one part of `hexlib/device/qdc/test_on_device.py` that CAN be run here. + +DELIBERATELY NOT NAMED `test_on_device_...`. `test_qdc_on_device_is_excluded.py` +proves the on-device file is never collected by asserting the literal string +`test_on_device.py` does not appear in `pytest --collect-only` output; a file +here whose own name shares that prefix is one rename away from making that +proof fail on a correctly-excluded repo. + +THE ON-DEVICE FILE ITSELF CANNOT BE. It executes inside the QDC artifact, +against a real phone, and the root `conftest.py` deliberately keeps pytest from +collecting it at all (see `test_qdc_on_device_is_excluded.py` for the mechanism +and the test that proves it). Nothing in this file changes that, and nothing +here runs a device test: `sh()` and `write_qdc_log()` are replaced by stubs and +no `test_*` function from that module is ever called. + +WHAT IS RUN, AND WHY IT IS WORTH RUNNING. `assert_cycles_total_is_a_real_ +measurement` is a pure function of one string -- the captured stdout of a +`hexlib_run` invocation -- so its behaviour is fully determined offline. It is +also the load-bearing new device assertion: until 2026-08-11 NOTHING on device +asserted the `cycles_total=` line at all (the test's docstring said `main.c` +never printed it, which had been false since an earlier commit), so deleting +`main.c`'s two `printf` lines would have kept all five on-device tests green +while flipping `hexlib/cli.py`'s post-job check to exit 1. + +A SOURCE ASSERTION WOULD NOT HAVE BEEN ENOUGH, for the same reason +`test_coherency_lane_classification.py` exists: "the file contains the string +`cycles_total`" cannot tell a check that rejects `cycles_total=0` apart from one +that accepts it, and accepting zero is precisely the defect -- the DSP reads +PCYCLE in a user-mode unsigned PD, where `SYSCFG.PCYCLEEN` cannot be set, so +zero is the expected reading if the counter is dead on this silicon. So the +function is imported and CALLED against real strings, including `cycles_total=0`. + +The rest of the on-device file's assertions genuinely cannot be executed here +and are reviewed only; that is stated plainly rather than implied by this +file's existence. +""" +import ast +import importlib.util +import inspect +import pathlib +import sys +import textwrap +import types + +import pytest + +ON_DEVICE = pathlib.Path("hexlib/device/qdc/test_on_device.py") + + +@pytest.fixture(scope="module") +def on_device(): + """Load the on-device module by path, with a stub `utils` standing in for + the flat module that sits beside it in the artifact zip. + + NOT imported as `hexlib.device.qdc.test_on_device`: on the farm the + artifact is one flat directory and `from utils import sh, write_qdc_log` + is what works there, so the file is correct as written and must not be + changed to import differently (conftest.py's own docstring says so). The + stubs raise if called, so a future refactor that made module import time + shell out would fail here rather than silently run something.""" + + def _no(*a, **kw): + raise AssertionError( + "the on-device module must not run shell commands at import time" + ) + + stub = types.ModuleType("utils") + stub.sh = _no + stub.write_qdc_log = _no + # `push` joined these when the on-device model was corrected: pytest runs + # on the QDC RUNNER, so binaries reach the phone by `adb push`, not `cp`. + # Stubbed like the others so an import-time call fails here rather than + # shelling out. + stub.push = _no + + saved = sys.modules.get("utils") + sys.modules["utils"] = stub + try: + spec = importlib.util.spec_from_file_location( + "hexlib_on_device_under_test", ON_DEVICE + ) + mod = importlib.util.module_from_spec(spec) + spec.loader.exec_module(mod) + finally: + if saved is None: + del sys.modules["utils"] + else: + sys.modules["utils"] = saved + return mod + + +def test_a_real_measurement_is_accepted_and_returned(on_device): + out = ( + "hexlib: --self-test: PASS (4100 values, bit-exact)\n" + "hexlib: --self-test: cycles_total=1287\n" + "RC=0\n" + ) + assert on_device.assert_cycles_total_is_a_real_measurement(out, "x") == 1287 + + +def test_a_zero_measurement_is_rejected(on_device): + """THE DEFECT. `cycles_total=0` contains the substring `cycles_total=`, so + any presence-only check accepts it -- and a run in which the DSP's cycle + counter never advanced measured nothing. The failure message must name + PCYCLEEN, because that is the actionable finding: not "hexlib is broken" + but "the counter does not work in this PD, so no stage-1 cycle figure + transfers".""" + out = ( + "hexlib: --self-test: PASS (4100 values, bit-exact)\n" + "hexlib: --self-test: cycles_total=0\n" + "RC=0\n" + ) + with pytest.raises(AssertionError) as e: + on_device.assert_cycles_total_is_a_real_measurement(out, "x") + assert "PCYCLEEN" in str(e.value), ( + "a zero cycle count must be reported as the PCYCLEEN/unsigned-PD " + "finding it is, not as a generic assertion failure" + ) + + +def test_an_absent_line_is_rejected_with_a_different_message(on_device): + """Absent and zero are DIFFERENT findings -- one means hexlib stopped + printing the line (or got no response), the other means the hardware + counter is dead -- and must not be reported as each other.""" + out = "hexlib: --self-test: PASS (4100 values, bit-exact)\nRC=0\n" + with pytest.raises(AssertionError) as e: + on_device.assert_cycles_total_is_a_real_measurement(out, "x") + msg = str(e.value) + assert "no `cycles_total=` line" in msg + assert "PCYCLEEN" not in msg + + +def test_a_positive_line_alongside_a_zero_one_is_accepted(on_device): + """One captured run can legitimately carry more than one `cycles_total=` + line. At least one genuine measurement is the bar; if the counter were + dead, EVERY line would read 0 and the zero test above still catches it.""" + out = "cycles_total=0\ncycles_total=1287\n" + assert on_device.assert_cycles_total_is_a_real_measurement(out, "x") == 1287 + + +def test_the_regex_does_not_match_a_non_numeric_value(on_device): + """`cycles_total=` must not be silently read as a measurement. + The regex captures digits only, so a malformed value looks ABSENT to this + helper -- which fails, which is the safe direction. (hexlib/cli.py + distinguishes malformed from absent on its side, where it has the whole + job's logs and can say which.)""" + out = "hexlib: --self-test: cycles_total=\n" + with pytest.raises(AssertionError): + on_device.assert_cycles_total_is_a_real_measurement(out, "x") + + +_HELPER = "assert_cycles_total_is_a_real_measurement" + + +def _calls_in(fn): + """The set of function names CALLED in `fn`'s body, read out of its parsed + syntax tree. + + WHY AN AST AND NOT A SUBSTRING SEARCH. This test used to be + `_HELPER + "(" in src`, over `inspect.getsource` text with only the + DOCSTRING removed. That is comment-blind: deleting the real call and leaving + `# assert_cycles_total_is_a_real_measurement(out, tag)` behind kept this + file at 6 passed while nothing on device checked the measurement at all -- + the same defect class the C source-assertion tests were rewritten twice for, + reappearing in the one place the subject is Python. + + WHY NOT `csource`, WHICH IS WHERE THE C SIDE'S ANSWER LIVES. It is a C + lexer: its comment tokens are `/* */` and `//`, and Python's is `#`. Handing + it Python source blanks the string literals and leaves every `#` comment + exactly where it was -- so it would not close this hole, only appear to. An + AST closes it by construction instead: a commented-out call is not a Call + node, a call named inside a string is not a Call node, and a docstring is + not a Call node, so none of the three need special handling. It is also + stricter than the text check ever was, because `_HELPER` appearing in an + unrelated expression (an f-string, a variable name) no longer counts. + + Attribute calls (`utils.foo()`) contribute their attribute name, so a call + reached through a module or object alias is still seen.""" + tree = ast.parse(textwrap.dedent(inspect.getsource(fn))) + found = set() + for node in ast.walk(tree): + if isinstance(node, ast.Call): + func = node.func + if isinstance(func, ast.Name): + found.add(func.id) + elif isinstance(func, ast.Attribute): + found.add(func.attr) + return found + + +def test_both_device_invocations_assert_the_measurement(on_device): + """Both `--self-test` and `--self-test --coherency-check` must call the + helper -- reviewed by source here, since the calls themselves can only run + on a device. Scoped to each function's own syntax tree, so one call cannot + cover for the other's absence, and a commented-out call cannot cover for + either (see `_calls_in`).""" + for name in ( + "test_scale_fp16_runs_on_the_dsp_and_is_correct", + "test_cache_coherency_is_independent_of_marshalling_and_of_any_kernel", + ): + fn = getattr(on_device, name) + assert _HELPER in _calls_in(fn), ( + f"{name} does not CALL {_HELPER}() -- nothing on device would then " + f"check the measurement at all, and a mention of it in a comment or " + f"a docstring is not a call" + ) diff --git a/hexlib/tests/test_device_skel_so_name.py b/hexlib/tests/test_device_skel_so_name.py new file mode 100644 index 0000000..e1eb135 --- /dev/null +++ b/hexlib/tests/test_device_skel_so_name.py @@ -0,0 +1,87 @@ +# hexlib/tests/test_device_skel_so_name.py +"""The skel's filename, bound to the one FastRPC will actually dlopen. + +THE DEFECT THIS EXISTS FOR WAS FOUND ON SILICON AND COULD NOT HAVE BEEN FOUND +ANYWHERE ELSE. `_build_device_skel_so` linked `libhexlib_skel.so`. The URI qaic +generates into `hexlib_iface.h` -- `hexlib_iface_URI`, which `session.c:163` +passes to `remote_handle64_open` -- names the library after the IDL, so the +device looks for `libhexlib_iface_skel.so`. `hexlib_iface_open` returned +rc -2147482618 until the file was pushed under both names by hand. + +WHY NO SIMULATOR TEST COULD CATCH IT: stage 1's build LINKS the skel directly +(`simhost.c` calls `hexlib_iface_open/_start/_invoke/...` as plain C functions, +bound by the linker straight to `skel.c`), so the filename never participates in +that path at all. The name only matters when something loads it BY NAME, and +only a device does. + +So the name is now DERIVED from the IDL stem in one place, and every other copy +is checked against that derivation here rather than trusted. +""" +import pathlib +import re + +from hexlib.runtime.build import HEXLIB_IDL_STEM, device_skel_so_name + +REPO = pathlib.Path(__file__).resolve().parents[2] + + +def test_the_name_follows_qaics_own_convention(): + """qaic names its outputs `.h` / `_stub.c` / `_skel.c` + from the IDL, and the URI it writes names `lib_skel.so`. This pins the + derivation itself, so a future change to the helper has to be deliberate.""" + assert device_skel_so_name() == "libhexlib_iface_skel.so" + assert device_skel_so_name("some_other_iface") == "libsome_other_iface_skel.so" + + +def test_the_idl_that_stem_comes_from_actually_exists(): + """A derivation from a stem that names no IDL would be a convention with + nothing behind it -- and qaic's URI comes from the real file's name.""" + idl = REPO / "hexlib" / "runtime" / "idl" / f"{HEXLIB_IDL_STEM}.idl" + assert idl.is_file(), f"{idl} does not exist, so the derived name is a guess" + + +def test_the_linker_is_told_the_derived_name_and_not_a_literal(): + """`build.py` must call the helper. A hardcoded string here would compile + and link perfectly and fail only on a device, which is the whole history of + this bug.""" + src = (REPO / "hexlib" / "runtime" / "build.py").read_text(encoding="utf-8") + assert "device_skel_so_name()" in src + assert 'os.path.join(out_dir, "libhexlib_skel.so")' not in src, ( + "build.py still links explicitly to the old name" + ) + + +def test_the_on_device_module_spells_the_same_name(): + """`test_on_device.py` runs on the QDC runner without hexlib installed, so + it cannot import the helper and has to carry a literal. That literal is read + back here and compared -- which is the only thing that keeps the two in step, + since nothing else imports both.""" + src = (REPO / "hexlib" / "device" / "qdc" / "test_on_device.py").read_text( + encoding="utf-8" + ) + m = re.search(r'^SKEL_SO\s*=\s*"([^"]+)"', src, re.M) + assert m, "test_on_device.py declares no SKEL_SO literal" + assert m.group(1) == device_skel_so_name(), ( + f"the on-device module pushes {m.group(1)!r} but the linker produces " + f"{device_skel_so_name()!r}; the device would dlopen a file that is " + f"not there, exactly as it did before this was bound" + ) + + +def test_nothing_still_names_the_old_library(): + """The old name in a *comment* is fine and deliberate -- several of them + explain the bug. What must not survive is a live reference in code that + packs, pushes or links the file.""" + offenders = [] + for rel in ("hexlib/cli.py", "hexlib/device/qdc/test_on_device.py", + "hexlib/runtime/build.py"): + for i, line in enumerate((REPO / rel).read_text(encoding="utf-8").splitlines(), 1): + stripped = line.strip() + if stripped.startswith("#") or stripped.startswith("*"): + continue + if '"libhexlib_skel.so"' in line or "'libhexlib_skel.so'" in line: + offenders.append(f"{rel}:{i}: {stripped}") + assert not offenders, ( + "these still name the pre-fix library as a string literal:\n" + + "\n".join(offenders) + ) diff --git a/hexlib/tests/test_dsp_sim.py b/hexlib/tests/test_dsp_sim.py new file mode 100644 index 0000000..fedac3c --- /dev/null +++ b/hexlib/tests/test_dsp_sim.py @@ -0,0 +1,706 @@ +# hexlib/tests/test_dsp_sim.py +"""Stage 1 acceptance: scale_fp16 through the skel's batch path, on the simulator. + +NOT "through FastRPC". The simulator omits the qaic stub (spec 0.1), so these +calls bind directly to the skel implementation and no marshalling occurs. What is +proven here is OUR code: batch parsing, the buffer table, the dispatch table, the +kernel adapter, the values, and PCYCLE. Marshalling is stage 3's. + +THE TEST THAT MATTERS IS test_an_unmapped_fd_is_refused. Under the simulator the +host and the DSP share one address space, so a skel that used the host's pointer +instead of resolving an fd would return the RIGHT ANSWER to a request whose +buffer was never mapped -- and would fail instantly on silicon. That test fails +exactly when the invariant is broken, which is what makes everything else here +transferable. Without it these are tests of an interface that might only work in +one address space. + +Artifacts (the skel archive, the QuRT-hosted .so, the sim configs) are built ONCE +per module via the `backend` fixture -- construction is the slow part; each test +below is one additional `hexagon-sim` launch reusing them. +""" +import os + +import numpy as np +import pytest + +from hexlib import toolchain as tc +from hexlib.exec import dsp as dspmod + +HAS_SDK = os.path.isdir(tc.default_sdk_root()) +sdk = pytest.mark.skipif(not HAS_SDK, reason="Hexagon SDK not present") + +N = 4100 # SCALE_N: 64*64 + 4, so a tail exists +FACTOR = 0.125 # SCALE_FACTOR, a power of two -> exact in fp16 + + +@pytest.fixture(scope="module") +def backend(tmp_path_factory): + return dspmod.DspSimBackend(["scale_fp16"], + str(tmp_path_factory.mktemp("dspsim"))) + + +@sdk +def test_scale_fp16_matches_numpy_exactly(backend): + """Error is exactly 0 because 0.125 is a power of two: only the exponent + changes and no mantissa bit is lost -- PROVIDED the scaled result stays in + fp16's NORMAL range. It does not always: `rng.standard_normal` puts ~0.8% + of elements below 0.01 in magnitude, and 0.125 * that underflows fp16's + normal floor (2**-14) into SUBNORMAL territory, where the real HVX + kernel's qf32-narrow rounding measurably diverges from numpy's by one ULP + (confirmed empirically: `hexagon.backend_for("scale")`, the already-proven + standalone-ELF path, shows the IDENTICAL one-ULP divergence from numpy at + the identical index for the identical seed -- so this is a genuine kernel + HW rounding edge case the original harness's deterministic, always-normal + inputs never exercised, not a marshalling bug in this new path). That is + a real finding, reported rather than hidden, but it is orthogonal to what + this test exists to check -- so inputs are kept clear of the subnormal + boundary, at every index still independently random (a dropped tail or a + byte-order swap needs that), rather than narrowing the test's claim.""" + rng = np.random.default_rng(0) + x = rng.standard_normal(N).astype(np.float32) + too_small = np.abs(x) < 0.01 + x = np.where(too_small, np.sign(x) * 0.01 + x, x).astype(np.float16) + y, stats = backend.run("scale", [x], {"factor": FACTOR}) + expect = (x.astype(np.float32) * FACTOR).astype(np.float16) + assert y.dtype == np.float16 + assert np.array_equal(y, expect), f"max diff {np.abs(y.astype(np.float32) - expect.astype(np.float32)).max()}" + + +@sdk +def test_an_unmapped_fd_is_refused(backend): + """THE DISCRIMINATOR. See the module docstring. A skel leaning on the shared + address space passes the request; a correct one refuses it.""" + res = backend.run_unmapped("scale", N, FACTOR) + assert res.status != dspmod.wire.STATUS["OK"], ( + "an invoke naming a never-mapped fd returned a RESULT. The skel resolved " + "an address it was not given — which works only because the simulator " + "shares one address space, and will fail on silicon." + ) + assert res.status == dspmod.wire.STATUS["ERR_UNMAPPED"] + + +@sdk +def test_the_dsp_reports_v75_and_a_real_vtcm_size(backend): + info = backend.hwinfo() + assert info.arch == 75 + assert info.vtcm == 8388608, "acquired VTCM should be the whole 8 MB page" + + +@sdk +def test_cycles_are_in_the_right_order_of_magnitude(backend): + """hexlib.sim reports 886 for this kernel. An order-of-magnitude difference + means PCYCLE is not bracketing what we think it is.""" + rng = np.random.default_rng(1) + x = rng.standard_normal(N).astype(np.float16) + _, stats = backend.run("scale", [x], {"factor": FACTOR}) + assert 200 < stats.cycles < 20000, f"got {stats.cycles}, sim reports 886" + + +@sdk +def test_it_agrees_with_the_standalone_elf_path_bit_for_bit(backend, tmp_path): + """The two transports must not disagree. While both exist, this is the + signal that says so. + + NOTE: the original draft called `old(["scale"], [x], {"factor": FACTOR})` + against `hexagon.backend_for`'s returned callable, which takes exactly two + positional arguments (`arrays`, `attrs`) -- that call would raise + TypeError before comparing anything, for any implementation, correct or + not. Fixed here to the real two-argument signature so this test can + actually fail when the two paths disagree, which is the only thing it + exists to check. + """ + from hexlib.exec import hexagon + + rng = np.random.default_rng(2) + x = rng.standard_normal(N).astype(np.float16) + y_rpc, _ = backend.run("scale", [x], {"factor": FACTOR}) + old = hexagon.backend_for("scale", work_dir=str(tmp_path)) + assert old is not None, "hexagon.backend_for('scale') found no RunnerSpec" + (y_elf,) = old([x], {"factor": FACTOR}) + assert np.array_equal(y_rpc, y_elf) + + +@sdk +def test_a_bad_magic_is_refused_without_running_an_op(backend): + res = backend.run_raw(b"NOPE" + b"\x00" * 60) + assert res.status == dspmod.wire.STATUS["ERR_BAD_MAGIC"] + assert res.n_ops == 0 + + +@sdk +def test_a_truncated_batch_is_refused(backend): + blob = backend.build_batch("scale", N, FACTOR) + res = backend.run_raw(blob[: len(blob) - 8]) + assert res.status == dspmod.wire.STATUS["ERR_TRUNCATED"] + + +@sdk +def test_an_unknown_kind_is_refused(backend): + res = backend.run_raw(backend.build_batch("scale", N, FACTOR, kind_override=999)) + assert res.status == dspmod.wire.STATUS["ERR_NO_KERNEL"] + + +@sdk +def test_a_response_that_was_never_written_cannot_read_as_success(backend): + """Belt and braces on the structural guarantee: status 0 is not a status.""" + with pytest.raises(dspmod.wire.WireError): + dspmod.wire.unpack_response(b"\x00" * 32) + + +# --------------------------------------------------------------------------- +# layernorm: three inputs, mixed input dtypes, and the dim: scalar sources +# --------------------------------------------------------------------------- +# +# Everything above drives `scale`: one input, one float attr. That left the +# general run() path -- multiple buffers, per-input dtypes, dimension-derived +# scalars -- carried on the simulator by nothing, which STATE.md recorded as an +# open gap. layernorm is the first op here with THREE inputs and the first with +# MIXED input dtypes (fp16 data, fp32 affine parameters), so it exercises the +# generated entry's per-buffer dtype check against buffers that genuinely differ. + +LN_EPS = 1e-6 + + +def _ln_reference(x, w, b, eps, ddof=0): + """The op registry's own formula, accumulated in fp64. + + `ddof=1` produces the UNBIASED-variance near-miss that + kernels/layernorm_fp16/ keeps as a rejected variant. It is a parameter here + so the test below can prove its own threshold discriminates. + """ + xf = x.astype(np.float64) + mean = xf.mean(axis=-1, keepdims=True) + centred = xf - mean + n = xf.shape[-1] + var = (centred * centred).sum(axis=-1, keepdims=True) / (n - ddof) + return ((centred / np.sqrt(var + eps)) * w + b).astype(np.float16) + + +def _ln_inputs(R, C, seed=7): + rng = np.random.default_rng(seed) + return (rng.standard_normal((R, C)).astype(np.float16), + rng.standard_normal(C).astype(np.float32), + rng.standard_normal(C).astype(np.float32)) + + +@sdk +@pytest.mark.parametrize("R,C", [(4, 768), (8, 64), (256, 768)]) +def test_layernorm_agrees_with_the_reference_and_rejects_the_near_miss(backend, R, C): + """A MAX-ERROR TOLERANCE CANNOT CHECK THIS KERNEL, so this does not use one. + + One fp16 ULP is 9.8e-4 relative. The unbiased-variance near-miss -- divide + by C-1 instead of C -- is only 6.5e-4 at C=768. So ANY per-element tolerance + loose enough to admit the kernel's genuine 1-ULP noise is already looser + than the bug it must catch. This repo has paid for that once: that near-miss + was WRONGLY ACCEPTED on its first run for exactly this reason. + + What separates them is the SHAPE of the disagreement, not its size. The + kernel's error is 1-ULP noise on a handful of elements; the near-miss is a + systematic shift on every element. So the statistic is the fraction of + BIT-EXACT elements, and the test asserts both directions -- that the kernel + clears the threshold, and that the near-miss does not. The second assertion + is what stops the threshold from being quietly argued downwards later: + lower it far enough to admit the bug and this test fails. + + Measured through this path, for the record: 99.805% bit-exact at (4,768), + 99.917% at (256,768), 100% at (8,64); the near-miss scores 49.7%, 47.8% and + 11.1%. Max error is 1 ULP at (4,768) and 6 ULPs at (256,768) -- the relative + error is the same, there are simply 64x more rows for the tail to appear in. + """ + x, w, b = _ln_inputs(R, C) + y, _ = backend.run("layernorm", [x, w, b], {"eps": LN_EPS}) + + assert y.dtype == np.float16 + assert y.shape == (R, C) + + good = _ln_reference(x, w, b, LN_EPS, ddof=0) + near_miss = _ln_reference(x, w, b, LN_EPS, ddof=1) + + frac_good = float((y == good).sum()) / y.size + frac_bad = float((y == near_miss).sum()) / y.size + + assert frac_good >= 0.99, ( + f"layernorm at R={R} C={C} matched the reference bit-for-bit on only " + f"{frac_good:.4%} of elements; 1-ULP noise on a few is expected, a " + f"systematic disagreement is not" + ) + assert frac_bad < 0.90, ( + f"THE THRESHOLD NO LONGER DISCRIMINATES: the unbiased-variance " + f"near-miss scores {frac_bad:.4%}, which the 0.99 bound above would " + f"not obviously reject. Tighten the check or find a better statistic " + f"-- do not relax it." + ) + assert frac_good - frac_bad > 0.4, ( + f"correct {frac_good:.4%} vs near-miss {frac_bad:.4%}: the separation " + f"this test relies on has collapsed" + ) + + +@sdk +def test_layernorm_reaches_the_kernel_with_its_dimensions_and_eps_intact(backend): + """The scalars, checked by consequence rather than by reading the blob. + + R and C arrive as `dim:0:0` and `dim:0:1` and eps as `attr:eps` -- the first + use of the dimension-derived sources on this transport. A swap of R and C + would be invisible to a square input and to any shape-only assertion, so + this uses a NON-SQUARE shape whose transpose is not even the same length, + and a deliberately large eps whose effect on the output is unmistakable. + """ + R, C = 8, 64 + x, w, b = _ln_inputs(R, C) + + y_small = backend.run("layernorm", [x, w, b], {"eps": 1e-6})[0] + y_huge = backend.run("layernorm", [x, w, b], {"eps": 4.0})[0] + + assert y_small.shape == (R, C) + # eps sits inside the sqrt, so a large one shrinks every normalised value + # towards zero before the affine term. If eps were dropped or read as an + # int, these two would be identical. + assert not np.array_equal(y_small, y_huge), ( + "eps=1e-6 and eps=4.0 produced identical output, so the attr scalar is " + "not reaching the kernel" + ) + assert np.allclose(y_huge.astype(np.float32), + _ln_reference(x, w, b, 4.0).astype(np.float32), + atol=2e-3), "eps=4.0 did not match the reference" + + +# --------------------------------------------------------------------------- +# The host's own fd-patch loop, bounded +# --------------------------------------------------------------------------- +# +# `run_raw` exists so a bad magic, a truncated blob or an unknown op kind +# exercises the DSP'S OWN validation. That makes n_bufs and off_bufs untrusted +# BY DESIGN on this path -- and simhost.c's fd-patch loop read both straight +# out of the blob and wrote 24 bytes per iteration into a fixed 64 KiB static +# array BEFORE `hexlib_iface_invoke` handed them to the code that validates +# them. So the two tests below could not have reported what they claim to: +# they would have crashed hexagon-sim inside the host, and a crash is not the +# DSP refusing anything. +# +# skel_dispatch.c has always validated these fields correctly, and main.c does +# on the device path. The asymmetry was the defect. + +_HDR_I_N_BUFS = 3 # struct order in wire._HDR: magic, version, total, +_HDR_I_OFF_BUFS = 6 # n_bufs, n_tensors, n_ops, off_bufs, ... + + +def _patch_hdr_word(blob: bytes, index: int, value: int) -> bytes: + out = bytearray(blob) + out[index * 4:(index + 1) * 4] = int(value).to_bytes(4, "little") + return bytes(out) + + +@sdk +def test_an_out_of_range_off_bufs_is_recognised_by_the_host_and_left_unpatched( + backend, +): + """off_bufs=0xFFFFFF00 with n_bufs=1, and THE STATUS ALONE DOES NOT PIN IT. + + That was this test's first form and removing the bound left it green, which + is the only reason the real behaviour here is written down. `size_t` is 32 + bits on this target, so `g_batch + 0xFFFFFF00` is `g_batch - 256`: the + unbounded loop did not fault at all, it wrote 24 bytes into whatever static + lives before `g_batch` and then invoked normally, and the skel returned the + same ERR_TRUNCATED it returns now. Silent corruption of a neighbouring + object, reported as a clean refusal -- worse than the crash the finding + described, and invisible to any assertion on the response. + + So this pins the HOST's own recognition, printed by simhost.c, which is the + thing that actually differs between patched and unpatched. The skel's + verdict is asserted too: an unpatched batch must still be SENT, because + refusing to send it here would substitute the host's judgement for the + DSP's, and `run_raw` exists to observe the DSP's. + """ + blob = backend.build_batch("scale", N, FACTOR) + res, out = backend.run_raw_verbose(_patch_hdr_word(blob, _HDR_I_OFF_BUFS, 0xFFFFFF00)) + assert "bufs_out_of_range_not_patched" in out, ( + "simhost patched (or silently wrapped past) a buffer table that does " + "not lie inside the blob; the loop is unbounded again. stdout:\n" + + out[-2000:] + ) + assert res.status == dspmod.wire.STATUS["ERR_TRUNCATED"], ( + f"the unpatched batch must still reach the skel and be refused BY the " + f"skel; got {dspmod.wire.STATUS_NAME.get(res.status, res.status)}" + ) + + +@sdk +def test_an_enormous_n_bufs_is_refused_by_the_dsp_not_by_a_host_crash(backend): + """n_bufs=0x01000000 with off_bufs left valid -- 384 MiB of forward walk. + + The other shape, and the one that does NOT trip the section-bounds check + first: the skel answers `ERR_INVAL_PARAMS` from `n_bufs > HEXLIB_MAX_BUFS`. + Before the bound, simhost's loop would have stepped 24 bytes at a time over + g_rsp and the skel's own static bufs[]/tens[] on the way there. + """ + blob = backend.build_batch("scale", N, FACTOR) + res, out = backend.run_raw_verbose(_patch_hdr_word(blob, _HDR_I_N_BUFS, 0x01000000)) + assert "bufs_out_of_range_not_patched" in out, ( + "simhost tried to patch 16.7M buffer descriptors out of a 64 KiB " + "array. stdout:\n" + out[-2000:] + ) + assert res.status in ( + dspmod.wire.STATUS["ERR_INVAL_PARAMS"], + dspmod.wire.STATUS["ERR_TRUNCATED"], + ), ( + f"expected the skel to refuse the buffer count, got " + f"{dspmod.wire.STATUS_NAME.get(res.status, res.status)}" + ) + + +# =========================================================================== +# THE FOUR KERNELS ADDED FOR THE ENCODER, EACH CHECKED AGAINST THE OP REGISTRY +# =========================================================================== +# +# WHY THE REGISTRY IS THE ORACLE HERE rather than a reference written in this +# file. Every one of these ops has a numpy `reference` in +# `hexlib/graph/opdefs/`, and that reference IS the specification -- it is what +# the eager executor runs, what the plan executor was validated against, and +# what the kernel author was told to implement. A second reference written here +# would be a second chance to get the same convention wrong, and for three of +# these four the convention is precisely the dangerous part: +# +# patchify `merge` reorders the output ROWS into 2x2 spatial-merge-block +# order; ignoring it gives the right shape and byte count. +# rope_2d split-half pairing (i, i+D/2), not adjacent pairs. Same shape +# either way. +# transpose_hd perm(0,2,1) vs perm(1,0,2) -- and with two equal dims, even a +# stride confusion returns the right answer. +# +# So these tests compare the C kernel, reached through the real batch wire, with +# the numpy oracle the graph itself uses. Disagreement means one of them is +# wrong, which is the finding either way. +# +# AND WHAT THEY PROVE THAT THE KERNEL GATE DOES NOT. The gate compiles a kernel +# against its own harness and never touches the batch path. `layernorm_fp16` +# gated green for a day while `KIND_ID["layernorm"]` answered ERR_NO_KERNEL on +# the wire, because a kernel is dispatchable only once it has a RunnerSpec. Each +# test below drives the op through `pack_batch` -> the skel -> the generated +# entry -> the kernel, so it fails if the spec's scalars, dtypes, layouts or +# buffer order are wrong even when the kernel itself is perfect. + + +def _oracle(kind, arrays, attrs): + """The op registry's own numpy reference for `kind`.""" + import hexlib.graph.opdefs # noqa: F401 -- registers the op definitions + from hexlib.graph.ops import REGISTRY + + return REGISTRY.get(kind).reference(tuple(arrays), dict(attrs))[0] + + +def _frac_bit_exact(got, want): + return float((np.asarray(got) == np.asarray(want)).sum()) / np.asarray(got).size + + +@sdk +def test_transpose_hd_dispatches_and_matches_the_registry(backend): + """perm(0,2,1), the variant `select()` has to route away from its sibling. + + B, T and D are three DIFFERENT numbers, because with T == D a kernel that + confuses the two strides still returns the right answer -- and the sibling + kernel keeps a near-miss that is exactly that confusion. Exact comparison: + this op does no arithmetic, so any difference at all is a bug. + + The routing is the other half of what this checks. `transpose` and + `transpose_hd` are two kernels behind one op kind and the wire carries no + perm, so the host resolves the variant and names it with its own KIND_ID. + Send the wrong id and this returns a correctly-shaped transposed-the-other- + way answer. + """ + B, T, D = 3, 8, 5 + rng = np.random.default_rng(11) + x = rng.standard_normal((B, T, D)).astype(np.float16) + + y, _ = backend.run("transpose", [x], {"perm": (0, 2, 1)}) + + assert y.shape == (B, D, T) + want = _oracle("transpose", [x], {"perm": (0, 2, 1)}) + assert np.array_equal(y, want.astype(np.float16)), ( + "the perm(0,2,1) op did not match the registry -- either the kernel is " + "wrong or it was dispatched to the perm(1,0,2) kernel" + ) + # And it must NOT equal the other permutation, which is the failure that + # would otherwise look like success on a square input. + if B == D: + other = np.transpose(x, (1, 0, 2)) + assert not np.array_equal(y, other) + + +@sdk +def test_softmax_dispatches_and_matches_the_registry(backend): + """Row softmax over the last axis, through the wire. + + NON-SQUARE ON PURPOSE. The encoder's real shape is (12,256,256), whose last + two dims are equal -- so a softmax along the wrong axis has the SAME shape + and the same byte count and nothing downstream could notice. (2,3,64) makes + the wrong axis a different shape, so `_out_shape` and the byte-count check + would catch it even before the values were compared. + + Also the first exercise of the `rows:` scalar source on this transport: R is + the product of the two leading axes (2*3 = 6), which no single `dim:` can + express, and the DSP computes it from `ne` rather than trusting a number the + host asserted. A wrong `rows:` gives a kernel that softmaxes over the wrong + number of rows, which on a 3-D input is a plausible wrong answer. + """ + rng = np.random.default_rng(12) + x = (rng.standard_normal((2, 3, 64)) * 3.0).astype(np.float16) + + y, _ = backend.run("softmax", [x], {"axis": -1}) + + assert y.shape == (2, 3, 64) + assert y.dtype == np.float16 + want = _oracle("softmax", [x], {"axis": -1}).astype(np.float16) + + # A tolerance is used here rather than bit-exactness, and the reason is + # written down: the kernel narrows through Q6_Vhf_equals_Wqf32, whose + # rounding is NOT IEEE round-to-nearest-even, so a 1-ULP disagreement with + # numpy is expected and is not a defect. 1 ULP at these magnitudes is ~1e-3 + # relative; the bound below is well inside what a real bug would exceed -- + # kernels/softmax_fp16/harness.c measures its own fp16-accumulation + # near-miss at 7.7% relative, about 80x this bound. + err = np.abs(y.astype(np.float32) - want.astype(np.float32)) + assert err.max() < 1e-3, f"max abs error {err.max()}" + + # THE PROPERTY, INDEPENDENT OF THE ORACLE: every row sums to 1. This is what + # catches a normalisation that divides by the count, or by a stale sum, in a + # way that comparing against a reference computed the same way would not. + sums = y.astype(np.float32).sum(axis=-1) + assert np.allclose(sums, 1.0, atol=2e-3), f"row sums {sums}" + + +@sdk +def test_rope_2d_dispatches_and_matches_the_registry(backend): + """The split-half rotation, with three inputs and mixed dtypes. + + T, H and D are all DIFFERENT, so a token/head index swap cannot pass by + coincidence -- and note the cos/sin tables have NO head axis, so indexing + them by head instead of by token is a plausible stride slip that the + kernel's own harness keeps as a near-miss. + + D must be even for the split-half pairing to exist at all, and the kernel + only vectorises D=64; other D take a correct scalar path. D=64 is used here + because it is the encoder's own head_dim and the path that actually ships. + """ + T, H, D = 5, 3, 64 + rng = np.random.default_rng(13) + x = rng.standard_normal((T, H, D)).astype(np.float16) + # REAL ROTATION TABLES, AND "REAL" MEANS DUPLICATED ACROSS THE TWO HALVES. + # A split-half rotation pairs element i with i + D/2 and applies cos[i], + # sin[i] to both, so the pair is a genuine 2-D rotation only when + # cos[i + D/2] == cos[i] and sin[i + D/2] == sin[i] -- which is exactly how + # RoPE tables are built, each frequency written into both halves. + # + # This is worth the comment because the first version of this test varied + # the angle across all D=64 positions. The kernel still MATCHED THE ORACLE + # (max error 3.9e-3, inside the bound), and the norm assertion below failed + # anyway -- because with cos[i + 32] != cos[i] the operation being applied + # is not a rotation and has no reason to preserve anything. The test was + # wrong, not the kernel. A property assertion is only as good as the inputs + # that make the property true. + half = D // 2 + ang_half = (np.arange(T)[:, None] * 0.1 + np.arange(half)[None, :] * 0.02) + ang = np.concatenate([ang_half, ang_half], axis=1) + cos = np.cos(ang).astype(np.float32) + sin = np.sin(ang).astype(np.float32) + + y, _ = backend.run("rope_2d", [x, cos, sin], {}) + + assert y.shape == (T, H, D) + assert y.dtype == np.float16 + want = _oracle("rope_2d", [x, cos, sin], {}).astype(np.float16) + err = np.abs(y.astype(np.float32) - want.astype(np.float32)) + assert err.max() < 4e-3, f"max abs error {err.max()}" + + # THE PROPERTY: a rotation preserves the norm of each (i, i+D/2) pair. This + # holds for the split-half convention and FAILS for adjacent pairing, so it + # is an oracle-independent check on the one thing most likely to be wrong. + def pair_norms(arr): + a = arr.astype(np.float32) + return a[..., :half] ** 2 + a[..., half:] ** 2 + + assert np.allclose(pair_norms(y), pair_norms(x), rtol=5e-2, atol=5e-3), ( + "the split-half pair norms changed, so this is not a rotation of the " + "(i, i+D/2) pairs -- the likely cause is adjacent pairing" + ) + + +@sdk +def test_patchify_dispatches_and_matches_the_registry(backend): + """The encoder's first op: rank-4 fp32 in, fp32 out, eight scalars. + + THE ONLY TEST HERE THAT REACHES `dim:0:3`, and the only one with four attr + params. `patch`, `merge`, `grid_h` and `grid_w` cannot be recovered from the + shapes, so all four cross the wire -- and `merge` CHANGES THE ANSWER rather + than describing it, reordering the output rows into 2x2 spatial-merge-block + order. A dropped `merge` param gives raster order: right shape, right bytes, + wrong rows. + + A small shape, because patchify at the encoder's own (3,2,256,256) costs + about 2.0M cycles and this is a dispatch test, not a benchmark. + + THE SHAPE HAD TO BE CHOSEN CAREFULLY AND THE FIRST CHOICE WAS DEGENERATE. + With grid_h=4, grid_w=2, merge=2 the merge-block order is IDENTICAL to + raster order: Bw = grid_w/merge = 1, so bw is always 0 and + `((bh*Bw + bw)*merge + mh)*merge + mw` collapses to the raster index. The + kernel was correct and the reordering assertion below could not see it + either way. grid_h=6, grid_w=4 gives Bh=3, Bw=2 -- both greater than one, so + the two orders genuinely differ -- and they stay DIFFERENT from each other so + a grid transpose cannot pass either. merge=2 divides both, which the registry + requires. + """ + C, T, patch, merge = 3, 2, 3, 2 + grid_h, grid_w = 6, 4 + H, W = grid_h * patch, grid_w * patch + attrs = {"patch": patch, "merge": merge, "grid_h": grid_h, "grid_w": grid_w, + "temporal_patch": T} + rng = np.random.default_rng(14) + img = rng.standard_normal((C, T, H, W)).astype(np.float32) + + y, _ = backend.run("patchify", [img], attrs) + + assert y.shape == (grid_h * grid_w, C * T * patch * patch) + assert y.dtype == np.float32 + want = _oracle("patchify", [img], attrs).astype(np.float32) + # fp32 throughout and no arithmetic at all, so this must be BIT-EXACT. + assert np.array_equal(y, want), ( + f"patchify disagreed with the registry on " + f"{(y != want).sum()} of {y.size} elements. Pure data movement in fp32 " + f"has no rounding to blame." + ) + + # AND THE MERGE REORDERING SPECIFICALLY. Raster order is what a kernel that + # ignores `merge` produces; it is the same shape, so only the values differ. + x = img.reshape(C, T, grid_h, patch, grid_w, patch) + raster = x.transpose(2, 4, 0, 1, 3, 5).reshape(grid_h * grid_w, -1) + assert not np.array_equal(y, raster), ( + "the output is in raster order, so `merge` was ignored -- the " + "downstream merger is a pure reshape and needs 2x2 blocks" + ) + + +@sdk +def test_matmul_dispatches_and_matches_the_reference(backend): + """A batched fp16 matmul through the DSP batch path. + + Bn > 1 so a wrong batch stride cannot pass, and K is not a multiple of the + kernel's 64-wide accumulator block so the scalar tail runs. + """ + rng = np.random.default_rng(5) + Bn, M, K, N = 3, 8, 70, 128 + a = rng.standard_normal((Bn, M, K)).astype(np.float16) + b = rng.standard_normal((Bn, K, N)).astype(np.float16) + + y, _ = backend.run("matmul", [a, b], {}) + want = a.astype(np.float32) @ b.astype(np.float32) + + assert y.shape == want.shape, f"{y.shape} != {want.shape}" + assert np.max(np.abs(y - want)) < 1e-2 * max(1.0, float(np.max(np.abs(want)))) + + +@sdk +def test_matmul_reduces_over_k_and_not_over_a_transposed_operand(backend): + """Oracle-independent. Build B so that every column is a distinct constant: + then C[b,m,n] must equal n * sum(A[b,m,:]), which a kernel that read B as + [N,K] cannot reproduce for a non-square operand.""" + rng = np.random.default_rng(6) + # N IS A MULTIPLE OF 64 ON PURPOSE. kernels/matmul_fp16/kernel.c loads B + # rows with an ALIGNED vector read (`(const HVX_Vector *) brow` then + # `bv[i]`), so a row is only correctly aligned when N % 64 == 0. N=96 + # trips that and this test would fail for a reason that has nothing to do + # with the property it exists to check. The bug is real and is Task 6; + # both encoder matmul shapes use N=256 and N=64, so it does not affect + # dispatch. + Bn, M, K, N = 2, 4, 32, 128 + a = rng.standard_normal((Bn, M, K)).astype(np.float16) + b = np.tile(np.arange(N, dtype=np.float16), (Bn, K, 1)) + + y, _ = backend.run("matmul", [a, b], {}) + row_sums = a.astype(np.float32).sum(axis=2) # (Bn, M) + want = row_sums[:, :, None] * np.arange(N, dtype=np.float32) + + assert y.shape == (Bn, M, N) + assert np.max(np.abs(y - want)) < 1e-2 * max(1.0, float(np.max(np.abs(want)))) + + +@sdk +def test_matmul_is_correct_when_n_is_not_a_multiple_of_64(backend): + """N=96 is not a multiple of 64, so B's rows are not 128-byte aligned. + + The vectorised path used to load them with an ALIGNED read, so the first + nvec64*64 columns were computed from shifted data while the scalar tail + was correct. Regression guard for that fix; the gate now covers this + shape too (spec.json's N is 200). + """ + rng = np.random.default_rng(21) + Bn, M, K, N = 2, 4, 32, 96 + a = rng.standard_normal((Bn, M, K)).astype(np.float16) + b = rng.standard_normal((Bn, K, N)).astype(np.float16) + + y, _ = backend.run("matmul", [a, b], {}) + want = a.astype(np.float32) @ b.astype(np.float32) + assert np.max(np.abs(y - want)) < 1e-2 * max(1.0, float(np.max(np.abs(want)))) + + +# --------------------------------------------------------------------------- +# matmul_epilogue: fp16 activations, a q4_0 weight, an fp32 bias, and a string +# `act` attr that has to cross the wire as an int code. +# +# THE ORACLE TRAP: the registry's own `matmul_epilogue` reference multiplies by +# the FULL-PRECISION weight. The kernel multiplies by the q4_0-QUANTIZED +# weight, whose per-block rounding error dwarfs anything a kernel bug could +# add. So the expected value here is built from `dequantize_q4_0` of the SAME +# bytes handed to the kernel -- the only way left to compare is the +# arithmetic, not the quantization format. +# --------------------------------------------------------------------------- + + +@sdk +@pytest.mark.parametrize("act", ["none", "gelu_tanh", "gelu_erf"]) +def test_matmul_epilogue_dispatches_for_every_activation(backend, act): + """Bias then activation, against a reference built from the SAME q4_0 + bytes the kernel gets -- see the oracle trap above.""" + from hexlib.exec.quant import dequantize_q4_0, quantize_q4_0 + from hexlib.exec.runner import RawTensor + from hexlib.graph.ops import get + + rng = np.random.default_rng(11) + M, K, N = 12, 96, 160 + a = rng.standard_normal((M, K)).astype(np.float16) + w = rng.standard_normal((K, N)).astype(np.float32) + bias = rng.standard_normal((N,)).astype(np.float32) + + w_bytes = quantize_q4_0(w) + w_raw = RawTensor(dtype="q4_0", shape=w.shape, data=w_bytes) + + y, _ = backend.run("matmul_epilogue", [a, w_raw, bias], {"act": act}) + + w_eff = dequantize_q4_0(w_bytes, w.shape) + ref = a.astype(np.float32) @ w_eff + bias + want = ref if act == "none" else get(act).reference((ref,), {})[0] + + assert y.shape == (M, N) + assert np.max(np.abs(y - want)) < 1e-2 * max(1.0, float(np.max(np.abs(want)))) + + +@sdk +def test_matmul_epilogue_applies_bias_before_activation(backend): + """Oracle-independent: with gelu_erf and a large negative bias every output + is driven to ~0. Adding the bias AFTER the activation cannot produce that -- + the bias would still be visible in the result.""" + from hexlib.exec.quant import quantize_q4_0 + from hexlib.exec.runner import RawTensor + + rng = np.random.default_rng(12) + M, K, N = 8, 64, 64 + a = rng.standard_normal((M, K)).astype(np.float16) + w = rng.standard_normal((K, N)).astype(np.float32) + bias = np.full((N,), -50.0, dtype=np.float32) + + w_raw = RawTensor(dtype="q4_0", shape=w.shape, data=quantize_q4_0(w)) + y, _ = backend.run("matmul_epilogue", [a, w_raw, bias], {"act": "gelu_erf"}) + assert np.max(np.abs(y)) < 1.0, ( + "a large negative bias applied BEFORE gelu must collapse the output; " + f"max |y| = {float(np.max(np.abs(y)))} means bias came after the " + "activation" + ) diff --git a/hexlib/tests/test_encoder_dispatch_coverage.py b/hexlib/tests/test_encoder_dispatch_coverage.py new file mode 100644 index 0000000..5a72d94 --- /dev/null +++ b/hexlib/tests/test_encoder_dispatch_coverage.py @@ -0,0 +1,57 @@ +"""Every real-work op in the compiled encoder must select a kernel. + +WHY THIS EXISTS. "Has a gated kernel" is not "can be dispatched" -- a +RunnerSpec is what makes an op reachable on the DSP batch path, and this +project has published a wrong coverage number three times by counting kernel +directories instead. layernorm was gated and unreachable; softmax and rope_2d +repeated it the same week; matmul and matmul_epilogue repeated it again. + +Counting is what goes wrong, so this test does not count. It asks select() +about every step of the real compiled plan. +""" +import collections + +import pytest + +import hexlib.graph.opdefs # noqa: F401 -- registers the op definitions +from hexlib.exec.runner import select +from hexlib.graph.pipeline import compile_model +from hexlib.graph.plan import V75_VTCM_TOTAL_BYTES +from hexlib.models.qwen35 import qwen35_at +from hexlib.models.vit import build_vision_encoder + + +@pytest.fixture(scope="module") +def encoder_plan(): + graph = build_vision_encoder(qwen35_at(256)) + compiled = compile_model(graph, budget=V75_VTCM_TOTAL_BYTES) + assert not hasattr(compiled, "reason"), ( + f"compile: {getattr(compiled, 'detail', compiled)}" + ) + return compiled.plan + + +def test_every_non_reshape_step_selects_a_kernel(encoder_plan): + undispatchable = collections.Counter() + for step in encoder_plan.steps: + kind = step.op.kind + if kind == "reshape": + continue + try: + select(kind, dict(step.op.attrs or {})) + except Exception: + undispatchable[kind] += 1 + assert not undispatchable, ( + f"ops with no reachable kernel: {dict(undispatchable)}. A gated kernel " + f"is not enough -- each needs a RunnerSpec in hexlib/exec/runner.py." + ) + + +def test_the_plan_is_the_expected_size(encoder_plan): + """A guard on the guard: if the encoder shrank, the test above could pass + while covering almost nothing.""" + kinds = collections.Counter(s.op.kind for s in encoder_plan.steps) + assert len(encoder_plan.steps) == 308 + assert kinds["reshape"] == 49 + assert kinds["matmul"] == 24 + assert kinds["matmul_epilogue"] == 75 diff --git a/hexlib/tests/test_encoder_on_sim.py b/hexlib/tests/test_encoder_on_sim.py new file mode 100644 index 0000000..4fa96d8 --- /dev/null +++ b/hexlib/tests/test_encoder_on_sim.py @@ -0,0 +1,283 @@ +# hexlib/tests/test_encoder_on_sim.py +"""THE WHOLE ENCODER, end to end, with every kernel that exists running on the +Hexagon simulator's DSP batch path. + +WHAT THIS IS FOR. Everything else that touches the simulator drives ONE op. That +proves a kernel and its wiring; it does not prove the encoder. The failures this +catches are the ones that only exist between ops: + + * a kernel that is correct in isolation and wrong on the shape the encoder + actually hands it (the gate ran layernorm at R=4; the encoder needs R=256); + * a spec whose scalars are right for the harness's shape and wrong for the + graph's, which no single-op test at a hand-chosen shape can see; + * an op whose output feeds another kernel rather than a test assertion, so a + systematic error is only visible after it has propagated; + * a variant routed to the wrong kernel for SOME of its ops -- `transpose` is + two kernels and the encoder uses both, in the same graph, interleaved. + +HOW IT COMPARES. Twice through `interpreter.run` over the same compiled plan and +the same feeds: once with DSP backends for every kind that has a kernel, once +with none, so the second run is the op registry's numpy reference throughout. +The reference path is the one the plan executor was validated against, so a +disagreement is the DSP's -- and the comparison is over the encoder's declared +OUTPUTS, after every intermediate has passed through the DSP and back. + +AT A TINY CONFIG, DELIBERATELY, AND THE REASON IS NOT CONVENIENCE. Every call to +a DSP backend is one `hexagon-sim` launch -- process start, QuRT boot, run, tear +down. The 256x256 encoder is 259 real-work ops, so a full-size run is 259 +launches of a simulator that takes seconds each even before the work, plus +patchify at ~2.0M cycles. Hours, for a correctness signal that a 58-step graph +gives in minutes. What a tiny config does NOT cover is recorded in the test that +needs it: shape-dependent behaviour still belongs in the per-kernel gate and in +`test_dsp_sim.py`, which drives the encoder's real shapes one op at a time. + +THIS TEST GETS STRONGER ON ITS OWN. It asks `runner.SPECS` which kinds have +kernels rather than listing them, so registering a kernel moves ops onto the DSP +here with no edit. It also ASSERTS how many ops went to the DSP, so a spec that +silently stops matching -- a `requires` that no longer fits the graph, a variant +whose attrs drifted -- fails loudly instead of quietly falling back to the +reference and still passing. +""" +import os + +import numpy as np +import pytest + +import hexlib.graph.opdefs # noqa: F401 -- registers the op definitions +from hexlib import toolchain as tc +from hexlib.exec import dsp as dspmod +from hexlib.exec import interpreter +from hexlib.exec.quant import dequantize_q4_0, quantize_q4_0 +from hexlib.exec.runner import SPECS, select +from hexlib.graph.pipeline import compile_model +from hexlib.models.vit import VitConfig, build_vision_encoder + +HAS_SDK = os.path.isdir(tc.default_sdk_root()) +sdk = pytest.mark.skipif(not HAS_SDK, reason="Hexagon SDK not present") + +VTCM_BUDGET = 8 * 1024 * 1024 + + +def _tiny_cfg() -> VitConfig: + """The same tiny shape `test_models_vit.py` uses, with the encoder's REAL + dtypes: fp16 activations and q4_0 weights. The dtypes matter more than the + sizes here -- they are what decide which kernel each op selects and whether + the block-quantized staging path is exercised at all.""" + return VitConfig( + depth=2, + hidden_size=64, + num_heads=4, + intermediate_size=128, + patch_size=4, + temporal_patch_size=2, + in_channels=3, + spatial_merge_size=2, + out_hidden_size=32, + layernorm_eps=1e-6, + rope_theta=10000.0, + image_size=32, + act_dtype="fp16", + weight_dtype="q4_0", + ) + + +def _compiled(): + """`compile_model`, NOT `compile_graph` + `Compiled(...)` by hand. + + `compile_graph` returns the plan alone and its docstring says why that is not + enough: "fusion creates ops that appear in no graph the caller holds, so the + plan on its own is not executable." Pairing a plan with the PRE-fusion graph + builds a `Compiled` that validates fine -- every tensor a step names is + declared -- and then fails at run time, because `interpreter.run` resolves + backends by walking `graph.ops` while `_run_op` looks them up by the PLAN + step's kind. `matmul_epilogue` exists only after fusion, so the lookup raises + `KeyError: 'matmul_epilogue'` on the third step. Ask for the post-fusion pair. + """ + graph = build_vision_encoder(_tiny_cfg()) + assert not hasattr(graph, "reason"), f"graph: {getattr(graph, 'detail', graph)}" + compiled = compile_model( + graph, budget=VTCM_BUDGET, + order_policy="min_peak", alloc_policy="largest_first", + ) + assert not hasattr(compiled, "reason"), ( + f"compile: {getattr(compiled, 'detail', compiled)}" + ) + return compiled, compiled.graph, compiled.plan + + +def _feeds(graph, plan, seed=3): + """Every graph input and every const, in the shapes the graph declares. + + Consts are RANDOM rather than zero or one. A zero weight makes every matmul + return zeros, which agrees with any reference for any reason; a weight of one + makes a transposed operand undetectable. Neither would fail if the DSP were + wrong. + + WEIGHT CONSTS ARE PRE-QUANTIZED. `matmul_epilogue`'s weight input (index 1, + see `hexlib.graph.opdefs.fused._infer`) crosses the wire as q4_0 -- `dsp.py` + quantizes it on the fly, on the way to the simulator (see + `interpreter_backends`'s docstring). The reference path never quantizes + anything, so comparing it against the fp32 weight measures the q4_0 FORMAT's + own ~1/16-of-range error, not the kernel. Running the same round trip here, + on the feed, makes both paths multiply by the identical q4_0-quantized + values: dsp.py's on-the-wire quantization of an already-quantized array + reproduces the same bytes, so what is left to compare is arithmetic and fp16 + rounding -- what the tolerance below is actually for. + """ + weight_names = { + step.op.inputs[1] + for step in plan.steps + if step.op.kind == "matmul_epilogue" + } + rng = np.random.default_rng(seed) + feeds = {} + for name in list(graph.inputs) + [ + t.name for t in graph.tensors.values() if t.const + ]: + spec = graph.tensor(name) + w = (rng.standard_normal(spec.shape) * 0.5).astype(np.float32) + if name in weight_names: + w = dequantize_q4_0(quantize_q4_0(w), w.shape).astype(w.dtype) + feeds[name] = w + return feeds + + +def _dispatchable(plan): + """(ops that will go to the DSP, ops that will fall back), by kind.""" + on_dsp, fallback = {}, {} + for step in plan.steps: + kind = step.op.kind + if kind == "reshape": + continue # a view; the interpreter needs no kernel + try: + name, _ = select(kind, dict(step.op.attrs)) + except (KeyError, ValueError): + fallback[kind] = fallback.get(kind, 0) + 1 + else: + on_dsp[name] = on_dsp.get(name, 0) + 1 + return on_dsp, fallback + + +def test_the_tiny_encoder_plan_is_the_shape_this_test_assumes(): + """Runs with no SDK. If the graph or the pass pipeline changes what this + encoder compiles to, the SDK-gated test below would start measuring something + else while still passing -- so the shape is pinned here, cheaply, where CI + can see it.""" + _, graph, plan = _compiled() + on_dsp, fallback = _dispatchable(plan) + total = sum(on_dsp.values()) + sum(fallback.values()) + + assert len(plan.steps) > 40, f"only {len(plan.steps)} plan steps" + assert total > 30, f"only {total} real-work ops" + # Both transpose variants must appear, since routing between them in one + # graph is a thing only this test exercises. + assert on_dsp.get("transpose", 0) > 0, "no perm(1,0,2) transpose in the plan" + assert on_dsp.get("transpose_hd", 0) > 0, ( + "no perm(0,2,1) transpose in the plan -- the variant-routing claim below " + "would be vacuous" + ) + # NO GAPS LEFT. matmul and matmul_epilogue got RunnerSpecs on 2026-08-12, + # so this goes from "no UNEXPECTED fallback" to "no fallback at all" -- + # which is the property that makes the SDK test below an end-to-end DSP + # run rather than a partial one. The `known_gaps` set is gone rather than + # emptied: an empty allowlist and a plain emptiness check are the same + # assertion, and keeping both leaves a dead name for the next reader to + # wonder about. + assert not fallback, ( + f"these ops fell back to the numpy reference instead of dispatching: " + f"{dict(fallback)}. Every real-work op is supposed to reach the DSP; " + f"either a kernel regressed out of SPECS or the graph grew a new kind." + ) + + +@sdk +def test_the_whole_encoder_agrees_with_the_reference_with_every_kernel_on_the_dsp(): + """THE END-TO-END RUN. + + Both paths execute the same compiled plan over the same feeds. The DSP path + routes every op with a kernel through `hexagon-sim`; the reference path uses + the op registry throughout. The encoder's declared outputs must agree. + + The tolerance is not tight and should not be: the DSP path computes in fp16 + while the reference accumulates in the interpreter's compute dtype, and the + error compounds across a 2-layer encoder -- every layernorm, softmax and + rotation narrows to fp16 and feeds the next op. What this test is for is + catching a WRONG op, not measuring the last ULP; per-op accuracy is pinned + at the encoder's real shapes by `test_dsp_sim.py` and by each kernel's gate, + both of which compare against a reference at a single op where the bound can + actually be tight. + + So the assertions are: finite, right shape, and close in a relative sense + that a genuinely wrong kernel cannot satisfy -- plus a correlation floor, + which is the assertion that survives a rescaling and would catch an output + that is the right size and the wrong content. + """ + compiled, graph, plan = _compiled() + on_dsp, fallback = _dispatchable(plan) + feeds = _feeds(graph, plan) + + sim = dspmod.DspSimBackend( + sorted({os.path.basename(s.kernel_dir) for s in SPECS.values()}), + os.environ.get("HEXLIB_SIM_WORK") or _work_dir(), + ) + backends = dspmod.interpreter_backends(sim) + + dsp_report = interpreter.run(compiled, feeds, backends=backends) + assert not hasattr(dsp_report, "reason"), ( + f"the DSP run failed: {getattr(dsp_report, 'reason', '')} — " + f"{getattr(dsp_report, 'detail', '')}" + ) + ref_report = interpreter.run(compiled, feeds) + assert not hasattr(ref_report, "reason"), ( + f"the reference run failed: {getattr(ref_report, 'detail', ref_report)}" + ) + + # THE COVERAGE CLAIM, ASSERTED. Without this, a spec that stopped matching + # would fall back to the reference and this test would compare the reference + # against itself and pass. + used = dsp_report.backend_used + supplied = sorted(k for k, v in used.items() if v == "supplied") + expected = sorted({SPECS[n].kind for n in on_dsp}) + assert supplied == expected, ( + f"kinds actually sent to the DSP were {supplied}, expected {expected}" + ) + assert len(supplied) >= 6, f"only {len(supplied)} kinds ran on the DSP" + + outs = list(graph.outputs) + assert outs, "the encoder graph declares no outputs" + for name in outs: + got = np.asarray(_result(dsp_report, name), dtype=np.float64) + want = np.asarray(_result(ref_report, name), dtype=np.float64) + assert got.shape == want.shape + assert np.all(np.isfinite(got)), f"{name}: DSP produced non-finite values" + + denom = max(float(np.abs(want).max()), 1e-6) + rel = float(np.abs(got - want).max()) / denom + assert rel < 0.05, ( + f"{name}: max relative error {rel:.4f} between the DSP path and the " + f"reference over the whole encoder" + ) + # Survives a rescaling, unlike the bound above: a kernel returning a + # scaled or permuted version of the right answer fails here. + gv, wv = got.reshape(-1), want.reshape(-1) + if gv.size > 1 and wv.std() > 0: + corr = float(np.corrcoef(gv, wv)[0, 1]) + assert corr > 0.99, f"{name}: correlation with the reference {corr:.4f}" + + +def _result(report, name): + """The named output, from whichever attribute the report carries it in.""" + for attr in ("outputs", "results", "ddr"): + table = getattr(report, attr, None) + if isinstance(table, dict) and name in table: + return table[name] + raise AssertionError( + f"ExecReport has no output {name!r}; attributes are " + f"{[a for a in dir(report) if not a.startswith('_')]}" + ) + + +def _work_dir(): + import tempfile + + return tempfile.mkdtemp(prefix="hexlib_encoder_sim_") diff --git a/hexlib/tests/test_exec_dsp_host.py b/hexlib/tests/test_exec_dsp_host.py new file mode 100644 index 0000000..cb9556c --- /dev/null +++ b/hexlib/tests/test_exec_dsp_host.py @@ -0,0 +1,369 @@ +# hexlib/tests/test_exec_dsp_host.py +"""Everything `DspSimBackend.run()` decides BEFORE and AFTER the one simulator +launch, with the launch itself replaced by a fake. + +WHY A SEPARATE FILE FROM test_dsp_sim.py. That file is the acceptance gate: it +builds the skel, the QuRT-hosted `.so` and the sim configs and then launches +`hexagon-sim` once per test -- minutes, and an SDK. None of the decisions +checked here need any of that, and every one of them is a decision whose +failure mode is a CORRECTLY-SHAPED WRONG ANSWER rather than a crash: + + * dispatching an op to a kernel that implements a different permutation + (`requires`, enforced on the host because `hexlib_args` carries no field + for a perm at all -- see genentry.py's own note); + * reading a PREVIOUS call's `hexlib_out.bin` as this call's result; + * accepting a response that answers for a different op kind, or for a + different number of ops, than the batch asked about; + * silently ignoring an extra input array. + +Each of those returns plausible values with an OK status, so only a test that +inspects the host's own bookkeeping can catch it. Replacing `run_sim` is what +makes that testable at all: it lets a run be given a deliberately stale, absent, +or mismatched artifact, which a real simulator would never produce on demand. + +WHAT THIS FILE DOES NOT PROVE. That the real simulator writes the files this +fake writes -- `simhost.c` does (it writes `hexlib_rsp.bin` on every invoke that +returns, and `hexlib_out.bin` only when the batch status is OK, and returns +`rh.status == HEXLIB_DSP_OK ? 0 : 1`), and test_dsp_sim.py is what checks it end +to end. This file checks what `run()` does with those artifacts. +""" +import os +import struct + +import numpy as np +import pytest + +from hexlib.exec import dsp as dspmod +from hexlib.runtime import wire +from hexlib.runtime.genentry import KIND_ID + +OK = wire.STATUS["OK"] + + +def _backend(work_dir): + """A `DspSimBackend` with no artifacts built. + + `__init__` compiles the skel archive, links the QuRT-hosted `.so` and + writes the sim configs; none of that is reachable from the host-side + decisions under test. Every attribute `run()` reads is set explicitly here, + so if `run()` grows a dependency on another one this raises AttributeError + rather than quietly skipping a check. + """ + b = object.__new__(dspmod.DspSimBackend) + b.work_dir = str(work_dir) + b.sdk_root = os.path.join(str(work_dir), "no-such-sdk") # run_sim is faked + os.makedirs(b.work_dir, exist_ok=True) + return b + + +def _rsp(results, status=OK, n_ops=None): + """A batch response blob, packed with `wire.py`'s OWN format strings rather + than a retyped copy of them -- so this helper cannot drift from the format + `unpack_response` reads. `n_ops` defaults to `len(results)`; passing it + explicitly is how the "claims more results than it carries" case is built. + """ + raw = struct.pack( + wire._RSP_HDR, wire.BATCH_MAGIC, wire.BATCH_VERSION, status, + len(results) if n_ops is None else n_ops, 886, 75, 0, + ) + for kind, st, cycles in results: + raw += struct.pack(wire._RESULT, kind, st, cycles) + return raw + + +def _scale_buffer(x, factor): + """The whole shared rpcmem buffer as `simhost.c` dumps it: the input + payload, zero-padded to the 128-byte boundary `run()` aligns the output to, + then the output region. Built from `dspmod._align_up` rather than a literal + so it tracks the backend's own alignment rule.""" + y = (x.astype(np.float32) * factor).astype(np.float16) + in_end = dspmod._align_up(x.nbytes) + return x.tobytes().ljust(in_end, b"\x00") + y.tobytes(), y + + +class _FakeSim: + """One stand-in for `dsp.run_sim`. Records every launch, optionally writes + an output and/or a response file, and returns whatever `SimHostResult` the + test asks for -- including the combination `simhost.c` produces when it + prints an OK invoke line and then dies before writing a file.""" + + def __init__(self, work_dir, out=None, rsp=None, status=OK, exit_code=0): + self.work_dir = str(work_dir) + self.out = out + self.rsp = rsp + self.status = status + self.exit_code = exit_code + self.launches = 0 + + def __call__(self, work_dir, extra_args=(), sdk_root=None): + self.launches += 1 + if self.out is not None: + with open(os.path.join(self.work_dir, dspmod.OUT_NAME), "wb") as f: + f.write(self.out) + if self.rsp is not None: + with open(os.path.join(self.work_dir, dspmod.RSP_NAME), "wb") as f: + f.write(self.rsp) + return dspmod.SimHostResult( + status=self.status, cycles=886, arch=75, vtcm=8388608, + stdout="SIMHOST fake", exit_code=self.exit_code, + ) + + +class _NeverLaunches: + """A `run_sim` replacement that fails if it is ever called. Used by every + test whose claim is that a request is refused BEFORE the simulator runs: + asserting only that an exception was raised would also pass if the refusal + happened afterwards, on the wrong grounds.""" + + def __call__(self, *a, **kw): + raise AssertionError( + "the simulator was launched for a request that must be refused on " + "the host, before anything is packed" + ) + + +# --- F1: `requires` is enforced on the host, because nothing else can --------- + + +def test_run_refuses_a_perm_NO_kernel_implements(tmp_path, monkeypatch): + """THE FINDING, with the example moved because the old one got a kernel. + + A transpose op whose perm no kernel implements has a DIFFERENT output shape, + which `_out_shape` computes from the REQUESTED perm -- so the byte count + matches whatever kernel it lands on, the status is OK, and the caller gets an + attention layout with the wrong permutation that every downstream shape check + accepts. + + The DSP cannot catch this: `hexlib_args` has no field carrying a + permutation (genentry.py emits an honest comment instead of a check that + could not fail). So the host is the only place it can be refused, and + `hexlib/exec/hexagon.py` has always done so -- this path did not. + + WHY THE PERM CHANGED. This used to pass perm (0,2,1), which was then + unimplemented. `kernels/transpose_hd_fp16` implements it now, so that op is + legitimately accepted and `runner.select` routes it to the second variant -- + the test was asserting the absence of a kernel, not the guard. (2,1,0) is a + real permutation of three axes that no kernel serves, so the guard is still + the only thing standing between this call and a wrong answer. The + companion below checks the (0,2,1) op is now ROUTED rather than refused, + which is what stops this pair from being weakened in the other direction. + """ + monkeypatch.setattr(dspmod, "run_sim", _NeverLaunches()) + b = _backend(tmp_path) + x = np.zeros((4, 3, 2), dtype=np.float16) + with pytest.raises(ValueError, match=r"perm"): + b.run("transpose", [x], {"perm": (2, 1, 0)}) + assert not os.listdir(tmp_path), ( + "the batch must not even be written for an op no kernel can serve" + ) + + +def test_the_two_transpose_perms_route_to_their_own_kernels(tmp_path, monkeypatch): + """The other half: a perm that IS implemented must reach its OWN kernel. + + One op kind, two kernels, and the wire carries no perm -- so the variant is + resolved on the host by `runner.select` and then named on the wire by its own + `KIND_ID`. Sending KIND_ID["transpose"] for a perm(0,2,1) op would dispatch + it to the perm(1,0,2) kernel: right shape, right status, wrong answer, and + nothing downstream could tell. So this asserts the mapping directly rather + than through a run, because the mapping is the whole mechanism. + """ + from hexlib.exec.runner import select + from hexlib.runtime.genentry import KIND_ID + + th_name, th_spec = select("transpose", {"perm": (1, 0, 2)}) + hd_name, hd_spec = select("transpose", {"perm": (0, 2, 1)}) + + assert th_spec.kernel_dir == "kernels/transpose_th_fp16" + assert hd_spec.kernel_dir == "kernels/transpose_hd_fp16" + assert KIND_ID[th_name] != KIND_ID[hd_name], ( + "both transpose variants would go on the wire as the same kind id, so " + "the DSP would run one kernel for both perms" + ) + + +def test_run_refuses_a_4d_input_to_the_3d_transpose_kernel(tmp_path, monkeypatch): + """The variant: the entry passes T,H,D from `ne[0][0..2]` and ignores + `ne[0][3]`, so a 4-D input moves a fraction of its elements and reports OK. + A 4-D op cannot have perm (1,0,2) (a permutation names every axis), so the + same host check refuses it -- which is the point: the check is on the + ATTRIBUTE the caller supplied, so it covers shapes the kernel never + considered.""" + monkeypatch.setattr(dspmod, "run_sim", _NeverLaunches()) + b = _backend(tmp_path) + x = np.zeros((2, 4, 3, 2), dtype=np.float16) + with pytest.raises(ValueError, match=r"perm"): + b.run("transpose", [x], {"perm": (1, 0, 2, 3)}) + + +def test_run_refuses_a_cast_to_a_dtype_the_kernel_does_not_produce(tmp_path, monkeypatch): + """THE SECOND FINDING, AND WHY THE DSP-SIDE CHECK CANNOT COVER IT. + `cast` is a general op kind; this kernel only does fp32 -> fp16. The + generated entry does check `a->dtype[out]`, but `run()` stamps that field + from `spec.out_dtype` -- so on the wire it compares the spec against + itself and passes by construction, whatever the caller asked for. Only a + check against the CALLER'S OWN attr can fail here, and that check lives on + the host.""" + monkeypatch.setattr(dspmod, "run_sim", _NeverLaunches()) + b = _backend(tmp_path) + x = np.zeros(16, dtype=np.float32) + with pytest.raises(ValueError, match=r"dtype"): + b.run("cast", [x], {"dtype": "fp32"}) + + +def test_the_permutation_the_kernel_does_implement_still_runs(tmp_path, monkeypatch): + """The control. A guard that refused everything would satisfy the two tests + above, so the accepted case must be shown to reach the simulator and come + back with the reordered shape.""" + b = _backend(tmp_path) + x = np.arange(2 * 3 * 4, dtype=np.float16).reshape(2, 3, 4) + in_end = dspmod._align_up(x.nbytes) + moved = np.transpose(x, (1, 0, 2)) + buf = x.tobytes().ljust(in_end, b"\x00") + moved.tobytes() + fake = _FakeSim(tmp_path, out=buf, + rsp=_rsp([(KIND_ID["transpose"], OK, 886)])) + monkeypatch.setattr(dspmod, "run_sim", fake) + + y, stats = b.run("transpose", [x], {"perm": (1, 0, 2)}) + assert fake.launches == 1 + assert y.shape == (3, 2, 4) + assert np.array_equal(y, moved) + assert stats.calls == 1 and stats.cycles == 886 + + +def test_scale_round_trips_through_the_faked_launch(tmp_path, monkeypatch): + """The other control: an op with no `requires` at all is unaffected, and + the output is sliced at the aligned offset the batch declared.""" + b = _backend(tmp_path) + x = np.arange(37, dtype=np.float16) + buf, expect = _scale_buffer(x, 0.5) + fake = _FakeSim(tmp_path, out=buf, rsp=_rsp([(KIND_ID["scale"], OK, 886)])) + monkeypatch.setattr(dspmod, "run_sim", fake) + + y, _ = b.run("scale", [x], {"factor": 0.5}) + assert np.array_equal(y, expect) + + +# --- F3: a stale artifact is not this call's result --------------------------- + + +def test_a_stale_output_is_not_read_as_this_calls_result(tmp_path, monkeypatch): + """THE FINDING. One backend, two calls. Call 1 wrote `hexlib_out.bin`. Call + 2's simulator prints its `SIMHOST invoke ... status=1` line and then dies + before `fopen("hexlib_out.bin","wb")`. Nothing deleted the old file, so + `run()` sliced call 1's buffer at the same offset and returned it as call + 2's answer -- with `res.exit_code` sitting unread in the result.""" + b = _backend(tmp_path) + x1 = np.full(37, 4.0, dtype=np.float16) + stale, stale_y = _scale_buffer(x1, 0.5) + with open(tmp_path / dspmod.OUT_NAME, "wb") as f: + f.write(stale) + with open(tmp_path / dspmod.RSP_NAME, "wb") as f: + f.write(_rsp([(KIND_ID["scale"], OK, 886)])) + + # Call 2: an OK status line, then death. Writes nothing. + monkeypatch.setattr(dspmod, "run_sim", + _FakeSim(tmp_path, out=None, rsp=None, exit_code=1)) + x2 = np.full(37, 1.0, dtype=np.float16) + assert stale_y[0] == 2.0 # call 1's values: what must never be returned here + with pytest.raises(dspmod.DspSimError): + b.run("scale", [x2], {"factor": 0.5}) + assert not os.path.exists(tmp_path / dspmod.OUT_NAME), ( + "the previous call's output must be GONE before the launch, not merely " + "unread -- otherwise the next code path that reads it inherits the bug" + ) + + +def test_a_stale_response_is_not_read_as_this_calls_result(tmp_path, monkeypatch): + """The same hazard one file over. `hexlib_rsp.bin` is what says WHICH op + answered and with what status; a leftover one from a previous call would + vouch for a launch that never wrote anything.""" + b = _backend(tmp_path) + with open(tmp_path / dspmod.RSP_NAME, "wb") as f: + f.write(_rsp([(KIND_ID["scale"], OK, 886)])) + x = np.arange(37, dtype=np.float16) + buf, _ = _scale_buffer(x, 0.5) + monkeypatch.setattr(dspmod, "run_sim", _FakeSim(tmp_path, out=buf, rsp=None)) + with pytest.raises(dspmod.DspSimError, match=dspmod.RSP_NAME): + b.run("scale", [x], {"factor": 0.5}) + + +def test_a_nonzero_exit_code_is_not_a_success(tmp_path, monkeypatch): + """`simhost.c` returns 0 if and only if the batch status was + HEXLIB_DSP_OK, so an OK status line together with a nonzero exit means the + process died after printing it. The result carried `exit_code` and nothing + read it.""" + b = _backend(tmp_path) + x = np.arange(37, dtype=np.float16) + buf, _ = _scale_buffer(x, 0.5) + monkeypatch.setattr(dspmod, "run_sim", _FakeSim( + tmp_path, out=buf, rsp=_rsp([(KIND_ID["scale"], OK, 886)]), exit_code=1)) + with pytest.raises(dspmod.DspSimError, match=r"exit"): + b.run("scale", [x], {"factor": 0.5}) + + +# --- F2: the response must answer for the op that was asked ------------------ + + +def test_a_response_for_a_different_kind_is_refused(tmp_path, monkeypatch): + """THE RENUMBER SCENARIO, host side. If the DSP's dispatch table and this + host's `KIND_ID` ever disagree, an op is served by the wrong kernel: the + element count matches, both pointers are non-null, the status is OK. The + response carries the kind the DSP actually dispatched (`skel_dispatch.c` + fills `results[i].kind = op.kind` before the lookup), so comparing it with + what was packed is the one host-side check that can see the drift.""" + b = _backend(tmp_path) + x = np.arange(37, dtype=np.float16) + buf, _ = _scale_buffer(x, 0.5) + monkeypatch.setattr(dspmod, "run_sim", _FakeSim( + tmp_path, out=buf, rsp=_rsp([(KIND_ID["transpose"], OK, 886)]))) + with pytest.raises(dspmod.DspSimError, match=r"kind"): + b.run("scale", [x], {"factor": 0.5}) + + +def test_a_response_for_a_different_number_of_ops_is_refused(tmp_path, monkeypatch): + b = _backend(tmp_path) + x = np.arange(37, dtype=np.float16) + buf, _ = _scale_buffer(x, 0.5) + monkeypatch.setattr(dspmod, "run_sim", _FakeSim( + tmp_path, out=buf, + rsp=_rsp([(KIND_ID["scale"], OK, 886), (KIND_ID["scale"], OK, 12)]))) + with pytest.raises(dspmod.DspSimError, match=r"1 op|n_ops"): + b.run("scale", [x], {"factor": 0.5}) + + +def test_a_per_op_failure_under_an_ok_batch_status_is_refused(tmp_path, monkeypatch): + """The batch header's status and the per-op status are two different + fields. `wire.BatchResponse.ok` already requires both; `run()` only ever + looked at the one parsed off stdout.""" + b = _backend(tmp_path) + x = np.arange(37, dtype=np.float16) + buf, _ = _scale_buffer(x, 0.5) + monkeypatch.setattr(dspmod, "run_sim", _FakeSim( + tmp_path, out=buf, + rsp=_rsp([(KIND_ID["scale"], wire.STATUS["ERR_REQUIRES"], 0)]))) + with pytest.raises(dspmod.DspSimError, match=r"ERR_REQUIRES"): + b.run("scale", [x], {"factor": 0.5}) + + +# --- Minor: an arity mismatch is refused, not truncated ---------------------- + + +def test_more_input_arrays_than_the_spec_declares_is_refused(tmp_path, monkeypatch): + """`zip(arrays, spec.inputs)` stops at the shorter one, so an `add` op + mis-wired with three inputs computed a+b, ignored c, and returned the right + shape with no error. `RunnerSpec.payload` raises on the other transport for + exactly this.""" + monkeypatch.setattr(dspmod, "run_sim", _NeverLaunches()) + b = _backend(tmp_path) + a = np.zeros(8, dtype=np.float16) + with pytest.raises(dspmod.DspSimError, match=r"3|inputs"): + b.run("add", [a, a, a], {}) + + +def test_fewer_input_arrays_than_the_spec_declares_is_refused(tmp_path, monkeypatch): + monkeypatch.setattr(dspmod, "run_sim", _NeverLaunches()) + b = _backend(tmp_path) + a = np.zeros(8, dtype=np.float16) + with pytest.raises(dspmod.DspSimError, match=r"1|inputs"): + b.run("add", [a], {}) diff --git a/hexlib/tests/test_genentry_entry_probe.py b/hexlib/tests/test_genentry_entry_probe.py new file mode 100644 index 0000000..820b02d --- /dev/null +++ b/hexlib/tests/test_genentry_entry_probe.py @@ -0,0 +1,707 @@ +# hexlib/tests/test_genentry_entry_probe.py +"""BEHAVIOURAL test of the GENERATED DSP entry points: compile them with a host +C compiler, call them with a hand-built `hexlib_args`, and check what they +return. + +WHY THIS EXISTS RATHER THAN MORE SOURCE ASSERTIONS. test_runtime_genentry.py +checks the emitted TEXT ("this `if` is present, in this order"). Text cannot +answer the only question that matters about a guard: does it fire? The output +dtype check has been emitted since the generator was written and CANNOT fire +through `hexlib/exec/dsp.py`, because that serializer fills the field it reads +from the same `spec.out_dtype` the check was generated from -- a check that +compares a constant with itself. So "the `if` is there" and "the guard works" +are genuinely different claims here, and the new per-input dtype checks needed +the second one. + +The proof is offline. Same recipe as test_wire_struct_layout.py, +test_session_arch_decode.py and test_coherency_lane_classification.py: emit a +small C program, compile it with a host `cc`, RUN it, and compare real returned +values against what Python expects. No SDK, no simulator, no device -- and +`hexagon-sim` could not answer this question much better anyway, since driving a +deliberately-wrong dtype through it needs a hand-built blob either way. + +WHAT IS REAL HERE AND WHAT IS A STAND-IN. Real: the entry source, verbatim from +`genentry.emit_entry`, and `hexlib_dsp.h` itself (so `hexlib_args`'s real layout, +the real `dtype[]` array, and the real status enum). A stand-in: `kernel_api.h`, +written below, which typedefs `hexlib_hf` as `unsigned short` and declares the +three kernel prototypes. The real per-kernel headers typedef it as `__fp16`, +which mainstream x86 gcc does not accept -- and nothing here depends on the +type's arithmetic, only on the entry's control flow before the call. The kernels +themselves are recording stubs, because "the kernel was not called" is half of +every assertion below. + +TWO PROBES, AND THE SECOND ONE EXISTS BECAUSE THE FIRST'S CLAIM WAS HALF TRUE. +`test_buffers_are_packed_sources_then_destinations` says the src-then-dst +contract is "PINNED FROM BOTH SIDES AT ONCE". It was not: only `genentry`'s +generated entries were compiled, so it pinned `genentry`'s `out_idx = n_in` and +nothing else. `skel_dispatch.c`'s fill loop -- the OTHER side, and the only +statement anywhere that `a->buf[]` holds sources followed by destinations -- was +never built here, and INVERTING IT (destinations packed first) left this file at +6 passed. The claim was written by the same hand that wrote the test and it was +simply wrong. + +So there is now a second fixture, `both_sides`, that compiles the REAL +`skel_dispatch.c` and `skel_bufs.c` together with a generated entry and drives a +REAL batch blob from `hexlib.runtime.wire.pack_batch` through +`hexlib_dispatch_batch`. Nothing about the ordering is asserted textually: the +recording kernel reports which ADDRESS it received as its input and which as its +output, and those addresses are checked against the fd-plus-offset arithmetic the +batch declared. Inverting either side -- the fill loop or `out_idx` -- makes the +kernel receive the other tensor and fails it. That is what pinning both sides at +once means, and it also makes this the one offline test that drives the host +serializer, the DSP-side fd-to-address mapping, the dispatcher and a generated +entry in a single run. + +THE STANDING-IN GOES ONE LEVEL FURTHER FOR THAT SECOND PROBE, and here is +exactly how far. Stubbed: `HAP_farf.h` (FARF discards its arguments), +`HAP_perf.h` (a monotonic counter), `HAP_mem.h` (HAP_mmap2 returns a small +distinct fake address per fd, so `hexlib_tensor.data` -- a uint32_t -- can hold +it), and `hexlib_vtcm_acquire`/`hexlib_vtcm_release`, which this probe is not +about. Real and compiled from the repo: skel_dispatch.c, skel_bufs.c, +hexlib_dsp.h, skel_internal.h, the generated entry. The addresses are never +dereferenced -- the kernel stub records the pointer and returns -- so a fake +mapping is enough to make identity meaningful, which is the only property under +test. +""" +import pathlib +import re +import shutil +import struct +import subprocess + +import pytest + +from hexlib.exec import runner as rn +from hexlib.runtime import genentry as ge +from hexlib.runtime import wire +from hexlib.runtime.wire import DTYPE_ID, STATUS + +SKEL = pathlib.Path("hexlib/runtime/skel") + +HOST_CC = shutil.which("cc") or shutil.which("gcc") or shutil.which("clang") +needs_cc = pytest.mark.skipif( + HOST_CC is None, + reason=( + "no host C compiler found (tried: cc, gcc, clang); the generated " + "entries' dtype guards are then covered only by the source assertions " + "in test_runtime_genentry.py, which can see that an `if` was emitted " + "but not that it fires. Install a host C compiler to restore this." + ), +) + +# A stand-in kernel_api.h -- see the module docstring on why hexlib_hf is not +# __fp16 here. +KERNEL_API_H = """\ +#ifndef PROBE_KERNEL_API_H +#define PROBE_KERNEL_API_H +typedef unsigned short hexlib_hf; +void scale_fp16(const hexlib_hf *x, hexlib_hf *y, int n, float factor); +void cast_f32_f16(const float *x, hexlib_hf *y, int n); +void add_fp16(const hexlib_hf *a, const hexlib_hf *b, hexlib_hf *y, int n); +#endif +""" + +PROBE_C = """\ +#include "hexlib_dsp.h" +#include "kernel_api.h" +#include +#include + +/* Recording stubs. "the kernel was NOT called" is half of every assertion. */ +static int g_calls; +static int g_n; +static float g_factor; + +/* WHICH BUFFER EACH ARGUMENT ACTUALLY WAS. The src-then-dst packing contract is + * stated in exactly one place -- skel_dispatch.c's fill loop -- and mirrored by + * genentry's `out_idx = n_in`; nothing checked that the two agree. Recording the + * pointers turns an "outputs first" refactor from a silent swap into a failure. */ +static const void *g_in0; +static const void *g_in1; +static const void *g_out; + +void scale_fp16(const hexlib_hf *x, hexlib_hf *y, int n, float factor) { + g_calls++; g_n = n; g_factor = factor; g_in0 = x; g_out = y; +} +void cast_f32_f16(const float *x, hexlib_hf *y, int n) { + g_calls++; g_n = n; g_in0 = x; g_out = y; +} +void add_fp16(const hexlib_hf *a, const hexlib_hf *b, hexlib_hf *y, int n) { + g_calls++; g_n = n; g_in0 = a; g_in1 = b; g_out = y; +} + +extern int scale_fp16_entry(const hexlib_args *); +extern int cast_f32_f16_entry(const hexlib_args *); +extern int add_fp16_entry(const hexlib_args *); + +static unsigned char b0[4096], b1[4096], b2[4096]; +static float params[4] = { 0.125f, 0.0f, 0.0f, 0.0f }; + +/* Every buffer supplied, every extent 17 (so the derived n is checkable and is + * not 0 by accident), params holding the scale factor. Each case then breaks + * exactly one thing. */ +static void base(hexlib_args *a, unsigned int n_buf) { + memset(a, 0, sizeof(*a)); + a->n_buf = n_buf; + a->buf[0] = b0; a->buf[1] = b1; a->buf[2] = b2; + for (unsigned int i = 0; i < HEXLIB_MAX_BUFS; i++) { + a->ne[i][0] = 17; a->ne[i][1] = 1; a->ne[i][2] = 1; a->ne[i][3] = 1; + } + a->params = params; +} + +/* putchar(10) emits the newline: this C is carried inside a Python + * string literal, so an escape sequence here round-trips badly. */ +static void report_order(const char *label, int ok) { + printf("%s=%d", label, ok ? 1 : 0); + putchar(10); +} + +static void report(const char *label, int rc) { + printf("case=%s rc=%d calls=%d n=%d\\n", label, rc, g_calls, g_n); +} + +int main(void) { + hexlib_args a; + int rc; + + g_calls = 0; g_n = -1; + base(&a, 2); a.dtype[0] = ID_FP16; a.dtype[1] = ID_FP16; + rc = scale_fp16_entry(&a); + report("scale_ok", rc); + report_order("scale_order", + g_in0 == (const void *) b0 && g_out == (const void *) b1); + printf("factor_ok=%d\\n", g_factor == 0.125f ? 1 : 0); + + g_calls = 0; g_n = -1; + base(&a, 2); a.dtype[0] = ID_FP32; a.dtype[1] = ID_FP16; + rc = scale_fp16_entry(&a); + report("scale_input_fp32", rc); + + g_calls = 0; g_n = -1; + base(&a, 2); a.dtype[0] = ID_FP16; a.dtype[1] = ID_FP32; + rc = scale_fp16_entry(&a); + report("scale_output_fp32", rc); + + g_calls = 0; g_n = -1; + base(&a, 2); a.dtype[0] = ID_FP32; a.dtype[1] = ID_FP16; + rc = cast_f32_f16_entry(&a); + report("cast_ok", rc); + + g_calls = 0; g_n = -1; + base(&a, 2); a.dtype[0] = ID_FP16; a.dtype[1] = ID_FP16; + rc = cast_f32_f16_entry(&a); + report("cast_input_fp16", rc); + + g_calls = 0; g_n = -1; + base(&a, 3); a.dtype[0] = ID_FP16; a.dtype[1] = ID_FP16; a.dtype[2] = ID_FP16; + rc = add_fp16_entry(&a); + report("add_ok", rc); + report_order("add_order", + g_in0 == (const void *) b0 && g_in1 == (const void *) b1 + && g_out == (const void *) b2); + + g_calls = 0; g_n = -1; + base(&a, 3); a.dtype[0] = ID_FP16; a.dtype[1] = ID_FP32; a.dtype[2] = ID_FP16; + rc = add_fp16_entry(&a); + report("add_second_input_fp32", rc); + + g_calls = 0; g_n = -1; + base(&a, 1); a.dtype[0] = ID_FP16; a.dtype[1] = ID_FP16; + rc = scale_fp16_entry(&a); + report("scale_one_buffer", rc); + + g_calls = 0; g_n = -1; + base(&a, 2); a.dtype[0] = ID_FP16; a.dtype[1] = ID_FP16; a.buf[1] = 0; + rc = scale_fp16_entry(&a); + report("scale_null_output", rc); + + return 0; +} +""" + +CASE_RE = re.compile(r"case=(\S+) rc=(-?\d+) calls=(-?\d+) n=(-?\d+)") + + +@pytest.fixture(scope="module") +def probe(tmp_path_factory): + """Compile the real generated entries plus the probe, run it once, and + return {label: (rc, calls, n)} together with the factor flag. + + The dtype IDS are passed in with -D from `wire.DTYPE_ID` rather than typed + into the C, so this cannot silently test a stale table.""" + if HOST_CC is None: + pytest.skip("no host C compiler") + d = tmp_path_factory.mktemp("entryprobe") + (d / "kernel_api.h").write_text(KERNEL_API_H) + (d / "probe.c").write_text(PROBE_C) + + sources = [str(d / "probe.c")] + for kind in ("scale", "cast", "add"): + spec = rn.SPECS[kind] + name = spec.kernel_dir.split("/")[-1] + path = d / f"{name}_entry.c" + path.write_text(ge.emit_entry(kind, spec)) + sources.append(str(path)) + + exe = str(d / "probe.exe") + cmd = [ + HOST_CC, "-std=c11", "-O0", + f"-DID_FP16={DTYPE_ID['fp16']}u", f"-DID_FP32={DTYPE_ID['fp32']}u", + "-I", str(d), "-I", str(SKEL.resolve()), + *sources, "-o", exe, + ] + cp = subprocess.run(cmd, capture_output=True, text=True) + assert cp.returncode == 0, f"probe did not compile:\n{cp.stderr}" + run = subprocess.run([exe], capture_output=True, text=True) + assert run.returncode == 0, f"probe crashed:\n{run.stdout}\n{run.stderr}" + + cases = { + m.group(1): (int(m.group(2)), int(m.group(3)), int(m.group(4))) + for m in CASE_RE.finditer(run.stdout) + } + assert len(cases) == 9, f"probe printed {sorted(cases)}:\n{run.stdout}" + cases["_factor_ok"] = ("factor_ok=1" in run.stdout, 0, 0) + cases["_scale_order"] = ("scale_order=1" in run.stdout, 0, 0) + cases["_add_order"] = ("add_order=1" in run.stdout, 0, 0) + return cases + + +@needs_cc +def test_a_well_formed_request_reaches_the_kernel(probe): + """The control, and it has to come first: every refusal below is only + meaningful because the accepted case is accepted. `n` proves the extent is + derived from `ne` on the DSP side, and the factor proves the params blob is + read as float bits and not as an int.""" + for label in ("scale_ok", "cast_ok", "add_ok"): + rc, calls, n = probe[label] + assert rc == STATUS["OK"], f"{label} returned {rc}" + assert calls == 1, f"{label} did not call its kernel" + assert n == 17, f"{label} derived n={n} from ne, expected 17" + assert probe["_factor_ok"][0], "the attr scalar did not arrive as 0.125f" + + +@needs_cc +def test_an_input_declared_with_the_wrong_dtype_is_refused_before_the_cast(probe): + """THE FINDING, PROVEN BY RETURN VALUE. `scale_fp16` casts buf[0] to + `const hexlib_hf *`. A batch declaring that buffer fp32 -- reachable from + `run_raw`, from main.c's hand-built blob, or from a future planner -- used to + be read at half stride: half the tensor, HEXLIB_DSP_OK, plausible values. + Now the entry refuses it and never calls the kernel. + + THIS IS THE FALSIFIABLE ONE. Delete the input dtype check from + `genentry._dtype_check`/`emit_entry` and this test fails with rc=1 and + calls=1 -- unlike the output check, which no serializer can violate.""" + rc, calls, _ = probe["scale_input_fp32"] + assert rc == STATUS["ERR_REQUIRES"], ( + f"an fp32 buffer cast to hexlib_hf* returned {rc}, not ERR_REQUIRES" + ) + assert calls == 0, "the kernel ran on a buffer of the wrong dtype" + + rc, calls, _ = probe["cast_input_fp16"] + assert rc == STATUS["ERR_REQUIRES"], f"cast's fp32 input check returned {rc}" + assert calls == 0 + + +@needs_cc +def test_the_dtype_check_is_per_buffer_not_just_the_first_one(probe): + """`add` has two inputs. A guard written for buf[0] alone would leave the + right-hand operand -- the one carrying the learned pos_embed in this + graph -- unchecked.""" + rc, calls, _ = probe["add_second_input_fp32"] + assert rc == STATUS["ERR_REQUIRES"], ( + f"add's SECOND input was cast without a dtype check (rc={rc})" + ) + assert calls == 0 + + +@needs_cc +def test_an_output_declared_with_the_wrong_dtype_is_refused_too(probe): + """The same hazard on the write side: an fp32 output buffer written through + `hexlib_hf *` gets half of it filled and half left as whatever was there. + Unreachable through `hexlib/exec/dsp.py` (which stamps this field from + `spec.out_dtype`), reachable from any hand-built batch -- which is exactly + what this probe is.""" + rc, calls, _ = probe["scale_output_fp32"] + assert rc == STATUS["ERR_REQUIRES"], f"returned {rc}" + assert calls == 0 + + +@needs_cc +def test_the_structural_checks_still_come_first(probe): + """A dtype check on a buffer the batch never supplied would be reading + uninitialised `hexlib_args` fields. The count and null checks must still be + the ones that answer these, with their own distinct status.""" + rc, calls, _ = probe["scale_one_buffer"] + assert rc == STATUS["ERR_INVAL_PARAMS"], f"n_buf=1 returned {rc}" + assert calls == 0 + rc, calls, _ = probe["scale_null_output"] + assert rc == STATUS["ERR_INVAL_PARAMS"], f"a null output returned {rc}" + assert calls == 0 + + +@needs_cc +def test_buffers_are_packed_sources_then_destinations(probe): + """THE CONTRACT, PINNED FROM BOTH SIDES AT ONCE. + + `skel_dispatch.c`'s fill loop is the only statement anywhere that `a->buf[]` + holds sources followed by destinations, and `genentry.py` hardcodes the + mirror image as `out_idx = n_in`. Neither referenced the other and no test + compared them, so an "outputs first" refactor could swap input and output + pointers in every generated entry at once: `scale_fp16` would write into its + own input and return the zero-filled output region, at the right length, + with status OK. Only the @sdk-gated numeric test would have noticed, and CI + does not run it. + + This checks the POINTERS the kernel actually received, so it fails on the + swap rather than on the spelling of any particular index expression. b0/b1/b2 + are distinct static arrays, which is what makes identity meaningful. + + ONE SIDE, HONESTLY LABELLED. The heading above used to say both sides were + pinned here. They were not: this probe hands `hexlib_args` to the entry + DIRECTLY, so it pins `genentry`'s half of the contract -- that the entry + reads its inputs from indices 0..n_in-1 and its output from index n_in -- and + nothing about how `a->buf[]` came to be filled. `skel_dispatch.c`'s fill loop + was not compiled by this file at all, and inverting it left every test here + passing. `test_the_dispatcher_and_the_generated_entry_agree_on_src_then_dst` + below is the other side, and the two together are what the heading claims.""" + assert probe["_scale_order"][0], ( + "scale_fp16 must receive buf[0] as its input and buf[1] as its output " + "(1 source, then 1 destination)" + ) + assert probe["_add_order"][0], ( + "add_fp16 must receive buf[0] and buf[1] as its two inputs and buf[2] as " + "its output -- the destination sits at index n_in, not index 0" + ) + + +# ============================================================================== +# THE OTHER SIDE OF THE SAME CONTRACT: skel_dispatch.c's fill loop, compiled and +# driven with a real batch blob. See the module docstring for what is stubbed. +# ============================================================================== + +_STUB_HEADERS = { + # FARF discards its arguments: nothing here reads a device log. + "HAP_farf.h": ( + "#ifndef HEXLIB_PROBE_HAP_FARF_H\n" + "#define HEXLIB_PROBE_HAP_FARF_H\n" + "#define FARF(...) do { } while (0)\n" + "#endif\n" + ), + # A monotonic counter, so the PCYCLE bracket produces a nonzero delta and + # the response's cycles_total is checkable without a real counter. + "HAP_perf.h": ( + "#ifndef HEXLIB_PROBE_HAP_PERF_H\n" + "#define HEXLIB_PROBE_HAP_PERF_H\n" + "static unsigned long long hexlib_probe_pcycles;\n" + "static inline unsigned long long HAP_perf_get_pcycles(void) {\n" + " hexlib_probe_pcycles += 1287; return hexlib_probe_pcycles;\n" + "}\n" + "#endif\n" + ), + # A distinct small fake address per fd. SMALL ON PURPOSE: hexlib_tensor.data + # is a uint32_t, so a real 64-bit host address would be truncated by + # skel_bufs.c's own (uint32_t) cast and identity would stop meaning + # anything. Nothing dereferences these. + "HAP_mem.h": ( + "#ifndef HEXLIB_PROBE_HAP_MEM_H\n" + "#define HEXLIB_PROBE_HAP_MEM_H\n" + "#include \n" + "#define HAP_PROT_READ 1\n" + "#define HAP_PROT_WRITE 2\n" + "#define HEXLIB_PROBE_BASE(fd) " + "(0x01000000u + 0x00010000u * (unsigned) (fd))\n" + "static inline void *HAP_mmap2(void *a, size_t l, int p, int f,\n" + " int fd, long o) {\n" + " (void) a; (void) l; (void) p; (void) f; (void) o;\n" + " return (void *) (size_t) HEXLIB_PROBE_BASE(fd);\n" + "}\n" + "static inline void *HAP_mmap(void *a, int l, int p, int f,\n" + " int fd, long o) {\n" + " (void) a; (void) l; (void) p; (void) f; (void) o;\n" + " return (void *) (size_t) HEXLIB_PROBE_BASE(fd);\n" + "}\n" + "static inline int HAP_munmap2(void *a, size_t l) {\n" + " (void) a; (void) l; return 0;\n" + "}\n" + "static inline int HAP_munmap(void *a, int l) {\n" + " (void) a; (void) l; return 0;\n" + "}\n" + "#endif\n" + ), +} + +# Only the one kernel this probe drives, so `kernel_api.h` above is not reused +# (it declares three). +_DISPATCH_KERNEL_API_H = """\ +#ifndef PROBE_DISPATCH_KERNEL_API_H +#define PROBE_DISPATCH_KERNEL_API_H +typedef unsigned short hexlib_hf; +void scale_fp16(const hexlib_hf *x, hexlib_hf *y, int n, float factor); +#endif +""" + +_DISPATCH_PROBE_C = r""" +#include "skel_internal.h" +#include "kernel_api.h" +#include +#include +#include + +/* Recording stub. The addresses are never dereferenced -- WHICH buffer arrived + * where is the entire question. */ +static int g_calls; +static const void *g_in0; +static const void *g_out; +static int g_n; +static float g_factor; + +void scale_fp16(const hexlib_hf *x, hexlib_hf *y, int n, float factor) { + g_calls++; g_in0 = x; g_out = y; g_n = n; g_factor = factor; +} + +extern int scale_fp16_entry(const hexlib_args *); + +/* The real table shape from hexlib_dsp.h, with the one generated entry. */ +const struct hexlib_kernel_entry hexlib_kernel_table[] = { + { PROBE_KIND_SCALE, "scale_fp16", scale_fp16_entry }, +}; +const uint32_t hexlib_kernel_table_len = 1; + +/* Not what this probe is about; skel_vtcm.c needs the real HAP_compute_res. */ +int hexlib_vtcm_acquire(struct hexlib_ctx *c) { (void) c; return HEXLIB_DSP_OK; } +void hexlib_vtcm_release(struct hexlib_ctx *c) { (void) c; } + +static struct hexlib_ctx g_ctx; +static unsigned char g_batch[65536]; +/* uint64-aligned: the dispatcher casts rsp + sizeof(hdr) to + * struct hexlib_op_result *, which contains a uint64_t. */ +static unsigned long long g_rsp[1024]; + +int main(int argc, char **argv) { + FILE *f; + size_t len; + int i; + uint32_t rsp_len = 0; + int rc; + + if (argc < 3) return 2; + f = fopen(argv[1], "rb"); + if (!f) return 3; + len = fread(g_batch, 1, sizeof(g_batch), f); + fclose(f); + + memset(&g_ctx, 0, sizeof(g_ctx)); + /* Register every fd the batch names, exactly as hexlib_iface_mmap would. */ + for (i = 2; i < argc; i++) { + unsigned fd = (unsigned) strtoul(argv[i], 0, 10); + printf("register fd=%u rc=%d\n", fd, + hexlib_bufs_register(&g_ctx, fd, PROBE_BUF_SIZE)); + } + /* The DSP-side mapping table, so the expected addresses are read out of the + * real skel_bufs.c state rather than recomputed by the test. */ + for (i = 0; i < HEXLIB_MAX_MMAPS; i++) { + if (g_ctx.mmap[i].size) { + printf("mapped fd=%d base=%llu\n", (int) g_ctx.mmap[i].fd, + (unsigned long long) g_ctx.mmap[i].base); + } + } + + g_ctx.started = 1; + rc = hexlib_dispatch_batch(&g_ctx, g_batch, (uint32_t) len, + (unsigned char *) g_rsp, + (uint32_t) sizeof(g_rsp), &rsp_len); + printf("dispatch rc=%d rsp_len=%u calls=%d n=%d factor_ok=%d\n", + rc, rsp_len, g_calls, g_n, g_factor == 0.125f ? 1 : 0); + printf("in0=%llu out=%llu\n", + (unsigned long long) (size_t) g_in0, + (unsigned long long) (size_t) g_out); + printf("rsp="); + for (i = 0; i < (int) rsp_len; i++) { + printf("%02x", ((unsigned char *) g_rsp)[i]); + } + printf("\n"); + return 0; +} +""" + +# Distinct fds and distinct NONZERO offsets: the input's address and the +# output's must be different numbers for identity to prove anything, and an +# offset of 0 on both would make a base-only bug invisible. +_FD_IN, _FD_OUT = 11, 22 +_OFF_IN, _OFF_OUT = 128, 256 +_BUF_SIZE = 4096 +_NE = (17, 1, 1, 1) +_FACTOR = 0.125 # a power of two, exact in fp16 -- same value run_self_test uses + + +@pytest.fixture(scope="module") +def both_sides(tmp_path_factory): + """Compile skel_dispatch.c + skel_bufs.c + a generated entry, build a real + batch with `wire.pack_batch`, run it through `hexlib_dispatch_batch`, and + return what the kernel saw plus the raw response bytes.""" + if HOST_CC is None: + pytest.skip("no host C compiler") + d = tmp_path_factory.mktemp("dispatchprobe") + for name, text in _STUB_HEADERS.items(): + (d / name).write_text(text) + (d / "kernel_api.h").write_text(_DISPATCH_KERNEL_API_H) + (d / "probe.c").write_text(_DISPATCH_PROBE_C) + (d / "scale_entry.c").write_text(ge.emit_entry("scale", rn.SPECS["scale"])) + + kind = ge.KIND_ID["scale"] + exe = str(d / "probe.exe") + cmd = [ + HOST_CC, "-std=c11", "-O0", + f"-DPROBE_KIND_SCALE={kind}u", f"-DPROBE_BUF_SIZE={_BUF_SIZE}u", + # The arch this "binary" was built for (hexlib_write_rsp_hdr reads it) + # and the HVX level that selects skel_bufs.c's HAP_mmap2 branch -- the + # v75 branch, which is the one that runs on the target part. + "-D__HEXAGON_ARCH__=75", "-D__HVX_ARCH__=75", + "-I", str(d), "-I", str(SKEL.resolve()), + str(d / "probe.c"), str(d / "scale_entry.c"), + str((SKEL / "skel_dispatch.c").resolve()), + str((SKEL / "skel_bufs.c").resolve()), + "-o", exe, + ] + cp = subprocess.run(cmd, capture_output=True, text=True) + assert cp.returncode == 0, ( + "the skel dispatch probe did not compile. THIS FIXTURE IMPOSES A REAL " + "CONSTRAINT AND THAT IS DELIBERATE: skel_dispatch.c and skel_bufs.c must " + "stay buildable by a plain host C compiler, i.e. straight-line C with no " + "Hexagon intrinsics and no inline asm, with every SDK dependency behind " + "one of the stubbed headers above. That is already true and is worth " + "keeping -- it is the same property that makes the cycle counter go " + "through HAP_perf_get_pcycles() rather than a hand-rolled `c15:14` read. " + "If a kernel-side intrinsic genuinely belongs in one of these two files, " + "it needs to move behind a helper this probe can stub, not be absorbed " + f"by deleting this test:\n{cp.stderr}" + ) + + bufs = [wire.BufDesc(fd=_FD_IN, size=_BUF_SIZE), + wire.BufDesc(fd=_FD_OUT, size=_BUF_SIZE)] + nbytes = 2 * _NE[0] + tensors = [ + wire.TensorDesc(bi=0, offset=_OFF_IN, nbytes=nbytes, dtype="fp16", + layout="row_major", ne=_NE), + wire.TensorDesc(bi=1, offset=_OFF_OUT, nbytes=nbytes, dtype="fp16", + layout="row_major", ne=_NE), + ] + factor_bits = struct.unpack("buf[]`; + `genentry` reads the output back out at `out_idx = n_in`. Neither references + the other. Invert either and `scale_fp16` writes into its own input and + returns the untouched output region -- at the right length, with status OK, + on real silicon. Only the @sdk-gated numeric test would have noticed, and CI + does not run it. + + What is checked is the ADDRESS the kernel received for each argument, + against the fd-plus-offset arithmetic the batch declared, with the bases read + out of skel_bufs.c's own mapping table. No index expression, no field name + and no source text is matched, so this fails on the swap itself rather than + on how anyone spelled it.""" + base_in = both_sides["bases"][_FD_IN] + base_out = both_sides["bases"][_FD_OUT] + assert base_in != base_out, "the two fds must map to different addresses" + + assert both_sides["calls"] == 1, "the kernel was not called exactly once" + assert both_sides["in0"] == base_in + _OFF_IN, ( + f"scale_fp16 received {both_sides['in0']} as its INPUT; the batch's one " + f"source tensor is at fd {_FD_IN} + {_OFF_IN} = {base_in + _OFF_IN}. " + f"(The output tensor is at {base_out + _OFF_OUT} -- if that is what " + f"arrived, sources and destinations are packed the other way round on " + f"one of the two sides, and the kernel is reading what it should be " + f"writing.)" + ) + assert both_sides["out"] == base_out + _OFF_OUT, ( + f"scale_fp16 received {both_sides['out']} as its OUTPUT, expected " + f"{base_out + _OFF_OUT} -- the destination sits at buf[n_in], filled " + f"from op.dst after op.src" + ) + + +@needs_cc +def test_the_dispatcher_resolves_a_tensor_from_the_dsp_side_mapping_only(both_sides): + """The corollary, and the invariant skel_bufs.c exists for: the address the + kernel got is `base + offset` where `base` came from the DSP's OWN mmap + table, keyed by fd. `wire.pack_batch` writes zero into the `base` wire slot + and there is no field for a host address, so an implementation that leaned on + one could not even be expressed here -- what this adds is that the address + actually used is the mapped one, measured, rather than 0 + offset (upstream's + silent fallthrough) or the offset alone.""" + base_in = both_sides["bases"][_FD_IN] + assert both_sides["in0"] not in (0, _OFF_IN), ( + "the kernel's pointer must be a real mapped address plus the offset, " + "not the offset alone or a zero base" + ) + assert both_sides["in0"] - _OFF_IN == base_in + assert both_sides["n"] == _NE[0], ( + f"the extent must be derived from the tensor's own ne on the DSP side; " + f"got n={both_sides['n']}, expected {_NE[0]}" + ) + assert both_sides["factor_ok"] == 1, ( + "the op's params blob must reach the kernel as float bits -- it is " + "carried as int32[] on the wire and cast on the DSP side" + ) + + +@needs_cc +def test_the_response_the_dispatcher_wrote_is_what_wire_py_unpacks(both_sides): + """END TO END, IN BOTH DIRECTIONS, IN ONE RUN: `wire.pack_batch` built the + blob, the real dispatcher walked it, and the response bytes it wrote go back + through `wire.unpack_response`. test_wire_struct_layout.py pins the two + descriptions of these bytes against each other by size and offset; this pins + them against a real dispatcher's real output. + + `cycles_total > 0` because the probe's HAP_perf stub advances -- which checks + that the PCYCLE bracket's delta actually reaches the response header, the + thing `hexlib/cli.py` refuses a device job over.""" + rsp = wire.unpack_response(both_sides["rsp"]) + assert rsp.status == STATUS["OK"], f"batch status {rsp.status}" + assert rsp.n_ops == 1 + assert rsp.arch == 75, ( + "the response must carry the arch the skel was BUILT for " + "(__HEXAGON_ARCH__), never a caller-supplied value" + ) + assert len(rsp.results) == 1 + assert rsp.results[0].kind == ge.KIND_ID["scale"] + assert rsp.results[0].status == STATUS["OK"] + assert rsp.results[0].cycles > 0 + assert rsp.cycles_total == rsp.results[0].cycles, ( + "one op, so the batch total is that op's own bracket" + ) + assert both_sides["rc"] == STATUS["OK"] diff --git a/hexlib/tests/test_graph_ir.py b/hexlib/tests/test_graph_ir.py index 7b82f6f..08c1485 100644 --- a/hexlib/tests/test_graph_ir.py +++ b/hexlib/tests/test_graph_ir.py @@ -69,8 +69,16 @@ def test_tensor_rejects_nonpositive_dim(): assert "x" in str(e.value) -def test_dtypes_are_exactly_the_four(): - assert DTYPES == frozenset({"fp32", "fp16", "int32", "q4_0"}) +def test_dtypes_are_exactly_these(): + """AN EXHAUSTIVE PIN, DELIBERATELY. A dtype added here has to be carried + through `wire.DTYPE_ID`, `runner.WIRE_DTYPE` or `WIRE_RAW`, and + `genentry._CTYPE` before anything can use it -- `test_runtime_wire.py` binds + those three -- so a silent addition is a dtype the graph accepts and the + wire cannot carry. Failing here is the intended way to be reminded. + + q8_0 was added on 2026-08-13 and this test caught it, which is the check + working rather than the check being in the way.""" + assert DTYPES == frozenset({"fp32", "fp16", "int32", "q4_0", "q8_0"}) def test_nbytes_dense(): diff --git a/hexlib/tests/test_host_source.py b/hexlib/tests/test_host_source.py new file mode 100644 index 0000000..a58eba5 --- /dev/null +++ b/hexlib/tests/test_host_source.py @@ -0,0 +1,992 @@ +# hexlib/tests/test_host_source.py +"""The CPU-side driver. Source assertions -- it cannot be RUN without a +device, which is exactly why stage 2 exists as a separate gate: so stage 3 +spends minutes on one unknown rather than five. + +TIGHTENED TWICE. A first draft of these checked for bare substrings anywhere +in a file -- which a comment, a dead branch, or a FARF log line naming the +right constant would also satisfy. Every earlier task in this plan had the +same problem and needed the same fix (see test_skel_bufs_source.py, +test_skel_vtcm_source.py), so these became function-scoped wherever the +underlying claim is about ONE function's behaviour, checking actual `return`s +and call ORDER rather than mere presence. + +THE SECOND TIGHTENING, AND WHY IT WAS NEEDED. Function scope alone was not +enough, because the checks still ran against comment-BEARING text. Three +mutations proved it: + + * reverting hexlib_decode_bcd_arch (session.c) to the shipped bug + (`return arch_ver;`) with the arithmetic left in a comment INSIDE the body + still matched the `>> 4` / `* 10` / `& 0x0f` regexes below; + * reverting hexlib_classify_coherency_lane (main.c) to its `bits == 0x0000u` + bug, old code left in a body comment, left this whole file at 22 passed; + * `handle = dlopen(...)` -> `handle = NULL;` passed, because `"dlopen(" in + body` was satisfied by driver.c's own dlopen-failed error format string; + same shape for `"dlsym(" in driver`. + +So every fixture here is COMMENT-BLANKED (`csource.code_only`), every slice +inherits that, and presence checks that used to be bare tokens are now call- +or assignment-shaped. The single test that legitimately inspects COMMENTS -- +test_coherency_check_documents_its_own_scope_limits, whose whole claim is that +a caveat is written down for a human reader -- takes the `main_comments` +fixture instead and says so. + +THE THIRD TIGHTENING: STRING LITERALS ARE BLANKED NOW TOO. `csource.code_only` +used to leave string and character literals intact, so the very defect the +second tightening's third bullet describes -- `"dlopen(" in body` satisfied by +driver.c's own error format string -- was still available to every other check +in this file, and to a `}` inside a literal truncating any slice. Fixed in +`csource` (see its module docstring for the three mechanisms and the +mutations). What that means HERE is that `code_only` now hands back text with +no literal payload in it at all, and the handful of checks in this file whose +subject genuinely IS literal text -- a printed line another test asserts on, a +command-line flag string, the dlopen candidate path, the `&_dom=cdsp` spelling +that must NOT appear -- take one of the `*_strings` fixtures below and pass +`keep_strings=True` to the slicer, saying so at the call site. Every other +check keeps the ordinary fixture, because for those a token inside a format +string is evidence of nothing. +""" +import pathlib +import re + +import pytest + +from hexlib.tests.csource import block_from as _block_from +from hexlib.tests.csource import calls as _calls +from hexlib.tests.csource import code_only as _code_only +from hexlib.tests.csource import code_only_keeping_strings as _code_with_strings +from hexlib.tests.csource import function_body as _function_body + +H = pathlib.Path("hexlib/runtime/host") + + +@pytest.fixture(scope="module") +def driver(): + return _code_only((H / "driver.c").read_text()) + + +@pytest.fixture(scope="module") +def session(): + return _code_only((H / "session.c").read_text()) + + +@pytest.fixture(scope="module") +def buffers(): + return _code_only((H / "buffers.c").read_text()) + + +@pytest.fixture(scope="module") +def main(): + return _code_only((H / "main.c").read_text()) + + +# -------------------------------------------------------------------------- +# THE `*_strings` FIXTURES: comments blanked, STRING LITERALS INTACT. For the +# few claims whose subject IS literal text. Blanking is length-preserving in +# both views, so an offset found in a `*_strings` slice is valid in the +# ordinary slice of the same function and vice versa -- which is how a test can +# locate a flag string in one view and then brace-slice the block in the other. +# Never use these for a "this guard is here" / "this constant is absent" check; +# that is what the plain fixtures are for. See csource.py's docstring. +# -------------------------------------------------------------------------- + + +@pytest.fixture(scope="module") +def driver_strings(): + return _code_with_strings((H / "driver.c").read_text()) + + +@pytest.fixture(scope="module") +def session_strings(): + return _code_with_strings((H / "session.c").read_text()) + + +@pytest.fixture(scope="module") +def main_strings(): + return _code_with_strings((H / "main.c").read_text()) + + +@pytest.fixture(scope="module") +def main_comments(): + """main.c WITH its comments, for the one test whose subject IS a comment + (test_coherency_check_documents_its_own_scope_limits). Every other check + in this file must use the `main` fixture above -- see the module + docstring.""" + return (H / "main.c").read_text() + + +def _macro_body(src, name): + """Slice an object/function-like `#define` by following backslash line + continuations. `_function_body`'s brace-counting does not apply to a + macro definition (its own braces are a `do { ... } while (0)` wrapper, + not the boundary we want), so this is a narrowly-scoped sibling rather + than a reuse -- there is exactly one macro these tests need to isolate. + + Comment-blanked like everything else here: `csource.code_only` preserves + length, and a blanked `/* ... */` inside a macro leaves any trailing + backslash continuation exactly where it was, so the line walk is + unaffected.""" + src = _code_only(src) + m = re.search(rf"#define\s+{re.escape(name)}\b", src) + assert m, f"could not find #define {name} in the source" + lines = src[m.start():].splitlines() + out = [] + for line in lines: + out.append(line) + if not line.rstrip().endswith("\\"): + break + return "\n".join(out) + + +def test_libcdsprpc_is_dlopened_not_linked(driver, driver_strings): + """A missing driver becomes a readable message instead of a loader + failure with no output -- this is why a well-built capability probe works + the first time it runs on real hardware. Scoped to hexlib_drv_init(): + dlopen() and the candidate path must both live in the function that + actually loads the driver, not merely somewhere in the file (e.g. a + comment mentioning both). AND a NULL handle -- every candidate path + failed -- must actually fail hexlib_drv_init from inside its own check, + not merely be logged: a build that calls dlopen() and ignores a NULL + result would otherwise satisfy the presence checks above and still crash + the first time a dlsym() runs against it. + + THE HANDLE MUST COME FROM THE CALL. `"dlopen(" in body` was satisfied by + driver.c's own `"hexlib: dlopen(%s) failed: %s\\n"` format string, three + lines below the real call -- so `handle = dlopen(candidates[i], RTLD_NOW);` + could be replaced outright with `handle = NULL;` and this test still + passed, with the readable-message property it exists to protect gone and + the loader never consulted at all.""" + body = _function_body(driver, "hexlib_drv_init") + assert re.search(r"\bhandle\s*=\s*dlopen\s*\(", body), ( + "the driver handle must be ASSIGNED from a real dlopen() call -- a " + "mention of dlopen in an error message is not loading anything" + ) + # THE ONE CHECK HERE WHOSE SUBJECT IS A LITERAL: the candidate path is a + # string, so it has to be looked for in the literal-bearing view. The + # `handle = dlopen(...)` check above deliberately does NOT -- that is the + # one the format string used to satisfy. + body_strings = _function_body(driver_strings, "hexlib_drv_init", + keep_strings=True) + assert '"libcdsprpc.so"' in body_strings, ( + "the candidate path must be a real string literal in the loading " + "function, not merely named in prose" + ) + + null_check = re.search(r"handle\s*==\s*NULL", body) + assert null_check, "a failed dlopen() must be checked, not assumed to succeed" + null_block = _block_from(body, null_check.end()) + assert re.search(r"return\s+-1\s*;", null_block), ( + "a NULL driver handle must actually fail hexlib_drv_init, not just " + "be logged" + ) + + +def test_every_symbol_is_resolved_by_name_and_checked(driver): + """Each required symbol must be the subject of an actual HEXLIB_DLSYM(...) + call inside hexlib_drv_init -- not merely named somewhere in the file, + which a stale comment or a typedef alone would also satisfy. + + AND THE MACRO MUST RESOLVE BY NAME, THROUGH dlsym, INTO THE POINTER. + `"dlsym(" in driver` was whole-file and was satisfied by the macro's own + `"hexlib: dlsym(%s) failed: %s\\n"` error format string, so the real + `(pfn) = (__typeof__(pfn)) dlsym(handle, #symbol);` could be replaced with + anything at all -- including `(pfn) = NULL;`, which would make every + symbol below "resolve" and then null-call later, the exact bug the macro + exists to prevent.""" + body = _function_body(driver, "hexlib_drv_init") + for sym in ( + "rpcmem_alloc", "rpcmem_free", "rpcmem_to_fd", "fastrpc_mmap", + "remote_handle64_open", "remote_handle64_invoke", + "remote_handle_control", "remote_session_control", + ): + assert re.search(rf"HEXLIB_DLSYM\([^;]*\b{re.escape(sym)}\b", body), sym + + macro = _macro_body(driver, "HEXLIB_DLSYM") + assert re.search(r"\(\s*pfn\s*\)\s*=[^;]*\bdlsym\s*\(", macro), ( + "HEXLIB_DLSYM must assign the function pointer from an actual " + "dlsym() call -- naming dlsym in its own failure message is not " + "resolving anything" + ) + assert re.search(r"\bdlsym\s*\(\s*handle\s*,\s*#\s*symbol\s*\)", macro), ( + "the symbol must be resolved BY NAME out of the dlopen'd handle " + "(dlsym(handle, #symbol)), which is what makes a missing symbol a " + "named error rather than a null call later" + ) + + +def test_a_missing_symbol_is_an_error_not_a_null_call(driver): + """The dlsym-and-check macro itself, not just the word `dlerror` anywhere + in the file, must report via dlerror() AND actually fail (`return -1`) on + a required symbol -- logging and continuing would reintroduce exactly the + null-call-later bug this exists to prevent.""" + macro = _macro_body(driver, "HEXLIB_DLSYM") + assert "dlerror()" in macro + assert "return -1;" in macro + # The failure path must be reachable from the `required` branch, not + # dead code after an unconditional return. + assert re.search(r"if\s*\(required\)", macro) + + +def test_cdsp_domain_three_and_unsigned_pd(session): + """CDSP only. ADSP is a v73 part with UNSIGNED_PD_SUPPORT = 0, so + targeting it would silently be a different measurement on different + hardware. Scoped to hexlib_open(): the domain check and the unsigned-PD + request must both be real control flow in the function that opens a + session, not just mentioned in the file somewhere.""" + body = _function_body(session, "hexlib_open") + assert "CDSP_DOMAIN_ID" in body + reject = re.search(r"domain\s*!=\s*CDSP_DOMAIN_ID", body) + assert reject, "hexlib_open must refuse any domain that isn't CDSP_DOMAIN_ID" + # A real refusal, not a comment: the `if` block guarded by the comparison + # above -- and nothing outside it, which a later unrelated `return -1;` + # (e.g. from the driver-init check further down) could otherwise satisfy + # by accident -- must itself contain the error return. + reject_block = _block_from(body, reject.end()) + assert re.search(r"return\s+-1\s*;", reject_block), ( + "the domain check must actually refuse inside its own if-block, not " + "merely be followed eventually by some other return" + ) + + unsigned_call = re.search(r"enable_unsigned_pd\s*\(", body) + assert unsigned_call, "hexlib_open must request an unsigned PD" + open_call = re.search(r"\bhexlib_iface_open\s*\(", body) + assert open_call, "hexlib_open must actually open the qaic handle" + assert unsigned_call.start() < open_call.start(), ( + "unsigned PD must be requested BEFORE the handle is opened -- " + "afterwards the PD already exists" + ) + + enable_body = _function_body(session, "enable_unsigned_pd") + assert "DSPRPC_CONTROL_UNSIGNED_MODULE" in enable_body + + +def test_the_uri_is_built_not_hardcoded_with_a_domain(session, session_strings): + """The URI must be assembled from hexlib_iface_URI (qaic-generated) and + CDSP_DOMAIN ('s own "&_dom=cdsp" macro) as adjacent string + literals -- never spelled out as a literal "&_dom=cdsp" string, which + would silently stop tracking either constant if it ever changed. + + THIS CHECKS CONSTRUCTION *FLOWING INTO USE*, not mere co-occurrence in + the file: it is not enough for `hexlib_iface_URI CDSP_DOMAIN` to appear + somewhere in session.c if the variable it builds is dead code and + hexlib_iface_open() is actually called with something else entirely. + So this captures the constructed variable's name and asserts THAT name + is what is passed as hexlib_iface_open's first argument, in hexlib_open + itself, after the construction.""" + body = _function_body(session, "hexlib_open") + construct = re.search( + r"(\w+)\s*\[\]\s*=\s*hexlib_iface_URI\s+CDSP_DOMAIN\b", body + ) + assert construct, ( + "hexlib_open must build the URI from hexlib_iface_URI and CDSP_DOMAIN" + ) + # LITERAL-BEARING VIEW, DELIBERATELY. This negative is about a STRING + # SPELLING -- the whole claim is that nobody wrote the domain suffix out by + # hand -- so it must be checked against text in which literals survive. + # Against `code_only` text it would pass vacuously (every literal blanked, + # so no literal can ever be found), which is a weaker check than the one + # this line was written to make. + assert '"&_dom=cdsp"' not in session_strings + + var = construct.group(1) + open_call = re.search(rf"\bhexlib_iface_open\s*\(\s*{re.escape(var)}\s*,", body) + assert open_call, ( + f"the constructed URI variable ({var!r}) must be passed as the " + "first argument to hexlib_iface_open -- not merely constructed and " + "left unused while something else is opened" + ) + assert construct.start() < open_call.start(), ( + "the URI must be constructed before it is passed to hexlib_iface_open" + ) + + +def test_arch_is_queried_from_the_driver_not_assumed(session): + """hexlib_query_caps must actually issue the ARCH_VER / DSPRPC_GET_DSP_INFO + query (function-scoped, not just present in the file), and hexlib_open + must cross-check that value against what the skel itself reports, failing + on disagreement rather than trusting either side alone. + + THE DECODE ITSELF IS PINNED HERE, NOT JUST THE COMPARISON. `arch` (skel + hwinfo, plain decimal, e.g. 75) and `caps.arch_ver` (driver ARCH_VER, BCD + nibble-packed, e.g. 0x8c75 = 35957) are in different encodings -- + comparing them raw is unconditionally false on every real device (see + session.c's own file header). A test that only checked "a comparison and + a `return -1` exist" could not see that the two operands were + incommensurable; that is exactly the shape of the bug this pins. See + test_session_arch_decode.py for a genuine, compiled-and-run behavioural + test of the same decode function against the one measured value + (0x8c75 -> 75).""" + caps_body = _function_body(session, "hexlib_query_caps") + assert "ARCH_VER" in caps_body + assert "DSPRPC_GET_DSP_INFO" in caps_body + + decode_body = _function_body(session, "hexlib_decode_bcd_arch") + assert re.search(r">>\s*4", decode_body), "must extract the high BCD nibble" + assert re.search(r"\*\s*10", decode_body), "must weight the high nibble by 10" + assert re.search(r"&\s*0x0f\b", decode_body), "must extract the low BCD nibble" + + open_body = _function_body(session, "hexlib_open") + decode_call = re.search( + r"hexlib_decode_bcd_arch\s*\(\s*caps\.arch_ver\s*\)", open_body + ) + assert decode_call, ( + "hexlib_open must decode caps.arch_ver through hexlib_decode_bcd_arch " + "before comparing it against the skel's arch -- comparing the raw " + "ARCH_VER directly against __HEXAGON_ARCH__ can never agree " + "(0x8c75 != 75) and would refuse every device session unconditionally" + ) + mismatch = re.search(r"arch\s*!=\s*\w+", open_body) + assert mismatch, "hexlib_open must cross-check driver arch against skel arch" + assert not re.search(r"arch\s*!=\s*caps\.arch_ver\b", open_body), ( + "hexlib_open must never compare the skel's arch against the raw, " + "undecoded caps.arch_ver" + ) + assert mismatch.start() > decode_call.start(), ( + "the decoded value, not the raw caps.arch_ver, must be what gets " + "compared against arch" + ) + mismatch_block = _block_from(open_body, mismatch.end()) + assert re.search(r"return\s+-1\s*;", mismatch_block), ( + "an arch mismatch must actually fail hexlib_open from inside its own " + "if-block, not just be logged" + ) + + +def test_no_literal_request_ids(session): + """An earlier probe in this project hardcoded DSPRPC_GET_DSP_INFO as 11 by + counting an enum in a doc comment; the real value is 2. A wrong request id + does not fail loudly -- it queries something else. So every request id + must come from 's own enums, never a bare number passed + straight to the control APIs. + + NEGATIVE HALF (absence of the bad pattern) paired with a POSITIVE HALF + (presence of the right one): a purely negative check would pass + vacuously if the real `remote_handle_control`/`remote_session_control` + calls were deleted outright, which is exactly the kind of gutted + implementation these tests exist to catch.""" + assert "#include " in session + assert not re.search(r"=\s*11\b", session) + # Nothing may pass a literal digit as the request id argument itself -- + # that would dodge the "= 11" check above while still hardcoding a + # different id the same way. + assert not re.search(r"remote_(handle|session)_control\s*\(\s*\d", session) + + caps_body = _function_body(session, "hexlib_query_caps") + assert re.search( + r"hexlib_remote_handle_control\s*\(\s*DSPRPC_GET_DSP_INFO\s*,", caps_body + ), "hexlib_query_caps must actually call remote_handle_control with DSPRPC_GET_DSP_INFO" + + enable_body = _function_body(session, "enable_unsigned_pd") + assert re.search( + r"hexlib_remote_session_control\s*\(\s*DSPRPC_CONTROL_UNSIGNED_MODULE\s*,", + enable_body, + ), ( + "enable_unsigned_pd must actually call remote_session_control with " + "DSPRPC_CONTROL_UNSIGNED_MODULE" + ) + + +def test_buffers_use_rpcmem_and_fastrpc_mmap(buffers): + """Scoped to hexlib_alloc(): the real allocation sequence must be + rpcmem_alloc, then rpcmem_to_fd, then fastrpc_mmap -- each an actual call + in the function that allocates a buffer, in that order, not merely + present somewhere in the file (e.g. in hexlib_free's teardown calls, + which mention fastrpc_mmap's sibling but not in this order).""" + body = _function_body(buffers, "hexlib_alloc") + i_alloc = body.index("hexlib_rpcmem_alloc(") + i_fd = body.index("hexlib_rpcmem_to_fd(") + i_mmap = body.index("hexlib_fastrpc_mmap(") + assert i_alloc < i_fd < i_mmap + + +def test_a_size_that_would_truncate_on_the_way_down_is_refused(buffers): + """`size` is a `size_t` and is narrowed TWICE: `(int) size` for + rpcmem_alloc and `(uint32_t) size` for hexlib_iface_mmap (qaic's own + generated signature from the IDL). Neither cast can report a loss, and they + can disagree with each other -- 2 GiB or more would register a mapping of + one length for a buffer allocated at another, and `(int) size` can go + negative outright. + + Unreachable today (every call site passes a plan-computed tensor size) and + it fails closed on the DSP side if it ever were not (skel_bufs.c's + `b->size > m->size` check), so the fix is deliberately a guard rather than + a widening of the wire. What this pins is that the guard runs BEFORE the + first cast: a check placed after rpcmem_alloc protects nothing, because the + truncation has already happened by then.""" + body = _function_body(buffers, "hexlib_alloc") + # `[^{]*` rather than `[^)]*`: the bound is written with a cast in it + # (`(size_t) INT_MAX`), so the condition legitimately contains parens. + m = re.search(r"if\s*\([^{]*\bsize\s*>[^{]*\)\s*\{", body) + assert m, ( + "hexlib_alloc must refuse a size too large for the int/uint32 casts " + "below it -- neither cast can report the truncation" + ) + assert m.start() < body.index("hexlib_rpcmem_alloc("), ( + "the size guard must run before the first narrowing cast, not after it" + ) + guard = _block_from(body, m.end() - 1) + assert "return -1;" in guard, ( + "an out-of-range size must be refused, not merely logged" + ) + assert "INT_MAX" in body, ( + "the bound must be the narrower of the two casts (rpcmem_alloc's int), " + "not uint32's -- a value that fits uint32 can still be negative as an " + "int" + ) + + +def test_the_host_never_puts_an_address_on_the_wire(buffers): + """hexlib_buf_to_desc -- the one place a hexlib_buf_desc is filled in from + this side -- must zero `base` itself, first (right after the memset, not + merely somewhere before the struct is used), and nothing in the file may + derive `base` from the host pointer (`buf->ptr`/`ptr`). + + TWO INDEPENDENT WEAKNESSES, BOTH PROVEN, BOTH FIXED HERE. `d->base = + (uint64_t)(uintptr_t) buf->ptr;` -- the host's own virtual address on the + wire, which works perfectly under the simulator's shared address space and + can only fail on silicon -- passed all 28 tests in this file when written + two ways at once: + + * the `base = 0;` positive was satisfied by an `fprintf` format string + containing that text (`csource` used to hand literals back intact; it + no longer does), and + * the whole-file `base\\s*=[^;]*\\bptr\\b` negative was satisfied by + routing the pointer through a temp named `hostaddr`, because the + forbidden token no longer appeared on the assignment's own line. A + negative check written as a pattern over the RHS can always be dodged + by a rename; that is a property of the shape of the check, not of the + name chosen. + + So the check is inverted into an EXHAUSTIVE one, which a rename cannot + dodge: enumerate every assignment to `d->base` in this function and require + that the complete set of right-hand sides is exactly `0`. And require that + this function never reads the host pointer AT ALL -- no `ptr` token in its + body -- so there is nothing available to launder through a temp under any + name. `hexlib_buf_to_desc` legitimately needs only `buf->size` and + `buf->fd`.""" + body = _function_body(buffers, "hexlib_buf_to_desc") + assigned = [rhs.strip() for rhs in re.findall(r"d->base\s*=\s*([^;]+);", body)] + assert assigned == ["0"], ( + f"hexlib_buf_to_desc must assign d->base exactly once, and the value " + f"must be 0 -- the DSP fills it in from its own mapping table " + f"(skel_bufs.c). Found right-hand sides {assigned!r}" + ) + memset_end = body.index(";", body.index("memset(")) + 1 + base_clear = re.search(r"\bbase\s*=\s*0\s*;", body) + assert base_clear, "base must be explicitly cleared, not left to memset alone" + assert base_clear.start() < body.index("d->size", memset_end), ( + "base must be cleared before the other fields are filled in" + ) + assert "ptr" not in body, ( + "hexlib_buf_to_desc must not so much as READ the host pointer -- it " + "needs buf->size and buf->fd and nothing else, and a function that " + "cannot see the address cannot put it on the wire under any variable " + "name" + ) + # Kept as a belt, and honestly labelled: this is the rename-defeatable + # form. The exhaustive check above is the one that holds. + assert not re.search(r"base\s*=[^;]*\bptr\b", buffers), ( + "base must never be derived from a host pointer anywhere in this file" + ) + + +def test_absence_of_a_response_is_a_failure(main): + """response_is_valid() must check HEXLIB_BATCH_MAGIC and refuse (return a + falsy value) both when the response is absent/short AND when the magic is + wrong -- and the --batch path must not even attempt to write the output + file until AFTER that check has passed, so an invalid response can never + leave a stale or garbage file behind.""" + valid_body = _function_body(main, "response_is_valid") + assert "HEXLIB_BATCH_MAGIC" in valid_body + assert valid_body.count("return 0;") >= 2, ( + "both the too-short case and the bad-magic case must return falsy" + ) + + batch_body = _function_body(main, "run_batch_file") + check_call = re.search(r"response_is_valid\s*\(", batch_body) + write_call = re.search(r"fopen\s*\(\s*out_path", batch_body) + assert check_call and write_call + assert check_call.start() < write_call.start(), ( + "the output file must never be opened before the response is " + "confirmed valid" + ) + + +# ============================================================================== +# --unmapped, --self-test's printed cycles_total, and --coherency-check. +# +# NONE of these three can be exercised by actually running hexlib_run (no +# device here) -- see test_runtime_device_build.py's own "NEITHER ARTIFACT IS +# EVER RUN HERE" note. So, like every other test in this file, these check +# SOURCE STRUCTURE: real call order, real branches, real string literals -- +# never merely "the flag's name appears somewhere in the file", which a stale +# comment or a dead branch would also satisfy. +# ============================================================================== + + +def test_unmapped_alloc_skips_only_the_dsp_registration_call(main): + """alloc_maybe_unmapped() must still run the ordinary CPU-side + rpcmem_alloc -> rpcmem_to_fd -> fastrpc_mmap sequence in full ("the host + allocates its rpcmem buffer and gets its fd as usual") -- only the + DSP-side hexlib_iface_mmap() registration (hexlib_bufs_register's table, + skel_bufs.c) is withheld when skip_dsp_register is true. Scoped to the + function itself and to each branch of its own if/else, not merely + "hexlib_iface_mmap is absent somewhere in the file", which the sibling + branch satisfying it would also make trivially true.""" + body = _function_body(main, "alloc_maybe_unmapped") + i_alloc = body.index("hexlib_rpcmem_alloc(") + i_fd = body.index("hexlib_rpcmem_to_fd(") + i_map = body.index("hexlib_fastrpc_mmap(") + assert i_alloc < i_fd < i_map, ( + "the CPU-side allocation sequence must run in the same order " + "hexlib_alloc() (buffers.c) uses, unconditionally" + ) + + skip_if = re.search(r"if\s*\(\s*skip_dsp_register\s*\)", body) + assert skip_if, "alloc_maybe_unmapped must branch on skip_dsp_register" + skip_block = _block_from(body, skip_if.end()) + assert "hexlib_iface_mmap(" not in skip_block, ( + "the skip_dsp_register branch must NOT call hexlib_iface_mmap -- " + "withholding exactly that call is the whole point of --unmapped" + ) + # AND NOTHING THAT COULD REGISTER IT UNDER ANOTHER NAME. The named ban above + # is satisfied by moving the registration into a one-line helper and calling + # THAT from here, which would silently un-break --unmapped: the fd would be + # registered after all, hexlib_bufs_map would find it, and the mode would + # report success for the case it exists to make fail. An exhaustive callee + # set cannot be dodged by a rename (see csource.calls()). + assert _calls(skip_block) == {"printf"}, ( + f"the --unmapped branch must do nothing but say so on stdout; found " + f"calls to {_calls(skip_block)!r}" + ) + + else_pos = body.index("else", skip_if.end()) + else_block = _block_from(body, else_pos) + assert "hexlib_iface_mmap(" in else_block, ( + "the ordinary (mapped) branch must still register the fd with the " + "skel, exactly like hexlib_alloc() does" + ) + + +def test_run_self_test_unmapped_path_uses_the_unmapped_allocator(main): + """run_self_test(unmapped): when the flag is set, BOTH self-test buffers + must go through alloc_maybe_unmapped(..., 1)/free_maybe_unmapped(..., 1) + -- not the ordinary hexlib_alloc()/hexlib_free() -- so the DSP's + hexlib_bufs_map() table lookup (skel_bufs.c) genuinely has nothing to + find for either fd.""" + body = _function_body(main, "run_self_test") + assert "alloc_maybe_unmapped(ctx, &bx, nbytes, 1)" in body + assert "alloc_maybe_unmapped(ctx, &by, nbytes, 1)" in body + assert "free_maybe_unmapped(ctx, bx, 1)" in body + assert "free_maybe_unmapped(ctx, by, 1)" in body + + +def test_self_test_prints_cycles_total_after_the_existing_pass_line(main, main_strings): + """The response header's cycles_total (skel_dispatch.c's PCYCLE bracket + around the kernel call) must be printed AFTER, never instead of, the + existing 'PASS (%d values, bit-exact)' line -- so the exact success + string test_on_device.py's `test_scale_fp16_runs_on_the_dsp_and_is_ + correct` already asserts on stays byte-for-byte intact, and the new + cycles line is strictly additive. + + BOTH ORDERED THINGS ARE PRINTF FORMAT STRINGS, so this is one of the few + checks that must run against the literal-bearing view -- the claim is + literally about what gets printed and in what order. The one non-literal + half (the value printed comes off the response header, not a constant) + stays on the ordinary view.""" + body_strings = _function_body(main_strings, "run_self_test", keep_strings=True) + pass_idx = body_strings.index("PASS (%d values, bit-exact)") + cycles_idx = body_strings.index("cycles_total=%llu", pass_idx) + assert pass_idx < cycles_idx + body = _function_body(main, "run_self_test") + assert "full_hdr.cycles_total" in body + + +def test_self_test_flag_parsing_routes_unmapped_and_coherency_correctly(main, main_strings): + """main()'s --self-test branch must recognize both --unmapped and + --coherency-check past argv[1], route --coherency-check to + run_coherency_check(), and thread the --unmapped flag straight into + run_self_test(unmapped) -- not merely mention both flag strings + somewhere in the function, which a comment or an unreachable branch + would also satisfy. + + THE FLAG NAMES ARE STRING LITERALS -- argv is compared against them -- so + finding them needs the literal-bearing view. The ROUTING half (the branch + exists and calls the right function) stays on the ordinary, blanked view, + which is the half a FARF or a usage() line could otherwise satisfy. The two + views are the same length, so the `"--self-test"` offset found in one is + the right offset to brace-slice the other from.""" + body = _function_body(main, "main") + body_strings = _function_body(main_strings, "main", keep_strings=True) + assert len(body) == len(body_strings) + self_test_pos = body_strings.index('"--self-test"') + self_test_block = _block_from(body, self_test_pos) + self_test_block_strings = _block_from(body_strings, self_test_pos, + keep_strings=True) + + assert '"--unmapped"' in self_test_block_strings + assert '"--coherency-check"' in self_test_block_strings + assert re.search(r"run_coherency_check\s*\(\s*\)", self_test_block) + assert re.search(r"run_self_test\s*\(\s*unmapped\s*\)", self_test_block) + + coherency_if = re.search(r"if\s*\(\s*coherency\s*\)", self_test_block) + assert coherency_if, "--coherency-check must be checked as its own branch" + coherency_block = _block_from(self_test_block, coherency_if.end()) + assert re.search(r"run_coherency_check\s*\(\s*\)", coherency_block), ( + "the coherency branch must actually call run_coherency_check(), not " + "merely check the flag and fall through" + ) + + +def test_usage_mentions_the_new_self_test_modifiers(main_strings): + """LITERAL-BEARING VIEW BY NATURE: usage() text is nothing but string + literals, and what this asserts is that a human running --help is told + about both modifiers.""" + body = _function_body(main_strings, "usage", keep_strings=True) + assert "--unmapped" in body + assert "--coherency-check" in body + + +def test_the_hosts_scale_kind_id_is_the_same_number_the_dsp_dispatches_on(main): + """THE ONE HAND-COPIED KIND ID, BOUND TO ITS SOURCE OF TRUTH. + + `genentry.KIND_ID` is where kind ids live; `genentry.emit_table` writes the + DSP's dispatch table straight from it. This host binary has no generated + header to read, so `#define HEXLIB_KIND_SCALE 9u` is a hand-copy, pinned by + a comment and -- until now -- by nothing executable. Mutating it to `10u` + left the entire offline suite green. + + A WRONG VALUE HERE IS NOT ALWAYS LOUD. main.c's own comment argues it is + (`hexlib_dispatch_batch` would answer ERR_NO_KERNEL, which --self-test + reports as a failure), and that is true only while the number it drifts to + is unregistered. 10 is `softmax` and 11 is `transpose`: as soon as either + has a kernel, a `scale` request dispatches to it, the buffer count matches, + both pointers are non-null, and the status is HEXLIB_DSP_OK. The wire + carries no table version and `skel_dispatch.c` matches on the id alone, so + nothing else in the system can notice. + + Read off the COMMENT-BLANKED source, so the "== 9" in the explanatory + comment above the `#define` cannot satisfy this. Mutating either side -- + the `#define` or the Python dict -- fails it. + """ + from hexlib.runtime.genentry import KIND_ID + + m = re.search(r"#define\s+HEXLIB_KIND_SCALE\s+(\d+)u\b", main) + assert m, "main.c no longer defines HEXLIB_KIND_SCALE as a decimal literal" + assert int(m.group(1)) == KIND_ID["scale"], ( + f"main.c dispatches scale as kind {m.group(1)}; genentry.KIND_ID says " + f"{KIND_ID['scale']}. One of the two copies drifted, and the DSP obeys " + f"the id it is sent." + ) + + +def test_build_scale_batch_factor_is_call_site_specific(main): + """run_self_test must build its batch with SELF_TEST_FACTOR (0.125f, a + power of two, exact in fp16) and run_coherency_check must use + COHERENCY_FACTOR (0.0f) -- never the other's constant, since a nonzero + factor in the coherency check would let a wrong result be blamed on + kernel arithmetic instead of ruling that out entirely.""" + self_test_body = _function_body(main, "run_self_test") + coherency_body = _function_body(main, "run_coherency_check") + assert re.search(r"build_scale_batch\([^)]*SELF_TEST_FACTOR", self_test_body) + assert re.search(r"build_scale_batch\([^)]*COHERENCY_FACTOR", coherency_body) + assert "SELF_TEST_FACTOR" not in coherency_body, ( + "the coherency check must never fall back to the self-test's own " + "nonzero factor" + ) + + +def test_coherency_check_writes_the_sentinel_before_invoking(main): + """The sentinel must be written into the OUTPUT buffer strictly before + hexlib_invoke() -- writing it afterwards would prove nothing about + whether the DSP's own write reached the host.""" + body = _function_body(main, "run_coherency_check") + sentinel_idx = body.index("COHERENCY_SENTINEL") + invoke_idx = body.index("hexlib_invoke(") + assert sentinel_idx < invoke_idx + + +def test_coherency_check_reads_the_sentinel_only_after_both_statuses_are_ok( + main, main_strings +): + """The sentinel read-back (and both printed verdict lines) must live + strictly inside the branch reached only once the batch-level status AND + the op's own result status are both confirmed HEXLIB_DSP_OK -- reading it + any earlier would make a marshalling failure indistinguishable from a + coherency one, exactly the confusion this check exists to resolve.""" + body = _function_body(main, "run_coherency_check") + op_ok_check = re.search(r"result->status\s*!=\s*HEXLIB_DSP_OK", body) + assert op_ok_check, "must check the op's own status, not merely the batch-level one" + else_pos = body.index("else", op_ok_check.end()) + success_block = _block_from(body, else_pos) + + assert "hexlib_classify_coherency_lane(" in success_block, ( + "the sentinel must only be read back (and classified) once both " + "statuses are confirmed OK" + ) + # The three ORDERED things are printf format strings, so their relative + # order is a claim about literal text and is checked in the literal-bearing + # view. Same offsets (blanking preserves length), so the `else` boundary + # found above is the right one to slice there too. + body_strings = _function_body(main_strings, "run_coherency_check", + keep_strings=True) + success_block_strings = _block_from(body_strings, else_pos, keep_strings=True) + cycles_idx = success_block_strings.index("cycles_total=%llu") + overwritten_idx = success_block_strings.index('"COHERENCY sentinel_overwritten\\n"') + unchanged_idx = success_block_strings.index('"COHERENCY sentinel_unchanged\\n"') + assert cycles_idx < overwritten_idx + assert cycles_idx < unchanged_idx, ( + "cycles_total must be printed before either COHERENCY verdict line " + "-- it is the signal that tells a dispatch bug (0 cycles) apart from " + "a genuine coherency miss (>0 cycles), and both must be visible " + "together regardless of which branch runs" + ) + + +def test_coherency_miss_has_its_own_distinct_exit_code(main): + """A coherency miss (or a dispatch bug -- see the function's own header + comment on why cycles_total, not this exit code, is what tells the two + apart) must exit with something other than 0-5, which are all already + claimed by other outcomes.""" + assert re.search(r"HEXLIB_EXIT_COHERENCY_MISS\s*=\s*6", main) + body = _function_body(main, "run_coherency_check") + assert "exit_code = HEXLIB_EXIT_COHERENCY_MISS;" in body + + +def test_coherency_check_treats_negative_zero_as_the_expected_zero_result(main): + """`x * 0.0f` is -0.0, not +0.0, whenever `x` is negative -- true of many + lanes of the self-test's own input -- and there is no -ffast-math here + (toolchain.py) to make that not so. A bit-exact compare of the read-back + buffer against `(__fp16) 0.0f` would misclassify that HEALTHY result as + "sentinel unchanged" and report a coherency miss that never happened. + + THIS IS A SOURCE ASSERTION ONLY -- it can confirm the sign-bit mask + exists in hexlib_classify_coherency_lane(), never that it actually + classifies 0x8000 as zero. See + hexlib/tests/test_coherency_lane_classification.py for the genuine, + compiled-and-run behavioural test of that exact function against that + exact bit pattern -- the same defect class as the arch-decode fix (see + test_session_arch_decode.py), closed the same way.""" + classify_body = _function_body(main, "hexlib_classify_coherency_lane") + assert re.search(r"&\s*0x7[Ff]{3}[Uu]?\b", classify_body), ( + "the expected-zero classification must mask off the sign bit " + "(0x7FFF), not compare bit-exact equality to +0.0 alone -- see this " + "function's own header comment on why -0.0 must count as zero" + ) + body = _function_body(main, "run_coherency_check") + # THIS NEGATIVE WAS VACUOUS AND IS NOW BOUND TO SOMETHING REAL. It used to + # be `not re.search(r"memcmp\(&yr\[i\],\s*&zero\b", main)` -- text that + # has never existed anywhere in main.c, in any revision, so the assertion + # could not fail no matter what the C did. What it MEANT to forbid is a + # bit-exact byte compare standing in for the magnitude classification, so + # forbid that: run_coherency_check's read-back loop must reach its verdict + # only through hexlib_classify_coherency_lane(). It uses memcpy (to get at + # raw bits) and never memcmp, so any memcmp appearing in this function is + # a comparison that has bypassed the classifier -- which is exactly the + # regression. Verified by mutation: inserting a memcmp here fails this. + assert "memcmp(" not in body, ( + "run_coherency_check must not compare the read-back buffer with " + "memcmp -- every lane's verdict goes through " + "hexlib_classify_coherency_lane(), and a bit-exact byte compare is " + "how the -0.0 false coherency miss happened the first time" + ) + assert re.search(r"hexlib_classify_coherency_lane\s*\(", body), ( + "run_coherency_check must classify each lane through " + "hexlib_classify_coherency_lane(), not reimplement the check inline " + "-- see test_coherency_lane_classification.py for why that function " + "must stay the one thing exercised behaviourally" + ) + + +def test_coherency_check_verifies_the_surviving_bytes_are_really_the_sentinel( + main, main_strings +): + """A buffer that is neither the expected zero result nor the intact + sentinel (garbled, or partially written) must not be folded into the + 'sentinel_unchanged' / coherency-miss verdict just because it failed the + zero check -- it is a third, distinct outcome and must be its own + branch with its own exit code.""" + classify_body = _function_body(main, "hexlib_classify_coherency_lane") + assert re.search(r"bits\s*==\s*sentinel_bits", classify_body), ( + "the surviving bits must be compared, bit-exact, against the real " + "sentinel's own bits -- not merely assumed to be the sentinel " + "because they were not zero" + ) + assert "HEXLIB_LANE_OTHER" in classify_body, ( + "a lane that is neither zero nor the sentinel must be its own, " + "third classification -- not folded into either of the other two" + ) + + body = _function_body(main, "run_coherency_check") + assert "HEXLIB_EXIT_COHERENCY_GARBLED" in main + assert "exit_code = HEXLIB_EXIT_COHERENCY_GARBLED;" in body + # The printed verdict line itself -- a literal, so the literal-bearing view. + body_strings = _function_body(main_strings, "run_coherency_check", + keep_strings=True) + assert '"COHERENCY buffer_garbled\\n"' in body_strings + + +def test_caps_reports_a_driver_failure_through_its_exit_code(main, main_strings): + """`--caps` EXITED 0 WHEN THE DRIVER FAILED TO LOAD. `print_caps()` + returned `void`, both failure branches printed to stderr and returned, and + `main()` returned HEXLIB_EXIT_OK regardless -- so on a device whose image + has no `libcdsprpc.so` for this ABI, `./hexlib_run --caps; echo RC=$?` + printed "could not load the FastRPC driver" and `RC=0`, which any `set -e` + wrapper or CI step reads as a pass. It is also the first mode run on + unfamiliar silicon and the one most likely to fail there. + + Three things are checked, because the defect needed all three to be wrong: + the function returns int, EVERY early return in it carries a nonzero exit + constant, and main() actually propagates the value instead of discarding + it.""" + m = re.search(r"\bstatic\s+int\s+print_caps\s*\(\s*void\s*\)", main) + assert m, ( + "print_caps must return an exit code, not void -- a void return is " + "why a driver-load failure exited 0" + ) + body = _function_body(main, "print_caps") + returns = re.findall(r"return\s+([^;]+);", body) + assert returns, "print_caps must return something" + assert all(r.strip().startswith("HEXLIB_EXIT_") for r in returns), ( + f"every return in print_caps must be a named exit code, got {returns!r}" + ) + # The two failure branches must NOT return OK; the last (success) one must. + assert returns[-1].strip() == "HEXLIB_EXIT_OK", ( + f"print_caps's final, success return must be OK, got {returns[-1]!r}" + ) + for r in returns[:-1]: + assert r.strip() != "HEXLIB_EXIT_OK", ( + "a print_caps failure branch returns HEXLIB_EXIT_OK -- that is the " + "original defect, moved rather than fixed" + ) + + # The flag itself is a literal, so it is located in the literal-bearing + # view; the block's CONTENT is then checked in the blanked one, where + # neither a comment nor a log line can supply the return this is about. + main_body = _function_body(main, "main") + main_body_strings = _function_body(main_strings, "main", keep_strings=True) + caps_pos = main_body_strings.index('"--caps"') + caps_block = _block_from(main_body, caps_pos) + assert re.search(r"return\s+print_caps\s*\(\s*\)\s*;", caps_block), ( + "main() must RETURN print_caps()'s value -- calling it and then " + "returning HEXLIB_EXIT_OK is exactly the bug" + ) + assert "HEXLIB_EXIT_OK" not in caps_block, ( + "main()'s --caps branch must not name a constant exit code at all; " + "the code comes from print_caps()" + ) + + +# The places §6.1's coherency table is written down. A doc claiming a +# guarantee the code does not deliver is, on this project, a defect at the same +# weight as a code bug -- so the correction has to land in all of them or the +# stale one becomes the one someone reads on the first device job. +# +# THERE WERE THREE. The design spec +# (`docs/superpowers/specs/2026-08-10-silicon-path-runtime-design.md`) was the +# third, and it left the repository entirely when `docs/` was untracked and +# purged from history. A test cannot assert against a file the repository does +# not contain: it passes on the machine that still has the untracked copy on +# disk and fails on every fresh clone, which is the CI-only failure this suite +# exists to avoid. The site is dropped rather than made conditional -- a +# `skipif` here would be a check that silently protects nothing everywhere it +# actually runs. +_COHERENCY_TABLE_SITES = ( + pathlib.Path("hexlib/runtime/host/main.c"), + pathlib.Path("hexlib/device/qdc/test_on_device.py"), +) + +# Each element of the correction, and why it must be present in every copy: +# "unreachable" -- row 1 (`cycles 0` + sentinel intact -> "a dispatch +# bug") cannot happen: reaching the read-back at all +# requires both statuses OK, which requires k->fn to +# have been called and returned OK, and PCYCLE brackets +# exactly that call. +# "not discriminated" -- the `sentinel_unchanged` row is consistent with a +# coherency miss AND with a kernel/entry that returned +# OK without writing `y`. Calling it "COHERENCY" is the +# misattribution §6.1 exists to prevent. +# "PCYCLEEN" -- if the counter does not advance in the unsigned PD, +# every row inverts; that is why a zero is asserted +# against rather than assumed impossible. +# "buffer_garbled" -- the third outcome a previous fix added must appear in +# the table too, or the table is still incomplete. +_CORRECTION_ELEMENTS = ("unreachable", "not discriminated", "pcycleen", "buffer_garbled") + + +# IDS ARE HAND-WRITTEN, NOT DERIVED FROM THE FILENAME. `ids=lambda p: p.name` +# put the literal text `test_on_device.py` into a node id, and +# test_qdc_on_device_is_excluded.py asserts that exact string never appears in +# `pytest --collect-only` output (its way of proving the on-device file is not +# collected) -- so a parametrize id here made THAT test fail, on a file that +# was correctly excluded. Reproduced before this comment existed. +@pytest.mark.parametrize( + "path", _COHERENCY_TABLE_SITES, ids=("host_main", "device_test") +) +def test_the_coherency_table_correction_landed_everywhere_it_is_written_down(path): + """READ WITH COMMENTS ON, DELIBERATELY -- unlike every other check in this + file. The subject IS the prose: §6.1's table is a claim made to a human + about what the first device job's output will mean, and it was asserting a + separation the code does not achieve. BOTH remaining copies are comments + (main.c's `run_coherency_check` header, test_on_device.py's docstring), so + blanking comments would make this assert nothing. + + Deleting the correction from EITHER fails this.""" + text = path.read_text(encoding="utf-8").lower() + missing = [e for e in _CORRECTION_ELEMENTS if e not in text] + assert not missing, ( + f"{path} is missing part of §6.1's corrected coherency table: " + f"{missing!r}. Both copies must say the same thing -- a stale one " + f"is the copy someone reads while triaging job 1." + ) + + +def test_coherency_check_documents_its_own_scope_limits(main_comments): + """Design doc §6.1 (corrected 2026-08-11): the table that makes + cycles_total load-bearing covers ONLY the DSP-write -> host-read + direction, for scale_fp16's own write pattern -- not the reverse + direction, and not every kernel. That caveat must live in this file's + own comments, not only in the on-device test's docstring, or a future + reader of just this file could believe a pass here is a general + coherency proof. + + THE ONE TEST IN THIS FILE THAT TAKES `main_comments`, NOT `main`. Its + subject IS the comment text -- a prose caveat written for a human reader -- + so blanking comments out would make it assert nothing and it would fail + immediately. Every other check here must use `main`; see the module + docstring.""" + main = main_comments + assert "DSP-write" in main and "host-read" in main + assert "host-write" in main and "DSP-read" in main + assert "kernel-independent" in main.lower() + + +def test_the_self_test_batch_names_its_layout_instead_of_writing_a_bare_zero(main): + """`tens[i].layout` must be spelled with the macro, not a literal. + + This was `tens[i].layout = 0;` with a trailing comment naming + `LAYOUT_ID["row_major"]` -- and unlike the `dtype = 1` literal beside it, + which genentry's emitted `a->dtype[0] != 1u` guard rejects loudly at run + time, NOTHING checked layout at all. `grep -c layout` over this file was 0. + + A bare 0 is not wrong today; it is unbound. Insert a layout ahead of + row_major in `wire.LAYOUT_ID` and `pack_batch` emits 1 while this file + keeps emitting 0, and `--self-test` still prints `PASS (4100 values, + bit-exact)` over a buffer the batch declared as something else. The enum + exists (per hexlib_dsp.h's own header) so that un-repacked weights are a + plan-time error rather than silent corruption, and `LAYOUT_ID` already + carries `q4_0_repacked` for the matmul that comes next. + + The macro's VALUE is bound to `wire.LAYOUT_ID` by a compiled probe in + test_wire_struct_layout.py; this test only pins that main.c goes through + the macro. Read off comment-blanked source, so the explanatory comment + cannot satisfy it. + """ + body = _function_body(main, "build_scale_batch") + assert "HEXLIB_LAYOUT_ROW_MAJOR" in body, ( + "build_scale_batch must set tens[].layout from HEXLIB_LAYOUT_ROW_MAJOR, " + "not from a bare integer whose only tie to wire.LAYOUT_ID is a comment" + ) + assert not re.search(r"\.layout\s*=\s*\d", body), ( + "a numeric literal is being assigned to .layout again; use the macro" + ) diff --git a/hexlib/tests/test_hostload.py b/hexlib/tests/test_hostload.py new file mode 100644 index 0000000..1952d66 --- /dev/null +++ b/hexlib/tests/test_hostload.py @@ -0,0 +1,96 @@ +"""A timeout must be attributed from measurement, never from a guess. + +These are the cases that actually occurred on matmul_fp16 in one session: +a stage that was genuinely over budget on an idle host, and a stage that had +already passed and only failed because a game was launched while it ran. +""" +import pytest + +from hexlib import hostload +from hexlib.hostload import LoadStats + + +def test_cpu_share_is_cpu_seconds_per_wall_second(): + assert LoadStats(wall_s=100.0, cpu_s=98.0, samples=50).cpu_share == pytest.approx(0.98) + + +def test_unmeasured_share_is_none_not_zero(): + """None means UNKNOWN. Zero would mean 'fully starved' and convict the host.""" + stats = LoadStats(wall_s=100.0, cpu_s=None, samples=0) + assert stats.cpu_share is None + assert stats.starved is None + + +def test_zero_wall_time_does_not_divide_by_zero(): + assert LoadStats(wall_s=0.0, cpu_s=1.0, samples=1).cpu_share is None + + +def test_starved_is_tri_state(): + assert LoadStats(wall_s=100.0, cpu_s=40.0, samples=9).starved is True + assert LoadStats(wall_s=100.0, cpu_s=99.0, samples=9).starved is False + assert LoadStats(wall_s=100.0, cpu_s=None, samples=0).starved is None + + +def test_contended_host_is_not_blamed_on_the_kernel(): + """nearmiss_wrong_batch_stride: passed in 825s, then blew 1800s because a + game launched mid-stage. The kernel must not be implicated.""" + stats = LoadStats(wall_s=1800.0, cpu_s=700.0, samples=400) + msg = hostload.timeout_diagnosis(1800.0, stats) + assert "HOST was contended" in msg + assert "idle host" in msg + assert "not evidence that the kernel fails to terminate" in msg + + +def test_an_undescheduled_timeout_points_at_the_code_without_exonerating_the_host(): + """A full CPU share means NOT DESCHEDULED, which is weaker than "the host + was idle" -- and the message must not overclaim. + + This assertion was originally `"not starving it" in msg`, and the message + it pinned was wrong. On 2026-08-13 a byte-identical near-miss ELF ran 1218s + once and exceeded 1800s twice, all at ~99% CPU share: memory-bandwidth + contention and SMT siblings slow a process that is never descheduled. A + diagnosis that reads "the host was not starving it" sends the next reader + to hunt a kernel bug that is not there. + """ + stats = LoadStats(wall_s=900.0, cpu_s=890.0, samples=400) + msg = hostload.timeout_diagnosis(900.0, stats) + assert "not DESCHEDULED" in msg + assert "WITHOUT ruling out a loaded host" in msg + + +def test_unmeasured_timeout_reports_the_ambiguity(): + """Without psutil the honest answer is 'cannot tell', not a default blame.""" + stats = LoadStats(wall_s=900.0, cpu_s=None, samples=0) + msg = hostload.timeout_diagnosis(900.0, stats) + assert "could not be measured" in msg + assert "does NOT distinguish" in msg + + +def test_threshold_boundary_is_not_starved(): + stats = LoadStats(wall_s=100.0, cpu_s=hostload.STARVED_BELOW * 100.0, samples=9) + assert stats.starved is False + + +def test_monitor_without_psutil_degrades_to_unmeasured(monkeypatch): + """An absent optional dependency must not fail a gate.""" + monkeypatch.setattr(hostload, "psutil", None) + with hostload.SimLoadMonitor() as mon: + pass + stats = mon.stats() + assert stats.cpu_s is None + assert stats.starved is None + assert stats.wall_s >= 0.0 + + +def test_monitor_survives_a_sampler_that_raises(monkeypatch): + """Sampling is diagnostic; an error in it must never be why a gate fails.""" + class Boom: + @staticmethod + def Process(*a, **k): + raise RuntimeError("no such process") + Error = RuntimeError + + monkeypatch.setattr(hostload, "psutil", Boom) + with hostload.SimLoadMonitor(interval_s=0.01) as mon: + pass + assert mon.stats().cpu_s is None diff --git a/hexlib/tests/test_kernels.py b/hexlib/tests/test_kernels.py index 4afd9f1..2b7ff43 100644 --- a/hexlib/tests/test_kernels.py +++ b/hexlib/tests/test_kernels.py @@ -7,32 +7,27 @@ impractical (the defect below is a static out-of-bounds read that the standalone Hexagon simulator has no sanitizer to catch, and which is unreachable under every conforming call anyway). + +THE SLICER IS SHARED, NOT COPIED. This file carried a fourth private copy of +the brace-counting slicer `hexlib/tests/csource.py` exists to consolidate, and +the weakest of the four: `src.index("void " + name + "(")` with no comment +handling at all. That mattered here specifically, because the assertion below +is `"else" not in body` -- a NEGATIVE check, which any comment inside +rmsnorm_fp16 containing the word "else" (or "otherwise... else", or a prose +mention of the removed branch) would trip, and which the comment-blanked slice +from `csource.function_body` cannot be tripped by. Migrated. """ from __future__ import annotations import pathlib import re +from hexlib.tests.csource import function_body as _function_body + REPO_ROOT = pathlib.Path(__file__).resolve().parents[2] RMSNORM_KERNEL_C = REPO_ROOT / "kernels" / "rmsnorm_fp16" / "kernel.c" -def _function_body(src: str, name: str) -> str: - start = src.index(f"void {name}(") - # Body starts at the first '{' after the signature and ends at the - # matching '}' (no nested braces of that name appear before it here). - brace = src.index("{", start) - depth = 0 - for i in range(brace, len(src)): - if src[i] == "{": - depth += 1 - elif src[i] == "}": - depth -= 1 - if depth == 0: - return src[brace:i + 1] - raise AssertionError(f"unbalanced braces in {name}") - - def test_rmsnorm_fp16_has_no_dead_and_unsafe_else_branch(): """Ledger #3 (final whole-branch review): `if (nb > 0) {...} else { accSq = Q6_Vqf16_vmpy_VhfVhf(xv[0], xv[0]); }` inside the row loop was diff --git a/hexlib/tests/test_qdc.py b/hexlib/tests/test_qdc.py new file mode 100644 index 0000000..3441581 --- /dev/null +++ b/hexlib/tests/test_qdc.py @@ -0,0 +1,639 @@ +# hexlib/tests/test_qdc.py +"""QDC submission and completion detection, against a fake API client. + +WHY COMPLETION IS DETECTED FROM LOG FILES. `get_job_status` returns +`state=None`; `get_jobs_list` lagged more than 30 minutes on both observed jobs. +Polling either means either hanging forever or declaring success early. A job +that ran zero tests once reported passing on this account, which is the failure +mode all of this exists to make impossible. +""" +import ast +import collections +import math +import pathlib +import re +import zipfile + +import pytest + +from hexlib.device.qdc import artifact, job + + +def test_target_is_sm8650_and_the_id_is_pinned(): + assert job.DEVICE == "SM8650" + assert job.TARGET_ID == 3625030 + + +def test_stage_produces_a_zip_containing_every_binary(tmp_path): + (tmp_path / "hexlib_run").write_bytes(b"\x7fELF fake") + (tmp_path / "libhexlib_skel.so").write_bytes(b"\x7fELF fake") + test_py = tmp_path / "test_on_device.py" + test_py.write_text("def test_x(): pass\n") + z = artifact.stage( + [str(tmp_path / "hexlib_run"), str(tmp_path / "libhexlib_skel.so")], + str(test_py), str(tmp_path / "job"), + ) + names = zipfile.ZipFile(z).namelist() + assert any("hexlib_run" in n for n in names) + assert any("libhexlib_skel.so" in n for n in names) + assert any("test_on_device.py" in n for n in names) + assert any("pytest.ini" in n for n in names) + + +def test_stage_refuses_a_missing_binary(tmp_path): + with pytest.raises(artifact.StagingError, match="not found"): + artifact.stage([str(tmp_path / "nope")], None, str(tmp_path / "job")) + + +# --- 0-byte inputs ------------------------------------------------------- +# +# `stage` checked existence only, and was VERIFIED to accept four 0-byte files +# and produce a perfectly submittable zip. A link or copy that fails part way +# leaves exactly that: a `libhexlib_skel.so` of length zero, present, correctly +# named, and completely unrunnable -- discovered on the device as a dlopen +# failure with no obvious cause, after the minutes are spent. + + +def test_stage_refuses_a_zero_byte_binary(tmp_path): + (tmp_path / "hexlib_run").write_bytes(b"\x7fELF fake") + (tmp_path / "libhexlib_skel.so").write_bytes(b"") # a truncated link + with pytest.raises(artifact.StagingError, match="0 bytes"): + artifact.stage( + [str(tmp_path / "hexlib_run"), str(tmp_path / "libhexlib_skel.so")], + None, str(tmp_path / "job"), + ) + + +def test_stage_refuses_a_zero_byte_test_script(tmp_path): + (tmp_path / "hexlib_run").write_bytes(b"\x7fELF fake") + (tmp_path / "test_on_device.py").write_text("") + with pytest.raises(artifact.StagingError, match="0 bytes"): + artifact.stage( + [str(tmp_path / "hexlib_run")], + str(tmp_path / "test_on_device.py"), str(tmp_path / "job"), + ) + + +def test_stage_produces_no_zip_at_all_when_an_input_is_empty(tmp_path): + """The refusal must happen BEFORE anything submittable exists on disk -- + a zip left behind by a failed staging run is a zip somebody can submit.""" + (tmp_path / "hexlib_run").write_bytes(b"") + out_base = tmp_path / "job" + with pytest.raises(artifact.StagingError): + artifact.stage([str(tmp_path / "hexlib_run")], None, str(out_base)) + assert not (tmp_path / "job.zip").exists() + + +def test_submission_requires_an_explicit_timeout(tmp_path): + z = tmp_path / "a.zip" + z.write_bytes(b"PK") + with pytest.raises(TypeError): + job.submit(str(z)) # timeout_min is required, not defaulted + + +def test_submission_refuses_a_zero_or_absurd_timeout(tmp_path, monkeypatch): + z = tmp_path / "a.zip" + z.write_bytes(b"PK") + with pytest.raises(job.QdcError, match="timeout"): + job.submit(str(z), timeout_min=0) + with pytest.raises(job.QdcError, match="timeout"): + job.submit(str(z), timeout_min=10000) + + +def test_wait_polls_log_files_and_never_job_status(monkeypatch): + calls = {"logs": 0} + + class F: + filename = "TestLogs/results.xml" + + def fake_logs(client, job_id): + calls["logs"] += 1 + return [F()] if calls["logs"] >= 2 else [] + + def boom(*a, **k): + raise AssertionError("get_job_status must never be polled") + + monkeypatch.setattr(job, "_client", lambda: object()) + monkeypatch.setattr(job.qdc_api, "get_job_log_files", fake_logs, raising=False) + monkeypatch.setattr(job.qdc_api, "get_job_status", boom, raising=False) + monkeypatch.setattr(job, "POLL_S", 0) + assert job.wait(1234, cap_s=10) is True + assert calls["logs"] >= 2 + + +def test_wait_never_polls_the_jobs_list_either(monkeypatch): + # get_jobs_list lagged more than 30 minutes on both jobs observed on + # this account -- as dangerous as get_job_status, and nothing stops a + # future edit from wiring it into wait() by mistake. This guard exists + # so that edit fails a test instead of silently reintroducing the + # exact failure mode this module was built to make impossible. + calls = {"logs": 0} + + class F: + filename = "TestLogs/results.xml" + + def fake_logs(client, job_id): + calls["logs"] += 1 + return [F()] if calls["logs"] >= 2 else [] + + def boom(*a, **k): + raise AssertionError("get_jobs_list must never be polled") + + monkeypatch.setattr(job, "_client", lambda: object()) + monkeypatch.setattr(job.qdc_api, "get_job_log_files", fake_logs, raising=False) + monkeypatch.setattr(job.qdc_api, "get_jobs_list", boom, raising=False) + monkeypatch.setattr(job, "POLL_S", 0) + assert job.wait(1234, cap_s=10) is True + assert calls["logs"] >= 2 + + +def test_wait_returns_false_at_the_cap_rather_than_hanging(monkeypatch): + monkeypatch.setattr(job, "_client", lambda: object()) + monkeypatch.setattr(job.qdc_api, "get_job_log_files", + lambda c, j: [], raising=False) + monkeypatch.setattr(job, "POLL_S", 0) + assert job.wait(1234, cap_s=0) is False + + +def test_a_job_with_no_results_xml_is_not_complete(monkeypatch): + class F: + filename = "TestLogs/logcat.txt" + + monkeypatch.setattr(job, "_client", lambda: object()) + monkeypatch.setattr(job.qdc_api, "get_job_log_files", + lambda c, j: [F()], raising=False) + monkeypatch.setattr(job, "POLL_S", 0) + assert job.wait(1234, cap_s=0) is False + + +# --- wait() is a completion detector, NOT a verdict ---------------------- +# +# `_has_results` was `RESULTS_MARKER in filename`, a bare substring test, so +# `TestLogs/results.xml.part` -- the half-written intermediate whose appearance +# is the one thing a completion detector must not fire on -- counted as +# "finished". And even a genuine match proves only that a NAME appeared: +# `wait()` never opens the file, so a zero-byte results.xml satisfies it. That +# is why the real check lives in `hexlib/cli.py::_qdc_check_results`, and why +# job.py's docstrings now say so instead of claiming wait() makes a false pass +# impossible. + + +@pytest.mark.parametrize("name", [ + "TestLogs/results.xml.part", + "TestLogs/results.xml.tmp", + "TestLogs/results.xml.gz", + "TestLogs/results.xmlx", + "TestLogs/my_results.xml.bak", +]) +def test_a_partially_written_results_file_is_not_completion(monkeypatch, name): + class F: + filename = None + + F.filename = name + monkeypatch.setattr(job, "_client", lambda: object()) + monkeypatch.setattr(job.qdc_api, "get_job_log_files", + lambda c, j: [F()], raising=False) + monkeypatch.setattr(job, "POLL_S", 0) + assert job.wait(1234, cap_s=0) is False, ( + f"{name!r} is not TestLogs/results.xml -- a substring match on it " + "declares a job complete off a half-written file" + ) + + +@pytest.mark.parametrize("name", [ + "TestLogs/results.xml", + "job-1234/TestLogs/results.xml", + "job-1234\\TestLogs\\results.xml", +]) +def test_the_real_results_file_is_recognized_however_it_is_pathed(monkeypatch, name): + class F: + filename = None + + F.filename = name + monkeypatch.setattr(job, "_client", lambda: object()) + monkeypatch.setattr(job.qdc_api, "get_job_log_files", + lambda c, j: [F()], raising=False) + monkeypatch.setattr(job, "POLL_S", 0) + assert job.wait(1234, cap_s=0) is True + + +def test_a_log_entry_with_no_filename_at_all_does_not_crash_wait(monkeypatch): + class F: + filename = None + + class G: + pass + + monkeypatch.setattr(job, "_client", lambda: object()) + monkeypatch.setattr(job.qdc_api, "get_job_log_files", + lambda c, j: [F(), G()], raising=False) + monkeypatch.setattr(job, "POLL_S", 0) + assert job.wait(1234, cap_s=0) is False + + +# --- fetch() mirrors QDC's directory layout ------------------------------ +# +# `os.path.join(dest, os.path.basename(name))` FLATTENED it. QDC's listings are +# not flat, so `TestLogs/results.xml` and `logs/results.xml` both became +# `dest/results.xml`: one silently overwrote the other, the returned `paths` +# held two entries pointing at ONE file, and cli.py's results check then read +# whichever download happened to land last. The one file this whole path exists +# to read is identified by that basename. + + +def _fake_downloads(monkeypatch, names): + """A fake QDC that lists `names` and 'downloads' each by writing its own + remote name into the local file, so a collision is detectable by content.""" + class F: + def __init__(self, filename): + self.filename = filename + + monkeypatch.setattr(job, "_client", lambda: object()) + monkeypatch.setattr(job.qdc_api, "get_job_log_files", + lambda c, j: [F(n) for n in names], raising=False) + + def fake_download(client, remote, local): + with open(local, "w", encoding="utf-8") as f: + f.write(remote) + return True + + monkeypatch.setattr(job.qdc_api, "download_log_file", fake_download, + raising=False) + + +def test_fetch_does_not_let_two_logs_with_one_basename_overwrite_each_other( + monkeypatch, tmp_path +): + _fake_downloads(monkeypatch, ["TestLogs/results.xml", "logs/results.xml"]) + paths = job.fetch(1234, str(tmp_path / "d")) + + assert len(set(paths)) == 2, ( + f"two distinct remote logs collapsed onto one local path: {paths}" + ) + contents = sorted(open(p, encoding="utf-8").read() for p in paths) + assert contents == ["TestLogs/results.xml", "logs/results.xml"], ( + "one download overwrote the other -- the returned paths pointed at a " + "single file holding whichever finished last" + ) + + +def test_fetch_keeps_the_results_basename_findable_by_the_cli(monkeypatch, tmp_path): + """cli._qdc_check_results locates the report with + `os.path.basename(p) == "results.xml"`, so mirroring the directory layout + must not change what that sees.""" + import os as _os + + _fake_downloads(monkeypatch, ["TestLogs/results.xml", "TestLogs/logcat.txt"]) + paths = job.fetch(1234, str(tmp_path / "d")) + assert any(_os.path.basename(p) == "results.xml" for p in paths) + assert all(_os.path.isfile(p) for p in paths) + + +@pytest.mark.parametrize("evil", [ + "../../escaped.txt", + "TestLogs/../../escaped.txt", + "/etc/passwd", + "C:/Windows/System32/evil.txt", + "", + " ", +]) +def test_fetch_refuses_a_remote_name_that_would_escape_the_destination( + monkeypatch, tmp_path, evil +): + """QDC's own names have never looked like this, which is exactly why + nothing would notice if one did.""" + dest = tmp_path / "d" + _fake_downloads(monkeypatch, [evil]) + paths = job.fetch(1234, str(dest)) + assert paths == [] + assert not (tmp_path / "escaped.txt").exists() + + +def _inject_fake_sdk(monkeypatch, *, get_public_api_client_using_api_key, client_ctor=None): + """Inject a fake qualcomm_device_cloud_sdk package into sys.modules so + job._client()'s internal lazy imports resolve to fakes -- proving + _client()'s default-vs-override branching without the real SDK + installed or reachable, and without network access.""" + import sys + import types as _types + + vendor = _types.SimpleNamespace( + get_public_api_client_using_api_key=get_public_api_client_using_api_key, + ) + api_pkg = _types.SimpleNamespace(qdc_api=vendor) + sdk_pkg = _types.ModuleType("qualcomm_device_cloud_sdk") + sdk_pkg.api = api_pkg + if client_ctor is not None: + sdk_pkg.Client = client_ctor + + monkeypatch.setitem(sys.modules, "qualcomm_device_cloud_sdk", sdk_pkg) + monkeypatch.setitem(sys.modules, "qualcomm_device_cloud_sdk.api", api_pkg) + monkeypatch.setitem(sys.modules, "qualcomm_device_cloud_sdk.api.qdc_api", vendor) + + +def test_client_uses_the_sdks_default_endpoint_when_not_overridden(monkeypatch): + monkeypatch.delenv("QDC_BASE_URL", raising=False) + monkeypatch.setenv("QDC_API_KEY", "irrelevant-for-this-test") + calls = {} + + def fake_default(**kwargs): + calls["kwargs"] = kwargs + return "vendor-client" + + _inject_fake_sdk(monkeypatch, get_public_api_client_using_api_key=fake_default) + assert job._client() == "vendor-client" + assert calls["kwargs"]["api_key_header"] == "irrelevant-for-this-test" + + +def test_client_honors_a_base_url_override_without_the_sdk_default(monkeypatch): + monkeypatch.setenv("QDC_BASE_URL", "https://private-tenant.example/qdc") + monkeypatch.setenv("QDC_API_KEY", "irrelevant-for-this-test") + + def boom(**kwargs): + raise AssertionError( + "the SDK's default-endpoint client must not be built when " + "QDC_BASE_URL is set" + ) + + seen = {} + + class FakeClient: + def __init__(self, base_url=None, headers=None): + seen["base_url"] = base_url + seen["headers"] = headers + + _inject_fake_sdk(monkeypatch, get_public_api_client_using_api_key=boom, + client_ctor=FakeClient) + job._client() + assert seen["base_url"] == "https://private-tenant.example/qdc" + assert seen["headers"]["Authorization"] == "irrelevant-for-this-test" + + +def test_the_api_key_is_read_from_the_environment_not_committed(monkeypatch): + monkeypatch.delenv("QDC_API_KEY", raising=False) + monkeypatch.setattr(job, "_key_file", lambda: None) + with pytest.raises(job.QdcError, match="QDC_API_KEY"): + job._api_key() + + +# --- the credential scan ------------------------------------------------- +# +# WHAT THIS REPLACED, AND WHY IT HAD TO GO. This was: +# +# assert "qdc_api_key" not in src.lower() or "environ" in src or "home()" in src +# assert "Bearer " not in src +# +# job.py contains `os.environ`, so the first disjunction was UNCONDITIONALLY +# TRUE for the only file that could ever carry a credential: adding +# `_FALLBACK_KEY = ""` to job.py left this file +# reporting 1 passed. Proven by mutation, not inferred. This is the only test +# guarding "no credential lands in the repository", and it could not fail. +# +# THE `Bearer ` ASSERTION IS GONE ON PURPOSE, NOT OVERLOOKED. It was a vacuous +# negative -- nothing under hexlib/device has ever spelled it, so it could +# only ever pass -- and the property it gestured at is already checked +# behaviourally, with a real binding, by +# test_client_honors_a_base_url_override_without_the_sdk_default above: that +# test asserts `seen["headers"]["Authorization"] == "irrelevant-for-this-test"`, +# i.e. the raw key with NO prefix, which is QDC's actual header scheme (a bare +# Authorization value plus X-QCOM-TokenType: apikey -- see job.py's docstring). +# An `assert "Bearer " not in src` cannot distinguish "we correctly don't use +# OAuth-style prefixes" from "this file happens not to contain that word", and +# keeping a second, weaker, source-text version of an assertion that is already +# made behaviourally is how a suite accumulates tests that only look like +# coverage. +# +# CREDENTIALS IN THIS PROJECT ARE PERSONAL AND LOCAL: read from QDC_API_KEY or +# ~/.qdc_api_key, never committed, never in CI. Nothing below reads either one. + +DEVICE_DIR = pathlib.Path("hexlib/device") + +# Identifier fragments that mean "this name holds a credential". Matched against +# the whole lowercased name AND against its underscore-separated words, so +# `_FALLBACK_KEY`, `apiKey`, `SECRET_TOKEN` and `qdc_api_key` all hit. +_CRED_WORDS = frozenset({ + "key", "keys", "secret", "secrets", "token", "tokens", "password", + "passwd", "pwd", "credential", "credentials", "cred", "auth", "bearer", + "signature", "sig", +}) +_CRED_FRAGMENTS = ( + "apikey", "api_key", "access_key", "accesskey", "private_key", + "privatekey", "secret", "password", "passwd", "credential", "authtoken", + "auth_token", "token", +) + +# Prefixes real credentials from real providers actually carry. Checked against +# every string in every file under hexlib/device REGARDLESS of what it is +# assigned to, since a key pasted into an innocuously-named variable (or a +# non-Python file) is the same leak. Kept to unmistakable markers so this +# cannot fire on a legitimate constant. +_SECRET_PREFIXES = ( + "sk-", "sk_live_", "sk_test_", "rk_live_", "ghp_", "gho_", "ghs_", + "github_pat_", "xoxb-", "xoxp-", "xoxa-", "AKIA", "ASIA", "AIza", + "ya29.", "eyJhbGciO", # a JWT's own base64 header + "-----BEGIN", # any PEM private key block +) + +_ENV_VAR_NAME = re.compile(r"\A[A-Z][A-Z0-9_]*\Z") + + +def _shannon_entropy_bits_per_char(s): + counts = collections.Counter(s) + n = len(s) + return -sum((c / n) * math.log2(c / n) for c in counts.values()) + + +def _why_this_looks_like_a_secret(value): + """Return a reason string if `value` has the SHAPE of a credential, else + None. Every exclusion below exists to keep a legitimate constant in this + project from tripping it -- the point is a scan that can fail on a real + key, not one that fails on a URL and gets deleted six weeks later. + + Deliberately shape-based and value-blind: nothing here has, or needs, any + knowledge of what a real QDC key looks like.""" + if len(value) < 16: + return None # QDC_API_KEY, apikey, X-QCOM-* etc. + if "://" in value or value.startswith(("http", "www.", "/", ".", "-I", "--")): + return None # URLs, paths, flags + if " " in value or "\n" in value or "\t" in value: + return None # prose: error messages, shell lines + if "/" in value or "\\" in value: + return None # /data/local/tmp/... and friends + if value.isdigit(): + return None # TARGET_ID 3625030, timeouts, sizes + if _ENV_VAR_NAME.match(value): + return None # the NAME of an env var, e.g. QDC_API_KEY + if re.fullmatch(r"[A-Za-z0-9_.]*\.[A-Za-z0-9]{1,6}", value): + return None # filenames: results.xml, pytest.ini + + classes = sum(( + bool(re.search(r"[a-z]", value)), + bool(re.search(r"[A-Z]", value)), + bool(re.search(r"[0-9]", value)), + )) + if classes < 2: + return None # all-lowercase words, SCREAMING_CASE + + bits = _shannon_entropy_bits_per_char(value) + if bits < 3.0: + return None + return f"{len(value)} chars, {bits:.2f} bits/char, {classes} character classes" + + +def _credential_shaped(name): + low = name.lower().lstrip("_") + if any(f in low for f in _CRED_FRAGMENTS): + return True + return bool(_CRED_WORDS & set(w for w in low.split("_") if w)) + + +def _named_string_constants(tree): + """Yield (name, value, lineno) for every string literal in `tree` that is + bound to a NAME: an assignment target (`X = "..."`, `self.x = "..."`, + annotated or not), a keyword argument (`f(api_key="...")`), or a dict entry + with a literal string key (`{"Authorization": "..."}`). Those are the + places a credential actually gets written; using `ast` rather than text + matching means a comment or a docstring cannot trip it, and equally cannot + hide one.""" + def target_names(node): + if isinstance(node, ast.Name): + yield node.id + elif isinstance(node, ast.Attribute): + yield node.attr + elif isinstance(node, (ast.Tuple, ast.List)): + for e in node.elts: + yield from target_names(e) + + for node in ast.walk(tree): + if isinstance(node, (ast.Assign, ast.AnnAssign)): + targets = node.targets if isinstance(node, ast.Assign) else [node.target] + if isinstance(node.value, ast.Constant) and isinstance(node.value.value, str): + for t in targets: + for name in target_names(t): + yield name, node.value.value, node.value.lineno + elif isinstance(node, ast.Call): + for kw in node.keywords: + if (kw.arg and isinstance(kw.value, ast.Constant) + and isinstance(kw.value.value, str)): + yield kw.arg, kw.value.value, kw.value.lineno + elif isinstance(node, ast.Dict): + for k, v in zip(node.keys, node.values): + if (isinstance(k, ast.Constant) and isinstance(k.value, str) + and isinstance(v, ast.Constant) and isinstance(v.value, str)): + yield k.value, v.value, v.lineno + + +def _scan_python_source(path, src): + """Credential-shaped NAME bound to a secret-shaped VALUE.""" + findings = [] + for name, value, lineno in _named_string_constants(ast.parse(src)): + if not _credential_shaped(name): + continue + why = _why_this_looks_like_a_secret(value) + if why: + findings.append(f"{path}:{lineno}: {name} = a {why} string literal") + return findings + + +def _scan_raw_text(path, src): + """A provider's own credential prefix, anywhere, under any name -- covers + the case the AST scan cannot: a key assigned to a name nobody would flag, + or sitting in a non-Python file.""" + findings = [] + for lineno, line in enumerate(src.split("\n"), start=1): + for prefix in _SECRET_PREFIXES: + idx = line.find(prefix) + if idx != -1 and _why_this_looks_like_a_secret(line[idx:].strip().strip("'\"")): + findings.append(f"{path}:{lineno}: a literal beginning {prefix!r}") + return findings + + +def _scan_device_tree(): + """Every text file under hexlib/device, not just *.py: a credential in a + shell script, an .ini or a .json staged into the artifact is the same leak. + Skips __pycache__ and anything that is not decodable as UTF-8.""" + findings = [] + for p in sorted(DEVICE_DIR.rglob("*")): + if not p.is_file() or "__pycache__" in p.parts: + continue + try: + src = p.read_text(encoding="utf-8") + except UnicodeDecodeError: + continue + posix = p.as_posix() + if p.suffix == ".py": + findings += _scan_python_source(posix, src) + findings += _scan_raw_text(posix, src) + return findings + + +def test_no_credential_appears_anywhere_in_the_device_source(): + """THE ONLY test guarding "no credential lands in the repository". It has + to be able to fail; the version this replaced could not (see the comment + block above). Its companion below proves it can, by planting one.""" + findings = _scan_device_tree() + assert not findings, ( + "credential-shaped literal(s) found under hexlib/device -- QDC keys are " + "personal and are read only from QDC_API_KEY or ~/.qdc_api_key, never " + "committed:\n " + "\n ".join(findings) + ) + + +def test_the_credential_scan_can_actually_fail(tmp_path, monkeypatch): + """MUTATION-VERIFY, kept as a test rather than done once by hand -- same + pattern as test_coherency_lane_classification.py's own mutation check. + Plants the exact shape of the leak that defeated the previous assertion + (`_FALLBACK_KEY = ""` in a file that also uses `os.environ`, which was + what made the old disjunction unconditionally true) and confirms the scan + reports it. + + THE PLANTED VALUE IS SYNTHETIC AND OBVIOUSLY SO: a 32-char hex string whose + nibbles simply count down and then up (0f1e2d3c...). It is not a QDC key, + not any provider's key, and no real credential is read, constructed or + stored anywhere in this file. It exists only to have the right SHAPE -- + length and entropy -- for the detector to bite on.""" + synthetic = "0f1e2d3c4b5a6978" + "8796a5b4c3d2e1f0" + planted = tmp_path / "qdc" / "leak.py" + planted.parent.mkdir(parents=True) + planted.write_text( + "import os\n" + "def _api_key():\n" + " return os.environ.get('QDC_API_KEY') or _FALLBACK_KEY\n" + f'_FALLBACK_KEY = "{synthetic}"\n' + ) + monkeypatch.setattr("hexlib.tests.test_qdc.DEVICE_DIR", tmp_path) + + findings = _scan_device_tree() + assert findings, ( + "the credential scan did not notice a planted, credential-shaped " + "literal -- so a green result from it means nothing. This is exactly " + "the state the assertion it replaced was in." + ) + assert any("_FALLBACK_KEY" in f for f in findings) + + # And the clean tree really is clean for the right reason: remove the + # planted file and the same scan over the same directory goes quiet, so the + # assertion above is detecting THAT literal and not merely anything at all. + planted.unlink() + assert not _scan_device_tree() + + +def test_the_credential_scan_does_not_fire_on_this_projects_real_constants(): + """The other half of "can fail": a scan that flags legitimate constants + gets deleted. Pins the specific shapes hexlib/device really contains -- + the NAME of an env var, QDC's header names, the numeric target id, a device + path -- so a future tightening of the heuristic that breaks them fails here + instead of in someone's unrelated PR.""" + for name, value in ( + ("_API_KEY_ENV", "QDC_API_KEY"), # an env var's NAME, not a key + ("_KEY_FILE_NAME", ".qdc_api_key"), # a filename, not a key + ("X-QCOM-TokenType", "apikey"), # the token TYPE, not a token + ("TARGET_ID_KEY", "3625030"), # a measured, public target id + ("KEY_PATH", "/data/local/tmp/hexlib"), + ("api_key_header", "QDC_BASE_URL"), + ): + assert _credential_shaped(name), f"{name} should be treated as sensitive" + assert _why_this_looks_like_a_secret(value) is None, ( + f"{name} = {value!r} is a legitimate constant in this project and " + "must not be reported as a credential" + ) diff --git a/hexlib/tests/test_qdc_on_device_is_excluded.py b/hexlib/tests/test_qdc_on_device_is_excluded.py new file mode 100644 index 0000000..07a0271 --- /dev/null +++ b/hexlib/tests/test_qdc_on_device_is_excluded.py @@ -0,0 +1,275 @@ +# hexlib/tests/test_qdc_on_device_is_excluded.py +"""`hexlib/device/qdc/test_on_device.py` runs ON THE PHONE, under the farm's +own pytest -- never here. This file proves that the mechanism keeping it out of +hexlib's own suite works, by running pytest in a subprocess and reading the +real collected node ids back. + +WHAT IS ACTUALLY GUARANTEED, AND WHAT THE PREVIOUS VERSION OF THIS DOCSTRING +CLAIMED. The previous version claimed these tests would survive "a future +`conftest.py` widened `rootdir`/`testpaths` to sweep it back in". They would +not have, for two reasons, and both were real defects rather than hypotheses: + + 1. THEY INSPECTED A DIFFERENT COMMAND THAN CI RUNS. They invoked pytest with + an explicit `hexlib/tests` path. `.github/workflows/ci.yml` runs + `pytest -q -m "not sdk"` with NO PATH, so collection starts at the repo + root and walks into `hexlib/device`, where `test_on_device.py`'s flat + `from utils import sh, write_qdc_log` (correct on the farm, where the + artifact is extracted unpackaged) raises ModuleNotFoundError at + collection time. Reproduced: `Interrupted: 1 error during collection`, + exit 2, ZERO tests run. Naming `hexlib/tests` on the command line hid + exactly the failure this file exists to prevent. + 2. EVERY ASSERTION WAS `assert "..." not in result.stdout`, AND + `result.returncode` WAS NEVER READ. Absence read as success: run the same + command with a deliberately broken `-p` plugin and you get rc=1, + len(stdout)==0, and all three assertions True. Any collection error -- + including the one in (1), had the command been the bare one -- turned this + file green. Verified by mutation, both before and after this rewrite. + +So what is guaranteed now is narrower and checkable: the EXACT command CI runs +exits 0, collects this very test file's own first test, and collects nothing +from `hexlib/device`. Plus the same for a `pytest hexlib` path invocation, +which is what pins the choice of mechanism (see the root conftest.py: a +`collect_ignore` there covers every invocation form, where a `testpaths` entry +in pyproject.toml would have covered only the bare one -- mutation-verified: +making that swap fails `test_naming_a_path_does_not_reach_the_on_device_test_ +either` below and nothing else). + +The self-referential node id is deliberate. Asserting on some OTHER file's test +name couples this file to a name it does not own; asserting that the +subprocess collected THIS file's own first test cannot drift, and is impossible +to satisfy with empty stdout or a collection error. + +THIRD DEFECT, FIXED 2026-08-11: THE ABSENCE CHECK MATCHED BARE SUBSTRINGS +AGAINST THE WHOLE OF STDOUT. It was `assert "test_on_device.py" not in +result.stdout` plus `assert "device/qdc" not in result.stdout`. Collection +output is a list of NODE IDS, but those assertions searched every byte of it, +so any new parametrize id, test name, or (in the failure path) traceback text +that merely NAMED that file or that directory tripped them -- and the failure +message then claimed the on-device file WAS collected when it was not, which is +a confusing thing to debug under merge pressure. It was hit for real. The +checks below parse the node ids out of stdout and match a node-id PREFIX +instead, and `test_the_absence_check_does_not_fire_on_a_mere_mention` / +`test_the_absence_check_still_fires_on_a_real_device_node_id` pin both +directions of that against fabricated output, so neither half is taken on +trust. + +THE EXCLUSION MECHANISM ITSELF IS UNCHANGED -- still the root conftest.py's +`collect_ignore`. Only how this file VERIFIES it changed. +""" +import os +import re +import subprocess +import sys + +import pytest + +REPO_ROOT = os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) +ON_DEVICE_TEST = os.path.join("hexlib", "device", "qdc", "test_on_device.py") + +# The exact arguments .github/workflows/ci.yml's `offline` job passes, minus +# --collect-only. NO PATH ARGUMENT: that is the whole point (see 1. above). +CI_PYTEST_ARGS = ["-q", "-m", "not sdk"] + +# This file's own module path, as pytest prints it: node ids are +# rootdir-relative with forward slashes on every platform, Windows included. +# Built from __file__ rather than typed out, so a rename cannot leave a stale +# literal behind that still "passes". +_THIS_MODULE_NODE = "hexlib/tests/" + os.path.basename(__file__) + + +def test_the_on_device_file_actually_exists(): + """A prerequisite, not the point of this file: if this ever goes + missing, every other assertion here about it being "excluded" would be + vacuously true for the wrong reason.""" + assert os.path.isfile(os.path.join(REPO_ROOT, ON_DEVICE_TEST)) + + +# Derived from the function object above, never retyped -- renaming that test +# moves this with it instead of leaving a node id that no longer exists (which +# would fail loudly, but for the wrong reason). +_KNOWN_GOOD_NODE_ID = ( + f"{_THIS_MODULE_NODE}::{test_the_on_device_file_actually_exists.__name__}" +) + + +def _collect(*args): + """Run `pytest --collect-only ARGS` at the repo root and return the + CompletedProcess. `--collect-only` is the only difference from the real + command: it makes the node ids readable without running 600+ tests inside + a test.""" + return subprocess.run( + [sys.executable, "-m", "pytest", "--collect-only", *args], + capture_output=True, text=True, cwd=REPO_ROOT, + ) + + +@pytest.fixture(scope="module") +def ci_collection(): + """Collection under the command CI actually runs: bare, no path.""" + return _collect(*CI_PYTEST_ARGS) + + +@pytest.fixture(scope="module") +def hexlib_path_collection(): + """Collection when a path IS given (`pytest hexlib`). `testpaths` is + ignored in this case; only a `collect_ignore` covers it.""" + return _collect("-q", "hexlib") + + +# A collected node id, as `--collect-only -q` prints one per line: +# `hexlib/tests/test_x.py::test_y`, or `...::test_y[param]` when parametrized. +# Anchored, with no whitespace before the `::`, so a line of prose that happens +# to contain both a `.py` and a `::` (a traceback, a message quoting a node id +# mid-sentence) is not mistaken for a collected test. +_NODE_ID_LINE = re.compile(r"\A(?P\S+\.py)::(?P\S+)\Z") + +# Node ids from the on-device tree, which is what WOULD appear if the root +# conftest.py's `collect_ignore` stopped working. The whole DIRECTORY, not just +# test_on_device.py's own name: a second on-device file added next to it must be +# caught without anyone having to remember to come back here. +_ON_DEVICE_NODE_PREFIX = "hexlib/device/" + + +def _collected_node_ids(stdout): + """Every collected node id in `--collect-only` output, separators + normalized to forward slashes. + + PARSED, NOT SUBSTRING-SEARCHED. This is the whole fix for the third defect + in this file's docstring: the previous version searched raw stdout, so any + line that merely mentioned a filename or a directory counted as evidence + that it had been collected. + """ + ids = [] + for raw in stdout.splitlines(): + m = _NODE_ID_LINE.match(raw.strip().replace("\\", "/")) + if m: + ids.append(m.group(0)) + return ids + + +def _on_device_node_ids(stdout): + return [ + n for n in _collected_node_ids(stdout) + if n.startswith(_ON_DEVICE_NODE_PREFIX) + ] + + +def _assert_collection_succeeded(result, how): + """The two things the old version of this file never checked. Order + matters: report the rc first, because a collection error is what makes + every "not in stdout" assertion vacuously true.""" + assert result.returncode == 0, ( + f"`pytest --collect-only {how}` exited {result.returncode}, not 0 -- " + "collection itself failed, so nothing below this line would have " + "proved anything about what was or was not collected:\n" + f"--- stdout ---\n{result.stdout}\n--- stderr ---\n{result.stderr}" + ) + collected = _collected_node_ids(result.stdout) + assert _KNOWN_GOOD_NODE_ID in collected, ( + f"`pytest --collect-only {how}` exited 0 but did not collect " + f"{_KNOWN_GOOD_NODE_ID} -- this file's own first test. Empty or " + "unrecognizable output must never be read as 'the on-device test was " + f"excluded'. Parsed {len(collected)} node id(s) from:\n" + f"--- stdout ---\n{result.stdout}" + ) + + +def _assert_device_qdc_absent(result, how): + offenders = _on_device_node_ids(result.stdout) + assert not offenders, ( + f"`pytest {how}` collected node id(s) under {_ON_DEVICE_NODE_PREFIX} " + f"-- those tests run only on the phone: {offenders}" + ) + + +def test_the_bare_command_ci_runs_collects_cleanly(ci_collection): + """THE LOAD-BEARING ONE. `pytest -q -m "not sdk"` -- CI's own command, no + path -- must exit 0 and collect real tests. This is exactly what was + broken: it exited 2 with zero tests run, and no test in this repo could + see it.""" + _assert_collection_succeeded(ci_collection, " ".join(CI_PYTEST_ARGS)) + + +def test_the_bare_command_ci_runs_does_not_collect_the_on_device_test(ci_collection): + """Binds to the mechanism: empty `collect_ignore` in the root conftest.py + and this fails -- on the rc, since the flat `import utils` breaks + collection outright, and on the node-id checks here if that import were + ever made to work. Mutation-verified both ways round.""" + _assert_collection_succeeded(ci_collection, " ".join(CI_PYTEST_ARGS)) + _assert_device_qdc_absent(ci_collection, " ".join(CI_PYTEST_ARGS)) + + +def test_naming_a_path_does_not_reach_the_on_device_test_either( + hexlib_path_collection, +): + """`pytest hexlib` -- a path argument, so `testpaths` would NOT apply. + This is the case that makes the root conftest.py's `collect_ignore` the + right mechanism rather than a `testpaths` entry; if someone swaps one for + the other, this test is the only thing that notices.""" + _assert_collection_succeeded(hexlib_path_collection, "hexlib") + _assert_device_qdc_absent(hexlib_path_collection, "hexlib") + + +# ============================================================================== +# BOTH DIRECTIONS OF THE ABSENCE CHECK, against fabricated output. +# +# The previous version of that check (`assert "test_on_device.py" not in +# result.stdout`) was wrong in the FALSE-POSITIVE direction: any test name, +# parametrize id or traceback line that merely NAMED the file failed it, with a +# message claiming the file had been collected when it had not. It was hit for +# real. Fixing that without also pinning the true-positive direction would just +# trade one silent failure for another, so both are checked here -- and neither +# needs a subprocess, so they cannot be skipped for being slow. +# ============================================================================== + + +def test_the_absence_check_does_not_fire_on_a_mere_mention(): + """A collected test whose NAME contains the on-device filename, plus prose + quoting the full path -- neither is a collected on-device node id.""" + stdout = ( + f"{_KNOWN_GOOD_NODE_ID}\n" + "hexlib/tests/test_cli_device_flag.py::test_error_names_test_on_device_py\n" + "hexlib/tests/test_x.py::test_paths[hexlib/device/qdc/test_on_device.py]\n" + " the staged script hexlib/device/qdc/test_on_device.py runs on the phone\n" + "3 tests collected in 0.42s\n" + ) + assert _on_device_node_ids(stdout) == [], ( + "a mention is not a collection -- this is the false positive that made " + "the previous check claim the on-device file had been collected when it " + "had not" + ) + assert _KNOWN_GOOD_NODE_ID in _collected_node_ids(stdout) + + +def test_the_absence_check_still_fires_on_a_real_device_node_id(): + """The direction that matters: an actually-collected on-device test must be + reported. Without this, the fix above could have been "match nothing".""" + stdout = ( + f"{_KNOWN_GOOD_NODE_ID}\n" + "hexlib/device/qdc/test_on_device.py::test_binaries_are_present\n" + "2 tests collected in 0.42s\n" + ) + offenders = _on_device_node_ids(stdout) + assert offenders == [ + "hexlib/device/qdc/test_on_device.py::test_binaries_are_present" + ] + + +def test_the_absence_check_catches_a_second_on_device_file_and_windows_paths(): + """The DIRECTORY, not one filename: a new on-device file next to + test_on_device.py is caught without anyone editing this test. Backslash + node ids are normalized rather than needing their own assertion.""" + stdout = ( + f"{_KNOWN_GOOD_NODE_ID}\n" + "hexlib/device/qdc/test_something_new.py::test_z\n" + "hexlib\\device\\qdc\\test_on_device.py::test_w\n" + ) + assert len(_on_device_node_ids(stdout)) == 2 + + +def test_a_collection_error_cannot_look_like_a_clean_exclusion(): + """Empty or error output yields ZERO node ids, so `_assert_collection_ + succeeded`'s known-good-id assertion fails -- absence is never read as + success here.""" + assert _collected_node_ids("") == [] + assert _collected_node_ids("Interrupted: 1 error during collection\n") == [] diff --git a/hexlib/tests/test_quant_q4_0.py b/hexlib/tests/test_quant_q4_0.py new file mode 100644 index 0000000..a085b41 --- /dev/null +++ b/hexlib/tests/test_quant_q4_0.py @@ -0,0 +1,243 @@ +# hexlib/tests/test_quant_q4_0.py +"""`hexlib.exec.quant` against an INDEPENDENT scalar transcription of +llama.cpp's `quantize_row_q4_0_ref`. + +WHY A SECOND IMPLEMENTATION RATHER THAN GOLDEN BYTES. A committed golden blob +would pin the output without saying what it means, and the failure mode here is +not "the bytes changed" — it is "the bytes are a valid q4_0 encoding of the wrong +thing". Three ways to get that, all of which round-trip to plausible values: + + * pairing nibble j with j+1 instead of j with j+16 (a permutation of the right + values: right norm, right histogram, wrong everywhere); + * `amax / 8` instead of `max / -8` (flips the sign of every value in blocks + whose extreme element is positive); + * `np.round` instead of truncate-after-adding-8.5 (banker's rounding, so it + disagrees on every exact .5 and nowhere else — invisible on random data). + +The scalar reference below is written from `ggml-quants.c:113-146` statement by +statement, in a loop, with no numpy vectorisation, so it shares no code and no +broadcasting with the implementation. Where they agree, they agree for a reason. + +The vectorised version is also the one hexlib will use to feed real q4_0 weights +to `matmul_epilogue`, so a disagreement here is a wrong weight matrix on the DSP. +""" +import numpy as np +import pytest + +from hexlib.exec.quant import ( + BLOCK_BYTES, + QK4_0, + dequantize_q4_0, + quantize_q4_0, +) +from hexlib.graph import ir + + +def _scalar_quantize(x): + """`quantize_row_q4_0_ref`, transcribed. One block at a time, no numpy.""" + flat = np.asarray(x, dtype=np.float32).reshape(-1) + assert flat.size % QK4_0 == 0 + out = bytearray() + for i in range(flat.size // QK4_0): + blk = flat[i * QK4_0 : (i + 1) * QK4_0] + amax, mx = 0.0, 0.0 + for v in blk: + v = float(v) + if amax < abs(v): + amax, mx = abs(v), v + d = np.float32(mx / -8.0) + # Stored as fp16, so everything downstream sees the narrowed value. + d16 = np.float16(d) + du = np.float32(d16) + idv = np.float32(1.0 / du) if du != 0.0 else np.float32(0.0) + out += bytes(np.array([d16], dtype=np.float16).view(np.uint8)) + for j in range(QK4_0 // 2): + x0 = np.float32(blk[j]) * idv + x1 = np.float32(blk[j + QK4_0 // 2]) * idv + # C: MIN(15, (int8_t)(x + 8.5f)) -- truncation toward zero. + xi0 = min(15, int(np.trunc(x0 + np.float32(8.5)))) + xi1 = min(15, int(np.trunc(x1 + np.float32(8.5)))) + xi0 = max(0, xi0) + xi1 = max(0, xi1) + out.append(xi0 | (xi1 << 4)) + return bytes(out) + + +def test_the_block_geometry_comes_from_ir(): + """18 bytes per 32 elements, and the constants are ir's, not a second copy. + A drifted block size is a wrong answer at full speed, not a crash.""" + assert QK4_0 == 32 + assert BLOCK_BYTES == 18 + assert ir.nbytes((1, 32), "q4_0") == BLOCK_BYTES + assert ir.nbytes((768, 768), "q4_0") == 768 * 768 // 32 * 18 + + +@pytest.mark.parametrize("shape", [(1, 32), (3, 64), (8, 96), (768, 32)]) +def test_agrees_with_the_scalar_transcription_byte_for_byte(shape): + rng = np.random.default_rng(5) + x = rng.standard_normal(shape).astype(np.float32) * 3.0 + got = quantize_q4_0(x) + want = _scalar_quantize(x) + assert len(got) == ir.nbytes(shape, "q4_0") + assert got == want, ( + f"the vectorised quantizer disagrees with the scalar transcription of " + f"llama.cpp's own loop on {sum(a != b for a, b in zip(got, want))} of " + f"{len(want)} bytes" + ) + + +def test_the_rounding_mode_is_truncate_after_adding_8_point_5(): + """THE THIRD HAZARD, AND IT NEEDS CONSTRUCTED INPUT TO BE VISIBLE AT ALL. + + `trunc(x + 8.5)` and `round(x + 8.0)` agree everywhere except where `x` is + exactly a half-integer — numpy's `round` is banker's rounding, so it sends + 0.5 to 0 and 1.5 to 2 while truncation-after-8.5 sends them to 1 and 2. On + random data that difference has measure zero, and swapping one for the other + passed every other test in this file. Found by mutation, which is the only + reason this test exists. + + So the input is built to land on exact halves. An extreme of -8.0 makes + `d = max / -8 = 1.0` (exactly representable in fp16, so the fp16 round trip + of the scale changes nothing) and therefore `id = 1.0` — which means the + scaled values ARE the input values, and half-integer inputs hit the boundary + directly. + + The expected codes are worked out by hand rather than taken from the + implementation: for value v, `trunc(v + 8.5)` clamped to [0, 15]. + """ + half_ints = np.array([-7.5, -6.5, -5.5, -4.5, -3.5, -2.5, -1.5, -0.5, + 0.5, 1.5, 2.5, 3.5, 4.5, 5.5, 6.5], dtype=np.float32) + # -8.0 sets the scale; the rest are exact halves. 32 elements total. + blk = np.concatenate([ + np.array([-8.0], dtype=np.float32), + half_ints, + np.array([-8.0], dtype=np.float32), + half_ints, + ]) + assert blk.size == QK4_0 + + raw = np.frombuffer(quantize_q4_0(blk.reshape(1, -1)), dtype=np.uint8) + d = raw[:2].copy().view(np.float16)[0] + assert d == np.float16(1.0), f"expected an exact scale of 1.0, got {d}" + + qs = raw[2:] + got = np.concatenate([(qs & 0x0F), (qs >> 4)]).astype(np.int32) + + expect = np.clip(np.trunc(blk + np.float32(8.5)), 0, 15).astype(np.int32) + assert np.array_equal(got, expect), ( + f"codes {got.tolist()} but truncate-after-8.5 gives {expect.tolist()}. " + f"numpy's round() would give " + f"{np.clip(np.round(blk + np.float32(8.0)), 0, 15).astype(int).tolist()}" + ) + # And the two rounding modes really do differ on this input, so the + # assertion above is not vacuous. + bankers = np.clip(np.round(blk + np.float32(8.0)), 0, 15).astype(np.int32) + assert not np.array_equal(expect, bankers), ( + "this input no longer distinguishes the two rounding modes, so the " + "assertion above proves nothing -- pick values that hit exact halves" + ) + + # The scalar transcription must agree byte for byte on it too. + assert quantize_q4_0(blk.reshape(1, -1)) == _scalar_quantize(blk) + + +def test_blocks_whose_extreme_value_is_POSITIVE_get_a_negative_scale(): + """`d = max / -8` with `max` SIGNED — the second hazard in the docstring. + + A block whose largest-magnitude element is positive must store a NEGATIVE + scale. `amax / 8` would store a positive one and flip the sign of every + dequantized value in that block, which is why this is asserted directly on + the stored bytes rather than only through a round trip. + """ + x = np.linspace(0.5, 4.0, QK4_0, dtype=np.float32) # all positive + raw = np.frombuffer(quantize_q4_0(x.reshape(1, -1)), dtype=np.uint8) + d = raw[:2].copy().view(np.float16)[0] + assert d < 0, f"scale {d} should be negative for an all-positive block" + + y = np.linspace(-4.0, -0.5, QK4_0, dtype=np.float32) # all negative + raw = np.frombuffer(quantize_q4_0(y.reshape(1, -1)), dtype=np.uint8) + d = raw[:2].copy().view(np.float16)[0] + assert d > 0, f"scale {d} should be positive for an all-negative block" + + +def test_the_low_nibble_is_element_j_and_the_high_nibble_is_element_j_plus_16(): + """THE PAIRING, asserted on the bytes. + + A block built so the two halves are clearly distinguishable: the first 16 + elements near the negative extreme (small quant codes) and the second 16 near + zero (codes near 8). If the pairing were j with j+1, the low and high nibbles + of each byte would BOTH come from the first half and both be small. + """ + x = np.concatenate([ + np.full(16, -1.0, dtype=np.float32), + np.full(16, 0.0, dtype=np.float32), + ]) + raw = np.frombuffer(quantize_q4_0(x.reshape(1, -1)), dtype=np.uint8) + qs = raw[2:] + lo = qs & 0x0F + hi = qs >> 4 + # first half is the extreme -> code 0; second half is zero -> code 8 + assert set(lo.tolist()) == {0}, f"low nibbles {sorted(set(lo.tolist()))}" + assert set(hi.tolist()) == {8}, f"high nibbles {sorted(set(hi.tolist()))}" + + +def test_a_dequantized_block_recovers_its_extreme_value_exactly(): + """The element that set the scale must come back essentially exact — its + quant code is 0 and `(0 - 8) * d == max`. This is the one value the format + represents without error, so it is the sharpest available check on the + scale.""" + rng = np.random.default_rng(6) + x = rng.standard_normal((4, 32)).astype(np.float32) * 2.0 + back = dequantize_q4_0(quantize_q4_0(x), x.shape) + for r in range(x.shape[0]): + j = int(np.abs(x[r]).argmax()) + # fp16 scale storage is the only loss here. + assert abs(back[r, j] - x[r, j]) <= abs(x[r, j]) * 1e-3 + 1e-6, ( + f"row {r}: extreme {x[r, j]} came back as {back[r, j]}" + ) + + +def test_the_round_trip_error_is_what_a_4_bit_format_can_do_and_no_worse(): + """A real bound, not a smoke test. 16 levels spanning [-8d, 7d] means the + quantization step is |d| and the worst case is about half a step, so the + error must be under ~amax/8 per element. Asserted as a fraction of each + block's own amax, because a global bound would be dominated by whichever + block happened to have the largest values.""" + rng = np.random.default_rng(7) + x = (rng.standard_normal((32, 64)) * 5.0).astype(np.float32) + back = dequantize_q4_0(quantize_q4_0(x), x.shape) + + blocks = x.reshape(-1, QK4_0) + got = back.reshape(-1, QK4_0) + amax = np.abs(blocks).max(axis=1) + err = np.abs(got - blocks).max(axis=1) + assert np.all(err <= amax / 8.0 * 1.02 + 1e-4), ( + f"worst block error {(err / amax).max():.4f} of amax; a 4-bit format " + f"with 16 levels should stay under 1/8" + ) + # And it must not be TRIVIALLY good, which would mean the test is measuring + # nothing -- 4-bit really does lose information. + assert err.max() > amax.max() / 100.0, ( + "round-trip error is suspiciously small for a 4-bit format; is the " + "dequantizer reading back a stored copy?" + ) + + +def test_an_all_zero_block_is_handled_and_round_trips_to_zero(): + """`d == 0` makes `1/d` a division by zero; llama.cpp guards it with + `id = d ? 1/d : 0`. Every code then lands on 8, and `(8-8)*0 == 0`.""" + x = np.zeros((2, 32), dtype=np.float32) + data = quantize_q4_0(x) + raw = np.frombuffer(data, dtype=np.uint8).reshape(-1, BLOCK_BYTES) + assert np.all(raw[:, 2:] == 0x88), "every quant code should be 8" + assert np.array_equal(dequantize_q4_0(data, x.shape), x) + + +def test_a_last_axis_that_is_not_a_multiple_of_the_block_is_refused(): + with pytest.raises(ValueError, match="multiple of 32"): + quantize_q4_0(np.zeros((4, 33), dtype=np.float32)) + + +def test_dequantize_refuses_a_buffer_of_the_wrong_length(): + with pytest.raises(ValueError, match="bytes"): + dequantize_q4_0(b"\x00" * 17, (1, 32)) diff --git a/hexlib/tests/test_quant_q8_0.py b/hexlib/tests/test_quant_q8_0.py new file mode 100644 index 0000000..e7ab0fc --- /dev/null +++ b/hexlib/tests/test_quant_q8_0.py @@ -0,0 +1,180 @@ +# hexlib/tests/test_quant_q8_0.py +"""q8_0, held to the ggml reference it was transcribed from. + +WHY THIS FORMAT EXISTS HERE. Measured against transformers on the shipped +Qwen3.5-0.8B vision weights at 256x256, fp32 arithmetic on both sides so nothing +but the weight format differs: + + q4_0 encoder output cosine 0.867606 + q8_0 encoder output cosine 0.999002 + +for 1.89x the weight bytes. Mixed precision was measured first and rejected: +the error is DIFFUSE, so keeping the merger, the patch embedding and all of +attention at higher precision while the MLPs stay q4_0 still only reaches 0.913. + +WHAT THESE TESTS ARE REALLY FOR. q8_0 sits next to q4_0 in the same file and +looks like its bigger sibling, which makes carrying q4_0's habits across the +obvious mistake. Three of them produce a plausible wrong answer rather than an +error, and each has a test below: the sign of `d`, the rounding mode, and the +element ordering. +""" +import numpy as np +import pytest + +from hexlib.exec.quant import ( + QK8_0, + Q8_0_BLOCK_BYTES, + dequantize_q8_0, + quantize_q8_0, +) +from hexlib.graph import ir + + +def _scalar_quantize(block: np.ndarray) -> bytes: + """`quantize_row_q8_0_ref` (ggml-quants.c:276-299) transcribed elementwise, + in Python, with no numpy vectorisation -- so it shares no code path with the + implementation and a shared bug cannot cancel.""" + import struct + + amax = 0.0 + for v in block: + amax = max(amax, abs(float(v))) + d = np.float16(amax / 127.0) + df = float(np.float32(d)) + idv = (1.0 / df) if df != 0.0 else 0.0 + out = bytearray(struct.pack(" 0, f"q8_0 scale must be positive, got {d}" + + +def test_the_rounding_is_half_away_from_zero_and_not_bankers(): + """`roundf`, not `np.round`. Built so the scaled values land on exact + halves: amax = 127 makes d = 1.0 exactly (representable in fp16), so the + scaled value IS the input and .5 inputs hit the boundary directly. + + The assertion is paired with a proof that the two modes actually differ on + this input -- otherwise it would pass for a banker's-rounding + implementation and prove nothing.""" + halves = np.array([0.5, 1.5, 2.5, 3.5, -0.5, -1.5, -2.5, -3.5], dtype=np.float32) + blk = np.concatenate([ + np.array([127.0], dtype=np.float32), # sets d = 1.0 + np.tile(halves, 3), + np.zeros(QK8_0 - 1 - 24, dtype=np.float32), + ]) + assert blk.size == QK8_0 + + raw = np.frombuffer(quantize_q8_0(blk.reshape(1, -1)), dtype=np.uint8) + assert raw[:2].copy().view(np.float16)[0] == np.float16(1.0) + got = raw[2:].copy().view(np.int8).astype(np.int64) + + away = (np.sign(blk) * np.floor(np.abs(blk) + 0.5)).astype(np.int64) + bankers = np.round(blk).astype(np.int64) + assert np.array_equal(got, away), f"got {got.tolist()}, away-from-zero {away.tolist()}" + assert not np.array_equal(away, bankers), ( + "this input no longer distinguishes the two rounding modes, so the " + "assertion above proves nothing" + ) + + +def test_the_quants_are_in_element_order_with_no_bias_and_no_nibble_pairing(): + """q4_0 packs element j and element j+16 into one byte and biases codes by + 8. q8_0 does neither: byte j is element j, signed. A block that is + monotonically increasing must come back monotonically increasing.""" + blk = np.linspace(-4.0, 4.0, QK8_0, dtype=np.float32) + raw = np.frombuffer(quantize_q8_0(blk.reshape(1, -1)), dtype=np.uint8) + q = raw[2:].copy().view(np.int8).astype(np.int64) + assert q.size == QK8_0 + assert np.all(np.diff(q) >= 0), f"not monotonic: {q.tolist()}" + assert q.min() < 0 < q.max(), "codes are signed, not biased into [0, 255]" + + +def test_the_extreme_element_does_not_wrap_to_minus_128(): + """`amax / 127` keeps |scaled| <= 127 in fp32, but `d` is NARROWED to fp16 + before use and narrowing downward makes the scaled value slightly larger -- + so the extreme can land on 128 and wrap to -128 without a clamp. A wrap + flips the sign of the single largest element in the block, which is the + worst possible element to get wrong.""" + rng = np.random.default_rng(7) + for trial in range(200): + blk = (rng.standard_normal(QK8_0) * 10 ** rng.uniform(-3, 3)).astype(np.float32) + raw = np.frombuffer(quantize_q8_0(blk.reshape(1, -1)), dtype=np.uint8) + q = raw[2:].copy().view(np.int8) + j = int(np.abs(blk).argmax()) + assert np.sign(int(q[j])) == np.sign(blk[j]) or blk[j] == 0, ( + f"trial {trial}: extreme {blk[j]} quantized to {q[j]} -- sign flipped" + ) + + +def test_the_round_trip_is_idempotent(): + """`test_encoder_on_sim.py` pre-quantizes its weight feeds so both paths + multiply identical values, which requires quantize(dequantize(q)) == q. q4_0 + has this property; a q8_0 that lost it would make that test flap rather than + fail, which is worse.""" + rng = np.random.default_rng(3) + for seed in range(6): + w = (rng.standard_normal((8, 64)) * 2.0).astype(np.float32) + a = quantize_q8_0(w) + assert quantize_q8_0(dequantize_q8_0(a, w.shape)) == a, f"seed {seed}" + + +def test_it_is_meaningfully_better_than_q4_0_on_the_same_data(): + """The whole reason this format was added. Not a tolerance -- a COMPARISON, + so it cannot pass by being loose.""" + from hexlib.exec.quant import dequantize_q4_0, quantize_q4_0 + + rng = np.random.default_rng(11) + w = (rng.standard_normal((32, 128)) * 1.5).astype(np.float32) + e4 = np.abs(dequantize_q4_0(quantize_q4_0(w), w.shape) - w).max() + e8 = np.abs(dequantize_q8_0(quantize_q8_0(w), w.shape) - w).max() + assert e8 < e4 / 4, ( + f"q8_0 max error {e8:.3e} is not meaningfully below q4_0's {e4:.3e}; " + f"four extra bits should buy roughly 16x" + ) + + +def test_a_last_axis_that_is_not_a_multiple_of_the_block_is_refused(): + with pytest.raises(ValueError, match="multiple of 32"): + quantize_q8_0(np.zeros((4, 30), dtype=np.float32)) + + +def test_a_wrong_length_buffer_is_refused_rather_than_reshaped(): + data = quantize_q8_0(np.zeros((4, 32), dtype=np.float32)) + with pytest.raises(ValueError, match="bytes"): + dequantize_q8_0(data, (4, 64)) diff --git a/hexlib/tests/test_raw_q4_0_staging.py b/hexlib/tests/test_raw_q4_0_staging.py new file mode 100644 index 0000000..cb9f783 --- /dev/null +++ b/hexlib/tests/test_raw_q4_0_staging.py @@ -0,0 +1,274 @@ +# hexlib/tests/test_raw_q4_0_staging.py +"""The block-quantized weight path: `RawTensor`, and what refuses a dense array. + +WHY THIS PATH EXISTS AT ALL. 75 of the encoder's 308 plan steps are +`matmul_epilogue`, and every one of them takes a q4_0 weight -- 48 at (768, 768), +12 at (768, 3072), 12 at (3072, 768), plus three singletons. `RunnerSpec` used to +refuse the dtype outright, with a message that named the missing work: +"block-quantized inputs are staged as raw bytes and need their own path". Nothing +downstream of that refusal could be built. + +WHAT MAKES THIS DANGEROUS RATHER THAN MERELY MISSING. Every other input on this +transport is a numpy array staged through `np.ascontiguousarray(a, dtype=...)`, +which converts. A q4_0 buffer cannot be converted -- it is 18-byte blocks of an +fp16 scale plus 32 4-bit values, `llama.cpp`'s `block_q4_0` -- so it has to be +passed through verbatim, and `.tobytes()` on ANY numpy array is willing to +produce bytes. An fp32 weight array staged down this path yields either a buffer +of the wrong size or, when the sizes happen to line up, a correctly-shaped answer +computed from noise, with no error anywhere. That is why the type is distinct and +why these tests spend most of their assertions on refusals. + +THE TWO NUMBERS ARE BOTH TRUE ON THE WIRE. `hexlib_tensor` carries `nbytes` and +`ne[]` as separate fields, and `wire.py` checks only that the buffer holds +`offset + nbytes` -- never that `nbytes == prod(ne) * itemsize`. So a (768, 768) +q4_0 weight says ne=(768,768) AND nbytes=331776. Putting the byte shape in `ne` +instead would be a lie every `dim:` scalar and every kernel reading `a->ne` +would inherit. +""" +import numpy as np +import pytest + +from hexlib.exec.runner import ( + RawTensor, + RunnerSpec, + Scalar, + WIRE_DTYPE, + WIRE_RAW, +) +from hexlib.graph import ir +from hexlib.runtime.genentry import emit_entry +from hexlib.runtime.wire import DTYPE_ID, LAYOUT_ID + +# The encoder's own weight shapes, from the compiled plan for qwen35 at 256x256. +ENCODER_WEIGHT_SHAPES = ((768, 768), (768, 3072), (3072, 768), (1536, 768), + (3072, 3072), (3072, 1024)) + + +def _spec(**kw): + base = dict( + kind="matmul_epilogue", + kernel_dir="kernels/matmul_epilogue_fp16", + inputs=("fp16", "q4_0", "fp32"), + out_dtype="fp16", + layouts=("row_major", "q4_0_repacked", "row_major", "row_major"), + ) + base.update(kw) + return RunnerSpec(**base) + + +def _weight(shape): + return RawTensor("q4_0", shape, b"\x5a" * ir.nbytes(shape, "q4_0")) + + +# --------------------------------------------------------------------------- +# RawTensor's own contract +# --------------------------------------------------------------------------- + + +@pytest.mark.parametrize("shape", ENCODER_WEIGHT_SHAPES) +def test_every_encoder_weight_shape_round_trips(shape): + """Not a smoke test: these are the six shapes the plan actually contains, + and `ir.nbytes` refuses a last dimension that is not a multiple of 32, so + this also confirms every one of them is expressible at all.""" + w = _weight(shape) + assert w.shape == shape + assert w.nbytes == ir.nbytes(shape, "q4_0") + assert w.size == shape[0] * shape[1] + # The whole point of the format: far smaller than the dense equivalent. + assert w.nbytes < shape[0] * shape[1] * 2 + + +def test_the_byte_count_comes_from_ir_and_not_from_a_second_copy_of_it(): + """`ir.nbytes` is the authority on the block size and it already refuses a + bad last dimension. If `RawTensor` recomputed the arithmetic instead of + asking, the two could drift -- and a drifted block size is a wrong answer, + not a crash. Checked by asserting the exact published constant: q4_0 is 18 + bytes per 32 elements, which `test_graph_ir.py` pins independently.""" + assert ir.nbytes((1, 32), "q4_0") == 18 + w = RawTensor("q4_0", (1, 32), b"\x00" * 18) + assert w.nbytes == 18 + + +@pytest.mark.parametrize("delta", [-18, -1, 1, 18]) +def test_a_buffer_that_is_not_exactly_the_block_size_is_refused(delta): + """Off by one byte or by one whole block, both refused at construction. + + A short weight is the realistic accident -- a truncated download, a slice + taken with the wrong stride -- and it is the one that produces a plausible + wrong answer rather than a crash, because the kernel reads whatever follows + it in the payload. + """ + n = ir.nbytes((768, 768), "q4_0") + with pytest.raises(ValueError, match="bytes of blocks"): + RawTensor("q4_0", (768, 768), b"\x00" * (n + delta)) + + +def test_a_last_dimension_that_is_not_a_multiple_of_the_block_is_refused(): + with pytest.raises(ValueError, match="multiple of the 32-element block"): + RawTensor("q4_0", (768, 33), b"\x00" * 400) + + +def test_rawtensor_refuses_a_dense_dtype(): + """`RawTensor` is for block-quantized storage only. fp16 has a numpy dtype + and belongs in the ordinary path; accepting it here would give two ways to + stage the same thing, one of which skips the conversion.""" + with pytest.raises(ValueError, match="WIRE_RAW"): + RawTensor("fp16", (16, 32), b"\x00" * 1024) + + +def test_the_raw_and_dense_dtype_tables_do_not_overlap(): + """A dtype in both tables would make `payload`'s branch order decide which + staging path a weight takes, which is exactly the kind of thing that is + correct until someone reorders it.""" + assert not (WIRE_RAW & set(WIRE_DTYPE)) + # And every raw dtype must still have a wire id, or it cannot be declared. + for dtype in WIRE_RAW: + assert dtype in DTYPE_ID, f"{dtype} has no DTYPE_ID and cannot cross" + + +# --------------------------------------------------------------------------- +# RunnerSpec: what may and may not be raw +# --------------------------------------------------------------------------- + + +def test_a_spec_may_read_q4_0(): + spec = _spec() + assert spec.inputs[1] == "q4_0" + assert spec.buf_layouts()[1] == "q4_0_repacked" + + +def test_a_spec_may_not_WRITE_q4_0(): + """`decode` reads the result back through `np.frombuffer` with a numpy + dtype, so a quantized output would be a result that cannot be decoded. No + kernel in this encoder writes one -- the weights arrive quantized and + everything computed is fp16.""" + with pytest.raises(ValueError, match="not a dense wire dtype"): + _spec(out_dtype="q4_0") + + +def test_an_unknown_dtype_is_still_refused_and_names_both_tables(): + """THE STAND-IN IS DERIVED, NOT SPELLED. This test used the literal "q8_0" + as its example of an unknown dtype, and on 2026-08-13 q8_0 was added as a + real block-quantized weight format -- so the test stopped exercising the + refusal and started asserting that a SUPPORTED dtype raises, which it no + longer did. It failed loudly, which was lucky; a test whose negative example + quietly becomes positive can just as easily keep passing for a new reason. + + So the unknown dtype is now chosen at run time as one that is in neither + table. That cannot go stale, because the day it becomes known it stops + being selected.""" + from hexlib.exec.runner import WIRE_DTYPE, WIRE_RAW + + known = set(WIRE_DTYPE) | set(WIRE_RAW) + unknown = next(c for c in ("q3_k", "q2_k", "nf4", "not_a_dtype") + if c not in known) + with pytest.raises(ValueError, match="neither a dense wire dtype"): + _spec(inputs=("fp16", unknown, "fp32")) + + +def test_payload_refuses_a_numpy_array_where_the_spec_declared_q4_0(): + """THE FAILURE THIS PATH EXISTS TO PREVENT. + + A dense array staged as blocks is not an error numpy will raise: + `.tobytes()` always works. So the refusal has to be explicit, and it has to + be here rather than only in `dsp.py`, because `RunnerSpec.payload` is the + other transport (the standalone-ELF runner) and both must agree. + """ + spec = _spec() + x = np.zeros((256, 768), dtype=np.float16) + dense_weight = np.zeros((768, 768), dtype=np.float16) + bias = np.zeros(768, dtype=np.float32) + with pytest.raises(ValueError, match="must be a RawTensor"): + spec.payload((x, dense_weight, bias)) + + +def test_payload_refuses_a_rawtensor_whose_dtype_is_not_the_declared_one(): + """Two raw dtypes will exist eventually (q8_0 is the obvious next one), and + at that point a mismatched RawTensor is a silent reinterpretation of one + block format as another. Refused now, while there is only one.""" + spec = _spec() + w = _weight((768, 768)) + object.__setattr__(w, "dtype", "q8_0") # frozen dataclass; forge the drift + with pytest.raises(ValueError, match="RawTensor says"): + spec.payload((np.zeros((256, 768), np.float16), w, + np.zeros(768, np.float32))) + + +def test_payload_stages_the_quantized_bytes_verbatim_and_in_order(): + """The weight's bytes must appear unchanged, and after the activation -- + src order is the contract the DSP walks. A conversion here would be + undetectable downstream: the buffer would be the right size and the values + would be wrong.""" + spec = _spec() + x = np.arange(8, dtype=np.float16) + w = RawTensor("q4_0", (1, 32), bytes(range(18))) + bias = np.arange(4, dtype=np.float32) + + blob = spec.payload((x, w, bias)) + assert blob == x.tobytes() + bytes(range(18)) + bias.tobytes() + + +def test_numel_and_dim_scalars_read_the_LOGICAL_shape_of_a_raw_input(): + """`ne` is the element shape, so a `dim:` scalar naming the weight must get + 768, not the byte count. This is the assertion that would fail if `ne` were + ever filled from the byte shape.""" + spec = _spec(scalars=(Scalar("dim:1:0", "int"), Scalar("dim:1:1", "int"), + Scalar("numel:1", "int"))) + w = _weight((768, 3072)) + header = spec.header((np.zeros((256, 768), np.float16), w, + np.zeros(3072, np.float32)), {}) + import struct + k, n, numel = struct.unpack("buf[1]" in c + assert "(const hexlib_hf *) a->buf[1]" not in c + # and the dense neighbours are still cast to their own types + assert "(const hexlib_hf *) a->buf[0]" in c + assert "(const float *) a->buf[2]" in c + + +def test_the_generated_entry_guards_the_weights_dtype_and_layout(): + """Both guards, with the real ids from the real tables. + + The dtype guard stops a dense buffer being read as blocks. The LAYOUT guard + is the one `hexlib_dsp.h` says the enum exists for -- "un-repacked weights + are a plan-time error rather than silent corruption" -- and without it a + row-major q4_0 weight handed to a kernel expecting the repacked order is a + correctly-shaped wrong answer. + """ + c = emit_entry("matmul_epilogue", _spec()) + assert f"a->dtype[1] != {DTYPE_ID['q4_0']}u" in c + assert f"a->layout[1] != {LAYOUT_ID['q4_0_repacked']}u" in c + # the activation beside it keeps its own, different, expectations + assert f"a->dtype[0] != {DTYPE_ID['fp16']}u" in c + assert f"a->layout[0] != {LAYOUT_ID['row_major']}u" in c + + +def test_the_generated_entry_compiles_as_C_for_a_q4_0_spec(): + """Cheap syntactic proof that the emitted cast and guards are real C rather + than a plausible-looking string. The full compile-and-run probe for entries + lives in test_genentry_entry_probe.py; this only needs to know that adding a + raw dtype did not produce something unparseable.""" + c = emit_entry("matmul_epilogue", _spec()) + # balanced braces and every guard statement terminated + assert c.count("{") == c.count("}") + for line in c.splitlines(): + s = line.strip() + if s.startswith("if (") and "return" in s: + assert s.endswith(";"), f"unterminated guard: {s}" diff --git a/hexlib/tests/test_runtime_build.py b/hexlib/tests/test_runtime_build.py new file mode 100644 index 0000000..5dbbd61 --- /dev/null +++ b/hexlib/tests/test_runtime_build.py @@ -0,0 +1,73 @@ +# hexlib/tests/test_runtime_build.py +"""qaic generation. SDK-gated, in the same shape as the existing SDK tests.""" +import os +import pathlib + +import pytest + +from hexlib import toolchain as tc +from hexlib.runtime import build as rb + +HAS_SDK = os.path.isdir(tc.default_sdk_root()) +sdk = pytest.mark.skipif(not HAS_SDK, reason="Hexagon SDK not present") + +IDL = "hexlib/runtime/idl/hexlib_iface.idl" + + +def test_the_idl_declares_invoke_and_not_a_queue_id(): + """A dsp_queue_id in `start` would mean we had drifted back to dspqueue, + which has no simulator path. Asserted so the drift is caught, not noticed.""" + src = pathlib.Path(IDL).read_text() + assert "invoke(" in src + assert "dsp_queue_id" not in src + assert "sequence batch" in src + + +def test_the_idl_hwinfo_reports_arch_and_acquired_vtcm(): + src = pathlib.Path(IDL).read_text() + assert "rout uint32 arch" in src + assert "rout uint64 vtcm_size" in src + + +def test_qaic_path_is_platform_correct(): + p = rb.qaic_path("/fake/sdk") + assert "qaic" in p + if os.name == "nt": + assert p.endswith("qaic.exe") and "bin" in p + else: + assert "Ubuntu" in p + + +@sdk +def test_qaic_generates_three_files(tmp_path): + out = rb.run_qaic(IDL, str(tmp_path)) + for f in (out.header, out.stub, out.skel): + assert os.path.isfile(f), f + hdr = pathlib.Path(out.header).read_text() + assert "hexlib_iface_invoke" in hdr + assert "hexlib_iface_hwinfo" in hdr + + +@sdk +def test_generated_skel_dispatches_invoke(tmp_path): + out = rb.run_qaic(IDL, str(tmp_path)) + skel = pathlib.Path(out.skel).read_text() + assert "hexlib_iface_invoke" in skel + + +@sdk +def test_missing_idl_is_an_error_not_an_empty_success(tmp_path): + with pytest.raises(rb.RuntimeBuildError, match="not found"): + rb.run_qaic(str(tmp_path / "nope.idl"), str(tmp_path)) + + +def test_qaic_exit_zero_without_files_still_raises(tmp_path, monkeypatch): + """Offline. qaic exiting 0 but writing nothing must still raise -- this is + the fail-closed check itself, not the happy path. Without this test, + deleting that check would leave every other test passing (they all rely + on the real qaic actually writing files), which is exactly the "absence + read as success" failure mode the check exists to prevent.""" + monkeypatch.setattr(rb, "qaic_path", lambda root: IDL) + monkeypatch.setattr(rb.tc, "run", lambda cmd, env, timeout=None: (0, "", "", False)) + with pytest.raises(rb.RuntimeBuildError, match="did not produce"): + rb.run_qaic(IDL, str(tmp_path)) diff --git a/hexlib/tests/test_runtime_device_build.py b/hexlib/tests/test_runtime_device_build.py new file mode 100644 index 0000000..0a84a83 --- /dev/null +++ b/hexlib/tests/test_runtime_device_build.py @@ -0,0 +1,305 @@ +# hexlib/tests/test_runtime_device_build.py +"""Task 10 -- STAGE 2 GATE: cross-compile hexlib_run (Android aarch64) and +libhexlib_iface_skel.so (Hexagon device shared object). NEITHER IS EVER RUN HERE -- +no device is available -- so every SDK-gated test below asserts the built +ARTIFACT and its machine type, never merely that a function returned a path +string that happens to exist. +""" +import os +import struct + +import pytest + +from hexlib import toolchain as tc +from hexlib.runtime import build as rb + +HAS_SDK = os.path.isdir(tc.default_sdk_root()) +sdk = pytest.mark.skipif(not HAS_SDK, reason="Hexagon SDK not present") + + +def test_ndk_is_discovered_inside_the_sdk_never_vendored(): + p = tc.ndk_root("/fake/sdk") + assert "android-ndk-r25c" in p + assert p.startswith("/fake/sdk") or p.startswith("\\fake") + + +def test_android_api_is_pinned(): + assert tc.ANDROID_API == 33 + + +def test_ndk_clang_name_is_api_specific_not_a_generic_alias(): + """A regression to some other clang alias (e.g. a bare `clang` or a + different API level) would still be "a file that exists" on a real SDK, + so the SDK-gated existence test below cannot by itself catch a wrong + name -- this pins the exact filename, offline.""" + p = rb.ndk_clang("/fake/sdk") + base = os.path.basename(p) + assert base.startswith(f"aarch64-linux-android{tc.ANDROID_API}-clang") + assert "android-ndk-r25c" in p + + +def test_ndk_clang_uses_the_cmd_wrapper_on_windows(): + """The bare-name driver next to the `.cmd` wrapper is a Bourne shell + script (confirmed with `file` against the real NDK) that a plain + subprocess call cannot execute on native Windows. A regression back to + the bare name would still look plausible in a path string, so this pins + the suffix directly rather than trusting the SDK-gated existence check to + notice (os.path.isfile is also true of the unusable bare-name file).""" + p = rb.ndk_clang("/fake/sdk") + if os.name == "nt": + assert p.endswith(".cmd"), p + else: + assert not p.endswith(".cmd"), p + + +def test_the_aarch64_host_link_treats_warnings_as_errors(): + """The Hexagon side gets -Wall -Werror from tc.HVX_CFLAGS; this aarch64 + link builds its own command line, so it needs them spelled out here or the + host half of the wire (host/*.c, which assembles the batch blob and the + buffer table by hand) keeps compiling with its diagnostics discarded -- + every caller of tc.run decides success from `rc != 0` alone. + + Source-text rather than behavioural because building hexlib_run needs the + SDK and the NDK, and this must fail on a machine with neither. Comment + lines are excluded for the same reason + test_runtime_sim_build.py's -fpic check excludes them: the flag appears in + this function's own explanatory comment, and a `re.search` over the whole + body would keep passing after the real flag was deleted. + """ + import inspect + import re + + live = [ + ln for ln in inspect.getsource(rb.build_device_binary).splitlines() + if not ln.strip().startswith("#") + ] + joined = "\n".join(live) + assert re.search(r'cmd\s*=\s*\[clang[^\]]*"-Werror"', joined), ( + "build_device_binary no longer passes -Werror to the aarch64 clang" + ) + assert '"-Wall"' in joined + + +def test_device_skel_link_flags_are_the_dll_recipe_not_the_sim_one(): + """Recovered from the SDK's OWN defines_hexagon_1_9.min DLL_LD_FLAGS, a + DIFFERENT recipe from SIM_SO_LINK_FLAGS -- the distinguishing content is + the five --wrap= flags (PD heap interposition a real FastRPC-loaded skel + needs and the QuRT-hosted sim .so does not). Asserting only "-shared" and + "-fpic" would also pass for SIM_SO_LINK_FLAGS and would not catch a + build_device_binary that accidentally reused the sim recipe for the + device skel -- a real risk since both produce a Hexagon .so from the same + kind of object files.""" + flags = rb.DEVICE_SKEL_LINK_FLAGS + assert "-shared" in flags + assert "-fpic" in flags + for wrapped in ("malloc", "calloc", "free", "realloc", "memalign"): + assert f"-Wl,--wrap={wrapped}" in flags, f"missing --wrap={wrapped}" + joined = " ".join(flags) + assert "--force-dynamic" not in joined + + +@sdk +def test_device_skel_so_actually_carries_the_symbolic_dynamic_flag(tmp_path): + """FIX 2 (coordinator review): the test above only inspects the + `DEVICE_SKEL_LINK_FLAGS` constant and would still pass if + `_build_device_skel_so` stopped splicing it into the real link command + (e.g. if it started building the flags list itself instead of using this + one). This closes that gap by reading `-Wl,-Bsymbolic`'s effect back OUT + OF THE ARTIFACT: a linker that honoured `-Bsymbolic` records a `DT_SYMBOLIC` + (0x10) tag in the `.so`'s own `PT_DYNAMIC` segment -- confirmed against a + real build with the Hexagon toolchain's own `hexagon-readelf -d` before + writing this parser. + + `--wrap=malloc/calloc/free/realloc/memalign` is DELIBERATELY NOT CHECKED + THIS WAY. Checked with `hexagon-nm -u` against a real build: none of + skel.c/skel_bufs.c/skel_vtcm.c/skel_dispatch.c or the generated entries + reference malloc/calloc/free/realloc/memalign at all (Hexagon's `--wrap` + only rewrites a call site that actually exists), so there is NO ARTIFACT + EVIDENCE the flags could leave in THIS SPECIFIC BUILD even when they are + genuinely present on the link line and doing exactly what they are meant + to. Asserting anything artifact-shaped here would be inventing a proxy, + which the review this test responds to explicitly said not to do. + """ + rb.build_device_binary(str(tmp_path)) + so = os.path.join(str(tmp_path), rb.device_skel_so_name()) + with open(so, "rb") as f: + data = f.read() + DT_SYMBOLIC = 0x10 + assert DT_SYMBOLIC in _elf32_dynamic_tags(data), ( + "the skel .so has no DT_SYMBOLIC dynamic tag -- -Wl,-Bsymbolic " + "from DEVICE_SKEL_LINK_FLAGS did not actually reach the link" + ) + + +def _elf32_dynamic_tags(data): + """Every DT_* tag present in an ELF32 file's PT_DYNAMIC segment (Hexagon + is ELFCLASS32 -- confirmed by reading ei_class, byte 4, == 1 -- so this + uses Elf32_Phdr/Elf32_Dyn layouts, not the 64-bit ones the aarch64 helper + below needs).""" + assert data[:4] == b"\x7fELF" + assert data[4] == 1, "expected ELFCLASS32 for a Hexagon ELF" + e_phoff = struct.unpack_from(" 4096, "hexlib_run is implausibly small" + assert os.path.getsize(so) > 4096, "the skel .so is implausibly small" + + +def _elf_header(path): + with open(path, "rb") as f: + head = f.read(20) + assert head[:4] == b"\x7fELF", f"{path} has no ELF magic" + ei_class = head[4] + e_machine = struct.unpack("buf[0]" in src + assert "(hexlib_hf *) a->buf[1]" in src + assert "HEXLIB_DSP_OK" in src + + +def test_entry_checks_buffer_count_before_dereferencing_any(): + src = ge.emit_entry("scale", rn.SPECS["scale"]) + idx = src.index("a->n_buf") + assert idx < src.index("scale_fp16("), "the count check must come first" + assert "HEXLIB_DSP_ERR_INVAL_PARAMS" in src + + +def test_numel_scalar_comes_from_ne_not_from_a_host_promise(): + """`n` is derived on the DSP from the tensor extent it was given, so a host + that lied about the length cannot make the kernel run past the buffer.""" + src = ge.emit_entry("scale", rn.SPECS["scale"]) + assert "a->ne[0][0]" in src + + +def test_attr_scalar_comes_from_params_blob(): + src = ge.emit_entry("scale", rn.SPECS["scale"]) + assert "a->params" in src + + +def test_perm_requirement_is_documented_as_unenforceable_not_faked(): + """transpose covers three signatures. Handing a perm (0,2,1) op to the perm + (1,0,2) kernel returns a correctly-shaped, silently WRONG layout that every + downstream shape check accepts -- but `hexlib_args` has no field that + carries a permutation (only per-buffer buf/ne/dtype/layout, plus n_buf, + vtcm, params, n_threads), so no `if` written here could ever fail on a + perm mismatch. Enforcement lives only on the host, in + `RunnerSpec.check_requires`, before the op is ever put on the wire. This + gap must be documented plainly, not papered over with a check that cannot + fail -- a check that cannot fail is indistinguishable from no check at all + except that it looks like protection.""" + src = ge.emit_entry("transpose", rn.SPECS["transpose"]) + assert "NOT VERIFIED ON THE DSP" in src + assert "perm" in src, "the gap should name the key it cannot verify" + checks = [line for line in src.splitlines() if line.strip().startswith("if (")] + assert not any("perm" in line for line in checks), ( + "a reachable `if` mentioning perm would be a check that cannot fail, " + "i.e. exactly the decorative check this generator must not emit" + ) + + +def test_cast_requires_fp16_dtype(): + """Unlike `perm`, `dtype` maps onto a real per-buffer field + (`hexlib_args.dtype[]`), so this one must be a reachable check, not just a + documented gap -- asserting only the status-code string would still pass + if the real check were downgraded to a comment, which is the one thing + this test exists to catch.""" + src = ge.emit_entry("cast", rn.SPECS["cast"]) + assert "HEXLIB_DSP_ERR_REQUIRES" in src + checks = [line for line in src.splitlines() if line.strip().startswith("if (")] + assert any("a->dtype[" in line for line in checks), ( + "the dtype requirement must be a reachable `if (a->dtype[...] ...)` " + "check, not only documented" + ) + + +def test_every_input_buffers_dtype_is_checked_before_it_is_cast(): + """THE FINDING. `a->dtype[i]` is carried per BUFFER and was read for exactly + one of them -- the output, and only when `requires` happened to mention it. + Meanwhile `emit_entry`'s own docstring names the hazard: "casting an fp32 + buffer to hexlib_hf* would halve every stride silently". A batch declaring a + `scale` input as fp32 with ne[0] = 4100 -- reachable from `run_raw`, from + main.c's hand-built blob, or from a future planner -- had 8200 of its 16400 + bytes read at half stride and got HEXLIB_DSP_OK back. + + So every buffer's declared dtype is now checked against the dtype the entry + is about to cast it to. Asserted as a reachable `if` per buffer INDEX, not + as a substring: a check on buf[0] only would still leave `add`'s second + operand and every output unguarded.""" + src = ge.emit_entry("cast", rn.SPECS["cast"]) + checks = [ln for ln in src.splitlines() if ln.strip().startswith("if (")] + assert any("a->dtype[0] !=" in ln for ln in checks), ( + "the fp32 INPUT's dtype must be checked before it is cast to float*" + ) + assert any("a->dtype[1] !=" in ln for ln in checks), ( + "the fp16 OUTPUT's dtype must be checked before it is cast to hexlib_hf*" + ) + # add has two inputs: the second one is the operand a single-buffer check + # would miss. + add = ge.emit_entry("add", rn.SPECS["add"]) + add_checks = [ln for ln in add.splitlines() if ln.strip().startswith("if (")] + for i in range(3): + assert any(f"a->dtype[{i}] !=" in ln for ln in add_checks), ( + f"buffer {i} of add is cast without its dtype being checked" + ) + + +def test_the_dtype_check_uses_the_wire_id_for_that_buffers_own_dtype(): + """The VALUE, bound to the buffer. `cast` is fp32 in, fp16 out, so the two + checks must compare against DIFFERENT ids -- a generator that used the + output's dtype for every buffer would emit two identical checks and refuse + every legitimate cast batch, and one that used the input's would let the + halved-stride write through.""" + from hexlib.runtime.wire import DTYPE_ID + + src = ge.emit_entry("cast", rn.SPECS["cast"]) + assert f"a->dtype[0] != {DTYPE_ID['fp32']}u" in src + assert f"a->dtype[1] != {DTYPE_ID['fp16']}u" in src + + +def test_the_buffer_count_and_null_checks_still_come_before_any_dtype_check(): + """`a->dtype[i]` is only meaningful for a buffer the batch actually + supplied, so the count check has to stay first.""" + src = ge.emit_entry("cast", rn.SPECS["cast"]) + assert src.index("a->n_buf") < src.index("a->dtype[") + assert src.index("!a->buf[0]") < src.index("a->dtype[") + assert src.index("a->dtype[") < src.index("cast_f32_f16(") + + +def test_a_dtype_requirement_that_disagrees_with_the_declared_dtype_is_refused(): + """THE RELATED MINOR, MADE LOUD. `_requires_check` hardcoded `out_idx` for + every key: correct for `cast` today, and silently wrong for any future + dtype requirement about an INPUT, which would have inspected the OUTPUT's + dtype instead. There is no way to say which buffer a `requires` entry is + about, so the generator refuses the ambiguous case rather than guessing -- + a spec whose dtype requirement is not its own declared output dtype is + either about an input (unexpressible) or a contradiction (it would refuse + every batch this serializer can build).""" + bad = rn.RunnerSpec( + kind="castish", kernel_dir="kernels/castish", inputs=("fp32",), + out_dtype="fp16", scalars=(rn.Scalar("numel:0", "int"),), + requires=(("dtype", "fp32"),), + ) + with pytest.raises(ge.GenError, match="dtype"): + ge.emit_entry("castish", bad) + + +def test_every_shipped_dtype_requirement_restates_its_own_declared_out_dtype(): + """Stated as a test because it is the reason the DSP-side dtype check + cannot fail through `hexlib/exec/dsp.py`: that serializer stamps the output + tensor's dtype from `spec.out_dtype`, which is the same value the + requirement holds, so the generated `if` compares the spec with itself. + The requirement is really enforced on the host, against the CALLER'S attr + (`RunnerSpec.check_requires`). The generated check is still worth having -- + it is reachable from a hand-built batch (main.c, `run_raw`) -- but its reach + should not be overstated, and this pins the fact.""" + for name, spec in rn.SPECS.items(): + for key, want in spec.requires: + if key == "dtype": + assert want == spec.out_dtype, name + + +def test_table_is_sorted_and_terminated(): + src = ge.emit_table({"scale": rn.SPECS["scale"], "add": rn.SPECS["add"]}) + assert "hexlib_kernel_table[]" in src + assert "hexlib_kernel_table_len" in src + assert src.index('"add"') < src.index('"scale"'), "sorted, so diffs are stable" + + +def test_kind_ids_are_stable_across_runs(): + """A kind id crossing the wire must not depend on dict ordering — a renumber + silently sends every op to the wrong kernel.""" + assert ge.KIND_ID == dict(sorted(ge.KIND_ID.items(), key=lambda kv: kv[1])) + ids = list(ge.KIND_ID.values()) + assert ids == sorted(ids) and len(set(ids)) == len(ids) + + +def test_every_spec_has_a_kind_id(): + for name in rn.SPECS: + assert name in ge.KIND_ID, f"{name} has no wire id" + + +# The ids as shipped. ON-WIRE IDS ARE AN ABI: `skel_dispatch.c` matches an op by +# id alone, the blob carries no table version, and nothing in a response would +# reveal a mismatch except the id itself. So a new kind is APPENDED and an +# existing one never moves. test_kind_ids_are_stable_across_runs below asserts +# only sortedness and uniqueness, which a wholesale renumber preserves -- and a +# renumber that moved `scale` from 9 onto `transpose`'s 11 would dispatch every +# scale op to transpose_th_fp16_entry with a matching buffer count, two non-null +# pointers, and HEXLIB_DSP_OK. This literal is what makes that fail. +SHIPPED_KIND_IDS = { + "add": 1, "cast": 2, "layernorm": 3, "matmul": 4, "matmul_epilogue": 5, + "patchify": 6, "reshape": 7, "rope_2d": 8, "scale": 9, "softmax": 10, + "transpose": 11, +} + + +def test_the_shipped_kind_ids_never_move(): + for name, want in SHIPPED_KIND_IDS.items(): + assert ge.KIND_ID.get(name) == want, ( + f"{name} was id {want} on the wire and is now " + f"{ge.KIND_ID.get(name)}. Append new kinds; never renumber." + ) + new = set(ge.KIND_ID) - set(SHIPPED_KIND_IDS) + assert all(ge.KIND_ID[n] > max(SHIPPED_KIND_IDS.values()) for n in new), ( + f"{sorted(new)} must take ids above {max(SHIPPED_KIND_IDS.values())}" + ) + + +def test_every_kind_a_COMPILED_PLAN_can_contain_has_a_wire_id(): + """THE COVERAGE CLAIM, CHECKED AGAINST THE REGISTRY RATHER THAN ASSUMED. + The op registry holds 13 kinds. The two with no id of their own are + `gelu_tanh` and `gelu_erf`, and both are in `fuse.FUSABLE_ACTS`: fusion + absorbs them into `matmul_epilogue`'s `act` attr, so neither can appear as a + standalone plan step and there is no live wire gap today. + + That is a claim about a PASS, though, not about the table, and it is exactly + the claim that stops being true the moment a new op kind is registered + without an id -- at which point `dsp.py` raises KeyError on a graph that + compiles fine. So the registry is compared here rather than trusted, and a + new kind that is neither fusable nor given an id fails this. + + `KIND_ID` IS KEYED BY KERNEL VARIANT, NOT BY OP KIND. This test used to + assert `set(KIND_ID) <= set(REGISTRY.all_kinds())` -- every id is an op kind + -- which was true only while every kind had at most one kernel. It stopped + being true with `transpose_hd`: the encoder's 60 transposes are two + permutations needing two genuinely different kernels, the wire carries no + perm for the DSP to branch on, so each variant needs its own id and + `runner.select` resolves which one on the host. + + The claim that assertion was really making -- that no id is a dead or + invented name -- is still worth holding, so it is made precisely instead of + being dropped: every id must name a `SPECS` variant, and every variant's + `.kind` must be a registered op kind. That still fails on a typo, on an id + left behind by a deleted kernel, and on a variant claiming a kind that does + not exist.""" + import hexlib.graph.opdefs # noqa: F401 -- registers the op defs + from hexlib.graph.fuse import FUSABLE_ACTS + from hexlib.graph.ops import REGISTRY + + kinds = set(REGISTRY.all_kinds()) + dispatchable = kinds - set(FUSABLE_ACTS) + + # Every dispatchable KIND must be reachable: at least one variant implements + # it and that variant has an id. Asking for the kind's own name in KIND_ID + # would now be wrong -- a kind served only by differently-named variants is + # still perfectly reachable. + unreachable = sorted( + k for k in dispatchable + if not any(n in ge.KIND_ID for n in rn.variants_for(k)) + and k in {s.kind for s in rn.SPECS.values()} + ) + assert not unreachable, ( + f"{unreachable} has a kernel variant but no wire id for it; dsp.py would " + f"raise KeyError on a graph that compiled cleanly" + ) + + # AND NO ID IS A NAME NOTHING CAN EVER MEAN. Two kinds of key are legitimate: + # an op kind's own name, which reserves an id before any kernel exists (the + # DSP answers ERR_NO_KERNEL for those, which is why `matmul` and `patchify` + # have ids and no spec), or a SPECS variant name whose `.kind` is registered. + # Anything else is a typo, or an id left behind by a deleted kernel. + meaningless = sorted( + n for n in ge.KIND_ID + if n not in kinds and (n not in rn.SPECS or rn.SPECS[n].kind not in kinds) + ) + assert not meaningless, ( + f"{meaningless} has a wire id but is neither a registered op kind nor a " + f"SPECS variant of one, so nothing can ever dispatch to it" + ) + + +def test_generated_entry_includes_the_kernel_api_header(): + src = ge.emit_entry("scale", rn.SPECS["scale"]) + assert '#include "kernel_api.h"' in src + assert '#include "hexlib_dsp.h"' in src + + +def test_input_dtype_not_output_dtype_decides_the_input_cast(): + """cast is fp32 in, fp16 out. Casting the input to hexlib_hf* would halve + every stride and read the wrong half of the buffer — a plausible wrong + answer, not a crash.""" + src = ge.emit_entry("cast", rn.SPECS["cast"]) + assert "(const float *) a->buf[0]" in src + assert "(hexlib_hf *) a->buf[1]" in src + + +def test_the_function_name_is_the_basename_of_the_kernel_dir(): + """spec.kernel_dir is a PATH, 'kernels/scale_fp16'. Using it verbatim would + emit `kernels/scale_fp16_entry`, which is not an identifier.""" + assert rn.SPECS["scale"].kernel_dir == "kernels/scale_fp16" + src = ge.emit_entry("scale", rn.SPECS["scale"]) + assert "int scale_fp16_entry(" in src + assert "kernels/" not in src.split("Source of truth")[1] + + +def test_a_hand_written_dsp_entry_wins(tmp_path): + """The escape hatch is real and its use is visible: a kernel whose argument + mapping is not expressible declaratively ships its own dsp_entry.c, and the + generator must not overwrite or shadow it. + + `expect=("scale",)` states what this one-kernel tmp tree actually claims to + hold -- without it, `generate` refuses the tree as a partial checkout, which + is the point of the test below.""" + kdir = tmp_path / "kernels" / "scale_fp16" + kdir.mkdir(parents=True) + (kdir / "dsp_entry.c").write_text("/* hand written */\n") + written = ge.generate(str(tmp_path), str(tmp_path / "out"), expect=("scale",)) + assert not any("scale_fp16_entry.c" in w for w in written) + assert any("hexlib_kernel_table.c" in w for w in written) + + +def test_generate_takes_the_REPO_root_not_a_kernels_root(tmp_path): + """kernel_dir is repo-relative. Passing `/kernels` would look for + `kernels/kernels/scale_fp16`, find nothing, and emit an EMPTY dispatch table + -- a build that links and then reports 'no kernel for kind 9' at run time.""" + (tmp_path / "kernels" / "scale_fp16").mkdir(parents=True) + ok = ge.generate(str(tmp_path), str(tmp_path / "out"), expect=("scale",)) + assert any("scale_fp16_entry.c" in w for w in ok) + with pytest.raises(ge.GenError, match="no kernel"): + ge.generate(str(tmp_path / "kernels"), str(tmp_path / "out2"), + expect=("scale",)) + + +def test_a_PARTIAL_kernel_tree_is_an_error_not_a_partial_dispatch_table(tmp_path): + """THE MINOR. `generate` raised only when NO kernel directory was found. A + tree missing SOME of them emitted a dispatch table missing those rows, with + no error at all -- so the ops answered HEXLIB_DSP_ERR_NO_KERNEL at run time, + which reads as "this kernel is broken" rather than "the generator was + pointed at an incomplete tree". Absence reported as partial success is the + same shape as absence reported as success, one step down. + + By default every kind with a `RunnerSpec` must have its directory; `expect` + narrows that for a caller that genuinely holds a subset (only the tests in + this file, today).""" + (tmp_path / "kernels" / "scale_fp16").mkdir(parents=True) + with pytest.raises(ge.GenError) as exc: + ge.generate(str(tmp_path), str(tmp_path / "out")) + # The MISSING list, not merely the names somewhere in the message: the one + # kernel that IS present must not be reported as absent. + # + # DERIVED FROM `SPECS`, NOT HARDCODED. This read `for ['add', 'cast', + # 'transpose']`, which is every spec except the one staged above -- so + # adding a legitimate new RunnerSpec (layernorm) failed it, on a change the + # test has no reason to care about. A test that has to be edited whenever + # the roster it is not testing grows is over-specified: it makes the test + # dictate the spec table. The property being checked is unchanged. + missing = sorted(k for k in rn.SPECS if k != "scale") + assert f"for {missing}" in str(exc.value) + # And the present one is reported as PRESENT. Asserting `"scale" not in + # message` would be wrong -- the message says `found ['scale']`, which is + # exactly the diagnostic a reader needs. + assert "found ['scale']" in str(exc.value), ( + "the kernel that IS present must be reported as found, so the operator " + "can tell an incomplete checkout from a wrong kernels root" + ) + assert not (tmp_path / "out" / "hexlib_kernel_table.c").exists(), ( + "a partial dispatch table must not be left behind for a build to link" + ) + + +def test_an_expect_naming_a_kind_with_no_spec_is_an_error(tmp_path): + """`expect` narrows a claim; it cannot invent one. A typo'd or stale name + would otherwise quietly narrow nothing.""" + (tmp_path / "kernels" / "scale_fp16").mkdir(parents=True) + with pytest.raises(ge.GenError, match="scal"): + ge.generate(str(tmp_path), str(tmp_path / "out"), expect=("scal",)) + + +def test_unknown_scalar_source_is_an_error_not_a_zero(): + bad = rn.RunnerSpec( + kind="bogus", kernel_dir="bogus_fp16", inputs=("fp16",), out_dtype="fp16", + scalars=(rn.Scalar(source="wat:1"),), + ) + with pytest.raises(ge.GenError, match="wat:1"): + ge.emit_entry("bogus", bad) diff --git a/hexlib/tests/test_runtime_sim_build.py b/hexlib/tests/test_runtime_sim_build.py new file mode 100644 index 0000000..248ec21 --- /dev/null +++ b/hexlib/tests/test_runtime_sim_build.py @@ -0,0 +1,355 @@ +# hexlib/tests/test_runtime_sim_build.py +"""Task 7b: the QuRT-hosted simulator build that actually reaches +hexlib_iface_start. + +Task 7's standalone `--force-dynamic` qexe (build_sim_qexe, gone now) built +and linked and reached hexlib_iface_open, but hexlib_iface_start could NEVER +succeed there: VTCM acquisition needs real QuRT thread/clock primitives a +standalone qexe cannot provide (see +.superpowers/sdd/2026-08-10-silicon-path-runtime/ +investigation-sim-vtcm-and-marshalling.md). build_sim_so below produces a +shared object dlopen'd by the SDK's own prebuilt run_main_on_hexagon_sim +under a real booted QuRT kernel instead -- the load-bearing test here, +test_sim_run_reaches_start_and_reports_real_vtcm, is the one that proves the +fix actually works, not merely that a function returned a path string. +""" +import contextlib +import inspect +import os +import re +import struct +import subprocess + +import pytest + +import hexlib +from hexlib import toolchain as tc +from hexlib.runtime import build as rb +from hexlib.runtime import wire + +HAS_SDK = os.path.isdir(tc.default_sdk_root()) +sdk = pytest.mark.skipif(not HAS_SDK, reason="Hexagon SDK not present") + +REPO_ROOT = os.path.dirname(os.path.dirname(os.path.abspath(hexlib.__file__))) + + +@contextlib.contextmanager +def _cwd(path): + """tc.run() (hexlib/toolchain.py, off limits here) takes no `cwd` kwarg + and always inherits the calling process's own working directory. As + `sim_qurt_command`'s own docstring now says: under the QuRT-hosted + launch, `fopen(..., "wb")` writes land in the LAUNCHING PROCESS'S cwd, + not in `--usefs`'s directory. Without this, running these tests from the + repo root (exactly what `python -m pytest hexlib/tests` does) would write + hexlib_rsp.bin/hexlib_out.bin into the repo's own working directory on + every run -- confirmed the hard way: an earlier run of this file did + exactly that and silently modified a tracked fixture file at the repo + root. This is the fix the docstring tells every future caller to make.""" + prev = os.getcwd() + os.chdir(path) + try: + yield + finally: + os.chdir(prev) + + +# ============================================================================ +# Pure tests: no SDK filesystem access, assertable anywhere. +# ============================================================================ + + +def test_sim_so_link_flags_are_pic_shared_not_a_standalone_exe(): + """SIM_SO_LINK_FLAGS is a shared-object recipe, recovered from the SDK's + own libs/run_main_on_hexagon example's test_main_so_link.txt -- NOT the + retired standalone-qexe recipe. A regression back to the old + --force-dynamic exe shape would defeat the whole point of this task (it + is exactly the shape that can never acquire VTCM), so this also asserts + the old flags are ABSENT, not just that the new ones are present.""" + flags = rb.SIM_SO_LINK_FLAGS + assert "-fpic" in flags + assert "-shared" in flags + assert "-Wl,-Bsymbolic" in flags + assert "-lc" in flags + joined = " ".join(flags) + assert "--force-dynamic" not in joined, ( + "this is the standalone-qexe flag that makes VTCM acquisition " + "impossible -- it must not reappear on the .so recipe" + ) + assert "--dynamic-linker=" not in joined + + +def test_build_skel_lib_compiles_position_independent_code(): + """Every object build_skel_lib compiles must carry -fpic, because the + only thing this archive is ever linked into is build_sim_so's shared + object -- non-PIC objects in a `-shared` link either fail to link outright + or (worse, silently) produce a .so with text-relocations a real device + loader would refuse. Reading the function's own source (not merely + grepping the whole file, which would also match this test module's own + docstring if it discussed -fpic) so a `-fpic` mentioned only in a comment, + with the actual compile call unchanged, would NOT be enough to pass this.""" + src = inspect.getsource(rb.build_skel_lib) + # Line-by-line, and only lines that do not START (after stripping + # leading whitespace) with a comment marker -- a commented-out call + # still contains this exact substring, so a plain `re.search` over the + # whole function body would NOT catch that regression. + live_lines = [ + ln for ln in src.splitlines() if not ln.strip().startswith("#") + ] + assert any( + re.search(r'cmd\.insert\(\s*1\s*,\s*"-fpic"\s*\)', ln) for ln in live_lines + ), "build_skel_lib no longer inserts -fpic into every compile command" + + +def test_sim_v_arch_is_the_sdks_own_lookup_not_a_mechanical_suffix(): + """Recovered from build/make.d.ext/hexagon/defines_hexagon_1_9.min's + SIM_V_ARCH table. Not reconstructable as "na_1" -- v68 and v81 use + entirely different suffixes -- so this pins the exact table, not a + plausible-looking formula that would happen to pass for v75 alone.""" + assert rb.sim_v_arch("v75") == "v75na_1" + assert rb.sim_v_arch("v68") == "v68n_1024" + assert rb.sim_v_arch("v81") == "v81qa_1" + + +def test_run_main_on_hexagon_sim_path_is_sdk_relative_never_vendored(): + p = rb.run_main_on_hexagon_sim_path("/sdk", "v75") + joined = p.replace("\\", "/") + assert joined == "/sdk/libs/run_main_on_hexagon/ship/hexagon_toolv19_v75/run_main_on_hexagon_sim" + + +def test_runelf_pbn_path_is_sdk_relative_never_vendored(): + p = rb.runelf_pbn_path("/sdk", "v75") + joined = p.replace("\\", "/") + assert joined == "/sdk/rtos/qurt/computev75/sdksim_bin/runelf.pbn" + + +def test_generated_sim_configs_are_actually_gitignored_by_name(): + """FIX 1 (coordinator review, task 7b): write_qurt_sim_configs's own + docstring used to claim osam.cfg/q6ss.cfg were "already git-ignored via + the same rules that already ignore *.o/*.a/*.so" -- no rule actually + matched *.cfg, so that was a stated guarantee with nothing behind it: a + caller pointing out_dir INSIDE the repo would have committed SDK-derived + config files, only avoided today by pytest's tmp_path happening to sit + outside the repo. + + Asks git DIRECTLY (`git check-ignore`) rather than re-parsing + `.gitignore` in Python, so this cannot drift from what git itself will + actually do -- reimplementing gitignore's own matching logic would risk + the test and the real behavior silently disagreeing. + + Uses a path nested under a directory ("_no_such_dir") that appears + nowhere else in `.gitignore`, and a same-directory sibling with a + different extension as a negative control, so a pass here can only be + explained by a rule that names these two files specifically -- not a + blanket `*.cfg`, and not some unrelated existing rule (e.g. `_work/`) + catching it by accident. + """ + for name in ("osam.cfg", "q6ss.cfg"): + rel = os.path.join("hexlib", "runtime", "_no_such_dir", name) + rc = subprocess.run( + ["git", "check-ignore", "-q", rel], cwd=REPO_ROOT + ).returncode + assert rc == 0, f"{rel!r} is not matched by .gitignore (rc={rc})" + + # Negative control: a same-shaped path that must NOT be ignored, proving + # the match above is specific to these two filenames. + control = os.path.join("hexlib", "runtime", "_no_such_dir", "not_a_generated_sim_config.cfg") + rc = subprocess.run(["git", "check-ignore", "-q", control], cwd=REPO_ROOT).returncode + assert rc != 0, ( + f"{control!r} is unexpectedly git-ignored -- the rule added for " + "osam.cfg/q6ss.cfg may have been written as a blanket *.cfg instead " + "of the tighter by-name pattern that was asked for" + ) + + +# ============================================================================ +# SDK-gated: actually build and run. +# ============================================================================ + + +@sdk +def test_skel_library_builds(tmp_path): + lib = rb.build_skel_lib(["scale_fp16"], str(tmp_path)) + assert os.path.isfile(lib) + assert os.path.getsize(lib) > 0 + # An ar archive, not merely a path that happens to exist: the magic bytes + # a mock or a `touch` would not reproduce. + with open(lib, "rb") as f: + assert f.read(8) == b"!\n" + + +@sdk +def test_sim_so_builds_a_real_hexagon_shared_object(tmp_path): + rb.build_skel_lib(["scale_fp16"], str(tmp_path)) + so = rb.build_sim_so(str(tmp_path)) + assert os.path.isfile(so) + assert os.path.getsize(so) > 0 + with open(so, "rb") as f: + header = f.read(20) + assert header[:4] == b"\x7fELF" + e_type = header[16] | (header[17] << 8) + e_machine = header[18] | (header[19] << 8) + # ET_DYN (3), not ET_EXEC (2): a build that regressed back to a + # standalone --force-dynamic executable would still be a valid Hexagon + # ELF and would still pass a bare "is this an ELF" check, but it could + # never be dlopen'd by run_main_on_hexagon_sim, which is the entire + # mechanism this task depends on. + assert e_type == 3, f"expected ET_DYN (shared object), got e_type={e_type}" + assert e_machine == 0xA4, f"expected EM_HEXAGON, got e_machine={e_machine}" + + +@sdk +def test_qurt_sim_configs_reference_real_sdk_artifacts_never_vendored(tmp_path): + """osam.cfg/q6ss.cfg must land in the build OUTPUT directory (never the + source tree), and must each be a thin reference to an SDK file that + genuinely exists on disk -- not a copy of its bytes.""" + osam_cfg, q6ss_cfg = rb.write_qurt_sim_configs(str(tmp_path)) + assert os.path.dirname(osam_cfg) == str(tmp_path) + assert os.path.dirname(q6ss_cfg) == str(tmp_path) + + osam_target = open(osam_cfg).read().strip() + assert os.path.isfile(osam_target), ( + f"osam.cfg names {osam_target!r}, which does not exist -- a dangling " + "reference is as bad as a vendored copy that went stale" + ) + assert osam_target.endswith("qurt_model.dll") + + q6ss_lines = open(q6ss_cfg).read().splitlines() + assert len(q6ss_lines) == 2 + for line in q6ss_lines: + path = line.split()[0] + assert os.path.isfile(path), f"q6ss.cfg names {path!r}, which does not exist" + assert q6ss_lines[0].endswith( + "qtimer.dll --csr_base=0xFC900000 --irq_p=3 --freq=19200000 --cnttid=1" + ) + assert q6ss_lines[1].endswith("l2vic.dll 32 0xFC910000") + + +@sdk +def test_sim_qurt_command_shape(tmp_path): + """Pure assembly, but needs the SDK to locate hexagon-sim -- checked + ordering matters: runelf.pbn loads run_main_on_hexagon_sim, which then + dlopen's the .so by (bare) name, and extra_args land after it.""" + so = str(tmp_path / "hexlib_sim.so") + cmd = rb.sim_qurt_command(str(tmp_path), so, extra_args=("--unmapped",)) + + def idx(needle): + for i, c in enumerate(cmd): + if needle in c: + return i + raise AssertionError(f"{needle!r} not found in {cmd}") + + i_runelf = idx("runelf.pbn") + i_run_main = idx("run_main_on_hexagon_sim") + i_so = cmd.index("hexlib_sim.so") + assert i_runelf < i_run_main < i_so, ( + "runelf.pbn must load run_main_on_hexagon_sim, which must dlopen the " + ".so, in that order" + ) + assert cmd[-1] == "--unmapped", "extra_args must come after the .so name" + assert "--usefs" in cmd and cmd[cmd.index("--usefs") + 1] == str(tmp_path) + assert "--rtos" in cmd + assert "--cosim_file" in cmd + assert "-mv75na_1" in cmd + + +@sdk +def test_sim_run_reaches_start_and_reports_real_vtcm(tmp_path): + """THE LOAD-BEARING TEST. Actually boots a real QuRT kernel under + hexagon-sim, dlopen's hexlib's own .so, and checks the SIMHOST output -- + not merely that build functions returned paths. A skel that still failed + to acquire VTCM (the exact defect this task exists to fix) would print + `SIMHOST error=start`, never reach the hwinfo line, and this test would + fail; a start that reported the wrong arch or the wrong VTCM size would + fail the regex match below even if `start` itself returned 0.""" + out_dir = str(tmp_path) + rb.build_skel_lib(["scale_fp16"], out_dir) + so = rb.build_sim_so(out_dir) + rb.write_qurt_sim_configs(out_dir) + + # An empty-but-well-formed batch: 0 bufs, 0 tensors, 0 ops. This is + # enough to drive open -> start -> hwinfo -> invoke -> stop -> close all + # the way through without needing rpcmem/fd plumbing, which is Task 8's + # concern, not this one's. + with open(os.path.join(out_dir, "hexlib_batch.bin"), "wb") as f: + f.write(wire.pack_batch([], [], [])) + with open(os.path.join(out_dir, "hexlib_in.bin"), "wb") as f: + f.write(b"") + + cmd = rb.sim_qurt_command(out_dir, so) + bin_dir = tc.find_toolchain_bin(tc.default_sdk_root()) + env = tc.toolchain_env(bin_dir) + # See sim_qurt_command's own docstring and _cwd's: writes land in the + # LAUNCHING process's cwd, not --usefs's directory, so this MUST run with + # cwd == out_dir or hexlib_rsp.bin/hexlib_out.bin land in the repo root. + with _cwd(out_dir): + rc, out, err, timed_out = tc.run(cmd, env, timeout=tc.SIM_TIMEOUT_MAX_S) + combined = out + err + + assert not timed_out, combined + assert "SIMHOST error=open" not in combined, combined + assert "SIMHOST error=start" not in combined, combined + + m = re.search(r"SIMHOST hwinfo arch=(\d+) threads=(\d+) vtcm=(\d+)", combined) + assert m, f"no SIMHOST hwinfo line recovered -- start did not succeed:\n{combined}" + assert int(m.group(1)) == 75, f"expected arch=75, got {m.group(0)}" + assert int(m.group(3)) == 8388608, f"expected vtcm=8388608, got {m.group(0)}" + + assert "SIMHOST invoke rc=0" in combined, combined + assert "SIMHOST done" in combined, combined + assert rc == 0, f"process exited {rc}:\n{combined}" + + +@sdk +def test_an_unmapped_fd_batch_is_still_refused_under_the_new_build(tmp_path): + """The property task-7b was told to preserve: --unmapped must still + DELIBERATELY SKIP hexlib_iface_mmap, and the skel must still refuse. + Only a real op naming a real (registered) buffer can exercise the + discriminator, so this builds one real "scale" op -- if the new + QuRT-hosted packaging accidentally made mapping a no-op (e.g. by + resolving the raw fd as an address the way a shared-address-space bug + would), this invoke would return OK instead of ERR_UNMAPPED.""" + from hexlib.runtime.genentry import KIND_ID + + out_dir = str(tmp_path) + rb.build_skel_lib(["scale_fp16"], out_dir) + so = rb.build_sim_so(out_dir) + rb.write_qurt_sim_configs(out_dir) + + n = 8 + bufs = [wire.BufDesc(fd=0, size=n * 2 * 2)] + tensors = [ + wire.TensorDesc(bi=0, offset=0, nbytes=n * 2, dtype="fp16", + layout="row_major", ne=(n, 1, 1, 1)), + wire.TensorDesc(bi=0, offset=n * 2, nbytes=n * 2, dtype="fp16", + layout="row_major", ne=(n, 1, 1, 1)), + ] + factor_bits = struct.unpack("buf[]` at + `nb >= HEXLIB_MAX_BUFS`), indistinguishable from a dozen other causes. This + module's whole reason for existing is that a refusal belongs where it can + name the numbers rather than where it can only answer with one.""" + bufs = [wire.BufDesc(fd=0, size=4096)] + tensors = [ + wire.TensorDesc(bi=0, offset=64 * i, nbytes=64, dtype="fp16", + layout="row_major", ne=(32, 1, 1, 1)) + for i in range(9) + ] + with pytest.raises(wire.WireError, match=r"6 sources \+ 3 destinations"): + wire.pack_batch( + bufs=bufs, tensors=tensors, + ops=[wire.OpDesc(kind=1, src=(0, 1, 2, 3, 4, 5), dst=(6, 7, 8))], + ) + # 6 + 2 is exactly HEXLIB_MAX_BUFS: the boundary is allowed, so the check + # cannot be an off-by-one that refuses a legal fused op. + wire.pack_batch( + bufs=bufs, tensors=tensors, + ops=[wire.OpDesc(kind=1, src=(0, 1, 2, 3, 4, 5), dst=(6, 7))], + ) + + +def test_the_dtype_table_uses_the_same_SPELLING_as_the_runner_and_the_generator(): + """THREE TABLES, ONE SET OF KEYS. `wire.DTYPE_ID` names the ids that cross + the wire; `runner.WIRE_DTYPE` names the numpy form of the same dtype; + `genentry._CTYPE` names its C form. genentry.py's own comment claims they + are keyed alike -- they were not: `DTYPE_ID` spelled int32 as "i32". + + The first `RunnerSpec` declaring an int32 input would then fail twice, in + two different places, for the same reason: `pack_batch` raising "unknown + dtype 'int32'" and `_requires_check` raising KeyError at generate time. + Loud, but it means the wire cannot carry a tensor `RunnerSpec` accepts -- + and `rope_2d` and `patchify`, both about to be written, are the kernels that + would hit it. + + The BLOCK-QUANTIZED dtypes are the deliberate asymmetry, asserted rather + than tolerated: they have a wire id because a block-quantized weight is a + real tensor on the DSP, and no numpy/C scalar form because they are staged + as raw bytes. + + That assertion used to read `== {"q4_0"}`, a literal, and q8_0 broke it the + moment it was added -- correctly, since a new wire id with no numpy form is + exactly what the check is watching for. It is bound to `WIRE_RAW` now, which + is the set that decides the question everywhere else (genentry picks + `unsigned char *` from it, and `RawTensor.__post_init__` refuses anything + outside it). A literal here would have to be edited for every future format + and is satisfied by editing it; the binding is not. + """ + from hexlib.exec.runner import WIRE_DTYPE, WIRE_RAW + from hexlib.runtime.genentry import _CTYPE + + assert set(WIRE_DTYPE) <= set(wire.DTYPE_ID), ( + f"{sorted(set(WIRE_DTYPE) - set(wire.DTYPE_ID))} can be declared by a " + "RunnerSpec but cannot be serialized" + ) + assert set(_CTYPE) <= set(wire.DTYPE_ID), ( + f"{sorted(set(_CTYPE) - set(wire.DTYPE_ID))} has a C type in the " + "generated entry but no wire id" + ) + assert set(WIRE_DTYPE) == set(_CTYPE), ( + "every dtype a spec can declare needs a C type in the generated entry, " + "and vice versa" + ) + assert set(wire.DTYPE_ID) - set(WIRE_DTYPE) == set(WIRE_RAW), ( + "every wire dtype without a numpy form must be a declared raw " + "block-quantized one, and every raw one must have a wire id" + ) + assert WIRE_RAW, "WIRE_RAW is empty, so the assertion above is vacuous" + + +def test_tensor_naming_a_nonexistent_buffer_is_refused(): + with pytest.raises(wire.WireError, match="buffer index"): + wire.pack_batch( + bufs=[wire.BufDesc(fd=7, size=64)], + tensors=[wire.TensorDesc(bi=3, offset=0, nbytes=8, dtype="fp16", + layout="row_major", ne=(4, 1, 1, 1))], + ops=[], + ) + + +def test_tensor_running_past_its_buffer_is_refused(): + with pytest.raises(wire.WireError, match="past the end"): + wire.pack_batch( + bufs=[wire.BufDesc(fd=7, size=64)], + tensors=[wire.TensorDesc(bi=0, offset=32, nbytes=64, dtype="fp16", + layout="row_major", ne=(32, 1, 1, 1))], + ops=[], + ) + + +def test_c_header_agrees_with_python_on_every_constant(): + """One source of truth, checked. A silent disagreement here is a wrong answer + on the DSP, not a compile error.""" + src = csource.code_only(DSP_H.read_text()) + assert "0x424C5848u" in src + assert "#define HEXLIB_MAX_BUFS 8" in src + assert "#define HEXLIB_MAX_SRC 6" in src + assert "#define HEXLIB_MAX_DST 4" in src + assert "#define HEXLIB_MAX_PARAMS 16" in src + assert "HEXLIB_DSP_OK = 1" in src + + +def test_every_status_name_is_bound_to_its_own_value_in_the_c_header(): + """NAME BOUND TO VALUE, not "the number appears somewhere". This assertion + used to be `assert f"= {val}" in src` for each value, which never bound a + value to a name: every integer 1..14 appears in the enum no matter how they + are permuted, so swapping ERR_UNMAPPED = 7 and ERR_NO_MMAP_SLOT = 8 in + hexlib_dsp.h left this file reporting 14 passed. Proven by mutation, twice + (that swap, and a four-way permutation). + + WHY THAT PERMUTATION IS NOT COSMETIC. 7 is the load-bearing unmapped-fd + discriminator -- the whole point of the staged gate is that a host address + crossing the wire is refused with ERR_UNMAPPED rather than silently working + under a shared address space. With the swap live, hexlib/exec/dsp.py:361-362 + reports every genuine INVAL_PARAMS as ERR_UNMAPPED, and + device/qdc/test_on_device.py's hardcoded `status 7` assertion passes on the + wrong condition. The only other tests that would notice + (test_dsp_sim.py:79, test_runtime_sim_build.py:352) are both @sdk-gated, so + CI never runs them: this assertion is the only unconditional guard there is. + + Scoped to the enum's own braces via csource, and comment-blanked, so a + number left behind in a comment cannot satisfy it either.""" + src = csource.code_only(DSP_H.read_text()) + enum_block = csource.block_from(src, src.index("enum hexlib_dsp_status")) + + for name, val in wire.STATUS.items(): + assert re.search(rf"\bHEXLIB_DSP_{name}\s*=\s*{val}\b", enum_block), ( + f"the C header does not bind HEXLIB_DSP_{name} to {val} " + f"(wire.py's STATUS says {val}). A permutation here is a wrong " + f"status NAME on the DSP, not a compile error:\n{enum_block}" + ) + + # And the other direction: a status added to the C enum but never taught to + # wire.py would make unpack_response() refuse a response the DSP considers + # legitimate ("status N is not a known status"). The loop above cannot see + # that, because it only iterates over what Python already knows. + in_header = { + m.group(1): int(m.group(2)) + for m in re.finditer(r"\bHEXLIB_DSP_(\w+)\s*=\s*(\d+)", enum_block) + } + assert in_header == wire.STATUS, ( + "the C enum and wire.py's STATUS are not the same mapping:\n" + f" only in the C header: {sorted(set(in_header) - set(wire.STATUS))}\n" + f" only in wire.py: {sorted(set(wire.STATUS) - set(in_header))}\n" + f" disagreeing values: " + f"{ {k: (in_header[k], wire.STATUS[k]) for k in set(in_header) & set(wire.STATUS) if in_header[k] != wire.STATUS[k]} }" + ) diff --git a/hexlib/tests/test_session_arch_decode.py b/hexlib/tests/test_session_arch_decode.py new file mode 100644 index 0000000..51ec7fb --- /dev/null +++ b/hexlib/tests/test_session_arch_decode.py @@ -0,0 +1,184 @@ +# hexlib/tests/test_session_arch_decode.py +"""BEHAVIOURAL test for the driver ARCH_VER BCD decode in session.c. + +WHY A SOURCE ASSERTION IS NOT ENOUGH. test_host_source.py can only check that +a comparison and a `return -1` exist in hexlib_open() -- it cannot see that +the two operands being compared are in incommensurable encodings. That is +exactly the shape of the original bug: `arch` (from hexlib_iface_hwinfo, +__HEXAGON_ARCH__, plain decimal, e.g. 75 -- see skel.c, test_dsp_sim.py's own +`info.arch == 75`) was compared directly against `caps.arch_ver` (from +DSPRPC_GET_DSP_INFO/ARCH_VER, a BCD-nibble-packed byte, e.g. 0x8c75 = 35957 +-- see device/qdc/test_on_device.py's own `35957` / `0x8c75`). 75 != 35957 +unconditionally, so every device session refused before measuring anything. +A test that only checks "a comparison and a failure path exist" cannot catch +that; a test that compiles and RUNS the actual decode can. + +WHAT THIS DOES. `hexlib_decode_bcd_arch()` is extracted straight out of +session.c with `csource.function_body` -- the SAME comment-aware slicer +every other source-assertion test in this project uses, never a second +hand-rolled copy (see csource.py's own module docstring for why that +matters) -- dropped into a tiny standalone .c file next to a one-line +harness that exports it under a stable name, compiled with a host C +compiler into a shared library, and called through ctypes with the ONE +measured value this project has on record (0x8c75, from +device/qdc/test_on_device.py) -- never a value invented for this test alone. + +THE DECISIVE PROPERTY. If the decode is ever removed -- e.g. reverted to +comparing the raw ARCH_VER against `arch` directly, the original bug -- +`hexlib_decode_bcd_arch()` no longer exists in session.c, and +`_function_body` raises before this test ever gets to compile or call +anything. If the decode is present but wrong, the compiled call below +returns something other than 75. Either way, this test fails; it does not +merely fail to notice. + +THE COMPILER-INDEPENDENT GUARD BELOW WAS ONCE FOOLABLE BY A COMMENT, AND IS +NOT ANY MORE -- SAY SO RATHER THAN LET IT BE REDISCOVERED. +`test_the_decode_function_the_behavioural_test_depends_on_still_exists` calls +itself compiler-independent, and it is: it runs with no `cc` on PATH, which on +such a machine makes it the ONLY guard on this fix. But when +`csource.function_body` returned the raw, comment-BEARING body, that guard +could be defeated exactly as easily as the source assertion in +test_host_source.py it exists to back up: reverting the body to `return +arch_ver;` and leaving `(val >> 4) * 10 + (val & 0x0f)` behind in a comment +INSIDE the body satisfied all three regexes below, and this test passed while +the three behavioural tests correctly failed -- so on a machine with no host C +compiler the regression really was invisible, which is the one thing this +test's docstring promised it could not be. `csource.function_body` now returns +comment-BLANKED text by default (see its module docstring), so the three +regexes below see only code. Verified by mutation, not by inspection: that +exact revert now fails this test. + +Adapted from llama.cpp's own htpdrv_get_arch (ggml-hexagon/htp-drv.cpp: +412-413, MIT; see ATTRIBUTION.md): `val = arch_ver & 0xff; arch = (val >> 4) +* 10 + (val & 0x0f)`. +""" +import ctypes +import pathlib +import re +import shutil +import subprocess +import sys + +import pytest + +from hexlib.tests.csource import function_body as _function_body + +SESSION_C = pathlib.Path("hexlib/runtime/host/session.c") + +HOST_CC = shutil.which("cc") or shutil.which("gcc") or shutil.which("clang") +needs_cc = pytest.mark.skipif( + HOST_CC is None, + reason=( + "no host C compiler found (tried: gcc, cc, clang); the BEHAVIOURAL " + "arch-decode test is skipped and only the weaker source assertion " + "in test_host_source.py (test_arch_is_queried_from_the_driver_not_" + "assumed) covers this. Install a host C compiler to restore it." + ), +) + + +def test_the_decode_function_the_behavioural_test_depends_on_still_exists( + decode_fn_source, +): + """COMPILER-INDEPENDENT -- runs even when needs_cc above would skip + every other test in this file, so a rename or removal of + hexlib_decode_bcd_arch() is never invisible on a machine with no host C + compiler on PATH. `decode_fn_source` itself already raises (failing this + test) if the function is gone; this test additionally pins that its body + still contains real BCD-decode arithmetic -- as CODE, not as a comment + left behind by whoever deleted it, which is a hole this test had until + `csource.function_body` began returning comment-blanked text (see this + module's docstring) -- not merely SOME function by that name that could + compile into anything. Pairs the exact NAME the + behavioural tests below extract with what session.c actually contains, + so the pair cannot silently drift apart -- see this module's docstring's + "decisive property" and the module docstring's WHY for the full + rationale.""" + assert re.search(r">>\s*4", decode_fn_source), "must extract the high BCD nibble" + assert re.search(r"\*\s*10", decode_fn_source), "must weight the high nibble by 10" + assert re.search(r"&\s*0x0f\b", decode_fn_source), "must extract the low BCD nibble" + + +@pytest.fixture(scope="module") +def decode_fn_source(): + """The real hexlib_decode_bcd_arch() BODY (from its opening `{` through + the matching `}` -- `csource.function_body`'s own documented slice; + it deliberately does not include the signature line, since every other + caller of this shared helper only ever inspects a body's control flow), + sliced straight out of session.c and never retyped here, so a change to + the real logic is exactly what this test exercises, not a + hand-maintained duplicate that could silently drift from it. Raises + (failing the test) if the function has been removed or renamed -- see + the module docstring's "decisive property".""" + src = SESSION_C.read_text() + return _function_body(src, "hexlib_decode_bcd_arch") + + +@pytest.fixture +def compiled_decode(decode_fn_source, tmp_path): + """Compile the extracted function into a shared library and return a + ctypes callable bound to it. The signature wrapped around the extracted + body below (`static uint32_t hexlib_decode_bcd_arch(uint32_t arch_ver)`) + is copied verbatim from session.c's own declaration -- see + hexlib_host.h/session.c -- only the BODY, the actual decode logic, comes + from the slice; nothing about the arithmetic is retyped here.""" + c_path = tmp_path / "decode.c" + c_path.write_text( + "#include \n" + "static uint32_t hexlib_decode_bcd_arch(uint32_t arch_ver)\n" + f"{decode_fn_source}\n" + "#if defined(_WIN32)\n" + "__declspec(dllexport)\n" + "#endif\n" + "uint32_t harness_decode(uint32_t arch_ver) {\n" + " return hexlib_decode_bcd_arch(arch_ver);\n" + "}\n" + ) + lib_path = tmp_path / ("decode.dll" if sys.platform == "win32" else "decode.so") + cmd = [HOST_CC, "-shared", "-fPIC", "-o", str(lib_path), str(c_path)] + result = subprocess.run(cmd, capture_output=True, text=True) + assert result.returncode == 0, ( + f"compiling the extracted hexlib_decode_bcd_arch() failed:\n" + f"{result.stdout}\n{result.stderr}" + ) + + lib = ctypes.CDLL(str(lib_path)) + lib.harness_decode.restype = ctypes.c_uint32 + lib.harness_decode.argtypes = [ctypes.c_uint32] + return lib.harness_decode + + +@needs_cc +def test_decode_of_the_measured_arch_ver_is_75(compiled_decode): + """0x8c75 is the exact ARCH_VER this project has measured on real + silicon (device/qdc/test_on_device.py); 75 is what __HEXAGON_ARCH__ + reports for the same part (skel.c / test_dsp_sim.py). This is the pair + that hexlib_open() must agree on for a device session to ever open.""" + assert compiled_decode(0x8C75) == 75, ( + "hexlib_decode_bcd_arch(0x8c75) must be 75 -- the exact " + "ARCH_VER/__HEXAGON_ARCH__ pair measured on real silicon" + ) + + +@needs_cc +def test_decode_is_a_real_bcd_decode_not_a_lookup_of_one_value(compiled_decode): + """A couple of adjacent points so a decode that merely happens to get + 0x8c75 right (e.g. a one-entry lookup table, or `& 0xff` alone without + the nibble split) cannot pass.""" + assert compiled_decode(0x8C73) == 73 + assert compiled_decode(0x0075) == 75 + # Only the LOW byte matters (0x1234 & 0xff == 0x34): high nibble 3, + # low nibble 4 -> 3*10 + 4 == 34. A lookup keyed on the whole 32-bit + # value, or one that used the high byte instead, would get this wrong. + assert compiled_decode(0x1234) == 34 + + +@needs_cc +def test_decode_does_not_return_the_raw_arch_ver(compiled_decode): + """Guards directly against the original defect: if the decode were + accidentally bypassed (the function body compiled but just returned its + input), this value would be 35957, not 75.""" + raw = 0x8C75 + decoded = compiled_decode(raw) + assert decoded != raw + assert decoded == 75 diff --git a/hexlib/tests/test_skel_bufs_source.py b/hexlib/tests/test_skel_bufs_source.py new file mode 100644 index 0000000..27dcaf8 --- /dev/null +++ b/hexlib/tests/test_skel_bufs_source.py @@ -0,0 +1,260 @@ +# hexlib/tests/test_skel_bufs_source.py +"""The pointer-free invariant, asserted against the source. + +These are source assertions, not behavioural ones — the behavioural test is +Task 8's unmapped-fd run on the simulator. They exist because the invariant is +easy to break in a way that PASSES on the simulator: host and DSP share one +address space there, so a skel that trusted the host's `base` would return the +right answer and only fail on silicon. Two independent guards, at two levels. + +TWICE REWRITTEN, BOTH TIMES FOR THE SAME REASON. A first version checked bare +substrings anywhere in the file; a second scoped some of them to a function +but still ran every check against comment-BEARING text. Both were defeated by +the same mutation: replace hexlib_bufs_map's unmapped-fd refusal with +`b->base = (uint64_t) b->fd; continue;` -- the shared-address-space bug this +whole file exists to catch -- and all eight tests still passed, because +`"HAP_mmap" in src` was satisfied by a comment, `_returns(src, ...)` was +whole-file and satisfied by an unrelated return in a different function, and +`"nbytes" in src` was satisfied by a FARF format string. So: + +A THIRD TIME, FOR THE SAME REASON AGAIN, AND THIS ONE IS THE POINT. The second +rewrite's third bullet below says every presence check is a call or an +assignment shape "so a mention in a log-message format string is not evidence +of anything" -- and it was still possible, because `csource` handed string +literals back untouched. `b->base = 0;` replaced by a FARF PRINTING THAT EXACT +TEXT, with the real `b->base = m->base;` deleted, kept this file at 8 passed: +the host's address survives into the descriptor and reaches a kernel, which is +the single invariant this file exists for. Separately, a `}` inside a FARF +truncated `find_by_fd`'s slice, so keying the lookup off the host's `base` (with +an `if (0)` decoy holding the by-fd comparison) also passed. Both are fixed in +`csource` -- literals are blanked now, and brace counting ignores them -- and +the checks below no longer rely on nobody having thought of it. + + * every fixture and every slice is COMMENT- AND LITERAL-BLANKED (csource + strips by default; `code_only` does the whole file), so nothing a mutation + leaves behind as a comment, a log line or a format string can satisfy + anything here; + * every check is scoped to the ONE function -- usually the one `if`-block -- + whose behaviour the claim is about, never the file; + * every presence check is a CALL or an ASSIGNMENT shape, never a bare token, + so a mention in a log-message format string is not evidence of anything. +""" +import pathlib +import re + +import pytest + +from hexlib.tests.csource import block_from as _block_from +from hexlib.tests.csource import calls as _calls +from hexlib.tests.csource import code_only as _code_only +from hexlib.tests.csource import function_body as _function_body + +SRC = pathlib.Path("hexlib/runtime/skel/skel_bufs.c") + + +@pytest.fixture(scope="module") +def src(): + """Comment-blanked, so every check below is about code. Blanking preserves + length, so `csource`'s offsets stay valid against this text.""" + return _code_only(SRC.read_text()) + + +def _returns(fragment, constant): + """A RETURN of the given status constant, inside `fragment` -- which must + be a function body or (better) the one `if`-block the claim is about. + NEVER pass the whole file: this was whole-file once, and an unrelated + `return HEXLIB_DSP_ERR_UNMAPPED;` in hexlib_bufs_unregister then stood in + for the one hexlib_bufs_map is supposed to have, which is how a mutation + that deleted the real one passed.""" + return re.search(rf"return\s+{re.escape(constant)}\s*;", fragment) is not None + + +def test_base_is_cleared_before_any_lookup(src): + """hexlib_bufs_map must destroy whatever base the host sent before it can + be read by the fd lookup. This is a behavioural claim about ONE function's + body, not about the file's layout — checking whole-file text order would + incidentally constrain where helpers like find_by_fd get defined, which is + not the invariant. Slice hexlib_bufs_map itself and check order there. + + AN ASSIGNMENT STATEMENT, NOT THE TEXT OF ONE. This was `"b->base = 0" in + body`, and `FARF(HIGH, "hexlib: b->base = 0 before any lookup");` satisfied + it while the real clear was gone -- 8 passed, with the host's address left + in the descriptor for the rest of the function to hand onward. The + statement terminator is part of the requirement for that reason. + + AND THE OTHER HALF OF THE INVARIANT, WHICH NOTHING USED TO CHECK: clearing + `base` is only half the job. The descriptor must then be filled in from the + DSP-side MAPPING (`m->base`, the table only hexlib_bufs_register writes), + after the lookup. With the clear demoted to a log line and this assignment + simply deleted, every check in this file still passed -- so both the destroy + and the re-fill are pinned here now, in that order.""" + body = _function_body(src, "hexlib_bufs_map") + clear = re.search(r"b->base\s*=\s*(?:0|NULL)\s*;", body) + assert clear, ( + "hexlib_bufs_map must destroy the host's `base` with a real assignment " + "statement before anything reads it -- a log line spelling out the " + "assignment is not one" + ) + lookup = re.search(r"\bfind_by_fd\s*\(", body) + assert lookup, "hexlib_bufs_map does not appear to look the buffer up at all" + assert clear.start() < lookup.start(), "clear base before looking the buffer up" + + fill = re.search(r"b->base\s*=\s*m->base\s*;", body) + assert fill, ( + "hexlib_bufs_map must set the descriptor's base FROM the DSP-side " + "mapping (b->base = m->base) -- without it the descriptor keeps " + "whatever the host wrote, which is the one thing this file exists to " + "prevent" + ) + assert lookup.start() < fill.start(), ( + "the base may only be filled in from a mapping the lookup actually " + "found, so it must come after the lookup" + ) + + +def test_lookup_is_by_fd(src): + """The match must be an actual `==` comparison against the slot's fd, + inside find_by_fd itself -- not the token `->fd ==` anywhere in the file, + which any unrelated comparison would satisfy.""" + body = _function_body(src, "find_by_fd") + assert re.search(r"->fd\s*==", body), ( + "find_by_fd must match slots by comparing the stored fd, never by " + "anything the host supplied as an address" + ) + assert not re.search(r"->base\s*==", body), ( + "find_by_fd must never key its lookup off `base` -- that is the host's " + "value and the whole point of this file is that it is not trusted" + ) + + +def test_the_dsp_maps_the_fd_itself(src): + """A REAL CALL, whose result is assigned -- not the token `HAP_mmap` + anywhere in the file. That check passed for a gutted hexlib_bufs_register + with both calls deleted, because skel_bufs.c's own comment about + `HAP_mmap`'s `len` argument spells the name in prose. + + Both spellings are required because both are live: HAP_mmap2 on + `__HVX_ARCH__ > 73` and HAP_mmap below it (see the file's own comment on + the `int` vs `size_t` length argument). Deleting either silently removes + the mapping on one arch.""" + body = _function_body(src, "hexlib_bufs_register") + assert re.search(r"=\s*HAP_mmap2\s*\(", body), ( + "hexlib_bufs_register must map the fd itself via HAP_mmap2 on the " + "v75+ branch, assigning the result -- not merely name it" + ) + assert re.search(r"=\s*HAP_mmap\s*\(", body), ( + "hexlib_bufs_register must map the fd itself via HAP_mmap on the " + "pre-v75 branch, assigning the result -- not merely name it" + ) + + +def test_an_unmapped_fd_is_an_error_not_a_zero_base(src): + """Upstream returns silently with base == 0 when no slot is free, and the + caller then computes 0 + offset and reads a small bogus address. Fixed. + + EACH STATUS IS PINNED TO THE BLOCK THAT MUST REPORT IT, not to the file. + This was three whole-file `_returns` calls, and hexlib_bufs_unregister's + own `return HEXLIB_DSP_ERR_UNMAPPED;` stood in for hexlib_bufs_map's -- + so replacing hexlib_bufs_map's refusal with "FARF and continue, trusting + the host's fd as an address" passed. That is precisely the + shared-address-space bug the simulator cannot see.""" + map_body = _function_body(src, "hexlib_bufs_map") + miss = re.search(r"if\s*\(\s*!\s*m\s*\)\s*\{", map_body) + assert miss, ( + "hexlib_bufs_map must branch on find_by_fd() having found nothing" + ) + miss_block = _block_from(map_body, miss.end() - 1) + assert _returns(miss_block, "HEXLIB_DSP_ERR_UNMAPPED"), ( + "an fd the DSP never mapped must be refused from inside that branch " + "-- logging and continuing (with or without an address derived from " + "the fd) is the upstream bug this file exists to have fixed" + ) + assert not re.search(r"\bcontinue\s*;", miss_block), ( + "the unmapped-fd branch must not continue the loop: the buffer would " + "be handed to a kernel with whatever base was left in it" + ) + # Exhaustive callee set, so the refusal cannot be routed through a helper + # that maps the fd on demand -- the thing this branch exists NOT to do, and + # the thing a scoped `X not in block` ban cannot see (csource.calls()). + assert _calls(miss_block) == {"FARF"}, ( + f"the unmapped-fd branch may log and return, and must call nothing " + f"else: mapping on demand here is the upstream bug, and on the " + f"simulator it would silently work; found {_calls(miss_block)!r}" + ) + + reg_body = _function_body(src, "hexlib_bufs_register") + assert _returns(reg_body, "HEXLIB_DSP_ERR_NO_MMAP_SLOT"), ( + "running out of mmap slots must be reported by hexlib_bufs_register " + "itself, not left as a silent base == 0 fallthrough" + ) + mmap_failed = re.search(r"if\s*\(\s*va\s*==", reg_body) + assert mmap_failed, "the result of the mapping call must be checked" + mmap_failed_block = _block_from(reg_body, mmap_failed.end()) + assert _returns(mmap_failed_block, "HEXLIB_DSP_ERR_MMAP_FAILED"), ( + "a failed mapping must be reported from inside its own check" + ) + + +def test_no_abort_on_a_failed_mapping(src): + """Upstream abort()s. Fail closed means returning a status, not killing the + process and leaving the host to interpret a dead session. Whole-file + negative, over code only -- a comment discussing upstream's abort() (this + file's header does) is not an abort().""" + assert "abort()" not in src + + +def test_tensor_data_is_computed_from_base_plus_offset(src): + """The kernel-visible address must be DERIVED, in one assignment, from the + mapped base and the tensor's own offset. `"base" in src and "offset" in + src` was satisfied by the file-header comment, and `"->data =" in src` by + the mutation `t->data = 0;` itself -- the gutting that check was supposed + to catch.""" + body = _function_body(src, "hexlib_tensors_resolve") + assert re.search(r"->data\s*=\s*[^;]*->base\s*\+\s*[^;]*->offset", body), ( + "the tensor's data address must be computed as the DSP-side mapped " + "base plus the tensor's offset, in that one assignment" + ) + + +def test_resolution_bounds_checks_the_offset(src): + """A tensor whose offset+nbytes exceeds its buffer must be refused on the + DSP too. The host checks it, but the host is not the thing being trusted. + + THE GUARD ITSELF, AND ITS OWN RETURN. `_returns(src, TRUNCATED)` was + whole-file and was satisfied by a different check's return; `"nbytes" in + src` was satisfied by the word `nbytes` inside a FARF format string. So a + version that dropped `nbytes` from the bound and deleted this return + passed.""" + body = _function_body(src, "hexlib_tensors_resolve") + guard = re.search(r"if\s*\([^;{]*->nbytes[^;{]*\)\s*\{", body) + assert guard, ( + "hexlib_tensors_resolve must have an `if` whose condition involves the " + "tensor's own nbytes -- a bound on offset alone is not a bound" + ) + cond = guard.group(0) + for token in ("->offset", "->nbytes", "->size", "+", ">"): + assert token in cond, ( + f"the bound must compare offset + nbytes against the buffer size; " + f"{token!r} is missing from {cond!r}" + ) + guard_block = _block_from(body, guard.end() - 1) + assert _returns(guard_block, "HEXLIB_DSP_ERR_TRUNCATED") or _returns( + guard_block, "HEXLIB_DSP_ERR_INVAL_PARAMS" + ), "the out-of-bounds case must return a status from inside its own block" + + +def test_buffer_index_is_range_checked(src): + """The out-of-range case must be a real comparison against the buffer + count, with its own return. `"n_bufs" in src` was satisfied by + hexlib_tensors_resolve's own PARAMETER NAME, so deleting the guard + entirely left this test passing.""" + body = _function_body(src, "hexlib_tensors_resolve") + guard = re.search(r"if\s*\([^;{]*->bi\s*>=\s*n_bufs\s*\)\s*\{", body) + assert guard, ( + "hexlib_tensors_resolve must refuse a tensor naming a buffer index " + "at or past n_bufs -- the parameter merely being named is not a check" + ) + guard_block = _block_from(body, guard.end() - 1) + assert _returns(guard_block, "HEXLIB_DSP_ERR_INVAL_PARAMS") or _returns( + guard_block, "HEXLIB_DSP_ERR_UNMAPPED" + ), "the out-of-range case must actually return an error, not just log one" diff --git a/hexlib/tests/test_skel_dispatch_source.py b/hexlib/tests/test_skel_dispatch_source.py new file mode 100644 index 0000000..4f86438 --- /dev/null +++ b/hexlib/tests/test_skel_dispatch_source.py @@ -0,0 +1,417 @@ +# hexlib/tests/test_skel_dispatch_source.py +"""The batch walk and the FastRPC entry points, asserted against the source. + +Source assertions, not behavioural ones -- the behavioural run is Task 8's. They +exist because the invariants here are easy to satisfy with a log line instead of +a real guard, and a log line passes on the simulator (host and DSP share an +address space there) right up until it fails on silicon. Every check below that +looks for a status constant requires it inside a `return`, an assignment to a +status field, or a genuine call to the shared header-writer -- never merely +"the token appears somewhere in the file", which a FARF-only downgrade would +still satisfy. + +Comments are blanked out of both fixtures before any check runs, in both +directions: a mutation cannot satisfy a positive check ("X must be assigned") +by demoting the assignment to a comment, and a mutation cannot trip a negative +check ("X must not appear here") merely by mentioning X in prose -- which +happened during development of this file (a comment in the invoke-before-start +refusal that named `hexlib_dispatch_batch` in prose briefly failed +test_invoke_before_start_is_refused for exactly that reason). + +AND SO ARE STRING LITERALS, AS OF THE MERGE-GATE REVIEW THAT FOUND THEM DOING +THE SAME JOB. `csource.code_only` used to hand literals back intact, so this +file's whole promise above was available in a second vehicle: the total_size +guard could log HEXLIB_DSP_ERR_TRUNCATED and fall through (12 passed), and a +`}` inside `FARF(HIGH, "pcycle }")` truncated `function_body`'s slice so the +"no raw register read here" negatives were answered by a fragment that stopped +at the literal (12 passed, with `__asm__("%0 = c15:14")` back in the wrapper). +Both are fixed in `csource`; see its docstring. + +FUNCTION SCOPE IS NOT REACHABILITY, WHICH IS THE THIRD THING THIS FILE HAD +WRONG. Moving the raw register read into a new `hexlib_raw_pcycle()` helper +that `hexlib_read_pcycle` calls left this file at 12 passed with no comment or +literal trick at all -- a scoped negative cannot see a rename. Every negative +here is now paired with either a whole-FILE ban (for a construct that must not +exist anywhere) or an exhaustive `csource.calls()` set (for one that must not +be REACHED from a particular block). + +THE SLICER IS SHARED, NOT COPIED. This file carried its own private +`_strip_comments`/`_function_body`/`_brace_block` -- the third copy of the +slicer `hexlib/tests/csource.py` was written to consolidate, and a WEAKER one: +its `_strip_comments` DELETED comment text rather than blanking it, so every +offset in the stripped text was shifted relative to the real file and no +offset could be reported back against the source. Migrated to `csource` +(`code_only` for the fixtures, `function_body`/`block_from` for the slices), +which keeps the same-length blanking property. See csource.py's own module +docstring for the full history, including the payload-check hole that stripping +comments only at the BOUNDARIES left open. +""" +import pathlib +import re + +import pytest + +from hexlib.tests.csource import block_from as _block_from +from hexlib.tests.csource import calls as _calls +from hexlib.tests.csource import code_only as _code_only +from hexlib.tests.csource import function_body as _function_body + +DISPATCH = pathlib.Path("hexlib/runtime/skel/skel_dispatch.c") +SKEL = pathlib.Path("hexlib/runtime/skel/skel.c") + + +@pytest.fixture(scope="module") +def d(): + return _code_only(DISPATCH.read_text()) + + +@pytest.fixture(scope="module") +def s(): + return _code_only(SKEL.read_text()) + + +def test_the_response_is_written_before_any_op_runs(d): + """The magic and a NON-OK status go down first, so a batch that dies + partway leaves a readable failure rather than a zero-filled buffer the + host would have to interpret. Scoped to hexlib_dispatch_batch's own body + -- not just file order -- so this cannot pass merely because the + header-writer helper happens to be defined above the dispatcher in the + file, regardless of what the dispatcher itself does first.""" + body = _function_body(d, "hexlib_dispatch_batch") + assert body.index("hexlib_write_rsp_hdr") < body.index("hexlib_kernel_table") + + +def test_status_is_never_left_as_zero(d): + """HEXLIB_DSP_OK is 1, not 0 (hexlib_dsp.h), so a response buffer that is + never written cannot read as success. The dispatcher's own default must + be an actual call passing HEXLIB_DSP_ERR_INTERNAL to the header-writer -- + not a comment or a FARF line naming the constant -- and it must happen + before the magic/version checks, so every early return after it still + leaves a specific status rather than reverting to a zeroed buffer.""" + m = re.search(r"hexlib_write_rsp_hdr\s*\([^;]*HEXLIB_DSP_ERR_INTERNAL", d) + assert m, "no call to hexlib_write_rsp_hdr passes HEXLIB_DSP_ERR_INTERNAL" + assert m.start() < d.index("HEXLIB_DSP_ERR_BAD_MAGIC") + + +def test_magic_and_version_are_checked_before_the_offsets_are_used(d): + body = _function_body(d, "hexlib_dispatch_batch") + assert body.index("HEXLIB_DSP_ERR_BAD_MAGIC") < body.index("off_bufs") + + +def test_total_size_is_checked_against_the_actual_length(d): + """Not just "the tokens appear somewhere" -- an actual `if` comparing + hdr.total_size to len, whose body reports HEXLIB_DSP_ERR_TRUNCATED.""" + body = _function_body(d, "hexlib_dispatch_batch") + m = re.search(r"if\s*\(\s*hdr\.total_size\s*!=\s*len\s*\)\s*\{", body) + assert m, "no guard comparing hdr.total_size against the actual length" + guard = _block_from(body, m.end() - 1) + # WAS `assert "HEXLIB_DSP_ERR_TRUNCATED" in guard`, which is the bare-token + # check this file's own docstring says it does not do: a guard that FARFs + # the constant's name and FALLS THROUGH satisfied it (12 passed), letting a + # batch whose declared size disagrees with its actual length go on to be + # walked. Both halves of reporting are now required, in the shapes the + # docstring promises -- the header-writer call the host reads its status + # from, and the return that stops the walk. + assert re.search( + r"hexlib_write_rsp_hdr\s*\([^;]*HEXLIB_DSP_ERR_TRUNCATED", guard + ), ( + "the length disagreement must be written into the response header the " + "host actually reads, not merely logged" + ) + assert re.search(r"return\s+HEXLIB_DSP_ERR_TRUNCATED\s*;", guard), ( + "and it must RETURN -- a guard that reports and then falls through " + "walks the batch anyway, which is the whole thing this check is for" + ) + + +def test_the_cycle_counter_is_read_through_the_sdks_own_api(d): + """NOT hand-rolled inline asm. `__asm__("%0 = c15:14")` only advances if + SYSCFG.PCYCLEEN is set, and a user-mode unsigned PD -- which is what the + skel runs in -- cannot set that bit. This project's own + include/hexlib/hexlib_harness.h sets it explicitly for the standalone-ELF + runtime, so the register reading 0 with the bit clear is a fact this repo + already records. The simulator measures a plausible four-figure number + either way and therefore cannot discriminate. + + So the read must go through HAP_perf_get_pcycles() + ($HEXAGON_SDK_ROOT/incs/HAP_perf.h), which issues the identical + instruction: same mechanism, but a documented SDK API rather than an + invented one, so a zero on silicon is a reportable platform fact about the + PD instead of an indistinguishable bug of ours. + + BOTH HALVES ARE ASSERTED, and both are scoped to the wrapper's own body + (comments and literals already blanked by `code_only`), so neither can be + satisfied by prose or by a log line: the SDK call must be PRESENT, and the + raw register read must be ABSENT. A revert to inline asm fails the second + half even if the first is left behind as dead code. + + AND THE NEGATIVE HALF IS ALSO ASSERTED AT FILE SCOPE, WHICH IS THE ONLY + SCOPE THAT MEANS ANYTHING FOR IT. A scoped negative asks "is the forbidden + construct in THIS function", and the answer is no as soon as it is moved + into a helper this function calls -- proven, and it needed no comment and no + string literal: `hexlib_raw_pcycle()` holding the `__asm__` while + `hexlib_read_pcycle` called it (with a dead `if (0)` branch keeping the + positive half green) left this file at 12 passed, with the wrapper reading a + register that cannot advance in the PD the skel actually runs in. The claim + was never really about this function: it is that NOTHING in this + translation unit reads the counter by hand. So it is checked that way, over + the whole comment- and literal-blanked file.""" + body = _function_body(d, "hexlib_read_pcycle") + assert re.search(r"\bHAP_perf_get_pcycles\s*\(\s*\)", body), ( + "hexlib_read_pcycle must read the counter through the SDK's own " + f"HAP_perf_get_pcycles(), got: {body!r}" + ) + assert "__asm__" not in body and "asm" not in body, ( + "the cycle counter must not be read by hand-rolled inline asm -- " + "SYSCFG.PCYCLEEN is unsettable from a user-mode unsigned PD, so a raw " + f"`c15:14` read may simply return 0 there: {body!r}" + ) + assert "c15:14" not in body and "C15:14" not in body, ( + f"no raw register read may survive in this wrapper: {body!r}" + ) + # THE SAME TWO BANS, AT FILE SCOPE. `d` is comment- AND literal-blanked, so + # this file's own header comment discussing the `__asm__("%0 = c15:14")` it + # replaced does not trip these, and neither would a FARF quoting it. + assert "c15:14" not in d and "C15:14" not in d, ( + "no raw c15:14 read may survive anywhere in skel_dispatch.c -- moving " + "it into a helper the wrapper calls is the same bug with a new name" + ) + assert "asm" not in d, ( + "no inline asm anywhere in skel_dispatch.c: the counter's one legal " + "read is HAP_perf_get_pcycles(), and a hand-rolled read one call level " + "away is still a hand-rolled read" + ) + # And the wrapper must be the ONLY thing that reads the counter, so the + # bracketing test below is measuring what it thinks it is. + assert _calls(body) == {"HAP_perf_get_pcycles"}, ( + f"hexlib_read_pcycle must call the SDK's reader and nothing else -- an " + f"extra callee here is where a hand-rolled read hides: {_calls(body)!r}" + ) + # And the include that makes it legal, in code rather than in a comment. + # (`#include "HAP_perf.h"` is a header-name, not a string literal, so + # `code_only` leaves it intact on purpose -- see csource.py.) + assert re.search(r'#\s*include\s+"HAP_perf\.h"', d), ( + "HAP_perf.h must actually be included, not merely referred to" + ) + + +def test_pcycle_brackets_only_the_kernel_call(d): + """Harness and RPC overhead is roughly constant, so including it + manufactures ratios out of nothing. Same counter and same placement as + hexlib.sim. Scoped to hexlib_dispatch_batch's own body: the read helper's + definition contains the literal text "hexlib_read_pcycle" too (it is the + function's own name), so an unscoped first/last index() over the whole + file would anchor "lo" on that definition instead of the first real call, + and everything from the definition onward -- including the tensor-resolve + call -- would land "between" the two markers, defeating the very check + this test exists to make. Slicing the dispatcher's body first removes the + definition from consideration entirely, and the `()` (no-arg call syntax, + vs. the definition's `(void)`) requirement in the pattern is a second, + independent guard against the same confusion.""" + # WAS `assert "c15:14" in d or "PCYCLE" in d` -- both halves were + # satisfiable by a comment before the fixtures were switched to + # `code_only`, and the first half pinned the hand-rolled asm this file now + # bans outright. The counter's provenance is + # test_the_cycle_counter_is_read_through_the_sdks_own_api's job above; this + # test is only about WHERE the pair of reads sits. + body = _function_body(d, "hexlib_dispatch_batch") + calls = [m.start() for m in re.finditer(r"hexlib_read_pcycle\s*\(\s*\)", body)] + assert len(calls) >= 2, "expected at least a before/after pair of calls" + lo, hi = calls[0], calls[-1] + between = body[lo:hi] + assert re.search(r"\bk->fn\s*\(", between), ( + "the kernel call must be inside the bracket -- and it must be the call " + "through the table's own function pointer, not merely the text `->fn(`" + ) + # THE EXHAUSTIVE CALLEE SET, NOT THREE NAMED BANS. The three below were + # named because they were the three things that existed when this was + # written; anything else that got moved between the reads -- including a + # one-line helper wrapping the resolve, which is exactly how the same + # escape was proven against the pcycle wrapper above -- would have gone + # unnoticed while inflating every measured cycle count on silicon. + assert _calls(between) == {"hexlib_read_pcycle", "fn"}, ( + f"only the opening pcycle read and the kernel call itself may sit " + f"inside the bracket; found {_calls(between)!r}" + ) + assert "hexlib_tensors_resolve" not in between, "resolution must be outside it" + assert "hexlib_bufs_map" not in between, "buffer mapping must be outside it" + assert "hexlib_write_rsp_hdr" not in between, "the response write must be outside it" + + +def test_an_unknown_kind_is_refused(d): + """Not just "the constant appears" -- the null-kernel-pointer guard must + itself assign HEXLIB_DSP_ERR_NO_KERNEL to the per-op result AND to the + batch-level status, which is what actually stops the batch and reports + the refusal on the wire rather than silently skipping the op.""" + body = _function_body(d, "hexlib_dispatch_batch") + m = re.search(r"if\s*\(\s*!\s*k\s*\)\s*\{", body) + assert m, "no null-kernel-pointer guard (`if (!k)`) found" + guard = _block_from(body, m.end() - 1) + assert re.search(r"results\[i\]\.status\s*=\s*HEXLIB_DSP_ERR_NO_KERNEL", guard) + assert re.search(r"batch_status\s*=\s*HEXLIB_DSP_ERR_NO_KERNEL", guard) + assert re.search(r"\bbreak\s*;", guard), ( + "an unknown kind must stop the batch, not continue it -- and it must be " + "an actual `break;` statement" + ) + + +def test_vtcm_reclaim_is_reported_not_ignored(d): + """The release callback in skel_vtcm.c only RECORDS a reclaim request + (`vtcm_needs_release = 1`); it deliberately does not release memory a + batch in flight is still using. This dispatcher is what must notice the + flag, stop at an op boundary, actually give the VTCM back (call + hexlib_vtcm_release), and report HEXLIB_DSP_ERR_VTCM_RECLAIMED as the + batch status -- not just mention the flag or the constant somewhere.""" + body = _function_body(d, "hexlib_dispatch_batch") + m = re.search(r"if\s*\(\s*ctx->vtcm_needs_release\s*\)\s*\{", body) + assert m, "no check of ctx->vtcm_needs_release inside the dispatcher" + guard = _block_from(body, m.end() - 1) + # A CALL STATEMENT ON THE SESSION CONTEXT, not the token. Deleting the call + # and leaving its name inside the FARF beside it -- `FARF(HIGH, "hexlib: + # hexlib_vtcm_release(ctx) deferred ...")` -- satisfied `"hexlib_vtcm_ + # release(" in guard` and left this file at 12 passed, with the batch + # stopping while still holding the reservation the competing session is + # blocked on. That is the whole failure this test is named for, and it is + # invisible to the simulator, where nothing else wants VTCM. + assert re.search(r"\bhexlib_vtcm_release\s*\(\s*ctx\s*\)\s*;", guard), ( + "the reclaim path must actually call hexlib_vtcm_release(ctx) -- " + "stopping the batch without giving the memory back leaves the " + "competing session waiting on a reservation nobody will release" + ) + assert re.search(r"batch_status\s*=\s*HEXLIB_DSP_ERR_VTCM_RECLAIMED", guard) + assert re.search(r"\bbreak\s*;", guard), ( + "must stop at the op boundary with an actual `break;`, not continue" + ) + # And nothing else happens in here: an exhaustive callee set, so the release + # cannot be swapped for a helper that only logs (see csource.calls()). + assert _calls(guard) == {"FARF", "hexlib_vtcm_release"}, ( + f"the reclaim path must log and release, and do nothing else at an op " + f"boundary; found {_calls(guard)!r}" + ) + + +def test_invoke_before_start_is_refused(s): + """An invoke before start must not run any op, and the response must + actually carry HEXLIB_DSP_ERR_NOT_STARTED as its status -- not merely a + FARF line naming the constant while the batch runs anyway. This is the + task's known loose end: a version that logs the constant and then + dispatches the batch regardless must fail this test.""" + body = _function_body(s, "hexlib_iface_invoke") + m = re.search(r"if\s*\(\s*!\s*ctx->started\s*\)\s*\{", body) + assert m, "hexlib_iface_invoke does not guard on ctx->started" + guard = _block_from(body, m.end() - 1) + assert re.search( + r"hexlib_write_rsp_hdr\s*\([^;]*HEXLIB_DSP_ERR_NOT_STARTED", guard + ), "the refusal must write NOT_STARTED into the response, not just log it" + assert "hexlib_dispatch_batch" not in guard, "invoke-before-start must not run any op" + # AND NOT VIA ANYTHING ELSE EITHER. The named ban above cannot see a + # one-line helper that dispatches; the exhaustive callee set can. + assert _calls(guard) == {"FARF", "hexlib_write_rsp_hdr"}, ( + f"the refusal may log and write the response header, and must call " + f"nothing else -- anything else is a path to running an op; found " + f"{_calls(guard)!r}" + ) + + +def test_both_wire_lengths_are_checked_for_a_negative_value(s): + """`batchLen` and `resultLen` are both `int` on the wire (qaic spells + `sequence` as a pointer plus a signed length), and only `resultLen` + was checked. A negative `batchLen` cast to uint32_t becomes an enormous + length, which PASSES hexlib_dispatch_batch's `len < sizeof(struct + hexlib_batch_hdr)` test -- so the dispatcher memcpy()s the full 40-byte + header out of `batch` before `hdr.total_size != len` can reject anything. + That is an out-of-bounds read of a buffer the host may have made much + shorter, and this entry point is the last place the sign is still visible: + after the cast the information is gone. + + Both guards must be inside hexlib_iface_invoke's own body and must + precede the cast, so this checks position as well as presence -- a check + added after the call to hexlib_dispatch_batch would protect nothing.""" + body = _function_body(s, "hexlib_iface_invoke") + dispatch_pos = body.index("hexlib_dispatch_batch") + for name in ("batchLen", "resultLen"): + m = re.search(rf"\b{name}\s*<\s*0\b", body) + assert m, ( + f"hexlib_iface_invoke does not reject a negative {name} -- cast to " + f"uint32_t it becomes a huge length that passes every subsequent " + f"size test" + ) + assert m.start() < dispatch_pos, ( + f"the negative-{name} guard must run BEFORE hexlib_dispatch_batch " + f"is called with the cast value, not after" + ) + # And the refusal must be reported, not merely detected: the batch-length + # path has a valid response buffer (resultLen was already checked above + # it), so it must write a real status the host can read off the wire. + m = re.search(r"if\s*\(\s*batchLen\s*<\s*0\s*\)\s*\{", body) + assert m, "the negative-batchLen guard must be its own `if` block" + guard = _block_from(body, m.end() - 1) + assert re.search(r"hexlib_write_rsp_hdr\s*\([^;]*HEXLIB_DSP_ERR_", guard), ( + "a negative batchLen must be reported in the response header, not " + "merely logged and dropped" + ) + assert "hexlib_dispatch_batch" not in guard, ( + "a negative batchLen must not reach the dispatcher at all" + ) + assert _calls(guard) == {"FARF", "hexlib_write_rsp_hdr"}, ( + f"same as the invoke-before-start refusal: log, write the header, call " + f"nothing else -- a helper that dispatches would satisfy the named ban " + f"above; found {_calls(guard)!r}" + ) + + +def test_hwinfo_reports_the_acquired_vtcm_size(s): + """The size on the wire must be READ OUT of the session context that + skel_vtcm.c filled in from HAP_compute_res, never a constant. + + `"vtcm_size" in s` was satisfied by the qaic-generated OUT-PARAMETER's own + name in hexlib_iface_hwinfo's signature, and the negative half banned only + one spelling of one constant -- so `*vtcm_size = (uint64)(8*1024*1024);` + passed both. The check is now the assignment itself: whatever + hexlib_iface_hwinfo writes through that pointer must be derived from + ctx->vtcm_size, which is the only value the acquisition path ever sets.""" + body = _function_body(s, "hexlib_iface_hwinfo") + m = re.search(r"\*\s*vtcm_size\s*=\s*([^;]+);", body) + assert m, "hexlib_iface_hwinfo must write something through *vtcm_size" + rhs = m.group(1) + assert "ctx->vtcm_size" in rhs, ( + f"hwinfo must report the ACQUIRED size (ctx->vtcm_size, set by " + f"skel_vtcm.c from HAP_compute_res), not `{rhs.strip()}` -- the part " + f"total is not the usable budget, in any spelling" + ) + assert "8388608" not in s, "hwinfo must report what was acquired, not a constant" + + +# Each qaic entry point, paired with the one thing it must actually DO. The +# entry points are thin by design -- they exist to delegate -- so the call each +# one delegates to IS its whole content, and a body that does not contain it is +# a stub regardless of what it returns. +_IFACE_DELEGATIONS = { + "hexlib_iface_open": r"\*\s*handle\s*=", + "hexlib_iface_close": r"\bhexlib_vtcm_free\s*\(", + "hexlib_iface_start": r"\bhexlib_vtcm_alloc\s*\(", + "hexlib_iface_stop": r"\bhexlib_vtcm_free\s*\(", + "hexlib_iface_mmap": r"\bhexlib_bufs_register\s*\(", + "hexlib_iface_munmap": r"\bhexlib_bufs_unregister\s*\(", + "hexlib_iface_hwinfo": r"__HEXAGON_ARCH__", + "hexlib_iface_invoke": r"\bhexlib_dispatch_batch\s*\(", +} + + +def test_skel_defines_the_iface_symbols_qaic_expects(s): + """Each symbol must be a real DEFINITION that does its own job -- not + merely a token present in the file. + + THIS WAS EIGHT `sym in s` CHECKS, AND EACH WAS SATISFIED BY THAT + FUNCTION'S OWN SIGNATURE. Gutting every body in skel.c to `return + AEE_SUCCESS;` passed all eight: the names were still there, on the empty + shells. `function_body` raising on a removed or renamed symbol covers the + presence half properly; the delegation table above covers the "and it + still does something" half, one required call per entry point.""" + for sym, required in _IFACE_DELEGATIONS.items(): + body = _function_body(s, sym) # raises if the definition is gone + assert re.search(required, body), ( + f"{sym}() is defined but does not {required!r} -- a FastRPC entry " + f"point that returns AEE_SUCCESS without delegating is a stub, and " + f"a stub reports success for work that never happened" + ) diff --git a/hexlib/tests/test_skel_vtcm_source.py b/hexlib/tests/test_skel_vtcm_source.py new file mode 100644 index 0000000..50a3865 --- /dev/null +++ b/hexlib/tests/test_skel_vtcm_source.py @@ -0,0 +1,234 @@ +"""VTCM acquisition. Source assertions; the behaviour is Task 8's hwinfo check. + +EVERY CHECK HERE RUNS AGAINST COMMENT-BLANKED, FUNCTION-SCOPED TEXT. Two +proven mutations got through the earlier version of this file. (1) Deleting +the HAP_compute_res_query_VTCM call outright and hardcoding +`vtcm_size = 4*1024*1024` passed, because the constant-ban below only banned +the 8 MiB spelling and the presence check was satisfied by the FARF string +that names the function in its error message. (2) Gutting `release_callback` +and moving `ctx->vtcm_needs_release = 1;` into `hexlib_vtcm_alloc` passed, +because `callback_body = src[:registered_at]` was not a body at all -- it was +the whole file up to the registration call, so anything defined above it +counted. Both are now scoped to the function whose behaviour is claimed. + +AND (3) -- FOUND LATER, AT THE MERGE GATE -- BOTH OF THOSE FIXES LEANED ON +FUNCTION SCOPE WHILE `csource` STILL HANDED STRING LITERALS BACK INTACT, so the +FARF vector described in (1) was never actually closed for anything: any +`X in body` check here remained satisfiable by a format string, and a `}` +inside one truncated any slice. Literals are blanked now (see csource.py). The +remaining hole of the same shape is that a scoped negative cannot see a +construct moved into a helper, so `release_callback`'s "must not release VTCM +itself" is stated as an exhaustive callee set rather than two named bans. +""" +import pathlib +import re + +import pytest + +from hexlib.tests.csource import block_after_call as _block_after_call +from hexlib.tests.csource import block_from as _block_from +from hexlib.tests.csource import calls as _calls +from hexlib.tests.csource import code_only as _code_only +from hexlib.tests.csource import function_body as _function_body + +SRC = pathlib.Path("hexlib/runtime/skel/skel_vtcm.c") +INTERNAL = pathlib.Path("hexlib/runtime/skel/skel_internal.h") + + +@pytest.fixture(scope="module") +def src(): + """Comment-blanked. Length-preserving, so csource's offsets stay valid.""" + return _code_only(SRC.read_text()) + + +@pytest.fixture(scope="module") +def internal(): + """skel_internal.h, comment-blanked -- the reclaim flag's DECLARATION is + part of the reclaim mechanism this file is about, and the one property that + cannot be checked from skel_vtcm.c alone.""" + return _code_only(INTERNAL.read_text()) + + +def test_the_reclaim_flag_is_declared_volatile(internal): + """CROSS-THREAD, AND IT WAS A PLAIN `int`. `release_callback` (skel_vtcm.c) + sets `ctx->vtcm_needs_release` from HAP_compute_res's own QuRT thread; + hexlib_dispatch_batch's per-op loop reads it. Nothing in the source made + the reader re-read memory -- it worked only because the loop body calls + `k->fn(&a)` through a function pointer the compiler cannot see into, which + forces a reload of anything that has escaped. An inlined or + constant-propagated kernel removes that accidental barrier and the load + can be hoisted out of the loop, at which point a mid-batch reclaim request + is never observed, the competing session waits on VTCM this one will not + return, and no source line looks any different. + + Checked against the comment-blanked header, so the word "volatile" + appearing in the explanatory comment beside it cannot satisfy this.""" + m = re.search(r"^\s*(.*?)\bvtcm_needs_release\s*;", internal, re.MULTILINE) + assert m, "skel_internal.h no longer declares vtcm_needs_release" + assert "volatile" in m.group(1), ( + f"vtcm_needs_release is written from release_callback's QuRT thread " + f"and read in the dispatch loop, so its declaration must be volatile; " + f"found `{m.group(0).strip()}`" + ) + + +def test_size_comes_from_the_runtime_never_a_constant(src): + """`STATE.md`: the part total is not the usable budget. VTCM is acquired at + session start, so the size must come from the runtime. + + THE OUT-PARAMETER, AND NOTHING ELSE, MAY SET THE REQUESTED SIZE. + `"HAP_compute_res_query_VTCM" in src` was satisfied by this file's own FARF + error string, and the literal bans below only covered 8 MiB -- so deleting + the call and writing `= 4*1024*1024` passed. The positive check now requires + the real call with that variable's address among its arguments, and the + negative check enumerates every assignment to it and allows only the `= 0` + initializer: any other constant, of any magnitude or spelling, fails. + + NAME-AGNOSTIC, deliberately. This used to hardcode the local as `vtcm_size`, + and broke when the fix for the "absolute requirement" bug split it into + `vtcm_total` and `vtcm_avail` -- a rename that satisfies the requirement + fully. A test that fails on a rename it should not care about is + over-specified: it makes the test dictate code layout, which is the defect, + not the code. So the name is now DERIVED from the size actually handed to + `set_vtcm_param_v2`, and the requirement is unchanged and applies to + whatever that variable is called.""" + alloc = _function_body(src, "hexlib_vtcm_alloc") + + param = re.search(r"HAP_compute_res_attr_set_vtcm_param_v2\s*\(([^()]*)\)", alloc) + assert param, "hexlib_vtcm_alloc must set the v2 VTCM params" + requested = [a.strip() for a in param.group(1).split(",")][1] + assert re.fullmatch(r"[A-Za-z_]\w*", requested), ( + f"the requested VTCM size must be a variable, not the expression " + f"{requested!r} -- a constant here is the bug this test exists for" + ) + + assert re.search( + rf"HAP_compute_res_query_VTCM\s*\([^;]*&\s*{re.escape(requested)}\b", alloc + ), ( + f"hexlib_vtcm_alloc must ask the runtime for `{requested}`, passing its " + f"address as an out-parameter -- naming the function in a log message is " + f"not asking it" + ) + + # `(?.])` so this sees the LOCAL, not `ctx->...`. + for m in re.finditer(rf"(?.]){re.escape(requested)}\s*=\s*([^;=]+);", alloc): + rhs = m.group(1).strip() + assert rhs == "0", ( + f"`{requested}` must only ever be set by the runtime query's " + f"out-parameter (the `= 0` initializer aside); found " + f"`{requested} = {rhs};`" + ) + # Belt: the 8 MiB part total, in both the decimal and hex forms the v75 + # spec and the address quote it in, must not appear anywhere in the code. + assert "8388608" not in src + assert "0x800000" not in src.lower() + + +def test_the_hardcoded_vtcm_address_appears_nowhere(src): + assert "0xd9000000" not in src.lower() + + +def test_a_release_callback_is_registered(src): + """A competing QNN-HTP or GGML-HTP session can reclaim VTCM mid-run. Not + registering the callback does not make that stop happening; it makes it + silent. + + THE CALLBACK'S OWN BODY, AND THE REGISTRATION THAT NAMES IT. This was + `src[:registered_at]` -- the whole file prefix, not a body -- so gutting + release_callback and hoisting the flag into hexlib_vtcm_alloc passed. + + The callback must flip the flag ON (not just mention the field, e.g. only + ever clearing it) and must not release VTCM itself: the batch in flight + may still be using the memory, so releasing is the dispatcher's job at an + op boundary (Task 6), not the callback's.""" + alloc = _function_body(src, "hexlib_vtcm_alloc") + assert re.search( + r"HAP_compute_res_attr_set_release_callback\s*\([^;]*\brelease_callback\b", + alloc, + ), ( + "hexlib_vtcm_alloc must register release_callback itself with the " + "compute-res attributes -- not merely name the setter" + ) + + callback_body = _function_body(src, "release_callback") + assert re.search(r"->vtcm_needs_release\s*=\s*1\s*;", callback_body), ( + "release_callback must record the reclaim request by setting " + "ctx->vtcm_needs_release = 1 -- if some other function sets it, the " + "reclaim request itself is being dropped on the floor" + ) + # THE CALLBACK CALLS NOTHING AT ALL, which is both what the C actually does + # and the only form of this claim a rename cannot dodge. The two named bans + # this replaces asked "does this body mention either release function", and + # the answer is no the moment the release moves into a helper the callback + # calls -- the same escape proven against skel_dispatch.c's pcycle wrapper. + # Releasing from here frees memory the batch in flight is still reading. + assert _calls(callback_body) == set(), ( + f"release_callback must only RECORD the request: it runs on " + f"HAP_compute_res's own QuRT thread while a batch may be mid-op, so " + f"anything it calls is a candidate for freeing memory a kernel is " + f"still using. Releasing is the dispatcher's job at an op boundary. " + f"Found calls to {_calls(callback_body)!r}" + ) + assert "HAP_compute_res_release(" not in callback_body + assert "HAP_compute_res_release_cached(" not in callback_body + + +def test_every_hap_failure_path_returns_a_status(src): + """Each `HAP_compute_res_*` call this file inspects the result of must + propagate a non-OK status to the caller when that check fails, not just + mention an error constant somewhere in the file -- the file-wide version + of this check would also pass for an implementation that calls every + HAP function, discards every return value, always returns + HEXLIB_DSP_OK, and happens to reference an error constant once in a dead + branch or a comment. 'Log and continue' -- FARF the failure and fall + through to `return HEXLIB_DSP_OK;` -- is exactly the regression class + this exists to catch; it is the same defect fixed in skel_bufs.c's + upstream (see ATTRIBUTION.md) and it would hand a kernel a null or + zero-length VTCM base pointer if reintroduced here.""" + alloc = _function_body(src, "hexlib_vtcm_alloc") + + query_block = _block_after_call(alloc, "HAP_compute_res_query_VTCM") + assert re.search(r"return\s+HEXLIB_DSP_ERR_\w+\s*;", query_block) + + acquire_block = _block_after_call(alloc, "HAP_compute_res_acquire") + assert re.search(r"return\s+HEXLIB_DSP_ERR_\w+\s*;", acquire_block) + + ptr_block = _block_after_call(alloc, "HAP_compute_res_attr_get_vtcm_ptr_v2") + assert re.search(r"return\s+HEXLIB_DSP_ERR_\w+\s*;", ptr_block) + + +def test_hmx_is_requested_only_when_the_session_asked_for_it(src): + """No kernel on this branch needs HMX, and `session.c` always passes + `n_hmx = 0` to `hexlib_iface_start`. Requesting HMX unconditionally here + risks `HAP_compute_res_acquire` refusing the WHOLE reservation for an + HMX-availability reason indistinguishable, from its single status code + alone, from a VTCM-size failure -- the operator would see a VTCM error + for what was actually an HMX one. `HAP_compute_res_attr_set_hmx_param` + must therefore be guarded by a real check of `ctx->n_hmx`, not called + unconditionally in `hexlib_vtcm_alloc`.""" + alloc = _function_body(src, "hexlib_vtcm_alloc") + guard = re.search(r"if\s*\(\s*ctx->n_hmx\s*>\s*0\s*\)", alloc) + assert guard, ( + "HAP_compute_res_attr_set_hmx_param must be guarded by ctx->n_hmx > 0" + ) + guarded_block = _block_from(alloc, guard.end()) + assert "HAP_compute_res_attr_set_hmx_param" in guarded_block, ( + "the HMX request itself must live inside the ctx->n_hmx > 0 guard, " + "not merely have an unrelated if-block near it" + ) + # And nowhere else in the function, unguarded -- a duplicate call outside + # the guard would defeat the whole point. + outside = alloc.replace(guarded_block, "", 1) + assert "HAP_compute_res_attr_set_hmx_param" not in outside + + +def test_no_abort_or_assert_anywhere_in_the_file(src): + """Fail closed means returning a status, not killing the process -- + upstream aborts on failure; we must not. This is a whole-file negative + check over CODE ONLY (this file's header discusses upstream's abort, and + a comment saying so is not an abort), and it legitimately passes for any + file that simply never spells abort()/assert() -- it does not by itself + prove a failure is detected or propagated. See + test_every_hap_failure_path_returns_a_status for that half.""" + assert "abort()" not in src + assert "assert(" not in src diff --git a/hexlib/tests/test_toolchain.py b/hexlib/tests/test_toolchain.py index 6228ad8..915a213 100644 --- a/hexlib/tests/test_toolchain.py +++ b/hexlib/tests/test_toolchain.py @@ -8,9 +8,42 @@ def test_base_flags_are_pinned(): assert tc.HVX_CFLAGS == [ "-mv75", "-mhvx", "-mhvx-length=128B", "-std=gnu11", "-O2", + "-Wall", "-Werror", ] +def test_warnings_are_errors_because_nothing_else_reads_them(): + """-Werror IS LOAD-BEARING, not tidiness. Every caller of `tc.run` decides + success from `rc != 0` alone (hexlib/runtime/build.py's compile loops, + hexlib/build.py, hexlib/exec/hexagon.py), so a warning is emitted and then + discarded. The diagnostic that matters is + -Wincompatible-pointer-types-discards-qualifiers: genentry.py emits each + kernel's DSP entry by ARGUMENT ORDER, and swapping the `const` input with + the mutable output is not a crash and not a bad status -- it is a plausible + wrong answer, and that warning is the only automatic notice of it. + + MEASURED on toolchain 19.0.04 with the real generated scale_fp16_entry.c: + with the two buffer casts swapped, `-Wall -Werror` -> rc=1 with that exact + diagnostic; the same file under the OLD flags -> rc=0 with the same text as + a warning. Removing either flag here restores that silence. + """ + assert "-Werror" in tc.HVX_CFLAGS + assert "-Wall" in tc.HVX_CFLAGS + # -Wextra and -Wpedantic were measured and rejected -- see toolchain.py's + # own comment for the numbers. Pinned so re-adding one is a deliberate act. + assert "-Wpedantic" not in tc.HVX_CFLAGS + assert "-Wextra" not in tc.HVX_CFLAGS + + +def test_the_hmx_flag_still_lands_after_the_warning_flags(): + """`cflags_for_caps` appends -mhmx, and `test_hmx_cap_adds_compiler_and_sim_ + flags` asserts it is LAST. Adding flags to HVX_CFLAGS must not have quietly + changed which flag that is.""" + flags = tc.cflags_for_caps(["hmx"]) + assert flags[-1] == "-mhmx" + assert flags[:-1] == tc.HVX_CFLAGS + + def test_compiler_is_the_c_driver(): """Kernels are GNU C. The vendored ggml-hexagon headers use the `asm` keyword and void* arithmetic, which are errors in C++; and every v6 expert diff --git a/hexlib/tests/test_vision_oracle.py b/hexlib/tests/test_vision_oracle.py index 0e12430..bbf5e87 100644 --- a/hexlib/tests/test_vision_oracle.py +++ b/hexlib/tests/test_vision_oracle.py @@ -8,10 +8,10 @@ import os import numpy as np -import pytest import hexlib.graph.opdefs # noqa: F401 from hexlib.graph import eager +from hexlib.models import loader from hexlib.models.vit import VitConfig, build_vision_encoder from hexlib.result import Err @@ -150,201 +150,26 @@ def test_a_wrong_epsilon_would_be_caught(): assert not np.allclose(out[out_name], z["expected_merged"], rtol=1e-4, atol=1e-5) -def _vision_bilinear_indices_and_weights( - grid: int, num_grid_per_side: int, merge: int -) -> tuple[np.ndarray, np.ndarray]: - """Numpy port of `get_vision_bilinear_indices_and_weights`. - - transformers/vision_utils.py:147-217 (single image, grid_thw = [[1, grid, - grid]], so the `t` loop there runs once and `.repeat(t)` is a no-op). - Read, not reinvented: the four-corner bilinear indices/weights over the - learned `side x side` grid, then reordered into merge-block token order - (the same order `patchify` and `get_vision_position_ids` use) by - `vision_utils.py:207-209`. - """ - side = num_grid_per_side - h = w = grid - - h_grid = np.linspace(0, side - 1, h) - w_grid = np.linspace(0, side - 1, w) - h_floor = h_grid.astype(np.int64) - w_floor = w_grid.astype(np.int64) - h_ceil = np.minimum(h_floor + 1, side - 1) - w_ceil = np.minimum(w_floor + 1, side - 1) - h_frac = h_grid - h_floor - w_frac = w_grid - w_floor - - h_floor_offset = h_floor * side - h_ceil_offset = h_ceil * side - - corner_indices = [ - (h_floor_offset[:, None] + w_floor[None, :]).flatten(), - (h_floor_offset[:, None] + w_ceil[None, :]).flatten(), - (h_ceil_offset[:, None] + w_floor[None, :]).flatten(), - (h_ceil_offset[:, None] + w_ceil[None, :]).flatten(), - ] - corner_weights = [ - ((1 - h_frac)[:, None] * (1 - w_frac)[None, :]).flatten(), - ((1 - h_frac)[:, None] * w_frac[None, :]).flatten(), - (h_frac[:, None] * (1 - w_frac)[None, :]).flatten(), - (h_frac[:, None] * w_frac[None, :]).flatten(), - ] - - h_idx = np.arange(h).reshape(h // merge, merge) - w_idx = np.arange(w).reshape(w // merge, merge) - base = h_idx[:, :, None, None] * w + w_idx[None, None, :, :] - reorder = base.transpose(0, 2, 1, 3).flatten() - - indices = np.stack([c[reorder] for c in corner_indices]) - weights = np.stack([c[reorder] for c in corner_weights]) - return indices, weights - - -def _vision_position_ids(grid: int, merge: int) -> np.ndarray: - """Numpy port of `get_vision_position_ids`, transformers/vision_utils.py:76-81. - - Single image, grid_thw = [[1, grid, grid]] -- the `t` loop runs once and - `.repeat(t)` is a no-op. - """ - h = w = grid - hpos_ids = np.broadcast_to(np.arange(h)[:, None], (h, w)) - hpos_ids = hpos_ids.reshape(h // merge, merge, w // merge, merge).transpose(0, 2, 1, 3).flatten() - - wpos_ids = np.broadcast_to(np.arange(w)[None, :], (h, w)) - wpos_ids = wpos_ids.reshape(h // merge, merge, w // merge, merge).transpose(0, 2, 1, 3).flatten() - - return np.stack([hpos_ids, wpos_ids], axis=-1) - - -def _rope_tables(cfg: VitConfig) -> tuple[np.ndarray, np.ndarray]: - """rope_cos, rope_sin -- modeling_qwen3_5.py:84-92 (inv_freq, rotary forward) - and :1096-1108 (position ids -> freqs -> cos/sin), reproduced on the host. - - `Qwen3_5VisionRotaryEmbedding(head_dim // 2)`: inv_freq has - `dim = head_dim // 2` elements halved again by the `arange(0, dim, 2)` - step, so `inv_freq` has `head_dim // 4` entries. Each token's (row, col) - position id is multiplied by inv_freq and flattened row-major (row block, - then col block) to `[n, head_dim // 2]`, then duplicated by - `cat((freqs, freqs), dim=-1)` to `[n, head_dim]` before cos/sin. - """ - d = cfg.head_dim - dim = d // 2 - inv_freq = 1.0 / (cfg.rope_theta ** (np.arange(0, dim, 2, dtype=np.float64) / dim)) - - position_ids = _vision_position_ids(cfg.grid, cfg.spatial_merge_size).astype(np.float64) - freqs = position_ids[:, :, None] * inv_freq[None, None, :] # [n, 2, dim//2] - n = position_ids.shape[0] - flat = freqs.reshape(n, -1) # [n, head_dim//2] - emb = np.concatenate([flat, flat], axis=-1) # [n, head_dim] - return np.cos(emb).astype(np.float32), np.sin(emb).astype(np.float32) - - -def _pos_embed_table(cfg: VitConfig, table: np.ndarray) -> np.ndarray: - """pos_embed -- modeling_qwen3_5.py:1090, :1100 and - `get_vision_bilinear_indices_and_weights` (transformers/vision_utils.py:147-217), - reproduced on the host: bilinearly resample the learned - `num_grid_per_side x num_grid_per_side` grid onto this config's patch grid, - in merge-block token order. - - `num_grid_per_side` is derived from `table.shape[0]` (the golden's own - `pos_embed.weight` row count), not hardcoded, exactly as - `self.num_grid_per_side = int(config.num_position_embeddings**0.5)` - (modeling_qwen3_5.py:1039) computes it from the config. A hardcoded - constant here would let a future regeneration with a different learned - grid size leave `side` silently wrong while `table`'s shape still passes - every shape check -- shape-correct but numerically wrong, the exact - failure class this oracle otherwise eliminates. - """ - side = int(table.shape[0] ** 0.5) - if side * side != table.shape[0]: - raise ValueError( - f"pos_embed table has {table.shape[0]} rows, which is not a perfect " - "square; num_grid_per_side is undefined" - ) - indices, weights = _vision_bilinear_indices_and_weights(cfg.grid, side, cfg.spatial_merge_size) - out = np.zeros((indices.shape[1], table.shape[1]), dtype=np.float64) - for i in range(4): - out += table[indices[i]].astype(np.float64) * weights[i][:, None] - return out.astype(np.float32) +# THE MAPPING NOW LIVES IN hexlib/models/loader.py. It was written here, but +# every part of it -- the Conv3d flatten, the fused-qkv split, each transpose, +# the two folded tables -- is exactly what a real checkpoint needs, and a second +# copy written for the checkpoint would be a second chance to get a transpose +# wrong. These aliases keep this file's tests reading as they did; the code they +# call is the same code the checkpoint loader calls, which is the point. +_vision_bilinear_indices_and_weights = loader.vision_bilinear_indices_and_weights +_vision_position_ids = loader.vision_position_ids +_rope_tables = loader.rope_tables +_pos_embed_table = loader.pos_embed_table def _feeds_from_golden(graph, z) -> dict[str, np.ndarray]: - """Map the upstream parameter names onto hexlib's tensor names. - - Returns a feed for the graph's image input and for every const tensor. - hexlib's graph starts from the IMAGE and patchifies it itself, so the feed - is z["image"] -- not z["patches"]. That is deliberate: it puts patchify - inside the differential rather than beside it. + """The golden npz, through the library's mapping. - Every Linear weight is TRANSPOSED (torch stores [out, in]; hexlib stores - [k, n]). The fused qkv weight [3H, H] is split into three [H, H] blocks in - q, k, v order and transposed -- modeling_qwen3_5.py:929 reshapes the - linear's output to (seq, 3, heads, -1) and unbinds along the `3` axis, so - contiguous output-row blocks [0:H], [H:2H], [2H:3H] are q, k, v - respectively; the same split applies to the weight's rows and the bias. - pos_embed and rope_cos/sin are computed here, on the host, exactly as - modeling_qwen3_5.py:1090-1108 does -- that is the point of folding them. - """ - p = lambda name: z[f"param::{name}"] - H = TINY_CFG.hidden_size - - feeds: dict[str, np.ndarray] = {} + `image` is a graph INPUT rather than a const, so it is supplied here and not + by `feeds_from_params`, which knows only about weights.""" + feeds = loader.feeds_from_params(graph, TINY_CFG, + lambda name: z[f"param::{name}"]) feeds["image"] = z["image"] + return feeds - # Patch embedding: Conv3d weight [embed, C, T, ph, pw] flattens (row-major) - # to [embed, C*T*ph*pw] -- exactly patchify's feature order -- then - # transposes to hexlib's [feat, embed]. - conv_w = p("patch_embed.proj.weight") - feeds["w_patch_embed"] = conv_w.reshape(conv_w.shape[0], -1).T - feeds["b_patch_embed"] = p("patch_embed.proj.bias") - - feeds["pos_embed"] = _pos_embed_table(TINY_CFG, p("pos_embed.weight")) - cos, sin = _rope_tables(TINY_CFG) - feeds["rope_cos"] = cos - feeds["rope_sin"] = sin - - for layer in range(TINY_CFG.depth): - b = f"blocks.{layer}" - blk = f"blk{layer}" - - qkv_w = p(f"{b}.attn.qkv.weight") # [3H, H] - qkv_b = p(f"{b}.attn.qkv.bias") # [3H] - for i, which in enumerate(("q", "k", "v")): - feeds[f"{blk}.w{which}"] = qkv_w[i * H:(i + 1) * H, :].T - feeds[f"{blk}.b{which}"] = qkv_b[i * H:(i + 1) * H] - - feeds[f"{blk}.wo"] = p(f"{b}.attn.proj.weight").T - feeds[f"{blk}.bo"] = p(f"{b}.attn.proj.bias") - - feeds[f"{blk}.w_fc1"] = p(f"{b}.mlp.linear_fc1.weight").T - feeds[f"{blk}.b_fc1"] = p(f"{b}.mlp.linear_fc1.bias") - feeds[f"{blk}.w_fc2"] = p(f"{b}.mlp.linear_fc2.weight").T - feeds[f"{blk}.b_fc2"] = p(f"{b}.mlp.linear_fc2.bias") - - feeds[f"{blk}.ln1_w"] = p(f"{b}.norm1.weight") - feeds[f"{blk}.ln1_b"] = p(f"{b}.norm1.bias") - feeds[f"{blk}.ln2_w"] = p(f"{b}.norm2.weight") - feeds[f"{blk}.ln2_b"] = p(f"{b}.norm2.bias") - - feeds["m_ln_w"] = p("merger.norm.weight") - feeds["m_ln_b"] = p("merger.norm.bias") - feeds["w_m1"] = p("merger.linear_fc1.weight").T - feeds["b_m1"] = p("merger.linear_fc1.bias") - feeds["w_m2"] = p("merger.linear_fc2.weight").T - feeds["b_m2"] = p("merger.linear_fc2.bias") - - const_names = {t.name for t in graph.tensors.values() if t.const} - missing = const_names - set(feeds) - assert not missing, f"_feeds_from_golden supplied no feed for: {sorted(missing)}" - extra = set(feeds) - const_names - set(graph.inputs) - assert not extra, f"_feeds_from_golden supplied a feed for non-const tensors: {sorted(extra)}" - - for name, array in feeds.items(): - want_shape = graph.tensor(name).shape - got_shape = tuple(np.asarray(array).shape) - assert got_shape == want_shape, ( - f"feed {name!r} has shape {got_shape}, but the graph declares {want_shape} " - "-- a shape mismatch here is almost always a forgotten transpose" - ) - return feeds diff --git a/hexlib/tests/test_vtcm_contention.py b/hexlib/tests/test_vtcm_contention.py new file mode 100644 index 0000000..3debd99 --- /dev/null +++ b/hexlib/tests/test_vtcm_contention.py @@ -0,0 +1,389 @@ +"""VTCM acquisition under contention, and the status that survives the RPC boundary. + +TWO THINGS THE SIMULATOR CANNOT TELL US, so they are tested here instead. + +The skel used to ask for the part's ENTIRE VTCM with `min_vtcm_size = 0`, which +`HAP_compute_res.h:544-546` defines as "the size is an absolute requirement". +On a shared CDSP one other client holding a single 4 KB page therefore made +`HAP_compute_res_acquire` fail, `hexlib_iface_start` fail, and every mode exit at +session open. The simulator cannot reproduce it -- nothing else there holds VTCM, +which is exactly why stage 1 was green while this was live -- so what is checked +here is the SHAPE of the request (both queried sizes used, the floor derived from +`avail` rather than a constant), plus the arithmetic of the status encoding, +COMPILED AND RUN rather than pattern-matched. + +The source assertions use csource so they are comment-blind and function-scoped: +a claim satisfied by a comment is the defect this project keeps rediscovering. +""" +import re +import shutil +import subprocess + +import pytest + +from hexlib.tests import csource + +# Same discovery order as test_session_arch_decode.py and +# test_coherency_lane_classification.py, so a machine without a host compiler +# skips the compiled checks uniformly rather than in three different ways. +HOST_CC = shutil.which("cc") or shutil.which("gcc") or shutil.which("clang") +needs_cc = pytest.mark.skipif( + HOST_CC is None, + reason=( + "no host C compiler found (tried: cc, gcc, clang); the BEHAVIOURAL " + "status-encoding checks need one. The source-shape checks above do not " + "and still run." + ), +) + +VTCM_C = "hexlib/runtime/skel/skel_vtcm.c" +SKEL_C = "hexlib/runtime/skel/skel.c" +DSP_H = "hexlib/runtime/skel/hexlib_dsp.h" + + +def _src(path): + with open(path, encoding="utf-8") as f: + return f.read() + + +def _alloc_body(): + return csource.function_body(_src(VTCM_C), "hexlib_vtcm_alloc") + + +# -------------------------------------------------------------------------- +# The request shape +# -------------------------------------------------------------------------- + +def test_the_query_asks_for_the_available_size_not_only_the_total(): + """`avail_block_size` is the 4th out-parameter and it used to be 0. + + Fails if anyone reverts to `HAP_compute_res_query_VTCM(0, &x, 0, 0, 0)`, + which is what made the total the only number the skel knew. + """ + body = _alloc_body() + m = re.search(r"HAP_compute_res_query_VTCM\s*\(([^()]*)\)", body) + assert m, "hexlib_vtcm_alloc must query VTCM sizes" + args = [a.strip() for a in m.group(1).split(",")] + assert len(args) == 5, f"expected 5 arguments, got {args}" + assert args[3] != "0", ( + "the 4th argument is avail_block_size -- the SDK's 'largest contiguous " + "memory chunk available'. Passing 0 discards it, which is what made the " + "skel demand the part total as an absolute requirement" + ) + assert args[3].startswith("&"), f"avail must be an out-parameter, got {args[3]}" + + +def _query_out_params(body): + """The names (without `&`) of HAP_compute_res_query_VTCM's total and avail + out-parameters, read off the real call. Derived rather than hardcoded: the + fix for the "absolute requirement" bug split one local into two, and a test + that names them dictates code layout instead of checking behaviour (see + test_skel_vtcm_source.py's note on the same rename).""" + m = re.search(r"HAP_compute_res_query_VTCM\s*\(([^()]*)\)", body) + assert m, "hexlib_vtcm_alloc must query VTCM sizes" + args = [a.strip() for a in m.group(1).split(",")] + assert len(args) == 5, f"expected 5 arguments, got {args}" + # Signature (HAP_compute_res.h:1087-1106): (application_id, + # total_block_size, total_block_layout, avail_block_size, + # avail_block_layout). + return args[1].lstrip("&").strip(), args[3].lstrip("&").strip() + + +def test_the_min_vtcm_size_floor_is_below_the_request_and_comes_from_the_query(): + """`min_vtcm_size = 0` means "absolute requirement" -- the bug. + + THIS PINNED THE BUG BY ARGUMENT SPELLING, AND THE BUG'S SEMANTIC EQUIVALENT + PASSED. The old form asserted only that the floor was not the literal `0` + and not a numeric constant. `min_vtcm_size = vtcm_total` satisfies both and + re-demands the WHOLE partition: it is the original defect restored, since a + floor equal to the request means any contention at all fails + HAP_compute_res_acquire, which fails hexlib_iface_start, which exits every + mode at session open. The identifier differing from `0` was never the + requirement. + + THE REQUIREMENT, STATED AS TWO RELATIONS INSTEAD OF ONE SPELLING. Ask for + the total the runtime reported, and accept down to the AVAIL the runtime + reported -- so the floor is (a) derived from the query's availability + out-parameter, and (b) a different quantity from the request, which is what + makes it a floor at all. `avail <= total` is the SDK's own guarantee about + those two out-parameters ("largest contiguous memory chunk available" vs the + partition total), so pinning WHICH out-parameter each argument is pins the + inequality without this test having to know either number. + + Name-agnostic in both directions: both names are read off the query call, so + a rename that keeps the semantics passes and a swap that keeps the names + fails.""" + body = _alloc_body() + total, avail = _query_out_params(body) + assert total != avail, ( + f"the total and available sizes must be two distinct out-parameters -- " + f"HAP_compute_res_query_VTCM was passed {total!r} for both, so there is " + f"no availability figure for the floor to come from" + ) + + m = re.search(r"HAP_compute_res_attr_set_vtcm_param_v2\s*\(([^()]*)\)", body) + assert m, "must set the v2 VTCM params" + args = [a.strip() for a in m.group(1).split(",")] + assert len(args) == 4, f"expected 4 arguments, got {args}" + requested, floor = args[1], args[3] + + assert requested == total, ( + f"the request (total_block_size) must be the total the runtime just " + f"reported ({total!r}), not {requested!r} -- asking for `avail` directly " + f"caps the session at a value that can go stale between query and " + f"acquire" + ) + assert floor == avail, ( + f"min_vtcm_size must be the runtime's own AVAILABLE size ({avail!r}), " + f"not {floor!r}. 0 is 'the size is an absolute requirement' " + f"(HAP_compute_res.h:544-546); {total!r} is the same thing spelled " + f"differently, since a floor equal to the request refuses any " + f"contention at all; a constant would violate this file's governing " + f"rule that the size comes from the runtime" + ) + assert floor != requested, ( + "the floor must be BELOW the request, not equal to it -- a floor equal " + "to the request is the absolute-requirement bug with a variable name on " + "it" + ) + + +def test_the_reservation_is_actually_acquired_and_its_results_stored(): + """A REQUEST SHAPE IS NOT AN ACQUISITION. Every check above reads arguments + off two `HAP_compute_res_attr_set_*` calls, and attribute setters acquire + nothing: replacing the rest of hexlib_vtcm_alloc with `return + HEXLIB_DSP_OK;` -- so the session never holds VTCM and every kernel gets a + null base with size 0 -- left this whole file green. The floor being right + is only interesting if the request built from it is submitted, checked, and + its results recorded on the session. + + Everything here is derived from the calls themselves rather than named, for + the same reason as the floor check above.""" + body = _alloc_body() + m = re.search(r"HAP_compute_res_attr_set_vtcm_param_v2\s*\(([^()]*)\)", body) + assert m, "must set the v2 VTCM params" + attr = [a.strip() for a in m.group(1).split(",")][0].lstrip("&").strip() + + acquire = re.search( + rf"(\w+)\s*=\s*HAP_compute_res_acquire\s*\(\s*&\s*{re.escape(attr)}\b", body + ) + assert acquire, ( + f"hexlib_vtcm_alloc must submit the attributes it just built " + f"(&{attr}) to HAP_compute_res_acquire and keep the result -- a " + f"discarded reservation context cannot be released or re-acquired later" + ) + rctx = acquire.group(1) + assert re.search(rf"if\s*\(\s*!\s*{re.escape(rctx)}\s*\)", body), ( + f"a failed acquire returns 0, so `{rctx}` must be checked for it -- " + f"HAP_compute_res_acquire burns its full timeout before failing and " + f"then every kernel would run with no VTCM at all" + ) + + ptr_query = re.search( + r"HAP_compute_res_attr_get_vtcm_ptr_v2\s*\(([^()]*)\)", body + ) + assert ptr_query, "the acquired VTCM's base and size must be read back" + ptr_args = [a.strip().lstrip("&").strip() for a in ptr_query.group(1).split(",")] + assert len(ptr_args) == 3, f"expected 3 arguments, got {ptr_args}" + got_ptr, got_size = ptr_args[1], ptr_args[2] + + for field, var, why in ( + ("vtcm_base", got_ptr, "no kernel can use VTCM it has no pointer to"), + ("vtcm_size", got_size, "hwinfo reports this number to the host, and " + "the M1 allocator's budget is it"), + ("vtcm_rctx", rctx, "without the reservation context the dispatcher " + "cannot release VTCM at an op boundary, which is " + "what a competing session waits on"), + ): + assert re.search( + rf"ctx->{field}\s*=\s*(?:\([^;)]*\)\s*)?{re.escape(var)}\s*;", body + ), ( + f"ctx->{field} must be set from `{var}` -- {why}" + ) + + +def test_a_fully_contended_partition_is_refused_with_its_own_status(): + """avail == 0 is distinguishable from a query failure.""" + body = _alloc_body() + zero_check = re.search(r"if\s*\(\s*\w*avail\w*\s*==\s*0\s*\)", body) + assert zero_check, "a fully contended partition (avail == 0) must be refused" + guarded = csource.block_from(body, zero_check.start()) + # A RETURN, not the token. `"X" in guarded` was satisfiable by a FARF + # naming the constant while the function carried on to acquire a + # reservation it had just proven impossible -- the file-wide shape of + # defect this whole area was reviewed for. (csource blanks literals now, so + # the FARF vector is closed at the source; requiring the return closes the + # "assign it to an unused local" one too.) + assert re.search(r"return\s+HEXLIB_DSP_ERR_VTCM_TOO_SMALL\s*;", guarded), ( + "refusing with a specific status is what lets a device operator tell " + "contention from a load failure -- and it must be RETURNED from inside " + "this branch, not merely named in it" + ) + + +# -------------------------------------------------------------------------- +# The simulator/device asymmetry, pinned so it stays a decision +# -------------------------------------------------------------------------- + +SIMHOST_C = "hexlib/runtime/simhost/simhost.c" +SESSION_C = "hexlib/runtime/host/session.c" + + +def _start_args(path, _fn=None): + """The argument list of the hexlib_iface_start CALL in `path`. + + Whole-file, comment-blanked: the call sites are in different functions on + the two sides and the argument list contains a cast (`(uint64) MAX_BLOB`), + so this anchors on the `);` that ends the statement rather than on the + first close paren. + """ + src = csource.code_only(_src(path)) + m = re.search(r"hexlib_iface_start\s*\((.*?)\)\s*;", src, re.DOTALL) + assert m, f"{path} must call hexlib_iface_start" + return [a.strip() for a in m.group(1).split(",")] + + +def test_the_simulator_requests_hmx_and_the_device_host_does_not(): + """THE GATE EXERCISES A DIFFERENT ACQUISITION PATH THAN PRODUCTION. + + `skel_vtcm.c` only calls `HAP_compute_res_attr_set_hmx_param` when + `ctx->n_hmx > 0`, and n_hmx is whatever start() was passed. simhost passes + 1, the device host passes 0 -- so stage 1 acquires VTCM *with* an HMX + request and a device never has. That is a real difference in the one call + most likely to fail first on unfamiliar silicon, and it was pinned by + nothing. + + This test does not judge which is right. It fails if either side changes + silently, so the asymmetry stays a recorded decision. The trigger to + revisit is the first HMX kernel, which is what skel_vtcm.c also says. + """ + sim = _start_args(SIMHOST_C, "main") + assert len(sim) == 5, f"unexpected simhost start signature: {sim}" + assert sim[3] == "1", ( + f"simhost passes n_hmx={sim[3]}; this test and skel.c's hwinfo comment " + "both record it as 1. If you changed it, the asymmetry note needs updating" + ) + + dev = _start_args(SESSION_C, "hexlib_open") + n_hmx = dev[3] + assert re.fullmatch(r"/\*\s*n_hmx\s*\*/\s*0|0", n_hmx), ( + f"the device host passes n_hmx={n_hmx!r}; it was 0, meaning the device " + "never requests HMX. Changing this changes compute-res acquisition on " + "hardware that has never run this code -- see skel.c's hwinfo comment" + ) + + +def test_hwinfo_does_not_claim_the_echoed_fields_are_dsp_facts(): + """The IDL said "what the DSP says about itself" for five fields when it is + true of one. A doc asserting a guarantee the code does not deliver counts + the same as a code defect here, because these files are the handoff record.""" + idl = _src("hexlib/runtime/idl/hexlib_iface.idl") + assert "ONLY vtcm_size IS A DSP FACT" in idl, ( + "the hwinfo block must state which outputs are queried and which are " + "compile-time constants or host echoes" + ) + body = csource.function_body(_src(SKEL_C), "hexlib_iface_hwinfo") + assert re.search(r"\*n_threads\s*=\s*1\s*;", body), ( + "n_threads is hardcoded; if that changed, the IDL note must change too" + ) + + +def test_start_does_not_flatten_the_vtcm_status_into_a_bare_failure(): + """`return AEE_EFAILED` threw away which of 14 statuses occurred.""" + body = csource.function_body(_src(SKEL_C), "hexlib_iface_start") + assert "HEXLIB_AEE_FROM_STATUS" in body, ( + "start() must carry the specific status out; a bare AEE_EFAILED is the " + "result the host already prints for a dozen unrelated causes" + ) + assert not re.search(r"return\s+AEE_EFAILED\s*;", body), ( + "the flattening return is what this fixes" + ) + + +# -------------------------------------------------------------------------- +# The status encoding, compiled and driven with real values +# -------------------------------------------------------------------------- + +_PROBE = r""" +#include +#include +%s +int main(void) { + /* every status in the enum, plus the tag boundaries */ + for (int s = 1; s <= 14; s++) { + int r = HEXLIB_AEE_FROM_STATUS(s); + printf("%%d %%u %%d %%d\n", s, (unsigned) r, + HEXLIB_AEE_IS_STATUS(r) ? 1 : 0, HEXLIB_AEE_STATUS(r)); + } + /* things that must NOT decode as our status */ + unsigned others[] = {0u, 0x80000008u, 0x8FA10000u, 0x0FA00000u}; + for (int i = 0; i < 4; i++) { + printf("other %%u %%d\n", others[i], HEXLIB_AEE_IS_STATUS(others[i]) ? 1 : 0); + } + return 0; +} +""" + + +def _macros(): + """The four macros, sliced out of the real header.""" + src = _src(DSP_H) + lines = [ + ln for ln in src.splitlines() + if re.match(r"\s*#define\s+HEXLIB_AEE_", ln) + ] + assert len(lines) == 4, f"expected 4 HEXLIB_AEE_ macros, found {len(lines)}" + return "\n".join(lines) + + +@needs_cc +def test_every_status_round_trips_through_the_aee_encoding(tmp_path): + """COMPILED AND RUN, not pattern-matched. + + A source assertion cannot see that a mask and a shift disagree -- which is + precisely how the arch cross-check shipped comparing 75 against 0x8c75. + """ + src = tmp_path / "probe.c" + src.write_text(_PROBE % _macros(), encoding="utf-8") + exe = tmp_path / "probe.exe" + subprocess.run([HOST_CC, str(src), "-o", str(exe)], check=True, + capture_output=True) + out = subprocess.run([str(exe)], check=True, capture_output=True, + text=True).stdout + + seen = {} + for line in out.strip().splitlines(): + parts = line.split() + if parts[0] == "other": + assert parts[2] == "0", ( + f"{parts[1]} must not decode as a hexlib status -- it would make " + "an RPC-layer failure look like a VTCM one" + ) + continue + s, raw, is_status, decoded = (int(p) for p in parts) + assert is_status == 1, f"status {s} must be recognised by its own tag" + assert decoded == s, f"status {s} decoded as {decoded}" + assert raw != 0, "the encoded value must be nonzero so every caller still fails" + seen[s] = raw + + assert len(seen) == 14 + assert len(set(seen.values())) == 14, "each status must encode distinctly" + + +@needs_cc +def test_the_encoding_is_never_zero_so_a_failure_is_never_read_as_success(tmp_path): + """AEE_SUCCESS is 0. An encoding that produced 0 for some status would turn + a VTCM failure into a successful session -- absence read as success, in the + place it would cost a device job.""" + src = tmp_path / "probe.c" + src.write_text(_PROBE % _macros(), encoding="utf-8") + exe = tmp_path / "probe.exe" + subprocess.run([HOST_CC, str(src), "-o", str(exe)], check=True, + capture_output=True) + out = subprocess.run([str(exe)], check=True, capture_output=True, + text=True).stdout + for line in out.strip().splitlines(): + parts = line.split() + if parts[0] == "other": + continue + assert int(parts[1]) != 0, f"status {parts[0]} encoded to 0 (== AEE_SUCCESS)" diff --git a/hexlib/tests/test_wholeplan.py b/hexlib/tests/test_wholeplan.py new file mode 100644 index 0000000..99a0613 --- /dev/null +++ b/hexlib/tests/test_wholeplan.py @@ -0,0 +1,217 @@ +# hexlib/tests/test_wholeplan.py +"""The whole plan as ONE batch: a single entry point into the DSP, one exit. + +WHAT THESE PROVE THAT `test_encoder_on_sim.py` DOES NOT. That file runs the +encoder with one simulator launch per op. It proves the kernels and the wiring; +it cannot prove that 259 ops in a SINGLE blob address each other correctly, +because it never builds one. The failure modes are different in kind: a tensor +index that is right per-op and wrong in a shared table, an arena offset that +collides with a live value, a reshape elided into an address someone else owns. + +Most of these need no SDK. The arena is built and checked on the host; only the +last test launches a simulator. +""" +import os +import struct + +import numpy as np +import pytest + +import hexlib.graph.opdefs # noqa: F401 -- registers the op definitions +from hexlib import toolchain as tc +from hexlib.exec.wholeplan import WholePlanError, build_whole_plan_batch +from hexlib.graph.pipeline import compile_model +from hexlib.models.vit import build_vision_encoder +from hexlib.runtime import wire +from hexlib.tests.test_encoder_on_sim import _feeds, _tiny_cfg + +HAS_SDK = os.path.isdir(tc.default_sdk_root()) +sdk = pytest.mark.skipif(not HAS_SDK, reason="Hexagon SDK not present") + +VTCM_BUDGET = 8 * 1024 * 1024 + + +def _built(): + graph = build_vision_encoder(_tiny_cfg()) + compiled = compile_model( + graph, budget=VTCM_BUDGET, + order_policy="min_peak", alloc_policy="largest_first", + ) + feeds = _feeds(compiled.graph, compiled.plan) + return compiled, feeds, build_whole_plan_batch(compiled, feeds) + + +def test_every_real_work_op_is_in_one_batch(): + """The count, asserted rather than eyeballed. A spec that stopped matching + would quietly shrink this batch, and the run would still succeed -- on + fewer ops than the encoder has.""" + compiled, _, b = _built() + steps = [s for s in compiled.plan.steps if getattr(s, "op", None) is not None] + reshapes = [s for s in steps if s.op.kind == "reshape"] + assert b.n_ops == len(steps) - len(reshapes) + assert b.skipped_reshapes == len(reshapes) + assert b.n_ops > 30, f"only {b.n_ops} ops reached the batch" + + +def test_the_blob_declares_the_ops_it_carries(): + """`wire.py` has `pack_batch` and `unpack_response` and no batch-header + DECODER -- the only thing that parses a batch is `skel_dispatch.c`. So this + reads `n_ops` out of the packed header directly rather than skipping. + + An earlier version of this test called a decoder that does not exist and + `pytest.skip`ped when `hasattr` said so, which made it a test that could + never run and never fail -- this project's own named failure mode, in the + file that exists to catch that class of thing.""" + _, _, b = _built() + hdr = struct.unpack(wire._HDR, b.blob[:wire.HDR_SIZE]) + assert b.n_ops in hdr, ( + f"n_ops={b.n_ops} appears nowhere in the packed header {hdr}" + ) + + +def test_no_tensor_runs_past_the_end_of_the_arena(): + """`pack_batch` checks this per tensor against the declared buffer size. The + check here is the complementary one: the arena is actually that big.""" + _, _, b = _built() + assert len(b.payload) == b.arena_bytes + for name, p in b.placements.items(): + assert p.offset + p.nbytes <= b.arena_bytes, name + + +def test_two_live_tensors_never_share_an_address(): + """THE INVARIANT THE WHOLE ARENA RESTS ON, checked directly rather than + inferred from the allocator having run. + + With `alias_activations=False` no activation shares an address at all, so + this is a strong statement: any overlap is a placement bug.""" + _, _, b = _built() + acts = [p for p in b.placements.values() if p.region == "act"] + acts.sort(key=lambda p: p.offset) + for a, c in zip(acts, acts[1:]): + assert a.offset + a.nbytes <= c.offset, ( + f"{a.name} [{a.offset}, {a.offset + a.nbytes}) overlaps " + f"{c.name} at {c.offset}" + ) + + +def test_a_reshape_output_shares_its_inputs_bytes_and_emits_no_op(): + """A reshape is a reinterpretation, so its output must land on its input's + storage with the SAME byte count and a DIFFERENT ne. If it ever gets its own + offset, the value is silently never written there.""" + compiled, _, b = _built() + pairs = [ + (s.op.outputs[0], s.op.inputs[0]) + for s in compiled.plan.steps + if getattr(s, "op", None) is not None and s.op.kind == "reshape" + ] + assert pairs, "the tiny encoder has no reshape; this test is vacuous" + for out, inp in pairs: + po, pi = b.placements[out], b.placements[inp] + assert po.offset == pi.offset, f"{out} does not share {inp}'s storage" + assert po.nbytes == pi.nbytes + + +def test_aliasing_the_plans_vtcm_slots_is_refused_not_silently_wrong(): + """THE BUG THIS GUARD EXISTS FOR, PINNED. + + Reusing the plan's VTCM offsets as flat-arena addresses looks obviously + right and is not: this builder moves a reshape output onto its input, which + keeps the input's slot live past the point the allocator was told it died. + Measured at the tiny config, the encoder's output correlation with the + reference fell from 1.000000 to 0.277 -- a wrong answer indistinguishable + from a kernel bug. + + The allocator is NOT at fault; its own slots have zero overlapping live + ranges. So this asserts the REFUSAL, because a mode that silently corrupts + is worse than one that does not exist.""" + compiled, feeds, _ = _built() + with pytest.raises(WholePlanError, match="live"): + build_whole_plan_batch(compiled, feeds, alias_activations=True) + + +def test_the_plans_own_slots_do_not_overlap_in_live_range(): + """The companion to the test above, and the reason it can name a culprit. + Without this, 'aliasing is unsafe' would be equally consistent with a broken + allocator, and the fix would have been attempted in the wrong file.""" + compiled, _, _ = _built() + slots = list(compiled.plan.vtcm) + by_offset = {} + for s in slots: + by_offset.setdefault(s.offset, []).append(s) + for offset, group in by_offset.items(): + for i in range(len(group)): + for j in range(i + 1, len(group)): + a, c = group[i], group[j] + assert not (a.first_use <= c.last_use and c.first_use <= a.last_use), ( + f"the allocator put {a.tensor} and {c.tensor} at offset " + f"{offset} with overlapping live ranges" + ) + + +def test_a_const_is_staged_in_the_dtype_its_consumer_declares(): + """`pos_embed` is fp32 in the graph and `add` declares both inputs fp16. + The per-op path coerces at call time; an arena has to do it at staging, and + getting this wrong is an ERR_REQUIRES from the DSP rather than a wrong + answer -- which is why it is worth pinning that it stays right.""" + _, _, b = _built() + p = b.placements["pos_embed"] + assert p.dtype == "fp16", ( + f"pos_embed staged as {p.dtype}; add declares fp16 inputs and the DSP's " + f"generated entry checks it" + ) + + +@sdk +def test_the_whole_encoder_runs_as_a_single_invoke_and_matches_the_reference(): + """ONE batch, ONE simulator launch, 49 ops, compared against the op + registry's numpy reference over the encoder's declared outputs. + + The tolerance is the same 5% `test_encoder_on_sim.py` uses and for the same + reason -- fp16 storage compounding across two layers. What makes this test + worth its runtime is not the bound but that the single-blob path reaches it + at all: every tensor index, every arena offset and every elided reshape has + to be right simultaneously for the answer to land anywhere near.""" + from hexlib.exec import dsp as dspmod + from hexlib.exec import interpreter + from hexlib.exec.runner import SPECS, WIRE_DTYPE + from hexlib.tests.test_encoder_on_sim import _result, _work_dir + + compiled, feeds, b = _built() + work = os.environ.get("HEXLIB_SIM_WORK") or _work_dir() + sim = dspmod.DspSimBackend( + sorted({os.path.basename(s.kernel_dir) for s in SPECS.values()}), work + ) + sim._write_call(b.blob, bytes(b.payload)) + res = dspmod.run_sim(work, sdk_root=sim.sdk_root) + assert res.status == wire.STATUS["OK"], ( + f"batch status {wire.STATUS_NAME.get(res.status, res.status)}" + ) + + rsp = sim._read_response() + assert len(rsp.results) == b.n_ops + bad = [(i, r) for i, r in enumerate(rsp.results) if not r.ok] + assert not bad, ( + "ops failed on the DSP: " + + ", ".join(f"{b.op_names[i]}=" + f"{wire.STATUS_NAME.get(r.status, r.status)}" for i, r in bad[:5]) + ) + assert res.cycles > 0, "cycles_total is zero; nothing was measured" + + arena = open(os.path.join(work, "hexlib_out.bin"), "rb").read() + assert len(arena) == b.arena_bytes + + ref = interpreter.run(compiled, feeds) + assert not hasattr(ref, "reason"), getattr(ref, "detail", ref) + + for name in b.outputs: + p = b.placements[name] + got = np.frombuffer( + arena[p.offset:p.offset + p.nbytes], dtype=WIRE_DTYPE[p.dtype] + ).astype(np.float64) + want = np.asarray(_result(ref, name), dtype=np.float64).reshape(-1) + assert got.shape == want.shape + assert np.all(np.isfinite(got)), f"{name}: non-finite values" + rel = float(np.abs(got - want).max()) / max(float(np.abs(want).max()), 1e-6) + assert rel < 0.05, f"{name}: max relative error {rel:.4e}" + corr = float(np.corrcoef(got, want)[0, 1]) + assert corr > 0.99, f"{name}: correlation {corr:.6f}" diff --git a/hexlib/tests/test_wire_struct_layout.py b/hexlib/tests/test_wire_struct_layout.py new file mode 100644 index 0000000..b6b095a --- /dev/null +++ b/hexlib/tests/test_wire_struct_layout.py @@ -0,0 +1,735 @@ +# hexlib/tests/test_wire_struct_layout.py +"""BEHAVIOURAL test for the HOST/DSP STRUCT SEAM: every `struct` format in +hexlib/runtime/wire.py against the real C structs in +hexlib/runtime/skel/hexlib_dsp.h, compiled by a host C compiler. + +WHY THIS EXISTS. `wire.py` serializes a batch with hand-written `struct` format +strings (`_HDR`, `_BUF`, `_TENSOR`, `_OP`, `_RSP_HDR`, `_RESULT`); the DSP casts +the same bytes to `struct hexlib_batch_hdr`, `struct hexlib_buf_desc`, +`struct hexlib_tensor`, `struct hexlib_op_desc`, `struct hexlib_batch_rsp_hdr` +and `struct hexlib_op_result`. Nothing checked that those two descriptions of +the same bytes agreed. They do agree today -- every size and every offset was +confirmed by compiling the header -- so this file is not fixing a live bug; it +is closing the gap that would let one appear silently. + +AND IT WOULD BE SILENT. Reorder `hexlib_tensor`'s `dtype` and `layout` fields +and every test in this repo stayed green while the DSP read a tensor's dtype as +its layout: a q4_0 weight interpreted as row_major, or vice versa. That is not +a crash and not a compile error -- it is a plausible wrong answer, produced at +full speed. test_runtime_wire.py's `test_c_header_agrees_with_python_on_every_ +constant` checks the #defines and the status enum; the LAYOUT was never checked +by anything, in either direction. + +THIS MATTERS MORE FROM HERE ON. The batch format is about to carry a 308-op +encoder plan rather than one op, so `_OP`'s 92 bytes get multiplied by 308 and +every offset in it is exercised 308 times per invoke. A one-field disagreement +that is survivable-looking with a single op is a garbled plan at that size. + +WHAT THIS DOES, AND WHY IT IS THE SAME RECIPE AS ITS TWO SIBLINGS. Follows +test_session_arch_decode.py and test_coherency_lane_classification.py exactly: +generate a small standalone C program, compile it with a host C compiler, RUN +it, and compare real measured numbers against what Python claims -- rather than +asserting on source text. Here the program `#include`s hexlib_dsp.h itself +(never a retyped copy of the structs) and prints `sizeof` for each struct plus +`offsetof` and the member `sizeof` for every field. Same HOST_CC lookup and the +same `needs_cc` skipif as both siblings, for the same reason: a machine with no +host compiler must skip cleanly and say what it lost, not fail and not silently +pass. + +THE BRIDGE, AND WHY IT IS NOT A HAND-MAINTAINED SECOND COPY. `wire.py`'s +formats carry no field NAMES, and the C structs carry no format string, so +something has to pair them: `WIRE_STRUCTS` below lists, per struct, the C field +names in the order the host writes them and how many format elements each field +occupies. It deliberately does NOT restate the types. Every type, size and +offset on the Python side is derived from `wire.py`'s own format strings by +`_flatten`, and `_assert_every_format_element_is_accounted_for` fails if a +field is added to or removed from a format without being added here -- so this +table cannot quietly drift out of agreement with wire.py, it can only stop +compiling against it. + +`<` MEANS NO PADDING, WHICH IS THE ONE ASSUMPTION WORTH NAMING. Every format +in wire.py is little-endian-with-no-alignment (`<`), so Python's offsets are +just cumulative sizes. The C structs are naturally aligned by the compiler. The +two agree only because every field in every wire struct happens to be laid out +so that natural alignment introduces no padding (e.g. `hexlib_batch_rsp_hdr` +has exactly four uint32 before its uint64, so the uint64 lands on 16). That is +a real property of these structs and not a general one -- it is exactly what +this file measures rather than assumes. + +COMPILER-INDEPENDENT HALF. `test_the_header_declares_every_wire_field_in_the_ +order_the_host_writes_them` runs with no `cc` on PATH and catches the +dtype/layout reorder on its own, by reading the field order out of the header +text with `csource.block_from` (comment-aware and comment-blanked, so a field +name left behind in a comment cannot satisfy it). It cannot catch a TYPE change +or a padding change -- `uint32_t offset` becoming `uint64_t offset` keeps the +order intact -- which is what the compiled tests below are for. + +THE THIRD DESCRIPTION OF THE SAME BYTES, WHICH NOTHING CHECKED AT ALL. Every +test above compares the C structs against wire.py's FORMAT STRINGS. There is a +third description in play and it is the one that actually runs: the ARGUMENT +ORDER of `pack_batch`'s `struct.pack` calls, and the unpacking order in +`unpack_response`. A format string says "eleven uint32 in a row"; it does not say +which value goes in which. Swapping `dtype` and `layout` in +`pack_batch`'s tensor `struct.pack(...)` call -- so every tensor's dtype is +written into the DSP's `layout` field and vice versa -- left the whole offline +suite at 810 passed. It was caught only by the @sdk-gated `test_dsp_sim.py`, +which needs the Hexagon SDK and which CI does not run: on any machine without +the SDK, and in CI, a q4_0 weight read as row_major (or vice versa) was +completely unpinned. The layout tests here could not see it, because the bytes +still had the right SIZE at the right OFFSETS -- they just meant different +things. + +`test_pack_batch_writes_every_value_into_the_field_it_belongs_to` closes that, +with no compiler needed. It packs a batch in which every field of every record +holds a DISTINCT recognizable value, then reads each field back at the offset +this file's own bridge table implies and checks it is the value that field was +given. Any two fields exchanged in a `struct.pack` call swaps two distinct +values and fails. `test_unpack_response_reads_every_field_from_the_slot_the_dsp_ +wrote_it_in` is the same idea in the other direction, on the response path. +""" +import pathlib +import re +import shutil +import struct +import subprocess +import sys + +import pytest + +from hexlib.runtime import wire +from hexlib.tests import csource + +DSP_H = pathlib.Path("hexlib/runtime/skel/hexlib_dsp.h") + +HOST_CC = shutil.which("cc") or shutil.which("gcc") or shutil.which("clang") +needs_cc = pytest.mark.skipif( + HOST_CC is None, + reason=( + "no host C compiler found (tried: cc, gcc, clang); the BEHAVIOURAL " + "host/DSP struct-layout test is skipped, and only the weaker " + "field-ORDER check in this same file " + "(test_the_header_declares_every_wire_field_in_the_order_the_host_" + "writes_them) covers this -- which cannot see a field's TYPE or " + "alignment change. Install a host C compiler to restore it." + ), +) + + +class _Wire: + """One wire struct: the name of wire.py's format string, the C struct it + describes, and the C field names in the order the host writes them paired + with how many format elements each consumes (4 for `ne[4]`, 16 for + `params[16]`, 1 for a scalar). No types: those come from wire.py.""" + + def __init__(self, fmt_attr, c_name, fields): + self.fmt_attr = fmt_attr + self.c_name = c_name + self.fields = fields + + @property + def fmt(self): + return getattr(wire, self.fmt_attr) + + def __repr__(self): + return f"{self.c_name} (wire.{self.fmt_attr})" + + +WIRE_STRUCTS = ( + _Wire("_HDR", "hexlib_batch_hdr", ( + ("magic", 1), ("version", 1), ("total_size", 1), ("n_bufs", 1), + ("n_tensors", 1), ("n_ops", 1), ("off_bufs", 1), ("off_tensors", 1), + ("off_ops", 1), ("flags", 1), + )), + _Wire("_BUF", "hexlib_buf_desc", ( + # `base` first, and it is DSP-side scratch the host writes 0 into -- + # see wire.py's module docstring. Its OFFSET being right is what makes + # "the host cannot express an address" true on the wire and not just in + # the dataclass. + ("base", 1), ("size", 1), ("fd", 1), ("flags", 1), + )), + _Wire("_TENSOR", "hexlib_tensor", ( + ("bi", 1), ("offset", 1), ("nbytes", 1), ("dtype", 1), ("layout", 1), + ("ne", 4), ("data", 1), ("pad", 1), + )), + _Wire("_OP", "hexlib_op_desc", ( + ("kind", 1), ("flags", 1), ("params", wire.MAX_PARAMS), + ("src", wire.MAX_SRC), ("dst", wire.MAX_DST), + )), + _Wire("_RSP_HDR", "hexlib_batch_rsp_hdr", ( + ("magic", 1), ("version", 1), ("status", 1), ("n_ops", 1), + ("cycles_total", 1), ("arch", 1), ("pad", 1), + )), + _Wire("_RESULT", "hexlib_op_result", ( + ("kind", 1), ("status", 1), ("cycles", 1), + )), +) + +_COUNTED_CODE = re.compile(r"(\d*)([a-zA-Z?])") + + +def _flatten(fmt): + """Expand a struct format into one code per element: `" + `["I","I","i"*16...,"H"*6...,"H"*4...]`. Repeat counts are the only thing + that makes a format's element count differ from its character count, and + getting that wrong is how a bridge table silently stops lining up. + + Refuses anything but a `<` prefix: the whole comparison below assumes no + alignment padding on the Python side, so a format that quietly changed to + `@` or `=` must fail loudly here rather than produce offsets that look + plausible.""" + assert fmt.startswith("<"), ( + f"{fmt!r} is not little-endian-no-padding ('<'); every offset computed " + "in this file assumes that, so a change of byte-order character must " + "be dealt with here explicitly, not absorbed" + ) + out = [] + for count, code in _COUNTED_CODE.findall(fmt[1:]): + out.extend([code] * (int(count) if count else 1)) + return out + + +def _python_layout(w): + """What wire.py's own format implies: (total size, {field: (offset, size)}). + Every number here comes from `struct.calcsize` over a slice of the REAL + format string -- nothing is typed in.""" + codes = _flatten(w.fmt) + layout = {} + i = 0 + for name, n in w.fields: + layout[name] = ( + struct.calcsize("<" + "".join(codes[:i])), + struct.calcsize("<" + "".join(codes[i:i + n])), + ) + i += n + assert i == len(codes), ( + f"{w!r}: this file's field table accounts for {i} of the " + f"{len(codes)} elements in wire.{w.fmt_attr} -- a field was added to " + "or removed from the format without being added here, so the " + "comparison below would have checked a prefix and called it a match" + ) + return struct.calcsize(w.fmt), layout + + +def test_every_wire_format_element_is_accounted_for_by_this_files_field_table(): + """Runs with or without a compiler. `_python_layout`'s own trailing + assertion is the point: it is what stops this file's bridge table from + silently describing a subset of wire.py's formats.""" + for w in WIRE_STRUCTS: + total, layout = _python_layout(w) + assert total > 0 + assert len(layout) == len(w.fields) + + +def test_the_size_constants_wire_py_exports_match_its_own_formats(): + """Cheap, but it pins the pairing the rest of this file (and pack_batch's + own offset arithmetic) relies on: HDR_SIZE really is calcsize(_HDR), etc.""" + for attr, size_attr in ( + ("_HDR", "HDR_SIZE"), ("_BUF", "BUF_SIZE"), ("_TENSOR", "TENSOR_SIZE"), + ("_OP", "OP_SIZE"), ("_RSP_HDR", "RSP_HDR_SIZE"), + ("_RESULT", "RESULT_SIZE"), + ): + assert getattr(wire, size_attr) == struct.calcsize(getattr(wire, attr)) + + +# ============================================================================== +# WHAT pack_batch / unpack_response ACTUALLY PUT IN EACH SLOT. No compiler +# needed: this is Python's own serializer checked against this file's field +# table, which the compiled tests above have already checked against the C. +# ============================================================================== + +_BY_NAME = {w.c_name: w for w in WIRE_STRUCTS} + + +def _read_record(c_name, blob, offset): + """`{field: value}` for one record of `struct c_name` at `offset` in `blob`, + unpacked with wire.py's own format and named by this file's field table. + Array fields come back as tuples. This is the only place the two are + paired, which is what makes a swapped pair of `struct.pack` arguments + visible: the format cannot tell them apart, the names can.""" + w = _BY_NAME[c_name] + values = struct.unpack_from(w.fmt, blob, offset) + out, i = {}, 0 + for name, n in w.fields: + out[name] = values[i] if n == 1 else tuple(values[i:i + n]) + i += n + assert i == len(values) + return out + + +def _assert_distinct(record, name, exempt=()): + """Every scalar in `record` must be a DIFFERENT value, or a swap of the two + that match would pass. `exempt` names fields whose value is fixed by the + contract (the DSP-side scratch slots, which are both required to be 0) and + so cannot be made distinct.""" + scalars = {k: v for k, v in record.items() + if isinstance(v, int) and k not in exempt} + assert len(set(scalars.values())) == len(scalars), ( + f"this test's own {name} values are not all distinct, so a swapped pair " + f"of fields would pass it: {scalars!r}" + ) + + +# Chosen so that within every record every scalar differs from every other -- +# including `version` (1) against `n_ops`, which is why there are two ops and +# three buffers rather than one of each. +_PACK_BUFS = ( + dict(fd=11, size=4096, flags=5), + dict(fd=22, size=8192, flags=6), + dict(fd=33, size=2048, flags=7), +) +_PACK_TENSORS = ( + dict(bi=0, offset=32, nbytes=16, dtype="fp16", layout="q4_0_repacked", + ne=(3, 5, 7, 9)), + # THE EXHAUSTIVELY-CHECKED ONE: every scalar in it is a different number, + # and its dtype and layout ids differ from each other AND from the other + # tensor's, so neither can be a constant and the two cannot be exchanged. + dict(bi=2, offset=64, nbytes=128, dtype="int32", layout="tiled_32x32", + ne=(11, 13, 17, 19)), + dict(bi=1, offset=256, nbytes=512, dtype="fp32", layout="row_major", + ne=(23, 29, 31, 37)), + dict(bi=1, offset=1024, nbytes=48, dtype="q4_0", layout="q4_0_repacked", + ne=(41, 43, 47, 53)), +) +_PACK_OPS = ( + dict(kind=9, flags=3, params=(101, 102, 103), src=(1, 2), dst=(0,)), + dict(kind=10, flags=4, params=(201, 202), src=(0, 3), dst=(1,)), +) + + +@pytest.fixture(scope="module") +def packed(): + """One real `pack_batch` blob built from the tables above.""" + bufs = [wire.BufDesc(**b) for b in _PACK_BUFS] + tensors = [wire.TensorDesc(**t) for t in _PACK_TENSORS] + ops = [wire.OpDesc(**o) for o in _PACK_OPS] + return wire.pack_batch(bufs, tensors, ops) + + +def test_pack_batch_writes_every_value_into_the_field_it_belongs_to(packed): + """THE THIRD DESCRIPTION, CHECKED. See the module docstring: swapping + `dtype` and `layout` in pack_batch's tensor pack call was 810-green offline + and caught only by the SDK-gated simulator test CI does not run. + + Every assertion here is a whole-record equality, not a field-by-field spot + check, so a field this test forgot cannot be the one that drifts -- and + `_assert_distinct` refuses to let the test pass on values that could not + tell a swap apart in the first place.""" + hdr = _read_record("hexlib_batch_hdr", packed, 0) + _assert_distinct(hdr, "header", exempt=("flags",)) + assert hdr == { + "magic": wire.BATCH_MAGIC, + "version": wire.BATCH_VERSION, + "total_size": len(packed), + "n_bufs": len(_PACK_BUFS), + "n_tensors": len(_PACK_TENSORS), + "n_ops": len(_PACK_OPS), + "off_bufs": wire.HDR_SIZE, + "off_tensors": wire.HDR_SIZE + wire.BUF_SIZE * len(_PACK_BUFS), + "off_ops": (wire.HDR_SIZE + wire.BUF_SIZE * len(_PACK_BUFS) + + wire.TENSOR_SIZE * len(_PACK_TENSORS)), + "flags": 0, + } + + for i, b in enumerate(_PACK_BUFS): + rec = _read_record("hexlib_buf_desc", packed, + hdr["off_bufs"] + wire.BUF_SIZE * i) + _assert_distinct(rec, f"buffer {i}", exempt=("base",)) + assert rec == {"base": 0, "size": b["size"], "fd": b["fd"], + "flags": b["flags"]}, ( + f"buffer {i}: pack_batch put the values somewhere other than the " + f"fields they name. `base` must be 0 -- see wire.py's docstring: " + f"there is no field for a host address, and the DSP fills this one" + ) + + for i, t in enumerate(_PACK_TENSORS): + rec = _read_record("hexlib_tensor", packed, + hdr["off_tensors"] + wire.TENSOR_SIZE * i) + if i == 1: + _assert_distinct(rec, f"tensor {i}", exempt=("data", "pad")) + assert rec == { + "bi": t["bi"], "offset": t["offset"], "nbytes": t["nbytes"], + "dtype": wire.DTYPE_ID[t["dtype"]], + "layout": wire.LAYOUT_ID[t["layout"]], + "ne": t["ne"], "data": 0, "pad": 0, + }, ( + f"tensor {i}: the DSP reads these eleven uint32 by NAME " + f"(hexlib_dsp.h) and pack_batch wrote them in a different order. " + f"dtype/layout exchanged is a q4_0 weight read as row_major -- a " + f"plausible wrong answer at full speed, not a crash" + ) + + for i, op in enumerate(_PACK_OPS): + rec = _read_record("hexlib_op_desc", packed, + hdr["off_ops"] + wire.OP_SIZE * i) + assert rec == { + "kind": op["kind"], + "flags": op["flags"], + "params": (tuple(op["params"]) + + (0,) * (wire.MAX_PARAMS - len(op["params"]))), + "src": (tuple(op["src"]) + + (0xFFFF,) * (wire.MAX_SRC - len(op["src"]))), + "dst": (tuple(op["dst"]) + + (0xFFFF,) * (wire.MAX_DST - len(op["dst"]))), + }, ( + f"op {i}: `kind` is what skel_dispatch.c matches the kernel table " + f"on and src/dst are what its fill loop walks in that order, so an " + f"exchange here dispatches the wrong kernel or feeds it the wrong " + f"buffers" + ) + + +def test_unpack_response_reads_every_field_from_the_slot_the_dsp_wrote_it_in(): + """THE RETURN PATH, SAME PROPERTY. `unpack_response` names its fields by + tuple position (`magic, version, status, n_ops, cycles, arch, _ = ...`), so + exchanging two of those names is invisible to any format-string comparison + -- and reading `status` out of the `n_ops` slot would report a two-op batch + as ERR_INTERNAL, or a failure as success. + + The bytes are built HERE, in this file's own field order (the order the + compiled tests above have already checked against hexlib_dsp.h), rather than + by wire.py -- otherwise a matching pair of mistakes on both sides would + cancel out and this would pass.""" + hdr_fields = { + "magic": wire.BATCH_MAGIC, "version": wire.BATCH_VERSION, + "status": wire.STATUS["ERR_REQUIRES"], "n_ops": 2, + "cycles_total": 1287, "arch": 75, "pad": 0, + } + _assert_distinct(hdr_fields, "response header", exempt=("pad",)) + results = ( + {"kind": 9, "status": wire.STATUS["OK"], "cycles": 101}, + {"kind": 10, "status": wire.STATUS["ERR_REQUIRES"], "cycles": 202}, + ) + for i, r in enumerate(results): + _assert_distinct(r, f"result {i}") + + def _pack(c_name, values): + w = _BY_NAME[c_name] + flat = [] + for name, n in w.fields: + v = values[name] + flat.extend(v if n > 1 else [v]) + return struct.pack(w.fmt, *flat) + + blob = _pack("hexlib_batch_rsp_hdr", hdr_fields) + for r in results: + blob += _pack("hexlib_op_result", r) + + rsp = wire.unpack_response(blob) + assert rsp.status == hdr_fields["status"], ( + "unpack_response read `status` out of a different slot than the one " + "hexlib_dsp.h declares it in" + ) + assert rsp.n_ops == hdr_fields["n_ops"] + assert rsp.cycles_total == hdr_fields["cycles_total"] + assert rsp.arch == hdr_fields["arch"] + assert len(rsp.results) == len(results) + for got, want in zip(rsp.results, results): + assert (got.kind, got.status, got.cycles) == ( + want["kind"], want["status"], want["cycles"] + ), ( + "an OpResult's kind/status/cycles came back in a different order " + "than the DSP wrote them" + ) + assert not rsp.ok, "a non-OK batch status must not read as ok" + + +@pytest.fixture(scope="module") +def header_source(): + """Comment-BLANKED header text. Every payload check in this file runs + against this, never the raw text, so a struct field name that survives only + inside a comment cannot satisfy an assertion -- see csource.py's module + docstring for the mutation that made this the default.""" + return csource.code_only(DSP_H.read_text()) + + +def _declared_fields(header, c_name): + """The field names of `struct c_name`, in declaration order, sliced out of + the header with `csource.block_from` (the shared comment-aware slicer, not + a fourth private copy -- see csource.py's docstring). Array declarators are + reduced to their name, so `uint32_t ne[4];` reads as `ne`.""" + marker = f"struct {c_name} {{" + assert marker in header, f"the header no longer declares `struct {c_name}`" + block = csource.block_from(header, header.index(marker)) + return [m.group(1) for m in re.finditer(r"\b(\w+)\s*(?:\[[^\]]*\])?\s*;", block)] + + +def test_the_header_declares_every_wire_field_in_the_order_the_host_writes_them( + header_source, +): + """COMPILER-INDEPENDENT -- runs even when `needs_cc` skips everything else, + so a field REORDER (the dtype/layout swap that motivated this file) is never + invisible on a machine with no host compiler. Order, and exactly these + fields: an extra field in the C struct that the host never writes is just as + much a seam break as a missing one, because it shifts everything after it.""" + for w in WIRE_STRUCTS: + expected = [name for name, _ in w.fields] + assert _declared_fields(header_source, w.c_name) == expected, ( + f"{w!r}: the C struct's fields are not the fields wire." + f"{w.fmt_attr} writes, in that order. Host and DSP disagree about " + "what the same bytes mean -- which is a wrong answer at full " + "speed, not a compile error" + ) + + +def _emit_probe_c(): + """A standalone C program that includes the REAL header and prints what the + compiler actually laid out. Generated from WIRE_STRUCTS so a field added + there is probed automatically. + + `(unsigned long)` + `%lu` rather than `%zu`: the host compiler here is + mingw gcc, whose `%zu` support depends on which stdio it was built against + (see test_coherency_lane_classification.py's note on this same toolchain + lacking `__fp16`). Every number printed is an offset or a small size, so a + 32-bit-safe cast costs nothing and removes the variable.""" + lines = [ + "#include ", + "#include ", + '#include "hexlib_dsp.h"', + "int main(void) {", + ] + for w in WIRE_STRUCTS: + lines.append( + f' printf("{w.c_name} . %lu 0\\n", ' + f"(unsigned long) sizeof(struct {w.c_name}));" + ) + for name, _ in w.fields: + lines.append( + f' printf("{w.c_name} {name} %lu %lu\\n", ' + f"(unsigned long) offsetof(struct {w.c_name}, {name}), " + f"(unsigned long) sizeof(((struct {w.c_name} *) 0)->{name}));" + ) + lines += [" return 0;", "}", ""] + return "\n".join(lines) + + +@pytest.fixture(scope="module") +def measured_layout(tmp_path_factory): + """Compile and RUN the probe, and return + {c_struct: (sizeof, {field: (offset, member_sizeof)})} as the host compiler + really laid it out. `-I` points at the header's own directory so the + `#include` resolves to the checked-in file and nothing else.""" + tmp_path = tmp_path_factory.mktemp("wire_layout") + c_path = tmp_path / "probe.c" + c_path.write_text(_emit_probe_c()) + exe = tmp_path / ("probe.exe" if sys.platform == "win32" else "probe") + + compile_result = subprocess.run( + [HOST_CC, "-o", str(exe), str(c_path), "-I", str(DSP_H.parent.resolve())], + capture_output=True, text=True, + ) + assert compile_result.returncode == 0, ( + "compiling the hexlib_dsp.h struct-layout probe failed -- the header " + "itself may not compile, which is a finding, not a reason to skip:\n" + f"{compile_result.stdout}\n{compile_result.stderr}" + ) + + run_result = subprocess.run([str(exe)], capture_output=True, text=True) + assert run_result.returncode == 0, ( + f"the struct-layout probe exited {run_result.returncode}:\n" + f"{run_result.stdout}\n{run_result.stderr}" + ) + + out = {} + for line in run_result.stdout.split("\n"): + parts = line.split() + if len(parts) != 4: + continue + c_name, field, a, b = parts + sizeof, fields = out.setdefault(c_name, (None, {})) + if field == ".": + out[c_name] = (int(a), fields) + else: + fields[field] = (int(a), int(b)) + assert out, ( + "the struct-layout probe compiled and ran but printed nothing " + f"parseable -- refusing to read that as agreement:\n{run_result.stdout}" + ) + return out + + +@needs_cc +@pytest.mark.parametrize("w", WIRE_STRUCTS, ids=lambda w: w.c_name) +def test_the_c_struct_is_the_size_wire_py_serializes(w, measured_layout): + """A size disagreement means the DSP reads op N at the wrong address for + every N > 0 -- and with a 308-op plan that is 307 garbled ops behind one + correct one.""" + expected_size, _ = _python_layout(w) + assert w.c_name in measured_layout, f"the probe printed nothing for {w!r}" + measured_size, _ = measured_layout[w.c_name] + assert measured_size == expected_size, ( + f"sizeof(struct {w.c_name}) is {measured_size} on the host compiler, " + f"but wire.{w.fmt_attr} serializes {expected_size} bytes" + ) + + +@needs_cc +@pytest.mark.parametrize("w", WIRE_STRUCTS, ids=lambda w: w.c_name) +def test_every_c_field_is_at_the_offset_and_width_wire_py_writes(w, measured_layout): + """THE DECISIVE ONE. Offset AND member width, per field. Offset catches a + reorder or unexpected padding; width catches a type change that a reorder + check cannot see (`uint32_t offset` -> `uint64_t offset` shifts nothing + before it and everything after).""" + _, expected = _python_layout(w) + _, measured = measured_layout[w.c_name] + for name, (exp_off, exp_size) in expected.items(): + assert name in measured, f"the probe printed no offset for {w.c_name}.{name}" + got_off, got_size = measured[name] + assert got_off == exp_off, ( + f"offsetof(struct {w.c_name}, {name}) is {got_off} in the C " + f"header, but wire.{w.fmt_attr} writes that field at {exp_off}. " + "The DSP is reading a different field than the host wrote" + ) + assert got_size == exp_size, ( + f"sizeof(struct {w.c_name}.{name}) is {got_size} in the C header, " + f"but wire.{w.fmt_attr} writes {exp_size} bytes there" + ) + + +@needs_cc +def test_the_dsp_side_scratch_fields_are_where_the_host_writes_its_zeros( + measured_layout, +): + """`hexlib_buf_desc.base` and `hexlib_tensor.data` are DSP-side scratch; + the host writes zeros into those exact wire slots (see wire.py's + pack_batch) so no host address can cross. test_runtime_wire.py's + `test_host_writes_zero_into_tensor_data` reads `data` back at a HARDCODED + `off_t + 9 * 4` -- correct today, and true only because of this layout. + Pin the offsets that hardcoding depends on, so if the header moves `data` + that test starts checking a different field's bytes and this one says why.""" + _, buf = measured_layout["hexlib_buf_desc"] + assert buf["base"][0] == 0, "the host's zero must land on `base`" + _, tensor = measured_layout["hexlib_tensor"] + assert tensor["data"] == (9 * 4, 4), ( + "hexlib_tensor.data is no longer the 10th uint32 -- " + "test_runtime_wire.py::test_host_writes_zero_into_tensor_data reads " + f"it at a hardcoded offset 36 and would now read {tensor['data']}" + ) + + +# --------------------------------------------------------------------------- +# The layout enum, on both sides at once +# --------------------------------------------------------------------------- + + +@needs_cc +def test_the_layout_enum_has_the_same_values_in_c_as_on_the_wire(tmp_path): + """`HEXLIB_LAYOUT_*` in the header vs `wire.LAYOUT_ID`, COMPILED. + + `main.c` used to write `tens[i].layout = 0` as a bare literal whose only + tie to `LAYOUT_ID["row_major"]` was a trailing comment, and unlike the + `dtype` literal beside it nothing checked layout anywhere: `grep -c layout + hexlib/tests/test_host_source.py` was 0. Inserting a value ahead of + row_major would have left `pack_batch` emitting 1 while `main.c` kept + emitting 0, and `--self-test` would still have printed `PASS (4100 values, + bit-exact)` over a buffer the batch declared as a different layout. + + Compiled rather than grepped for the reason this whole file exists: a + source assertion over `#define` lines is satisfied by a comment, and was + twice satisfied by a string literal. The preprocessor's own numbers are the + only thing that cannot be faked. Every entry in `LAYOUT_ID` must have a + macro, so ADDING a Python-side layout without adding the C one fails here + too -- which is the direction the next kernel takes (`q4_0_repacked`). + """ + names = {k: "HEXLIB_LAYOUT_" + k.upper() for k in wire.LAYOUT_ID} + lines = ['#include ', '#include "hexlib_dsp.h"', "int main(void) {"] + for key, macro in names.items(): + lines.append(f' printf("{key} %lu' + r'\n' + f'", (unsigned long) {macro});') + lines += [" return 0;", "}", ""] + + c_path = tmp_path / "layout_probe.c" + c_path.write_text("\n".join(lines)) + exe = tmp_path / ("lp.exe" if sys.platform == "win32" else "lp") + cc = subprocess.run( + [HOST_CC, "-o", str(exe), str(c_path), "-I", str(DSP_H.parent.resolve())], + capture_output=True, text=True, + ) + assert cc.returncode == 0, ( + "the layout-enum probe did not compile. Every name in wire.LAYOUT_ID " + "must have a HEXLIB_LAYOUT_ macro in hexlib_dsp.h -- a layout " + "that exists only on the Python side is one main.c cannot spell:\n" + f"{cc.stdout}\n{cc.stderr}" + ) + run = subprocess.run([str(exe)], capture_output=True, text=True) + assert run.returncode == 0, f"layout probe exited {run.returncode}" + + got = {} + for line in run.stdout.split("\n"): + parts = line.split() + if len(parts) == 2: + got[parts[0]] = int(parts[1]) + + assert got == dict(wire.LAYOUT_ID), ( + f"the C layout macros are {got} but wire.LAYOUT_ID is " + f"{dict(wire.LAYOUT_ID)}. These cross the wire in " + "hexlib_tensor.layout; a mismatch is a correctly-shaped wrong answer, " + "not a compile error." + ) + + +# --------------------------------------------------------------------------- +# The status names, on both sides at once +# --------------------------------------------------------------------------- + + +@needs_cc +def test_the_status_names_agree_between_c_and_the_wire_table(tmp_path): + """`hexlib_dsp_status_name` vs `wire.STATUS`, COMPILED. + + The host prints this string when the DSP tags a real status into an + AEEResult (`session.c`, via `HEXLIB_AEE_IS_STATUS`). Before that decode + existed, every cause of a failed `start()` printed the same bare negative + number -- a VTCM contention failure looked exactly like a signing failure or + a missing skel, which is the operator confusion the tag was added to remove. + A wrong NAME here is a different flavour of the same problem: it sends + someone to debug the wrong subsystem. + + Compiled rather than grepped, for the reason this whole file exists: a source + assertion over `case` labels is satisfiable by a comment, and twice was by a + string literal. This drives the real function with every value in + `wire.STATUS` and compares what it actually returns. + + Both directions are checked. Every wire status must have a name (a status + added to the Python table and not the switch), and the switch must not + invent one for a value the table does not have -- the `default` returns + "UNKNOWN", so an out-of-range value is reported as unknown rather than + silently reading a neighbouring string. + """ + names = sorted(wire.STATUS.items(), key=lambda kv: kv[1]) + lines = ['#include ', '#include "hexlib_dsp.h"', "int main(void) {"] + for _, value in names: + lines.append(f' printf("%d %s' + r'\n' + f'", {value}, ' + f"hexlib_dsp_status_name({value}));") + # And one value deliberately outside the enum. + lines.append(' printf("%d %s' + r'\n' + '", 999, hexlib_dsp_status_name(999));') + lines += [" return 0;", "}", ""] + + c_path = tmp_path / "status_probe.c" + c_path.write_text("\n".join(lines)) + exe = tmp_path / ("sp.exe" if sys.platform == "win32" else "sp") + cc = subprocess.run( + [HOST_CC, "-o", str(exe), str(c_path), "-I", str(DSP_H.parent.resolve())], + capture_output=True, text=True, + ) + assert cc.returncode == 0, ( + "the status-name probe did not compile -- hexlib_dsp.h itself may not, " + f"which is a finding and not a reason to skip:\n{cc.stdout}\n{cc.stderr}" + ) + run = subprocess.run([str(exe)], capture_output=True, text=True) + assert run.returncode == 0, f"status probe exited {run.returncode}" + + got = {} + for line in run.stdout.split("\n"): + parts = line.split() + if len(parts) == 2: + got[int(parts[0])] = parts[1] + + for name, value in names: + assert got.get(value) == name, ( + f"wire.STATUS says {value} is {name!r} but hexlib_dsp_status_name " + f"returns {got.get(value)!r}. The host prints this string to tell an " + f"operator which subsystem failed." + ) + assert got.get(999) == "UNKNOWN", ( + f"a status outside the enum returned {got.get(999)!r}; it must be " + f"reported as unknown rather than resolving to some other name" + ) diff --git a/hexlib/toolchain.py b/hexlib/toolchain.py index aeb43c9..701862b 100644 --- a/hexlib/toolchain.py +++ b/hexlib/toolchain.py @@ -25,6 +25,27 @@ DSP_ARCH = "v75" TOOLCHAIN_VERSION = "19.0.04" +# The Android NDK the Hexagon SDK ships alongside its own toolchain, used +# ONLY for Task 10's aarch64 cross-compile (hexlib_run). VERIFIED against the +# actual installed SDK, not assumed: `android-ndk-r25c/toolchains/llvm/ +# prebuilt/windows-x86_64/bin` genuinely contains +# `aarch64-linux-android21-clang` through `...34-clang`/`...35-clang`, so 33 +# exists and is used as pinned below -- this was checked, not carried over +# from a draft that guessed it. Task 9's host code itself has no NDK-version +# dependency (it is plain C against //), so this +# pin is a policy choice (a stable, well-supported API level), not something +# forced by the source. +NDK_VERSION = "r25c" +ANDROID_API = 33 + + +def ndk_root(sdk_root: str) -> str: + """The NDK bundled with the Hexagon SDK. Discovered, never vendored -- + the SDK is license-restricted and this path is inside it; nothing here + is fetched or copied out.""" + return os.path.join(sdk_root, "tools", f"android-ndk-{NDK_VERSION}") + + # Engages the cycle-approximate microarchitectural model (caches + bus latency). # With it off the simulator idealizes memory, which is misleading for # bandwidth-bound kernels. @@ -43,12 +64,74 @@ STD = "gnu11" COMPILER = "hexagon-clang" -HVX_CFLAGS = [f"-m{DSP_ARCH}", "-mhvx", "-mhvx-length=128B", f"-std={STD}", "-O2"] +# -Wall -Werror: THE ONLY AUTOMATIC BACKSTOP AGAINST A MIS-ORDERED GENERATED +# KERNEL CALL, and until 2026-08-11 it was thrown away. `hexlib/runtime/ +# genentry.py` emits each kernel's DSP entry point by ORDER -- inputs (cast to +# `const T *`) then the output (cast to `T *`) -- and getting that order wrong +# is not a crash and not a wrong status; it is a plausible wrong answer. What +# catches it is the compiler: passing the `const` input where the mutable +# output belongs is `-Wincompatible-pointer-types-discards-qualifiers`. Every +# caller of `tc.run` checks only `rc != 0`, and a warning leaves rc == 0, so +# that diagnostic was emitted and discarded on every build. +# +# MEASURED, NOT ASSUMED (2026-08-11, toolchain 19.0.04, the real generated +# scale_fp16_entry.c with its two buffer casts swapped): +# original, -Wall -Werror -> rc=0 +# swapped casts, -Wall -Werror -> rc=1, "error: passing 'const hexlib_hf *' +# ... discards qualifiers [-Werror, +# -Wincompatible-pointer-types-discards- +# qualifiers]" +# swapped casts, old flags -> rc=0, the SAME text as a warning +# So the mis-order was, and is, exactly one warning away from shipping. +# +# -Werror WHOLESALE, NOT A HAND-PICKED LIST, because it turned out not to +# break anything: the full SDK-gated build set (kernel ELFs via +# hexlib/build.py, the QuRT-hosted simulator .so, the device skel .so, and the +# aarch64 hexlib_run) compiles with ZERO warnings under -Wall on this +# toolchain, so there was no existing diagnostic to grandfather in and no +# reason to enumerate a subset that would then quietly not cover the next one. +# If a future kernel legitimately needs a warning suppressed, suppress THAT +# warning at THAT site (`#pragma clang diagnostic`) where a reader can see it +# -- do not widen this list back out. +# +# TWO FLAGS CONSIDERED AND REJECTED, both MEASURED rather than guessed at: +# +# -Wpedantic: 29 warnings across the skel + the six kernels today +# (26 -Wgnu-zero-variadic-macro-arguments, 3 -Wlanguage-extension-token). +# Under -Werror that is an immediate build failure, and the two classes are +# inherent: this codebase is GNU C on purpose (see STD above) and the HVX +# types are extensions. Rejected. +# +# -Wextra: measured CLEAN (0 warnings) on the same set, so it would build +# today -- and it is still rejected, because what it adds over -Wall is +# dominated by -Wunused-parameter/-Wsign-compare, which say nothing about +# the argument-order class this exists to catch, while making -Werror fire +# on ordinary in-progress kernel code (a stubbed kernel that ignores a +# parameter). It costs a contributor a build for no safety. Revisit only +# with a specific defect it would have caught. +HVX_CFLAGS = [f"-m{DSP_ARCH}", "-mhvx", "-mhvx-length=128B", f"-std={STD}", "-O2", + "-Wall", "-Werror"] SIM_TIMEOUT_S = 60 # An XL kernel gets more time, but this still kills genuine infinite loops. # Measured need: DMA/VTCM kernels have run 204-406s under the timing model. -SIM_TIMEOUT_MAX_S = 900 +# +# Raised 900 -> 1800 for matmul_fp16, whose SCALAR near-misses are the most +# expensive code the gate runs. At its harness shape (Bn=3, M=40, K=128, +# N=192 = 2.95M inner iterations) the three near-misses measured 825s, 862s +# and 1195s. The 1195s one is nearmiss_fp16_accumulate: it narrows to fp16 on +# every multiply-add rather than once per output element, costing 185,300,776 +# simulated cycles against the real kernel's 995,714 (186x). At 900s it was +# killed 295s from the end and reported INCONCLUSIVE -- a correct near-miss +# scored as a gate failure. Given room it terminates and rejects properly +# (n_wrong 64, max_err 0.125). +# +# The cost of this headroom is that a genuine infinite loop now burns 30 +# minutes instead of 15. That was accepted deliberately: the alternative was +# shrinking matmul_fp16's harness shape, and K=128 cannot move without +# destroying the adversarial element (one dominant product plus exactly 127 +# followers) that catches the fp16-accumulation bug in the first place. +SIM_TIMEOUT_MAX_S = 1800 def default_sdk_root() -> str: diff --git a/kernels/hmx_matmul_fp16/README.md b/kernels/hmx_matmul_fp16/README.md new file mode 100644 index 0000000..573cf22 --- /dev/null +++ b/kernels/hmx_matmul_fp16/README.md @@ -0,0 +1,99 @@ +# hmx_matmul_fp16 — committed, and it does NOT pass its gate + +**There is no `RESULT.md` in this directory, deliberately.** Every other kernel here has +one, because `RESULT.md` is the tool-generated record of a kernel that cleared gates 3 +and 5. This one does not clear them: the simulator faults before the harness prints +`HEXLIB_VERDICT`, and the gate reports that honestly — + +> no verdict recovered from the simulator: the harness never printed HEXLIB_VERDICT, so +> nothing was actually checked. This is a failure, not a pass. + +Writing a `RESULT.md` by hand would make an ungated kernel indistinguishable from a +gated one in exactly the place a reader goes to check. The record lives here instead. + +The kernel source is committed anyway because **what it found on the way is worth more +than the code**, and because the code is probably fine — it was being run somewhere it +can never work. + +## What it is + +The fp16 HMX tile sequence at the smallest shape whose scalar reference is trivially +checkable: `M=32, N=32, K=64`, two 2048-byte dot tiles, `range = 2048 * n_dot_tiles - 1`. +Deliberately *before* weight repacking, deep-K accumulation or `matmul_epilogue` +integration are built on top of it. Operands are symmetric in fp16 mode — both are +2048-byte tiles — unlike the int8 path, where the activation is masked at 2047 and the +weight at 1023. + +## Two real defects, found and fixed + +Both would have cost far more later, and both were found by reading upstream rather than +by guessing at the fault. + +1. **`HMX_SET_BIAS` does not take a bias tile.** It takes a **256-byte** area, not a + 32×32 one. `hmx_init_column_scales` (llama.cpp `hmx-utils.h:19-23`) writes one HVX + vector of per-**column** packed 32-bit words, then one zero vector — and each word is + an fp16 *pair*: low half a multiplicative scale, high half an additive bias. That is + what upstream's `Q6_V_vsplat_R(0x3c00)` means when its comment reads "scale: 1.0, + bias: 0.0 in FP16". The first version passed a 2048-byte 32×32 array, which is a + different operand entirely. + + `docs/hardware/hmx-int8.md` had recorded "the bias tile is a scale, not an additive + bias" from the earlier int8 probing. That was half the story: it is **both**, one pair + per column. + +2. **Tile-sized alignment.** `mxmem` addresses a 2048-byte tile; this project's standard + `HEXLIB_ALIGN` is 128. The first fault was at `badva0=04114a68`, whose low 11 bits are + not zero. + +## The wall, and it is structural + +With both fixed it still faults, and the fault **moved to a stack address** +(`badva0=04115868`) while the tiles are now 2048-aligned — so it is no longer the +operands. Exception code `0x18` in `ssr=80740018`, `ccr=00130000`. + +The reason is not a hypothesis any more. Reading +`../llama.cpp/ggml/src/ggml-hexagon/htp` gives the whole protocol, and **a standalone +simulator ELF — which is how every other kernel here is gated — has none of its four +parts**: + +| | what HMX needs | upstream | in hexlib | +|---|---|---|---| +| 1 | **Power**, separate from HVX's own request and guarded on `__HVX_ARCH__ >= 75`: `HAP_power_set_HMX_v2` with all three DCVS corners at `VCORNER_MAX`, perf mode `HAP_CLK_PERF_HIGH` | `main.c:473-492` | **does not exist** | +| 2 | **Acquisition** in the *same* compute-res attr as VTCM: `HAP_compute_res_attr_set_hmx_param(&attr, 1)` before `HAP_compute_res_acquire` | `main.c:259-291` | `skel_vtcm.c:102` — written, commented *"REVISIT THIS when an HMX kernel first lands"*, and **never once executed** | +| 3 | **An explicit lock** around every use: `HAP_compute_res_hmx_lock` / `_hmx_unlock` | `hmx-queue.c:17-30` | **does not exist** | +| 4 | **A dedicated thread** owning that lock — upstream queues all HMX work to `hmx_queue_thread`, created only `if (n_hmx)` | `main.c:386-394` | n/a | + +The pieces map almost one-to-one onto code that already exists here, which is why this +is a short list rather than a redesign: llama.cpp's +`htp_iface_start(..., n_hvx, n_hmx, max_vmem)` is the *same* signature hexlib ported, and +`n_hmx` was always meant for exactly this — `session.c` passes `0` unconditionally. + +**Consequence: an HMX kernel belongs on the QuRT-hosted batch path, not on `hexlib +test`'s standalone-ELF gate path.** That is a structural difference from all twelve +gated kernels in this repository. They are gated by a program that boots, computes and +prints a verdict with no protection domain around it, and HMX cannot run in one. + +## Open, and deliberately not guessed + +**Is `HAP_compute_res_hmx_lock` thread-scoped?** That decides whether hexlib's +single-threaded skel can hold it inline or has to stand up a queue thread the way +upstream does. Writing the four steps into hexlib before knowing this would be building +on the same kind of assumption that produced both defects above. + +**The cheapest next move** is to wire `n_hmx=1`, add the power request and an inline lock +in `skel_dispatch.c`, and run this existing tile through the **batch** path on the +simulator. If it computes, HMX is unblocked. + +## Why `nearmiss_split_packet.c` is kept + +Nothing can run it yet. It is kept because it preserves the failure +`docs/hardware/hmx-int8.md` spent four probe rounds on: an activation load and a weight +load issued as **two packets instead of one** does not degrade the accumulator, it +**clears** it. That reads as a hardware limitation rather than a coding error, and it is +the kind of thing that is expensive to rediscover. + +## What HMX is and is not for + +It is a **speed** lever, not an accuracy one. The encoder's accuracy problem is the +weight format — see the accuracy table in the top-level `README.md`, where q4_0 costs +0.13 of cosine and fp16 activations cost 9e-06. No amount of HMX moves that. diff --git a/kernels/hmx_matmul_fp16/baseline.c b/kernels/hmx_matmul_fp16/baseline.c new file mode 100644 index 0000000..3273cd3 --- /dev/null +++ b/kernels/hmx_matmul_fp16/baseline.c @@ -0,0 +1,40 @@ +/* kernels/hmx_matmul_fp16/baseline.c + * + * The scalar reference, in the TILED layout the engine reads, so the comparison + * is of arithmetic and not of a layout the baseline invented. + * + * LAYOUT, stated so it can be wrong loudly rather than quietly: both operands + * are `n_dot_tiles` consecutive 32x32 row-major tiles. Activation tile t holds + * a[m][32*t + j] at element (m, j); weight tile t holds b[32*t + j][n] at + * element (j, n). Accumulation runs over all tiles and all 32 elements within + * each, so K = 32 * n_dot_tiles. + * + * If the hardware disagrees -- llama.cpp's own weight repacker mentions a + * "crouton" order with every two rows transposed -- the gate FAILS rather than + * silently comparing two wrong things, which is the point of writing the + * reference against a named layout instead of against the kernel. + */ +#include "kernel_api.h" + +void hmx_matmul_fp16_baseline(const hexlib_hf *act, const hexlib_hf *wt, + const unsigned int *scales, hexlib_hf *out) { + for (int m = 0; m < HMX_MM_M; ++m) { + for (int n = 0; n < HMX_MM_N; ++n) { + /* Word n is the fp16 pair for COLUMN n: low half scale, high half + * bias. Decoded through a union rather than a cast so the fp16 + * bit pattern is read as fp16 and not reinterpreted. */ + union { unsigned short u; __fp16 h; } sc, bi; + sc.u = (unsigned short) (scales[n] & 0xFFFFu); + bi.u = (unsigned short) (scales[n] >> 16); + float sum = 0.0f; + for (int t = 0; t < HMX_MM_DOT_TILES; ++t) { + const hexlib_hf *a = act + (long) t * HMX_TILE_ELMS; + const hexlib_hf *b = wt + (long) t * HMX_TILE_ELMS; + for (int j = 0; j < 32; ++j) { + sum += (float) a[m * 32 + j] * (float) b[j * 32 + n]; + } + } + out[m * HMX_MM_N + n] = (hexlib_hf) (sum * (float) sc.h + (float) bi.h); + } + } +} diff --git a/kernels/hmx_matmul_fp16/harness.c b/kernels/hmx_matmul_fp16/harness.c new file mode 100644 index 0000000..a1b620b --- /dev/null +++ b/kernels/hmx_matmul_fp16/harness.c @@ -0,0 +1,81 @@ +/* kernels/hmx_matmul_fp16/harness.c + * + * THREE THINGS THIS HARNESS DOES DELIBERATELY: + * + * 1. THE INPUT IS NOT UNIFORM. A tile filled with one repeated value gives the + * same answer under ANY permutation of its elements, so a kernel with a + * completely wrong tile layout would pass. Every element here is distinct + * modulo a small period and the two operands use DIFFERENT periods, so a + * transposed tile, a swapped operand pair or a wrong dot-tile stride all + * move the answer. + * + * 2. THE BIAS IS NON-ZERO AND NOT CONSTANT. `bias = mxmem2()` is set once + * before the multiply and it is easy to write a kernel that ignores it and + * still matches on a zero bias. + * + * 3. THE OUTPUT IS POISONED before the call. A kernel that writes nothing -- + * which is exactly what the split-packet failure looks like on the readout + * path -- fails on the poison rather than passing on a zeroed buffer. + * + * TOLERANCE. HMX accumulates internally and narrows once at the store, while + * the baseline accumulates in fp32 and narrows per element. Over K=64 those + * differ by more than one fp16 ULP, so the comparison uses the same + * hexlib_close_f16 (rel 0.02, abs 1e-3) every fp16 kernel here uses. The check + * that a wrong LAYOUT cannot hide inside that tolerance is point 1, not the + * tolerance. + */ +#include "hexlib/hexlib_harness.h" +#include "kernel_api.h" + +/* TILE-SIZED ALIGNMENT, NOT HEXLIB_ALIGN'S 128 BYTES. `mxmem` addresses a + * 2048-byte tile and the first version of this harness used the project's + * standard 128-byte alignment, which faulted with badva0=04114a68 -- an address + * whose low 11 bits are not zero. */ +#define HMX_TILE_ALIGN __attribute__((aligned(2048))) +static hexlib_hf ACT [HMX_MM_DOT_TILES * HMX_TILE_ELMS] HMX_TILE_ALIGN; +static hexlib_hf WT [HMX_MM_DOT_TILES * HMX_TILE_ELMS] HMX_TILE_ALIGN; +/* 256 bytes, per-column (scale, bias) fp16 pairs -- NOT a 32x32 tile. */ +static unsigned int SCALES[HMX_SCALES_WORDS] __attribute__((aligned(256))); +static hexlib_hf OUT [HMX_MM_M * HMX_MM_N] HMX_TILE_ALIGN; +static hexlib_hf REF [HMX_MM_M * HMX_MM_N] HEXLIB_ALIGN; + +int main(void) { + /* Small magnitudes: K=64 accumulations of products must stay inside fp16's + * range, and this kernel is establishing a SEQUENCE, not overflow behaviour. */ + for (int i = 0; i < HMX_MM_DOT_TILES * HMX_TILE_ELMS; ++i) { + ACT[i] = (hexlib_hf) (((float) ((i % 11) - 5)) * 0.125f); + WT[i] = (hexlib_hf) (((float) ((i % 7) - 3)) * 0.250f); + } + /* Scale 1.0 (0x3c00) with a per-column bias that VARIES, so a kernel that + * ignores the scale operand cannot match. Columns beyond 32 are padding. */ + for (int i = 0; i < HMX_SCALES_WORDS; ++i) { + SCALES[i] = 0u; + } + for (int n = 0; n < HMX_MM_N; ++n) { + union { unsigned short u; __fp16 h; } bi; + bi.h = (__fp16) (((float) ((n % 5) - 2)) * 0.5f); + SCALES[n] = 0x3c00u | ((unsigned int) bi.u << 16); + } + for (int i = 0; i < HMX_MM_M * HMX_MM_N; ++i) { + OUT[i] = (hexlib_hf) 12345.0f; + } + + hmx_matmul_fp16_baseline(ACT, WT, SCALES, REF); + + unsigned long long kcyc = 0; + HEXLIB_TIME_KERNEL(kcyc, hmx_matmul_fp16(ACT, WT, SCALES, OUT)); + + int n_wrong = 0; + double max_err = 0.0; + for (int i = 0; i < HMX_MM_M * HMX_MM_N; ++i) { + if (!hexlib_close_f16((float) OUT[i], (float) REF[i], 0.02f, 1e-3f)) { + ++n_wrong; + } + double d = (double) (float) OUT[i] - (double) (float) REF[i]; + if (d < 0.0) d = -d; + if (d > max_err) max_err = d; + } + + hexlib_report(n_wrong == 0, n_wrong, max_err, kcyc); + return 0; +} diff --git a/kernels/hmx_matmul_fp16/kernel.c b/kernels/hmx_matmul_fp16/kernel.c new file mode 100644 index 0000000..5695a8e --- /dev/null +++ b/kernels/hmx_matmul_fp16/kernel.c @@ -0,0 +1,47 @@ +/* kernels/hmx_matmul_fp16/kernel.c + * + * One 32x32 fp16 output tile through the HMX tile engine. See kernel_api.h for + * the sequence and for why the one-packet rule is the whole point. + */ +#include "kernel_api.h" + +/* Transcribed from ../llama.cpp/ggml/src/ggml-hexagon/htp/hmx-utils.h:207-220 + * (MIT; see ATTRIBUTION.md), not paraphrased -- the brace placement is load + * bearing and a reformatting is a behaviour change. */ +#define HMX_LOAD_MPY_DEEP_F16(act, wt, range) \ + "{\n" \ + " activation.hf = mxmem(" act ", " range "):deep\n" \ + " weight.hf = mxmem(" wt ", " range ")\n" \ + "}\n" + +#define HMX_STORE_AFTER_F16(out, scale_reg) \ + "mxmem(" out ", " scale_reg "):after.hf = acc\n" + +#define HMX_SET_BIAS(scales) \ + "bias = mxmem2(" scales ")\n" + +#define HMX_CLRACC_F16() \ + "mxclracc.hf\n" + +void hmx_matmul_fp16(const hexlib_hf *act, const hexlib_hf *wt, + const unsigned int *scales, hexlib_hf *out) { + /* `range` spans the WHOLE dot depth in one instruction: the `:deep` variant + * walks n_dot_tiles consecutive tiles itself rather than needing a loop, so + * long as n_dot_tiles <= 32 (llama.cpp's own bound, matmul-ops.h and the + * `__builtin_assume(n_dot_tiles <= 32)` in core_dot_chunk_fp16_short). The + * encoder's K of 768 and 3072 are 24 and 96 dot tiles, so 3072 will need the + * outer accumulation loop core_dot_chunk_fp16 already shows -- that is the + * next kernel, not this one. */ + const unsigned int range = (unsigned int) (HMX_TILE_BYTES * HMX_MM_DOT_TILES - 1); + + /* BIAS IS SET ONCE, BEFORE the accumulator is cleared and before the + * multiply -- llama.cpp sets it outside both of its tile loops. Setting it + * after mxclracc, or per tile, is a different operation. */ + asm volatile(HMX_SET_BIAS("%0") :: "r"(scales)); + + asm volatile(HMX_CLRACC_F16()); + asm volatile(HMX_LOAD_MPY_DEEP_F16("%1", "%2", "%0") + :: "r"(range), "r"(act), "r"(wt)); + asm volatile(HMX_STORE_AFTER_F16("%0", "%1") + :: "r"(out), "r"(0) : "memory"); +} diff --git a/kernels/hmx_matmul_fp16/kernel_api.h b/kernels/hmx_matmul_fp16/kernel_api.h new file mode 100644 index 0000000..b1db8a7 --- /dev/null +++ b/kernels/hmx_matmul_fp16/kernel_api.h @@ -0,0 +1,123 @@ +/* kernels/hmx_matmul_fp16/kernel_api.h */ +#ifndef HEXLIB_HMX_MATMUL_FP16_API_H +#define HEXLIB_HMX_MATMUL_FP16_API_H + +typedef __fp16 hexlib_hf; + +/* THE FIRST HMX KERNEL IN THIS PROJECT. Deliberately the smallest shape that + * exercises the whole instruction sequence: one 32x32 fp16 output tile, a + * K depth of two 32-element dot tiles. + * + * WHY SO SMALL. Everything downstream -- weight repacking, deep-K accumulation + * past 32 dot tiles, folding into matmul_epilogue -- rests on the tile + * sequence being right, and that sequence has ALREADY been got wrong once in + * this project's own history. `docs/hardware/hmx-int8.md` records four probe + * rounds that concluded the tile engine "could not be made to accumulate" and + * derived a precise, wrong law for it. The cause was issuing the activation and + * weight loads as two separate packets. So this kernel establishes the sequence + * at a shape where a scalar reference is trivially checkable, before anything is + * built on top of it. + * + * THE SEQUENCE, from ../llama.cpp/ggml/src/ggml-hexagon/htp/hmx-utils.h:207-220 + * (MIT; see ATTRIBUTION.md) and its caller `core_dot_chunk_fp16_short` + * (hmx-mm-kernels-tiled.h:604-629): + * + * bias = mxmem2(scales) <- ONCE, before the loops + * mxclracc.hf <- per OUTPUT tile + * { activation.hf = mxmem(act, range):deep + * weight.hf = mxmem(wt, range) } <- ONE PACKET. see below. + * mxmem(out, 0):after.hf = acc + * + * with `range = 2048 * n_dot_tiles - 1`. + * + * THE ONE-PACKET RULE IS THE WHOLE POINT. The braces are not formatting. HMX + * forms the multiply from an activation/weight pair issued together in a single + * instruction packet; split across two packets the accumulator's contribution is + * not degraded, it is CLEARED -- which reads as "the readout works but nothing + * accumulates", exactly the false conclusion the int8 investigation reached. + * `nearmiss_split_packet.c` preserves that failure. + * + * THE "BIAS" OPERAND IS NOT A BIAS TILE, AND GETTING THIS WRONG CRASHED THE + * FIRST VERSION OF THIS KERNEL. `HMX_SET_BIAS` takes a 256-BYTE area, not a + * 32x32 tile: `hmx_init_column_scales` (hmx-utils.h:19-23) writes one HVX + * vector of per-COLUMN packed words then one zero vector. Each 32-bit word is + * an fp16 PAIR -- low half the multiplicative scale, high half the additive + * bias -- which is what upstream's `Q6_V_vsplat_R(0x3c00)` means when its + * comment says "scale: 1.0, bias: 0.0 in FP16" (0x3c00 is fp16 1.0 in the low + * half, 0 in the high). 32 words covers 32 columns = 128 bytes; the second + * vector is padding. + * + * `docs/hardware/hmx-int8.md` recorded "the bias tile is a scale, not an + * additive bias" from int8 probing. That was half of it: it is BOTH, one pair + * per column. + * + * ========================================================================== + * WHY THIS KERNEL CANNOT BE GATED BY `hexlib test`, ESTABLISHED BY READING + * llama.cpp RATHER THAN BY MORE PROBING. + * ========================================================================== + * HMX needs FOUR things before a single `mxmem` will execute, and a standalone + * simulator ELF -- which is what every other kernel in this repository is + * gated as -- has none of them. All four are in + * ../llama.cpp/ggml/src/ggml-hexagon/htp (MIT; see ATTRIBUTION.md): + * + * 1. POWER. `main.c:473-492`, guarded `#if __HVX_ARCH__ >= 75`, and SEPARATE + * from the HVX power request beside it: + * request.type = HAP_power_set_HMX_v2; + * request.hmx_v2.set_power = TRUE; .power_up = TRUE; + * request.hmx_v2.set_clock = TRUE; + * request.hmx_v2.target_corner = HAP_DCVS_EXP_VCORNER_MAX; (min, max too) + * request.hmx_v2.perf_mode = HAP_CLK_PERF_HIGH; + * HAP_power_set(ctx, &request); + * + * 2. ACQUISITION, in the SAME compute-res attr as VTCM, not a separate one + * (`vtcm_alloc`, main.c:259-291): + * HAP_compute_res_attr_set_hmx_param(&attr, 1); + * rctx = HAP_compute_res_acquire(&attr, 1000000); + * + * 3. AN EXPLICIT LOCK around every use (`hmx-queue.c:17-30`): + * HAP_compute_res_hmx_lock(rctx); ... mxmem ... _hmx_unlock(rctx); + * + * 4. A DEDICATED THREAD owning that lock -- upstream queues all HMX work to + * `hmx_queue_thread`, created only `if (n_hmx)` (main.c:386-394). Whether + * the lock is thread-scoped, and therefore whether hexlib's + * single-threaded skel can hold it inline, IS NOT SETTLED and must not be + * assumed either way. + * + * WHAT THAT MEANS FOR hexlib. The pieces map almost one-to-one onto code that + * already exists here, which is why this is a short list and not a redesign: + * + * - `session.c` passes n_hmx = 0 unconditionally today. llama.cpp's + * `htp_iface_start(..., n_hvx, n_hmx, max_vmem)` is the same signature + * hexlib ported, and `n_hmx` was always meant for exactly this. + * - `skel_vtcm.c:102` ALREADY has `if (ctx->n_hmx > 0) + * HAP_compute_res_attr_set_hmx_param(&attr, 1);` -- requirement 2, written + * and never once executed. + * - Requirements 1 and 3 do not exist in hexlib at all. + * + * So an HMX kernel belongs on the QuRT-hosted BATCH path (skel + VTCM + a real + * compute-res context), not on the standalone-ELF gate path. The kernel source + * below is probably fine; it was being run somewhere it can never work. The + * fault it produces there is exception 0x18 with badva on the stack. + * + * TILE GEOMETRY. A 32x32 fp16 tile is 1024 elements and 2048 bytes + * (HTP_MM_HMX_TILE_N_ELMS = 1024, matmul-ops.h:19). Operands are SYMMETRIC in + * fp16 mode -- both 2048 bytes -- unlike the int8 path, where the activation + * tile is masked at 2047 and the weight at 1023. + */ +#define HMX_MM_M 32 /* output rows */ +#define HMX_MM_N 32 /* output cols */ +#define HMX_MM_DOT_TILES 2 /* K = 32 * this */ +#define HMX_MM_K (32 * HMX_MM_DOT_TILES) +#define HMX_TILE_ELMS 1024 +#define HMX_TILE_BYTES 2048 +/* One HVX vector of per-column (scale, bias) words + one vector of padding. */ +#define HMX_SCALES_BYTES 256 +#define HMX_SCALES_WORDS (HMX_SCALES_BYTES / 4) + +void hmx_matmul_fp16(const hexlib_hf *act, const hexlib_hf *wt, + const unsigned int *scales, hexlib_hf *out); + +void hmx_matmul_fp16_baseline(const hexlib_hf *act, const hexlib_hf *wt, + const unsigned int *scales, hexlib_hf *out); + +#endif diff --git a/kernels/hmx_matmul_fp16/nearmiss_split_packet.c b/kernels/hmx_matmul_fp16/nearmiss_split_packet.c new file mode 100644 index 0000000..f459f87 --- /dev/null +++ b/kernels/hmx_matmul_fp16/nearmiss_split_packet.c @@ -0,0 +1,28 @@ +/* kernels/hmx_matmul_fp16/nearmiss_split_packet.c + * + * THE FAILURE THIS PROJECT ALREADY MADE ONCE, PRESERVED. + * + * Identical to kernel.c except the activation and weight loads are issued as + * TWO packets instead of one. `docs/hardware/hmx-int8.md` records four rounds of + * probing that concluded from exactly this that the tile engine "could not be + * made to accumulate", and derived a precise law (`out = bias_high >> 7`) for + * behaviour that was purely an artifact of the split. + * + * It is the most dangerous kind of wrong: it compiles, it runs, it writes an + * output of the right shape, and the readout path works. Only the values are + * wrong, and they are wrong in a way that looks like a hardware limitation + * rather than a coding error. + */ +#include "kernel_api.h" + +void hmx_matmul_fp16(const hexlib_hf *act, const hexlib_hf *wt, + const unsigned int *scales, hexlib_hf *out) { + const unsigned int range = (unsigned int) (HMX_TILE_BYTES * HMX_MM_DOT_TILES - 1); + + asm volatile("bias = mxmem2(%0)\n" :: "r"(scales)); + asm volatile("mxclracc.hf\n"); + /* THE DEFECT: two packets where the working kernel has one set of braces. */ + asm volatile("activation.hf = mxmem(%1, %0):deep\n" :: "r"(range), "r"(act)); + asm volatile("weight.hf = mxmem(%1, %0)\n" :: "r"(range), "r"(wt)); + asm volatile("mxmem(%0, %1):after.hf = acc\n" :: "r"(out), "r"(0) : "memory"); +} diff --git a/kernels/hmx_matmul_fp16/spec.json b/kernels/hmx_matmul_fp16/spec.json new file mode 100644 index 0000000..eea2e8a --- /dev/null +++ b/kernels/hmx_matmul_fp16/spec.json @@ -0,0 +1,19 @@ +{ + "task_id": "hmx_matmul_fp16", + "dtype": "fp16", + "caps": ["hmx"], + "mechanisms": ["hmx"], + "params": { + "M": 32, + "N": 32, + "K": 64, + "n_dot_tiles": 2, + "tile_bytes": 2048, + "range": 4095, + "note": "range = 2048 * n_dot_tiles - 1. Operands are symmetric in fp16 mode (both 2048-byte tiles), unlike the int8 path where the activation is masked at 2047 and the weight at 1023.", + "purpose": "Establish the fp16 HMX tile sequence at a shape whose scalar reference is trivially checkable, before weight repacking, deep-K accumulation or matmul_epilogue integration are built on it." + }, + "expert_kernel_cycles": null, + "tolerance": "hexlib_close_f16", + "tags": ["matmul", "hmx", "tile", "probe", "first-rung"] +} diff --git a/kernels/matmul_epilogue_fp16/RESULT.md b/kernels/matmul_epilogue_fp16/RESULT.md new file mode 100644 index 0000000..36bc353 --- /dev/null +++ b/kernels/matmul_epilogue_fp16/RESULT.md @@ -0,0 +1,18 @@ +### hexlib verify — matmul_epilogue_fp16 + +| gate | result | +|---|---| +| correct | PASS | +| max abs error | 0.000244141 (n_wrong 0) | +| kernel_cycles | 14310406 | +| accel (ELF-proven) | hvx, hvx-compute | +| near-miss `nearmiss_bias_after_activation.c` | correctly rejected | +| near-miss `nearmiss_fp16_accumulation.c` | correctly rejected | +| near-miss `nearmiss_gelu_swap.c` | correctly rejected | +| near-miss `nearmiss_scale_off_by_one_block.c` | correctly rejected | +| near-miss `nearmiss_swapped_nibble_order.c` | correctly rejected | +| **gate** | **PASS** | + +target `v75` · toolchain `19.0.04` · SDK `6.4.0.2` · host `sriha@Heathcliff` · `2026-08-12T05:28:34Z` + +Measured on the hexagon simulator under the pinned bus model (buspenalty 75, busratio 2). The simulator is cycle-approximate; these numbers are reproducible, not silicon measurements. diff --git a/kernels/matmul_epilogue_fp16/baseline.c b/kernels/matmul_epilogue_fp16/baseline.c new file mode 100644 index 0000000..ce2d712 --- /dev/null +++ b/kernels/matmul_epilogue_fp16/baseline.c @@ -0,0 +1,97 @@ +#include "kernel_api.h" + +#include +#include + +/* Scalar reference. Correct and obvious, never fast. + * + * Same contract as kernel.c (see kernel_api.h): fp32 accumulate over K, bias + * added BEFORE the activation, activation computed in fp32 via libm's real + * tanhf/erff (not a polynomial approximation -- this file is what kernel.c's + * own tanh/erf approximations are checked against, so it must not share their + * approximation error), one narrow to fp16 at the very end. + * + * The q4_0 dequant is its OWN copy of the formula (not a call into kernel.c), + * exactly like kernels/softmax_fp16/baseline.c reimplements its own expf loop + * rather than sharing kernel.c's polynomial: a baseline that calls into the + * thing it is checking would not be an independent reference. + * + * THE DEQUANTIZED WEIGHT IS ROUNDED THROUGH fp16 HERE TOO, DELIBERATELY. + * kernel.c's `wbuf` is `hexlib_hf` -- it MUST be, because the only HVX + * multiply primitive available (`Q6_Wqf32_vmpy_VhfVhf`) takes two fp16 + * vectors as input, so every dequantized weight value is rounded to fp16 + * before it is ever multiplied, not just at the final output. That is a + * real extra rounding step this kernel's hardware path pays that a + * hypothetical float32-weight-storage kernel would not, and it must be + * reflected here for this file to be a fair reference for THIS kernel's + * actual numeric contract: a baseline that kept the dequantized weight in + * float32 the whole way through would be measuring a DIFFERENT, more + * accurate algorithm than the one kernel.c implements, and would + * systematically look "wrong" by an amount that has nothing to do with a + * bug -- exactly what was first measured as up to ~2.2e-3 absolute error at + * K=128 before this fix (a few ULP of weight-rounding noise per term, + * accumulated over many terms, is not negligible at the largest K this + * harness tests). + */ +static void mm_dequant_block_baseline(const unsigned char *blk, float *out32) { + __fp16 d; + memcpy(&d, blk, sizeof(d)); + const float df = (float) d; + const unsigned char *qs = blk + 2; + for (int j = 0; j < 16; ++j) { + const int lo = (int) (qs[j] & 0x0F) - 8; + const int hi = (int) ((qs[j] >> 4) & 0x0F) - 8; + out32[j] = (float) (hexlib_hf) (df * (float) lo); + out32[16 + j] = (float) (hexlib_hf) (df * (float) hi); + } +} + +void matmul_epilogue_fp16_baseline(const hexlib_hf *a, const unsigned char *w, + const float *bias, hexlib_hf *out, + int M, int K, int N, int act) { + if (M <= 0 || K <= 0 || N <= 0 || N % MM_Q4_0_BLOCK != 0) { + return; + } + const int nblocks = N / MM_Q4_0_BLOCK; + const long row_stride = (long) nblocks * MM_Q4_0_BLOCK_BYTES; + + float wblock[MM_Q4_0_BLOCK]; + + for (int m = 0; m < M; ++m) { + const hexlib_hf *arow = a + (long) m * K; + hexlib_hf *orow = out + (long) m * N; + + for (int bb = 0; bb < nblocks; ++bb) { + float acc[MM_Q4_0_BLOCK]; + for (int j = 0; j < MM_Q4_0_BLOCK; ++j) { + acc[j] = 0.0f; + } + + const long bb_off = (long) bb * MM_Q4_0_BLOCK_BYTES; + for (int k = 0; k < K; ++k) { + const unsigned char *blk = w + (long) k * row_stride + bb_off; + mm_dequant_block_baseline(blk, wblock); + const float av = (float) arow[k]; + for (int j = 0; j < MM_Q4_0_BLOCK; ++j) { + acc[j] += av * wblock[j]; + } + } + + for (int j = 0; j < MM_Q4_0_BLOCK; ++j) { + float r = acc[j] + bias[(long) bb * MM_Q4_0_BLOCK + j]; + if (act == MM_ACT_GELU_TANH) { + const double x = (double) r; + /* sqrt(2/pi), literal rather than M_PI: M_PI is not + * guaranteed to be declared by without a + * feature-test macro on every toolchain. */ + const double inner = 0.7978845608028654 * (x + 0.044715 * x * x * x); + r = (float) (0.5 * x * (1.0 + tanh(inner))); + } else if (act == MM_ACT_GELU_ERF) { + const double x = (double) r; + r = (float) (0.5 * x * (1.0 + erf(x / sqrt(2.0)))); + } + orow[(long) bb * MM_Q4_0_BLOCK + j] = (hexlib_hf) r; + } + } + } +} diff --git a/kernels/matmul_epilogue_fp16/harness.c b/kernels/matmul_epilogue_fp16/harness.c new file mode 100644 index 0000000..0935ae0 --- /dev/null +++ b/kernels/matmul_epilogue_fp16/harness.c @@ -0,0 +1,308 @@ +/* kernels/matmul_epilogue_fp16/harness.c + * + * Builds inputs, runs the baseline for reference, times ONLY the kernel calls, + * compares with tolerance, and prints the two lines the driver parses. + * + * THREE SHAPES, THREE ACTIVATIONS, ALL DIFFERENT (M, K, N), K A MULTIPLE OF 32: + * + * Test 1: M=8, K=64, N=128, act=none -- K fits in one 64-lane vector + * Test 2: M=12, K=96, N=160, act=gelu_tanh -- K spans 3 blocks of 32 + * Test 3: M=20, K=128, N=96, act=gelu_erf -- K spans 4 blocks of 32 + * + * M, K, N are pairwise distinct within every test (per-test, not just overall) + * specifically so a transposed-operand bug cannot return a same-shape, + * same-size, wrong-content answer the way it could if e.g. M == N. + * + * ========================================================================== + * WHY THESE NUMBERS DISCRIMINATE, NOT JUST "LOOK ADVERSARIAL" -- MEASURED. + * ========================================================================== + * This repo has been burned once already (layernorm_fp16's unbiased-variance + * near-miss, ~0.065% error at C=768, WRONGLY ACCEPTED on its first run because + * that was SMALLER than fp16's own ~0.05% ULP noise). The tolerance below + * (rel=1%, abs=3e-4) and the two adversarial cells that follow were derived + * the same way kernels/softmax_fp16 derived its own: measured in Python + * against the exact algorithm this file and kernel.c implement, not loosened + * until something passed. + * + * CELL 1 -- Test 1, row m=0, output column n=0: THE fp16-ACCUMULATION-OVER-K + * TRIGGER. a[0,k] = 1.0 for every k (see fill_test1). Column 0's dequantized + * weight value at k=0 is exactly 1.0 (scale d=1.0, code 9 -> code-8=1); at + * every k in [1,64) it is exactly 2^-12 = 0.000244140625 (scale d=2^-12, same + * code 9). So the true (real-number) dot product at (m=0, n=0) is exactly + * 1.0 + 63 * 2^-12 = 1.015380859375 + * Measured in Python, running the exact two algorithms: + * - fp32-accumulate-then-narrow-once (this kernel's own contract): result + * 1.015625, error +0.000244140625 (+0.024% relative) -- an ordinary + * single narrowing-to-fp16 rounding, 0.25 ULP at this magnitude + * (ulp(1.0) = 0.0009765625). + * - fp16-running-accumulation (nearmiss_fp16_accumulation.c's bug): every + * one of the 63 additions of 2^-12 is below half the accumulator's own + * ULP once it reaches 1.0, so EVERY one of them rounds back to exactly + * 1.0 and the near-miss's running sum never leaves 1.0. Final result: + * 1.0, error -0.015380859375 (-1.51% relative) -- 63x the correct + * kernel's own noise, and both the 1% relative AND the 3e-4 absolute + * tolerance branches reject it (1.51% > 1%; 0.0156 > 3e-4). + * + * CELL 2 -- Test 3, row m=0: THE gelu_tanh/gelu_erf-SWAP TRIGGER. a[0,k] = 0 + * for every k (all of row 0's activation is zero), so the matmul contributes + * nothing and every output in that row is exactly act(bias[n]) -- a direct + * probe of the activation function alone. bias[0] is set to exactly -2.7f. + * Measured in Python (real math.tanh/math.erf, the same formulas kernel_api.h + * cites): over x in [-8, 8] scanned at 0.01 resolution, the GLOBAL MAXIMUM + * absolute difference between gelu_tanh(x) and gelu_erf(x) is only 4.73e-4, + * at x = -2.70 -- genuinely quiet in absolute terms (kernel_api.h's own + * warning: "these two agree to within ~5e-4 almost everywhere"). But at that + * SAME x, gelu_erf(-2.7) = -0.0093608..., i.e. a SMALL output magnitude, so: + * - fp16 ULP at that output's own magnitude is only 7.6e-6. + * - the tanh/erf formula difference there, in fp16, is 4.73e-4 -- SIXTY-TWO + * of those ULPs, and 5.2% relative to the correct (erf) value. + * - this kernel's OWN legitimate noise at that point (fp32 tanh/erf via + * hvx_vec_exp_f32's measured ~1e-6 relative error, one final narrow) is + * at most a couple of ULPs there, i.e. order 1e-5 -- far under both the + * 1% relative and 3e-4 absolute tolerance branches, while the swapped + * formula (5.2% relative, 4.7e-4 absolute) fails BOTH. + * This is exactly why the tolerance uses an ABSOLUTE bound small enough to + * bite near zero (3e-4, not the 1e-3-or-looser bound that would admit this + * bug by absolute value alone) alongside the RELATIVE bound that does the + * real work away from zero -- see hexlib_close_f16's own OR-of-two-branches + * definition. A single loose absolute tolerance could not do both jobs at + * once; that is the exact trap this comment exists to name. + * + * All other near-misses (swapped nibble order, bias-after-activation, + * scale-off-by-one) are NOT given a dedicated adversarial cell: measured in + * Python against this exact algorithm on ordinary (non-adversarial, formula- + * generated, mixed-sign) weight and activation data, each already disagrees + * with the correct reference on 60-85% of ALL output elements, at a max + * relative error in the hundreds-to-millions-of-percent range (a swapped + * nibble or a scale from the wrong block is not a rounding-sized mistake). + * They do not need to be quiet to be real bugs; only the two above do. + */ +#include "hexlib/hexlib_harness.h" +#include "kernel_api.h" + +#include + +void matmul_epilogue_fp16_baseline(const hexlib_hf *, const unsigned char *, + const float *, hexlib_hf *, + int, int, int, int); + +/* --- q4_0 quantizer, TEST-DATA CONSTRUCTION ONLY ----------------------- + * Not part of the kernel's contract (the kernel and baseline only ever + * DEQUANTIZE). Implements the same reference formula kernel_api.h cites + * (llama.cpp's quantize_row_q4_0_ref, adapted): amax-derived scale, + * round-half-away-from-zero via the classic "+8.5, truncate" trick, clamp to + * [0,15]. Used to build "generic" blocks from a chosen set of 32 target + * float values; the two adversarial cells above are built directly (exact + * scale and code chosen by hand) rather than through this quantizer, because + * they need EXACT dequantized values, not "whatever this round-trips to". + */ +static void mm_quantize_block(const float *vals32, unsigned char *blk18) { + float amax = 0.0f; + for (int j = 0; j < 32; ++j) { + float av = vals32[j] < 0.0f ? -vals32[j] : vals32[j]; + if (av > amax) amax = av; + } + __fp16 dh = (__fp16) (amax / -8.0f); + float id = (float) dh != 0.0f ? 1.0f / (float) dh : 0.0f; + memcpy(blk18, &dh, sizeof(dh)); + for (int j = 0; j < 16; ++j) { + int lo = (int) (vals32[j] * id + 8.5f); + int hi = (int) (vals32[j + 16] * id + 8.5f); + if (lo < 0) lo = 0; if (lo > 15) lo = 15; + if (hi < 0) hi = 0; if (hi > 15) hi = 15; + blk18[2 + j] = (unsigned char) (((hi & 0x0F) << 4) | (lo & 0x0F)); + } +} + +/* Build one block by hand: exact scale, exact 32 codes. Used for the two + * adversarial cells, where the test needs an EXACT dequantized value rather + * than whatever mm_quantize_block happens to round a target to. */ +static void mm_build_block(float d, const int codes[32], unsigned char *blk18) { + __fp16 dh = (__fp16) d; + memcpy(blk18, &dh, sizeof(dh)); + for (int j = 0; j < 16; ++j) { + int lo = codes[j] & 0x0F; + int hi = codes[j + 16] & 0x0F; + blk18[2 + j] = (unsigned char) ((hi << 4) | lo); + } +} + +/* ======================= Test 1: M=8, K=64, N=128, act=none ============ */ +#define T1_M 8 +#define T1_K 64 +#define T1_N 128 +#define T1_NBLOCKS (T1_N / MM_Q4_0_BLOCK) +#define T1_ROWSTRIDE (T1_NBLOCKS * MM_Q4_0_BLOCK_BYTES) + +static hexlib_hf T1_A[T1_M * T1_K] HEXLIB_ALIGN; +static unsigned char T1_W[T1_K * T1_ROWSTRIDE] HEXLIB_ALIGN; +static float T1_BIAS[T1_N] HEXLIB_ALIGN; +static hexlib_hf T1_OUT[T1_M * T1_N] HEXLIB_ALIGN; +static hexlib_hf T1_REF[T1_M * T1_N] HEXLIB_ALIGN; + +static void fill_test1(void) { + for (int m = 0; m < T1_M; ++m) { + for (int k = 0; k < T1_K; ++k) { + /* Row 0 is the fp16-accumulation stress row: a[0,k] = 1.0 for + * every k, so the dot product at column 0 is a direct sum of + * that column's dequantized weight values (see CELL 1 above). */ + float v = (m == 0) ? 1.0f : 0.05f * (float) (((m * 7 + k * 3) % 23) - 11); + T1_A[m * T1_K + k] = (hexlib_hf) v; + } + } + for (int n = 0; n < T1_N; ++n) { + T1_BIAS[n] = 0.01f * (float) (((n * 5) % 19) - 9); + } + + for (int k = 0; k < T1_K; ++k) { + for (int bb = 0; bb < T1_NBLOCKS; ++bb) { + unsigned char *blk = T1_W + (long) k * T1_ROWSTRIDE + (long) bb * MM_Q4_0_BLOCK_BYTES; + if (bb == 0) { + /* CELL 1: column 0 (index 0 of this block) carries the + * stress value; the other 31 codes in the block are a + * generic varying pattern sharing the same scale. */ + int codes[32]; + for (int j = 0; j < 32; ++j) { + codes[j] = (j * 5 + k * 3) % 16; + } + codes[0] = 9; /* code-8 = 1 */ + float d = (k == 0) ? 1.0f : 0.000244140625f; /* 1.0 or 2^-12 */ + mm_build_block(d, codes, blk); + } else { + float vals[32]; + for (int j = 0; j < 32; ++j) { + vals[j] = 0.3f * (float) (((k * 3 + bb * 7 + j) % 13) - 6); + } + mm_quantize_block(vals, blk); + } + } + } +} + +/* ================= Test 2: M=12, K=96, N=160, act=gelu_tanh ============ */ +#define T2_M 12 +#define T2_K 96 +#define T2_N 160 +#define T2_NBLOCKS (T2_N / MM_Q4_0_BLOCK) +#define T2_ROWSTRIDE (T2_NBLOCKS * MM_Q4_0_BLOCK_BYTES) + +static hexlib_hf T2_A[T2_M * T2_K] HEXLIB_ALIGN; +static unsigned char T2_W[T2_K * T2_ROWSTRIDE] HEXLIB_ALIGN; +static float T2_BIAS[T2_N] HEXLIB_ALIGN; +static hexlib_hf T2_OUT[T2_M * T2_N] HEXLIB_ALIGN; +static hexlib_hf T2_REF[T2_M * T2_N] HEXLIB_ALIGN; + +static void fill_test2(void) { + for (int m = 0; m < T2_M; ++m) { + for (int k = 0; k < T2_K; ++k) { + float v = 0.04f * (float) (((m * 11 + k * 5) % 29) - 14); + T2_A[m * T2_K + k] = (hexlib_hf) v; + } + } + for (int n = 0; n < T2_N; ++n) { + T2_BIAS[n] = 0.02f * (float) (((n * 7) % 23) - 11); + } + for (int k = 0; k < T2_K; ++k) { + for (int bb = 0; bb < T2_NBLOCKS; ++bb) { + unsigned char *blk = T2_W + (long) k * T2_ROWSTRIDE + (long) bb * MM_Q4_0_BLOCK_BYTES; + float vals[32]; + for (int j = 0; j < 32; ++j) { + vals[j] = 0.25f * (float) (((k * 5 + bb * 3 + j * 2) % 17) - 8); + } + mm_quantize_block(vals, blk); + } + } +} + +/* ================= Test 3: M=20, K=128, N=96, act=gelu_erf ============= */ +#define T3_M 20 +#define T3_K 128 +#define T3_N 96 +#define T3_NBLOCKS (T3_N / MM_Q4_0_BLOCK) +#define T3_ROWSTRIDE (T3_NBLOCKS * MM_Q4_0_BLOCK_BYTES) + +static hexlib_hf T3_A[T3_M * T3_K] HEXLIB_ALIGN; +static unsigned char T3_W[T3_K * T3_ROWSTRIDE] HEXLIB_ALIGN; +static float T3_BIAS[T3_N] HEXLIB_ALIGN; +static hexlib_hf T3_OUT[T3_M * T3_N] HEXLIB_ALIGN; +static hexlib_hf T3_REF[T3_M * T3_N] HEXLIB_ALIGN; + +static void fill_test3(void) { + for (int m = 0; m < T3_M; ++m) { + for (int k = 0; k < T3_K; ++k) { + /* Row 0 is the gelu_tanh/gelu_erf-swap stress row: a[0,k] = 0 for + * every k, so every output in that row is exactly act(bias[n]) + * (see CELL 2 above). */ + float v = (m == 0) ? 0.0f : 0.03f * (float) (((m * 13 + k * 3) % 31) - 15); + T3_A[m * T3_K + k] = (hexlib_hf) v; + } + } + for (int n = 0; n < T3_N; ++n) { + T3_BIAS[n] = 0.05f * (float) (((n * 9) % 27) - 13); + } + T3_BIAS[0] = -2.7f; /* CELL 2: exact probe point, see header comment */ + + for (int k = 0; k < T3_K; ++k) { + for (int bb = 0; bb < T3_NBLOCKS; ++bb) { + unsigned char *blk = T3_W + (long) k * T3_ROWSTRIDE + (long) bb * MM_Q4_0_BLOCK_BYTES; + float vals[32]; + for (int j = 0; j < 32; ++j) { + vals[j] = 0.2f * (float) (((k * 7 + bb * 5 + j * 3) % 19) - 9); + } + mm_quantize_block(vals, blk); + } + } +} + +/* rel=1%, abs=3e-4: derived above from measured numbers, not loosened until + * something passed. See the header comment for CELL 1 and CELL 2's numbers. */ +#define MM_REL_TOL 0.01f +#define MM_ABS_TOL 3e-4f + +static void poison(hexlib_hf *buf, int n) { + for (int i = 0; i < n; ++i) { + buf[i] = (hexlib_hf) 12345.0f; + } +} + +int main(void) { + fill_test1(); + fill_test2(); + fill_test3(); + poison(T1_OUT, T1_M * T1_N); + poison(T2_OUT, T2_M * T2_N); + poison(T3_OUT, T3_M * T3_N); + + matmul_epilogue_fp16_baseline(T1_A, T1_W, T1_BIAS, T1_REF, T1_M, T1_K, T1_N, MM_ACT_NONE); + matmul_epilogue_fp16_baseline(T2_A, T2_W, T2_BIAS, T2_REF, T2_M, T2_K, T2_N, MM_ACT_GELU_TANH); + matmul_epilogue_fp16_baseline(T3_A, T3_W, T3_BIAS, T3_REF, T3_M, T3_K, T3_N, MM_ACT_GELU_ERF); + + unsigned long long kcyc1 = 0, kcyc2 = 0, kcyc3 = 0; + HEXLIB_TIME_KERNEL(kcyc1, matmul_epilogue_fp16(T1_A, T1_W, T1_BIAS, T1_OUT, T1_M, T1_K, T1_N, MM_ACT_NONE)); + HEXLIB_TIME_KERNEL(kcyc2, matmul_epilogue_fp16(T2_A, T2_W, T2_BIAS, T2_OUT, T2_M, T2_K, T2_N, MM_ACT_GELU_TANH)); + HEXLIB_TIME_KERNEL(kcyc3, matmul_epilogue_fp16(T3_A, T3_W, T3_BIAS, T3_OUT, T3_M, T3_K, T3_N, MM_ACT_GELU_ERF)); + unsigned long long kcyc = kcyc1 + kcyc2 + kcyc3; + + int n_wrong = 0; + double max_err = 0.0; + + hexlib_hf *outs[3] = { T1_OUT, T2_OUT, T3_OUT }; + hexlib_hf *refs[3] = { T1_REF, T2_REF, T3_REF }; + int counts[3] = { T1_M * T1_N, T2_M * T2_N, T3_M * T3_N }; + + for (int t = 0; t < 3; ++t) { + for (int i = 0; i < counts[t]; ++i) { + float yv = (float) outs[t][i]; + float rv = (float) refs[t][i]; + if (!hexlib_close_f16(yv, rv, MM_REL_TOL, MM_ABS_TOL)) { + ++n_wrong; + } + double d = (double) yv - (double) rv; + if (d < 0.0) d = -d; + if (d > max_err) max_err = d; + } + } + + hexlib_report(n_wrong == 0, n_wrong, max_err, kcyc); + return 0; +} diff --git a/kernels/matmul_epilogue_fp16/kernel.c b/kernels/matmul_epilogue_fp16/kernel.c new file mode 100644 index 0000000..ac7a53d --- /dev/null +++ b/kernels/matmul_epilogue_fp16/kernel.c @@ -0,0 +1,268 @@ +/* matmul + bias + optional activation, q4_0 block-quantized weight. + * See kernel_api.h for the full spec, citations, and the layout/precision + * decisions this file implements. + * + * STRUCTURE. One q4_0 block (32 output columns) is processed per inner + * iteration: for a fixed row m and a fixed block of 32 output columns, the + * kernel walks all of K, dequantizing that row's block to fp16, multiplying + * by the broadcast activation scalar a[m,k], and accumulating in fp32; after + * the K loop it adds the bias, applies the activation, and narrows once. This + * is deliberately the SIMPLEST correct structure, not the fastest one: the + * weight block at (k, block) is re-dequantized once per output ROW m rather + * than once and reused across all M rows. Fixing that (dequantize each (k, + * block) once, hold the M partial sums live across a re-ordered loop) is the + * obvious next optimisation and is not attempted here -- see kernel_api.h's + * "WHY HVX-COMPUTE, NOT HMX" for the same "first rung, not a result" framing + * kernels/softmax_fp16 and kernels/layernorm_fp16 use for their own + * unfinished optimisations. + * + * LANE ORDER: THE SAME SHUFFLE-BEFORE / DEAL-AFTER CONVENTION AS + * kernels/layernorm_fp16/kernel.c AND kernels/rope_2d_fp16/kernel.c. + * `Q6_Wqf32_vmpy_VhfVhf` (the only fp16->qf32 widening primitive) and + * `Q6_Vhf_equals_Wqf32` (the only qf32->fp16 narrowing primitive) both + * PERMUTE lanes -- element k does not land in lane k (verified against those + * two kernels' own header comments, same toolchain, same arch). That is + * invisible to a PURE elementwise op with no other operand (add_fp16, + * scale_fp16 use the raw primitives directly and get away with it, because + * widen-then-narrow with no shuffle in between is its own inverse). It is + * NOT invisible here: after the K-reduction, this kernel adds a bias vector + * that was loaded PLAINLY from memory (true column order), so the + * accumulated qf32 partial sums MUST already be in true column order before + * that add, or bias[n] would land on the wrong accumulated column -- a + * correctly-shaped, silently-wrong answer no shape check would catch. So + * every widen here shuffles its non-uniform operand first (`widen_mul_ + * ordered`, adapted from layernorm_fp16's `widen_ordered`), and the one + * narrow at the very end deals after narrowing (`narrow_ordered`, copied + * from the same file, cited there as line-by-line verified in kernels/ + * softmax_fp16/kernel.c's own header comment). + * + * EXP / TANH / ERF: hvx_vec_exp_f32, NEVER hvx_vec_exp2_f16. hvx_vec_exp2_f16 + * (hvx-exp.h) has a wrong E5 coefficient (0x5082 where upstream calls for + * 0x090c, 262% error at fractional input 0.7 -- see kernels/softmax_fp16's + * header for the same finding) and this kernel's gelu_tanh needs a tanh, + * which this file builds from `hvx_vec_exp_f32` directly (tanh(u) = + * (1-exp(-2|u|))/(1+exp(-2|u|)), sign copied back in) rather than via + * hvx-sigmoid.h's `hvx_vec_fast_sigmoid_f16`/`hvx_vec_tanh_f16`, which route + * through the broken `hvx_vec_exp2_f16`. `hvx_vec_exp_f32` itself is + * measured at ~1e-6 relative over the input range that matters (kernels/ + * softmax_fp16/kernel.c's header). gelu_erf uses the Abramowitz-Stegun 7.1.26 + * rational approximation to erf (max absolute error in erf itself: 1.5e-7, + * a standard published approximation, not derived here), also built on + * `hvx_vec_exp_f32`. The two activations therefore share the same exp + * primitive but are otherwise genuinely different formulas, per kernel_api.h + * -- see nearmiss_gelu_swap.c for what happens if they are swapped. + * + * NO Q6_Vhf_vadd_VhfVhf anywhere (does not exist on v75, crashes clang + * 19.0.04 exit code 70 -- add_fp16/kernel.c's header). All fp16/fp32 + * arithmetic here goes through the qf32 path via hvx-base.h's helpers or the + * hand-rolled widen/narrow below. + */ +#include "kernel_api.h" + +#include +#include +#include + +#include "hexlib/hvx/hvx-base.h" +#include "hexlib/hvx/hvx-exp.h" +#include "hexlib/hvx/hvx-inverse.h" + +#define LANES_FP16 64 +#define LANES_FP32 32 + +/* --- q4_0 dequant, one 32-element block, scalar ----------------------- + * + * blk: 18 bytes -- fp16 scale d, then 16 bytes of packed nibbles. + * out32: 32 hexlib_hf, written in full (indices [0,32)). + * + * Formula from kernel_api.h's SPEC SOURCE 3 (adapted from llama.cpp's + * dequantize_row_q4_0, cited there): low nibble of byte j -> index j, high + * nibble of byte j -> index j+16. NOT interleaved. See + * nearmiss_swapped_nibble_order.c. + */ +static inline void mm_dequant_block(const unsigned char *blk, hexlib_hf *out32) { + __fp16 d; + memcpy(&d, blk, sizeof(d)); + const float df = (float) d; + const unsigned char *qs = blk + 2; + for (int j = 0; j < 16; ++j) { + const int lo = (int) (qs[j] & 0x0F) - 8; + const int hi = (int) ((qs[j] >> 4) & 0x0F) - 8; + out32[j] = (hexlib_hf) (df * (float) lo); + out32[16 + j] = (hexlib_hf) (df * (float) hi); + } +} + +/* --- ordered widen/narrow, adapted from kernels/layernorm_fp16/kernel.c --- + * (that file's own header explains why the shuffle/deal is needed; see this + * file's header for why it applies here too). + */ + +/* (x_bcast_fp16 * y_fp16), widened to fp32, TRUE element order. + * x is assumed UNIFORM across all 64 lanes (a broadcast scalar), so it needs + * no shuffle of its own -- shuffling a vector whose lanes are all equal is a + * no-op, exactly how layernorm_fp16's widen_ordered treats its `one` operand. + * out[0] = elements [0,32) of x*y as fp32; out[1] = elements [32,64). */ +static inline void mm_widen_mul_ordered(HVX_Vector x_bcast, HVX_Vector y, + HVX_Vector *out) { + HVX_VectorPair p = Q6_Wqf32_vmpy_VhfVhf(Q6_Vh_vshuff_Vh(y), x_bcast); + out[0] = Q6_Vsf_equals_Vqf32(Q6_V_lo_W(p)); + out[1] = Q6_Vsf_equals_Vqf32(Q6_V_hi_W(p)); +} + +/* Two IEEE fp32 vectors (TRUE element order) -> one fp16 vector (TRUE order). */ +static inline HVX_Vector mm_narrow_ordered(HVX_Vector lo, HVX_Vector hi) { + const HVX_Vector zero = Q6_V_vzero(); + HVX_Vector qlo = Q6_Vqf32_vadd_VsfVsf(lo, zero); + HVX_Vector qhi = Q6_Vqf32_vadd_VsfVsf(hi, zero); + return Q6_Vh_vdeal_Vh(Q6_Vhf_equals_Wqf32(Q6_W_vcombine_VV(qhi, qlo))); +} + +/* One fp16 value splatted to all 64 lanes, as a bit pattern (scale_fp16's + * own splat_fp16, same reasoning: Q6_Vh_vsplat_R wants the bit pattern, a + * numeric cast would splat the wrong thing). */ +static inline HVX_Vector mm_splat_hf(float v) { + /* float, not hexlib_hf: hexagon-clang rejects __fp16 as a by-value + * parameter outright (see include/hexlib/hexlib_harness.h's own note + * on the same restriction). */ + union { hexlib_hf h; unsigned short u; } bits; + bits.h = (hexlib_hf) v; + return Q6_Vh_vsplat_R((int) bits.u); +} + +/* --- tanh and erf, both built on hvx_vec_exp_f32 (fp32, accurate) ------ */ + +/* tanh(u) = (1 - e)/(1 + e), e = exp(-2*|u|); sign copied back in bitwise. + * Numerically stable for either sign: e is always in (0, 1]. */ +static inline HVX_Vector mm_tanh_f32(HVX_Vector u) { + HVX_Vector absu = hvx_vec_abs_f32(u); + HVX_Vector e = hvx_vec_exp_f32(hvx_vec_mul_f32_f32(absu, hvx_vec_splat_f32(-2.0f))); + HVX_Vector num = hvx_vec_sub_f32_f32(hvx_vec_splat_f32(1.0f), e); + HVX_Vector den = hvx_vec_add_f32_f32(hvx_vec_splat_f32(1.0f), e); + HVX_Vector t = hvx_vec_mul_f32_f32(num, hvx_vec_inverse_f32(den)); /* tanh(|u|), >= 0 */ + HVX_Vector sign_bits = Q6_V_vand_VV(u, Q6_V_vsplat_R(0x80000000)); + return Q6_V_vor_VV(t, sign_bits); +} + +/* gelu_tanh(x) = 0.5*x*(1 + tanh(sqrt(2/pi)*(x + 0.044715*x^3))). + * kernel_api.h SPEC SOURCE 2 / elementwise.py:110-114. */ +static inline HVX_Vector mm_gelu_tanh_f32(HVX_Vector x) { + HVX_Vector x2 = hvx_vec_mul_f32_f32(x, x); + HVX_Vector x3 = hvx_vec_mul_f32_f32(x2, x); + HVX_Vector inner = hvx_vec_add_f32_f32( + x, hvx_vec_mul_f32_f32(x3, hvx_vec_splat_f32(0.044715f))); + inner = hvx_vec_mul_f32_f32(inner, hvx_vec_splat_f32(0.7978845608028654f)); + HVX_Vector t = mm_tanh_f32(inner); + HVX_Vector one_plus_t = hvx_vec_add_f32_f32(hvx_vec_splat_f32(1.0f), t); + HVX_Vector half_x = hvx_vec_mul_f32_f32(x, hvx_vec_splat_f32(0.5f)); + return hvx_vec_mul_f32_f32(half_x, one_plus_t); +} + +/* erf(x), x>=0 form via Abramowitz & Stegun 7.1.26 (published rational + * approximation, max abs error 1.5e-7 in erf itself): t=1/(1+p*x), + * erf(x) = 1 - (a1*t+a2*t^2+a3*t^3+a4*t^4+a5*t^5)*exp(-x^2). Horner form + * below; sign copied back in bitwise for x<0 (erf is odd). */ +static inline HVX_Vector mm_erf_f32(HVX_Vector x) { + HVX_Vector absx = hvx_vec_abs_f32(x); + HVX_Vector denom = hvx_vec_add_f32_f32( + hvx_vec_splat_f32(1.0f), + hvx_vec_mul_f32_f32(hvx_vec_splat_f32(0.3275911f), absx)); + HVX_Vector t = hvx_vec_inverse_f32(denom); + + HVX_Vector poly = hvx_vec_splat_f32(1.061405429f); + poly = hvx_vec_mul_f32_f32(poly, t); + poly = hvx_vec_add_f32_f32(poly, hvx_vec_splat_f32(-1.453152027f)); + poly = hvx_vec_mul_f32_f32(poly, t); + poly = hvx_vec_add_f32_f32(poly, hvx_vec_splat_f32(1.421413741f)); + poly = hvx_vec_mul_f32_f32(poly, t); + poly = hvx_vec_add_f32_f32(poly, hvx_vec_splat_f32(-0.284496736f)); + poly = hvx_vec_mul_f32_f32(poly, t); + poly = hvx_vec_add_f32_f32(poly, hvx_vec_splat_f32(0.254829592f)); + poly = hvx_vec_mul_f32_f32(poly, t); + + HVX_Vector neg_x2 = hvx_vec_mul_f32_f32( + hvx_vec_mul_f32_f32(absx, absx), hvx_vec_splat_f32(-1.0f)); + HVX_Vector exp_neg_x2 = hvx_vec_exp_f32(neg_x2); + HVX_Vector erf_abs = hvx_vec_sub_f32_f32(hvx_vec_splat_f32(1.0f), + hvx_vec_mul_f32_f32(poly, exp_neg_x2)); + HVX_Vector sign_bits = Q6_V_vand_VV(x, Q6_V_vsplat_R(0x80000000)); + return Q6_V_vor_VV(erf_abs, sign_bits); +} + +/* gelu_erf(x) = 0.5*x*(1 + erf(x/sqrt(2))). + * kernel_api.h SPEC SOURCE 2 / elementwise.py:117-125. */ +static inline HVX_Vector mm_gelu_erf_f32(HVX_Vector x) { + HVX_Vector arg = hvx_vec_mul_f32_f32(x, hvx_vec_splat_f32(0.7071067811865476f)); + HVX_Vector e = mm_erf_f32(arg); + HVX_Vector one_plus_e = hvx_vec_add_f32_f32(hvx_vec_splat_f32(1.0f), e); + HVX_Vector half_x = hvx_vec_mul_f32_f32(x, hvx_vec_splat_f32(0.5f)); + return hvx_vec_mul_f32_f32(half_x, one_plus_e); +} + +void matmul_epilogue_fp16(const hexlib_hf *a, const unsigned char *w, + const float *bias, hexlib_hf *out, + int M, int K, int N, int act) { + if (M <= 0 || K <= 0 || N <= 0) { + return; + } + /* ir.nbytes (ir.py:47) already refused a caller whose N is not a + * multiple of 32 before this weight buffer could even be constructed; + * this is a defensive check on the kernel's own contract, not a new + * requirement. */ + if (N % MM_Q4_0_BLOCK != 0) { + return; + } + + const int nblocks = N / MM_Q4_0_BLOCK; + const long row_stride = (long) nblocks * MM_Q4_0_BLOCK_BYTES; + + /* Scratch for one dequantized 32-column block, upper half of the 64-lane + * fp16 vector left at zero: only ONE block (32 real columns) is live at a + * time, so the widen-multiply's high 32 lanes always see 0 * anything = + * 0 and never contribute to the accumulator that gets read out below. + * Zeroed once, outside every loop -- mm_dequant_block only ever writes + * indices [0,32). */ + hexlib_hf wbuf[LANES_FP16] __attribute__((aligned(128))); + for (int j = MM_Q4_0_BLOCK; j < LANES_FP16; ++j) { + wbuf[j] = (hexlib_hf) 0.0f; + } + HVX_Vector *wv_slot = (HVX_Vector *) wbuf; + + for (int m = 0; m < M; ++m) { + const hexlib_hf *arow = a + (long) m * K; + hexlib_hf *orow = out + (long) m * N; + + for (int bb = 0; bb < nblocks; ++bb) { + const long bb_off = (long) bb * MM_Q4_0_BLOCK_BYTES; + HVX_Vector acc_lo = Q6_V_vzero(); + + for (int k = 0; k < K; ++k) { + const unsigned char *blk = w + (long) k * row_stride + bb_off; + mm_dequant_block(blk, wbuf); + HVX_Vector wvec = *wv_slot; + HVX_Vector abcast = mm_splat_hf(arow[k]); + + HVX_Vector prod[2]; + mm_widen_mul_ordered(abcast, wvec, prod); + acc_lo = hvx_vec_add_f32_f32(acc_lo, prod[0]); + /* prod[1] (elements [32,64), always the product of `abcast` + * with the zeroed high half of wbuf) is exactly zero and is + * never read -- see the wbuf comment above. */ + } + + HVX_Vector biasv = hvx_vmemu(bias + (long) bb * MM_Q4_0_BLOCK); + HVX_Vector r = hvx_vec_add_f32_f32(acc_lo, biasv); + + if (act == MM_ACT_GELU_TANH) { + r = mm_gelu_tanh_f32(r); + } else if (act == MM_ACT_GELU_ERF) { + r = mm_gelu_erf_f32(r); + } + /* act == MM_ACT_NONE: r is already the value to store. */ + + HVX_Vector outv = mm_narrow_ordered(r, Q6_V_vzero()); + hvx_vec_store_u(orow + (long) bb * MM_Q4_0_BLOCK, + MM_Q4_0_BLOCK * (uint32_t) sizeof(hexlib_hf), outv); + } + } +} diff --git a/kernels/matmul_epilogue_fp16/kernel_api.h b/kernels/matmul_epilogue_fp16/kernel_api.h new file mode 100644 index 0000000..89e768b --- /dev/null +++ b/kernels/matmul_epilogue_fp16/kernel_api.h @@ -0,0 +1,192 @@ +/* kernels/matmul_epilogue_fp16/kernel_api.h */ +#ifndef HEXLIB_MATMUL_EPILOGUE_FP16_API_H +#define HEXLIB_MATMUL_EPILOGUE_FP16_API_H + +typedef __fp16 hexlib_hf; + +/* matmul + bias + optional activation, with a q4_0 block-quantized weight. + * 75 of the encoder's 259 real-work plan steps are this op (48x (256,768)x + * (768,768), 12x (256,768)x(768,3072) act=gelu_tanh, 12x (256,3072)x(3072,768), + * 1x (256,1536)x(1536,768), 1x (64,3072)x(3072,3072) act=gelu_erf, 1x + * (64,3072)x(3072,1024)) -- more than a quarter of the whole encoder's work, + * and 55.9 of its 58.6 MB of weights. + * + * ========================================================================== + * SPEC SOURCE 1: the fused op's own reference, ORDER OF OPERATIONS. + * ========================================================================== + * hexlib/graph/opdefs/fused.py:44-53 (`_reference`): + * + * out = (a @ b + bias).astype(a.dtype) <- BIAS ADDED, THEN CAST + * if act == "none": return out + * return get(act).reference((out,), {}) <- ACTIVATION APPLIED AFTER + * + * i.e. for every element: y[m,n] = act(sum_k a[m,k]*w[k,n] + bias[n]). + * BIAS IS ADDED BEFORE THE ACTIVATION, NEVER AFTER -- this is the one order + * `fuse.py` (hexlib/graph/fuse.py:13,61-76) is even ALLOWED to produce: it only + * folds a matmul -> add(bias) -> {gelu_tanh,gelu_erf} chain, bias-then-act by + * construction. See nearmiss_bias_after_activation.c for what applying it in + * the other order does to the output. + * + * ========================================================================== + * SPEC SOURCE 2: the two activation formulas (DIFFERENT FUNCTIONS). + * ========================================================================== + * hexlib/graph/opdefs/elementwise.py:110-125. gelu_tanh and gelu_erf are two + * different functions used in two different places in the model (line 4 of + * that file's own header), not one op with a flag: + * + * gelu_tanh(x) = 0.5*x*(1 + tanh( sqrt(2/pi) * (x + 0.044715*x^3) )) + * -- ACT2FN["gelu_pytorch_tanh"], the blocks' MLP + * (modeling_qwen3_5.py:849, cited at elementwise.py:111) + * gelu_erf(x) = 0.5*x*(1 + erf(x / sqrt(2))) + * -- plain nn.GELU(), approximate='none', the merger + * (modeling_qwen3_5.py:882, cited at elementwise.py:118-122) + * + * These two agree to within ~5e-4 absolute almost everywhere (see harness.c's + * header for the measured max, ~4.7e-4 at x ~= -2.7) -- a genuinely quiet + * near-miss if the two formulas are ever swapped for each other. See + * nearmiss_gelu_swap.c and harness.c for how this kernel's tolerance is shown + * to still catch it. + * + * ========================================================================== + * SPEC SOURCE 3: the q4_0 block format. + * ========================================================================== + * hexlib/graph/ir.py:20-23 (`Q4_0_BLOCK = 32`, `Q4_0_BLOCK_BYTES = 18`): + * "32 four-bit values (16 bytes) + one fp16 scale (2 bytes) = 18 bytes... + * matches llama.cpp's block_q4_0." Dequant formula (adapted, not copied, from + * llama.cpp's `dequantize_row_q4_0`, `ggml-quants.c`, per + * docs/research/quantization.md section 1, "Q4_0 -- plain affine RTN"): + * + * for a block of 32 values with scale d and 16 code bytes qs[0..15]: + * for j in [0, 16): + * value[j] = d * ((qs[j] & 0x0F) - 8) <- LOW nibble -> index j + * value[j+16] = d * ((qs[j] >> 4) - 8) <- HIGH nibble -> index j+16 + * + * NOT interleaved (2*j, 2*j+1). See nearmiss_swapped_nibble_order.c for what + * reading it the other way does. + * + * ========================================================================== + * THE LAYOUT DECISION THIS KERNEL COMMITS TO: PLAIN ROW-MAJOR q4_0, NOT + * ggml-hexagon's 576-BYTE REPACKED TILE ORDER. + * ========================================================================== + * hexlib_dsp.h defines HEXLIB_LAYOUT_ROW_MAJOR (0) and HEXLIB_LAYOUT_ + * Q4_0_REPACKED (2) as distinct enum values precisely so "un-repacked weights + * are a plan-time error rather than silent corruption" -- this kernel expects + * HEXLIB_LAYOUT_ROW_MAJOR for its weight operand. + * + * The weight tensor's LOGICAL shape is (K, N) -- reduction dim first, output + * dim second, numpy/C row-major -- exactly as `hexlib/exec/runner.py`'s own + * q4_0 staging tests declare it (ENCODER_WEIGHT_SHAPES in + * test_raw_q4_0_staging.py: (768,768), (768,3072), (3072,768), (1536,768), + * (3072,3072), (3072,1024), all (K,N)). `ir.nbytes` requires shape[-1] % 32 == + * 0 (ir.py:47), i.e. the LAST axis -- N, the output/free dimension -- is the + * one split into 32-element blocks, not K. Concretely: row k of the weight + * occupies `(N/32)*18` contiguous bytes; block b of row k (bytes + * `[b*18, b*18+18)` within that row) covers output columns `[b*32, b*32+32)`, + * ALL AT THE SAME REDUCTION INDEX k, sharing one scale. + * + * WHY THIS AXIS, NOT llama.cpp's: llama.cpp's own Linear weight is stored + * (out_features, in_features) and blocks along in_features (the reduction + * axis) because its consumer is HMX/GEMM hardware that wants a scale + * per reduction-strip. This kernel does the opposite deliberately: blocking + * along N means that for a FIXED k, one 18-byte block dequantizes directly + * into a 32-lane HVX vector representing 32 *output columns*, which is + * multiplied by the single broadcast scalar a[m,k] and accumulated into a + * 32-lane fp32 partial-output vector -- exactly the per-k multiply-accumulate + * structure this (HVX-compute, not HMX) kernel is built around. See + * "WHY HVX-COMPUTE, NOT HMX" below. + * + * PLAIN ROW-MAJOR, NOT REPACKED: this kernel reads the 18-byte blocks in their + * natural (k, block-of-32-columns) order, exactly as `ir.nbytes` lays them + * out. It does NOT expect llama.cpp/ggml-hexagon's 32x32-tile, 576-byte + * repacked order (`docs/research/quantization.md` section 0, `repack_q4_0_ + * tiled`) -- that repacking exists to feed HMX's fixed 32x32 tile shape, and + * this kernel does not drive HMX (see below). A caller MUST set the wire + * layout id to HEXLIB_LAYOUT_ROW_MAJOR (0) for this weight, not + * HEXLIB_LAYOUT_Q4_0_REPACKED (2) -- the latter would be a correctly-shaped + * wrong answer, exactly the failure mode hexlib_dsp.h's enum exists to make + * loud instead of silent. + * + * ========================================================================== + * WHY HVX-COMPUTE, NOT HMX. + * ========================================================================== + * HMX needs weights pre-dequantized to fp16 tiles (q4_0 nibbles are not a + * type HMX accepts directly -- docs/research/quantization.md section 0/2), + * needs the activation load and weight load braced into ONE packet (silently + * zeroing the accumulator otherwise), and needs an explicit outer reduction + * loop for K > 32*32 = 1024 (K=3072 here needs 3 strips of <=32 dot-tiles, + * accumulated) -- three separate, individually error-prone pieces of new + * machinery. Per this task's own scope note: "an HVX-compute kernel that + * dequantizes q4_0 blocks to fp16 and does a vectorised multiply-accumulate + * is a COMPLETE AND VALUABLE result... HMX is the optimisation." This kernel + * takes that path: HVX vector compute (widen-multiply-accumulate in qf32, + * narrow once), no HMX. It makes all 75 plan steps dispatchable; a follow-up + * kernel can add the HMX path later without changing this one's contract. + * + * ========================================================================== + * ACCUMULATION PRECISION AND ROUNDING. + * ========================================================================== + * fp32 (Hexagon's qf32 pipeline) THE ENTIRE WAY THROUGH, WITH ONE UNAVOIDABLE + * EXTRA ROUNDING: the only HVX primitive available to multiply the + * dequantized weight by the activation scalar (`Q6_Wqf32_vmpy_VhfVhf`) takes + * two fp16 VECTORS, so each dequantized weight value is stored as fp16 (one + * rounding) BEFORE it is multiplied, not only at the final output. baseline.c + * replicates this same intermediate fp16 rounding of the weight for exactly + * this reason -- omitting it there was tried first and cost up to ~2.2e-3 + * absolute error at K=128 (a few ULP of weight-rounding noise per reduction + * term, accumulated over many terms, is not negligible at the largest K this + * harness tests), which would have been comparing this kernel against a + * different, more-accurate-than-actual algorithm. Each dequantized fp16 + * weight value is then widened and multiplied by the fp16 activation scalar + * into qf32, accumulated into a running fp32 sum over all of K, THEN the + * fp32 bias is added, THEN the activation (also computed in fp32, via + * `hvx_vec_exp_f32` -- never `hvx_vec_exp2_f16`, which has a wrong E5 + * coefficient, see kernel.c) is applied, and the fp16 STORE is the one and + * only narrowing step. This matches kernels/softmax_fp16 and kernels/ + * layernorm_fp16's own established precedent ("float32 intermediate, narrow + * once") and is provably more accurate than rounding to fp16 after the bias + * add and again after the activation. The oracle itself + * (hexlib/graph/eager.py:24-29) computes every dtype as float32/float64 and + * is blind to this choice; harness.c's header comment gives the actual + * measured numbers for what this choice costs against a scalar fp32 + * baseline, and why that is far below the size of a real bug. + * + * See kernel.c's header for the exact HVX lane-order mechanics (the + * fp16<->qf32 widen/narrow permutes lanes; this kernel follows the same + * shuffle-before/deal-after convention as kernels/layernorm_fp16 and + * kernels/rope_2d_fp16, because it mixes computed (permuted-domain) values + * with a plainly-loaded bias vector, exactly the situation those two kernels' + * own comments warn about). + * + * ========================================================================== + * SHAPES AND ALIGNMENT. + * ========================================================================== + * a: fp16 (M, K), row-major, 128-byte aligned. + * w: q4_0 raw bytes, LOGICAL shape (K, N), row-major blocks per above. K and N + * are both required to be multiples of 32 (`ir.nbytes`'s own constraint; + * every encoder shape satisfies it for both dims). + * bias: fp32 (N,), 128-byte aligned is not required (loaded unaligned). + * out: fp16 (M, N), row-major, 128-byte aligned. + * act: one of MM_ACT_NONE, MM_ACT_GELU_TANH, MM_ACT_GELU_ERF (below). + * + * THE CROUTON LAYOUT ("every two rows transposed") DOES NOT APPLY HERE. It is + * called out in `ai_fp16_matmul_gelu/spec.json` (../HVX-clean/data/v6/tasks/) + * as an edge case for THAT kernel's activation operand layout. This kernel's + * activation operand `a` is required to be plain row-major fp16 (declared + * above and enforced by the wire's `row_major` layout id on that input, per + * hexlib/tests/test_raw_q4_0_staging.py's own `_spec()`); nothing here ever + * reads `a` two rows at a time or reinterprets it as anything other than + * (M, K) row-major. There is no crouton-shaped operand in this kernel's + * contract to get wrong. + */ +#define MM_ACT_NONE 0 +#define MM_ACT_GELU_TANH 1 +#define MM_ACT_GELU_ERF 2 + +#define MM_Q4_0_BLOCK 32 +#define MM_Q4_0_BLOCK_BYTES 18 + +void matmul_epilogue_fp16(const hexlib_hf *a, const unsigned char *w, + const float *bias, hexlib_hf *out, + int M, int K, int N, int act); + +#endif diff --git a/kernels/matmul_epilogue_fp16/nearmiss_bias_after_activation.c b/kernels/matmul_epilogue_fp16/nearmiss_bias_after_activation.c new file mode 100644 index 0000000..c5bc720 --- /dev/null +++ b/kernels/matmul_epilogue_fp16/nearmiss_bias_after_activation.c @@ -0,0 +1,190 @@ +/* A plausible WRONG implementation the harness must reject. + * + * THE MISTAKE: applying the activation BEFORE adding the bias, instead of + * after. kernel_api.h SPEC SOURCE 1 (hexlib/graph/opdefs/fused.py:44-53) is + * explicit about the order: `out = (a@b + bias).astype(dtype)`, THEN the + * activation is applied to `out`. This file computes `act(a@b) + bias` + * instead of `act(a@b + bias)`. + * + * WHY ANYONE WOULD WRITE IT. Both bias-add and activation are "the + * epilogue" in casual description, and nothing about the C source makes + * their relative order look load-bearing -- it is one extra add, in the + * same function, moved a few lines. `fuse.py` (hexlib/graph/fuse.py:61-76) + * only ever folds a matmul -> add(bias) -> activation chain in that exact + * order, so this file's order is not even a case the fusion pass could have + * legally produced, but nothing in this kernel's own C source signals that. + * + * WHY THIS IS EASY TO CATCH HERE. For act != none, moving the bias to the + * other side of a nonlinear function changes the argument the nonlinearity + * actually sees whenever bias is not negligibly small relative to the + * pre-activation matmul output -- which is true almost everywhere on this + * harness's data (biases are formula-generated with the same order of + * magnitude as the matmul outputs, not vanishingly small). Measured in + * Python against this exact algorithm on ordinary (non-adversarial) data at + * this kernel's own test shapes: 61% of ALL output elements (across the two + * activation-bearing test shapes) disagree with the correct reference, with + * a maximum relative error in the hundreds-of-thousands-of-percent range + * (bias moved across a nonlinearity is not a rounding-sized mistake). No + * dedicated adversarial cell needed. (Test 1, act=none, is unaffected by + * this bug by construction -- there is no activation for the two orders to + * disagree about there -- but Tests 2 and 3 both use a real activation and + * both show the disagreement.) + */ +#include "kernel_api.h" + +#include +#include +#include + +#include "hexlib/hvx/hvx-base.h" +#include "hexlib/hvx/hvx-exp.h" +#include "hexlib/hvx/hvx-inverse.h" + +#define LANES_FP16 64 +#define LANES_FP32 32 + +static inline void mm_dequant_block(const unsigned char *blk, hexlib_hf *out32) { + __fp16 d; + memcpy(&d, blk, sizeof(d)); + const float df = (float) d; + const unsigned char *qs = blk + 2; + for (int j = 0; j < 16; ++j) { + const int lo = (int) (qs[j] & 0x0F) - 8; + const int hi = (int) ((qs[j] >> 4) & 0x0F) - 8; + out32[j] = (hexlib_hf) (df * (float) lo); + out32[16 + j] = (hexlib_hf) (df * (float) hi); + } +} + +static inline void mm_widen_mul_ordered(HVX_Vector x_bcast, HVX_Vector y, + HVX_Vector *out) { + HVX_VectorPair p = Q6_Wqf32_vmpy_VhfVhf(Q6_Vh_vshuff_Vh(y), x_bcast); + out[0] = Q6_Vsf_equals_Vqf32(Q6_V_lo_W(p)); + out[1] = Q6_Vsf_equals_Vqf32(Q6_V_hi_W(p)); +} + +static inline HVX_Vector mm_narrow_ordered(HVX_Vector lo, HVX_Vector hi) { + const HVX_Vector zero = Q6_V_vzero(); + HVX_Vector qlo = Q6_Vqf32_vadd_VsfVsf(lo, zero); + HVX_Vector qhi = Q6_Vqf32_vadd_VsfVsf(hi, zero); + return Q6_Vh_vdeal_Vh(Q6_Vhf_equals_Wqf32(Q6_W_vcombine_VV(qhi, qlo))); +} + +static inline HVX_Vector mm_splat_hf(float v) { + /* float, not hexlib_hf: hexagon-clang rejects __fp16 as a by-value + * parameter outright (see include/hexlib/hexlib_harness.h's own note + * on the same restriction). */ + union { hexlib_hf h; unsigned short u; } bits; + bits.h = (hexlib_hf) v; + return Q6_Vh_vsplat_R((int) bits.u); +} + +static inline HVX_Vector mm_tanh_f32(HVX_Vector u) { + HVX_Vector absu = hvx_vec_abs_f32(u); + HVX_Vector e = hvx_vec_exp_f32(hvx_vec_mul_f32_f32(absu, hvx_vec_splat_f32(-2.0f))); + HVX_Vector num = hvx_vec_sub_f32_f32(hvx_vec_splat_f32(1.0f), e); + HVX_Vector den = hvx_vec_add_f32_f32(hvx_vec_splat_f32(1.0f), e); + HVX_Vector t = hvx_vec_mul_f32_f32(num, hvx_vec_inverse_f32(den)); + HVX_Vector sign_bits = Q6_V_vand_VV(u, Q6_V_vsplat_R(0x80000000)); + return Q6_V_vor_VV(t, sign_bits); +} + +static inline HVX_Vector mm_gelu_tanh_f32(HVX_Vector x) { + HVX_Vector x2 = hvx_vec_mul_f32_f32(x, x); + HVX_Vector x3 = hvx_vec_mul_f32_f32(x2, x); + HVX_Vector inner = hvx_vec_add_f32_f32( + x, hvx_vec_mul_f32_f32(x3, hvx_vec_splat_f32(0.044715f))); + inner = hvx_vec_mul_f32_f32(inner, hvx_vec_splat_f32(0.7978845608028654f)); + HVX_Vector t = mm_tanh_f32(inner); + HVX_Vector one_plus_t = hvx_vec_add_f32_f32(hvx_vec_splat_f32(1.0f), t); + HVX_Vector half_x = hvx_vec_mul_f32_f32(x, hvx_vec_splat_f32(0.5f)); + return hvx_vec_mul_f32_f32(half_x, one_plus_t); +} + +static inline HVX_Vector mm_erf_f32(HVX_Vector x) { + HVX_Vector absx = hvx_vec_abs_f32(x); + HVX_Vector denom = hvx_vec_add_f32_f32( + hvx_vec_splat_f32(1.0f), + hvx_vec_mul_f32_f32(hvx_vec_splat_f32(0.3275911f), absx)); + HVX_Vector t = hvx_vec_inverse_f32(denom); + + HVX_Vector poly = hvx_vec_splat_f32(1.061405429f); + poly = hvx_vec_mul_f32_f32(poly, t); + poly = hvx_vec_add_f32_f32(poly, hvx_vec_splat_f32(-1.453152027f)); + poly = hvx_vec_mul_f32_f32(poly, t); + poly = hvx_vec_add_f32_f32(poly, hvx_vec_splat_f32(1.421413741f)); + poly = hvx_vec_mul_f32_f32(poly, t); + poly = hvx_vec_add_f32_f32(poly, hvx_vec_splat_f32(-0.284496736f)); + poly = hvx_vec_mul_f32_f32(poly, t); + poly = hvx_vec_add_f32_f32(poly, hvx_vec_splat_f32(0.254829592f)); + poly = hvx_vec_mul_f32_f32(poly, t); + + HVX_Vector neg_x2 = hvx_vec_mul_f32_f32( + hvx_vec_mul_f32_f32(absx, absx), hvx_vec_splat_f32(-1.0f)); + HVX_Vector exp_neg_x2 = hvx_vec_exp_f32(neg_x2); + HVX_Vector erf_abs = hvx_vec_sub_f32_f32(hvx_vec_splat_f32(1.0f), + hvx_vec_mul_f32_f32(poly, exp_neg_x2)); + HVX_Vector sign_bits = Q6_V_vand_VV(x, Q6_V_vsplat_R(0x80000000)); + return Q6_V_vor_VV(erf_abs, sign_bits); +} + +static inline HVX_Vector mm_gelu_erf_f32(HVX_Vector x) { + HVX_Vector arg = hvx_vec_mul_f32_f32(x, hvx_vec_splat_f32(0.7071067811865476f)); + HVX_Vector e = mm_erf_f32(arg); + HVX_Vector one_plus_e = hvx_vec_add_f32_f32(hvx_vec_splat_f32(1.0f), e); + HVX_Vector half_x = hvx_vec_mul_f32_f32(x, hvx_vec_splat_f32(0.5f)); + return hvx_vec_mul_f32_f32(half_x, one_plus_e); +} + +void matmul_epilogue_fp16(const hexlib_hf *a, const unsigned char *w, + const float *bias, hexlib_hf *out, + int M, int K, int N, int act) { + if (M <= 0 || K <= 0 || N <= 0 || N % MM_Q4_0_BLOCK != 0) { + return; + } + + const int nblocks = N / MM_Q4_0_BLOCK; + const long row_stride = (long) nblocks * MM_Q4_0_BLOCK_BYTES; + + hexlib_hf wbuf[LANES_FP16] __attribute__((aligned(128))); + for (int j = MM_Q4_0_BLOCK; j < LANES_FP16; ++j) { + wbuf[j] = (hexlib_hf) 0.0f; + } + HVX_Vector *wv_slot = (HVX_Vector *) wbuf; + + for (int m = 0; m < M; ++m) { + const hexlib_hf *arow = a + (long) m * K; + hexlib_hf *orow = out + (long) m * N; + + for (int bb = 0; bb < nblocks; ++bb) { + const long bb_off = (long) bb * MM_Q4_0_BLOCK_BYTES; + HVX_Vector acc_lo = Q6_V_vzero(); + + for (int k = 0; k < K; ++k) { + const unsigned char *blk = w + (long) k * row_stride + bb_off; + mm_dequant_block(blk, wbuf); + HVX_Vector wvec = *wv_slot; + HVX_Vector abcast = mm_splat_hf(arow[k]); + + HVX_Vector prod[2]; + mm_widen_mul_ordered(abcast, wvec, prod); + acc_lo = hvx_vec_add_f32_f32(acc_lo, prod[0]); + } + + /* WRONG: activation applied to the raw matmul output first, bias + * added afterward -- the opposite of fused.py's own order. */ + HVX_Vector r = acc_lo; + if (act == MM_ACT_GELU_TANH) { + r = mm_gelu_tanh_f32(r); + } else if (act == MM_ACT_GELU_ERF) { + r = mm_gelu_erf_f32(r); + } + HVX_Vector biasv = hvx_vmemu(bias + (long) bb * MM_Q4_0_BLOCK); + r = hvx_vec_add_f32_f32(r, biasv); + + HVX_Vector outv = mm_narrow_ordered(r, Q6_V_vzero()); + hvx_vec_store_u(orow + (long) bb * MM_Q4_0_BLOCK, + MM_Q4_0_BLOCK * (uint32_t) sizeof(hexlib_hf), outv); + } + } +} diff --git a/kernels/matmul_epilogue_fp16/nearmiss_fp16_accumulation.c b/kernels/matmul_epilogue_fp16/nearmiss_fp16_accumulation.c new file mode 100644 index 0000000..9c7dc12 --- /dev/null +++ b/kernels/matmul_epilogue_fp16/nearmiss_fp16_accumulation.c @@ -0,0 +1,193 @@ +/* A plausible WRONG implementation the harness must reject. + * + * THE MISTAKE: accumulating the K-reduction in __fp16 (rounding to fp16 + * after EVERY multiply-add) instead of carrying it in fp32 the whole way + * through. Everything else is IDENTICAL to kernel.c: same dequant, same bias + * order, same activation, same final narrow. + * + * WHY ANYONE WOULD WRITE IT. The activation operand and the weight are both + * fp16, so accumulating the running sum in fp16 "matches" everything else in + * sight, and on a part where fp16 vectors are twice as wide as fp32 ones, it + * looks like the natural width to keep a running sum in. Nothing about the C + * source looks unstable; the loop is the same loop. + * + * WHY THIS IS THE INTERESTING NEAR-MISS, WITH THE NUMBERS. See harness.c's + * header, "CELL 1": row 0, column 0 of Test 1 sums 1.0 (k=0) with 63 copies + * of 2^-12 (k=1..63). The true sum is 1.015380859375. The correct kernel + * (fp32 accumulate, one narrow) gets 1.015625, error +0.000244140625 + * (+0.024%) -- an ordinary 0.25-ULP narrowing rounding. THIS near-miss's + * fp16-per-step running sum rounds every one of those 63 additions back to + * exactly 1.0 (each increment is below half the accumulator's own ULP at + * that magnitude), so it never leaves 1.0: final error -0.015380859375 + * (-1.51%), 63x the correct kernel's own noise, and well past both the 1% + * relative and 3e-4 absolute tolerance branches. Measured in Python running + * this exact algorithm on this exact data (see harness.c). + */ +#include "kernel_api.h" + +#include +#include +#include + +#include "hexlib/hvx/hvx-base.h" +#include "hexlib/hvx/hvx-exp.h" +#include "hexlib/hvx/hvx-inverse.h" + +#define LANES_FP16 64 +#define LANES_FP32 32 + +static inline void mm_dequant_block(const unsigned char *blk, hexlib_hf *out32) { + __fp16 d; + memcpy(&d, blk, sizeof(d)); + const float df = (float) d; + const unsigned char *qs = blk + 2; + for (int j = 0; j < 16; ++j) { + const int lo = (int) (qs[j] & 0x0F) - 8; + const int hi = (int) ((qs[j] >> 4) & 0x0F) - 8; + out32[j] = (hexlib_hf) (df * (float) lo); + out32[16 + j] = (hexlib_hf) (df * (float) hi); + } +} + +static inline void mm_widen_mul_ordered(HVX_Vector x_bcast, HVX_Vector y, + HVX_Vector *out) { + HVX_VectorPair p = Q6_Wqf32_vmpy_VhfVhf(Q6_Vh_vshuff_Vh(y), x_bcast); + out[0] = Q6_Vsf_equals_Vqf32(Q6_V_lo_W(p)); + out[1] = Q6_Vsf_equals_Vqf32(Q6_V_hi_W(p)); +} + +static inline HVX_Vector mm_narrow_ordered(HVX_Vector lo, HVX_Vector hi) { + const HVX_Vector zero = Q6_V_vzero(); + HVX_Vector qlo = Q6_Vqf32_vadd_VsfVsf(lo, zero); + HVX_Vector qhi = Q6_Vqf32_vadd_VsfVsf(hi, zero); + return Q6_Vh_vdeal_Vh(Q6_Vhf_equals_Wqf32(Q6_W_vcombine_VV(qhi, qlo))); +} + +static inline HVX_Vector mm_splat_hf(float v) { + /* float, not hexlib_hf: hexagon-clang rejects __fp16 as a by-value + * parameter outright (see include/hexlib/hexlib_harness.h's own note + * on the same restriction). */ + union { hexlib_hf h; unsigned short u; } bits; + bits.h = (hexlib_hf) v; + return Q6_Vh_vsplat_R((int) bits.u); +} + +/* WRONG: round the running sum down to fp16 and back up to fp32, every + * single step -- the running accumulator is effectively __fp16, not fp32. */ +static inline HVX_Vector mm_round_trip_fp16(HVX_Vector v32) { + HVX_Vector as16 = mm_narrow_ordered(v32, Q6_V_vzero()); + HVX_Vector one = mm_splat_hf((hexlib_hf) 1.0f); + HVX_Vector widened[2]; + mm_widen_mul_ordered(one, as16, widened); + return widened[0]; +} + +static inline HVX_Vector mm_tanh_f32(HVX_Vector u) { + HVX_Vector absu = hvx_vec_abs_f32(u); + HVX_Vector e = hvx_vec_exp_f32(hvx_vec_mul_f32_f32(absu, hvx_vec_splat_f32(-2.0f))); + HVX_Vector num = hvx_vec_sub_f32_f32(hvx_vec_splat_f32(1.0f), e); + HVX_Vector den = hvx_vec_add_f32_f32(hvx_vec_splat_f32(1.0f), e); + HVX_Vector t = hvx_vec_mul_f32_f32(num, hvx_vec_inverse_f32(den)); + HVX_Vector sign_bits = Q6_V_vand_VV(u, Q6_V_vsplat_R(0x80000000)); + return Q6_V_vor_VV(t, sign_bits); +} + +static inline HVX_Vector mm_gelu_tanh_f32(HVX_Vector x) { + HVX_Vector x2 = hvx_vec_mul_f32_f32(x, x); + HVX_Vector x3 = hvx_vec_mul_f32_f32(x2, x); + HVX_Vector inner = hvx_vec_add_f32_f32( + x, hvx_vec_mul_f32_f32(x3, hvx_vec_splat_f32(0.044715f))); + inner = hvx_vec_mul_f32_f32(inner, hvx_vec_splat_f32(0.7978845608028654f)); + HVX_Vector t = mm_tanh_f32(inner); + HVX_Vector one_plus_t = hvx_vec_add_f32_f32(hvx_vec_splat_f32(1.0f), t); + HVX_Vector half_x = hvx_vec_mul_f32_f32(x, hvx_vec_splat_f32(0.5f)); + return hvx_vec_mul_f32_f32(half_x, one_plus_t); +} + +static inline HVX_Vector mm_erf_f32(HVX_Vector x) { + HVX_Vector absx = hvx_vec_abs_f32(x); + HVX_Vector denom = hvx_vec_add_f32_f32( + hvx_vec_splat_f32(1.0f), + hvx_vec_mul_f32_f32(hvx_vec_splat_f32(0.3275911f), absx)); + HVX_Vector t = hvx_vec_inverse_f32(denom); + + HVX_Vector poly = hvx_vec_splat_f32(1.061405429f); + poly = hvx_vec_mul_f32_f32(poly, t); + poly = hvx_vec_add_f32_f32(poly, hvx_vec_splat_f32(-1.453152027f)); + poly = hvx_vec_mul_f32_f32(poly, t); + poly = hvx_vec_add_f32_f32(poly, hvx_vec_splat_f32(1.421413741f)); + poly = hvx_vec_mul_f32_f32(poly, t); + poly = hvx_vec_add_f32_f32(poly, hvx_vec_splat_f32(-0.284496736f)); + poly = hvx_vec_mul_f32_f32(poly, t); + poly = hvx_vec_add_f32_f32(poly, hvx_vec_splat_f32(0.254829592f)); + poly = hvx_vec_mul_f32_f32(poly, t); + + HVX_Vector neg_x2 = hvx_vec_mul_f32_f32( + hvx_vec_mul_f32_f32(absx, absx), hvx_vec_splat_f32(-1.0f)); + HVX_Vector exp_neg_x2 = hvx_vec_exp_f32(neg_x2); + HVX_Vector erf_abs = hvx_vec_sub_f32_f32(hvx_vec_splat_f32(1.0f), + hvx_vec_mul_f32_f32(poly, exp_neg_x2)); + HVX_Vector sign_bits = Q6_V_vand_VV(x, Q6_V_vsplat_R(0x80000000)); + return Q6_V_vor_VV(erf_abs, sign_bits); +} + +static inline HVX_Vector mm_gelu_erf_f32(HVX_Vector x) { + HVX_Vector arg = hvx_vec_mul_f32_f32(x, hvx_vec_splat_f32(0.7071067811865476f)); + HVX_Vector e = mm_erf_f32(arg); + HVX_Vector one_plus_e = hvx_vec_add_f32_f32(hvx_vec_splat_f32(1.0f), e); + HVX_Vector half_x = hvx_vec_mul_f32_f32(x, hvx_vec_splat_f32(0.5f)); + return hvx_vec_mul_f32_f32(half_x, one_plus_e); +} + +void matmul_epilogue_fp16(const hexlib_hf *a, const unsigned char *w, + const float *bias, hexlib_hf *out, + int M, int K, int N, int act) { + if (M <= 0 || K <= 0 || N <= 0 || N % MM_Q4_0_BLOCK != 0) { + return; + } + + const int nblocks = N / MM_Q4_0_BLOCK; + const long row_stride = (long) nblocks * MM_Q4_0_BLOCK_BYTES; + + hexlib_hf wbuf[LANES_FP16] __attribute__((aligned(128))); + for (int j = MM_Q4_0_BLOCK; j < LANES_FP16; ++j) { + wbuf[j] = (hexlib_hf) 0.0f; + } + HVX_Vector *wv_slot = (HVX_Vector *) wbuf; + + for (int m = 0; m < M; ++m) { + const hexlib_hf *arow = a + (long) m * K; + hexlib_hf *orow = out + (long) m * N; + + for (int bb = 0; bb < nblocks; ++bb) { + const long bb_off = (long) bb * MM_Q4_0_BLOCK_BYTES; + HVX_Vector acc_lo = Q6_V_vzero(); + + for (int k = 0; k < K; ++k) { + const unsigned char *blk = w + (long) k * row_stride + bb_off; + mm_dequant_block(blk, wbuf); + HVX_Vector wvec = *wv_slot; + HVX_Vector abcast = mm_splat_hf(arow[k]); + + HVX_Vector prod[2]; + mm_widen_mul_ordered(abcast, wvec, prod); + acc_lo = hvx_vec_add_f32_f32(acc_lo, prod[0]); + /* WRONG: round the running sum through fp16 every step. */ + acc_lo = mm_round_trip_fp16(acc_lo); + } + + HVX_Vector biasv = hvx_vmemu(bias + (long) bb * MM_Q4_0_BLOCK); + HVX_Vector r = hvx_vec_add_f32_f32(acc_lo, biasv); + + if (act == MM_ACT_GELU_TANH) { + r = mm_gelu_tanh_f32(r); + } else if (act == MM_ACT_GELU_ERF) { + r = mm_gelu_erf_f32(r); + } + + HVX_Vector outv = mm_narrow_ordered(r, Q6_V_vzero()); + hvx_vec_store_u(orow + (long) bb * MM_Q4_0_BLOCK, + MM_Q4_0_BLOCK * (uint32_t) sizeof(hexlib_hf), outv); + } + } +} diff --git a/kernels/matmul_epilogue_fp16/nearmiss_gelu_swap.c b/kernels/matmul_epilogue_fp16/nearmiss_gelu_swap.c new file mode 100644 index 0000000..fb53be5 --- /dev/null +++ b/kernels/matmul_epilogue_fp16/nearmiss_gelu_swap.c @@ -0,0 +1,184 @@ +/* A plausible WRONG implementation the harness must reject. + * + * THE MISTAKE: using gelu_tanh's formula where gelu_erf belongs, and vice + * versa -- the MM_ACT_GELU_TANH branch calls mm_gelu_erf_f32 and the + * MM_ACT_GELU_ERF branch calls mm_gelu_tanh_f32. Everything else, including + * both activation implementations themselves, is byte-identical to kernel.c. + * + * WHY ANYONE WOULD WRITE IT. Both are "the GELU variant", both take one + * fp32 vector and return one fp32 vector, and a copy-paste or a mixed-up + * `if`/`else if` branch is the whole bug -- nothing about either function's + * signature or body hints which activation attribute it belongs under. + * kernel_api.h says outright that gelu_tanh and gelu_erf are "two different + * functions used in two different places in the model... not one op with a + * flag" precisely because this swap is easy to make and easy to miss. + * + * WHY THIS IS THE QUIET NEAR-MISS THIS KERNEL HAS TO WORK TO CATCH. See + * harness.c's header, "CELL 2": gelu_tanh and gelu_erf agree to within + * 4.73e-4 absolute EVERYWHERE (measured over x in [-8,8] at 0.01 + * resolution, real math.tanh/math.erf) -- smaller, in absolute terms, than + * one legitimate fp16 rounding at many magnitudes. What makes it catchable + * is that harness.c's Test 3 row 0 probes x=-2.7 specifically, where the + * correct (erf) output's own magnitude is small (~0.0094), so that same + * 4.73e-4 absolute difference is 62 ULPs and 5.2% relative AT THAT POINT -- + * both tolerance branches (1% relative, 3e-4 absolute) reject it there, even + * though a naive "how big is the raw number" glance at 4.73e-4 in isolation + * would suggest otherwise. + */ +#include "kernel_api.h" + +#include +#include +#include + +#include "hexlib/hvx/hvx-base.h" +#include "hexlib/hvx/hvx-exp.h" +#include "hexlib/hvx/hvx-inverse.h" + +#define LANES_FP16 64 +#define LANES_FP32 32 + +static inline void mm_dequant_block(const unsigned char *blk, hexlib_hf *out32) { + __fp16 d; + memcpy(&d, blk, sizeof(d)); + const float df = (float) d; + const unsigned char *qs = blk + 2; + for (int j = 0; j < 16; ++j) { + const int lo = (int) (qs[j] & 0x0F) - 8; + const int hi = (int) ((qs[j] >> 4) & 0x0F) - 8; + out32[j] = (hexlib_hf) (df * (float) lo); + out32[16 + j] = (hexlib_hf) (df * (float) hi); + } +} + +static inline void mm_widen_mul_ordered(HVX_Vector x_bcast, HVX_Vector y, + HVX_Vector *out) { + HVX_VectorPair p = Q6_Wqf32_vmpy_VhfVhf(Q6_Vh_vshuff_Vh(y), x_bcast); + out[0] = Q6_Vsf_equals_Vqf32(Q6_V_lo_W(p)); + out[1] = Q6_Vsf_equals_Vqf32(Q6_V_hi_W(p)); +} + +static inline HVX_Vector mm_narrow_ordered(HVX_Vector lo, HVX_Vector hi) { + const HVX_Vector zero = Q6_V_vzero(); + HVX_Vector qlo = Q6_Vqf32_vadd_VsfVsf(lo, zero); + HVX_Vector qhi = Q6_Vqf32_vadd_VsfVsf(hi, zero); + return Q6_Vh_vdeal_Vh(Q6_Vhf_equals_Wqf32(Q6_W_vcombine_VV(qhi, qlo))); +} + +static inline HVX_Vector mm_splat_hf(float v) { + /* float, not hexlib_hf: hexagon-clang rejects __fp16 as a by-value + * parameter outright (see include/hexlib/hexlib_harness.h's own note + * on the same restriction). */ + union { hexlib_hf h; unsigned short u; } bits; + bits.h = (hexlib_hf) v; + return Q6_Vh_vsplat_R((int) bits.u); +} + +static inline HVX_Vector mm_tanh_f32(HVX_Vector u) { + HVX_Vector absu = hvx_vec_abs_f32(u); + HVX_Vector e = hvx_vec_exp_f32(hvx_vec_mul_f32_f32(absu, hvx_vec_splat_f32(-2.0f))); + HVX_Vector num = hvx_vec_sub_f32_f32(hvx_vec_splat_f32(1.0f), e); + HVX_Vector den = hvx_vec_add_f32_f32(hvx_vec_splat_f32(1.0f), e); + HVX_Vector t = hvx_vec_mul_f32_f32(num, hvx_vec_inverse_f32(den)); + HVX_Vector sign_bits = Q6_V_vand_VV(u, Q6_V_vsplat_R(0x80000000)); + return Q6_V_vor_VV(t, sign_bits); +} + +static inline HVX_Vector mm_gelu_tanh_f32(HVX_Vector x) { + HVX_Vector x2 = hvx_vec_mul_f32_f32(x, x); + HVX_Vector x3 = hvx_vec_mul_f32_f32(x2, x); + HVX_Vector inner = hvx_vec_add_f32_f32( + x, hvx_vec_mul_f32_f32(x3, hvx_vec_splat_f32(0.044715f))); + inner = hvx_vec_mul_f32_f32(inner, hvx_vec_splat_f32(0.7978845608028654f)); + HVX_Vector t = mm_tanh_f32(inner); + HVX_Vector one_plus_t = hvx_vec_add_f32_f32(hvx_vec_splat_f32(1.0f), t); + HVX_Vector half_x = hvx_vec_mul_f32_f32(x, hvx_vec_splat_f32(0.5f)); + return hvx_vec_mul_f32_f32(half_x, one_plus_t); +} + +static inline HVX_Vector mm_erf_f32(HVX_Vector x) { + HVX_Vector absx = hvx_vec_abs_f32(x); + HVX_Vector denom = hvx_vec_add_f32_f32( + hvx_vec_splat_f32(1.0f), + hvx_vec_mul_f32_f32(hvx_vec_splat_f32(0.3275911f), absx)); + HVX_Vector t = hvx_vec_inverse_f32(denom); + + HVX_Vector poly = hvx_vec_splat_f32(1.061405429f); + poly = hvx_vec_mul_f32_f32(poly, t); + poly = hvx_vec_add_f32_f32(poly, hvx_vec_splat_f32(-1.453152027f)); + poly = hvx_vec_mul_f32_f32(poly, t); + poly = hvx_vec_add_f32_f32(poly, hvx_vec_splat_f32(1.421413741f)); + poly = hvx_vec_mul_f32_f32(poly, t); + poly = hvx_vec_add_f32_f32(poly, hvx_vec_splat_f32(-0.284496736f)); + poly = hvx_vec_mul_f32_f32(poly, t); + poly = hvx_vec_add_f32_f32(poly, hvx_vec_splat_f32(0.254829592f)); + poly = hvx_vec_mul_f32_f32(poly, t); + + HVX_Vector neg_x2 = hvx_vec_mul_f32_f32( + hvx_vec_mul_f32_f32(absx, absx), hvx_vec_splat_f32(-1.0f)); + HVX_Vector exp_neg_x2 = hvx_vec_exp_f32(neg_x2); + HVX_Vector erf_abs = hvx_vec_sub_f32_f32(hvx_vec_splat_f32(1.0f), + hvx_vec_mul_f32_f32(poly, exp_neg_x2)); + HVX_Vector sign_bits = Q6_V_vand_VV(x, Q6_V_vsplat_R(0x80000000)); + return Q6_V_vor_VV(erf_abs, sign_bits); +} + +static inline HVX_Vector mm_gelu_erf_f32(HVX_Vector x) { + HVX_Vector arg = hvx_vec_mul_f32_f32(x, hvx_vec_splat_f32(0.7071067811865476f)); + HVX_Vector e = mm_erf_f32(arg); + HVX_Vector one_plus_e = hvx_vec_add_f32_f32(hvx_vec_splat_f32(1.0f), e); + HVX_Vector half_x = hvx_vec_mul_f32_f32(x, hvx_vec_splat_f32(0.5f)); + return hvx_vec_mul_f32_f32(half_x, one_plus_e); +} + +void matmul_epilogue_fp16(const hexlib_hf *a, const unsigned char *w, + const float *bias, hexlib_hf *out, + int M, int K, int N, int act) { + if (M <= 0 || K <= 0 || N <= 0 || N % MM_Q4_0_BLOCK != 0) { + return; + } + + const int nblocks = N / MM_Q4_0_BLOCK; + const long row_stride = (long) nblocks * MM_Q4_0_BLOCK_BYTES; + + hexlib_hf wbuf[LANES_FP16] __attribute__((aligned(128))); + for (int j = MM_Q4_0_BLOCK; j < LANES_FP16; ++j) { + wbuf[j] = (hexlib_hf) 0.0f; + } + HVX_Vector *wv_slot = (HVX_Vector *) wbuf; + + for (int m = 0; m < M; ++m) { + const hexlib_hf *arow = a + (long) m * K; + hexlib_hf *orow = out + (long) m * N; + + for (int bb = 0; bb < nblocks; ++bb) { + const long bb_off = (long) bb * MM_Q4_0_BLOCK_BYTES; + HVX_Vector acc_lo = Q6_V_vzero(); + + for (int k = 0; k < K; ++k) { + const unsigned char *blk = w + (long) k * row_stride + bb_off; + mm_dequant_block(blk, wbuf); + HVX_Vector wvec = *wv_slot; + HVX_Vector abcast = mm_splat_hf(arow[k]); + + HVX_Vector prod[2]; + mm_widen_mul_ordered(abcast, wvec, prod); + acc_lo = hvx_vec_add_f32_f32(acc_lo, prod[0]); + } + + HVX_Vector biasv = hvx_vmemu(bias + (long) bb * MM_Q4_0_BLOCK); + HVX_Vector r = hvx_vec_add_f32_f32(acc_lo, biasv); + + /* WRONG: the two branches call each other's formula. */ + if (act == MM_ACT_GELU_TANH) { + r = mm_gelu_erf_f32(r); + } else if (act == MM_ACT_GELU_ERF) { + r = mm_gelu_tanh_f32(r); + } + + HVX_Vector outv = mm_narrow_ordered(r, Q6_V_vzero()); + hvx_vec_store_u(orow + (long) bb * MM_Q4_0_BLOCK, + MM_Q4_0_BLOCK * (uint32_t) sizeof(hexlib_hf), outv); + } + } +} diff --git a/kernels/matmul_epilogue_fp16/nearmiss_scale_off_by_one_block.c b/kernels/matmul_epilogue_fp16/nearmiss_scale_off_by_one_block.c new file mode 100644 index 0000000..f16ddd9 --- /dev/null +++ b/kernels/matmul_epilogue_fp16/nearmiss_scale_off_by_one_block.c @@ -0,0 +1,196 @@ +/* A plausible WRONG implementation the harness must reject. + * + * THE MISTAKE: dequantizing output-column-block bb's 32 codes with the + * SCALE from output-column-block (bb-1), instead of block bb's own scale. + * The codes read are still correct for block bb; only the 2-byte fp16 + * scale prefix comes from the wrong 18-byte block. + * + * WHY ANYONE WOULD WRITE IT. Pointer arithmetic over a stream of same-sized + * 18-byte records is exactly the kind of code where an index that should + * track the current iteration silently tracks the PREVIOUS one instead -- + * an off-by-one in which block's base address feeds the scale read, while + * the codes pointer (computed separately, correctly) does not share the + * mistake. Nothing about the surrounding loop looks wrong; the block being + * read is still 18 bytes, still at a valid offset within the same row. + * + * WHY THIS IS EASY TO CATCH HERE. This kernel's own blocks (kernel_api.h's + * layout decision) are quantized independently per (row k, output-block bb) + * with amax-derived scales, so adjacent blocks along a row generally have + * DIFFERENT scales (harness.c's weight-generating formula varies with `bb` + * specifically to make sure of this). Applying the wrong scale multiplies + * every one of that block's 32 dequantized values by (right_scale / + * wrong_scale) -- a systematic, block-wide distortion, not a rounding + * perturbation. Measured in Python against this exact algorithm on ordinary + * formula-generated data at this kernel's own test shapes: 61% of ALL + * output elements disagree with the correct reference, max relative error + * over 100000% (block 0 of every row, which has no "previous" block in this + * implementation's wrap-around indexing, borrows the LAST block's scale -- + * still a real, present mismatch, not a skipped case). No dedicated + * adversarial cell needed. + */ +#include "kernel_api.h" + +#include +#include +#include + +#include "hexlib/hvx/hvx-base.h" +#include "hexlib/hvx/hvx-exp.h" +#include "hexlib/hvx/hvx-inverse.h" + +#define LANES_FP16 64 +#define LANES_FP32 32 + +/* WRONG: scale comes from `scale_blk`, codes come from `codes_blk` -- the + * two pointers are the SAME block in the correct kernel, but the caller + * below passes them one block apart. */ +static inline void mm_dequant_block_wrong_scale(const unsigned char *scale_blk, + const unsigned char *codes_blk, + hexlib_hf *out32) { + __fp16 d; + memcpy(&d, scale_blk, sizeof(d)); + const float df = (float) d; + const unsigned char *qs = codes_blk + 2; + for (int j = 0; j < 16; ++j) { + const int lo = (int) (qs[j] & 0x0F) - 8; + const int hi = (int) ((qs[j] >> 4) & 0x0F) - 8; + out32[j] = (hexlib_hf) (df * (float) lo); + out32[16 + j] = (hexlib_hf) (df * (float) hi); + } +} + +static inline void mm_widen_mul_ordered(HVX_Vector x_bcast, HVX_Vector y, + HVX_Vector *out) { + HVX_VectorPair p = Q6_Wqf32_vmpy_VhfVhf(Q6_Vh_vshuff_Vh(y), x_bcast); + out[0] = Q6_Vsf_equals_Vqf32(Q6_V_lo_W(p)); + out[1] = Q6_Vsf_equals_Vqf32(Q6_V_hi_W(p)); +} + +static inline HVX_Vector mm_narrow_ordered(HVX_Vector lo, HVX_Vector hi) { + const HVX_Vector zero = Q6_V_vzero(); + HVX_Vector qlo = Q6_Vqf32_vadd_VsfVsf(lo, zero); + HVX_Vector qhi = Q6_Vqf32_vadd_VsfVsf(hi, zero); + return Q6_Vh_vdeal_Vh(Q6_Vhf_equals_Wqf32(Q6_W_vcombine_VV(qhi, qlo))); +} + +static inline HVX_Vector mm_splat_hf(float v) { + /* float, not hexlib_hf: hexagon-clang rejects __fp16 as a by-value + * parameter outright (see include/hexlib/hexlib_harness.h's own note + * on the same restriction). */ + union { hexlib_hf h; unsigned short u; } bits; + bits.h = (hexlib_hf) v; + return Q6_Vh_vsplat_R((int) bits.u); +} + +static inline HVX_Vector mm_tanh_f32(HVX_Vector u) { + HVX_Vector absu = hvx_vec_abs_f32(u); + HVX_Vector e = hvx_vec_exp_f32(hvx_vec_mul_f32_f32(absu, hvx_vec_splat_f32(-2.0f))); + HVX_Vector num = hvx_vec_sub_f32_f32(hvx_vec_splat_f32(1.0f), e); + HVX_Vector den = hvx_vec_add_f32_f32(hvx_vec_splat_f32(1.0f), e); + HVX_Vector t = hvx_vec_mul_f32_f32(num, hvx_vec_inverse_f32(den)); + HVX_Vector sign_bits = Q6_V_vand_VV(u, Q6_V_vsplat_R(0x80000000)); + return Q6_V_vor_VV(t, sign_bits); +} + +static inline HVX_Vector mm_gelu_tanh_f32(HVX_Vector x) { + HVX_Vector x2 = hvx_vec_mul_f32_f32(x, x); + HVX_Vector x3 = hvx_vec_mul_f32_f32(x2, x); + HVX_Vector inner = hvx_vec_add_f32_f32( + x, hvx_vec_mul_f32_f32(x3, hvx_vec_splat_f32(0.044715f))); + inner = hvx_vec_mul_f32_f32(inner, hvx_vec_splat_f32(0.7978845608028654f)); + HVX_Vector t = mm_tanh_f32(inner); + HVX_Vector one_plus_t = hvx_vec_add_f32_f32(hvx_vec_splat_f32(1.0f), t); + HVX_Vector half_x = hvx_vec_mul_f32_f32(x, hvx_vec_splat_f32(0.5f)); + return hvx_vec_mul_f32_f32(half_x, one_plus_t); +} + +static inline HVX_Vector mm_erf_f32(HVX_Vector x) { + HVX_Vector absx = hvx_vec_abs_f32(x); + HVX_Vector denom = hvx_vec_add_f32_f32( + hvx_vec_splat_f32(1.0f), + hvx_vec_mul_f32_f32(hvx_vec_splat_f32(0.3275911f), absx)); + HVX_Vector t = hvx_vec_inverse_f32(denom); + + HVX_Vector poly = hvx_vec_splat_f32(1.061405429f); + poly = hvx_vec_mul_f32_f32(poly, t); + poly = hvx_vec_add_f32_f32(poly, hvx_vec_splat_f32(-1.453152027f)); + poly = hvx_vec_mul_f32_f32(poly, t); + poly = hvx_vec_add_f32_f32(poly, hvx_vec_splat_f32(1.421413741f)); + poly = hvx_vec_mul_f32_f32(poly, t); + poly = hvx_vec_add_f32_f32(poly, hvx_vec_splat_f32(-0.284496736f)); + poly = hvx_vec_mul_f32_f32(poly, t); + poly = hvx_vec_add_f32_f32(poly, hvx_vec_splat_f32(0.254829592f)); + poly = hvx_vec_mul_f32_f32(poly, t); + + HVX_Vector neg_x2 = hvx_vec_mul_f32_f32( + hvx_vec_mul_f32_f32(absx, absx), hvx_vec_splat_f32(-1.0f)); + HVX_Vector exp_neg_x2 = hvx_vec_exp_f32(neg_x2); + HVX_Vector erf_abs = hvx_vec_sub_f32_f32(hvx_vec_splat_f32(1.0f), + hvx_vec_mul_f32_f32(poly, exp_neg_x2)); + HVX_Vector sign_bits = Q6_V_vand_VV(x, Q6_V_vsplat_R(0x80000000)); + return Q6_V_vor_VV(erf_abs, sign_bits); +} + +static inline HVX_Vector mm_gelu_erf_f32(HVX_Vector x) { + HVX_Vector arg = hvx_vec_mul_f32_f32(x, hvx_vec_splat_f32(0.7071067811865476f)); + HVX_Vector e = mm_erf_f32(arg); + HVX_Vector one_plus_e = hvx_vec_add_f32_f32(hvx_vec_splat_f32(1.0f), e); + HVX_Vector half_x = hvx_vec_mul_f32_f32(x, hvx_vec_splat_f32(0.5f)); + return hvx_vec_mul_f32_f32(half_x, one_plus_e); +} + +void matmul_epilogue_fp16(const hexlib_hf *a, const unsigned char *w, + const float *bias, hexlib_hf *out, + int M, int K, int N, int act) { + if (M <= 0 || K <= 0 || N <= 0 || N % MM_Q4_0_BLOCK != 0) { + return; + } + + const int nblocks = N / MM_Q4_0_BLOCK; + const long row_stride = (long) nblocks * MM_Q4_0_BLOCK_BYTES; + + hexlib_hf wbuf[LANES_FP16] __attribute__((aligned(128))); + for (int j = MM_Q4_0_BLOCK; j < LANES_FP16; ++j) { + wbuf[j] = (hexlib_hf) 0.0f; + } + HVX_Vector *wv_slot = (HVX_Vector *) wbuf; + + for (int m = 0; m < M; ++m) { + const hexlib_hf *arow = a + (long) m * K; + hexlib_hf *orow = out + (long) m * N; + + for (int bb = 0; bb < nblocks; ++bb) { + const long bb_off = (long) bb * MM_Q4_0_BLOCK_BYTES; + /* WRONG: the scale is read from the PREVIOUS block (wrapping + * around for bb==0), while the codes still come from bb. */ + const int scale_bb = (bb - 1 + nblocks) % nblocks; + const long scale_off = (long) scale_bb * MM_Q4_0_BLOCK_BYTES; + HVX_Vector acc_lo = Q6_V_vzero(); + + for (int k = 0; k < K; ++k) { + const unsigned char *codes_blk = w + (long) k * row_stride + bb_off; + const unsigned char *scale_blk = w + (long) k * row_stride + scale_off; + mm_dequant_block_wrong_scale(scale_blk, codes_blk, wbuf); + HVX_Vector wvec = *wv_slot; + HVX_Vector abcast = mm_splat_hf(arow[k]); + + HVX_Vector prod[2]; + mm_widen_mul_ordered(abcast, wvec, prod); + acc_lo = hvx_vec_add_f32_f32(acc_lo, prod[0]); + } + + HVX_Vector biasv = hvx_vmemu(bias + (long) bb * MM_Q4_0_BLOCK); + HVX_Vector r = hvx_vec_add_f32_f32(acc_lo, biasv); + + if (act == MM_ACT_GELU_TANH) { + r = mm_gelu_tanh_f32(r); + } else if (act == MM_ACT_GELU_ERF) { + r = mm_gelu_erf_f32(r); + } + + HVX_Vector outv = mm_narrow_ordered(r, Q6_V_vzero()); + hvx_vec_store_u(orow + (long) bb * MM_Q4_0_BLOCK, + MM_Q4_0_BLOCK * (uint32_t) sizeof(hexlib_hf), outv); + } + } +} diff --git a/kernels/matmul_epilogue_fp16/nearmiss_swapped_nibble_order.c b/kernels/matmul_epilogue_fp16/nearmiss_swapped_nibble_order.c new file mode 100644 index 0000000..6898429 --- /dev/null +++ b/kernels/matmul_epilogue_fp16/nearmiss_swapped_nibble_order.c @@ -0,0 +1,183 @@ +/* A plausible WRONG implementation the harness must reject. + * + * THE MISTAKE: swapping which nibble of each packed byte maps to which + * dequantized index. The correct mapping (kernel_api.h SPEC SOURCE 3, + * adapted from llama.cpp's dequantize_row_q4_0): low nibble of byte j -> + * index j, high nibble of byte j -> index j+16. This file writes it the + * other way: HIGH nibble -> index j, LOW nibble -> index j+16. + * + * WHY ANYONE WOULD WRITE IT. "byte j holds elements 2j and 2j+1" is the more + * common packing convention in adjacent-pair nibble formats, and swapping + * which nibble is "first" inside that assumption is an easy transcription + * slip that still compiles, still produces a same-shape output, and still + * looks locally sensible (the shift amount 4 and mask 0x0F are still exactly + * right -- only which RESULT INDEX each one is assigned to is wrong). + * + * WHY THIS IS EASY TO CATCH HERE (not quiet like the fp16/gelu near-misses). + * Unless a block's 32 target values happen to be symmetric under swapping + * (j, j+16), every block's dequantized values land at different indices + * entirely -- not a rounding-sized perturbation but a full permutation of + * which weight multiplies which activation term. Measured in Python against + * this exact algorithm on ordinary formula-generated (non-adversarial, + * mixed-sign) weight data at this kernel's own test shapes: 85% of ALL + * output elements disagree with the correct reference, max relative error + * in the tens of millions of percent (a wrong-index sum is not a small + * number near a right one). No dedicated adversarial cell needed. + */ +#include "kernel_api.h" + +#include +#include +#include + +#include "hexlib/hvx/hvx-base.h" +#include "hexlib/hvx/hvx-exp.h" +#include "hexlib/hvx/hvx-inverse.h" + +#define LANES_FP16 64 +#define LANES_FP32 32 + +/* WRONG: nibble roles swapped. HIGH nibble -> index j, LOW nibble -> j+16. */ +static inline void mm_dequant_block(const unsigned char *blk, hexlib_hf *out32) { + __fp16 d; + memcpy(&d, blk, sizeof(d)); + const float df = (float) d; + const unsigned char *qs = blk + 2; + for (int j = 0; j < 16; ++j) { + const int lo = (int) (qs[j] & 0x0F) - 8; + const int hi = (int) ((qs[j] >> 4) & 0x0F) - 8; + out32[j] = (hexlib_hf) (df * (float) hi); /* swapped */ + out32[16 + j] = (hexlib_hf) (df * (float) lo); /* swapped */ + } +} + +static inline void mm_widen_mul_ordered(HVX_Vector x_bcast, HVX_Vector y, + HVX_Vector *out) { + HVX_VectorPair p = Q6_Wqf32_vmpy_VhfVhf(Q6_Vh_vshuff_Vh(y), x_bcast); + out[0] = Q6_Vsf_equals_Vqf32(Q6_V_lo_W(p)); + out[1] = Q6_Vsf_equals_Vqf32(Q6_V_hi_W(p)); +} + +static inline HVX_Vector mm_narrow_ordered(HVX_Vector lo, HVX_Vector hi) { + const HVX_Vector zero = Q6_V_vzero(); + HVX_Vector qlo = Q6_Vqf32_vadd_VsfVsf(lo, zero); + HVX_Vector qhi = Q6_Vqf32_vadd_VsfVsf(hi, zero); + return Q6_Vh_vdeal_Vh(Q6_Vhf_equals_Wqf32(Q6_W_vcombine_VV(qhi, qlo))); +} + +static inline HVX_Vector mm_splat_hf(float v) { + /* float, not hexlib_hf: hexagon-clang rejects __fp16 as a by-value + * parameter outright (see include/hexlib/hexlib_harness.h's own note + * on the same restriction). */ + union { hexlib_hf h; unsigned short u; } bits; + bits.h = (hexlib_hf) v; + return Q6_Vh_vsplat_R((int) bits.u); +} + +static inline HVX_Vector mm_tanh_f32(HVX_Vector u) { + HVX_Vector absu = hvx_vec_abs_f32(u); + HVX_Vector e = hvx_vec_exp_f32(hvx_vec_mul_f32_f32(absu, hvx_vec_splat_f32(-2.0f))); + HVX_Vector num = hvx_vec_sub_f32_f32(hvx_vec_splat_f32(1.0f), e); + HVX_Vector den = hvx_vec_add_f32_f32(hvx_vec_splat_f32(1.0f), e); + HVX_Vector t = hvx_vec_mul_f32_f32(num, hvx_vec_inverse_f32(den)); + HVX_Vector sign_bits = Q6_V_vand_VV(u, Q6_V_vsplat_R(0x80000000)); + return Q6_V_vor_VV(t, sign_bits); +} + +static inline HVX_Vector mm_gelu_tanh_f32(HVX_Vector x) { + HVX_Vector x2 = hvx_vec_mul_f32_f32(x, x); + HVX_Vector x3 = hvx_vec_mul_f32_f32(x2, x); + HVX_Vector inner = hvx_vec_add_f32_f32( + x, hvx_vec_mul_f32_f32(x3, hvx_vec_splat_f32(0.044715f))); + inner = hvx_vec_mul_f32_f32(inner, hvx_vec_splat_f32(0.7978845608028654f)); + HVX_Vector t = mm_tanh_f32(inner); + HVX_Vector one_plus_t = hvx_vec_add_f32_f32(hvx_vec_splat_f32(1.0f), t); + HVX_Vector half_x = hvx_vec_mul_f32_f32(x, hvx_vec_splat_f32(0.5f)); + return hvx_vec_mul_f32_f32(half_x, one_plus_t); +} + +static inline HVX_Vector mm_erf_f32(HVX_Vector x) { + HVX_Vector absx = hvx_vec_abs_f32(x); + HVX_Vector denom = hvx_vec_add_f32_f32( + hvx_vec_splat_f32(1.0f), + hvx_vec_mul_f32_f32(hvx_vec_splat_f32(0.3275911f), absx)); + HVX_Vector t = hvx_vec_inverse_f32(denom); + + HVX_Vector poly = hvx_vec_splat_f32(1.061405429f); + poly = hvx_vec_mul_f32_f32(poly, t); + poly = hvx_vec_add_f32_f32(poly, hvx_vec_splat_f32(-1.453152027f)); + poly = hvx_vec_mul_f32_f32(poly, t); + poly = hvx_vec_add_f32_f32(poly, hvx_vec_splat_f32(1.421413741f)); + poly = hvx_vec_mul_f32_f32(poly, t); + poly = hvx_vec_add_f32_f32(poly, hvx_vec_splat_f32(-0.284496736f)); + poly = hvx_vec_mul_f32_f32(poly, t); + poly = hvx_vec_add_f32_f32(poly, hvx_vec_splat_f32(0.254829592f)); + poly = hvx_vec_mul_f32_f32(poly, t); + + HVX_Vector neg_x2 = hvx_vec_mul_f32_f32( + hvx_vec_mul_f32_f32(absx, absx), hvx_vec_splat_f32(-1.0f)); + HVX_Vector exp_neg_x2 = hvx_vec_exp_f32(neg_x2); + HVX_Vector erf_abs = hvx_vec_sub_f32_f32(hvx_vec_splat_f32(1.0f), + hvx_vec_mul_f32_f32(poly, exp_neg_x2)); + HVX_Vector sign_bits = Q6_V_vand_VV(x, Q6_V_vsplat_R(0x80000000)); + return Q6_V_vor_VV(erf_abs, sign_bits); +} + +static inline HVX_Vector mm_gelu_erf_f32(HVX_Vector x) { + HVX_Vector arg = hvx_vec_mul_f32_f32(x, hvx_vec_splat_f32(0.7071067811865476f)); + HVX_Vector e = mm_erf_f32(arg); + HVX_Vector one_plus_e = hvx_vec_add_f32_f32(hvx_vec_splat_f32(1.0f), e); + HVX_Vector half_x = hvx_vec_mul_f32_f32(x, hvx_vec_splat_f32(0.5f)); + return hvx_vec_mul_f32_f32(half_x, one_plus_e); +} + +void matmul_epilogue_fp16(const hexlib_hf *a, const unsigned char *w, + const float *bias, hexlib_hf *out, + int M, int K, int N, int act) { + if (M <= 0 || K <= 0 || N <= 0 || N % MM_Q4_0_BLOCK != 0) { + return; + } + + const int nblocks = N / MM_Q4_0_BLOCK; + const long row_stride = (long) nblocks * MM_Q4_0_BLOCK_BYTES; + + hexlib_hf wbuf[LANES_FP16] __attribute__((aligned(128))); + for (int j = MM_Q4_0_BLOCK; j < LANES_FP16; ++j) { + wbuf[j] = (hexlib_hf) 0.0f; + } + HVX_Vector *wv_slot = (HVX_Vector *) wbuf; + + for (int m = 0; m < M; ++m) { + const hexlib_hf *arow = a + (long) m * K; + hexlib_hf *orow = out + (long) m * N; + + for (int bb = 0; bb < nblocks; ++bb) { + const long bb_off = (long) bb * MM_Q4_0_BLOCK_BYTES; + HVX_Vector acc_lo = Q6_V_vzero(); + + for (int k = 0; k < K; ++k) { + const unsigned char *blk = w + (long) k * row_stride + bb_off; + mm_dequant_block(blk, wbuf); + HVX_Vector wvec = *wv_slot; + HVX_Vector abcast = mm_splat_hf(arow[k]); + + HVX_Vector prod[2]; + mm_widen_mul_ordered(abcast, wvec, prod); + acc_lo = hvx_vec_add_f32_f32(acc_lo, prod[0]); + } + + HVX_Vector biasv = hvx_vmemu(bias + (long) bb * MM_Q4_0_BLOCK); + HVX_Vector r = hvx_vec_add_f32_f32(acc_lo, biasv); + + if (act == MM_ACT_GELU_TANH) { + r = mm_gelu_tanh_f32(r); + } else if (act == MM_ACT_GELU_ERF) { + r = mm_gelu_erf_f32(r); + } + + HVX_Vector outv = mm_narrow_ordered(r, Q6_V_vzero()); + hvx_vec_store_u(orow + (long) bb * MM_Q4_0_BLOCK, + MM_Q4_0_BLOCK * (uint32_t) sizeof(hexlib_hf), outv); + } + } +} diff --git a/kernels/matmul_epilogue_fp16/spec.json b/kernels/matmul_epilogue_fp16/spec.json new file mode 100644 index 0000000..eaeeb7a --- /dev/null +++ b/kernels/matmul_epilogue_fp16/spec.json @@ -0,0 +1,27 @@ +{ + "task_id": "matmul_epilogue_fp16", + "dtype": "q4_0->fp16", + "caps": [], + "mechanisms": ["hvx"], + "params": { + "shapes_tested": [ + {"M": 8, "K": 64, "N": 128, "act": "none"}, + {"M": 12, "K": 96, "N": 160, "act": "gelu_tanh"}, + {"M": 20, "K": 128, "N": 96, "act": "gelu_erf"} + ], + "encoder_shapes": [ + "48x (256,768)x(768,768)+bias-> (256,768) act=none", + "12x (256,768)x(768,3072)+bias-> (256,3072) act=gelu_tanh", + "12x (256,3072)x(3072,768)+bias->(256,768) act=none", + "1x (256,1536)x(1536,768)+bias->(256,768) act=none", + "1x (64,3072)x(3072,3072)+bias->(64,3072) act=gelu_erf", + "1x (64,3072)x(3072,1024)+bias->(64,1024) act=none" + ], + "encoder_op_count": 75, + "weight_layout": "row_major q4_0, blocks along N (the output/free axis, the last logical dim of the (K,N) weight), NOT ggml-hexagon's 576-byte HMX-repacked tile order", + "accel_path": "HVX-compute (widen-multiply-accumulate in qf32, one narrow at the end); HMX not implemented -- see kernel_api.h 'WHY HVX-COMPUTE, NOT HMX'" + }, + "expert_kernel_cycles": null, + "tolerance": "hexlib_close_f16 (rel=0.01, abs=3e-4; derivation in harness.c's header)", + "tags": ["matmul", "bias", "gelu", "q4_0", "quantized-weight", "encoder", "first-rung"] +} diff --git a/kernels/matmul_fp16/RESULT.md b/kernels/matmul_fp16/RESULT.md new file mode 100644 index 0000000..333f798 --- /dev/null +++ b/kernels/matmul_fp16/RESULT.md @@ -0,0 +1,16 @@ +### hexlib verify — matmul_fp16 + +| gate | result | +|---|---| +| correct | PASS | +| max abs error | 0.000976562 (n_wrong 0) | +| kernel_cycles | 995714 | +| accel (ELF-proven) | hvx, hvx-compute | +| near-miss `nearmiss_fp16_accumulate.c` | correctly rejected | +| near-miss `nearmiss_transposed_operand.c` | correctly rejected | +| near-miss `nearmiss_wrong_batch_stride.c` | correctly rejected | +| **gate** | **PASS** | + +target `v75` · toolchain `19.0.04` · SDK `6.4.0.2` · host `sriha@Heathcliff` · `2026-08-12T09:35:31Z` + +Measured on the hexagon simulator under the pinned bus model (buspenalty 75, busratio 2). The simulator is cycle-approximate; these numbers are reproducible, not silicon measurements. diff --git a/kernels/matmul_fp16/baseline.c b/kernels/matmul_fp16/baseline.c new file mode 100644 index 0000000..9e33339 --- /dev/null +++ b/kernels/matmul_fp16/baseline.c @@ -0,0 +1,31 @@ +#include "kernel_api.h" + +/* Scalar reference. Correct and obvious, never fast. + * + * Matches hexlib/graph/opdefs/structural.py:55-61's reference lambda + * (`arrays[0] @ arrays[1]`) at the precision hexlib/graph/eager.py:20-29 fixes + * for it -- see kernel_api.h's SPEC comment for the derivation. The K-reduction + * is accumulated in `float` (never `hexlib_hf`) and rounded to fp16 exactly + * once, at the final store; that is the one property this file must not get + * wrong, because it is the file every near-miss and the kernel itself are + * checked against. + */ +void matmul_fp16_baseline(const hexlib_hf *A, const hexlib_hf *B, hexlib_hf *C, + int Bn, int M, int K, int N) { + for (int b = 0; b < Bn; ++b) { + const hexlib_hf *Ab = A + (long) b * M * K; + const hexlib_hf *Bb = B + (long) b * K * N; + hexlib_hf *Cb = C + (long) b * M * N; + for (int m = 0; m < M; ++m) { + const hexlib_hf *arow = Ab + (long) m * K; + hexlib_hf *crow = Cb + (long) m * N; + for (int n = 0; n < N; ++n) { + float acc = 0.0f; + for (int k = 0; k < K; ++k) { + acc += (float) arow[k] * (float) Bb[(long) k * N + n]; + } + crow[n] = (hexlib_hf) acc; + } + } + } +} diff --git a/kernels/matmul_fp16/harness.c b/kernels/matmul_fp16/harness.c new file mode 100644 index 0000000..4426a7a --- /dev/null +++ b/kernels/matmul_fp16/harness.c @@ -0,0 +1,155 @@ +/* kernels/matmul_fp16/harness.c + * + * Builds inputs, runs the baseline for reference, times ONLY the kernel call, + * compares with tolerance, and prints the two lines the driver parses. + * + * SHAPE: (Bn,M,K,N) = (MM_B,MM_M,MM_K,MM_N) = (3, 40, 128, 200). All four + * DELIBERATELY DIFFERENT numbers -- kernels/transpose_th_fp16's own + * convention (see its harness.c header comment): with any two of B/M/K/N + * equal, a stride-confusion or transposed-operand bug can produce a + * same-shape, same-size result that a shape check (or an unlucky data set) + * cannot see. All four distinct here means nearmiss_wrong_batch_stride.c and + * nearmiss_transposed_operand.c cannot pass by accident -- neither near-miss + * is HVX code at all (both are plain scalar C with no dependency on N's + * relationship to 64), so that guarantee holds regardless of what follows. + * + * N IS DELIBERATELY NOT A MULTIPLE OF 64 (200 = 3*64 + 8), unlike the real + * encoder shapes (N = 256 or 64, both multiples of 64) -- this is + * intentional, not an oversight. kernel.c's vectorised column-block loop + * used to dereference an aligned `HVX_Vector *` on both the B-row load and + * the C-row store, which lowers to a 128-byte-ALIGNED access, correct only + * when N % 64 == 0. At the old N=192, every row start (k * N * sizeof(hf) = + * k * 384 bytes) happens to be a multiple of 128 for every k, so that bug + * was invisible right here even though it silently corrupted any real, + * non-multiple-of-64 shape. Fixed in 9b6f7c0 by switching both the load and + * the store to the repo's `hvx_vmemu` unaligned wrapper + * (include/hexlib/hvx/hvx-base.h). At N=200, row starts (k * 400 bytes) are + * NOT all 128-byte-aligned, and this run now also exercises kernel.c's + * scalar tail (the last 8 columns, 192..199) in addition to its vectorised + * blocks -- so a regression back to an aligned dereference cannot hide + * behind this shape the way it hid behind N=192. + * + * GENERIC DATA. A[b][m][k] and B[b][k][n] are deterministic, small, and + * exact multiples of 0.25 (exactly representable in fp16), built from a + * formula that mixes b, m/k, and k/n so no two batches and no row/column look + * alike -- this is what makes nearmiss_wrong_batch_stride.c (always reads + * batch 0) and nearmiss_transposed_operand.c (reads B with (k,n) swapped) + * fail on ordinary data, without needing special-casing for them. + * + * THE ADVERSARIAL ELEMENT is C[1][5][7] -- batch 1, row 5, column 7 -- and it + * exists for exactly one purpose: to make accumulating the K-reduction in + * fp16 instead of float32 (nearmiss_fp16_accumulate.c) a LARGE, unmistakable + * error rather than a rounding-noise near-miss. Read this carefully, because + * a per-element tolerance loose enough to admit the real kernel's own + * legitimate noise can be looser than a real bug -- that already happened + * once in this repo (layernorm_fp16's unbiased-variance near-miss was wrongly + * accepted on its first run because the bug's size was smaller than fp16's + * own ULP noise; see kernels/softmax_fp16's harness.c for the same lesson + * applied to a fp16-accumulated sum). + * + * A[1][5][k] is set to 2.0 for k=0 and 2^-10 = 0.0009765625 (one fp16 ULP at + * magnitude ~1-2) for k=1..127 (127 identical followers); B[1][k][7] is set + * to 1.0 for every k. So C[1][5][7] = 2.0*1.0 + 127 * (2^-10 * 1.0). + * + * MEASURED IN PYTHON (numpy float32 / float16, same algorithm + * nearmiss_fp16_accumulate.c implements, K = MM_K = 128 terms: one 2.0 term + * plus 127 followers of 2^-10): + * float32 accumulate = 2.1240234375 + * correctly rounded ONCE to fp16 (this spec) = 2.125 (diff from the + * float32 sum: 0.0009765625, + * exactly one fp16 ULP -- the + * single unavoidable rounding + * every correct implementation + * pays exactly once) + * fp16-accumulated (the near-miss's bug) = 2.0 (diff from the + * float32 sum: 0.1240234375 -- + * 127x the correct kernel's + * own single-rounding error, + * ~5.8% relative) + * Every one of the 127 identical 2^-10 increments rounds away once the fp16 + * accumulator reaches magnitude ~2 (fp16 ULP there is 2^-9 = 0.001953125, so a + * 2^-10 increment is exactly half a ULP and never survives round-to-even) -- + * the same "many increments each near the accumulator's own ULP, added one at + * a time" shape that made kernels/softmax_fp16's row 1 discriminating. + * + * TOLERANCE (see hexlib_close_f16 calls below): rel=1e-2, abs=1e-3. The + * correct kernel's worst case on this element (one legitimate fp16 rounding, + * 0.0009765625 absolute) sits just under the absolute bound and far under the + * relative one (1e-2 * 2.125 = 0.02125); the near-miss's error (0.1240234375) + * clears BOTH by more than an order of magnitude (~124x the absolute bound, + * ~5.8x the relative one). This tolerance was derived from those two measured + * numbers, not loosened until something passed. + * + * Rows/columns/batches not involved in the adversarial element use the + * generic formula everywhere, including at batch 1 row 5 and column 7 outside + * k -- there is nothing special about this harness beyond the one element + * needed to discriminate the one quiet bug a friendly random data set would + * hide. + */ +#include "hexlib/hexlib_harness.h" +#include "kernel_api.h" + +void matmul_fp16_baseline(const hexlib_hf *, const hexlib_hf *, hexlib_hf *, + int, int, int, int); + +#define MM_ADV_B 1 +#define MM_ADV_M 5 +#define MM_ADV_N 7 + +static hexlib_hf A[MM_B * MM_M * MM_K] HEXLIB_ALIGN; +static hexlib_hf Bm[MM_B * MM_K * MM_N] HEXLIB_ALIGN; +static hexlib_hf C[MM_B * MM_M * MM_N] HEXLIB_ALIGN; +static hexlib_hf REF[MM_B * MM_M * MM_N] HEXLIB_ALIGN; + +static void fill(void) { + for (int b = 0; b < MM_B; ++b) { + for (int m = 0; m < MM_M; ++m) { + for (int k = 0; k < MM_K; ++k) { + int q = ((b * 131 + m * 17 + k * 7) % 13) - 6; /* -6..6 */ + A[(long) b * MM_M * MM_K + (long) m * MM_K + k] = + (hexlib_hf) ((float) q * 0.25f); + } + } + for (int k = 0; k < MM_K; ++k) { + for (int n = 0; n < MM_N; ++n) { + int q = ((b * 89 + k * 13 + n * 5) % 15) - 7; /* -7..7 */ + Bm[(long) b * MM_K * MM_N + (long) k * MM_N + n] = + (hexlib_hf) ((float) q * 0.25f); + } + } + } + + /* The adversarial row/column -- see the header comment for the numbers. */ + for (int k = 0; k < MM_K; ++k) { + float av = (k == 0) ? 2.0f : (1.0f / 1024.0f); /* 2^-10, exact in fp16 */ + A[(long) MM_ADV_B * MM_M * MM_K + (long) MM_ADV_M * MM_K + k] = (hexlib_hf) av; + Bm[(long) MM_ADV_B * MM_K * MM_N + (long) k * MM_N + MM_ADV_N] = (hexlib_hf) 1.0f; + } + + for (int i = 0; i < MM_B * MM_M * MM_N; ++i) { + C[i] = (hexlib_hf) 12345.0f; /* poison: a no-op kernel cannot pass */ + } +} + +int main(void) { + fill(); + + matmul_fp16_baseline(A, Bm, REF, MM_B, MM_M, MM_K, MM_N); + + unsigned long long kcyc = 0; + HEXLIB_TIME_KERNEL(kcyc, matmul_fp16(A, Bm, C, MM_B, MM_M, MM_K, MM_N)); + + int n_wrong = 0; + double max_err = 0.0; + for (int i = 0; i < MM_B * MM_M * MM_N; ++i) { + if (!hexlib_close_f16((float) C[i], (float) REF[i], 1e-2f, 1e-3f)) { + ++n_wrong; + } + double d = (double) (float) C[i] - (double) (float) REF[i]; + if (d < 0.0) d = -d; + if (d > max_err) max_err = d; + } + + hexlib_report(n_wrong == 0, n_wrong, max_err, kcyc); + return 0; +} diff --git a/kernels/matmul_fp16/kernel.c b/kernels/matmul_fp16/kernel.c new file mode 100644 index 0000000..829e87d --- /dev/null +++ b/kernels/matmul_fp16/kernel.c @@ -0,0 +1,146 @@ +/* kernels/matmul_fp16/kernel.c + * + * HVX-COMPUTE matmul. HMX was attempted first (see the report for what was + * tried and why it was abandoned: the HMX matrix-unit path needs the SSR.XE + * extension-context bit enabled from inside the kernel, and setting it from + * this standalone gate's runtime made the simulator hang rather than fault or + * proceed -- an environment difference from the two read-only reference + * projects this kernel started from, not a numerics problem, and not + * something safe to keep debugging blind against a 900s-per-run gate). Per + * the brief's own fallback guidance, this is the complete, HVX-compute + * deliverable: real vector ARITHMETIC (not just vector loads/stores), which + * is what `hexlib/anticheat.py`'s `used_hvx_compute` proof requires. + * + * SHAPE OF THE COMPUTE. HVX has no native dot-product-with-horizontal-reduce + * primitive worth using per output element here, so this vectorises across + * the OUTPUT ROW (the N axis) instead of the K reduction: for a fixed batch + * b and row m, accumulate `C[m][:] += A[m][k] * B[k][:]` for every k, where + * `A[m][k]` is a single scalar broadcast across a vector and `B[k][:]` is a + * real vector load -- an outer-product-style accumulation, one FMA-pair + * (mul + add) per 64-column block per k. This is the same "vectorise the + * elementwise axis, loop the reduction axis in scalar" shape + * kernels/softmax_fp16/kernel.c uses for its own reduction (max, then sum), + * just with the roles of "vectorised axis" and "reduction axis" swapped + * (columns vectorised, K reduced) because THIS op's reduction axis is not + * the last one. + * + * PRECISION. Accumulation is float32 throughout (never `hexlib_hf`), narrowed + * to fp16 exactly once at the end of the K loop -- see kernel_api.h's SPEC + * comment for where that requirement comes from (structural.py's matmul + * reference + eager.py's fp32-everywhere oracle). All fp32 arithmetic here + * goes through `hvx_vec_{add,mul}_f32_f32` (include/hexlib/hvx/hvx-base.h), + * which on this arch are `Q6_Vsf_equals_Vqf32(Q6_Vqf32_..._VsfVsf(...))` + * under the hood -- the qf32 path, never a native `Vhf`-typed accumulate. + * `Q6_Vhf_vadd_VhfVhf` does not exist on v75 and crashes clang 19.0.04 with + * exit code 70 (this repo's own hardware note); nothing here calls it. + * + * WIDEN/NARROW: reused from the vendored header, not reimplemented -- + * `hvx_vec_f16_to_f32` and `hvx_vec_f32_to_f16` (hvx-base.h) do the same + * shuffle-widen / deal-narrow dance kernels/softmax_fp16/kernel.c already + * verified line by line against layernorm_fp16's hand-rolled version. + * + * COLUMN BLOCKING. N is processed 64 fp16 lanes (= one HVX vector) at a time, + * each block held as a pair of float32 accumulator vectors (lo/hi 32 lanes). + * Both real encoder shapes have N a multiple of 64 (256 and 64), so the + * vectorised path covers them fully; a scalar tail below still handles a + * non-multiple-of-64 N correctly (not exercised by this harness, never + * silently wrong for a shape nobody vectorised -- the same tradeoff + * kernels/softmax_fp16/kernel.c documents for its own scratch cap). Rows (M) + * and the batches (Bn) are plain scalar loops: only the reduction's inner + * width (N) needs SIMD width, not the outer dimensions. + */ +#include "kernel_api.h" + +#include +#include + +#include "hexlib/hvx/hvx-base.h" + +#define LANES_FP16 64 + +/* Per-row float32 accumulator: one (lo, hi) fp32 vector pair per 64-column + * block. Sized for the encoder's own largest N (256 -> 4 blocks) with + * headroom; a wider N falls back to the plain scalar loop below rather than + * overflow this fixed array. */ +#define MM_MAX_NVEC64 8 + +void matmul_fp16(const hexlib_hf *A, const hexlib_hf *B, hexlib_hf *C, + int Bn, int M, int K, int N) { + if (Bn <= 0 || M <= 0 || K <= 0 || N <= 0) { + return; + } + + const int nvec64 = N / LANES_FP16; + const int vecN = nvec64 * LANES_FP16; + + if (nvec64 > MM_MAX_NVEC64) { + /* Wider than this kernel's fixed accumulator array -- correct but + * fully scalar rather than overflowing it. */ + for (int b = 0; b < Bn; ++b) { + const hexlib_hf *Ab = A + (long) b * M * K; + const hexlib_hf *Bb = B + (long) b * K * N; + hexlib_hf *Cb = C + (long) b * M * N; + for (int m = 0; m < M; ++m) { + for (int n = 0; n < N; ++n) { + float acc = 0.0f; + for (int k = 0; k < K; ++k) { + acc += (float) Ab[(long) m * K + k] * (float) Bb[(long) k * N + n]; + } + Cb[(long) m * N + n] = (hexlib_hf) acc; + } + } + } + return; + } + + HVX_Vector acc_lo[MM_MAX_NVEC64]; + HVX_Vector acc_hi[MM_MAX_NVEC64]; + float scalar_acc[LANES_FP16]; /* for the N % 64 tail, at most 63 live */ + + for (int b = 0; b < Bn; ++b) { + const hexlib_hf *Ab = A + (long) b * M * K; + const hexlib_hf *Bb = B + (long) b * K * N; + hexlib_hf *Cb = C + (long) b * M * N; + + for (int m = 0; m < M; ++m) { + const hexlib_hf *arow = Ab + (long) m * K; + hexlib_hf *crow = Cb + (long) m * N; + const HVX_Vector zero = Q6_V_vzero(); + + for (int i = 0; i < nvec64; ++i) { + acc_lo[i] = zero; + acc_hi[i] = zero; + } + for (int n = vecN; n < N; ++n) { + scalar_acc[n - vecN] = 0.0f; + } + + /* K is the reduction axis: loop it in scalar, vectorise every + * 64-column block of the row it touches. */ + for (int k = 0; k < K; ++k) { + const float av = (float) arow[k]; + const HVX_Vector va = hvx_vec_splat_f32(av); + const hexlib_hf *brow = Bb + (long) k * N; + + for (int i = 0; i < nvec64; ++i) { + HVX_Vector bvi = hvx_vmemu(brow + (long) i * LANES_FP16); + HVX_VectorPair bp = hvx_vec_f16_to_f32(bvi); + HVX_Vector blo = Q6_V_lo_W(bp); + HVX_Vector bhi = Q6_V_hi_W(bp); + acc_lo[i] = hvx_vec_add_f32_f32(acc_lo[i], hvx_vec_mul_f32_f32(va, blo)); + acc_hi[i] = hvx_vec_add_f32_f32(acc_hi[i], hvx_vec_mul_f32_f32(va, bhi)); + } + for (int n = vecN; n < N; ++n) { + scalar_acc[n - vecN] += av * (float) brow[n]; + } + } + + for (int i = 0; i < nvec64; ++i) { + hvx_vmemu(crow + (long) i * LANES_FP16) = hvx_vec_f32_to_f16(acc_lo[i], acc_hi[i]); + } + for (int n = vecN; n < N; ++n) { + crow[n] = (hexlib_hf) scalar_acc[n - vecN]; + } + } + } +} diff --git a/kernels/matmul_fp16/kernel_api.h b/kernels/matmul_fp16/kernel_api.h new file mode 100644 index 0000000..aa2c6eb --- /dev/null +++ b/kernels/matmul_fp16/kernel_api.h @@ -0,0 +1,80 @@ +/* kernels/matmul_fp16/kernel_api.h */ +#ifndef HEXLIB_MATMUL_FP16_API_H +#define HEXLIB_MATMUL_FP16_API_H + +typedef __fp16 hexlib_hf; + +/* Batched matmul, fp16 in and out, NO bias and NO activation (that fused op is + * `matmul_epilogue` -- hexlib/graph/opdefs/fused.py -- a different op and a + * different kernel, kernels/matmul_epilogue_fp16/, out of scope here). + * + * SPEC, taken from: + * - hexlib/graph/opdefs/structural.py:55-61 -- the `matmul` OpDef itself: + * reference=lambda arrays, attrs: (arrays[0] @ arrays[1]).astype(arrays[0].dtype) + * - hexlib/graph/eager.py:20-29 (`NUMPY_DTYPE`) -- the ACCUMULATION PRECISION + * this reference actually runs at. Its own comment: "Every dtype is fed and + * computed as fp32 in the oracle" -- `NUMPY_DTYPE["fp16"] = float32`. So by + * the time `arrays[0] @ arrays[1]` runs, both operands are already float32 + * arrays, `@` accumulates in (at least) float32, and `.astype(arrays[0].dtype)` + * is a no-op (arrays[0] is float32 already, not fp16). fp16 rounding happens + * exactly once, later, when `env.run` (eager.py:125) casts the op's output + * into the graph environment's fp16-tagged tensor -- never inside the matmul + * itself and never per partial sum. + * + * i.e. for each of the Bn independent batches (A is [M, K] row-major, B is + * [K, N] row-major -- B already arrives pre-transposed to [k, n], per + * structural.py's own module docstring, lines 3-5; C is [M, N] row-major): + * + * out[m][n] = sum_{k=0}^{K-1} (float) A[m][k] * (float) B[k][n] <- fp32 sum + * C[m][n] = (fp16) out[m][n] <- rounded ONCE + * + * THE QUIET WRONG ANSWER THIS RULES OUT: accumulating that K-reduction in fp16 + * instead of float32. At small K, on friendly data, that difference is smaller + * than fp16's own legitimate 1-ULP rounding noise and is invisible to a loose + * tolerance -- exactly how kernels/layernorm_fp16's unbiased-variance near-miss + * was wrongly accepted once already (see ROADMAP.md / that kernel's own + * comments). At this kernel's K (up to 256 in the real encoder; 128 in this + * harness) it is not invisible -- see harness.c and nearmiss_fp16_accumulate.c + * for the measured numbers that prove the harness discriminates it from + * ordinary rounding noise, not just asserts that it does. + * + * SHAPES. The compiled plan needs exactly two, both batched matmuls with no + * bias and no activation: + * fp16 (12, 256, 64) @ fp16 (12, 64, 256) -> fp16 (12, 256, 256) [QK^T] + * fp16 (12, 256, 256) @ fp16 (12, 256, 64) -> fp16 (12, 256, 64) [AV] + * i.e. B=12, (M,K,N) = (256,64,256) or (256,256,64). This kernel takes + * (Bn, M, K, N) as parameters and covers both from one implementation, because + * they are one op with two sizes, not two ops. + * + * MECHANISM. See kernel.c: this is an HVX-COMPUTE kernel (real vector + * arithmetic -- multiply and add -- never just vector loads/stores). An HMX + * (matrix-unit) version was attempted first; see the report for why it was + * abandoned in favor of this one. HVX vectorises across the OUTPUT ROW (the + * N axis) with the K reduction as a scalar outer loop: `C[m][:] += A[m][k] * + * B[k][:]` accumulated in float32 vectors, one 64-column block at a time. + * Both real encoder N values (256, 64) are multiples of 64 and take the fully + * vectorised path; a scalar tail below still handles any N that is not, for + * a future caller this kernel was not tuned for. + * + * ROUNDING. HVX's fp16<->fp32 widen/narrow (`hvx_vec_f16_to_f32` / + * `hvx_vec_f32_to_f16`, include/hexlib/hvx/hvx-base.h) goes through the qf32 + * path and its narrow interleaves lanes via `Q6_Vh_vdeal_Vh` -- not + * necessarily IEEE round-to-nearest-even, the same caveat this repo already + * documents for `Q6_Vhf_equals_Wqf32`. So this kernel's result can differ + * from the float32-accumulate-then-round-once reference by up to roughly one + * fp16 ULP. That is why it is tolerance-compared (hexlib_close_f16), never + * bit-exact. + * + * ALIGNMENT. A, B, and C must be 128-byte aligned; N should be a multiple of + * 64 to take the vectorised path for the whole row (both real encoder shapes + * satisfy this). + */ +#define MM_B 3 +#define MM_M 40 +#define MM_K 128 +#define MM_N 200 + +void matmul_fp16(const hexlib_hf *A, const hexlib_hf *B, hexlib_hf *C, + int Bn, int M, int K, int N); + +#endif diff --git a/kernels/matmul_fp16/nearmiss_fp16_accumulate.c b/kernels/matmul_fp16/nearmiss_fp16_accumulate.c new file mode 100644 index 0000000..804464e --- /dev/null +++ b/kernels/matmul_fp16/nearmiss_fp16_accumulate.c @@ -0,0 +1,50 @@ +/* A plausible WRONG implementation the harness must reject. + * + * THE MISTAKE: accumulating the K-reduction in fp16 (__fp16, rounding to fp16 + * after EVERY multiply-add) instead of float32. Everything else is IDENTICAL + * to baseline.c -- same loop nest, same data, same final store. Only the type + * of the running sum changes. + * + * WHY ANYONE WOULD WRITE IT. A and B are both fp16, the product of two fp16 + * values "looks like" it belongs in an fp16 accumulator, and on a target + * where an fp16 register is one lane width and float32 is two, accumulating + * in the storage dtype looks consistent rather than careless. Nothing about + * the C source looks unstable; the loop is the same loop. + * + * WHY THE HARNESS CATCHES IT: see harness.c's header comment for the full + * derivation. In short, its adversarial element (batch 1, row 5, column 7) is + * one dominant product (2.0) plus 127 identical followers, each exactly one + * fp16 ULP at the accumulator's own magnitude (2^-10 at ~1-2) -- individually + * too small to survive round-to-even once added to an accumulator that has + * already reached that magnitude. MEASURED (Python, numpy float32/float16, + * this exact algorithm): float32 sum 2.1240234375, this near-miss's fp16- + * accumulated sum 2.0 -- an absolute error of 0.1240234375, ~127x the correct + * kernel's own single-rounding error (0.0009765625) and ~5.8% relative, + * comfortably outside the harness's tolerance (rel=1e-2, abs=1e-3) on both + * measures. On the harness's OTHER (non-adversarial) elements this bug is far + * smaller and can be invisible -- that is exactly why the harness does not + * rely on generic data alone to catch it. + */ +#include "kernel_api.h" + +void matmul_fp16(const hexlib_hf *A, const hexlib_hf *B, hexlib_hf *C, + int Bn, int M, int K, int N) { + for (int b = 0; b < Bn; ++b) { + const hexlib_hf *Ab = A + (long) b * M * K; + const hexlib_hf *Bb = B + (long) b * K * N; + hexlib_hf *Cb = C + (long) b * M * N; + for (int m = 0; m < M; ++m) { + for (int n = 0; n < N; ++n) { + /* WRONG: the running sum is __fp16, so it rounds to fp16 + * after every single multiply-add instead of accumulating in + * float32. */ + hexlib_hf acc16 = (hexlib_hf) 0.0f; + for (int k = 0; k < K; ++k) { + float p = (float) Ab[(long) m * K + k] * (float) Bb[(long) k * N + n]; + acc16 = (hexlib_hf) ((float) acc16 + p); + } + Cb[(long) m * N + n] = acc16; + } + } + } +} diff --git a/kernels/matmul_fp16/nearmiss_transposed_operand.c b/kernels/matmul_fp16/nearmiss_transposed_operand.c new file mode 100644 index 0000000..2dcd2ee --- /dev/null +++ b/kernels/matmul_fp16/nearmiss_transposed_operand.c @@ -0,0 +1,45 @@ +/* A plausible WRONG implementation the harness must reject. + * + * THE MISTAKE: reading B as if it were stored [N, K] row-major (the "weight + * transposed" convention many BLAS-style APIs default to) instead of the + * [K, N] row-major layout this op's own spec requires (kernel_api.h's SPEC + * comment / hexlib/graph/opdefs/structural.py's "weights arrive pre- + * transposed to [k, n]" docstring). Concretely: `B[n*K + k]` instead of the + * correct `B[k*N + n]`. + * + * WHY ANYONE WOULD WRITE IT. "matmul with the second operand transposed" is + * such a common BLAS convention (sgemm's TRANSB) that swapping the stride + * pair is an easy slip, especially once QK^T is on your mind -- attention's + * QK^T literally IS A @ K^T, so a kernel author moving between the two matmul + * call sites can genuinely misremember which one this generic kernel expects. + * + * WHY THE HARNESS CATCHES IT: this indexing stays in-bounds regardless of the + * shape (max index n*K+k = (N-1)*K+(K-1) < N*K always), so it cannot be + * caught by a bounds check or a crash -- only by comparing values. The + * harness's B data is NOT symmetric under this transpose (kernels/ + * transpose_th_fp16's own convention: B/M/K/N are all different numbers here, + * and the fill formula mixes k and n asymmetrically), so this near-miss + * reads essentially unrelated elements and produces a wrong value at nearly + * every output position, not just the adversarial one. + */ +#include "kernel_api.h" + +void matmul_fp16(const hexlib_hf *A, const hexlib_hf *B, hexlib_hf *C, + int Bn, int M, int K, int N) { + for (int b = 0; b < Bn; ++b) { + const hexlib_hf *Ab = A + (long) b * M * K; + const hexlib_hf *Bb = B + (long) b * K * N; + hexlib_hf *Cb = C + (long) b * M * N; + for (int m = 0; m < M; ++m) { + for (int n = 0; n < N; ++n) { + float acc = 0.0f; + for (int k = 0; k < K; ++k) { + /* WRONG: should be Bb[k*N + n] -- B is [K, N], not + * [N, K]. */ + acc += (float) Ab[(long) m * K + k] * (float) Bb[(long) n * K + k]; + } + Cb[(long) m * N + n] = (hexlib_hf) acc; + } + } + } +} diff --git a/kernels/matmul_fp16/nearmiss_wrong_batch_stride.c b/kernels/matmul_fp16/nearmiss_wrong_batch_stride.c new file mode 100644 index 0000000..b410204 --- /dev/null +++ b/kernels/matmul_fp16/nearmiss_wrong_batch_stride.c @@ -0,0 +1,41 @@ +/* A plausible WRONG implementation the harness must reject. + * + * THE MISTAKE: computing the per-batch pointer offset for A from `m` and `k` + * (the loop variables already in scope) instead of from `b` -- so every batch + * reads and writes batch 0's slice of A. B and C use the correct `b`-scaled + * offset, which is what makes this "plausible": the bug is a single wrong + * multiplier on one of three pointers, not a structural rewrite. + * + * WHY ANYONE WOULD WRITE IT. `M * K` is the per-batch element count for A, + * and `(long) m * K` (the correct per-ROW offset within a batch) is a + * visually similar expression to `(long) b * M * K` (the correct per-BATCH + * offset) -- both are "some index times K, cast to long, added to a base + * pointer". A copy-paste that grabs the row-offset idiom for the batch-offset + * site compiles clean and is correct for b=0. + * + * WHY THE HARNESS CATCHES IT: it is caught only because A differs across + * batches. harness.c's fill formula mixes `b` into every element of A (and of + * B, though B's offset here is not the bug), so batch 1 and batch 2 read + * batch 0's A values entirely -- wrong on nearly every output element in + * those two batches, not just a rounding-sized difference. + */ +#include "kernel_api.h" + +void matmul_fp16(const hexlib_hf *A, const hexlib_hf *B, hexlib_hf *C, + int Bn, int M, int K, int N) { + for (int b = 0; b < Bn; ++b) { + /* WRONG: always batch 0 of A, regardless of b. */ + const hexlib_hf *Ab = A; + const hexlib_hf *Bb = B + (long) b * K * N; + hexlib_hf *Cb = C + (long) b * M * N; + for (int m = 0; m < M; ++m) { + for (int n = 0; n < N; ++n) { + float acc = 0.0f; + for (int k = 0; k < K; ++k) { + acc += (float) Ab[(long) m * K + k] * (float) Bb[(long) k * N + n]; + } + Cb[(long) m * N + n] = (hexlib_hf) acc; + } + } + } +} diff --git a/kernels/matmul_fp16/spec.json b/kernels/matmul_fp16/spec.json new file mode 100644 index 0000000..1ebe4dd --- /dev/null +++ b/kernels/matmul_fp16/spec.json @@ -0,0 +1,33 @@ +{ + "task_id": "matmul_fp16", + "dtype": "fp16", + "caps": [], + "mechanisms": [ + "hvx" + ], + "params": { + "Bn": 3, + "M": 40, + "K": 128, + "N": 200, + "encoder_shapes": [ + "fp16 (12, 256, 64) @ fp16 (12, 64, 256) -> fp16 (12, 256, 256) [QK^T]", + "fp16 (12, 256, 256) @ fp16 (12, 256, 64) -> fp16 (12, 256, 64) [AV]" + ], + "encoder_op_count": 24, + "no_bias_no_activation": "matmul_epilogue is a different op/kernel", + "mechanism": "hvx-compute: N vectorised 64-fp16-lanes-at-a-time (outer-product accumulation over K), qf32 accumulate, single narrow to fp16 at the end of each row's K loop", + "hmx_attempted": "yes -- abandoned after the standalone gate runtime hung when the kernel enabled the HMX/SSR.XE extension context from inside kernel.c; see the kernel-matmul-report.md for what was tried", + "fallback": "plain scalar float32-accumulate loop for N wider than this kernel's fixed 8-block (512-column) accumulator; the vectorised column blocks use unaligned HVX loads/stores (hvx_vmemu) precisely because N is not guaranteed to be a multiple of 64 -- an earlier version used an aligned HVX_Vector dereference here, which silently corrupted the vectorised columns for any N % 64 != 0; a per-column scalar tail still handles the leftover columns past the last full 64-wide block; N=200 (not a multiple of 64) is gated specifically to keep this from regressing", + "gate_shape_is_not_set_here": "Bn/M/K/N above are DESCRIPTIVE ONLY. The gate's real shape is #defined as MM_B/MM_M/MM_K/MM_N in kernel_api.h and nothing passes these values to the build -- editing them here changes NOTHING about what runs. Verified 2026-08-13 after changing N here 192->200 and M 40->20 and finding the rebuilt near-miss ELFs byte-identical to the previous run's. Widening the gate to a non-multiple-of-64 N (to cover the aligned-load bug fixed in 9b6f7c0) means editing MM_N in kernel_api.h, and re-checking that the harness's adversarial element (MM_ADV_N) still works at the new N. Done 2026-08-13: kernel_api.h's MM_N is now 200 (matches this descriptive value again); the adversarial element and harness.c's own shape reasoning were re-verified at N=200 -- see harness.c's header comment." + }, + "expert_kernel_cycles": null, + "tolerance": "hexlib_close_f16", + "tags": [ + "matmul", + "attention", + "encoder", + "qkt", + "av" + ] +} diff --git a/kernels/patchify_fp32/RESULT.md b/kernels/patchify_fp32/RESULT.md new file mode 100644 index 0000000..aac40dd --- /dev/null +++ b/kernels/patchify_fp32/RESULT.md @@ -0,0 +1,16 @@ +### hexlib verify — patchify_fp32 + +| gate | result | +|---|---| +| correct | PASS | +| max abs error | 0 (n_wrong 0) | +| kernel_cycles | 2018331 | +| accel (ELF-proven) | hvx · movement-only (no arithmetic) | +| near-miss `nearmiss_channel_temporal_swap.c` | correctly rejected | +| near-miss `nearmiss_merge_ignored.c` | correctly rejected | +| near-miss `nearmiss_patch_interior_swap.c` | correctly rejected | +| **gate** | **PASS** | + +target `v75` · toolchain `19.0.04` · SDK `6.4.0.2` · host `sriha@Heathcliff` · `2026-08-11T21:47:01Z` + +Measured on the hexagon simulator under the pinned bus model (buspenalty 75, busratio 2). The simulator is cycle-approximate; these numbers are reproducible, not silicon measurements. diff --git a/kernels/patchify_fp32/baseline.c b/kernels/patchify_fp32/baseline.c new file mode 100644 index 0000000..19c83bd --- /dev/null +++ b/kernels/patchify_fp32/baseline.c @@ -0,0 +1,54 @@ +#include "kernel_api.h" + +/* Scalar reference. Correct and obvious, never fast. + * + * Written directly against the grid, deliberately NOT reusing kernel.c's + * row-staging structure: for every (gh, gw) patch in the grid, in plain grid + * order, compute which output row it lands in (structural.py:204-206's merge + * reshape/transpose) and copy that one patch's (C, T, ph, pw) pixels + * (structural.py:202's H/W split) straight out of the image. No arithmetic + * anywhere -- the comparison against the kernel is exact. + * + * See kernel_api.h for the full citation of hexlib/graph/opdefs/ + * structural.py's patchify OpDef, whose numpy reference this mirrors. + */ +void patchify_fp32_baseline(const float *img, float *out, + int C, int T, int H, int W, + int patch, int merge, int grid_h, int grid_w) { + if (C <= 0 || T <= 0 || H <= 0 || W <= 0 || patch <= 0 || merge <= 0 + || grid_h <= 0 || grid_w <= 0) { + return; + } + + const int Bw = grid_w / merge; + const int out_cols = C * T * patch * patch; + + for (int gh = 0; gh < grid_h; ++gh) { + const int bh = gh / merge; + const int mh = gh % merge; + + for (int gw = 0; gw < grid_w; ++gw) { + const int bw = gw / merge; + const int mw = gw % merge; + const int token = ((bh * Bw + bw) * merge + mh) * merge + mw; + + for (int c = 0; c < C; ++c) { + for (int t = 0; t < T; ++t) { + for (int ph = 0; ph < patch; ++ph) { + const int h = gh * patch + ph; + const long src_row = (((long) c * T) + t) * H * W + + (long) h * W; + const long feat_row = (long) ((c * T + t) * patch + ph) + * patch; + + for (int pw = 0; pw < patch; ++pw) { + const int w = gw * patch + pw; + out[(long) token * out_cols + feat_row + pw] = + img[src_row + w]; + } + } + } + } + } + } +} diff --git a/kernels/patchify_fp32/harness.c b/kernels/patchify_fp32/harness.c new file mode 100644 index 0000000..08e035d --- /dev/null +++ b/kernels/patchify_fp32/harness.c @@ -0,0 +1,94 @@ +/* kernels/patchify_fp32/harness.c + * + * TWO SHAPES, ONE VERDICT. The encoder shape (PF_*) is W=256=8*32, so it + * exercises kernel.c's whole-vector row-staging path and is what the cycle + * count is measured on. The small shape (PF2_*) has W=12 -- not a multiple + * of the 32-lane fp32 vector -- so it exercises the scalar tail path that + * PF_* never reaches, and it uses C != T (2 vs 3) so a channel/temporal + * stride swap is actually detectable (see nearmiss_channel_temporal_swap.c's + * own note on why this matters). + * + * Every input element gets its own exact integer value (its flat index -- + * fp32 represents integers up to 2^24 exactly, and both shapes here are far + * smaller than that), so a comparison cannot pass by two different positions + * happening to hold equal numbers. The check is exact: this op does no + * arithmetic, so any difference at all is a bug. + */ +#include "hexlib/hexlib_harness.h" +#include "kernel_api.h" + +void patchify_fp32_baseline(const float *, float *, int, int, int, int, + int, int, int, int); + +#define PF_IN_N (PF_C * PF_T * PF_H * PF_W) +#define PF_OUT_N (PF_GRID_H * PF_GRID_W * PF_C * PF_T * PF_PATCH * PF_PATCH) + +#define PF2_IN_N (PF2_C * PF2_T * PF2_H * PF2_W) +#define PF2_OUT_N (PF2_GRID_H * PF2_GRID_W * PF2_C * PF2_T * PF2_PATCH * PF2_PATCH) + +#define MAXN (PF_IN_N > PF_OUT_N ? PF_IN_N : PF_OUT_N) + +static float X[MAXN] HEXLIB_ALIGN; +static float Y[MAXN] HEXLIB_ALIGN; +static float REF[MAXN] HEXLIB_ALIGN; + +static int check(int C, int T, int H, int W, int patch, int merge, + int grid_h, int grid_w, int n_in, int n_out, + int *n_wrong, double *max_err) { + for (int i = 0; i < n_in; ++i) { + X[i] = (float) i; /* distinct, exactly representable */ + } + for (int i = 0; i < n_out; ++i) { + Y[i] = 12345.0f; + REF[i] = 0.0f; + } + + patchify_fp32_baseline(X, REF, C, T, H, W, patch, merge, grid_h, grid_w); + patchify_fp32(X, Y, C, T, H, W, patch, merge, grid_h, grid_w); + + for (int i = 0; i < n_out; ++i) { + double d = (double) Y[i] - (double) REF[i]; + if (d != 0.0) { + ++(*n_wrong); + } + if (d < 0.0) d = -d; + if (d > *max_err) *max_err = d; + } + return n_out; +} + +int main(void) { + int n_wrong = 0; + double max_err = 0.0; + + unsigned long long kcyc = 0; + /* Timed call is the encoder's own shape (the whole-vector row path). The + * small shape is checked for correctness but deliberately not folded + * into the cycle count, which would make the number mean nothing. */ + { + for (int i = 0; i < PF_IN_N; ++i) { + X[i] = (float) i; + } + for (int i = 0; i < PF_OUT_N; ++i) { + Y[i] = 12345.0f; + } + patchify_fp32_baseline(X, REF, PF_C, PF_T, PF_H, PF_W, PF_PATCH, + PF_MERGE, PF_GRID_H, PF_GRID_W); + HEXLIB_TIME_KERNEL(kcyc, patchify_fp32(X, Y, PF_C, PF_T, PF_H, PF_W, + PF_PATCH, PF_MERGE, + PF_GRID_H, PF_GRID_W)); + for (int i = 0; i < PF_OUT_N; ++i) { + double d = (double) Y[i] - (double) REF[i]; + if (d != 0.0) ++n_wrong; + if (d < 0.0) d = -d; + if (d > max_err) max_err = d; + } + } + + /* Scalar-tail + non-trivial-merge shape. */ + check(PF2_C, PF2_T, PF2_H, PF2_W, PF2_PATCH, PF2_MERGE, + PF2_GRID_H, PF2_GRID_W, PF2_IN_N, PF2_OUT_N, &n_wrong, &max_err); + + hexlib_report(n_wrong == 0, n_wrong, max_err, kcyc); + return 0; +} diff --git a/kernels/patchify_fp32/kernel.c b/kernels/patchify_fp32/kernel.c new file mode 100644 index 0000000..7f4f060 --- /dev/null +++ b/kernels/patchify_fp32/kernel.c @@ -0,0 +1,88 @@ +/* [C, T, H, W] -> [grid_h*grid_w, C*T*patch*patch], fp32. See kernel_api.h + * for the full index-arithmetic derivation (taken verbatim from + * hexlib/graph/opdefs/structural.py's patchify OpDef, lines 145-207) and the + * "WHY IT IS FAST" note this implementation follows. + * + * SHAPE OF THE MOVEMENT. grid_w * patch == W, so a full row of the image, for + * one (c, t, h), is W contiguous fp32 with no arithmetic performed on it at + * all -- it just needs to land in grid_w different destination rows, sliced + * into patch-wide (16-float) pieces. The read side of that is a genuinely + * whole-vector-aligned bulk move whenever W is a multiple of the 32-lane fp32 + * vector (true for the encoder's only shape, W=256=8*32); the write side + * never is, because each destination row lives PF_PATCH*PF_PATCH*C*T floats + * away from its neighbours and no run longer than one patch-row (16 floats, + * half a vector) is ever contiguous in the destination either. So: HVX for + * the row-wide read, scalar for the redistribution -- there is nothing to + * gain from forcing the scatter into vector form (it would need real + * cross-lane shuffles for no benefit, since the store addresses are neither + * contiguous nor a fixed stride HVX can address directly). + */ +#include "kernel_api.h" + +#include +#include + +#define VEC_BYTES 128 +#define VEC_FLOATS (VEC_BYTES / (int) sizeof(float)) /* 32 */ + +/* The encoder's only shape has W = grid_w * patch = 256; this is a hard cap + * on the local row-staging buffer, not a general limit on the op itself. */ +#define PATCHIFY_ROWBUF_MAX 256 + +void patchify_fp32(const float *img, float *out, + int C, int T, int H, int W, + int patch, int merge, int grid_h, int grid_w) { + if (C <= 0 || T <= 0 || H <= 0 || W <= 0 || patch <= 0 || merge <= 0 + || grid_h <= 0 || grid_w <= 0 || W > PATCHIFY_ROWBUF_MAX) { + return; + } + + const int Bw = grid_w / merge; + const int out_cols = C * T * patch * patch; + + const int nvec = W / VEC_FLOATS; /* whole vectors per row */ + const int tail = W - nvec * VEC_FLOATS; + + float rowbuf[PATCHIFY_ROWBUF_MAX] __attribute__((aligned(VEC_BYTES))); + + for (int c = 0; c < C; ++c) { + for (int t = 0; t < T; ++t) { + const float *chan = img + (((long) c * T) + t) * H * W; + + for (int h = 0; h < H; ++h) { + const float *src_row = chan + (long) h * W; + + /* Bulk-load the whole row through the vector unit: every + * element of it is needed by SOME destination patch, so this + * is not speculative over-fetch, just wider instructions for + * work the scalar loop below would do one float at a time. */ + const HVX_Vector *sv = (const HVX_Vector *) src_row; + HVX_Vector *dv = (HVX_Vector *) rowbuf; + for (int v = 0; v < nvec; ++v) { + dv[v] = sv[v]; + } + for (int w = W - tail; w < W; ++w) { + rowbuf[w] = src_row[w]; + } + + const int gh = h / patch; + const int ph = h % patch; + const int bh = gh / merge; + const int mh = gh % merge; + const int feat_row_base = ((c * T + t) * patch + ph) * patch; + + for (int gw = 0; gw < grid_w; ++gw) { + const int bw = gw / merge; + const int mw = gw % merge; + const int token = ((bh * Bw + bw) * merge + mh) * merge + mw; + + float *dst = out + (long) token * out_cols + feat_row_base; + const float *rp = rowbuf + (long) gw * patch; + for (int pw = 0; pw < patch; ++pw) { + dst[pw] = rp[pw]; + } + } + } + } + } +} diff --git a/kernels/patchify_fp32/kernel_api.h b/kernels/patchify_fp32/kernel_api.h new file mode 100644 index 0000000..7ffb243 --- /dev/null +++ b/kernels/patchify_fp32/kernel_api.h @@ -0,0 +1,120 @@ +/* kernels/patchify_fp32/kernel_api.h + * + * The vision encoder's patch-embedding rearrangement: + * + * [C, T, H, W] -> [grid_h*grid_w, C*T*patch*patch] + * + * The encoder's only shape: C=3, T=2, H=W=256, patch=16, merge=2, + * grid_h=grid_w=16 -- so [3, 2, 256, 256] fp32 -> [256, 1536] fp32 + * (1536 = 3*2*16*16, 256 = 16*16). + * + * THE INDEX ARITHMETIC IS TAKEN VERBATIM FROM THE OP REGISTRY, NOT DERIVED + * HERE. `hexlib/graph/opdefs/structural.py`'s `patchify` OpDef is the + * authority on what this op means; the eager numpy implementation at + * `_patchify_reference` (structural.py:172-207) is the specification this + * kernel and its baseline must both match bit-for-bit. Restated as scalar + * index arithmetic: + * + * gh = h / patch, ph = h % patch (structural.py:202's first reshape + * gw = w / patch, pw = w % patch splits H into grid_h*patch and + * W into grid_w*patch) + * + * bh = gh / merge, mh = gh % merge (structural.py:204's second + * bw = gw / merge, mw = gw % merge reshape splits the grid into + * merge blocks) + * + * Bw = grid_w / merge + * token = ((bh * Bw + bw) * merge + mh) * merge + mw + * (structural.py:206's + * .transpose(2, 5, 3, 6, 0, 1, 4, 7) + * puts token axes in (bh, bw, mh, + * mw) order, then the trailing + * .reshape flattens them row-major) + * + * feat = ((c * T + t) * patch + ph) * patch + pw + * (the same transpose puts feature + * axes in (c, t, ph, pw) order) + * + * out[token][feat] = img[c][t][h][w] + * + * MERGE DOES AFFECT THE OUTPUT ROW ORDER -- it is not decorative metadata. + * structural.py:172-179's own docstring is explicit about why: the patch + * merger downstream is a pure reshape (cited there as + * modeling_qwen3_5.py:886), so consecutive runs of merge*merge=4 rows must + * already BE the 2x2 spatial block the merger will fold together. Per-patch + * FEATURE order, in contrast, is untouched by merge: each output row still + * holds exactly one patch's (C, T, ph, pw) pixels, never several patches' + * pixels concatenated into one row. Getting this backwards -- reordering + * features by merge instead of rows, or dropping the reordering and emitting + * plain raster order -- produces a correctly SHAPED wrong answer that no + * shape check catches; see nearmiss_merge_ignored.c. + * + * WHY IT IS FAST (movement_only -- no arithmetic anywhere in this op). + * grid_w * patch == W exactly (16 * 16 == 256), so a full source image row, + * for one fixed (c, t, h), is 256 contiguous fp32 = 8 whole 128-byte HVX + * vectors with NO remainder. The kernel bulk-loads that row through the + * vector unit (32 elements per instruction instead of one) into a local + * staging buffer, then redistributes its 16 patch-width (16-float = 64-byte, + * sub-vector) slices to their 16 different destination rows with scalar + * stores -- the redistribution has no contiguous run longer than one patch + * row in either operand, so there is nothing left to vectorise there. This + * mirrors transpose_th_fp16's rule: vectorise whatever genuinely maps onto + * whole aligned vectors, scalar for what does not. + * + * ALIGNMENT / SIZE CONTRACT. `img` must be 128-byte aligned (HEXLIB_ALIGN) + * and W <= PATCHIFY_ROWBUF_MAX (256, this op's only width, defined in + * kernel.c). A W not a multiple of 32 floats still works (via the scalar + * tail path over the last few elements of the row) but is never exercised + * on the encoder's actual shape, where W == 256 divides evenly. + * + * TOLERANCE. Pure data movement, no arithmetic anywhere -- the comparison + * against the baseline is EXACT, matching transpose_th_fp16's rationale. + */ +#ifndef HEXLIB_PATCHIFY_FP32_API_H +#define HEXLIB_PATCHIFY_FP32_API_H + +/* The encoder's one real shape. */ +#define PF_C 3 +#define PF_T 2 +#define PF_H 256 +#define PF_W 256 +#define PF_PATCH 16 +#define PF_MERGE 2 +#define PF_GRID_H 16 +#define PF_GRID_W 16 + +/* A second, deliberately small shape whose W is NOT a multiple of the + * 32-lane fp32 vector, so the harness exercises kernel.c's scalar tail path + * (never hit by the encoder's own W=256) as well as a non-trivial merge + * block (merge=2 with a grid taller/wider than one block) and a genuine + * channel/temporal asymmetry (C != T, so a stride swap between them is + * detectable -- see nearmiss_channel_temporal_swap.c). + */ +#define PF2_C 2 +#define PF2_T 3 +#define PF2_H 12 +#define PF2_W 12 +#define PF2_PATCH 3 +#define PF2_MERGE 2 +#define PF2_GRID_H 4 +#define PF2_GRID_W 4 + +/* + * img: [C, T, H, W], row-major, 128-byte aligned. + * out: [grid_h*grid_w, C*T*patch*patch], row-major. + * + * Caller-supplied C, T, H, W, patch, merge, grid_h, grid_w must satisfy + * grid_h*patch == H, grid_w*patch == W, T given == the T the image was + * packed with, and grid_h % merge == grid_w % merge == 0 -- exactly the + * invariants `_patchify_infer` (structural.py:145-169) checks in the + * registry. This kernel does not re-validate them at runtime (a DSP kernel + * is not the place for the error path the graph builder already owns) but + * degrades safely (returns without writing) if the obviously-fatal ones + * (non-positive dims, W larger than the row-buffer's fixed 256-float + * capacity) are violated. + */ +void patchify_fp32(const float *img, float *out, + int C, int T, int H, int W, + int patch, int merge, int grid_h, int grid_w); + +#endif diff --git a/kernels/patchify_fp32/nearmiss_channel_temporal_swap.c b/kernels/patchify_fp32/nearmiss_channel_temporal_swap.c new file mode 100644 index 0000000..42272c1 --- /dev/null +++ b/kernels/patchify_fp32/nearmiss_channel_temporal_swap.c @@ -0,0 +1,71 @@ +/* A plausible WRONG implementation the harness must reject. + * + * THE MISTAKE: computing the channel offset as if the image were packed + * [T, C, H, W] instead of the real [C, T, H, W] -- i.e. treating T as the + * OUTER stride (C*H*W... no, T*H*W) and C as the inner one, rather than the + * other way round. + * + * WHY ANYONE WOULD WRITE IT. C and T sit right next to each other at the + * front of the shape, both are small (3 and 2 for the encoder), and the + * combined index expression `(c * T + t) * H * W` genuinely looks + * symmetric with its wrong twin `(t * C + c) * H * W` -- swapping which of + * the two multiplies the OTHER's extent is a one-character stride + * confusion, not a logic error a reviewer would spot by eye. It is exactly + * the class of mistake this project's own task description calls out: + * "transposing the channel and temporal axes ... a stride confusion + * produces a same-sized wrong answer." + * + * WHY THE HARNESS CATCHES IT: only because C != T. Both orderings produce a + * [C*T, H, W]-shaped flattened channel axis of the same total size, so + * every shape check still passes -- but for C=3, T=2 the two strides (T*H*W + * vs C*H*W) differ, so almost every (c, t) pair reads from the wrong + * channel-major slab. At C == T the two expressions would be identical and + * this kernel would be silently CORRECT, which is why the harness's PF2_* + * shape deliberately uses C=2, T=3 (as well as the encoder's own C=3, T=2) + * rather than a square C==T shape that would let this slip through. + */ +#include "kernel_api.h" + +#define PATCHIFY_ROWBUF_MAX 256 + +void patchify_fp32(const float *img, float *out, + int C, int T, int H, int W, + int patch, int merge, int grid_h, int grid_w) { + if (C <= 0 || T <= 0 || H <= 0 || W <= 0 || patch <= 0 || merge <= 0 + || grid_h <= 0 || grid_w <= 0 || W > PATCHIFY_ROWBUF_MAX) { + return; + } + + const int Bw = grid_w / merge; + const int out_cols = C * T * patch * patch; + + for (int c = 0; c < C; ++c) { + for (int t = 0; t < T; ++t) { + /* WRONG: (t * C + c), as though the image were packed + * [T, C, H, W] rather than the real [C, T, H, W]. */ + const float *chan = img + (((long) t * C) + c) * H * W; + + for (int h = 0; h < H; ++h) { + const float *src_row = chan + (long) h * W; + + const int gh = h / patch; + const int ph = h % patch; + const int bh = gh / merge; + const int mh = gh % merge; + const int feat_row_base = ((c * T + t) * patch + ph) * patch; + + for (int gw = 0; gw < grid_w; ++gw) { + const int bw = gw / merge; + const int mw = gw % merge; + const int token = ((bh * Bw + bw) * merge + mh) * merge + mw; + + float *dst = out + (long) token * out_cols + feat_row_base; + const float *rp = src_row + (long) gw * patch; + for (int pw = 0; pw < patch; ++pw) { + dst[pw] = rp[pw]; + } + } + } + } + } +} diff --git a/kernels/patchify_fp32/nearmiss_merge_ignored.c b/kernels/patchify_fp32/nearmiss_merge_ignored.c new file mode 100644 index 0000000..454ac64 --- /dev/null +++ b/kernels/patchify_fp32/nearmiss_merge_ignored.c @@ -0,0 +1,69 @@ +/* A plausible WRONG implementation the harness must reject. + * + * THE MISTAKE: emitting output rows in plain raster order (token = gh * + * grid_w + gw) and never looking at `merge` at all. + * + * WHY ANYONE WOULD WRITE IT. `merge` looks, from the op's attrs alone, like + * it could be pure metadata for a downstream consumer -- "the merger will + * group these 2x2 later" -- rather than something THIS op has to act on. + * grid_h and grid_w are the only two attrs that appear in the output SHAPE + * (grid_h*grid_w rows), so it is easy to conclude the op is "just flatten + * the grid" and never notice that the registry's own reference + * (structural.py:172-207) reshapes the grid into merge blocks and + * transposes them into (bh, bw, mh, mw) order BEFORE flattening -- i.e. + * `merge` changes which row a given patch lands in, not just how a later op + * groups rows that are already in raster order. kernel_api.h's own + * docstring calls this out explicitly for exactly this reason. + * + * WHY THE HARNESS CATCHES IT: raster order and merge-block order agree only + * for grid cells inside the very first merge block (bh == bw == 0), where + * `gh * grid_w + gw` and the real block-order formula both evaluate to + * small, coincidentally-matching numbers for a couple of entries -- but they + * diverge everywhere else (e.g. at merge=2, grid_w=16: real gh=1, gw=0 lands + * at token 2, raster order puts it at token 16), so almost every row of a + * 16x16 or 4x4 grid ends up on the wrong output row. Every input element has + * a distinct value, so a row landing in the wrong place is not masked by two + * rows coincidentally holding the same numbers. + */ +#include "kernel_api.h" + +#define PATCHIFY_ROWBUF_MAX 256 + +void patchify_fp32(const float *img, float *out, + int C, int T, int H, int W, + int patch, int merge, int grid_h, int grid_w) { + (void) merge; /* WRONG: never consulted. */ + + if (C <= 0 || T <= 0 || H <= 0 || W <= 0 || patch <= 0 + || grid_h <= 0 || grid_w <= 0 || W > PATCHIFY_ROWBUF_MAX) { + return; + } + + const int out_cols = C * T * patch * patch; + + for (int c = 0; c < C; ++c) { + for (int t = 0; t < T; ++t) { + const float *chan = img + (((long) c * T) + t) * H * W; + + for (int h = 0; h < H; ++h) { + const float *src_row = chan + (long) h * W; + + const int gh = h / patch; + const int ph = h % patch; + const int feat_row_base = ((c * T + t) * patch + ph) * patch; + + for (int gw = 0; gw < grid_w; ++gw) { + /* WRONG: plain raster order, ignoring the merge-block + * reordering the registry's reference performs. */ + const int token = gh * grid_w + gw; + + float *dst = out + (long) token * out_cols + feat_row_base; + const float *rp = src_row + (long) gw * patch; + for (int pw = 0; pw < patch; ++pw) { + dst[pw] = rp[pw]; + } + } + } + } + } +} diff --git a/kernels/patchify_fp32/nearmiss_patch_interior_swap.c b/kernels/patchify_fp32/nearmiss_patch_interior_swap.c new file mode 100644 index 0000000..c68bfa4 --- /dev/null +++ b/kernels/patchify_fp32/nearmiss_patch_interior_swap.c @@ -0,0 +1,70 @@ +/* A plausible WRONG implementation the harness must reject. + * + * THE MISTAKE: writing each patch's pixels with the two INTERIOR patch axes + * swapped -- feature index `((c*T+t)*patch + pw)*patch + ph` instead of the + * real `((c*T+t)*patch + ph)*patch + pw`. Every row and token lands in + * exactly the right place (this kernel still runs the correct token + * arithmetic); only the 16x16 (or PF2's 3x3) patch itself comes out + * transposed inside its row. + * + * WHY ANYONE WOULD WRITE IT. The op moves through FOUR nested index + * variables per patch (c, t, ph, pw) and only the last two, ph and pw, are + * ever the same size (patch == patch) -- so a hand-written feature-index + * expression that puts them in the wrong order is dimensionally invisible: + * `(... * patch + ph) * patch + pw` and `(... * patch + pw) * patch + ph` + * both type-check, both produce a value in [0, patch*patch), and neither + * looks more "obviously right" than the other by inspection. This is the + * task's own "walking the patch in row-major when the layout wants + * patch-interior last" -- pw must be the fastest-varying (innermost) axis + * per kernel_api.h's derivation from structural.py:206's axis order, and + * this near-miss makes ph the innermost one instead. + * + * WHY THE HARNESS CATCHES IT: only the DIAGONAL of each patch (ph == pw) + * lands in its correct feature slot; every off-diagonal element is written + * to its transpose's slot instead. With distinct values at every input + * position, an off-diagonal swap is never masked by two positions + * coincidentally holding the same number. + */ +#include "kernel_api.h" + +void patchify_fp32(const float *img, float *out, + int C, int T, int H, int W, + int patch, int merge, int grid_h, int grid_w) { + if (C <= 0 || T <= 0 || H <= 0 || W <= 0 || patch <= 0 || merge <= 0 + || grid_h <= 0 || grid_w <= 0) { + return; + } + + const int Bw = grid_w / merge; + const int out_cols = C * T * patch * patch; + + for (int gh = 0; gh < grid_h; ++gh) { + const int bh = gh / merge; + const int mh = gh % merge; + + for (int gw = 0; gw < grid_w; ++gw) { + const int bw = gw / merge; + const int mw = gw % merge; + const int token = ((bh * Bw + bw) * merge + mh) * merge + mw; + + for (int c = 0; c < C; ++c) { + for (int t = 0; t < T; ++t) { + for (int ph = 0; ph < patch; ++ph) { + const int h = gh * patch + ph; + const long src_row = (((long) c * T) + t) * H * W + + (long) h * W; + + for (int pw = 0; pw < patch; ++pw) { + const int w = gw * patch + pw; + /* WRONG: pw and ph swapped in the feature index. */ + const long feat = (long) ((c * T + t) * patch + pw) + * patch + ph; + out[(long) token * out_cols + feat] = + img[src_row + w]; + } + } + } + } + } + } +} diff --git a/kernels/patchify_fp32/spec.json b/kernels/patchify_fp32/spec.json new file mode 100644 index 0000000..569b37e --- /dev/null +++ b/kernels/patchify_fp32/spec.json @@ -0,0 +1,23 @@ +{ + "task_id": "patchify_fp32", + "dtype": "fp32", + "caps": [], + "mechanisms": ["hvx"], + "params": { + "C": 3, + "T": 2, + "H": 256, + "W": 256, + "patch": 16, + "temporal_patch": 2, + "merge": 2, + "grid_h": 16, + "grid_w": 16, + "encoder_shape": "[3, 2, 256, 256] -> [256, 1536]", + "small_shape": "[2, 3, 12, 12] -> [16, 54], patch=3, merge=2 (scalar-tail path)" + }, + "expert_kernel_cycles": null, + "tolerance": "exact", + "movement_only": true, + "tags": ["layout", "encoder", "vision", "patch-embed"] +} diff --git a/kernels/rmsnorm_fp16/BAKEOFF.md b/kernels/rmsnorm_fp16/BAKEOFF.md index 7d75455..6bf630e 100644 --- a/kernels/rmsnorm_fp16/BAKEOFF.md +++ b/kernels/rmsnorm_fp16/BAKEOFF.md @@ -16,7 +16,7 @@ compared. |---|---|---|---|---|---| | scalar baseline | this repo, `baseline.c` copied to `kernel.c` for the Step 6 discrimination check | yes | 69443 | none | reference; both near-misses correctly rejected against it, confirming the harness discriminates before any HVX kernel existed | | v6 `rmsnorm_gain_fp16` (adapted) | HVX-clean v6, handwritten, `solutions/s2.c` (ror-shift butterfly reduce) | yes | 2231 | hvx, hvx-compute | **winner.** Original is R=6,C=80 with a PER-ROW scalar gain and recorded 9798 cycles at that shape/2.741x; adapted to R=8,C=128 with hexlib's PER-COLUMN gain (rewrote the scale epilogue as a real vector×vector `w[block]` multiply instead of a scalar-splat gain, since a per-row scalar cannot express a per-column vector) and generalized the reduction from reading only block 0 (correct only for the original's single-block C=80 shape) to accumulating qf16 sum-of-squares across all `nb=C/64` blocks before the one-time ror-shift reduce. eps threaded as a parameter instead of hardcoded `1e-3f`. See "Fix round 2" below for the 2021 -> 2231 move. | -| v6 `fp16_rmsnorm` (adapted) | HVX-clean v6, handwritten, block-accumulate + unpack-once scalar sum | yes | 10498 | hvx, hvx-compute | Original is n=2048 (single row) and recorded 2311 cycles at that shape/60.53x. **Correction to this task's brief:** the brief describes this candidate as "no gain vector — you must add the w[] multiply", but the actual `expert.c` already multiplies by a per-feature `gamma[]` in the scale epilogue, structurally identical to hexlib's per-column `w[c]`; no gain multiply had to be added. Wrapped the original single-vector body in a `for r in [0,R)` loop (x/y offset by `r*C`, `w[]` reused unchanged every row, exactly like the original's `gamma` not varying by call). eps threaded as a parameter instead of hardcoded `1e-3f`. Loses to candidate A by 5.19x at this shape: its reduction unpacks the qf16 accumulator to memory and finishes with a 64-iteration scalar add loop — negligible when paid once for n=2048, but paid once PER ROW here (8x), while candidate A's ror-shift butterfly never leaves the vector unit. | +| v6 `fp16_rmsnorm` (adapted) | HVX-clean v6, handwritten, block-accumulate + unpack-once scalar sum | yes | 10498 | hvx, hvx-compute | Original is n=2048 (single row) and recorded 2311 cycles at that shape/60.53x. **Correction to this task's brief:** the brief describes this candidate as "no gain vector — you must add the w[] multiply", but the actual `expert.c` already multiplies by a per-feature `gamma[]` in the scale epilogue, structurally identical to hexlib's per-column `w[c]`; no gain multiply had to be added. Wrapped the original single-vector body in a `for r in [0,R)` loop (x/y offset by `r*C`, `w[]` reused unchanged every row, exactly like the original's `gamma` not varying by call). eps threaded as a parameter instead of hardcoded `1e-3f`. Loses to candidate A by 4.71x at this shape (5.19x before Fix round 2 moved the winner 2021 -> 2231; see below): its reduction unpacks the qf16 accumulator to memory and finishes with a 64-iteration scalar add loop — negligible when paid once for n=2048, but paid once PER ROW here (8x), while candidate A's ror-shift butterfly never leaves the vector unit. | | ggml-hexagon `hvx-norm.h` | llama.cpp, MIT | — | — | — | **not evaluated: fp32 only.** `hvx_fast_rms_norm_mul_f32` (`include/hexlib/hvx/hvx-norm.h`) is an exact semantic match — RMSNorm with a per-column gain vector, reduction in `Vqf32`/`Vsf` — but there is no fp16 norm anywhere in the vendored ggml-hexagon set, and v6 has no fp32 norm at all (every v6 norm is fp16 or i8). Comparing an fp32 implementation against fp16 candidates would measure the dtype, not the implementation. Deferred to kernel #2, `rmsnorm_f32`, where it goes head to head with a hexlib implementation on identical fp32 shapes — the first direct measurement against production ggml-hexagon code. | **Fix round 1 (numbers above are post-fix).** `hexlib/build.py` originally linked diff --git a/kernels/rope_2d_fp16/RESULT.md b/kernels/rope_2d_fp16/RESULT.md new file mode 100644 index 0000000..bc8c463 --- /dev/null +++ b/kernels/rope_2d_fp16/RESULT.md @@ -0,0 +1,19 @@ +### hexlib verify — rope_2d_fp16 + +| gate | result | +|---|---| +| correct | PASS | +| max abs error | 6.10352e-05 (n_wrong 0) | +| kernel_cycles | 1212 | +| accel (ELF-proven) | hvx, hvx-compute | +| near-miss `nearmiss_adjacent_pairing.c` | correctly rejected | +| near-miss `nearmiss_fp16_accumulate.c` | correctly rejected | +| near-miss `nearmiss_negated_term.c` | correctly rejected | +| near-miss `nearmiss_partial_rotation.c` | correctly rejected | +| near-miss `nearmiss_swapped_cos_sin.c` | correctly rejected | +| near-miss `nearmiss_table_indexed_by_head.c` | correctly rejected | +| **gate** | **PASS** | + +target `v75` · toolchain `19.0.04` · SDK `6.4.0.2` · host `sriha@Heathcliff` · `2026-08-11T21:14:27Z` + +Measured on the hexagon simulator under the pinned bus model (buspenalty 75, busratio 2). The simulator is cycle-approximate; these numbers are reproducible, not silicon measurements. diff --git a/kernels/rope_2d_fp16/baseline.c b/kernels/rope_2d_fp16/baseline.c new file mode 100644 index 0000000..d58ab2f --- /dev/null +++ b/kernels/rope_2d_fp16/baseline.c @@ -0,0 +1,40 @@ +/* kernels/rope_2d_fp16/baseline.c */ +#include "kernel_api.h" + +/* Scalar reference. Correct and obvious, never fast. + * + * A direct transcription of the per-index form derived in kernel_api.h from + * structural.py's `_rope_2d_reference` -- see that header for the full + * derivation and the split-half pairing citation (structural.py:253, cross- + * checked against the forge2 reference and llama.cpp's HTP_ROPE_TYPE_VISION + * kernel). + * + * Everything is accumulated in `float`, matching the reference's cast to + * float32 before the rotation; only the store into `yr[...]` rounds to fp16. + * cos/sin are read at the SAME token row for every head (no head axis on the + * table), and at their OWN column -- cos[t,i] and cos[t,i+half] are two + * distinct reads, never the same value reused. + */ +void rope_2d_fp16_baseline(const hexlib_hf *x, const float *costab, + const float *sintab, hexlib_hf *y, + int T, int H, int D) { + if (T <= 0 || H <= 0 || D <= 0) { + return; + } + const int half = D / 2; + + for (int t = 0; t < T; ++t) { + const float *cr = costab + (long) t * D; + const float *sr = sintab + (long) t * D; + for (int h = 0; h < H; ++h) { + const hexlib_hf *xr = x + ((long) t * H + h) * D; + hexlib_hf *yr = y + ((long) t * H + h) * D; + for (int i = 0; i < half; ++i) { + const float x0 = (float) xr[i]; + const float x1 = (float) xr[i + half]; + yr[i] = (hexlib_hf) (x0 * cr[i] - x1 * sr[i]); + yr[i + half] = (hexlib_hf) (x1 * cr[i + half] + x0 * sr[i + half]); + } + } + } +} diff --git a/kernels/rope_2d_fp16/harness.c b/kernels/rope_2d_fp16/harness.c new file mode 100644 index 0000000..2687fa8 --- /dev/null +++ b/kernels/rope_2d_fp16/harness.c @@ -0,0 +1,163 @@ +/* kernels/rope_2d_fp16/harness.c + * + * T, H, D ARE ALL DIFFERENT NUMBERS (6, 3, 64), for the reason + * kernels/transpose_th_fp16 gives for its own T=8, H=3: with any two of the + * three axes equal, a kernel that confuses their strides -- indexing the + * cos/sin table by head instead of by token is exactly this op's version of + * that mistake, since the table has no head axis at all -- can still produce + * the right answer by accident. D=64 is additionally fixed by the encoder's + * only real shape and by kernel.c's fast path (see kernel_api.h). + * + * THE TABLE'S TWO HALVES GENUINELY DIFFER. cos[t,i] and cos[t,i+half] are + * built from different, unequal formulas below (not HF's `cat(freqs,freqs)` + * convention, where they coincide) -- see kernel_api.h's "2-D" note. A kernel + * that assumes the low half's angle also applies to the high half is caught. + * + * EVERY (t,h) GETS A DIFFERENT x ROW, so a kernel that mixes up which head's + * data meets which token's table entry is visible, not masked by repeated + * data. + * + * ========================================================================== + * THE DISCRIMINATING CHECK: fp16-vs-fp32 accumulation, made STRUCTURAL. + * ========================================================================== + * One real near-miss here is subtle on friendly data: rounding each + * intermediate product (x0*cos, x1*sin) to fp16 before combining, instead of + * keeping the whole rotation in float/qf32 and rounding only the final + * result. On generic small values that costs roughly one extra fp16 ULP + * (~1e-3 relative) on top of the correct kernel's own ~1e-3 narrowing noise + * (Q6_Vhf_equals_Wqf32 is not IEEE round-to-nearest and differs from numpy by + * 1 ULP -- see kernel_api.h / hexlib's HARD CONSTRAINTS). A tolerance loose + * enough to admit that legitimate noise is already loose enough to admit the + * bug too -- exactly the failure kernels/layernorm_fp16's + * nearmiss_unbiased_variance.c was WRONGLY ACCEPTED by once, on friendly + * data, before that kernel's harness was tightened. + * + * Rather than chase a tolerance that happens to sit between the two noise + * floors, this harness makes the DIFFERENCE STRUCTURAL: one token (the last + * one, t = ROPE_T - 1) is given a NEAR-CANCELLATION at column i=0 (and its + * pair i=half): both halves of x are set to the same large value (1000.0, + * exactly representable in fp16), and cos/sin are chosen so the two products + * being combined are individually large (~500) but nearly equal in magnitude, + * so the TRUE result is tiny (~0.1): + * + * x0 = x1 = 1000.0 + * cos[i] = 0.5, sin[i] = 0.5001 -> y[i] = 500.0 - 500.1 = -0.1 (true) + * cos[i+half] = 0.5, sin[i+half] = -0.4999 -> y[i+half] = 500.0 + (-499.9) = 0.1 (true) + * + * WHY THIS DISCRIMINATES, WORKED OUT ARITHMETICALLY (not just asserted): + * + * fp32/qf32 path (correct kernel and baseline): 500.0 and 500.1 (or 500.0 + * and -499.9) are each accurate to ~7 decimal digits, so their difference + * is accurate to a few times 1e-5 -- the tiny true result survives the + * subtraction essentially intact, and only THEN gets rounded to fp16 (whose + * ULP near 0.1 is ~1e-4). Error at this point: ~1e-4, well inside any + * tolerance this harness uses elsewhere. + * + * fp16-intermediate path (the near-miss): fp16's ULP at magnitude ~500 is + * 500 * 2^-10 =~ 0.49 (the representable grid there is spaced by 0.25, + * since 500 sits in the [256,512) binade). 500.1 is only 0.1 away from + * 500.0 and 0.15 away from the next grid point 500.25, so it rounds TO + * 500.0 -- indistinguishable from the other operand. The same happens to + * -499.9, which rounds to -500.0. The near-miss's subtraction/addition then + * sees 500.0-500.0=0 and 500.0+(-500.0)=0, LOSING THE ENTIRE SIGNAL: it + * reports 0.0 where the true answer is -0.1 / +0.1. Error: 0.1 -- a + * thousand times the correct kernel's error at the same point, and far + * larger than the loose per-element tolerance (rel 0.02, abs 1e-3) used for + * every other element in this harness. + * + * This is the same principle kernels/layernorm_fp16's C=64-narrow-width pass + * uses (pick a regime where the bug is bigger, not a tolerance you have + * talked yourself into) -- applied here as a single crafted VALUE rather than + * a second narrower SHAPE, since this op has a value-dependent (cancellation) + * failure mode rather than a size-dependent (1/C vs 1/(C-1)) one. + */ +#include "hexlib/hexlib_harness.h" +#include "kernel_api.h" + +#include + +void rope_2d_fp16_baseline(const hexlib_hf *, const float *, const float *, + hexlib_hf *, int, int, int); + +#define ROPE_N (ROPE_T * ROPE_H * ROPE_D) + +static hexlib_hf X[ROPE_N] HEXLIB_ALIGN; +static float COS[ROPE_T * ROPE_D] HEXLIB_ALIGN; +static float SIN[ROPE_T * ROPE_D] HEXLIB_ALIGN; +static hexlib_hf Y[ROPE_N] HEXLIB_ALIGN; +static hexlib_hf REF[ROPE_N] HEXLIB_ALIGN; + +int main(void) { + const int half = ROPE_D / 2; + + /* cos/sin table: the two halves of each row use DIFFERENT angle formulas + * (mimicking a real 2-D table where the low half carries one spatial axis + * and the high half the other -- see kernel_api.h's "2-D" note), so + * cos[t,i] != cos[t,i+half] in general and a kernel that reuses the low + * half's angle for the high half is caught. */ + for (int t = 0; t < ROPE_T; ++t) { + for (int i = 0; i < half; ++i) { + const float angle_lo = 0.15f * (float) (t + 1) * (float) (i + 1); + COS[t * ROPE_D + i] = cosf(angle_lo); + SIN[t * ROPE_D + i] = sinf(angle_lo); + + const float angle_hi = + 0.11f * (float) (t + 1) * (float) (i + 1) + 0.83f; + COS[t * ROPE_D + half + i] = cosf(angle_hi); + SIN[t * ROPE_D + half + i] = sinf(angle_hi); + } + } + + /* x: every (t,h,i) cell distinct, asymmetric, no repeated pattern that + * could make a wrong pairing or a wrong stride pass by coincidence. */ + for (int t = 0; t < ROPE_T; ++t) { + for (int h = 0; h < ROPE_H; ++h) { + for (int i = 0; i < ROPE_D; ++i) { + const int idx = (t * ROPE_H + h) * ROPE_D + i; + const float v = + (float) (((t * 7 + h * 13 + i * 3) % 23) - 11) * 0.3f; + X[idx] = (hexlib_hf) v; + } + } + } + + /* The near-cancellation cell: see the header comment above for the + * arithmetic. Overwritten AFTER the generic fill, at token + * t=ROPE_T-1, head 0, columns 0 and half. */ + { + const int t = ROPE_T - 1; + const int h = 0; + const int base = (t * ROPE_H + h) * ROPE_D; + X[base + 0] = (hexlib_hf) 1000.0f; + X[base + half] = (hexlib_hf) 1000.0f; + COS[t * ROPE_D + 0] = 0.5f; + SIN[t * ROPE_D + 0] = 0.5001f; + COS[t * ROPE_D + half] = 0.5f; + SIN[t * ROPE_D + half] = -0.4999f; + } + + /* Poison the output so a kernel that writes nothing cannot pass. */ + for (int i = 0; i < ROPE_N; ++i) { + Y[i] = (hexlib_hf) 12345.0f; + } + + rope_2d_fp16_baseline(X, COS, SIN, REF, ROPE_T, ROPE_H, ROPE_D); + + unsigned long long kcyc = 0; + HEXLIB_TIME_KERNEL(kcyc, + rope_2d_fp16(X, COS, SIN, Y, ROPE_T, ROPE_H, ROPE_D)); + + int n_wrong = 0; + double max_err = 0.0; + for (int i = 0; i < ROPE_N; ++i) { + if (!hexlib_close_f16((float) Y[i], (float) REF[i], 0.02f, 1e-3f)) { + ++n_wrong; + } + double d = (double) (float) Y[i] - (double) (float) REF[i]; + if (d < 0.0) d = -d; + if (d > max_err) max_err = d; + } + + hexlib_report(n_wrong == 0, n_wrong, max_err, kcyc); + return 0; +} diff --git a/kernels/rope_2d_fp16/kernel.c b/kernels/rope_2d_fp16/kernel.c new file mode 100644 index 0000000..9749e62 --- /dev/null +++ b/kernels/rope_2d_fp16/kernel.c @@ -0,0 +1,138 @@ +/* kernels/rope_2d_fp16/kernel.c + * + * WHAT IS VECTORISED AND WHAT IS NOT. This op has no reduction at all -- it is + * purely elementwise -- so unlike kernels/layernorm_fp16 there is no scalar + * "next thing to fix". The FAST PATH below is fully vectorised for the one + * shape the encoder ever calls this with (D == 64). Any other D falls back to + * a scalar loop, same order of work as layernorm_fp16's documented tail: a + * kernel that silently mangled an untested shape would be worse than one that + * is merely slow there. + * + * WHY D == 64 IS SPECIAL. The pairing is split-half: column i pairs with + * column i+D/2 (see kernel_api.h for the citation). For D == 64, half == 32, + * which is exactly LANES_FP32 -- so the whole pairing lives inside ONE 64-lane + * fp16 vector, and widening it to two 32-lane fp32 halves via widen_ordered + * below splits it EXACTLY at the pairing boundary: part[0] holds columns + * [0,32), part[1] holds columns [32,64). Those line up 1:1 with cos/sin's own + * 32-lane fp32 vectors (crv[0]/srv[0] = columns [0,32), crv[1]/srv[1] = + * columns [32,64)), so the whole rotation is four vector multiplies, an add, + * a subtract and one narrow -- no cross-vector shuffling of the pairing + * itself is needed. For any other multiple of 64, half would not equal 32 and + * this alignment would not hold in general, which is why the fast path is + * gated on D == 64 exactly rather than "D is a multiple of 64". + * + * WHY THE CONVERSIONS LOOK LIKE THAT. Adapted VERBATIM from + * kernels/layernorm_fp16/kernel.c's `widen_ordered`/`narrow_ordered` helpers, + * for the same reason they exist there: there is no fp16<->qf32 convert + * instruction (widening is a multiply by 1.0, narrowing is an add of 0.0), and + * both the widen (Q6_Wqf32_vmpy_VhfVhf) and the narrow (Q6_Vhf_equals_Wqf32) + * PERMUTE lanes -- element k does not land in lane k. Q6_Vh_vshuff_Vh before + * the widen and Q6_Vh_vdeal_Vh after the narrow put it back. Lane order is + * load-bearing HERE, exactly as it is in layernorm_fp16's affine epilogue and + * for the same reason: cos[t,i] and sin[t,i] are PER-COLUMN, so column i of x + * must meet column i (not some shuffled position) of cos/sin. Getting that + * wrong yields a correctly-shaped output where the rotation angle applied to + * a column is not the one that column was given, which no shape check can + * see -- exactly the failure mode layernorm_fp16's + * nearmiss_permuted_affine_lanes.c demonstrates for its own per-column w/b. + * + * v75 HAS NO fp16 ADD/SUB INSTRUCTION (Q6_Vhf_vadd_VhfVhf arrives at + * __HVX_ARCH__ 79 and crashes hexagon-clang 19.0.04's instruction selection + * with exit code 70 if used here). All arithmetic below is qf32: + * Q6_Vqf32_vmpy_VsfVsf multiplies two IEEE fp32 vectors into qf32, + * Q6_Vqf32_vsub_Vqf32Vqf32/Q6_Vqf32_vadd_Vqf32Vqf32 combine two qf32 vectors, + * and Q6_Vsf_equals_Vqf32 narrows qf32 back to IEEE fp32 before the final + * fp16 narrow (which itself goes through the qf32-pair narrow, per + * narrow_ordered). + */ +#include "kernel_api.h" + +#include +#include + +#define LANES_FP32 32 +#define LANES_FP16 64 +#define FP16_ONE 0x3C00 + +/* fp16 vector -> two IEEE fp32 vectors, IN ELEMENT ORDER. + * out[0] holds elements 0..31, out[1] holds 32..63. + * Adapted verbatim from kernels/layernorm_fp16/kernel.c. */ +static inline void widen_ordered(HVX_Vector v, HVX_Vector one, HVX_Vector *out) { + HVX_VectorPair p = Q6_Wqf32_vmpy_VhfVhf(Q6_Vh_vshuff_Vh(v), one); + out[0] = Q6_Vsf_equals_Vqf32(Q6_V_lo_W(p)); + out[1] = Q6_Vsf_equals_Vqf32(Q6_V_hi_W(p)); +} + +/* Two IEEE fp32 vectors -> one fp16 vector, IN ELEMENT ORDER. + * Adapted verbatim from kernels/layernorm_fp16/kernel.c. */ +static inline HVX_Vector narrow_ordered(HVX_Vector lo, HVX_Vector hi) { + const HVX_Vector zero = Q6_V_vzero(); + HVX_Vector qlo = Q6_Vqf32_vadd_VsfVsf(lo, zero); + HVX_Vector qhi = Q6_Vqf32_vadd_VsfVsf(hi, zero); + return Q6_Vh_vdeal_Vh(Q6_Vhf_equals_Wqf32(Q6_W_vcombine_VV(qhi, qlo))); +} + +void rope_2d_fp16(const hexlib_hf *x, const float *costab, const float *sintab, + hexlib_hf *y, int T, int H, int D) { + if (T <= 0 || H <= 0 || D <= 0) { + return; + } + const int half = D / 2; + + if (D == LANES_FP16) { + const HVX_Vector one = Q6_Vh_vsplat_R(FP16_ONE); + + for (int t = 0; t < T; ++t) { + /* cos[t,:] and sin[t,:] loaded once per token, reused for every + * head -- the table has no head axis (kernel_api.h). */ + const HVX_Vector *crv = (const HVX_Vector *) (costab + (long) t * D); + const HVX_Vector *srv = (const HVX_Vector *) (sintab + (long) t * D); + const HVX_Vector vc0 = crv[0]; /* cos[t, 0:32] */ + const HVX_Vector vc1 = crv[1]; /* cos[t, 32:64] */ + const HVX_Vector vs0 = srv[0]; /* sin[t, 0:32] */ + const HVX_Vector vs1 = srv[1]; /* sin[t, 32:64] */ + + for (int h = 0; h < H; ++h) { + const HVX_Vector *xv = + (const HVX_Vector *) (x + ((long) t * H + h) * D); + HVX_Vector *yv = (HVX_Vector *) (y + ((long) t * H + h) * D); + + HVX_Vector part[2]; + widen_ordered(xv[0], one, part); + /* part[0] = x[t,h,0:32], part[1] = x[t,h,32:64] */ + + /* y[i] = x[i] *cos[i] - x[i+half]*sin[i] */ + HVX_Vector lo = Q6_Vsf_equals_Vqf32( + Q6_Vqf32_vsub_Vqf32Vqf32( + Q6_Vqf32_vmpy_VsfVsf(part[0], vc0), + Q6_Vqf32_vmpy_VsfVsf(part[1], vs0))); + + /* y[i+half] = x[i+half]*cos[i+half] + x[i] *sin[i+half] */ + HVX_Vector hi = Q6_Vsf_equals_Vqf32( + Q6_Vqf32_vadd_Vqf32Vqf32( + Q6_Vqf32_vmpy_VsfVsf(part[1], vc1), + Q6_Vqf32_vmpy_VsfVsf(part[0], vs1))); + + yv[0] = narrow_ordered(lo, hi); + } + } + return; + } + + /* Scalar fallback for D != 64. Never exercised by the encoder (this op + * is always called at head_dim=64); kept correct rather than omitted. */ + for (int t = 0; t < T; ++t) { + const float *cr = costab + (long) t * D; + const float *sr = sintab + (long) t * D; + for (int h = 0; h < H; ++h) { + const hexlib_hf *xr = x + ((long) t * H + h) * D; + hexlib_hf *yr = y + ((long) t * H + h) * D; + for (int i = 0; i < half; ++i) { + const float x0 = (float) xr[i]; + const float x1 = (float) xr[i + half]; + yr[i] = (hexlib_hf) (x0 * cr[i] - x1 * sr[i]); + yr[i + half] = (hexlib_hf) (x1 * cr[i + half] + x0 * sr[i + half]); + } + } + } +} diff --git a/kernels/rope_2d_fp16/kernel_api.h b/kernels/rope_2d_fp16/kernel_api.h new file mode 100644 index 0000000..da0924b --- /dev/null +++ b/kernels/rope_2d_fp16/kernel_api.h @@ -0,0 +1,123 @@ +/* kernels/rope_2d_fp16/kernel_api.h */ +#ifndef HEXLIB_ROPE_2D_FP16_API_H +#define HEXLIB_ROPE_2D_FP16_API_H + +typedef __fp16 hexlib_hf; + +/* 2-D rotary position embedding applied to the vision-tower's attention Q/K. + * 24 ops in the encoder, all one shape: x fp16 [256, 12, 64] (tokens, heads, + * head_dim), cos/sin fp32 CONST [256, 64], output fp16 [256, 12, 64]. + * + * SPECIFICATION SOURCE: hexlib/graph/opdefs/structural.py:222-264, the + * `rope_2d` OpDef. `_rope_2d_infer` (222-237) fixes the shapes; the actual + * math is `_rope_2d_reference` (240-254), itself a transcription of + * `apply_rotary_pos_emb_vision`, modeling_qwen3_5.py:891-902, with + * `rotate_half` at modeling_qwen3_5.py:562 (cited at structural.py:246). + * + * for each token t in [0, T), head h in [0, H), let half = D / 2: + * for i in [0, half): + * y[t,h,i] = x[t,h,i] * cos[t,i] - x[t,h,i+half] * sin[t,i] + * y[t,h,i+half] = x[t,h,i+half] * cos[t,i+half] + x[t,h,i] * sin[t,i+half] + * + * (Equivalently, structural.py:253's form: + * rotated = concat([-x[..., half:], x[..., :half]], axis=-1) + * y = x * cos + rotated * sin + * -- the two are the same formula written two ways; the derivation above is + * just the concat/slice unrolled per index.) + * + * ========================================================================== + * THE PAIRING CONVENTION IS SPLIT-HALF (i, i+D/2), NOT ADJACENT (2i, 2i+1). + * ========================================================================== + * Read directly from structural.py:253: + * rotated = np.concatenate([-xf[..., half:], xf[..., :half]], axis=-1) + * `xf[..., :half]` is columns [0, half); `xf[..., half:]` is [half, D). That + * is GPT-NeoX-style rotation (element i pairs with element i+D/2), NOT GPT-J's + * interleaved pairing (element 2i pairs with 2i+1). CONFIRMED against two + * independent sources, not merely consistent with them: + * + * - ../HVX-clean/run_artifacts/hexlib_encoder/enc_rope2d_fp16/kernel.cpp + * (forge2's scalar reference for this EXACT op and shape, verified + * against a PyTorch golden -- see that directory's PROVENANCE.md). + * Lines 40-72: v_slice_1 reads x[.., 32:64] (v_x[i0*768+i1*64+32+i2]), + * v_neg negates it, v_slice_2 reads x[.., 0:32], v_cat writes + * NEGATED-SECOND-HALF into columns [0,32) and the ORIGINAL FIRST HALF + * into columns [32,64) -- i.e. `linalg.mlir`'s own concat, lines 13-21: + * `tensor.extract_slice %1[0, 0, 32] ... : ... to tensor<...x32xf32>` + * (the SECOND half, negated) concatenated with the extract at `[0,0,0]` + * (the FIRST half). Same split-half formula, unit for unit. + * - ../llama.cpp/ggml/src/ggml-hexagon/htp/rope-ops.c: mode + * HTP_ROPE_TYPE_VISION (line 27) routes through the NEOX-style pairing + * path (`is_vision`/`is_neox` at 474-476), whose HVX kernel + * `hvx_rope_neox_f32_aa` (lines 285-330) reads `v0` from the FIRST half + * (`src0[i]`) and `v1` from the SECOND half (`src0[he+i]`, `he = ne/2`) + * and computes `dst[i] = v0*cos - v1*sin`, `dst[he+i] = v0*sin + v1*cos` + * -- the identical split-half pairing, sign for sign, with `dst[i]` + * matching this file's `y[t,h,i]` and `dst[he+i]` matching + * `y[t,h,i+half]`. + * + * ALL THREE SOURCES AGREE. There is no disagreement to report. + * + * ========================================================================== + * THE TABLE IS INDEXED OVER THE FULL D=64, NOT D/2=32 REUSED TWICE. + * ========================================================================== + * cos/sin have shape (T, D), enforced at structural.py:230 (`cos.shape != + * (x.shape[0], x.shape[2])` raises). cos[t,i] and cos[t,i+half] are two + * DIFFERENT stored floats read at two different offsets, never the same + * value read twice. Some HF rotary tables happen to satisfy + * cos[t,i] == cos[t,i+half] (their `emb = cat(freqs, freqs)` convention), but + * this kernel must not assume that, and the harness deliberately uses a table + * where the two halves differ so a kernel that assumes the HF convention (and + * reads only the low half's frequency for both) is caught. + * + * ========================================================================== + * NO HEAD AXIS ON THE TABLE. + * ========================================================================== + * cos/sin are (T, D) -- structural.py:250-251 unsqueeze them on the HEAD axis + * (`cosf[:, None, :]`) before the broadcast multiply, meaning the SAME row + * cos[t,:], sin[t,:] is used for every head at token t. The table has no head + * axis at all, which makes "index by head instead of by token" a plausible + * stride slip: cos[h,:] is a legal, in-bounds read whenever h < T, and is + * simply the wrong row. + * + * ========================================================================== + * WHAT "2-D" MEANS HERE. + * ========================================================================== + * The encoder is a vision tower and each token's position is a (row, column) + * pair over a patch grid. In the model that builds cos/sin, the two halves of + * head_dim typically carry the two spatial axes -- e.g. columns [0, half) + * derive their frequency from the row position, columns [half, D) from the + * column position -- so cos[t,i] and cos[t,i+half] can differ not just in + * value but in WHICH axis of the 2-D position they encode. That split, + * however, happens UPSTREAM of this op: by the time cos and sin reach + * `rope_2d`, they are already flat (T, D) tables, and `_rope_2d_infer`/ + * `_rope_2d_reference` have no row/column-specific logic at all (structural. + * py:222-254) -- neither does this kernel. "2-D" describes where the table's + * values came from, not anything this op computes differently per axis; it is + * one 1-D rotation over the full head_dim, applied per token. + * + * ========================================================================== + * DTYPES AND ROUNDING. + * ========================================================================== + * x and y are fp16 (hexlib_hf); cos and sin are fp32. Per structural.py: + * 249-254 (`xf = x.astype(np.float32)`, ... , `.astype(x.dtype)` at the very + * end), the rotation is computed in float32 regardless of the activation + * dtype; only the STORED result is rounded to fp16. This kernel's HVX path + * keeps the arithmetic in Hexagon's qf32 the whole way through and narrows + * once, at the very end, matching that contract. + * + * D must be even (required by the op registry's own infer, structural.py: + * 235-236). The fast HVX path below additionally requires D == 64 exactly: + * that puts the entire pairing (i, i+32) inside ONE 64-lane fp16 vector, + * split by the widen step into two 32-lane fp32 halves that line up 1:1 with + * cos/sin's own 32-lane fp32 vectors -- and D=64 is the ONLY head_dim this op + * is ever called with in the encoder. Other D fall back to a scalar loop: + * correct, not vectorised. See kernel.c's header comment. + */ +#define ROPE_T 6 +#define ROPE_H 3 +#define ROPE_D 64 + +void rope_2d_fp16(const hexlib_hf *x, const float *costab, const float *sintab, + hexlib_hf *y, int T, int H, int D); + +#endif diff --git a/kernels/rope_2d_fp16/nearmiss_adjacent_pairing.c b/kernels/rope_2d_fp16/nearmiss_adjacent_pairing.c new file mode 100644 index 0000000..6c66cb0 --- /dev/null +++ b/kernels/rope_2d_fp16/nearmiss_adjacent_pairing.c @@ -0,0 +1,48 @@ +/* A plausible WRONG implementation the harness must reject. + * + * THE MISTAKE: pairing ADJACENT columns (2i, 2i+1) instead of SPLIT-HALF + * columns (i, i+D/2). + * + * WHY ANYONE WOULD WRITE IT. Rotary embedding has two conventions in the + * wild, and they are both called "RoPE" in casual writing: GPT-J's + * interleaved pairing (2i, 2i+1) and GPT-NeoX's split-half pairing + * (i, i+D/2). They are both real, both shipped in major model families, and + * nothing about the OP NAME `rope_2d` or its SHAPES distinguishes them -- a + * kernel author who has implemented the interleaved form before (it is, if + * anything, the more commonly taught one) will reach for it here too. Only + * reading the reference formula (structural.py:253's slice-negate-concat, or + * equivalently kernel_api.h's derivation) settles which one this op needs. + * + * WHY IT SURVIVES A SHAPE CHECK. Both conventions consume the same x, cos and + * sin tensors, of the same shapes, and produce a same-shaped output where + * every value is a plausible rotated float. There is no dimension mismatch, + * no NaN, nothing a shape assertion or an "is it finite" check would catch -- + * only comparison against a reference computed with the RIGHT pairing catches + * it, and only if the input data is rich enough that the two pairings + * actually diverge (which this harness's asymmetric x and cos/sin ensure). + */ +#include "kernel_api.h" + +void rope_2d_fp16(const hexlib_hf *x, const float *costab, const float *sintab, + hexlib_hf *y, int T, int H, int D) { + if (T <= 0 || H <= 0 || D <= 0) { + return; + } + + for (int t = 0; t < T; ++t) { + const float *cr = costab + (long) t * D; + const float *sr = sintab + (long) t * D; + for (int h = 0; h < H; ++h) { + const hexlib_hf *xr = x + ((long) t * H + h) * D; + hexlib_hf *yr = y + ((long) t * H + h) * D; + /* WRONG: pairs (2i, 2i+1), the GPT-J interleaved convention, + * instead of (i, i+D/2). */ + for (int i = 0; i + 1 < D; i += 2) { + const float x0 = (float) xr[i]; + const float x1 = (float) xr[i + 1]; + yr[i] = (hexlib_hf) (x0 * cr[i] - x1 * sr[i]); + yr[i + 1] = (hexlib_hf) (x1 * cr[i + 1] + x0 * sr[i + 1]); + } + } + } +} diff --git a/kernels/rope_2d_fp16/nearmiss_fp16_accumulate.c b/kernels/rope_2d_fp16/nearmiss_fp16_accumulate.c new file mode 100644 index 0000000..a96796c --- /dev/null +++ b/kernels/rope_2d_fp16/nearmiss_fp16_accumulate.c @@ -0,0 +1,61 @@ +/* A plausible WRONG implementation the harness must reject. + * + * THE MISTAKE: rounding each intermediate PRODUCT to fp16 before combining + * them, instead of accumulating the whole rotation in float (kernel_api.h / + * structural.py:249-254: the reference casts x, cos AND sin to float32 + * before the rotation and rounds only the FINAL result to fp16). + * + * WHY ANYONE WOULD WRITE IT. x and y are `hexlib_hf`; cos and sin are the odd + * ones out at `float`. A kernel author minimising type conversions -- or one + * who has just written a kernel where x, w and y were ALL fp16 and copies + * that shape of code -- writes each partial product straight into a + * `hexlib_hf` local "since that's what the surrounding types are", rounding + * it immediately rather than carrying it through as float. Nothing in the + * function signature stops this: `(hexlib_hf) (x0 * costab[i])` and + * `(float) ((hexlib_hf) (x0 * costab[i]))` differ only in when the fp16 round + * trip happens, and only the SECOND is wrong here. + * + * WHY THIS IS THE QUIET ONE. On generic small values this costs roughly one + * extra fp16 ULP on top of the correct kernel's own narrowing noise -- the + * SAME order of magnitude, not obviously separable by a loose per-element + * tolerance. See harness.c's header comment for the worked arithmetic: this + * harness catches it not by tightening the tolerance but by including one + * token (t = ROPE_T-1) where two large (~500) intermediate products nearly + * cancel to a tiny (~0.1) true result. fp32/qf32 keeps that cancellation + * accurate; fp16 rounds BOTH large products to the SAME grid point (fp16's + * ULP at magnitude 500 is ~0.49, far coarser than the 0.1-0.2 gap between + * them) and reports 0.0, losing the entire signal. That is what this near- + * miss is built to be rejected on. + */ +#include "kernel_api.h" + +void rope_2d_fp16(const hexlib_hf *x, const float *costab, const float *sintab, + hexlib_hf *y, int T, int H, int D) { + if (T <= 0 || H <= 0 || D <= 0) { + return; + } + const int half = D / 2; + + for (int t = 0; t < T; ++t) { + const float *cr = costab + (long) t * D; + const float *sr = sintab + (long) t * D; + for (int h = 0; h < H; ++h) { + const hexlib_hf *xr = x + ((long) t * H + h) * D; + hexlib_hf *yr = y + ((long) t * H + h) * D; + for (int i = 0; i < half; ++i) { + const float x0 = (float) xr[i]; + const float x1 = (float) xr[i + half]; + + /* WRONG: each product rounded to fp16 BEFORE combining, + * instead of staying in float through the whole rotation. */ + hexlib_hf p0_lo = (hexlib_hf) (x0 * cr[i]); + hexlib_hf p1_lo = (hexlib_hf) (x1 * sr[i]); + yr[i] = (hexlib_hf) ((float) p0_lo - (float) p1_lo); + + hexlib_hf p1_hi = (hexlib_hf) (x1 * cr[i + half]); + hexlib_hf p0_hi = (hexlib_hf) (x0 * sr[i + half]); + yr[i + half] = (hexlib_hf) ((float) p1_hi + (float) p0_hi); + } + } + } +} diff --git a/kernels/rope_2d_fp16/nearmiss_negated_term.c b/kernels/rope_2d_fp16/nearmiss_negated_term.c new file mode 100644 index 0000000..e6db645 --- /dev/null +++ b/kernels/rope_2d_fp16/nearmiss_negated_term.c @@ -0,0 +1,46 @@ +/* A plausible WRONG implementation the harness must reject. + * + * THE MISTAKE: negating the wrong term. The rotation is + * y[i] = x[i] *cos[i] - x[i+half]*sin[i] + * y[i+half] = x[i+half]*cos[i+half] + x[i] *sin[i+half] + * and this kernel drops the minus sign on the FIRST line, computing + * y[i] = x[i] *cos[i] + x[i+half]*sin[i] (WRONG) + * y[i+half] = x[i+half]*cos[i+half] + x[i] *sin[i+half] (still right) + * + * WHY ANYONE WOULD WRITE IT. The two output halves look almost symmetric -- + * both are "cos of my own half plus sin of the other half's contribution" -- + * and the ONE sign that breaks that symmetry (rotate_half's `-x[...,half:]` + * from structural.py:253) is easy to lose when writing the two lines side by + * side, especially by someone who has just finished proving the SPLIT-HALF + * pairing is right and is now transcribing the four products from memory + * rather than re-reading the slice/negate/concat. + * + * WHY IT SURVIVES A SHAPE CHECK. A same-shaped, plausible-looking rotated + * output; only a reference comparison with cos/sin that are not both zero at + * once catches the missing sign. + */ +#include "kernel_api.h" + +void rope_2d_fp16(const hexlib_hf *x, const float *costab, const float *sintab, + hexlib_hf *y, int T, int H, int D) { + if (T <= 0 || H <= 0 || D <= 0) { + return; + } + const int half = D / 2; + + for (int t = 0; t < T; ++t) { + const float *cr = costab + (long) t * D; + const float *sr = sintab + (long) t * D; + for (int h = 0; h < H; ++h) { + const hexlib_hf *xr = x + ((long) t * H + h) * D; + hexlib_hf *yr = y + ((long) t * H + h) * D; + for (int i = 0; i < half; ++i) { + const float x0 = (float) xr[i]; + const float x1 = (float) xr[i + half]; + /* WRONG: should be x0*cr[i] - x1*sr[i]. */ + yr[i] = (hexlib_hf) (x0 * cr[i] + x1 * sr[i]); + yr[i + half] = (hexlib_hf) (x1 * cr[i + half] + x0 * sr[i + half]); + } + } + } +} diff --git a/kernels/rope_2d_fp16/nearmiss_partial_rotation.c b/kernels/rope_2d_fp16/nearmiss_partial_rotation.c new file mode 100644 index 0000000..5f54582 --- /dev/null +++ b/kernels/rope_2d_fp16/nearmiss_partial_rotation.c @@ -0,0 +1,56 @@ +/* A plausible WRONG implementation the harness must reject. + * + * THE MISTAKE: rotating only PART of head_dim and copying the rest straight + * through, unrotated -- when this op's contract rotates the FULL head_dim, + * always (kernel_api.h; structural.py's `_rope_2d_reference` has no partial- + * rotary logic at all). + * + * WHY ANYONE WOULD WRITE IT. Partial rotary IS a real, shipped mechanism + * elsewhere -- ../llama.cpp/ggml/src/ggml-hexagon/htp/rope-ops.c's + * `rope_neox_f32` (lines 422-435) rotates only `rctx->n_dims` columns and then + * explicitly copies the remaining channels through unchanged when + * `n_dims < ne0` (line 432: "fill the remain channels with data from src + * tensor"). Someone porting a rope kernel FROM that codebase, or simply + * remembering that "some rope variants only rotate part of the head", could + * carry that guard into an op whose registry entry never asked for it. + * + * WHAT THIS KERNEL DOES: treats only the first D/2 columns as the "rotary + * dimension" and rotates WITHIN that half using split-half pairing at HALF + * the correct distance (D/4 instead of D/2), then copies columns [D/2, D) + * straight from x. Both halves of the mistake are structural: half the + * output columns are not rotated by any angle at all, and the columns that + * ARE rotated use the wrong pairing distance too. + */ +#include "kernel_api.h" + +void rope_2d_fp16(const hexlib_hf *x, const float *costab, const float *sintab, + hexlib_hf *y, int T, int H, int D) { + if (T <= 0 || H <= 0 || D <= 0) { + return; + } + const int rotary_dim = D / 2; /* WRONG: should be D, the whole head. */ + const int r_half = rotary_dim / 2; + + for (int t = 0; t < T; ++t) { + const float *cr = costab + (long) t * D; + const float *sr = sintab + (long) t * D; + for (int h = 0; h < H; ++h) { + const hexlib_hf *xr = x + ((long) t * H + h) * D; + hexlib_hf *yr = y + ((long) t * H + h) * D; + + /* Rotate only the first `rotary_dim` columns, paired at + * distance r_half within that sub-range. */ + for (int i = 0; i < r_half; ++i) { + const float x0 = (float) xr[i]; + const float x1 = (float) xr[i + r_half]; + yr[i] = (hexlib_hf) (x0 * cr[i] - x1 * sr[i]); + yr[i + r_half] = (hexlib_hf) (x1 * cr[i + r_half] + x0 * sr[i + r_half]); + } + /* WRONG: the remaining columns are copied through, unrotated -- + * this op has no such pass-through range. */ + for (int i = rotary_dim; i < D; ++i) { + yr[i] = xr[i]; + } + } + } +} diff --git a/kernels/rope_2d_fp16/nearmiss_swapped_cos_sin.c b/kernels/rope_2d_fp16/nearmiss_swapped_cos_sin.c new file mode 100644 index 0000000..b15efae --- /dev/null +++ b/kernels/rope_2d_fp16/nearmiss_swapped_cos_sin.c @@ -0,0 +1,45 @@ +/* A plausible WRONG implementation the harness must reject. + * + * THE MISTAKE: swapping the cos and sin table arguments -- using sin where + * cos belongs and cos where sin belongs, with the pairing and the signs + * otherwise correct. + * + * WHY ANYONE WOULD WRITE IT. `rope_2d_fp16(x, costab, sintab, y, T, H, D)` + * takes cos before sin; a transcription slip (copying from a reference that + * lists them the other way, or simply mistyping two adjacent identifiers + * that are the same shape and dtype) produces a function that compiles + * cleanly -- costab and sintab are both `const float *`, so nothing in the + * type system notices the swap. + * + * WHY IT SURVIVES A SHAPE CHECK. Same shapes, same dtypes, a same-shaped + * plausible-looking rotated output. It is caught only by comparison against + * the real cos/sin assignment, and only because this harness's cos and sin + * values are NOT symmetric (cos[t,i] != sin[t,i] almost everywhere) -- a + * harness built from, say, a 45-degree-only angle table (cos == sin) would + * let this bug pass by coincidence. + */ +#include "kernel_api.h" + +void rope_2d_fp16(const hexlib_hf *x, const float *costab, const float *sintab, + hexlib_hf *y, int T, int H, int D) { + if (T <= 0 || H <= 0 || D <= 0) { + return; + } + const int half = D / 2; + + for (int t = 0; t < T; ++t) { + /* WRONG: cr reads sintab, sr reads costab. */ + const float *cr = sintab + (long) t * D; + const float *sr = costab + (long) t * D; + for (int h = 0; h < H; ++h) { + const hexlib_hf *xr = x + ((long) t * H + h) * D; + hexlib_hf *yr = y + ((long) t * H + h) * D; + for (int i = 0; i < half; ++i) { + const float x0 = (float) xr[i]; + const float x1 = (float) xr[i + half]; + yr[i] = (hexlib_hf) (x0 * cr[i] - x1 * sr[i]); + yr[i + half] = (hexlib_hf) (x1 * cr[i + half] + x0 * sr[i + half]); + } + } + } +} diff --git a/kernels/rope_2d_fp16/nearmiss_table_indexed_by_head.c b/kernels/rope_2d_fp16/nearmiss_table_indexed_by_head.c new file mode 100644 index 0000000..0656562 --- /dev/null +++ b/kernels/rope_2d_fp16/nearmiss_table_indexed_by_head.c @@ -0,0 +1,48 @@ +/* A plausible WRONG implementation the harness must reject. + * + * THE MISTAKE: indexing the cos/sin table by HEAD instead of by TOKEN -- + * `costab + h*D` instead of `costab + t*D`. + * + * WHY ANYONE WOULD WRITE IT. x is [T, H, D] and the natural stride pattern + * for "the other tensor associated with this loop" is to match whichever + * index is closer at hand -- and this loop nests `h` inside `t`, so `h` is + * the index most recently touched when the table lookup is written. cos/sin + * are [T, D], with NO HEAD AXIS AT ALL (kernel_api.h), so `costab + h*D` is a + * LEGAL, IN-BOUNDS read whenever h < T (true here, H=3 <= T=6) -- it does not + * crash or read garbage, it silently reads the wrong row. + * + * WHY THE HARNESS CAN CATCH IT: only because T != H. This kernel reads row + * `h` (always in [0, H)) instead of row `t` (in [0, T)). For every token + * t >= H (here, t = 3, 4, 5 of 6), EVERY head reads one of rows {0, 1, 2} + * instead of its own row {3, 4, 5} -- entirely wrong, for half the tokens, at + * every head. Even for t < H it is wrong whenever h != t. A harness with + * T == H (or a table whose rows all happened to agree) would let this pass by + * coincidence; kernels/transpose_th_fp16's T=8,H=3 choice exists for exactly + * this reason and this kernel's T=6,H=3 follows it. + */ +#include "kernel_api.h" + +void rope_2d_fp16(const hexlib_hf *x, const float *costab, const float *sintab, + hexlib_hf *y, int T, int H, int D) { + if (T <= 0 || H <= 0 || D <= 0) { + return; + } + const int half = D / 2; + + for (int t = 0; t < T; ++t) { + for (int h = 0; h < H; ++h) { + /* WRONG: indexed by h, but cos/sin have no head axis -- the + * correct row is t. */ + const float *cr = costab + (long) h * D; + const float *sr = sintab + (long) h * D; + const hexlib_hf *xr = x + ((long) t * H + h) * D; + hexlib_hf *yr = y + ((long) t * H + h) * D; + for (int i = 0; i < half; ++i) { + const float x0 = (float) xr[i]; + const float x1 = (float) xr[i + half]; + yr[i] = (hexlib_hf) (x0 * cr[i] - x1 * sr[i]); + yr[i + half] = (hexlib_hf) (x1 * cr[i + half] + x0 * sr[i + half]); + } + } + } +} diff --git a/kernels/rope_2d_fp16/spec.json b/kernels/rope_2d_fp16/spec.json new file mode 100644 index 0000000..b52d467 --- /dev/null +++ b/kernels/rope_2d_fp16/spec.json @@ -0,0 +1,18 @@ +{ + "task_id": "rope_2d_fp16", + "dtype": "fp16", + "caps": [], + "mechanisms": ["hvx"], + "params": { + "T": 6, + "H": 3, + "D": 64, + "encoder_shape": "x fp16 [256, 12, 64]; cos/sin fp32 CONST [256, 64]", + "encoder_op_count": 24, + "pairing": "split-half (i, i+D/2), NEOX-style -- structural.py:253", + "table_axes": "cos/sin indexed by (token, head_dim); no head axis -- broadcast across all heads" + }, + "expert_kernel_cycles": null, + "tolerance": "hexlib_close_f16", + "tags": ["rope", "positional-encoding", "vision", "encoder", "elementwise"] +} diff --git a/kernels/softmax_fp16/RESULT.md b/kernels/softmax_fp16/RESULT.md new file mode 100644 index 0000000..22d7ec2 --- /dev/null +++ b/kernels/softmax_fp16/RESULT.md @@ -0,0 +1,16 @@ +### hexlib verify — softmax_fp16 + +| gate | result | +|---|---| +| correct | PASS | +| max abs error | 0 (n_wrong 0) | +| kernel_cycles | 11292 | +| accel (ELF-proven) | hvx, hvx-compute | +| near-miss `nearmiss_no_max_subtraction.c` | correctly rejected | +| near-miss `nearmiss_sum_fp16.c` | correctly rejected | +| near-miss `nearmiss_wrong_axis.c` | correctly rejected | +| **gate** | **PASS** | + +target `v75` · toolchain `19.0.04` · SDK `6.4.0.2` · host `sriha@Heathcliff` · `2026-08-11T21:02:03Z` + +Measured on the hexagon simulator under the pinned bus model (buspenalty 75, busratio 2). The simulator is cycle-approximate; these numbers are reproducible, not silicon measurements. diff --git a/kernels/softmax_fp16/baseline.c b/kernels/softmax_fp16/baseline.c new file mode 100644 index 0000000..c4efa4f --- /dev/null +++ b/kernels/softmax_fp16/baseline.c @@ -0,0 +1,42 @@ +#include "kernel_api.h" + +#include + +/* Scalar reference. Correct and obvious, never fast. + * + * Three passes over the row, matching hexlib/graph/opdefs/elementwise.py:141-146 + * exactly in ORDER OF OPERATIONS: find the max first, subtract it before the + * exponential (never after), sum the exponentials, then divide. Every + * intermediate (`m`, `e[c]`, `s`) is float32; only the stored result is rounded + * to fp16 -- see kernel_api.h's PRECISION note for why float32 rather than the + * reference's float64 is the right thing for this kernel to be held to. + * + * expf() here is the toolchain's own libm, not a polynomial approximation -- + * this file is the reference the polynomial in kernel.c is checked against, so + * it must not share the same approximation error. + */ +void softmax_fp16_baseline(const hexlib_hf *x, hexlib_hf *y, int R, int C) { + for (int r = 0; r < R; ++r) { + const hexlib_hf *xr = x + (long) r * C; + hexlib_hf *yr = y + (long) r * C; + + float m = (float) xr[0]; + for (int c = 1; c < C; ++c) { + float v = (float) xr[c]; + if (v > m) m = v; + } + + float e[C > 0 ? C : 1]; /* VLA: baseline is plain scalar C, no HVX + * alignment constraint to respect. */ + float s = 0.0f; + for (int c = 0; c < C; ++c) { + float v = expf((float) xr[c] - m); + e[c] = v; + s += v; + } + + for (int c = 0; c < C; ++c) { + yr[c] = (hexlib_hf) (e[c] / s); + } + } +} diff --git a/kernels/softmax_fp16/harness.c b/kernels/softmax_fp16/harness.c new file mode 100644 index 0000000..e150a2d --- /dev/null +++ b/kernels/softmax_fp16/harness.c @@ -0,0 +1,107 @@ +/* kernels/softmax_fp16/harness.c + * + * Builds inputs, runs the baseline for reference, times ONLY the kernel call, + * compares with tolerance, and prints the two lines the driver parses. + * + * SHAPE: SOFTMAX_R=6, SOFTMAX_C=256. C matches the encoder's real row width + * (fp16 (12, 256, 256), axis=-1); R is a small representative sample of the + * encoder's 3072 independent rows, same as layernorm_fp16 sampling R=4 of 256. + * R != C ON PURPOSE: the encoder's own last two dims ARE square (256x256), so + * a wrong-axis softmax there produces a same-shape, same-size output that a + * shape check cannot see. Making this harness's R and C UNEQUAL means a + * row-softmax and a column-softmax are reducing over different-sized groups + * (6 vs 256) no matter what the data looks like, so nearmiss_wrong_axis.c + * cannot pass here by accident -- see kernel_api.h's SHAPE note. + * + * ROW 0 IS THE "no max subtraction" TRIGGER. x[0][0] = 90.0f. expf(90) alone + * overflows float32 (FLT_MAX's ln is ~88.72), so a kernel that exponentiates + * before subtracting the row max produces +inf, then inf/inf = NaN. + * nearmiss_no_max_subtraction.c must fail here, loudly, via NaN != anything. + * + * ROW 1 IS THE "sum accumulated in fp16" TRIGGER, and it is the one that + * matters. Read this carefully, because a per-element tolerance loose enough + * to admit the real kernel's own legitimate noise CAN be looser than a real + * bug -- that already happened once in this repo (layernorm_fp16's + * unbiased-variance near-miss was wrongly accepted on its first run because + * the bug's size, ~0.065% at C=768, was smaller than fp16's own ~0.05% ULP + * noise). The same risk applies here: summing 256 fp16-rounded exp() values + * one at a time, rounding to fp16 AFTER EACH addition, can differ from a + * float32 accumulation by well under one ULP PER ELEMENT on "friendly" data + * (measured in Python: uniform small scores, max relative difference only + * ~0.09% -- indistinguishable from ordinary rounding noise, and a tolerance + * tight enough to catch it there would also reject the correct kernel). + * + * So row 1 is not friendly. It is one dominant score (20.0) and 255 IDENTICAL + * followers at 20.0 - 6.5 = 13.5 -- every follower's exp(shifted) is + * ~exp(-6.5) = 0.0015, individually tiny against a running sum near 1.0-2.0, + * which is exactly the shape of input that makes fp16 accumulation lose mass: + * many increments each close to the accumulator's own ULP, added one at a + * time, round away a little every single step, 255 times in a row. Measured + * in Python (float32 exp, float32 vs fp16-per-step accumulation, same + * algorithm nearmiss_sum_fp16.c implements): the two summed to 1.383384 vs. + * 1.498047 -- an 8.3% difference in the DENOMINATOR -- which lands as a 7.7% + * relative / 0.0552 absolute error on the dominant output element alone. That + * is not a rounding-noise near-miss: it is 90-150x the size of the real + * kernel's own error on the same row (measured max_abs_err ~4.9e-4, ~1 fp16 + * ULP at that magnitude, from the single unavoidable narrow-to-fp16 rounding + * every correct implementation pays exactly once). The tolerance below (1% + * relative, 1e-3 absolute) sits in between with more than an order of + * magnitude of headroom on each side -- it is not a number arrived at by + * relaxing it until the kernel passed. + * + * ROWS 2-5 are generic, mutually dissimilar fp16 attention-score-shaped data + * (mixed sign, mixed magnitude) -- coverage, not a discriminator on their own. + */ +#include "hexlib/hexlib_harness.h" +#include "kernel_api.h" + +void softmax_fp16_baseline(const hexlib_hf *, hexlib_hf *, int, int); + +static hexlib_hf X[SOFTMAX_R * SOFTMAX_C] HEXLIB_ALIGN; +static hexlib_hf Y[SOFTMAX_R * SOFTMAX_C] HEXLIB_ALIGN; +static hexlib_hf REF[SOFTMAX_R * SOFTMAX_C] HEXLIB_ALIGN; + +static void fill(void) { + for (int c = 0; c < SOFTMAX_C; ++c) { + /* Row 0: overflow trigger for "no max subtraction". */ + float v0 = (c == 0) ? 90.0f : 0.1f * (float) ((c % 13) - 6); + X[0 * SOFTMAX_C + c] = (hexlib_hf) v0; + + /* Row 1: fp16-sum trigger -- one dominant score, 255 identical + * followers 6.5 below it. See the header comment for the numbers. */ + float v1 = (c == 0) ? 20.0f : (20.0f - 6.5f); + X[1 * SOFTMAX_C + c] = (hexlib_hf) v1; + + /* Rows 2-5: generic coverage, mutually dissimilar. */ + X[2 * SOFTMAX_C + c] = (hexlib_hf) (2.0f * (float) ((c % 17) - 8) * 0.25f); + X[3 * SOFTMAX_C + c] = (hexlib_hf) (0.01f * (float) (c % 31)); + X[4 * SOFTMAX_C + c] = (hexlib_hf) (-1.5f + 0.03f * (float) (c % 41)); + X[5 * SOFTMAX_C + c] = (hexlib_hf) (5.0f * (float) ((c * 7) % 19) / 19.0f - 2.5f); + } + for (int i = 0; i < SOFTMAX_R * SOFTMAX_C; ++i) { + Y[i] = (hexlib_hf) 12345.0f; /* poison: a no-op kernel cannot pass */ + } +} + +int main(void) { + fill(); + + softmax_fp16_baseline(X, REF, SOFTMAX_R, SOFTMAX_C); + + unsigned long long kcyc = 0; + HEXLIB_TIME_KERNEL(kcyc, softmax_fp16(X, Y, SOFTMAX_R, SOFTMAX_C)); + + int n_wrong = 0; + double max_err = 0.0; + for (int i = 0; i < SOFTMAX_R * SOFTMAX_C; ++i) { + if (!hexlib_close_f16((float) Y[i], (float) REF[i], 0.01f, 1e-3f)) { + ++n_wrong; + } + double d = (double) (float) Y[i] - (double) (float) REF[i]; + if (d < 0.0) d = -d; + if (d > max_err) max_err = d; + } + + hexlib_report(n_wrong == 0, n_wrong, max_err, kcyc); + return 0; +} diff --git a/kernels/softmax_fp16/kernel.c b/kernels/softmax_fp16/kernel.c new file mode 100644 index 0000000..1cb1e53 --- /dev/null +++ b/kernels/softmax_fp16/kernel.c @@ -0,0 +1,169 @@ +/* Row-wise softmax, fp16 in/out, float32 max/exp/sum -- see kernel_api.h for + * the exact formula and where it comes from. + * + * THREE VECTORISED PASSES OVER EACH ROW, mirroring the scalar reference's own + * three passes (max, then exp+sum, then divide) rather than trying to fuse + * them -- the max must be known before any exp() call, and the sum must be + * known before any division, so there is no way to do this in one pass without + * re-deriving online-softmax's running-rescale trick, which this "first rung" + * version does not attempt (compare layernorm_fp16's own honesty about its + * unvectorised reductions). + * + * WIDEN/NARROW: reused, not reimplemented. `hvx_vec_f16_to_f32` and + * `hvx_vec_f32_to_f16` in the vendored include/hexlib/hvx/hvx-base.h already do + * exactly the shuffle-widen / narrow-deal dance that kernels/layernorm_fp16/ + * kernel.c hand-rolls as `widen_ordered`/`narrow_ordered` -- verified against + * that file line by line: hvx_vec_f16_to_f32's low lane group is elements + * 0..31, high is 32..63 (same as layernorm's out[0]/out[1]), and + * hvx_vec_f32_to_f16(lo, hi) performs the identical + * qf32-combine-then-Q6_Vh_vdeal_Vh as layernorm's narrow_ordered(lo, hi). Using + * the header's own versions instead of a local copy means one thing to keep in + * sync with the vendored source, not two. + * + * EXP: hvx_vec_exp_f32, NEVER hvx_vec_exp2_f16. The latter is in the same + * vendored header (hvx-exp.h) and IS BROKEN -- its E5 coefficient is 0x5082 + * where it should be 0x090c, 262% error at fractional input 0.7, and it is + * live in llama.cpp's own fp16 flash-attention softmax upstream. hvx_vec_exp_f32 + * is a different function with a different (natural-log-based, degree-7 + * Taylor) polynomial and does not share that bug. Measured in Python against + * real exp() (see scratch verification, same coefficients, same algorithm): + * ~1e-6 relative over shifted inputs in [-20, 0], which is every element that + * still matters after the max subtraction -- exp_f32's own intentional clamp + * below x=-88 only touches terms so far below the row's max that they + * underflow to 0 in fp16 regardless of how accurately they are computed. + * + * NO Q6_Vhf_vadd_VhfVhf, NO Vhf-typed accumulate anywhere. Per the repo's own + * hardware notes, that instruction does not exist on v75 and crashes clang + * 19.0.04 with exit code 70. All fp32 arithmetic here goes through + * hvx_vec_{add,sub,mul}_f32_f32 (hvx-base.h), which on this arch are + * Q6_Vsf_equals_Vqf32(Q6_Vqf32_..._VsfVsf(...)) under the hood -- the qf32 + * path, never a native Vhf op. + * + * SUM IS float32, ACCUMULATED FROM THE UNROUNDED exp() VALUES, EXACTLY ONCE + * ROUNDED TO fp16 AT THE FINAL DIVISION. This is the property the + * fp16-accumulated near-miss (nearmiss_sum_fp16.c) gets wrong, and the property + * the harness's adversarial row is built to make visible -- see harness.c's + * header comment and nearmiss_sum_fp16.c's for the numbers. + */ +#include "kernel_api.h" + +#include +#include +#include + +#include "hexlib/hvx/hvx-base.h" +#include "hexlib/hvx/hvx-exp.h" +#include "hexlib/hvx/hvx-reduce.h" + +#define LANES_FP16 64 + +/* Scratch for one row's unnormalised exp() values, float32, so the sum can be + * accumulated from full precision and the divide-then-narrow happens exactly + * once. Fixed-size and 128-byte aligned so the vectorised store/load into it is + * never the unaligned path. Sized for the encoder's real C=256 with 4x + * headroom; a row wider than this falls back to a fully scalar per-row + * computation below rather than corrupting memory or silently truncating -- + * "a kernel that silently mangles a different C is worse than one that is + * slow" (kernels/layernorm_fp16/kernel.c's own phrase for the same tradeoff). */ +#define SOFTMAX_SCRATCH_CAP 1024 + +static void softmax_row_scalar(const hexlib_hf *xr, hexlib_hf *yr, int C) { + float m = (float) xr[0]; + for (int c = 1; c < C; ++c) { + float v = (float) xr[c]; + if (v > m) m = v; + } + float s = 0.0f; + /* Re-derive e[c] on the second pass rather than storing it: this fallback + * exists only for C beyond the fast path's scratch cap, so it is not on + * any measured path and trading a second expf() for zero extra memory is + * the right call here. */ + for (int c = 0; c < C; ++c) { + s += expf((float) xr[c] - m); + } + for (int c = 0; c < C; ++c) { + yr[c] = (hexlib_hf) (expf((float) xr[c] - m) / s); + } +} + +void softmax_fp16(const hexlib_hf *x, hexlib_hf *y, int R, int C) { + if (R <= 0 || C <= 0) { + return; + } + if (C > SOFTMAX_SCRATCH_CAP) { + for (int r = 0; r < R; ++r) { + softmax_row_scalar(x + (long) r * C, y + (long) r * C, C); + } + return; + } + + float escratch[SOFTMAX_SCRATCH_CAP] __attribute__((aligned(128))); + const int nvec16 = C / LANES_FP16; /* fp16 vectors per row */ + const int vecC = nvec16 * LANES_FP16; /* elements covered by the vector path */ + + for (int r = 0; r < R; ++r) { + const hexlib_hf *xr = x + (long) r * C; + hexlib_hf *yr = y + (long) r * C; + const HVX_Vector *xv = (const HVX_Vector *) xr; + HVX_Vector *ev = (HVX_Vector *) escratch; + + /* --- pass 1: row max, vectorised over the 64-lane blocks --------- */ + float m; + if (nvec16 > 0) { + HVX_Vector accmax; + for (int i = 0; i < nvec16; ++i) { + HVX_VectorPair p = hvx_vec_f16_to_f32(xv[i]); + HVX_Vector blockmax = Q6_Vsf_vmax_VsfVsf(Q6_V_lo_W(p), Q6_V_hi_W(p)); + accmax = (i == 0) ? blockmax : Q6_Vsf_vmax_VsfVsf(accmax, blockmax); + } + HVX_Vector redmax = hvx_vec_reduce_max_f32(accmax); + m = hvx_vec_get_f32(redmax); + } else { + m = (float) xr[0]; + } + /* Scalar tail, C % 64 != 0. When nvec16 == 0 the vector path above + * never ran and m was already seeded from xr[0]; this loop still + * starts at vecC == 0 in that case and simply re-compares xr[0] + * against itself once, which is harmless. The encoder's C=256 has no + * tail (256 = 4*64), so this loop is untested by the harness's main + * shape and exists only so a future non-multiple-of-64 C is correct + * rather than lucky. */ + for (int c = vecC; c < C; ++c) { + float v = (float) xr[c]; + if (v > m) m = v; + } + + /* --- pass 2: shift, exp, accumulate the sum in float32 ----------- */ + const HVX_Vector vmax = hvx_vec_splat_f32(m); + HVX_Vector accsum; + for (int i = 0; i < nvec16; ++i) { + HVX_VectorPair p = hvx_vec_f16_to_f32(xv[i]); + HVX_Vector lo = hvx_vec_sub_f32_f32(Q6_V_lo_W(p), vmax); + HVX_Vector hi = hvx_vec_sub_f32_f32(Q6_V_hi_W(p), vmax); + HVX_Vector elo = hvx_vec_exp_f32(lo); + HVX_Vector ehi = hvx_vec_exp_f32(hi); + ev[2 * i] = elo; + ev[2 * i + 1] = ehi; + HVX_Vector blocksum = hvx_vec_add_f32_f32(elo, ehi); + accsum = (i == 0) ? blocksum : hvx_vec_add_f32_f32(accsum, blocksum); + } + float s = (nvec16 > 0) ? hvx_vec_get_f32(hvx_vec_reduce_sum_f32(accsum)) : 0.0f; + for (int c = vecC; c < C; ++c) { /* scalar tail */ + float v = expf((float) xr[c] - m); + escratch[c] = v; + s += v; + } + + /* --- pass 3: divide by the sum, narrow once to fp16 --------------- */ + const HVX_Vector vinv = hvx_vec_splat_f32(1.0f / s); + HVX_Vector *yv = (HVX_Vector *) yr; + for (int i = 0; i < nvec16; ++i) { + HVX_Vector olo = hvx_vec_mul_f32_f32(ev[2 * i], vinv); + HVX_Vector ohi = hvx_vec_mul_f32_f32(ev[2 * i + 1], vinv); + yv[i] = hvx_vec_f32_to_f16(olo, ohi); + } + for (int c = vecC; c < C; ++c) { /* scalar tail */ + yr[c] = (hexlib_hf) (escratch[c] / s); + } + } +} diff --git a/kernels/softmax_fp16/kernel_api.h b/kernels/softmax_fp16/kernel_api.h new file mode 100644 index 0000000..4e3a642 --- /dev/null +++ b/kernels/softmax_fp16/kernel_api.h @@ -0,0 +1,71 @@ +/* kernels/softmax_fp16/kernel_api.h */ +#ifndef HEXLIB_SOFTMAX_FP16_API_H +#define HEXLIB_SOFTMAX_FP16_API_H + +typedef __fp16 hexlib_hf; + +/* Softmax over the last axis, row-wise, fp16 in and out. + * + * SPEC. Taken from hexlib/graph/opdefs/elementwise.py:141-146 + * (`_softmax_reference`), which is the op registry's own reference and the + * eager/numpy implementation this kernel is held to: + * + * axis = attrs["axis"] (== -1 for this kernel: last axis) + * shifted = x - max(x, axis, keepdims=True) <- MAX SUBTRACTED BEFORE exp + * e = exp(shifted) + * out = e / sum(e, axis, keepdims=True) <- SUMMED AFTER exp, divided last + * + * i.e. for each row r in [0, R): + * m = max_c x[r][c] <- PER-ROW max, not global + * e[c] = exp((float) x[r][c] - m) + * s = sum_c e[c] + * y[r][c] = (hexlib_hf) (e[c] / s) + * + * PRECISION. The reference upcasts to float64 (numpy's default promotion) -- + * see elementwise.py line 143 `x = arrays[0].astype(np.float64)`. This kernel, + * like layernorm_fp16 and rmsnorm_fp16, computes in float32 instead: x and y are + * fp16 storage, but the max, the exponential, and the sum are all float32, and + * only the final quotient is rounded to fp16 once. float64 vs float32 is + * invisible at fp16 output resolution for any well-conditioned row; it is NOT + * invisible for a row engineered to make float32-vs-float16 summation differ, + * which is what this kernel's near-miss and harness exist to demonstrate (see + * harness.c and nearmiss_sum_fp16.c). + * + * EXP: NOT hvx_vec_exp2_f16. include/hexlib/hvx/hvx-exp.h's `hvx_vec_exp2_f16` + * has a wrong E5 polynomial coefficient (0x5082 where upstream calls for + * 0x090c) -- 262% error at fractional input 0.7, live in llama.cpp's own fp16 + * flash-attention softmax. This kernel uses `hvx_vec_exp_f32` instead (same + * header), a natural-log-based degree-7 Taylor polynomial in fp32 with its own, + * different and unaffected, coefficient table. Measured against real exp() in + * Python (see kernel.c's header comment for the numbers): ~1e-6 relative over + * the input range that matters after max-subtraction; the function's own + * intentional clamp below -88 only affects terms that underflow to 0 in fp16 + * anyway, so it costs nothing here. + * + * SHAPE. The encoder's actual op is fp16 (12, 256, 256), axis=-1: 12 batches of + * 256 rows of 256 elements, 3072 independent rows total, 12 ops. Rows are + * independent, so like layernorm_fp16 (LN_R=4 vs. the encoder's R=256) this + * kernel's own harness uses a SMALLER R than the real 3072 -- but the SAME + * C=256, because C (the reduction width) is what determines the numerics and + * the vector loop structure, not R. THE HARNESS'S R AND C ARE DELIBERATELY + * UNEQUAL (SM_R != SM_C below), even though the real encoder's last two dims + * ARE square (256x256): a wrong-axis near-miss on a square matrix produces a + * same-shape, same-total-size output, so a shape check cannot distinguish it, + * and even a value check could be fooled by an accidentally-symmetric test + * matrix. R != C makes a row-softmax and a column-softmax structurally + * different (different reduction group sizes) regardless of what the data + * looks like. See nearmiss_wrong_axis.c and harness.c. + * + * ALIGNMENT. x and y must be 128-byte aligned. C must be a multiple of 64 (the + * fp16 HVX vector width) for the vectorised path; the kernel still produces + * correct results for a non-multiple C via a scalar tail, and falls back to a + * fully scalar per-row computation if C exceeds SOFTMAX_SCRATCH_CAP (see + * kernel.c) -- slow, but never silently wrong for a shape nobody vectorised. + * Rows are independent. + */ +#define SOFTMAX_R 6 +#define SOFTMAX_C 256 + +void softmax_fp16(const hexlib_hf *x, hexlib_hf *y, int R, int C); + +#endif diff --git a/kernels/softmax_fp16/nearmiss_no_max_subtraction.c b/kernels/softmax_fp16/nearmiss_no_max_subtraction.c new file mode 100644 index 0000000..74e0bc6 --- /dev/null +++ b/kernels/softmax_fp16/nearmiss_no_max_subtraction.c @@ -0,0 +1,40 @@ +/* A plausible WRONG implementation the harness must reject. + * + * THE MISTAKE: exponentiating the raw score directly, without subtracting the + * row max first. Mathematically softmax is shift-invariant -- subtracting ANY + * per-row constant before dividing leaves the result unchanged -- so on paper + * this "simplification" looks free. It is not free in float32: exp() overflows + * long before the division would have cancelled it back out. + * + * WHY ANYONE WOULD WRITE IT. `e[c] = exp(x[c]); y[c] = e[c] / sum(e)` reads as + * a more direct transcription of "softmax(x)_c = exp(x_c) / sum(exp(x))" than + * the numerically-stable form with the max subtraction folded in -- the max + * subtraction is a stability trick, not part of the mathematical definition, + * and it is the kind of line a first draft omits. + * + * WHY IT IS LOUD, NOT SUBTLE (unlike nearmiss_sum_fp16.c). exp(90) alone + * overflows float32 (ln(FLT_MAX) is ~88.72), so harness.c's row 0 + * (x[0][0] = 90.0f) turns into +inf, and inf / inf is NaN by IEEE 754 -- every + * element of that row becomes NaN, and hexlib_close_f16 has no code path that + * calls NaN close to anything. This near-miss is deliberately the "too easy" + * one the task description warns about: it is included because it IS a real + * mistake, not because it is a hard one to catch. + */ +#include "kernel_api.h" + +#include + +void softmax_fp16(const hexlib_hf *x, hexlib_hf *y, int R, int C) { + for (int r = 0; r < R; ++r) { + const hexlib_hf *xr = x + (long) r * C; + hexlib_hf *yr = y + (long) r * C; + + float s = 0.0f; + for (int c = 0; c < C; ++c) { + s += expf((float) xr[c]); /* WRONG: no max subtraction */ + } + for (int c = 0; c < C; ++c) { + yr[c] = (hexlib_hf) (expf((float) xr[c]) / s); + } + } +} diff --git a/kernels/softmax_fp16/nearmiss_sum_fp16.c b/kernels/softmax_fp16/nearmiss_sum_fp16.c new file mode 100644 index 0000000..9aed02b --- /dev/null +++ b/kernels/softmax_fp16/nearmiss_sum_fp16.c @@ -0,0 +1,65 @@ +/* A plausible WRONG implementation the harness must reject. + * + * THE MISTAKE: accumulating the softmax denominator in fp16 (__fp16, rounding + * to fp16 after EVERY addition) instead of float32. Everything else here is + * IDENTICAL to baseline.c -- same max subtraction, same expf(), same division + * at the end. Only the type of the running sum changes. + * + * WHY ANYONE WOULD WRITE IT. x and y are both fp16; e[c] = exp(shifted) is a + * value the same "shape" as the data everywhere else in this kernel, so + * accumulating it in the storage dtype looks consistent rather than careless -- + * especially next to a hardware target where an fp16 accumulator is one lane + * width, and float32 is two. Nothing about the C source LOOKS unstable; the + * loop is the same loop. + * + * WHY THIS IS THE INTERESTING NEAR-MISS. On "friendly" data (measured in + * Python: 256 small values spread over roughly [-2, 2]) this bug differs from + * a correct float32 accumulation by well under one ULP on most elements -- + * genuinely invisible to a max-error check, for the same reason the + * unbiased-variance near-miss in kernels/layernorm_fp16/ was once wrongly + * accepted: the bug is smaller than fp16's own legitimate rounding noise on + * data that does not stress it. A tolerance loose enough to admit the real + * kernel's noise on friendly data would ALSO admit this bug there. + * + * THAT IS WHY harness.c's row 1 IS NOT FRIENDLY. It is built specifically to + * make 256 fp16-rounded additions lose real mass: one dominant term and 255 + * IDENTICAL followers each ~exp(-6.5) = 0.0015 -- individually close to the + * running accumulator's own fp16 ULP, added one at a time, 255 times, so a + * little rounds away on almost every step. Measured in Python running this + * exact algorithm on that exact row: float32 sum = 1.383384, fp16-per-step sum + * = 1.498047 -- an 8.3% difference in the denominator, landing as a 7.7% + * relative / 0.0552 absolute error on the dominant output element, some 90-150x + * the size of the real kernel's own single-fp16-rounding noise on the same + * row (~4.9e-4). See harness.c's header comment for the full derivation and + * why the chosen tolerance (1% relative, 1e-3 absolute) sits comfortably + * between the two rather than having been loosened until something passed. + */ +#include "kernel_api.h" + +#include + +void softmax_fp16(const hexlib_hf *x, hexlib_hf *y, int R, int C) { + for (int r = 0; r < R; ++r) { + const hexlib_hf *xr = x + (long) r * C; + hexlib_hf *yr = y + (long) r * C; + + float m = (float) xr[0]; + for (int c = 1; c < C; ++c) { + float v = (float) xr[c]; + if (v > m) m = v; + } + + /* WRONG: the running sum is __fp16, so it rounds to fp16 after every + * single addition instead of accumulating in float32. */ + hexlib_hf s16 = (hexlib_hf) 0.0f; + for (int c = 0; c < C; ++c) { + float e = expf((float) xr[c] - m); + s16 = (hexlib_hf) ((float) s16 + e); + } + float s = (float) s16; + + for (int c = 0; c < C; ++c) { + yr[c] = (hexlib_hf) (expf((float) xr[c] - m) / s); + } + } +} diff --git a/kernels/softmax_fp16/nearmiss_wrong_axis.c b/kernels/softmax_fp16/nearmiss_wrong_axis.c new file mode 100644 index 0000000..0f21f02 --- /dev/null +++ b/kernels/softmax_fp16/nearmiss_wrong_axis.c @@ -0,0 +1,56 @@ +/* A plausible WRONG implementation the harness must reject. + * + * THE MISTAKE: softmax reduced over axis 0 (down each column, across the R + * rows) instead of axis -1 (across each row, over the C columns). Attention + * softmax and "softmax the other way" are both a completely ordinary thing to + * write; get the loop nesting backwards -- outer over columns, inner over + * rows, normalising by the column instead of the row -- and every line still + * type-checks and every array access is still in bounds. + * + * WHY THE SHAPE MATTERS HERE. The encoder's real op is fp16 (12, 256, 256), + * axis=-1 -- the last two dims ARE square. On a square matrix, axis-0-softmax + * and axis-(-1)-softmax produce the SAME shape and the SAME total element + * count, so neither a shape assertion nor a naive "did the output resize + * correctly" check can tell them apart; only the VALUES differ, and only if + * the test data itself is not accidentally symmetric. kernel_api.h's own + * SHAPE note is about exactly this: harness.c deliberately uses R=6 != C=256, + * so a row-reduction and a column-reduction are summing over groups of very + * different sizes (6 vs 256) no matter what the data looks like -- this + * near-miss cannot pass by coincidence here the way it could on a square test + * shape with unlucky (symmetric) data. + * + * WHY IT FAILS BY A LOT, NOT A LITTLE. Averaged over a column of only 6 + * elements, a typical output magnitude is around 1/6 ~ 0.167; averaged over a + * row of 256, it is around 1/256 ~ 0.0039 -- a ~43x scale mismatch before + * even accounting for the different values being combined. This is the + * "too easy on its own" class the task description flags, included anyway + * because it is a real, easy-to-write mistake, and because it is the one that + * specifically requires the R != C harness shape to be caught reliably rather + * than by luck. + */ +#include "kernel_api.h" + +#include + +void softmax_fp16(const hexlib_hf *x, hexlib_hf *y, int R, int C) { + /* WRONG AXIS: outer loop over columns, inner loop over rows -- this + * normalises each COLUMN of R elements instead of each ROW of C + * elements. */ + for (int c = 0; c < C; ++c) { + float m = (float) x[0 * C + c]; + for (int r = 1; r < R; ++r) { + float v = (float) x[r * C + c]; + if (v > m) m = v; + } + + float s = 0.0f; + for (int r = 0; r < R; ++r) { + s += expf((float) x[r * C + c] - m); + } + + for (int r = 0; r < R; ++r) { + float e = expf((float) x[r * C + c] - m); + y[r * C + c] = (hexlib_hf) (e / s); + } + } +} diff --git a/kernels/softmax_fp16/spec.json b/kernels/softmax_fp16/spec.json new file mode 100644 index 0000000..d42d01a --- /dev/null +++ b/kernels/softmax_fp16/spec.json @@ -0,0 +1,18 @@ +{ + "task_id": "softmax_fp16", + "dtype": "fp16", + "caps": [], + "mechanisms": ["hvx"], + "params": { + "R": 6, + "C": 256, + "axis": -1, + "encoder_shape": "[12, 256, 256], fp16, axis=-1", + "encoder_op_count": 12, + "encoder_rows_total": 3072, + "reductions_vectorised": "mostly (vector max/sum reduction over full 64-lane blocks; scalar only for the C % 64 tail, which the encoder's C=256 never exercises)" + }, + "expert_kernel_cycles": null, + "tolerance": "hexlib_close_f16", + "tags": ["softmax", "attention", "encoder", "reduce-then-broadcast", "first-rung"] +} diff --git a/kernels/transpose_hd_fp16/RESULT.md b/kernels/transpose_hd_fp16/RESULT.md new file mode 100644 index 0000000..b6f1f1a --- /dev/null +++ b/kernels/transpose_hd_fp16/RESULT.md @@ -0,0 +1,16 @@ +### hexlib verify — transpose_hd_fp16 + +| gate | result | +|---|---| +| correct | PASS | +| max abs error | 0 (n_wrong 0) | +| kernel_cycles | 5606 | +| accel (ELF-proven) | hvx · movement-only (no arithmetic) | +| near-miss `nearmiss_plain_copy.c` | correctly rejected | +| near-miss `nearmiss_swapped_stride.c` | correctly rejected | +| near-miss `nearmiss_wrong_batch_stride.c` | correctly rejected | +| **gate** | **PASS** | + +target `v75` · toolchain `19.0.04` · SDK `6.4.0.2` · host `sriha@Heathcliff` · `2026-08-11T20:54:54Z` + +Measured on the hexagon simulator under the pinned bus model (buspenalty 75, busratio 2). The simulator is cycle-approximate; these numbers are reproducible, not silicon measurements. diff --git a/kernels/transpose_hd_fp16/baseline.c b/kernels/transpose_hd_fp16/baseline.c new file mode 100644 index 0000000..91e93ae --- /dev/null +++ b/kernels/transpose_hd_fp16/baseline.c @@ -0,0 +1,18 @@ +#include "kernel_api.h" + +/* Scalar reference. Correct and obvious, never fast. + * + * No arithmetic, so there is no rounding anywhere and the comparison against + * the kernel is EXACT: the output must be a permutation of the input's exact + * bytes. That makes this the one kernel in the set where a tolerance would + * hide a bug rather than accommodate the hardware. */ +void transpose_hd_fp16_baseline(const hexlib_hf *x, hexlib_hf *y, + int B, int T, int D) { + for (int b = 0; b < B; ++b) { + for (int t = 0; t < T; ++t) { + for (int d = 0; d < D; ++d) { + y[((long) b * D + d) * T + t] = x[((long) b * T + t) * D + d]; + } + } + } +} diff --git a/kernels/transpose_hd_fp16/harness.c b/kernels/transpose_hd_fp16/harness.c new file mode 100644 index 0000000..96dcbe1 --- /dev/null +++ b/kernels/transpose_hd_fp16/harness.c @@ -0,0 +1,85 @@ +/* kernels/transpose_hd_fp16/harness.c + * + * TWO SHAPES, ONE VERDICT. TR_T=64 makes each gathered chunk exactly one + * 128-byte HVX vector and exercises the vector-store path; TR_T_ODD=40 makes + * every chunk 80 bytes, so no whole vector ever fits and the scalar tail path + * runs for the entire row instead. Both are checked and the counts are + * accumulated into a single verdict, because the driver requires exactly one + * and would treat two as an error. + * + * B, T and D are deliberately three DIFFERENT numbers (5, 64, 7). A kernel + * that confuses which axis belongs in which stride -- the classic transpose + * bug -- produces the right answer whenever the two confused axes happen to + * have equal size, so the near-misses that make exactly that mistake would + * pass a harness built on equal dimensions. They are why all three differ. + * + * Every element is given a distinct value, so a comparison cannot pass by two + * different positions happening to hold the same number. The check is exact: + * this op does no arithmetic, so any difference at all is a bug. + */ +#include "hexlib/hexlib_harness.h" +#include "kernel_api.h" + +void transpose_hd_fp16_baseline(const hexlib_hf *, hexlib_hf *, int, int, int); + +#define MAXN (TR_B * TR_T * TR_D) + +static hexlib_hf X[MAXN] HEXLIB_ALIGN; +static hexlib_hf Y[MAXN] HEXLIB_ALIGN; +static hexlib_hf REF[MAXN] HEXLIB_ALIGN; + +static int check(int B, int T, int D, int *n_wrong, double *max_err) { + const int n = B * T * D; + for (int i = 0; i < n; ++i) { + /* Distinct and exactly representable in fp16: integers up to 2048 are. */ + X[i] = (hexlib_hf) (float) (i % 2048); + } + for (int i = 0; i < n; ++i) { + Y[i] = (hexlib_hf) 12345.0f; + REF[i] = (hexlib_hf) 0.0f; + } + + transpose_hd_fp16_baseline(X, REF, B, T, D); + transpose_hd_fp16(X, Y, B, T, D); + + for (int i = 0; i < n; ++i) { + double d = (double) (float) Y[i] - (double) (float) REF[i]; + if (d != 0.0) { + ++(*n_wrong); + } + if (d < 0.0) d = -d; + if (d > *max_err) *max_err = d; + } + return n; +} + +int main(void) { + int n_wrong = 0; + double max_err = 0.0; + + unsigned long long kcyc = 0; + /* Timed call is the whole-vector shape. The odd-T shape is checked for + * correctness but deliberately not folded into the cycle count, which + * would make the number mean nothing. */ + { + const int n = TR_B * TR_T * TR_D; + for (int i = 0; i < n; ++i) { + X[i] = (hexlib_hf) (float) (i % 2048); + Y[i] = (hexlib_hf) 12345.0f; + } + transpose_hd_fp16_baseline(X, REF, TR_B, TR_T, TR_D); + HEXLIB_TIME_KERNEL(kcyc, transpose_hd_fp16(X, Y, TR_B, TR_T, TR_D)); + for (int i = 0; i < n; ++i) { + double d = (double) (float) Y[i] - (double) (float) REF[i]; + if (d != 0.0) ++n_wrong; + if (d < 0.0) d = -d; + if (d > max_err) max_err = d; + } + } + + /* Scalar-only path: no chunk of the row is a whole number of vectors. */ + check(TR_B, TR_T_ODD, TR_D, &n_wrong, &max_err); + + hexlib_report(n_wrong == 0, n_wrong, max_err, kcyc); + return 0; +} diff --git a/kernels/transpose_hd_fp16/kernel.c b/kernels/transpose_hd_fp16/kernel.c new file mode 100644 index 0000000..ec12d39 --- /dev/null +++ b/kernels/transpose_hd_fp16/kernel.c @@ -0,0 +1,65 @@ +/* [B, T, D] -> [B, D, T] by gathering strided reads scalar-wise and committing + * them to the contiguous output row with HVX vector stores. + * + * WHY THE WRITE SIDE, NOT THE READ SIDE. See kernel_api.h for the full + * argument. In short: for a fixed (b, d), the output row y[b][d][:] is T + * contiguous elements, while the corresponding input column x[b][:][d] is T + * elements each D*2 bytes apart -- there is no run shared by both sides to + * hand a vector load. Exactly one side can be made contiguous by loop order, + * and it is cheaper to spend the vector unit on the side that is more + * expensive to get wrong (stores stall the store queue; loads have a + * prefetcher's help). + * + * HD_VLEN elements (one HVX vector's worth of fp16, 128 bytes / 2 = 64) are + * gathered into a small aligned stack buffer with scalar loads, then written + * out in one vector store via HVX_UVector -- the compiler-recognized + * "unaligned vector" pointer type from the SDK's hexagon_types.h, documented + * in this repo's own include/hexlib/hvx/hvx-base.h (`hvx_vmemu`). It is needed + * because T*2 bytes need not put every output row on a 128-byte boundary, so + * an aligned vector store is not always safe here. + * + * This is a genuine but PARTIAL acceleration: the store side is vectorized, + * the load side stays scalar. A full cross-lane HVX transpose (vdelta/vshuff) + * would vectorize both sides; that path was attempted and set aside -- see the + * kernel-transpose-hd report for what was tried and why. + */ +#include "kernel_api.h" + +#include +#include + +/* Elements per HVX vector at fp16: 128 bytes / 2 bytes each = 64. */ +#define HD_VLEN (128 / (int) sizeof(hexlib_hf)) + +void transpose_hd_fp16(const hexlib_hf *x, hexlib_hf *y, int B, int T, int D) { + if (B <= 0 || T <= 0 || D <= 0) { + return; + } + + /* Aligned so it can be read back as a single HVX_Vector below. */ + hexlib_hf buf[HD_VLEN] __attribute__((aligned(128))); + + for (int b = 0; b < B; ++b) { + const hexlib_hf *xb = x + (long) b * T * D; + hexlib_hf *yb = y + (long) b * D * T; + + for (int d = 0; d < D; ++d) { + const hexlib_hf *src_col = xb + d; /* stride D between t's */ + hexlib_hf *dst_row = yb + (long) d * T; /* contiguous over t */ + + int t = 0; + for (; t + HD_VLEN <= T; t += HD_VLEN) { + for (int i = 0; i < HD_VLEN; ++i) { + buf[i] = src_col[(long) (t + i) * D]; + } + const HVX_Vector *bv = (const HVX_Vector *) buf; + HVX_UVector *dv = (HVX_UVector *) (dst_row + t); + *dv = *bv; + } + /* Scalar tail: fewer than one whole vector's worth of t remains. */ + for (; t < T; ++t) { + dst_row[t] = src_col[(long) t * D]; + } + } + } +} diff --git a/kernels/transpose_hd_fp16/kernel_api.h b/kernels/transpose_hd_fp16/kernel_api.h new file mode 100644 index 0000000..f35468d --- /dev/null +++ b/kernels/transpose_hd_fp16/kernel_api.h @@ -0,0 +1,69 @@ +/* kernels/transpose_hd_fp16/kernel_api.h */ +#ifndef HEXLIB_TRANSPOSE_HD_FP16_API_H +#define HEXLIB_TRANSPOSE_HD_FP16_API_H + +typedef __fp16 hexlib_hf; + +/* [B, T, D] -> [B, D, T], fp16. perm (0, 2, 1) -- the QK^T operand layout move. + * + * y[b][d][t] = x[b][t][d] + * + * 12 ops in the encoder, at B=12 T=256 D=64. + * + * WHY THIS ONE IS HARD (see kernels/transpose_th_fp16/kernel_api.h for the easy + * sibling, perm (1,0,2), and read its header first -- this comment assumes it). + * There, D is untouched and innermost on BOTH operands, so a whole run of D + * contiguous elements moves as one unit. Here the permutation swaps the + * innermost two axes, so D is innermost on the INPUT and T is innermost on the + * OUTPUT: no run longer than one element is contiguous on both sides at once. + * A naive scalar loop is either a D*2-byte-strided load or a D*2-byte-strided + * store (D=64, fp16 -> 128 bytes = exactly one HVX vector at the encoder + * shape), and whichever side you make contiguous, the other is fully strided. + * There is no run of shared contiguity to hand to a vector load/store the way + * transpose_th_fp16 does. + * + * THE CHOICE MADE HERE: make the WRITE side contiguous and batch it into HVX + * vector stores; leave the READ side scalar. Two independent reasons: + * 1. For a fixed (b, d), y[b][d][:] is T contiguous elements -- the output's + * whole innermost run -- while x[b][:][d] is T elements each D*2 bytes + * apart, with no contiguity on the read side to exploit at any stride. + * Only one side of this op can ever be made contiguous by choice of loop + * order; here that side is the write. + * 2. transpose_th_fp16's own header argues that a store that misses is more + * expensive than a load that misses, because the load has a prefetcher's + * help and the store queue is what stalls. Batching the more expensive + * side into whole vectors is the side worth batching. + * So the inner loop gathers up to one HVX vector's worth of elements from the + * strided read (scalar, unavoidable -- there is no contiguous run to load) into + * a small aligned local buffer, then commits it with a SINGLE HVX vector store + * to the contiguous output row (unaligned-safe, via HVX_UVector, since T*2 + * bytes need not put every row on a 128-byte boundary). At the encoder's + * T=256, D=64 this turns 256 scalar stores per (b, d) row into 4 vector stores; + * the scalar reads are unchanged in count, but the effort that matters -- the + * store side -- is genuinely vectorized. + * + * NOT A FULL HVX TRANSPOSE. A true cross-lane 64x64 vector transpose (a + * vdelta/vshuff network) would vectorize the read side too, turning this into + * a real 64x speedup on both sides rather than one. That was attempted and set + * aside for this version -- see the accompanying report for what was tried and + * why it was not landed. This version's HVX use is real and ELF-provable (a + * genuine vector load and a genuine vector store per chunk) but PARTIAL: it + * accelerates the store side only, not the load side. + * + * ALIGNMENT: none required. x and y may start at any address; the gather + * buffer is aligned internally by the kernel, and the output store always uses + * HVX's unaligned vector-store form. + * + * A second shape whose run is NOT a whole number of vectors, so the harness + * exercises the scalar-only tail path too. TR_T_ODD * 2 = 80 bytes < 128, so no + * chunk of it ever reaches a whole vector. + */ +#define TR_B 5 +#define TR_T 64 +#define TR_D 7 + +#define TR_T_ODD 40 + +void transpose_hd_fp16(const hexlib_hf *x, hexlib_hf *y, int B, int T, int D); + +#endif diff --git a/kernels/transpose_hd_fp16/nearmiss_plain_copy.c b/kernels/transpose_hd_fp16/nearmiss_plain_copy.c new file mode 100644 index 0000000..e4dee9c --- /dev/null +++ b/kernels/transpose_hd_fp16/nearmiss_plain_copy.c @@ -0,0 +1,26 @@ +/* A plausible WRONG implementation the harness must reject. + * + * THE MISTAKE: copying the array through unchanged, as though a transpose were + * a reinterpretation of the same bytes. + * + * WHY ANYONE WOULD WRITE IT. This project establishes -- correctly -- that + * every `reshape` in the encoder is FREE, because at these shapes a reshape + * reads the same bytes in the same order and so moves nothing. `transpose` + * sits next to it in the IR, takes the same kind of `perm`-looking attribute, + * and also does no arithmetic. The step from "reshape is free" to "this layout + * op is free" is one inference, and it is wrong: a transpose reorders the + * bytes. + * + * Both ops also produce an output with the same ELEMENT COUNT as their input, + * so every shape check in the pipeline still passes. Nothing but the values + * catches this. + */ +#include "kernel_api.h" + +void transpose_hd_fp16(const hexlib_hf *x, hexlib_hf *y, int B, int T, int D) { + const long n = (long) B * T * D; + /* WRONG: same bytes, same order. */ + for (long i = 0; i < n; ++i) { + y[i] = x[i]; + } +} diff --git a/kernels/transpose_hd_fp16/nearmiss_swapped_stride.c b/kernels/transpose_hd_fp16/nearmiss_swapped_stride.c new file mode 100644 index 0000000..32c7130 --- /dev/null +++ b/kernels/transpose_hd_fp16/nearmiss_swapped_stride.c @@ -0,0 +1,38 @@ +/* A plausible WRONG implementation the harness must reject. + * + * THE MISTAKE: using D where T belongs in the OUTPUT stride -- the output row + * for a fixed (b, d) is placed at `((b * D + d) * D + t)` instead of + * `((b * D + d) * T + t)`. + * + * WHY ANYONE WOULD WRITE IT. The read side genuinely strides by D (successive + * t's in x are D elements apart), and that D is sitting right there in the + * expression one line above. The output's row stride is T -- the length of + * the row being written, not the length of the row being read -- but D is the + * variable that was just typed, and reusing it instead of reaching for T is + * exactly the kind of copy-paste-adjacent slip that survives a quick read: the + * two lines look symmetric when the wrong one is written and asymmetric when + * the right one is. + * + * WHY THE HARNESS CAN CATCH IT: only because T != D. At T == D the two + * expressions are identical and this kernel is CORRECT. The encoder's real + * shape is T=256, D=64, so they differ there -- but a harness built on a + * square shape would pass this and ship it. TR_T is 64 and TR_D is 7 for + * exactly this reason. + */ +#include "kernel_api.h" + +void transpose_hd_fp16(const hexlib_hf *x, hexlib_hf *y, int B, int T, int D) { + if (B <= 0 || T <= 0 || D <= 0) { + return; + } + for (int b = 0; b < B; ++b) { + for (int t = 0; t < T; ++t) { + for (int d = 0; d < D; ++d) { + hexlib_hf v = x[((long) b * T + t) * D + d]; + /* WRONG: D used as the output row stride, but the output row + * (fixed b, d, varying t) has length T, not D. */ + y[((long) b * D + d) * D + t] = v; + } + } + } +} diff --git a/kernels/transpose_hd_fp16/nearmiss_wrong_batch_stride.c b/kernels/transpose_hd_fp16/nearmiss_wrong_batch_stride.c new file mode 100644 index 0000000..7596c63 --- /dev/null +++ b/kernels/transpose_hd_fp16/nearmiss_wrong_batch_stride.c @@ -0,0 +1,37 @@ +/* A plausible WRONG implementation the harness must reject. + * + * THE MISTAKE: the transpose WITHIN each batch is correct -- t and d are + * swapped exactly as they should be -- but the batch dimension is walked as + * though y's flat layout were (T, D, B), batch LAST, instead of the actual + * (B, D, T), batch FIRST. + * + * WHY ANYONE WOULD WRITE IT. Someone gets the hard part right -- the + * inner two-axis swap that is the whole point of this kernel -- and then + * places the batch index using the same "outermost index gets the biggest + * stride" intuition that would be correct if batch actually varied slowest in + * memory, without checking that y is still allocated (B, D, T) and not some + * other order the intuition assumed. It is the mistake of getting the + * documented part of the contract right and the undocumented part (which axis + * is actually outermost in the buffer you were handed) wrong. + * + * WHY THE HARNESS CAN CATCH IT: because B, T and D are three distinct values, + * this lands every batch's data at a completely different -- and wrongly + * sized -- set of offsets than the real (B, D, T) layout, so almost nothing + * ends up where transpose_hd_fp16_baseline puts it. + */ +#include "kernel_api.h" + +void transpose_hd_fp16(const hexlib_hf *x, hexlib_hf *y, int B, int T, int D) { + if (B <= 0 || T <= 0 || D <= 0) { + return; + } + for (int b = 0; b < B; ++b) { + for (int t = 0; t < T; ++t) { + for (int d = 0; d < D; ++d) { + /* WRONG: walks y as though its shape were (T, D, B) -- batch + * last -- instead of the real (B, D, T) -- batch first. */ + y[((long) t * D + d) * B + b] = x[((long) b * T + t) * D + d]; + } + } + } +} diff --git a/kernels/transpose_hd_fp16/runner.c b/kernels/transpose_hd_fp16/runner.c new file mode 100644 index 0000000..17909b9 --- /dev/null +++ b/kernels/transpose_hd_fp16/runner.c @@ -0,0 +1,74 @@ +/* kernels/transpose_hd_fp16/runner.c -- the executor's entry point. + * + * PROTOCOL, little-endian, matching hexlib/exec/runner.py's spec shape for + * `transpose` (see kernels/transpose_th_fp16/runner.c for the sibling this + * mirrors): + * + * hexlib_in.bin int32 B, int32 T, int32 D + * fp16 x[B*T*D] + * hexlib_out.bin fp16 y[B*D*T] + * + * Three dimensions rather than one element count, because the permutation + * cannot be performed without knowing all three -- unlike the elementwise + * kernels, where n is enough. + */ +#include + +#include "hexlib/hexlib_harness.h" +#include "kernel_api.h" + +/* Encoder shape is 12*256*64 = 196608. */ +#define RUNNER_MAX_N 262144 + +static hexlib_hf X[RUNNER_MAX_N] HEXLIB_ALIGN; +static hexlib_hf Y[RUNNER_MAX_N] HEXLIB_ALIGN; + +int main(void) { + FILE *in = fopen("hexlib_in.bin", "rb"); + if (!in) { + printf("RUNNER error=no_input\n"); + return 2; + } + int B = 0, T = 0, D = 0; + if (fread(&B, sizeof(int), 1, in) != 1 + || fread(&T, sizeof(int), 1, in) != 1 + || fread(&D, sizeof(int), 1, in) != 1) { + printf("RUNNER error=short_header\n"); + fclose(in); + return 3; + } + if (B <= 0 || T <= 0 || D <= 0) { + printf("RUNNER error=bad_dims B=%d T=%d D=%d\n", B, T, D); + fclose(in); + return 4; + } + const long n = (long) B * T * D; + if (n > RUNNER_MAX_N) { + printf("RUNNER error=too_big n=%ld max=%d\n", n, RUNNER_MAX_N); + fclose(in); + return 4; + } + if (fread(X, sizeof(hexlib_hf), (size_t) n, in) != (size_t) n) { + printf("RUNNER error=short_payload n=%ld\n", n); + fclose(in); + return 5; + } + fclose(in); + + unsigned long long kcyc = 0; + HEXLIB_TIME_KERNEL(kcyc, transpose_hd_fp16(X, Y, B, T, D)); + + FILE *out = fopen("hexlib_out.bin", "wb"); + if (!out) { + printf("RUNNER error=no_output\n"); + return 6; + } + if (fwrite(Y, sizeof(hexlib_hf), (size_t) n, out) != (size_t) n) { + printf("RUNNER error=short_write\n"); + fclose(out); + return 7; + } + fclose(out); + printf("RUNNER ok n=%ld cycles=%llu\n", n, kcyc); + return 0; +} diff --git a/kernels/transpose_hd_fp16/spec.json b/kernels/transpose_hd_fp16/spec.json new file mode 100644 index 0000000..2f65dd2 --- /dev/null +++ b/kernels/transpose_hd_fp16/spec.json @@ -0,0 +1,18 @@ +{ + "task_id": "transpose_hd_fp16", + "dtype": "fp16", + "caps": [], + "mechanisms": ["hvx"], + "params": { + "B": 5, + "T": 64, + "D": 7, + "T_odd": 40, + "encoder_shape": "[12, 256, 64] -> [12, 64, 256]", + "encoder_op_count": 12 + }, + "expert_kernel_cycles": null, + "tolerance": "exact", + "movement_only": true, + "tags": ["layout", "encoder", "attention"] +} diff --git a/pyproject.toml b/pyproject.toml index b685c57..c28aa1e 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -16,12 +16,35 @@ hexlib = "hexlib.cli:main" include = ["hexlib*"] [project.optional-dependencies] +# Everything needed to run the offline test suite. numpy is a hard dependency +# already; pytest is the only addition. Deliberately NOT listed: pytest-asyncio +# (no test in this repo is async -- it only appears in local warning output +# because it happens to be installed in one dev environment), and +# `qualcomm_device_cloud_sdk`, which is account-gated and imported lazily inside +# hexlib/device/qdc/job.py so that importing the package never requires it. +# +# CI installs THIS extra. It must stay sufficient on its own: `.github/workflows +# /ci.yml` used to read `pip install -e ".[dev]" || pip install -e . pytest`, +# intending the second half as a safety net, but no `dev` extra existed and pip +# treats a missing extra as a WARNING with exit 0 -- so the `||` could never +# fire and pytest was never installed. The build went red on `No module named +# pytest` rather than falsely green, but the fallback was doing nothing. +dev = ["pytest>=8"] oracle = ["torch>=2.0", "transformers>=5.10"] [tool.setuptools.package-data] "hexlib.tests" = ["data/*.npz"] [tool.pytest.ini_options] +# NO `testpaths` HERE, ON PURPOSE. Keeping hexlib/device/qdc/test_on_device.py +# out of the offline suite (it runs on the phone, and its flat `import utils` +# breaks collection here) is done by `collect_ignore` in the ROOT conftest.py +# instead. `testpaths` would apply only when no path is given on the command +# line, so `pytest hexlib` or `pytest .` would walk back into it; the conftest +# covers every invocation form. See conftest.py's docstring for the full +# reasoning and hexlib/tests/test_qdc_on_device_is_excluded.py for the test +# that pins both -- adding `testpaths` in place of that mechanism fails +# test_naming_a_path_does_not_reach_the_on_device_test_either. markers = [ "sdk: requires the Hexagon SDK (hexagon-clang, hexagon-sim). Not run in CI.", ] diff --git a/scripts/qdc_interactive.py b/scripts/qdc_interactive.py new file mode 100644 index 0000000..8b43abc --- /dev/null +++ b/scripts/qdc_interactive.py @@ -0,0 +1,250 @@ +#!/usr/bin/env python3 +"""Run hexlib's device binaries on real Snapdragon silicon via a QDC session. + +FIRST WORKING PATH TO SILICON, 2026-08-13. Four Appium test-package jobs +(756124, 756159, 756206, 756221) all reached Completed with +result=UNSUCCESSFUL and published no logs of their own, so nothing inside the +package could be observed. The interactive session works instead, and this +file is the recipe. + +THREE THINGS ARE ALL REQUIRED. Each was a separate discovery and none is +obvious from the SDK signature: + + 1. THE KEY IS QDC'S, NOT YOURS. `submit_session(ssh_public_key=...)` refuses + a key of your own with "The provided SSH public key could not be found or + has not been created for user". QDC ISSUES the pair -- look for + `~/.ssh/qdc_id_.pem` dropped when a session was created from the web + UI. Probing ours showed the 2026-08-07 key registered and the 2026-08-06 + one not, so they expire or get replaced; if this script starts failing at + submit, make a session in the UI and use the new pem. + + 2. session_parameters=[SSHONLY] IS WHAT PROVISIONS SSH. Without it the + session is created, reaches Running, and NEVER publishes an sshConfigs + entry. Two ten-minute polls were burned on that (sessions 756450, + 756584) before the parameter was found. + + 3. IT IS AN ADB TUNNEL, NOT A SHELL. sshConfigs hands back + ssh -i \\ + -L ::5037 -N sshtunnel@ssh.qdc.qualcomm.com + which forwards a local port to the DEVICE'S ADB SERVER. There is no + remote host to scp to and no remote shell. Everything runs from THIS + machine through `adb -P `. + +BILLING. Sessions bill by the minute and are billed for the full timeout, not +for what you use (session 756450: 15 charged, ~10 used). complete_session runs +in a finally, and `--timeout` bounds the worst case even if this process dies. + +WHAT IT FOUND ON THE FIRST REAL RUN, so nobody re-derives it: + * `--caps` returns arch_ver 35957 (0x8c75) -- BIT-IDENTICAL to the + simulator, which is what job.py's fact 1 asserted and had never checked. + unsigned_pd_support=1, vtcm_total_bytes=8388608. + * The skel is built as `libhexlib_skel.so` but FastRPC dlopens + `libhexlib_iface_skel.so`. No simulator test can catch this: stage 1 links + the skel directly instead of loading it by name. Push it under both names + until runtime/build.py is fixed. + * cycles_total=14267 in a user-mode unsigned PD -- NON-ZERO. STATE.md called + this the most important thing a device job can report, because a dead + PCYCLE would invalidate every cycle figure stage 1 measured. + * The unmapped-fd refusal holds against real ION (batch status 7). + * `--self-test` returns 3859/4100 values not bit-exact and + `--coherency-check` exits 6 `sentinel_unchanged`. That is NOT a coherency + diagnosis: main.c:94 says exit 6 is equally consistent with a kernel or + generated entry returning OK without writing its output. Settling it needs + the skel-side echo op STATE.md records as deferred. + +usage: + python scripts/qdc_interactive.py --key ~/.ssh/qdc_id_2026-8-7_847.pem \\ + --bin-dir [--timeout-min 15] +""" +from __future__ import annotations + +import argparse +import json +import os +import socket +import subprocess +import sys +import time + +SSH_EXTRA = [ + "-o", "StrictHostKeyChecking=no", + "-o", "UserKnownHostsFile=/dev/null", + "-o", "IdentitiesOnly=yes", + "-o", "ExitOnForwardFailure=yes", + "-o", "ServerAliveInterval=15", + "-o", "LogLevel=ERROR", +] +DEV = "/data/local/tmp/hexlib" + + +def log(m: str) -> None: + print(f"[{time.strftime('%H:%M:%S')}] {m}", flush=True) + + +def adb(port: int, *args: str, timeout: int = 180): + cmd = ["adb", "-P", str(port), *args] + log("$ " + " ".join(cmd[:7]) + (" ..." if len(cmd) > 7 else "")) + p = subprocess.run(cmd, capture_output=True, text=True, + encoding="utf-8", errors="replace", timeout=timeout) + out = ((p.stdout or "") + (p.stderr or "")).rstrip() + if out: + print(out[:4000], flush=True) + log(f" -> exit {p.returncode}") + return p.returncode, out + + +def port_open(port: int) -> bool: + with socket.socket() as s: + s.settimeout(2) + return s.connect_ex(("127.0.0.1", port)) == 0 + + +def main() -> int: + ap = argparse.ArgumentParser() + ap.add_argument("--key", required=True, + help="QDC-ISSUED pem (~/.ssh/qdc_id_.pem), not your own key") + ap.add_argument("--bin-dir", required=True, + help="directory holding hexlib_run and the skel .so") + ap.add_argument("--stage-dir", + help="directory holding batch.bin + arena.bin (a whole-plan " + "batch from hexlib.exec.wholeplan). If given, the " + "session pushes them, runs --batch, and pulls the " + "arena back as out.bin") + ap.add_argument("--self-tests", action="store_true", + help="also run --self-test / --coherency-check / --unmapped") + ap.add_argument("--timeout-min", type=int, default=15, + help="session ceiling; you are billed for ALL of it") + ap.add_argument("--adb-port", type=int, default=15037) + ap.add_argument("--ready-wait-s", type=int, default=420) + args = ap.parse_args() + + from qualcomm_device_cloud_sdk.api import qdc_api as v + from qualcomm_device_cloud_sdk.models.session_submission_parameter import ( + SessionSubmissionParameter, + ) + from hexlib.device.qdc import job + from hexlib.runtime import build + + key = os.path.expanduser(args.key) + pub = subprocess.run(["ssh-keygen", "-y", "-f", key], + capture_output=True, text=True).stdout.strip() + if not pub: + log(f"cannot derive a public key from {key}") + return 1 + + client = job._client() + sid = v.submit_session( + public_api_client=client, + target_id=job.TARGET_ID, + session_name="hexlib interactive", # <= 32 chars or QDC answers 400 + timeout=args.timeout_min, + ssh_public_key=pub, + session_parameters=[SessionSubmissionParameter.SSHONLY], + ) + if sid is None: + log("submit_session returned None") + return 1 + log(f"session {sid} submitted (timeout {args.timeout_min} min)") + + tunnel = None + try: + cmd = None + deadline = time.time() + args.ready_wait_s + while time.time() < deadline: + d = json.loads(v.get_session_by_id(client, sid).content.decode()) + cfgs = d.get("sshConfigs") or [] + log(f"state={d.get('state')} sshConfigs={len(cfgs)}") + if cfgs: + cmd = cfgs[0].get("sshCommand") or cfgs[0].get("qualnetSshCommand") + break + if d.get("state") in ("Completed", "Canceled", "Failed"): + log(f"session ended early: {d.get('state')}") + return 1 + time.sleep(15) + if not cmd: + log("no sshConfigs before the cap -- is SSHONLY set and the key QDC's?") + return 1 + + real = (cmd.replace("", key) + .replace("", str(args.adb_port))) + parts = real.split() + tunnel = subprocess.Popen([parts[0], *SSH_EXTRA, *parts[1:]], + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, text=True) + for _ in range(30): + if port_open(args.adb_port): + break + if tunnel.poll() is not None: + log(f"tunnel died: {(tunnel.stdout.read() or '')[:800]}") + return 1 + time.sleep(2) + else: + log(f"port {args.adb_port} never opened") + return 1 + log(f"tunnel up: local {args.adb_port} -> device adb server") + + p = args.adb_port + rc, out = adb(p, "devices") + if rc != 0 or "\tdevice" not in out: + log("no device through the tunnel") + return 1 + adb(p, "shell", "getprop ro.product.model") + adb(p, "shell", f"mkdir -p {DEV}") + adb(p, "push", os.path.join(args.bin_dir, "hexlib_run"), f"{DEV}/") + # ONE NAME NOW. The build emits the name FastRPC dlopens + # (`device_skel_so_name()`, derived from the IDL stem), so the + # push-under-both-names workaround this script used to carry is gone. + # If it comes back, the bug is in runtime/build.py, not here. + skel = build.device_skel_so_name() + adb(p, "push", os.path.join(args.bin_dir, skel), f"{DEV}/") + adb(p, "shell", f"chmod 755 {DEV}/hexlib_run") + + env = f"cd {DEV} && ADSP_LIBRARY_PATH={DEV}" + log("=== hexlib_run --caps ===") + adb(p, "shell", f"{env} ./hexlib_run --caps; echo RC=$?", timeout=300) + + if args.self_tests: + for mode in ("--self-test", + "--self-test --coherency-check", + "--self-test --unmapped"): + log(f"=== hexlib_run {mode} ===") + adb(p, "shell", f"{env} ./hexlib_run {mode}; echo RC=$?", timeout=300) + + if args.stage_dir: + # THE WHOLE PLAN, ONE INVOKE. `arena.bin` is an IN-OUT buffer: it + # carries the weights and the graph input in, and the DSP writes + # every activation and the output back into it, so the same file + # is both `--in` and the thing pulled back afterwards. + batch = os.path.join(args.stage_dir, "batch.bin") + arena = os.path.join(args.stage_dir, "arena.bin") + adb(p, "push", batch, f"{DEV}/") + adb(p, "push", arena, f"{DEV}/") + log("=== hexlib_run --batch (the whole encoder, one invoke) ===") + rc, out = adb( + p, "shell", + f"{env} ./hexlib_run --batch batch.bin --in arena.bin " + f"--out out.bin; echo RC=$?", + timeout=900, + ) + if "RC=0" in out: + dest = os.path.join(args.stage_dir, "out.bin") + adb(p, "pull", f"{DEV}/out.bin", dest, timeout=300) + log(f"pulled {dest}") + else: + log("--batch did not exit 0; nothing pulled") + return 0 + finally: + if tunnel and tunnel.poll() is None: + log("closing tunnel") + tunnel.terminate() + log(f"completing session {sid}") + try: + v.complete_session(client, sid) + log("session completed") + except Exception as e: # noqa: BLE001 + log(f"complete_session FAILED: {e} -- session {sid} bills until its " + f"{args.timeout_min}-minute timeout") + + +if __name__ == "__main__": + sys.exit(main())