diff --git a/.gitignore b/.gitignore index bf804e07..6fd0322c 100644 --- a/.gitignore +++ b/.gitignore @@ -227,3 +227,6 @@ benchmarks/cross_framework # local e2e/bench artifacts (harnesses may run with repo cwd) /results/ + +# Local scratch / agent worktrees (never committed) +.claude/ diff --git a/benchmarks/README.md b/benchmarks/README.md index 6218903f..34e99fbb 100644 --- a/benchmarks/README.md +++ b/benchmarks/README.md @@ -27,5 +27,28 @@ batch size x miss rate. python benchmarks/bench_offload_cache_copy.py ``` +**`bench_ornith_attention.py`** — synthetic (no checkpoint): Ornith's exact attention +geometry (16 query heads, 2 KV heads, head_dim 256 — the GQA shape `decode_launch_config` +tunes packed-int4/Q4_0 decode for) through the production Triton kernels directly +(`decode_paged_attention` / `paged_attention` / `extend_paged_attention`, no server). +Sweeps decode context length x batch size x the `max_kv_splits` scratch ceiling, plus +representative prefill (fresh chunk) and extend (cached prefix + new chunk) cases, over +one or more `--kv-quant` pool formats (`int4`/`q4_0`, `q8_0`, `fp8_e4m3`, `bf16`). Every +quantized case is checked against the same kernel fed the pool's dequantized values +before it is timed — the correctness gate `test_ornith_q4_tuned_decode_matches_dequantized_oracle` +pins at unit scale, exercised here at benchmark scale. + +```bash +python benchmarks/bench_ornith_attention.py +python benchmarks/bench_ornith_attention.py --decode-lengths 8192 32768 131072 200000 \ + --kv-quant int4 q8_0 --batch-sizes 1 4 16 --json out.jsonl +``` + For host RAM vs PCIe bandwidth and the offload/hybrid backend pick, use `ft bench bw` instead — it writes the JSON profile the engine reads. + +`bench_decode_moe.py` also accepts `--max-context` (full-context `--max-seq-len-override` ++ `--num-tokens`), `--kv-cache-dtype`, `--prefill-chunk` (`--max-prefill-length`), and +`--prefill-hit-d2d`, for reproducing the long-context configurations in `docs/models.md` +(e.g. Ornith Q4_0 at 200K) through the real serving path — all optional, defaults +unchanged when omitted. diff --git a/benchmarks/bench_decode_moe.py b/benchmarks/bench_decode_moe.py index 723c2be5..25853523 100644 --- a/benchmarks/bench_decode_moe.py +++ b/benchmarks/bench_decode_moe.py @@ -112,6 +112,31 @@ def parse_args(argv: list[str] | None = None) -> argparse.Namespace: default=1800, help="seconds to wait for the spawned server to become ready", ) + p.add_argument( + "--max-context", + type=int, + default=None, + help="server --max-seq-len-override AND --num-tokens, for a full-context run " + "(e.g. a long-context Ornith/Laguna session); default keeps the prior " + "8192 + --decode sizing with the server's own --num-tokens default", + ) + p.add_argument( + "--kv-cache-dtype", + default=None, + help="server --kv-cache-dtype (auto|q8_0|fp8_e4m3|int4|q4_0); default leaves the " + "server's own default (auto, unquantized) in place", + ) + p.add_argument( + "--prefill-chunk", + type=int, + default=None, + help="server --max-prefill-length; default leaves the server's own default chunk size", + ) + p.add_argument( + "--prefill-hit-d2d", + action="store_true", + help="pass --moe-prefill-hit-d2d to the server (off by default, matching the server default)", + ) p.add_argument("--json", dest="json_out", default=None, help="append the result rows here") return p.parse_args(argv) @@ -172,17 +197,28 @@ def free_port() -> int: def serve_cmd(args: argparse.Namespace, backend: str, port: int) -> list[str]: + max_seq_len = args.max_context if args.max_context is not None else 8192 + args.decode cmd = [ sys.executable, "-m", "freetoken.cli", "serve", "--model", args.model, "--host", "127.0.0.1", "--port", str(port), "--moe-backend", backend, "--max-running-requests", "1", - "--max-seq-len-override", str(8192 + args.decode), + "--max-seq-len-override", str(max_seq_len), "--memory-ratio", str(args.mem_ratio), "--cuda-graph-max-bs", "0" if args.no_graph else "1", "--moe-hybrid-max-fetch", str(args.hybrid_fetch), ] + # Every flag below is opt-in and omitted unless passed, so a bare invocation keeps + # the server's own defaults exactly as before this option set existed. + if args.max_context is not None: + cmd += ["--num-tokens", str(args.max_context)] + if args.kv_cache_dtype is not None: + cmd += ["--kv-cache-dtype", args.kv_cache_dtype] + if args.prefill_chunk is not None: + cmd += ["--max-prefill-length", str(args.prefill_chunk)] + if args.prefill_hit_d2d: + cmd.append("--moe-prefill-hit-d2d") if args.cache > 0: cmd += ["--moe-cache-size", str(args.cache)] elif args.cache_rate is not None: @@ -309,6 +345,9 @@ def run_one(args: argparse.Namespace, backend: str) -> dict: f"[bench] model={args.model}\n" f"[bench] backend={backend} cache={args.cache or args.cache_rate or 'auto'} " f"mem_ratio={args.mem_ratio} decode={args.decode} graph={not args.no_graph}\n" + f"[bench] max_context={args.max_context or f'{8192 + args.decode} (default)'} " + f"kv_cache_dtype={args.kv_cache_dtype or 'auto (default)'} " + f"prefill_chunk={args.prefill_chunk or 'default'} prefill_hit_d2d={args.prefill_hit_d2d}\n" f"[bench] sampling={sampling} <- {sampling_src}\n" f"[bench] server log: {log_path}", flush=True, diff --git a/benchmarks/bench_gguf_gemm.py b/benchmarks/bench_gguf_gemm.py new file mode 100644 index 00000000..c8e133ba --- /dev/null +++ b/benchmarks/bench_gguf_gemm.py @@ -0,0 +1,171 @@ +"""GGUF quantized-matmul benchmark: int8-MMA MMQ vs DP4A MMQ vs dequant+cuBLAS vs MMVQ. + +Sweeps the dense (``fused_mul_mat_gguf`` seams) and grouped-MoE +(``_moe_matmul`` seams) kernel families over batch size on synthetic +random-but-safe packed weights (fp16 scale fields masked small, as in +``tests/kernels/test_gguf_quant_types.py``), at Ornith-1.5-35B geometry by +default. Every timed case is first cross-checked against the transient +dequantized weights (the oracle): rel error must stay below --tol. + +The int8-MMA columns need the ``freetoken_gguf_mmq`` extension (sm_75+ build, +sm_120-dispatched); they are skipped with a note where unavailable. + +Run: + CUDA_VISIBLE_DEVICES=0 PYTHONPATH=python python benchmarks/bench_gguf_gemm.py + ... --dense-rows 4 16 32 512 8192 --moe-tokens 16 320 8192 --json out.jsonl +""" + +from __future__ import annotations + +import argparse +import json +import statistics + + +def build_cases(args) -> list[dict]: + """Pure case list (unit-testable without CUDA).""" + cases = [] + for rows in args.dense_rows: + for qtype_name in args.dense_types: + cases.append({"op": "dense", "rows": rows, "qtype": qtype_name, + "out_features": args.dense_out, "in_features": args.hidden}) + for tokens in args.moe_tokens: + cases.append({"op": "moe", "tokens": tokens, "experts": args.experts, + "top_k": args.top_k, "hidden": args.hidden, "inter": args.inter}) + return cases + + +def parse_args(argv=None): + p = argparse.ArgumentParser(description=__doc__) + p.add_argument("--dense-rows", type=int, nargs="+", default=[4, 8, 16, 32, 128, 512, 2048, 8192]) + p.add_argument("--dense-types", nargs="+", default=["q4_k", "q6_k"], choices=["q4_k", "q6_k"]) + p.add_argument("--dense-out", type=int, default=8192) + p.add_argument("--moe-tokens", type=int, nargs="+", default=[16, 64, 320, 1024, 8192]) + p.add_argument("--experts", type=int, default=256) + p.add_argument("--top-k", type=int, default=8) + p.add_argument("--hidden", type=int, default=2048) + p.add_argument("--inter", type=int, default=512) + p.add_argument("--iters", type=int, default=20) + p.add_argument("--tol", type=float, default=0.02) + p.add_argument("--json", type=str, default=None) + return p.parse_args(argv) + + +def _med_ms(fn, iters): + import torch + + for _ in range(3): + fn() + torch.cuda.synchronize() + times = [] + for _ in range(iters): + s, e = torch.cuda.Event(True), torch.cuda.Event(True) + s.record() + fn() + e.record() + torch.cuda.synchronize() + times.append(s.elapsed_time(e)) + return statistics.median(times) + + +def _packed(qtype, rows_of_blocks, seed): + import numpy as np + import torch + from freetoken.models.gguf.dequant import BLOCK_SHAPE + + rng = np.random.default_rng(seed) + raw = rng.integers(0, 256, (rows_of_blocks, BLOCK_SHAPE[qtype][1]), dtype=np.uint8) + raw.view(np.uint16)[:] &= np.uint16(0x3BFF) + return torch.from_numpy(raw) + + +def main(argv=None) -> int: + args = parse_args(argv) + import torch + + from freetoken.kernel.gguf import ( + ggml_dequantize, + ggml_moe_a8, + ggml_moe_get_block_size, + ggml_mul_mat_a8, + ggml_mul_mat_vec_a8, + ) + from freetoken.models.gguf.dequant import BLOCK_SHAPE, GGML_Q4_K, GGML_Q6_K + from freetoken.moe.fused import moe_align_block_size + + qtypes = {"q4_k": GGML_Q4_K, "q6_k": GGML_Q6_K} + try: + from freetoken.kernel.gguf import ggml_moe_a8_mma, ggml_mul_mat_a8_mma + + mma_ok = True + except Exception as exc: # noqa: BLE001 - report and continue without MMA + print(f"# int8-MMA extension unavailable: {exc}") + mma_ok = False + + out_rows = [] + torch.manual_seed(0) + for case in build_cases(args): + if case["op"] == "dense": + qtype = qtypes[case["qtype"]] + block, ts = BLOCK_SHAPE[qtype] + out_f, in_f, rows = case["out_features"], case["in_features"], case["rows"] + w = _packed(qtype, out_f * in_f // block, seed=qtype).reshape(out_f, -1).cuda() + dense = ggml_dequantize(w, qtype, out_f, in_f, torch.float16) + x = torch.randn(rows, in_f, dtype=torch.float16, device="cuda") + ref = (x.float() @ dense.float().T) + res = {"op": "dense", "qtype": case["qtype"], "rows": rows} + + def check(name, y): + rel = ((y.float() - ref).norm() / ref.norm()).item() + assert rel < args.tol, (name, rel) + + y = ggml_mul_mat_a8(w, x, qtype, out_f) + check("dp4a", y) + res["dp4a_ms"] = _med_ms(lambda: ggml_mul_mat_a8(w, x, qtype, out_f), args.iters) + res["dequant_ms"] = _med_ms( + lambda: x @ ggml_dequantize(w, qtype, out_f, in_f, torch.float16).T, args.iters + ) + if rows <= 8: + y = ggml_mul_mat_vec_a8(w, x, qtype, out_f) + check("mmvq", y) + res["mmvq_ms"] = _med_ms(lambda: ggml_mul_mat_vec_a8(w, x, qtype, out_f), args.iters) + if mma_ok: + y = ggml_mul_mat_a8_mma(w, x.float(), qtype, out_f) + check("mma", y) + res["mma_ms"] = _med_ms(lambda: ggml_mul_mat_a8_mma(w, x, qtype, out_f), args.iters) + else: + experts, top_k = case["experts"], case["top_k"] + hidden, inter, tokens = case["hidden"], case["inter"], case["tokens"] + block, ts = BLOCK_SHAPE[GGML_Q4_K] + gu = _packed(GGML_Q4_K, experts * 2 * inter * hidden // block, seed=1).reshape(experts, -1).cuda() + x = torch.randn(tokens, hidden, dtype=torch.float16, device="cuda") + ids = torch.stack([torch.randperm(experts)[:top_k] for _ in range(tokens)]).int().cuda() + res = {"op": "moe", "tokens": tokens} + + def dp4a_moe(): + bs = ggml_moe_get_block_size(GGML_Q4_K) + s, ei, npad = moe_align_block_size(ids, bs, experts) + return ggml_moe_a8(x, gu, s, ei, npad, GGML_Q4_K, 2 * inter, top_k, tokens) + + ref_moe = dp4a_moe().float() + res["dp4a_ms"] = _med_ms(dp4a_moe, args.iters) + if mma_ok: + y = ggml_moe_a8_mma(x, gu, ids, top_k, GGML_Q4_K, 2 * inter, tokens, gu.shape[1], True) + rel = ((y.float() - ref_moe).norm() / ref_moe.norm()).item() + assert rel < args.tol, ("moe-mma", rel) + res["mma_ms"] = _med_ms( + lambda: ggml_moe_a8_mma(x, gu, ids, top_k, GGML_Q4_K, 2 * inter, tokens, gu.shape[1], True), + args.iters, + ) + print(res) + out_rows.append(res) + + if args.json: + with open(args.json, "w") as f: + for row in out_rows: + f.write(json.dumps(row) + "\n") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/benchmarks/bench_ornith_attention.py b/benchmarks/bench_ornith_attention.py new file mode 100644 index 00000000..1f640a43 --- /dev/null +++ b/benchmarks/bench_ornith_attention.py @@ -0,0 +1,417 @@ +"""Ornith Q4_0 attention benchmark: decode/prefill/extend at the exact 16Q/2KV/D256 geometry. + +Ornith (Qwen3.5-MoE-family GQA: 16 query heads, 2 KV heads, head_dim 256) is the shape +``decode_launch_config`` in ``kernel/triton/attention.py`` special-cases for the packed +int4 (GGML Q4_0 / ``--kv-cache-dtype q4_0``) KV pool: BLOCK_N=32 / 32 splits / 4 warps, +selected because the generic BLOCK_N=16 tuning silently corrupted this geometry's packed +loads (see ``tests/kernels/test_kv_quant.py::test_ornith_q4_tuned_decode_matches_dequantized_oracle``, +which this bench exercises at benchmark scale). This script calls the SAME production +Triton entry points (``decode_paged_attention`` / ``paged_attention`` / +``extend_paged_attention``) no server, no model weights. + +Three op families, matching how a long-context Ornith session actually drives attention: + + decode -- one new token against a cached context of length L (the tuned split-K + kernel above). Swept over context length, batch size, and the max_kv_splits + scratch ceiling (``launch_splits = min(preferred, ceiling)``, so a ceiling + below the tuned value reproduces the degraded-scratch/capture-buffer case + the kernel's docstring calls out). + prefill -- a fresh chunk attending only to itself (first chunk of a prompt, or a + --max-prefill-length-sized chunk), via the fused causal kernel. + extend -- a fresh chunk attending to a long CACHED (quantized) prefix plus itself, + via the split extend kernel -- the realistic shape of chunk N>1 of a long + prompt, or of extending a session's context. + +Correctness gate (always on unless --skip-verify): before ANY case is timed, its +production (quantized) call is checked against the SAME kernel fed the pool's +dequantized values -- the dequantized values are the oracle, not a separate reference +implementation, so this isolates "does the packed-Q4 dequant-and-dot path compute the +same attention" from "how fast is it". Tolerance matches the kernel's own regression +tests (rtol=atol=2e-2, bf16 accumulation). + +Run: + CUDA_VISIBLE_DEVICES=0 PYTHONPATH=python python benchmarks/bench_ornith_attention.py + ... --decode-lengths 8192 32768 131072 200000 --kv-quant int4 q8_0 --json out.jsonl + ... --ops decode --batch-sizes 1 4 16 --max-kv-splits 8 16 32 +""" + +from __future__ import annotations + +import argparse +import json +import statistics +from dataclasses import dataclass, field + +# Geometry this bench targets by default: Ornith-1.5-35B-A3B's attention (16 query +# heads, 2 KV heads, head_dim 256) -- the exact shape decode_launch_config tunes for. +Q_HEADS = 16 +KV_HEADS = 2 +HEAD_DIM = 256 + +QUANT_CHOICES = ("int4", "q4_0", "q8_0", "fp8_e4m3", "bf16") +# CLI spelling -> freetoken.kvcache.quant spec name. "q4_0"/"int4" are the same packed +# scheme (llama.cpp naming vs the internal one); "bf16" means the unquantized pool. +_QUANT_ALIAS = {"int4": "int4", "q4_0": "int4", "q8_0": "q8_0", "fp8_e4m3": "fp8_e4m3", "bf16": "auto"} + +OPS = ("decode", "prefill", "extend") + + +# -------------------------------------------------------------------------------------- +# Pure case-building (no torch/CUDA import) -- kept separate so it is unit-testable on a +# machine with no GPU. +# -------------------------------------------------------------------------------------- + + +@dataclass(frozen=True) +class DecodeCase: + op: str = field(default="decode", init=False) + quant: str = "" + ctx_len: int = 0 + batch: int = 0 + max_kv_splits: int | None = None # None = use the kernel's own tuned preference + + +@dataclass(frozen=True) +class PrefillCase: + op: str = field(default="prefill", init=False) + quant: str = "" + chunk_len: int = 0 + + +@dataclass(frozen=True) +class ExtendCase: + op: str = field(default="extend", init=False) + quant: str = "" + prefix_len: int = 0 + chunk_len: int = 0 + + +def build_cases(args: argparse.Namespace) -> list: + """Expand the CLI sweeps into one case object per (op, quant, config) point. + + Deterministic order (op, then quant, then the op's own axes) so --json output and + the printed table always list cases the same way for a given set of flags. + """ + cases: list = [] + quants = args.kv_quant # kept as the CLI spelling for display; runners resolve via _QUANT_ALIAS + max_kv_splits = args.max_kv_splits or [None] + + if "decode" in args.ops: + for quant in quants: + for ctx_len in args.decode_lengths: + for batch in args.batch_sizes: + for splits in max_kv_splits: + cases.append(DecodeCase(quant=quant, ctx_len=ctx_len, batch=batch, max_kv_splits=splits)) + if "prefill" in args.ops: + for quant in quants: + for chunk_len in args.prefill_chunk_sizes: + cases.append(PrefillCase(quant=quant, chunk_len=chunk_len)) + if "extend" in args.ops: + for quant in quants: + for prefix_len in args.decode_lengths: + for chunk_len in args.extend_chunk_sizes: + cases.append(ExtendCase(quant=quant, prefix_len=prefix_len, chunk_len=chunk_len)) + return cases + + +def parse_args(argv: list[str] | None = None) -> argparse.Namespace: + p = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) + p.add_argument("--device", type=int, default=0) + p.add_argument("--ops", nargs="+", default=list(OPS), choices=OPS) + p.add_argument( + "--kv-quant", nargs="+", default=["int4"], choices=QUANT_CHOICES, + help="pool format(s) to bench; 'bf16' is the unquantized baseline (no oracle check)", + ) + p.add_argument( + "--decode-lengths", type=int, nargs="+", default=[2048, 16384, 65536, 131072], + help="decode: cached context length before the new token. extend: cached prefix length.", + ) + p.add_argument("--batch-sizes", type=int, nargs="+", default=[1], help="decode: concurrent requests") + p.add_argument( + "--max-kv-splits", type=int, nargs="+", default=None, + help="decode: scratch ceiling for the split-K reduction (launch uses min(tuned, ceiling)); " + "default lets each case use its quant's own tuned preference", + ) + p.add_argument( + "--prefill-chunk-sizes", type=int, nargs="+", default=[512, 2048, 8192], + help="prefill: fresh (no-prefix) chunk sizes; 8192 matches the server's default --max-prefill-length", + ) + p.add_argument( + "--extend-chunk-sizes", type=int, nargs="+", default=[2048], + help="extend: freshly-computed chunk size attending to the cached --decode-lengths prefix", + ) + p.add_argument("--warmup", type=int, default=5) + p.add_argument("--iters", type=int, default=20) + p.add_argument("--seed", type=int, default=0) + p.add_argument("--skip-verify", action="store_true", help="skip the dequantized-oracle correctness gate") + p.add_argument("--json", dest="json_out", default=None, help="append one JSON line per case here") + return p.parse_args(argv) + + +# -------------------------------------------------------------------------------------- +# CUDA execution -- imports torch/triton lazily so --help and case-building stay usable +# without a GPU. +# -------------------------------------------------------------------------------------- + + +def _make_kv(tokens: int, heads: int, dim: int, device, seed: int): + import torch + + g = torch.Generator(device=device).manual_seed(seed) + return torch.randn(tokens, heads, dim, generator=g, device=device, dtype=torch.bfloat16) + + +def _quantize_pool(spec, k, v): + """Store bf16 k/v into a pool of the given quant spec. + + Returns ``(k_cache, k_scale, v_cache, v_scale, k_oracle, v_oracle)``. For the + unquantized spec the cache tensors ARE k/v and there is no oracle (nothing to check + the production path against) -- callers must skip the correctness gate in that case. + """ + import torch + + from freetoken.kernel.triton.kv_quant import store_kv_quant + from freetoken.kvcache.quant import BLOCK + + if not spec.enabled: + return k, None, v, None, None, None + + slots, heads, dim = k.shape + epb = spec.elements_per_byte + kq = torch.zeros(slots, heads, dim // epb, device=k.device, dtype=spec.storage_dtype) + vq = torch.zeros_like(kq) + ks = torch.zeros(slots, heads, dim // BLOCK, device=k.device, dtype=torch.float16) + vs = torch.zeros_like(ks) + indices = torch.arange(slots, device=k.device, dtype=torch.int32) + store_kv_quant(kq, ks, vq, vs, indices, k, v, spec) + k_oracle = spec.dequantize(kq.float(), ks).to(torch.bfloat16).reshape(slots, heads, dim) + v_oracle = spec.dequantize(vq.float(), vs).to(torch.bfloat16).reshape(slots, heads, dim) + return kq, ks, vq, vs, k_oracle, v_oracle + + +def _check_oracle(got, want, label: str) -> float: + import torch + + torch.testing.assert_close(got, want, rtol=2e-2, atol=2e-2, msg=lambda m: f"{label}: {m}") + return (got.float() - want.float()).abs().max().item() + + +def _time_ms(fn, warmup: int, iters: int) -> float: + import torch + + for _ in range(warmup): + fn() + torch.cuda.synchronize() + samples = [] + for _ in range(iters): + start = torch.cuda.Event(enable_timing=True) + end = torch.cuda.Event(enable_timing=True) + torch.cuda._sleep(10**7) # settle clocks before the timed call, like bench_offload_cache_copy + start.record() + fn() + end.record() + end.synchronize() + samples.append(start.elapsed_time(end)) + return statistics.median(samples) + + +def run_decode(case: DecodeCase, quant_specs, device, args) -> dict: + import torch + + from freetoken.kernel.triton.attention import decode_launch_config, decode_paged_attention + + spec = quant_specs[_QUANT_ALIAS[case.quant]] + slots = case.ctx_len * case.batch + k = _make_kv(slots, KV_HEADS, HEAD_DIM, device, args.seed) + v = _make_kv(slots, KV_HEADS, HEAD_DIM, device, args.seed + 1) + q = _make_kv(case.batch, Q_HEADS, HEAD_DIM, device, args.seed + 2) + k_cache, k_scale, v_cache, v_scale, k_oracle, v_oracle = _quantize_pool(spec, k, v) + + indptr = torch.arange(0, slots + 1, case.ctx_len, device=device, dtype=torch.int32) + indices = torch.arange(slots, device=device, dtype=torch.int32) + q_pos = torch.full((case.batch,), case.ctx_len - 1, device=device, dtype=torch.int32) + sm_scale = HEAD_DIM**-0.5 + + quant_name = "int4" if spec.enabled and spec.elements_per_byte == 2 else ("quant8" if spec.enabled else None) + preferred, block_n, num_warps = decode_launch_config( + quant_name=quant_name, head_dim=HEAD_DIM, num_q_heads=Q_HEADS, num_kv_heads=KV_HEADS + ) + ceiling = case.max_kv_splits if case.max_kv_splits is not None else preferred + + def call(k_c, v_c, k_s, v_s): + # decode_paged_attention picks its OWN preferred split count internally (it + # differs for the quantized call vs. the dequantized-oracle bf16 call, since + # decode_launch_config keys off whether a quant scale was given), so nsplits and + # the logits/lse scratch must be sized from that SAME per-call preference, not a + # value borrowed from the other call -- otherwise stage2 reduces over splits + # stage1 never wrote, reading uninitialized scratch. + qname = quant_name if k_s is not None else None + call_preferred, _, _ = decode_launch_config( + quant_name=qname, head_dim=HEAD_DIM, num_q_heads=Q_HEADS, num_kv_heads=KV_HEADS + ) + splits = max(min(call_preferred, ceiling), 1) + logits = torch.empty(case.batch, Q_HEADS, splits, HEAD_DIM, device=device, dtype=torch.float32) + lse = torch.empty(case.batch, Q_HEADS, splits, device=device, dtype=torch.float32) + nsplits = torch.full((case.batch,), splits, device=device, dtype=torch.int32) + return decode_paged_attention( + q, k_c, v_c, indptr, indices, q_pos, logits, lse, nsplits, splits, sm_scale, + k_scale=k_s, v_scale=v_s, + ) + + scratch_splits = max(min(preferred, ceiling), 1) # the quantized (production) path's actual launch + max_abs_err = None + if spec.enabled and not args.skip_verify: + got = call(k_cache, v_cache, k_scale, v_scale) + want = call(k_oracle, v_oracle, None, None) + max_abs_err = _check_oracle(got, want, f"decode quant={case.quant} ctx={case.ctx_len}") + + time_ms = _time_ms(lambda: call(k_cache, v_cache, k_scale, v_scale), args.warmup, args.iters) + bytes_moved = slots * KV_HEADS * HEAD_DIM * 2 * spec.bytes_per_element(torch.bfloat16) + return { + "op": "decode", "quant": case.quant, "ctx_len": case.ctx_len, "batch": case.batch, + "max_kv_splits_ceiling": ceiling, "launch_splits": scratch_splits, "tuned_block_n": block_n, + "tuned_num_warps": num_warps, "time_ms": time_ms, "gbps": bytes_moved / (time_ms * 1e6), + "max_abs_err": max_abs_err, + } + + +def run_prefill(case: PrefillCase, quant_specs, device, args) -> dict: + import torch + + from freetoken.kernel.triton.attention import paged_attention + + spec = quant_specs[_QUANT_ALIAS[case.quant]] + n = case.chunk_len + k = _make_kv(n, KV_HEADS, HEAD_DIM, device, args.seed) + v = _make_kv(n, KV_HEADS, HEAD_DIM, device, args.seed + 1) + q = _make_kv(n, Q_HEADS, HEAD_DIM, device, args.seed + 2) + k_cache, k_scale, v_cache, v_scale, k_oracle, v_oracle = _quantize_pool(spec, k, v) + + indptr = torch.tensor([0, n], device=device, dtype=torch.int32) + indices = torch.arange(n, device=device, dtype=torch.int32) + q_to_req = torch.zeros(n, device=device, dtype=torch.int32) + q_pos = torch.arange(n, device=device, dtype=torch.int32) + sm_scale = HEAD_DIM**-0.5 + kw = dict(indptr=indptr, indices=indices, q_to_req=q_to_req, q_positions=q_pos, sm_scale=sm_scale) + + def call(k_c, v_c, k_s, v_s): + return paged_attention(q=q, k_cache=k_c, v_cache=v_c, k_scale=k_s, v_scale=v_s, **kw) + + max_abs_err = None + if spec.enabled and not args.skip_verify: + got = call(k_cache, v_cache, k_scale, v_scale) + want = call(k_oracle, v_oracle, None, None) + max_abs_err = _check_oracle(got, want, f"prefill quant={case.quant} chunk={case.chunk_len}") + + time_ms = _time_ms(lambda: call(k_cache, v_cache, k_scale, v_scale), args.warmup, args.iters) + flops = 2 * 2 * n * n * Q_HEADS * HEAD_DIM # QK^T + PV, causal factor ignored (upper bound) + return { + "op": "prefill", "quant": case.quant, "chunk_len": case.chunk_len, + "time_ms": time_ms, "tflops": flops / (time_ms * 1e9), "max_abs_err": max_abs_err, + } + + +def run_extend(case: ExtendCase, quant_specs, device, args) -> dict: + import torch + + from freetoken.kernel.triton.attention import extend_paged_attention + + spec = quant_specs[_QUANT_ALIAS[case.quant]] + prefix, chunk = case.prefix_len, case.chunk_len + total = prefix + chunk + k = _make_kv(total, KV_HEADS, HEAD_DIM, device, args.seed) + v = _make_kv(total, KV_HEADS, HEAD_DIM, device, args.seed + 1) + q = _make_kv(chunk, Q_HEADS, HEAD_DIM, device, args.seed + 2) + # The cached prefix lives in the (quantized) pool; the new chunk is passed straight + # through in bf16 via k_extend/v_extend, matching how a real extend step supplies + # this step's freshly-projected K/V alongside the persistent KV cache. + k_extend, v_extend = k[prefix:], v[prefix:] + k_cache, k_scale, v_cache, v_scale, k_oracle, v_oracle = _quantize_pool(spec, k[:prefix], v[:prefix]) + + qo_indptr = torch.tensor([0, chunk], device=device, dtype=torch.int32) + kv_indptr = torch.tensor([0, prefix], device=device, dtype=torch.int32) + kv_indices = torch.arange(prefix, device=device, dtype=torch.int32) + prefix_lens = torch.tensor([prefix], device=device, dtype=torch.int32) + sm_scale = HEAD_DIM**-0.5 + kw = dict( + qo_indptr=qo_indptr, kv_indptr=kv_indptr, kv_indices=kv_indices, prefix_lens=prefix_lens, + max_q_len=chunk, sm_scale=sm_scale, k_extend=k_extend, v_extend=v_extend, + ) + + def call(k_c, v_c, k_s, v_s): + return extend_paged_attention(q=q, k_cache=k_c, v_cache=v_c, k_scale=k_s, v_scale=v_s, **kw) + + max_abs_err = None + if spec.enabled and not args.skip_verify: + got = call(k_cache, v_cache, k_scale, v_scale) + want = call(k_oracle, v_oracle, None, None) + max_abs_err = _check_oracle(got, want, f"extend quant={case.quant} prefix={prefix} chunk={chunk}") + + time_ms = _time_ms(lambda: call(k_cache, v_cache, k_scale, v_scale), args.warmup, args.iters) + # QK^T + PV against (cached prefix + causal self) KV, upper-bounded by the full window. + flops = 2 * 2 * chunk * total * Q_HEADS * HEAD_DIM + return { + "op": "extend", "quant": case.quant, "prefix_len": prefix, "chunk_len": chunk, + "time_ms": time_ms, "tflops": flops / (time_ms * 1e9), "max_abs_err": max_abs_err, + } + + +_RUNNERS = {"decode": run_decode, "prefill": run_prefill, "extend": run_extend} + + +def _print_row(row: dict) -> None: + op = row["op"] + err = f"{row['max_abs_err']:.4f}" if row["max_abs_err"] is not None else "n/a" + if op == "decode": + print( + f"decode quant={row['quant']:<9} ctx={row['ctx_len']:>7} batch={row['batch']:<3} " + f"splits={row['launch_splits']:>2}/{row['max_kv_splits_ceiling']:<3} " + f"{row['time_ms']:8.4f} ms {row['gbps']:7.1f} GB/s oracle_err={err}", + flush=True, + ) + elif op == "prefill": + print( + f"prefill quant={row['quant']:<9} chunk={row['chunk_len']:>7} " + f"{row['time_ms']:8.3f} ms {row['tflops']:7.2f} TFLOP/s oracle_err={err}", + flush=True, + ) + else: + print( + f"extend quant={row['quant']:<9} prefix={row['prefix_len']:>7} chunk={row['chunk_len']:>6} " + f"{row['time_ms']:8.3f} ms {row['tflops']:7.2f} TFLOP/s oracle_err={err}", + flush=True, + ) + + +def main(argv: list[str] | None = None) -> int: + args = parse_args(argv) + + import torch + + from freetoken.kvcache.quant import FP8_E4M3, INT4, NONE, Q8_0 + + assert torch.cuda.is_available(), "CUDA is required" + torch.cuda.set_device(args.device) + device = torch.device("cuda") + quant_specs = {"int4": INT4, "q8_0": Q8_0, "fp8_e4m3": FP8_E4M3, "auto": NONE} + + print(f"gpu {torch.cuda.get_device_name(device)} geometry q_heads={Q_HEADS} kv_heads={KV_HEADS} " + f"head_dim={HEAD_DIM} (Ornith)", flush=True) + + cases = build_cases(args) + rows = [] + for case in cases: + row = _RUNNERS[case.op](case, quant_specs, device, args) + _print_row(row) + rows.append(row) + if args.json_out: + with open(args.json_out, "a") as f: + f.write(json.dumps(row) + "\n") + + print(f"\n{len(rows)} cases run, {sum(1 for r in rows if r.get('max_abs_err') is not None)} " + f"oracle-verified before timing", flush=True) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/docs/cli.md b/docs/cli.md index cf4b27a2..64742091 100644 --- a/docs/cli.md +++ b/docs/cli.md @@ -68,6 +68,7 @@ See [models.md](models.md#moe-backends) for what each backend does. | `--kv-reserve-tokens` | 8192 | KV token floor reserved before `--moe-cache-auto` fills experts | | `--moe-cpu-threads` | physical cores | CPU worker threads for the cpu/hybrid executor | | `--moe-cpu-layers` | all on GPU | With `offload`: which MoE layers decode on CPU (`3,7,11`, a count, or a fraction) | +| `--moe-pageable-gpu` | off | On WSL pin-quota overflow, stage selected misses through a bounded pinned buffer so all expert math remains on GPU (disables CUDA graphs and prefill overlap) | | `--moe-hybrid-max-fetch` | auto | With `hybrid`: max experts fetched over PCIe per layer per step; rest computed on CPU | | `--moe-prefill-hit-d2d` | off | Prefill: copy cache-hit experts device-side, stream only misses (CUDA >= 13) | | `--disable-moe-prefill-overlap` | overlap on | Disable the two-buffer prefill copy overlap | @@ -153,4 +154,3 @@ expert format + GPU name, so a profile from different hardware is ignored rather than misapplied. Selection flags: `--dtype`, `--model`, `--formats`, `--isa`; decision rule: `--threshold` (default 2.0 — recommend hybrid when CPU bandwidth > 2× PCIe). - diff --git a/docs/models.md b/docs/models.md index e4850a12..5956c884 100644 --- a/docs/models.md +++ b/docs/models.md @@ -1,8 +1,9 @@ # Supported models FreeToken loads HF safetensors checkpoints directly (plus native GGUF for -Gemma-4). The checkpoints below are known-good — the prebuilt kernels are tuned -for them; other checkpoints of the same architectures work too. +Gemma-4, Qwen3.5-MoE/Ornith, and Laguna). The checkpoints below are known-good — +the prebuilt kernels are tuned for them; other checkpoints of the same architectures +work too. | Model | HF checkpoints | |---|---| @@ -10,10 +11,13 @@ for them; other checkpoints of the same architectures work too. | GLM-5.2 | [nvidia/GLM-5.2-NVFP4](https://huggingface.co/nvidia/GLM-5.2-NVFP4) | | GLM-4.7 | [nvidia/GLM-4.7-NVFP4](https://huggingface.co/nvidia/GLM-4.7-NVFP4) | | Qwen3.6 / Qwen3.5 MoE | [Qwen/Qwen3.6-35B-A3B](https://huggingface.co/Qwen/Qwen3.6-35B-A3B) ([-FP8](https://huggingface.co/Qwen/Qwen3.6-35B-A3B-FP8)), [nvidia/Qwen3.6-35B-A3B-NVFP4](https://huggingface.co/nvidia/Qwen3.6-35B-A3B-NVFP4), [Qwen/Qwen3.5-35B-A3B](https://huggingface.co/Qwen/Qwen3.5-35B-A3B) ([-FP8](https://huggingface.co/Qwen/Qwen3.5-35B-A3B-FP8)) | +| Ornith 1.5 35B-A3B | [ornith-ai/Ornith-1.5-35B-A3B-GGUF](https://huggingface.co/ornith-ai/Ornith-1.5-35B-A3B-GGUF) (native Q4_K_M GGUF) | | Qwen3.6 dense | [Qwen/Qwen3.6-27B](https://huggingface.co/Qwen/Qwen3.6-27B) ([-FP8](https://huggingface.co/Qwen/Qwen3.6-27B-FP8)), [nvidia/Qwen3.6-27B-NVFP4](https://huggingface.co/nvidia/Qwen3.6-27B-NVFP4) | | Qwen3-MoE | [Qwen/Qwen3-30B-A3B](https://huggingface.co/Qwen/Qwen3-30B-A3B) | | gpt-oss | [openai/gpt-oss-120b](https://huggingface.co/openai/gpt-oss-120b), [openai/gpt-oss-20b](https://huggingface.co/openai/gpt-oss-20b) | | Gemma-4 | [google/gemma-4-26B-A4B-it](https://huggingface.co/google/gemma-4-26B-A4B-it), [nvidia/Gemma-4-26B-A4B-NVFP4](https://huggingface.co/nvidia/Gemma-4-26B-A4B-NVFP4), [google/gemma-4-12B-it](https://huggingface.co/google/gemma-4-12B-it), [nvidia/Gemma-4-31B-IT-NVFP4](https://huggingface.co/nvidia/Gemma-4-31B-IT-NVFP4) .. | +| Poolside Laguna-S 2.1 | compressed-tensors INT4 safetensors (including its BF16 expert tail), native GGUF | +| NVIDIA Nemotron 3 Super | [nvidia/NVIDIA-Nemotron-3-Super-120B-A12B-NVFP4](https://huggingface.co/nvidia/NVIDIA-Nemotron-3-Super-120B-A12B-NVFP4) | | MiniMax-M2.5 | [nvidia/MiniMax-M2.5-NVFP4](https://huggingface.co/nvidia/MiniMax-M2.5-NVFP4) | | Muse-Glimmer | [meta-models/Muse-Glimmer-30B](https://huggingface.co/meta-models/Muse-Glimmer-30B), [RedHatAI/Muse-Glimmer-30B-NVFP4](https://huggingface.co/RedHatAI/Muse-Glimmer-30B-NVFP4) | @@ -38,3 +42,49 @@ for them; other checkpoints of the same architectures work too. - DeepSeek-V4 checkpoints must keep the `inference/config.json` subdir — the authoritative model args are read from there. - Multimodal checkpoints are served text-only. +- Laguna-S INT4 needs the `offload` backend. On WSL, FreeToken automatically + keeps enough layers on CPU when the mixed INT4/BF16 banks exceed the CUDA + pinned-memory budget. +- A single-session 200K Laguna configuration on a 16 GB GPU should reserve the + minimum 256 expert slots, use INT4 KV, and keep the SWA pool near its working-set + floor: `--max-running-requests 1 --max-seq-len-override 200000 --num-tokens 200000 + --kv-cache-dtype int4 --moe-cache-size 256 --disable-moe-prefill-overlap + --swa-full-tokens-ratio 0.006 --memory-ratio 0.95`. +- For Ornith Q4_K_M at 200K on a 16 GB GPU, use one request, Q4_0 KV, 5,000 + expert slots, and the default 8K prefill chunks: `--max-running-requests 1 + --max-seq-len-override 200000 --num-tokens 200000 --kv-cache-dtype q4_0 + --moe-backend offload --moe-cache-size 5000 --max-prefill-length 8192 + --memory-ratio 0.95`. On the RTX 2000 Ada/WSL test host, cold 32K TTFT was + 51.1 s at 8K chunks versus 54.6 s at 16K; `--moe-prefill-hit-d2d` was slower + on this stack and should remain disabled. Install the optional SGLang kernel + (`freetoken[sgl]`) for faster expert-route alignment. FreeToken's Q4_0 path matches + llama.cpp's block quantizer and is validated with normal answers, OpenAI tool calls, + and a 55.6K-token Claude Code Bash-tool round trip. The sm_89 attention tuning reduces + a synthetic 200K full-attention layer from 2.42 ms to 0.92 ms; the live 55.9K decode + ran at 36--48 tok/s after warmup. A coherent 169.9K-token live generation completed + in 425.8 s of prefill and decoded at 27--35 tok/s. `int4` remains an alias for `q4_0`. +- On Blackwell (sm_120, e.g. RTX 5080 16 GB) the same command serves the **full + 262,144-token window**: `--attention-backend triton --max-seq-len-override 262144 + --num-tokens 262144 --kv-cache-dtype q4_0 --max-running-requests 1 + --moe-backend offload --moe-cache-auto --max-prefill-length 8192`. Pass the + backend explicitly: sm_120 auto-resolves to FlashInfer, which cannot read the + quantized KV pool. The attention launch tables are + architecture-aware: the sm_120 Q4_0 decode launch (64 splits, 64-token tiles) + runs a synthetic 262K full-attention layer in 0.36 ms versus 0.82 ms with the + sm_89 tuning, and the extend/prefill kernels drop to 4 warps (1.12x on long-Q4 + prefix extension, 2x on cold chunks). BLOCK_N=16 silently corrupts the packed + Q4 loader on sm_120 exactly as on sm_89 and stays excluded. On sm_120 the + Q4_K/Q6_K GGUF matmuls (dense prefill and large routed-expert batches) run on + llama.cpp's int8-tensor-core MMQ (vendored under `kernel/csrc/gguf_mmq/`, + JIT-built on first use): ~13x over the DP4A kernels and ~1.3x over transient + dequant+cuBLAS at 8K-token chunks, with the same lossless packed weights. +- Nemotron 3 Super uses its native hybrid Mamba-2 / full-attention / latent-MoE + architecture. The NVFP4 release needs about 60 GiB of host RAM for expert banks and + 10.3 GiB of resident GPU weights. FreeToken currently serves one concurrent Nemotron + session. On WSL, `--moe-pageable-gpu` keeps the pin-budget overflow banks pageable, + stages only their routed misses through a small pinned buffer, and still executes every + ReLU² expert on GPU. This eager path disables CUDA graphs and prefill overlap. A minimal + all-GPU-compute launch is: + `ft serve --model nvidia/NVIDIA-Nemotron-3-Super-120B-A12B-NVFP4 + --max-running-requests 1 --moe-backend offload --moe-cpu-layers 0 + --moe-pageable-gpu --moe-cache-auto`. diff --git a/python/freetoken/attention/triton.py b/python/freetoken/attention/triton.py index 9eed1e1d..cf3d335e 100644 --- a/python/freetoken/attention/triton.py +++ b/python/freetoken/attention/triton.py @@ -87,7 +87,6 @@ def __init__(self, config: ModelConfig): self.capture: TritonCaptureData | None = None self.capture_bs: List[int] = [] self.max_graph_bs = 0 - self.max_kv_splits = 8 self.prefill_tile_min_q = 128 self.num_q_heads = int(getattr(config, "num_qo_heads", 1)) kv_groups = getattr(config, "kv_cache_group_specs", lambda: ())() @@ -95,6 +94,22 @@ def __init__(self, config: ModelConfig): (group.head_dim for group in kv_groups), default=int(getattr(config, "head_dim", 1)), ) + from freetoken.kernel.triton.attention import decode_launch_config + + quant = getattr(self.kvcache, "quant", None) + quant_name = getattr(quant, "name", None) if getattr(quant, "enabled", False) else None + capability = ( + torch.cuda.get_device_capability(self.device) + if self.device.type == "cuda" + else None + ) + self.max_kv_splits = decode_launch_config( + quant_name=quant_name, + head_dim=self.max_head_dim, + num_q_heads=self.num_q_heads, + num_kv_heads=int(getattr(config, "num_kv_heads", 1)), + compute_capability=capability, + )[0] def _ensure_decode_scratch( self, @@ -144,6 +159,7 @@ def forward( extend_paged_attention, paged_attention, ) + from freetoken.kvcache.quant import BLOCK as QBLOCK metadata = batch.attn_metadata assert isinstance(metadata, TritonMetadata) @@ -151,10 +167,24 @@ def forward( k_raw = self.kvcache.k_cache(layer_id) v_raw = self.kvcache.v_cache(layer_id) - kv_heads, head_dim = k_raw.shape[-2], k_raw.shape[-1] - assert head_dim == q.shape[-1] - k_cache = k_raw.view(-1, kv_heads, head_dim) - v_cache = v_raw.view(-1, kv_heads, head_dim) + kv_heads = k_raw.shape[-2] + quant = getattr(self.kvcache, "quant", None) + epb = 1 if quant is None or not quant.enabled else quant.elements_per_byte + # The slab's last dim is the STORAGE head dim -- it halves for packed int4. The + # kernels get logical element space (head_dim, offs_d, scales) plus the EPB + # constexpr, and only the loads divide by it. + head_dim = q.shape[-1] + assert k_raw.shape[-1] == head_dim // epb and v_raw.shape[-1] == head_dim // epb, ( + f"packed KV slab {k_raw.shape[-1]} != logical head_dim {head_dim} / epb {epb}" + ) + k_cache = k_raw.view(-1, kv_heads, head_dim // epb) + v_cache = v_raw.view(-1, kv_heads, head_dim // epb) + # A quantized pool hands its per-block scales alongside the slabs; an unquantized + # one has none, and the kernels compile their bf16 path unchanged. + k_scale = v_scale = None + if quant is not None and quant.enabled: + k_scale = self.kvcache.k_scale(layer_id).view(-1, kv_heads, head_dim // QBLOCK) + v_scale = self.kvcache.v_scale(layer_id).view(-1, kv_heads, head_dim // QBLOCK) spec = attn_spec or AttentionSpec() indices = metadata.indices @@ -181,6 +211,8 @@ def forward( sm_scale=scale, sliding_window=spec.sliding_window, sinks=spec.sinks, + k_scale=k_scale, + v_scale=v_scale, ) if ( (not metadata.is_decode) @@ -201,6 +233,8 @@ def forward( sinks=spec.sinks, k_extend=k.view(q.shape[0], kv_heads, head_dim), v_extend=v.view(q.shape[0], kv_heads, head_dim), + k_scale=k_scale, + v_scale=v_scale, ) return paged_attention( q=q, @@ -213,6 +247,8 @@ def forward( sm_scale=scale, sliding_window=spec.sliding_window, sinks=spec.sinks, + k_scale=k_scale, + v_scale=v_scale, ) def prepare_metadata(self, batch: Batch) -> None: diff --git a/python/freetoken/engine/config.py b/python/freetoken/engine/config.py index 543012f3..3142641f 100644 --- a/python/freetoken/engine/config.py +++ b/python/freetoken/engine/config.py @@ -48,6 +48,11 @@ class EngineConfig: # fraction ("0.5"). None/"" = all layers on GPU (plain offload). --moe-backend cpu # already means all layers on CPU and ignores this. moe_cpu_layers: str | None = None + # WSL fallback for expert banks that exceed the CUDA host-registration quota. + # Overflow layers stay pageable in RAM; decode gathers each step's misses through + # a small pinned staging buffer and still executes every expert on the GPU. + # This path is eager-only because the CPU gather cannot be CUDA-graph captured. + moe_pageable_gpu: bool = False # Hybrid MoE backend (--moe-backend hybrid): max experts fetched over PCIe per # (layer, decode step); the rest of that step's misses are computed on the CPU. # -1 (default) = auto: fetch the benched pcie_bw/cpu_bw fraction of each step's @@ -80,6 +85,16 @@ class EngineConfig: # KV capacity in tokens; resolved into num_page_override by _adjust_config once page_size # is final. Mutually exclusive with num_page_override. num_token_override: int | None = None + # KV element storage (--kv-cache-dtype): "auto" keeps the compute dtype; q8_0 and + # fp8_e4m3 store 8 bits, while int4/q4_0 use GGML Q4_0 with two values per byte. + # Every quantized scheme carries a per-block scale. Resolved by the pools and cost model. + kv_cache_dtype: str = "auto" + + @cached_property + def kv_quant(self): + from freetoken.kvcache.quant import resolve_kv_quant + + return resolve_kv_quant(self.kv_cache_dtype) @cached_property def hf_config(self): diff --git a/python/freetoken/engine/engine.py b/python/freetoken/engine/engine.py index 22c4a6c7..9e2992e6 100644 --- a/python/freetoken/engine/engine.py +++ b/python/freetoken/engine/engine.py @@ -3,6 +3,7 @@ import gc import math import os +import platform from datetime import timedelta from typing import Any, Dict, Iterable, NamedTuple, Tuple @@ -154,6 +155,44 @@ def _resolve_auto_attention_backend( ) +def _validate_kv_cache_dtype(config, model_config) -> None: + """Gate --kv-cache-dtype against what the quantized path actually implements. + + Compact KV storage (8-bit and packed int4) lives in the Triton attention kernels and + MHA/hybrid-SWA pools. Every other backend reads the KV slabs through its own kernels + (flashinfer's ``kv_data_type``, trtllm's fp8 path) which this has not been wired into, + and the MLA/DSA/DSV4/BSA pools have their own slab layouts. Reject those combinations + here at config time, rather than letting a wrong-dtype tensor reach a kernel. + """ + quant = getattr(config, "kv_quant", None) + if quant is None or not quant.enabled: + return + + from freetoken.kvcache.quant import BLOCK + + backends = [p.strip() for p in config.attention_backend.split(",")] + if any(b != "triton" for b in backends): + raise ValueError( + f"--kv-cache-dtype {quant.name} needs the triton attention backend, but the " + f"resolved backend is {config.attention_backend!r}. Pass " + "--attention-backend triton, or drop --kv-cache-dtype." + ) + + specs = [s for s in model_config.kv_cache_group_specs() if s.num_layers > 0] + if any(s.mla or s.index_head_dim > 0 for s in specs): + raise ValueError( + f"--kv-cache-dtype {quant.name} does not support MLA/DSA latent KV pools " + "(their slabs alias K and V and carry an index tier); use --kv-cache-dtype auto." + ) + bad = [s for s in specs if s.head_dim % BLOCK] + if bad: + names = ", ".join(f"{s.name} (head_dim {s.head_dim})" for s in bad) + raise ValueError( + f"--kv-cache-dtype {quant.name} needs every head_dim to be a multiple of " + f"{BLOCK}, the quantization block; this model has {names}." + ) + + def _validate_attention_backend_choice(config, override, required: frozenset[AttnType]) -> None: """Config-time type x backend capability check for the resolved (or explicit) backend string: every comma part must serve every required type and have its @@ -478,6 +517,16 @@ def _resolve_auto_moe_cache_size(self, config: EngineConfig, banks) -> tuple[int fixed_cache_size += state_pool_bytes(config) # sibling GDN state pool, engine-summed num_experts = config.model_config.num_experts total_experts = config.model_config.num_moe_layers * num_experts + # ``--num-tokens`` is resolved to num_page_override before model load. An + # explicit KV geometry is a hard floor for the joint auto plan, not merely + # something allocated after experts greedily consume a budget that reserved + # only the default 8K tokens. The latter overcommitted Ornith 200K by about + # 1.1 GiB and starved prefill kernels of all transient workspace. + kv_reserve_tokens = max(config.kv_reserve_tokens, min_reserve) + if config.num_page_override is not None: + kv_reserve_tokens = max( + kv_reserve_tokens, config.num_page_override * page_tokens + ) return resolve_moe_cache_auto( baseline_free=self._baseline_free, weights_bytes=self._weights_bytes, @@ -488,7 +537,7 @@ def _resolve_auto_moe_cache_size(self, config: EngineConfig, banks) -> tuple[int num_experts=num_experts, total_experts=total_experts, prefill_overlap=config.moe_prefill_overlap, - kv_reserve_tokens=max(config.kv_reserve_tokens, min_reserve), + kv_reserve_tokens=kv_reserve_tokens, page_size=page_tokens, quant_format=banks.quant_format, ) @@ -512,8 +561,21 @@ def _init_offload_moe_cache(self, config: EngineConfig) -> OffloadMoeCache: # layout; the GPU slot-cache GEMM reads those same native rows. decode_target also # gates the CPU executor build below. cpu_layer_ids = _resolve_cpu_layers(config, config.model_config.num_moe_layers) + pageable_gpu_layer_ids: frozenset[int] = frozenset() + if config.moe_pageable_gpu: + if config.moe_backend != "offload": + raise ValueError("--moe-pageable-gpu requires --moe-backend offload") + if cpu_layer_ids: + raise ValueError( + "--moe-pageable-gpu cannot be combined with non-zero " + "--moe-cpu-layers" + ) + pageable_gpu_layer_ids = _auto_pageable_gpu_layers( + config, config.model_config.num_moe_layers + ) if ( not cpu_layer_ids + and not config.moe_pageable_gpu and config.moe_cpu_layers is None and config.moe_backend in ("offload", "hybrid") and _pin_budget_bytes() is not None @@ -528,8 +590,9 @@ def _init_offload_moe_cache(self, config: EngineConfig) -> OffloadMoeCache: # split residency: where pinning is quota-capped (_pin_budget_bytes), pin only the GPU layers' banks and mlock the CPU layers' # uncapped hosts keep every bank pinned (CPU decode reads them the same; overlap prefill stays on) # not applied to plain --moe-backend cpu; all-locked under a cap = --moe-backend offload --moe-cpu-layers 1.0 + unpinned_layer_ids = cpu_layer_ids | pageable_gpu_layer_ids split_residency = ( - bool(cpu_layer_ids) + bool(unpinned_layer_ids) and config.moe_backend in ("offload", "hybrid") and _pin_budget_bytes() is not None ) @@ -550,8 +613,8 @@ def _init_offload_moe_cache(self, config: EngineConfig) -> OffloadMoeCache: if split_residency and config.moe_prefill_overlap: # locked (unregistered) layers cannot feed the async pinned H2D double buffer; their prefill is a synchronous pageable copy via materialize logger.info_rank0( - "--moe-cpu-layers split residency: disabling MoE prefill overlap " - "(locked layers prefill via synchronous pageable copies)" + "MoE split residency: disabling prefill overlap " + "(unpinned layers prefill via synchronous pageable copies)" ) object.__setattr__(config, "moe_prefill_overlap", False) if cache_factory is None: @@ -565,11 +628,14 @@ def _init_offload_moe_cache(self, config: EngineConfig) -> OffloadMoeCache: if split_residency: from freetoken.moe.host_banks import HostResidency - requested_residency = [ - HostResidency.LOCKED.value if i in cpu_layer_ids - else HostResidency.PINNED.value - for i in range(config.model_config.num_moe_layers) - ] + requested_residency = [] + for i in range(config.model_config.num_moe_layers): + if i in pageable_gpu_layer_ids: + requested_residency.append(HostResidency.PAGEABLE.value) + elif i in cpu_layer_ids: + requested_residency.append(HostResidency.LOCKED.value) + else: + requested_residency.append(HostResidency.PINNED.value) banks = load_expert_banks( config.model_path, config.model_config, @@ -612,6 +678,12 @@ def _init_offload_moe_cache(self, config: EngineConfig) -> OffloadMoeCache: decode_target=decode_target, hybrid_max_fetch=config.moe_hybrid_max_fetch, ) + # Per-layer mixed GGUF geometry is also consumed by Laguna's split + # Q4_0/BF16 CPU executor when WSL cannot pin every expert layer. + cache.gguf_expert_types = config.model_config.gguf_expert_types + cache.expert_hidden_size = config.model_config.hidden_size + cache.expert_intermediate_size = config.model_config.moe_intermediate_size + cache.pageable_gpu = config.moe_pageable_gpu # before set_bank_sources: the residency validation and the copy plan's skip of non-pinned layers key on the CPU-layer set cache.cpu_layer_ids = cpu_layer_ids cache.set_bank_sources(banks.sources, layer_residency=banks.layer_residency) @@ -673,7 +745,10 @@ def _init_cpu_moe_executor(self, config: EngineConfig, cache, layers) -> None: must be stable for the captured nodes. Buffers/tasks themselves are allocated lazily on the first (eager) forward at each batch size. """ - from freetoken.moe.cpu_executor import CpuMoeExecutor + from freetoken.moe.cpu_executor import ( + CpuMoeExecutor, + MixedGgufCpuMoeExecutor, + ) sample = layers[0] required = ("top_k", "activation", "apply_router_weight_on_input") @@ -686,7 +761,12 @@ def _init_cpu_moe_executor(self, config: EngineConfig, cache, layers) -> None: # round a batch up to the largest captured size; cover both. max_tokens = max(config.max_running_req, config.cuda_graph_max_bs or 0, 1) # gpt-oss mxfp4 carries clamped-swiglu scalars; other formats use the defaults. - executor = CpuMoeExecutor( + executor_cls = ( + MixedGgufCpuMoeExecutor + if cache.quant_format == "gguf" + else CpuMoeExecutor + ) + executor = executor_cls( cache, top_k=sample.top_k, activation=sample.activation, @@ -1011,6 +1091,13 @@ def _ensure_expandable_segments() -> None: """ if os.environ.get("PYTORCH_ALLOC_CONF") or os.environ.get("PYTORCH_CUDA_ALLOC_CONF"): return + # PyTorch 2.11 + CUDA 13 currently accepts this allocator setting under WSL but the + # first CUDA allocation then fails with ``CUDA driver error: unknown error``. Keep + # WSL on the native caching allocator until the driver/runtime combination supports + # expandable segments reliably. + if os.environ.get("WSL_DISTRO_NAME") or "microsoft" in platform.release().lower(): + logger.info_rank0("WSL detected; using the native CUDA caching allocator") + return try: torch.cuda.memory._set_allocator_settings("expandable_segments:True") except Exception as exc: # pragma: no cover - depends on torch build @@ -1121,7 +1208,7 @@ def _resolve_cpu_layers(config: EngineConfig, num_moe_layers: int) -> frozenset[ # expert activations the CPU MoE executor supports (csrc ActKind) _CPU_MOE_ACTS = ( - "silu", "swish", "gelu", "gelu_tanh", "gelu_pytorch_tanh", "swigluoai", + "silu", "swish", "gelu", "gelu_tanh", "gelu_pytorch_tanh", "swigluoai", "relu2", ) @@ -1143,7 +1230,7 @@ def _cpu_moe_executor_viable(model_config) -> bool: return False expert_quant = getattr(model_config, "expert_quant", "none") fmt = expert_quant if expert_quant != "none" else (moe_wfmt or "bf16") - return fmt == "mxfp4" or fmt in _WFMT_IDS + return fmt in ("mxfp4", "laguna_int4") or fmt in _WFMT_IDS def _pin_budget_bytes() -> int | None: @@ -1187,6 +1274,40 @@ def _auto_cpu_layers(config: EngineConfig, num_moe_layers: int) -> frozenset[int return ids +def _auto_pageable_gpu_layers( + config: EngineConfig, num_moe_layers: int +) -> frozenset[int]: + """Select the layers that cannot fit under the host pin quota. + + Unlike :func:`_auto_cpu_layers`, these layers still decode on the GPU. Their + selected cache misses take a pageable -> pinned staging -> VRAM route, so only + the layers that overflow the quota pay the extra RAM copy. + """ + from freetoken.moe.expert_banks import bank_bytes_estimate, ftw_bank_bytes + + bank_bytes = ftw_bank_bytes(config.model_path) or bank_bytes_estimate(config.model_config) + if not bank_bytes: + return frozenset() + budget = _pin_budget_bytes() + if budget is None or bank_bytes <= budget: + logger.info_rank0( + "--moe-pageable-gpu: expert banks fit the CUDA pin budget; keeping every " + "layer on the direct pinned RAM -> GPU path" + ) + return frozenset() + n = min(num_moe_layers, math.ceil(num_moe_layers * (1 - budget / bank_bytes))) + head = (n + 1) // 2 + ids = frozenset(range(head)) | frozenset( + range(num_moe_layers - (n - head), num_moe_layers) + ) + logger.info_rank0( + f"--moe-pageable-gpu: banks {bank_bytes / 2**30:.2f} GiB > pin budget " + f"{budget / 2**30:.2f} GiB; {n} head+tail MoE layers use pageable staging " + f"and GPU compute ({sorted(ids)})" + ) + return ids + + # MoE-only knobs and the value each resolves to on a dense model. moe_backend is handled # separately (its dense value is 'fused', but 'auto' resolves there without a warning). _DENSE_MOE_SETTINGS = { @@ -1194,6 +1315,7 @@ def _auto_cpu_layers(config: EngineConfig, num_moe_layers: int) -> frozenset[int "moe_cache_rate": None, "moe_cache_auto": False, "moe_cpu_layers": None, + "moe_pageable_gpu": False, "moe_cpu_threads": 0, "moe_hybrid_max_fetch": -1, "moe_prefill_overlap": True, @@ -1237,6 +1359,16 @@ def override(attr: str, value: Any): # this is dangerous, use with caution f"experts); ignoring MoE settings: {', '.join(dropped)}" ) + if config.moe_pageable_gpu: + # Decode staging reads the device-produced LRU copy plan back to the host, + # gathers pageable rows into a small pinned buffer, then resumes GPU work. + # Those host operations cannot be replayed from a CUDA graph. + override("cuda_graph_bs", []) + override("cuda_graph_max_bs", 0) + logger.info_rank0( + "--moe-pageable-gpu: disabling CUDA graphs (pageable miss staging is eager)" + ) + if single_stream_only: # The model runs one sequence at a time: it collapses the batch to one row and the # decode CUDA graph is captured at bs=1. Force the runtime knobs so the KV pool, page @@ -1306,6 +1438,7 @@ def override(attr: str, value: Any): # this is dangerous, use with caution ) logger.info_rank0(f"Auto-selected attention backend: {config.attention_backend}") _validate_attention_backend_choice(config, override, required_attn_types) + _validate_kv_cache_dtype(config, model_config) if config.moe_cache_rate is not None: total_experts = config.model_config.num_moe_layers * config.model_config.num_experts @@ -1427,6 +1560,18 @@ def override(attr: str, value: Any): # this is dangerous, use with caution override("moe_cache_rate", None) override("moe_cache_auto", False) + if ( + is_moe + and config.moe_backend == "cpu" + and expert_quant == "laguna_int4" + ): + raise ValueError( + "Laguna INT4/BF16 needs --moe-backend offload (or auto); the all-CPU " + "backend reserves a two-layer BF16-sized prefill cache that does not fit " + "on typical 16 GiB GPUs. Offload automatically assigns enough layers to " + "CPU decode under WSL's pin budget." + ) + if is_moe and config.moe_backend == "cpu": # CPU-compute decode keeps experts in host RAM and computes them on the CPU; # the GPU only holds the two-layer prefill double buffer. So the slot cache is @@ -1459,6 +1604,12 @@ def override(attr: str, value: Any): # this is dangerous, use with caution f"{config.moe_backend!r}); use --moe-backend cpu to run all layers on CPU" ) + if is_moe and config.moe_pageable_gpu and config.moe_backend != "offload": + raise ValueError( + "--moe-pageable-gpu requires --moe-backend offload; it is an " + "all-GPU-compute alternative to CPU/hybrid decode" + ) + if is_moe: object.__setattr__(model_config, "moe_backend", config.moe_backend) object.__setattr__(model_config, "nvfp4_backend", config.nvfp4_backend) diff --git a/python/freetoken/kernel/aot.py b/python/freetoken/kernel/aot.py index 5e87f8c9..6c363ba5 100644 --- a/python/freetoken/kernel/aot.py +++ b/python/freetoken/kernel/aot.py @@ -124,6 +124,27 @@ def build(build_directory: pathlib.Path) -> object: return KernelSpec(name=_make_name("fast_index_copy_multi", *args), build=build) +def _fast_index_copy_multi_strided_spec( + num_threads: int, blocks_per_bank: int +) -> KernelSpec: + args = make_cpp_args(num_threads, blocks_per_bank) + + def build(build_directory: pathlib.Path) -> object: + return load_jit( + "fast_index_copy_multi_strided", + *args, + cuda_files=["fast_index_copy.cuh"], + cuda_wrappers=[ + ("launch", f"&MultiIndexCopyStridedKernel<{args}>::run") + ], + build_directory=str(build_directory), + ) + + return KernelSpec( + name=_make_name("fast_index_copy_multi_strided", *args), build=build + ) + + def _batch_memcpy_spec() -> KernelSpec: def build(build_directory: pathlib.Path) -> object: return load_jit( @@ -153,6 +174,7 @@ def default_kernel_specs() -> tuple[KernelSpec, ...]: for feature_size in DEFAULT_FAST_INDEX_COPY_FEATURE_SIZES ) specs.append(_fast_index_copy_multi_spec(num_threads=1024, blocks_per_bank=8)) + specs.append(_fast_index_copy_multi_strided_spec(num_threads=1024, blocks_per_bank=8)) # prefill hit-D2D gather (HBM-bound: wide grid) + its miss-side batch H2D binding. specs.append(_fast_index_copy_multi_spec(num_threads=1024, blocks_per_bank=64)) specs.append(_batch_memcpy_spec()) diff --git a/python/freetoken/kernel/causal_conv1d.py b/python/freetoken/kernel/causal_conv1d.py index 81f1fab7..80c4a1fc 100644 --- a/python/freetoken/kernel/causal_conv1d.py +++ b/python/freetoken/kernel/causal_conv1d.py @@ -22,6 +22,7 @@ def causal_conv1d_varlen( cu_seqlens: torch.Tensor, # [batch+1] int32 prefix sums of per-request lengths cache_indices: torch.Tensor, # [batch] int32 slot id per request has_initial_state: torch.Tensor, # [batch] bool (carry conv state across chunks) + bias: torch.Tensor | None = None, ) -> torch.Tensor: """Varlen (prefill) depthwise causal conv with silu; writes silu(conv) into ``x`` in place and refreshes ``conv_states[cache_indices]`` with each request's tail.""" @@ -33,7 +34,7 @@ def causal_conv1d_varlen( ) return triton_causal_conv1d_varlen( - x, weight, conv_states, cu_seqlens, cache_indices, has_initial_state + x, weight, conv_states, cu_seqlens, cache_indices, has_initial_state, bias=bias ) from sgl_kernel import causal_conv1d_fwd @@ -41,7 +42,7 @@ def causal_conv1d_varlen( if x.stride(-1) != 1: x = x.contiguous() causal_conv1d_fwd( - x, weight, None, conv_states, + x, weight, bias, conv_states, cu_seqlens.to(torch.int32), cache_indices.to(torch.int32), has_initial_state, True, _PAD_SLOT_ID, ) @@ -53,6 +54,7 @@ def causal_conv1d_decode( conv_state: torch.Tensor, # [num_slots, conv_dim, state_len>=kernel-1] (in place) weight: torch.Tensor, # [conv_dim, kernel] conv_state_indices: torch.Tensor, # [batch] int32 slot id per request + bias: torch.Tensor | None = None, ) -> torch.Tensor: """Single-token (decode) causal conv update with silu; shifts+appends the new token into ``conv_state[conv_state_indices]`` in place and returns silu(conv).""" @@ -63,13 +65,15 @@ def causal_conv1d_decode( causal_conv1d_decode as triton_causal_conv1d_decode, ) - return triton_causal_conv1d_decode(x, conv_state, weight, conv_state_indices) + return triton_causal_conv1d_decode( + x, conv_state, weight, conv_state_indices, bias=bias + ) from sgl_kernel import causal_conv1d_update x = x.unsqueeze(-1) causal_conv1d_update( - x, conv_state, weight, None, True, None, + x, conv_state, weight, bias, True, None, conv_state_indices.to(torch.int32), _PAD_SLOT_ID, ) return x.squeeze(-1) diff --git a/python/freetoken/kernel/csrc/cpu_moe/cpu_moe_ext.cpp b/python/freetoken/kernel/csrc/cpu_moe/cpu_moe_ext.cpp index 880e8637..3b1caa6d 100644 --- a/python/freetoken/kernel/csrc/cpu_moe/cpu_moe_ext.cpp +++ b/python/freetoken/kernel/csrc/cpu_moe/cpu_moe_ext.cpp @@ -71,10 +71,20 @@ inline bf16_t f32_to_bf16(float f) { // MiniMax-M3): gate/up are combined jointly with the runtime alpha/limit // scalars, so it is handled in the do_pass1 epilogue (act_apply never sees it; // the mxfp4 kernel additionally fuses its own copy of the same math). -enum ActKind { ACT_SILU = 0, ACT_GELU = 1, ACT_GELU_TANH = 2, ACT_SWIGLUOAI = 3 }; +enum ActKind { + ACT_SILU = 0, + ACT_GELU = 1, + ACT_GELU_TANH = 2, + ACT_SWIGLUOAI = 3, + ACT_RELU2 = 4, +}; inline float act_apply(int act, float x) { if (act == ACT_SILU) return x / (1.0f + std::exp(-x)); + if (act == ACT_RELU2) { + const float y = std::max(0.0f, x); + return y * y; + } if (act == ACT_GELU) return 0.5f * x * (1.0f + std::erf(x * 0.70710678118654752440f)); // gelu_tanh @@ -1477,16 +1487,17 @@ struct CpuMoeExecutor { const uint8_t* gu_scale_l, const uint16_t* gu_global_l, int e, int row, const bf16_t* x, const float* xe, const float* xo, const int8_t* xi8, const float* xas) { + const int up_rows = act == ACT_RELU2 ? I : 2 * I; if (fmt == WF_BF16) { - const bf16_t* w = gate_up_l + ((size_t)e * (2 * I) + row) * H; + const bf16_t* w = gate_up_l + ((size_t)e * up_rows + row) * H; return dot(w, x, H); } if (fmt == WF_Q4_0) { const uint8_t* w = - gu_packed_l + ((size_t)e * (2 * I) + row) * (size_t)q4_gu_row_bytes; + gu_packed_l + ((size_t)e * up_rows + row) * (size_t)q4_gu_row_bytes; return q4dot(w, xi8, xas, H); // W4A8: int8 activations (Q8_0), scale in xas } - const size_t r = (size_t)e * (2 * I) + row; + const size_t r = (size_t)e * up_rows + row; if (use_vnni) return nvi8dot(gu_packed_l + r * (size_t)(H / 2), gu_scale_l + r * (size_t)(H / 16), fp16_to_f32(gu_global_l[r]), xi8, H, e4m3_lut, xas); @@ -1605,23 +1616,28 @@ struct CpuMoeExecutor { const int i0 = static_cast(ib) * IBLK; const int i1 = std::min(I, i0 + IBLK); const bool swigluoai = act == ACT_SWIGLUOAI; + const bool relu2 = act == ACT_RELU2; const float lim = swiglu_limit, alpha = swiglu_alpha; for (int i = i0; i < i1; ++i) { // gate = row i, up = row I+i float gate = gemm1_dot(gate_up_l, gu_packed_l, gu_scale_l, gu_global_l, e, i, x_row, xe, xo, xi8, xas) * w_in; - float up = gemm1_dot(gate_up_l, gu_packed_l, gu_scale_l, gu_global_l, e, I + i, x_row, - xe, xo, xi8, xas) * w_in; - if (swigluoai) { - // clamp(gate, max=lim) * sigmoid(alpha * gate) * (clamp(up, +-lim) + 1) - // -- same math as the mxfp4 kernel's fused epilogue (lim == +inf: no clamp). - if (gate > lim) gate = lim; - if (up > lim) up = lim; - else if (up < -lim) up = -lim; - const float glu = gate / (1.0f + std::exp(-gate * alpha)); - g_row[i] = f32_to_bf16(glu * (up + 1.0f)); + if (relu2) { + g_row[i] = f32_to_bf16(act_apply(act, gate)); } else { - g_row[i] = f32_to_bf16(act_apply(act, gate) * up); + float up = gemm1_dot(gate_up_l, gu_packed_l, gu_scale_l, gu_global_l, e, I + i, + x_row, xe, xo, xi8, xas) * w_in; + if (swigluoai) { + // clamp(gate, max=lim) * sigmoid(alpha * gate) * (clamp(up, +-lim) + 1) + // -- same math as the mxfp4 kernel's fused epilogue (lim == +inf: no clamp). + if (gate > lim) gate = lim; + if (up > lim) up = lim; + else if (up < -lim) up = -lim; + const float glu = gate / (1.0f + std::exp(-gate * alpha)); + g_row[i] = f32_to_bf16(glu * (up + 1.0f)); + } else { + g_row[i] = f32_to_bf16(act_apply(act, gate) * up); + } } } } @@ -2143,8 +2159,8 @@ PYBIND11_MODULE(TORCH_EXTENSION_NAME, m) { // ABI capability marker: the highest ActKind this build implements in the // GENERIC epilogue. CpuMoeExecutor.__init__ probes it before requesting an act // id the epilogue must handle -- a prebuilt .so from before ACT_SWIGLUOAI - // accepts id 3 without error and silently computes the wrong activation + // accepts a newer id without error and silently computes the wrong activation // (act_apply falls through to gelu_tanh); the probe turns a stale extension // into a loud rebuild instruction instead of wrong model outputs. - m.def("max_generic_act_id", []() { return static_cast(ACT_SWIGLUOAI); }); + m.def("max_generic_act_id", []() { return static_cast(ACT_RELU2); }); } diff --git a/python/freetoken/kernel/csrc/gguf/gguf_kernel.cu b/python/freetoken/kernel/csrc/gguf/gguf_kernel.cu index d88960d5..ecb1f325 100644 --- a/python/freetoken/kernel/csrc/gguf/gguf_kernel.cu +++ b/python/freetoken/kernel/csrc/gguf/gguf_kernel.cu @@ -545,7 +545,8 @@ torch::Tensor ggml_moe_a8_vec( int64_t top_k, int64_t type, int64_t row, - int64_t tokens) { + int64_t tokens, + int64_t expert_stride_bytes) { int col = X.sizes()[1]; const int padded = (col + 512 - 1) / 512 * 512; const at::cuda::OptionalCUDAGuard device_guard(device_of(X)); @@ -568,6 +569,7 @@ torch::Tensor ggml_moe_a8_vec( col, row, quant_X.stride(0), + expert_stride_bytes, stream); break; case 3: @@ -581,6 +583,7 @@ torch::Tensor ggml_moe_a8_vec( col, row, quant_X.stride(0), + expert_stride_bytes, stream); break; case 6: @@ -594,6 +597,7 @@ torch::Tensor ggml_moe_a8_vec( col, row, quant_X.stride(0), + expert_stride_bytes, stream); break; case 7: @@ -607,6 +611,7 @@ torch::Tensor ggml_moe_a8_vec( col, row, quant_X.stride(0), + expert_stride_bytes, stream); break; case 8: @@ -620,6 +625,7 @@ torch::Tensor ggml_moe_a8_vec( col, row, quant_X.stride(0), + expert_stride_bytes, stream); break; case 10: @@ -633,6 +639,7 @@ torch::Tensor ggml_moe_a8_vec( col, row, quant_X.stride(0), + expert_stride_bytes, stream); break; case 11: @@ -646,6 +653,7 @@ torch::Tensor ggml_moe_a8_vec( col, row, quant_X.stride(0), + expert_stride_bytes, stream); break; case 12: @@ -659,6 +667,7 @@ torch::Tensor ggml_moe_a8_vec( col, row, quant_X.stride(0), + expert_stride_bytes, stream); break; case 13: @@ -672,6 +681,7 @@ torch::Tensor ggml_moe_a8_vec( col, row, quant_X.stride(0), + expert_stride_bytes, stream); break; case 14: @@ -685,6 +695,7 @@ torch::Tensor ggml_moe_a8_vec( col, row, quant_X.stride(0), + expert_stride_bytes, stream); break; case 16: @@ -698,6 +709,7 @@ torch::Tensor ggml_moe_a8_vec( col, row, quant_X.stride(0), + expert_stride_bytes, stream); break; case 17: @@ -711,6 +723,7 @@ torch::Tensor ggml_moe_a8_vec( col, row, quant_X.stride(0), + expert_stride_bytes, stream); break; case 18: @@ -724,6 +737,7 @@ torch::Tensor ggml_moe_a8_vec( col, row, quant_X.stride(0), + expert_stride_bytes, stream); break; case 19: @@ -737,6 +751,7 @@ torch::Tensor ggml_moe_a8_vec( col, row, quant_X.stride(0), + expert_stride_bytes, stream); break; case 20: @@ -750,6 +765,7 @@ torch::Tensor ggml_moe_a8_vec( col, row, quant_X.stride(0), + expert_stride_bytes, stream); break; case 21: @@ -763,6 +779,7 @@ torch::Tensor ggml_moe_a8_vec( col, row, quant_X.stride(0), + expert_stride_bytes, stream); break; case 22: @@ -776,6 +793,7 @@ torch::Tensor ggml_moe_a8_vec( col, row, quant_X.stride(0), + expert_stride_bytes, stream); break; case 23: @@ -789,6 +807,7 @@ torch::Tensor ggml_moe_a8_vec( col, row, quant_X.stride(0), + expert_stride_bytes, stream); break; case 29: @@ -802,6 +821,7 @@ torch::Tensor ggml_moe_a8_vec( col, row, quant_X.stride(0), + expert_stride_bytes, stream); break; } diff --git a/python/freetoken/kernel/csrc/gguf/moe.cuh b/python/freetoken/kernel/csrc/gguf/moe.cuh index 91e434fa..29186ea7 100644 --- a/python/freetoken/kernel/csrc/gguf/moe.cuh +++ b/python/freetoken/kernel/csrc/gguf/moe.cuh @@ -43,6 +43,15 @@ static __device__ __forceinline__ void moe_q( const auto col_dst_0 = blockIdx.y * mmq_x; + // ``sorted_token_ids`` is capacity-sized, while num_tokens_post_padded is the + // live aligned prefix. The launch grid is built from that capacity and can + // therefore contain one block beginning exactly at the end of the live prefix. + // Reject it before reading its uninitialized token/expert entries. The old + // strict-``>`` check ran that block and could use a negative garbage token id as + // an input/output offset (observed as an illegal access in Qwen3.5-MoE's 128-token + // prefill warmup). + if (col_dst_0 >= num_tokens_post_padded[0]) return; + int token_offs[mmq_x / nwarps]; for (int i = 0; i < mmq_x; i += nwarps) { token_offs[i / nwarps] = sorted_token_ids[col_dst_0 + threadIdx.y + i]; @@ -50,7 +59,6 @@ static __device__ __forceinline__ void moe_q( const int exp_idx = expert_ids[blockIdx.y]; if (exp_idx > 255 || exp_idx < 0) return; - if (blockIdx.y * mmq_x > num_tokens_post_padded[0]) return; const block_q_t* x = (const block_q_t*)((char*)vx + exp_idx * exp_stride); const block_q8_1* y = (const block_q8_1*)(vy); diff --git a/python/freetoken/kernel/csrc/gguf/moe_vec.cuh b/python/freetoken/kernel/csrc/gguf/moe_vec.cuh index 8cef9e08..e3cae540 100644 --- a/python/freetoken/kernel/csrc/gguf/moe_vec.cuh +++ b/python/freetoken/kernel/csrc/gguf/moe_vec.cuh @@ -11,7 +11,8 @@ static __global__ void moe_vec_q( const int topk, const int ncols, const int nrows, - const int token_stride) { + const int token_stride, + const int64_t expert_stride_bytes) { const auto row = blockIdx.x * blockDim.y + threadIdx.y; const auto token = blockIdx.z / topk; @@ -27,7 +28,11 @@ static __global__ void moe_vec_q( // partial sum for each thread float tmp = 0.0f; - const block_q_t* x = ((const block_q_t*)vx) + expert * nrows * blocks_per_row; + // expert_stride_bytes == 0: dense contiguous banks (original layout). > 0: each + // expert starts at a fixed byte stride (padded banks for mixed-quant models). + const block_q_t* x = expert_stride_bytes > 0 + ? (const block_q_t*)((const char*)vx + (size_t)expert * expert_stride_bytes) + : ((const block_q_t*)vx) + expert * nrows * blocks_per_row; const block_q8_1* y = (const block_q8_1*)(((const int*)vy) + token * token_stride); for (auto i = threadIdx.x / (qi / vdr); i < blocks_per_row; i += blocks_per_warp) { @@ -62,12 +67,13 @@ static void moe_vec_q4_0_q8_1_cuda( const int ncols, const int nrows, const int token_stride, + const int64_t expert_stride_bytes, cudaStream_t stream) { const int block_num_y = (nrows + GGML_CUDA_MMV_Y - 1) / GGML_CUDA_MMV_Y; const dim3 block_nums(block_num_y, 1, tokens * top_k); const dim3 block_dims(WARP_SIZE, GGML_CUDA_MMV_Y, 1); moe_vec_q - <<>>(vx, vy, dst, topk_ids, top_k, ncols, nrows, token_stride); + <<>>(vx, vy, dst, topk_ids, top_k, ncols, nrows, token_stride, expert_stride_bytes); } template @@ -81,12 +87,13 @@ static void moe_vec_q4_1_q8_1_cuda( const int ncols, const int nrows, const int token_stride, + const int64_t expert_stride_bytes, cudaStream_t stream) { const int block_num_y = (nrows + GGML_CUDA_MMV_Y - 1) / GGML_CUDA_MMV_Y; const dim3 block_nums(block_num_y, 1, tokens * top_k); const dim3 block_dims(WARP_SIZE, GGML_CUDA_MMV_Y, 1); moe_vec_q - <<>>(vx, vy, dst, topk_ids, top_k, ncols, nrows, token_stride); + <<>>(vx, vy, dst, topk_ids, top_k, ncols, nrows, token_stride, expert_stride_bytes); } template @@ -100,12 +107,13 @@ static void moe_vec_q5_0_q8_1_cuda( const int ncols, const int nrows, const int token_stride, + const int64_t expert_stride_bytes, cudaStream_t stream) { const int block_num_y = (nrows + GGML_CUDA_MMV_Y - 1) / GGML_CUDA_MMV_Y; const dim3 block_nums(block_num_y, 1, tokens * top_k); const dim3 block_dims(WARP_SIZE, GGML_CUDA_MMV_Y, 1); moe_vec_q - <<>>(vx, vy, dst, topk_ids, top_k, ncols, nrows, token_stride); + <<>>(vx, vy, dst, topk_ids, top_k, ncols, nrows, token_stride, expert_stride_bytes); } template @@ -119,12 +127,13 @@ static void moe_vec_q5_1_q8_1_cuda( const int ncols, const int nrows, const int token_stride, + const int64_t expert_stride_bytes, cudaStream_t stream) { const int block_num_y = (nrows + GGML_CUDA_MMV_Y - 1) / GGML_CUDA_MMV_Y; const dim3 block_nums(block_num_y, 1, tokens * top_k); const dim3 block_dims(WARP_SIZE, GGML_CUDA_MMV_Y, 1); moe_vec_q - <<>>(vx, vy, dst, topk_ids, top_k, ncols, nrows, token_stride); + <<>>(vx, vy, dst, topk_ids, top_k, ncols, nrows, token_stride, expert_stride_bytes); } template @@ -138,12 +147,13 @@ static void moe_vec_q8_0_q8_1_cuda( const int ncols, const int nrows, const int token_stride, + const int64_t expert_stride_bytes, cudaStream_t stream) { const int block_num_y = (nrows + GGML_CUDA_MMV_Y - 1) / GGML_CUDA_MMV_Y; const dim3 block_nums(block_num_y, 1, tokens * top_k); const dim3 block_dims(WARP_SIZE, GGML_CUDA_MMV_Y, 1); moe_vec_q - <<>>(vx, vy, dst, topk_ids, top_k, ncols, nrows, token_stride); + <<>>(vx, vy, dst, topk_ids, top_k, ncols, nrows, token_stride, expert_stride_bytes); } template @@ -157,12 +167,13 @@ static void moe_vec_q2_K_q8_1_cuda( const int ncols, const int nrows, const int token_stride, + const int64_t expert_stride_bytes, cudaStream_t stream) { const int block_num_y = (nrows + GGML_CUDA_MMV_Y - 1) / GGML_CUDA_MMV_Y; const dim3 block_nums(block_num_y, 1, tokens * top_k); const dim3 block_dims(WARP_SIZE, GGML_CUDA_MMV_Y, 1); moe_vec_q - <<>>(vx, vy, dst, topk_ids, top_k, ncols, nrows, token_stride); + <<>>(vx, vy, dst, topk_ids, top_k, ncols, nrows, token_stride, expert_stride_bytes); } template @@ -176,12 +187,13 @@ static void moe_vec_q3_K_q8_1_cuda( const int ncols, const int nrows, const int token_stride, + const int64_t expert_stride_bytes, cudaStream_t stream) { const int block_num_y = (nrows + GGML_CUDA_MMV_Y - 1) / GGML_CUDA_MMV_Y; const dim3 block_nums(block_num_y, 1, tokens * top_k); const dim3 block_dims(WARP_SIZE, GGML_CUDA_MMV_Y, 1); moe_vec_q - <<>>(vx, vy, dst, topk_ids, top_k, ncols, nrows, token_stride); + <<>>(vx, vy, dst, topk_ids, top_k, ncols, nrows, token_stride, expert_stride_bytes); } template @@ -195,12 +207,13 @@ static void moe_vec_q4_K_q8_1_cuda( const int ncols, const int nrows, const int token_stride, + const int64_t expert_stride_bytes, cudaStream_t stream) { const int block_num_y = (nrows + GGML_CUDA_MMV_Y - 1) / GGML_CUDA_MMV_Y; const dim3 block_nums(block_num_y, 1, tokens * top_k); const dim3 block_dims(WARP_SIZE, GGML_CUDA_MMV_Y, 1); moe_vec_q - <<>>(vx, vy, dst, topk_ids, top_k, ncols, nrows, token_stride); + <<>>(vx, vy, dst, topk_ids, top_k, ncols, nrows, token_stride, expert_stride_bytes); } template @@ -214,12 +227,13 @@ static void moe_vec_q5_K_q8_1_cuda( const int ncols, const int nrows, const int token_stride, + const int64_t expert_stride_bytes, cudaStream_t stream) { const int block_num_y = (nrows + GGML_CUDA_MMV_Y - 1) / GGML_CUDA_MMV_Y; const dim3 block_nums(block_num_y, 1, tokens * top_k); const dim3 block_dims(WARP_SIZE, GGML_CUDA_MMV_Y, 1); moe_vec_q - <<>>(vx, vy, dst, topk_ids, top_k, ncols, nrows, token_stride); + <<>>(vx, vy, dst, topk_ids, top_k, ncols, nrows, token_stride, expert_stride_bytes); } template @@ -233,12 +247,13 @@ static void moe_vec_q6_K_q8_1_cuda( const int ncols, const int nrows, const int token_stride, + const int64_t expert_stride_bytes, cudaStream_t stream) { const int block_num_y = (nrows + GGML_CUDA_MMV_Y - 1) / GGML_CUDA_MMV_Y; const dim3 block_nums(block_num_y, 1, tokens * top_k); const dim3 block_dims(WARP_SIZE, GGML_CUDA_MMV_Y, 1); moe_vec_q - <<>>(vx, vy, dst, topk_ids, top_k, ncols, nrows, token_stride); + <<>>(vx, vy, dst, topk_ids, top_k, ncols, nrows, token_stride, expert_stride_bytes); } template @@ -252,12 +267,13 @@ static void moe_vec_iq2_xxs_q8_1_cuda( const int ncols, const int nrows, const int token_stride, + const int64_t expert_stride_bytes, cudaStream_t stream) { const int block_num_y = (nrows + GGML_CUDA_MMV_Y - 1) / GGML_CUDA_MMV_Y; const dim3 block_nums(block_num_y, 1, tokens * top_k); const dim3 block_dims(WARP_SIZE, GGML_CUDA_MMV_Y, 1); moe_vec_q - <<>>(vx, vy, dst, topk_ids, top_k, ncols, nrows, token_stride); + <<>>(vx, vy, dst, topk_ids, top_k, ncols, nrows, token_stride, expert_stride_bytes); } template @@ -271,12 +287,13 @@ static void moe_vec_iq2_xs_q8_1_cuda( const int ncols, const int nrows, const int token_stride, + const int64_t expert_stride_bytes, cudaStream_t stream) { const int block_num_y = (nrows + GGML_CUDA_MMV_Y - 1) / GGML_CUDA_MMV_Y; const dim3 block_nums(block_num_y, 1, tokens * top_k); const dim3 block_dims(WARP_SIZE, GGML_CUDA_MMV_Y, 1); moe_vec_q - <<>>(vx, vy, dst, topk_ids, top_k, ncols, nrows, token_stride); + <<>>(vx, vy, dst, topk_ids, top_k, ncols, nrows, token_stride, expert_stride_bytes); } template @@ -290,12 +307,13 @@ static void moe_vec_iq2_s_q8_1_cuda( const int ncols, const int nrows, const int token_stride, + const int64_t expert_stride_bytes, cudaStream_t stream) { const int block_num_y = (nrows + GGML_CUDA_MMV_Y - 1) / GGML_CUDA_MMV_Y; const dim3 block_nums(block_num_y, 1, tokens * top_k); const dim3 block_dims(WARP_SIZE, GGML_CUDA_MMV_Y, 1); moe_vec_q - <<>>(vx, vy, dst, topk_ids, top_k, ncols, nrows, token_stride); + <<>>(vx, vy, dst, topk_ids, top_k, ncols, nrows, token_stride, expert_stride_bytes); } template @@ -309,12 +327,13 @@ static void moe_vec_iq3_xxs_q8_1_cuda( const int ncols, const int nrows, const int token_stride, + const int64_t expert_stride_bytes, cudaStream_t stream) { const int block_num_y = (nrows + GGML_CUDA_MMV_Y - 1) / GGML_CUDA_MMV_Y; const dim3 block_nums(block_num_y, 1, tokens * top_k); const dim3 block_dims(WARP_SIZE, GGML_CUDA_MMV_Y, 1); moe_vec_q - <<>>(vx, vy, dst, topk_ids, top_k, ncols, nrows, token_stride); + <<>>(vx, vy, dst, topk_ids, top_k, ncols, nrows, token_stride, expert_stride_bytes); } template @@ -328,12 +347,13 @@ static void moe_vec_iq1_s_q8_1_cuda( const int ncols, const int nrows, const int token_stride, + const int64_t expert_stride_bytes, cudaStream_t stream) { const int block_num_y = (nrows + GGML_CUDA_MMV_Y - 1) / GGML_CUDA_MMV_Y; const dim3 block_nums(block_num_y, 1, tokens * top_k); const dim3 block_dims(WARP_SIZE, GGML_CUDA_MMV_Y, 1); moe_vec_q - <<>>(vx, vy, dst, topk_ids, top_k, ncols, nrows, token_stride); + <<>>(vx, vy, dst, topk_ids, top_k, ncols, nrows, token_stride, expert_stride_bytes); } template @@ -347,12 +367,13 @@ static void moe_vec_iq1_m_q8_1_cuda( const int ncols, const int nrows, const int token_stride, + const int64_t expert_stride_bytes, cudaStream_t stream) { const int block_num_y = (nrows + GGML_CUDA_MMV_Y - 1) / GGML_CUDA_MMV_Y; const dim3 block_nums(block_num_y, 1, tokens * top_k); const dim3 block_dims(WARP_SIZE, GGML_CUDA_MMV_Y, 1); moe_vec_q - <<>>(vx, vy, dst, topk_ids, top_k, ncols, nrows, token_stride); + <<>>(vx, vy, dst, topk_ids, top_k, ncols, nrows, token_stride, expert_stride_bytes); } template @@ -366,12 +387,13 @@ static void moe_vec_iq4_nl_q8_1_cuda( const int ncols, const int nrows, const int token_stride, + const int64_t expert_stride_bytes, cudaStream_t stream) { const int block_num_y = (nrows + GGML_CUDA_MMV_Y - 1) / GGML_CUDA_MMV_Y; const dim3 block_nums(block_num_y, 1, tokens * top_k); const dim3 block_dims(WARP_SIZE, GGML_CUDA_MMV_Y, 1); moe_vec_q - <<>>(vx, vy, dst, topk_ids, top_k, ncols, nrows, token_stride); + <<>>(vx, vy, dst, topk_ids, top_k, ncols, nrows, token_stride, expert_stride_bytes); } template @@ -385,12 +407,13 @@ static void moe_vec_iq4_xs_q8_1_cuda( const int ncols, const int nrows, const int token_stride, + const int64_t expert_stride_bytes, cudaStream_t stream) { const int block_num_y = (nrows + GGML_CUDA_MMV_Y - 1) / GGML_CUDA_MMV_Y; const dim3 block_nums(block_num_y, 1, tokens * top_k); const dim3 block_dims(WARP_SIZE, GGML_CUDA_MMV_Y, 1); moe_vec_q - <<>>(vx, vy, dst, topk_ids, top_k, ncols, nrows, token_stride); + <<>>(vx, vy, dst, topk_ids, top_k, ncols, nrows, token_stride, expert_stride_bytes); } template @@ -404,10 +427,11 @@ static void moe_vec_iq3_s_q8_1_cuda( const int ncols, const int nrows, const int token_stride, + const int64_t expert_stride_bytes, cudaStream_t stream) { const int block_num_y = (nrows + GGML_CUDA_MMV_Y - 1) / GGML_CUDA_MMV_Y; const dim3 block_nums(block_num_y, 1, tokens * top_k); const dim3 block_dims(WARP_SIZE, GGML_CUDA_MMV_Y, 1); moe_vec_q - <<>>(vx, vy, dst, topk_ids, top_k, ncols, nrows, token_stride); + <<>>(vx, vy, dst, topk_ids, top_k, ncols, nrows, token_stride, expert_stride_bytes); } diff --git a/python/freetoken/kernel/csrc/gguf_mmq/common.cuh b/python/freetoken/kernel/csrc/gguf_mmq/common.cuh new file mode 100644 index 00000000..14dd1098 --- /dev/null +++ b/python/freetoken/kernel/csrc/gguf_mmq/common.cuh @@ -0,0 +1,1676 @@ +#pragma once + +#include "ggml.h" +#include "ggml-impl.h" +#include "ggml-cuda.h" + +#include +#include +#include +#include + +#if defined(GGML_USE_HIP) +#define GGML_COMMON_DECL_HIP +#define GGML_COMMON_IMPL_HIP +#else +#define GGML_COMMON_DECL_CUDA +#define GGML_COMMON_IMPL_CUDA +#if defined(GGML_USE_MUSA) +#define GGML_COMMON_DECL_MUSA +#define GGML_COMMON_IMPL_MUSA +#endif +#endif +#include "ggml-common.h" + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#if defined(GGML_USE_HIP) +#include "vendors/hip.h" +#elif defined(GGML_USE_MUSA) +#include "vendors/musa.h" +#else +#include "vendors/cuda.h" +#endif // defined(GGML_USE_HIP) + +#define STRINGIZE_IMPL(...) #__VA_ARGS__ +#define STRINGIZE(...) STRINGIZE_IMPL(__VA_ARGS__) + +#define WARP_SIZE 32 +#define CUDART_HMAX 11070 // CUDA 11.7, min. ver. for which __hmax and __hmax2 are known to work (may be higher than needed) +#define CUDART_HMASK 12000 // CUDA 12.0, min. ver. for half2 -> uint mask comparisons + +#define GGML_CUDA_CC_PASCAL 600 +#define GGML_CUDA_CC_DP4A 610 // minimum compute capability for __dp4a, an intrinsic for byte-wise dot products +#define GGML_CUDA_CC_VOLTA 700 +#define GGML_CUDA_CC_TURING 750 +#define GGML_CUDA_CC_AMPERE 800 +#define GGML_CUDA_CC_ADA_LOVELACE 890 +#define GGML_CUDA_CC_HOPPER 900 +// While BW spans CC 1000, 1100 & 1200, we are integrating Tensor Core instructions available to 1200 family, see +// https://docs.nvidia.com/cutlass/media/docs/cpp/blackwell_functionality.html#blackwell-sm120-gemms +#define GGML_CUDA_CC_BLACKWELL 1200 +#define GGML_CUDA_CC_DGX_SPARK 1210 +#define GGML_CUDA_CC_RUBIN 1300 +#define GGML_CUDA_CC_OFFSET_AMD 0x1000000 +#define GGML_CUDA_CC_OFFSET_MTHREADS 0x0100000 +#define GGML_CUDA_CC_IS_NVIDIA(cc) (cc < GGML_CUDA_CC_OFFSET_MTHREADS) + +// AMD +// GCN/CDNA, wave size is 64 +#define GGML_CUDA_CC_GCN4 (GGML_CUDA_CC_OFFSET_AMD + 0x803) // Tonga, Fiji, Polaris, minimum for fast fp16 +#define GGML_CUDA_CC_VEGA (GGML_CUDA_CC_OFFSET_AMD + 0x900) // Vega56/64, minimum for fp16 dual issue +#define GGML_CUDA_CC_VEGA20 (GGML_CUDA_CC_OFFSET_AMD + 0x906) // MI50/Radeon VII, minimum for dp4a +#define GGML_CUDA_CC_CDNA1 (GGML_CUDA_CC_OFFSET_AMD + 0x908) // MI100, minimum for MFMA, acc registers +#define GGML_CUDA_CC_CDNA2 (GGML_CUDA_CC_OFFSET_AMD + 0x90a) // MI210 (gfx90a), minimum acc register renaming +#define GGML_CUDA_CC_CDNA3 (GGML_CUDA_CC_OFFSET_AMD + 0x942) // MI300 +#define GGML_CUDA_CC_CDNA4 (GGML_CUDA_CC_OFFSET_AMD + 0x950) // MI350X/MI355X + +// RDNA removes MFMA, dp4a, xnack, acc registers, wave size is 32 +#define GGML_CUDA_CC_RDNA1 (GGML_CUDA_CC_OFFSET_AMD + 0x1010) // RX 5000 +#define GGML_CUDA_CC_RDNA2 (GGML_CUDA_CC_OFFSET_AMD + 0x1030) // RX 6000, minimum for dp4a +#define GGML_CUDA_CC_RDNA3 (GGML_CUDA_CC_OFFSET_AMD + 0x1100) // RX 7000, minimum for WMMA +#define GGML_CUDA_CC_RDNA3_5 (GGML_CUDA_CC_OFFSET_AMD + 0x1150) // AI 370, AI Max 395 laptops. +#define GGML_CUDA_CC_RDNA4 (GGML_CUDA_CC_OFFSET_AMD + 0x1200) // RX 9000 + +#define GGML_CUDA_CC_IS_AMD(cc) (cc >= GGML_CUDA_CC_OFFSET_AMD) +#define GGML_CUDA_CC_IS_RDNA(cc) (cc >= GGML_CUDA_CC_RDNA1) +#define GGML_CUDA_CC_IS_RDNA1(cc) (cc >= GGML_CUDA_CC_RDNA1 && cc < GGML_CUDA_CC_RDNA2) +#define GGML_CUDA_CC_IS_RDNA2(cc) (cc >= GGML_CUDA_CC_RDNA2 && cc < GGML_CUDA_CC_RDNA3) +#define GGML_CUDA_CC_IS_RDNA3_0(cc) (cc >= GGML_CUDA_CC_RDNA3 && cc < GGML_CUDA_CC_RDNA3_5) +#define GGML_CUDA_CC_IS_RDNA3_5(cc) (cc >= GGML_CUDA_CC_RDNA3_5 && cc < GGML_CUDA_CC_RDNA4) +#define GGML_CUDA_CC_IS_RDNA3(cc) (GGML_CUDA_CC_IS_RDNA3_0(cc) || GGML_CUDA_CC_IS_RDNA3_5(cc)) +#define GGML_CUDA_CC_IS_RDNA4(cc) (cc >= GGML_CUDA_CC_RDNA4) +#define GGML_CUDA_CC_IS_GCN(cc) (cc > GGML_CUDA_CC_OFFSET_AMD && cc < GGML_CUDA_CC_CDNA1) +#define GGML_CUDA_CC_IS_CDNA(cc) (cc >= GGML_CUDA_CC_CDNA1 && cc < GGML_CUDA_CC_RDNA1) +#define GGML_CUDA_CC_IS_CDNA1(cc) (cc >= GGML_CUDA_CC_CDNA1 && cc < GGML_CUDA_CC_CDNA2) +#define GGML_CUDA_CC_IS_CDNA2(cc) (cc >= GGML_CUDA_CC_CDNA2 && cc < GGML_CUDA_CC_CDNA3) +#define GGML_CUDA_CC_IS_CDNA3(cc) (cc >= GGML_CUDA_CC_CDNA3 && cc < GGML_CUDA_CC_CDNA4) +#define GGML_CUDA_CC_IS_CDNA4(cc) (cc >= GGML_CUDA_CC_CDNA4 && cc < GGML_CUDA_CC_RDNA1) + +// Moore Threads +#define MUSART_HMASK 40300 // MUSA rc4.3, min. ver. for half2 -> uint mask comparisons + +#define GGML_CUDA_CC_QY1 (GGML_CUDA_CC_OFFSET_MTHREADS + 0x210) // MTT S80, MTT S3000 +#define GGML_CUDA_CC_QY2 (GGML_CUDA_CC_OFFSET_MTHREADS + 0x220) // MTT S4000 +#define GGML_CUDA_CC_PH1 (GGML_CUDA_CC_OFFSET_MTHREADS + 0x310) // MTT S5000 + +#define GGML_CUDA_CC_IS_MTHREADS(cc) (cc >= GGML_CUDA_CC_OFFSET_MTHREADS && cc < GGML_CUDA_CC_OFFSET_AMD) +#define GGML_CUDA_CC_IS_QY1(cc) (cc >= GGML_CUDA_CC_QY1 && cc < GGML_CUDA_CC_QY2) +#define GGML_CUDA_CC_IS_QY2(cc) (cc >= GGML_CUDA_CC_QY2 && cc < GGML_CUDA_CC_PH1) +#define GGML_CUDA_CC_IS_PH1(cc) (cc >= GGML_CUDA_CC_PH1) + +#if !defined(GGML_USE_HIP) && !defined(GGML_USE_MUSA) && CUDART_VERSION >= 11070 +# define GGML_CUDA_USE_CUB +#endif // !defined(GGML_USE_HIP) && !defined(GGML_USE_MUSA) && CUDART_VERSION >= 11070 + +// PDL host-side support (cudaLaunchKernelEx) requires CUDART >= 11.8. +// However, this has been bugged in CTK < 12.3 for MSVC builds, see +// https://github.com/ggml-org/llama.cpp/pull/22522#discussion_r3302393293 +// __CUDA_ARCH__ is undefined in host passes; GPU arch check happens in device-side code. +#if !defined(GGML_USE_HIP) && !defined(GGML_USE_MUSA) && \ + (CUDART_VERSION >= 12030 || (!(defined(_MSC_VER) && !defined(__clang__)) && CUDART_VERSION >= 11080)) +# define GGML_CUDA_USE_PDL +#endif // !defined(GGML_USE_HIP) && !defined(GGML_USE_MUSA) && (CUDART_VERSION >= 12030 || (!(defined(_MSC_VER) && !defined(__clang__)) && CUDART_VERSION >= 11080)) + +static __device__ __forceinline__ void ggml_cuda_pdl_sync() { +#if defined(GGML_CUDA_USE_PDL) && defined(__CUDA_ARCH__) && __CUDA_ARCH__ >= GGML_CUDA_CC_HOPPER + cudaGridDependencySynchronize(); +#endif // defined(GGML_CUDA_USE_PDL) && defined(__CUDA_ARCH__) && __CUDA_ARCH__ >= GGML_CUDA_CC_HOPPER +} + +static __device__ __forceinline__ void ggml_cuda_pdl_lc() { +#if defined(GGML_CUDA_USE_PDL) && defined(__CUDA_ARCH__) && __CUDA_ARCH__ >= GGML_CUDA_CC_HOPPER + cudaTriggerProgrammaticLaunchCompletion(); +#endif // defined(GGML_CUDA_USE_PDL) && defined(__CUDA_ARCH__) && __CUDA_ARCH__ >= GGML_CUDA_CC_HOPPER +} + +#ifdef __CUDA_ARCH_LIST__ +constexpr bool ggml_cuda_has_arch_impl(int) { + return false; +} + +template +constexpr bool ggml_cuda_has_arch_impl(const int arch, const int first, Archs... rest) { + return arch == first || ggml_cuda_has_arch_impl(arch, rest...); +} + +constexpr bool ggml_cuda_has_arch(const int arch) { + return ggml_cuda_has_arch_impl(arch, __CUDA_ARCH_LIST__); +} + +constexpr int ggml_cuda_highest_compiled_arch_impl(const int /*arch*/, const int cur) { + if (cur == 0) { + return -1; + } + return cur; +} + +template +constexpr int ggml_cuda_highest_compiled_arch_impl(const int arch, const int cur, const int first, Archs... rest) { + if (first <= arch && first > cur) { + return ggml_cuda_highest_compiled_arch_impl(arch, first, rest...); + } else { + return ggml_cuda_highest_compiled_arch_impl(arch, cur, rest...); + } +} + +constexpr int ggml_cuda_highest_compiled_arch(const int arch) { + return ggml_cuda_highest_compiled_arch_impl(arch, 0, __CUDA_ARCH_LIST__); +} +#else +static int ggml_cuda_highest_compiled_arch(const int arch) { + return arch; +} +#endif // __CUDA_ARCH_LIST__ + +// --------------------------------------------------------------------------------------------------------- + +#define MATRIX_ROW_PADDING 512 // last row of quant. matrices is a multiple of this to avoid out-of-bounds memory accesses + +#define GGML_CUDA_MAX_STREAMS 8 + +[[noreturn]] +void ggml_cuda_error(const char * stmt, const char * func, const char * file, int line, const char * msg); + +#define CUDA_CHECK_GEN(err, success, error_fn) \ + do { \ + auto err_ = (err); \ + if (err_ != (success)) { \ + ggml_cuda_error(#err, __func__, __FILE__, __LINE__, error_fn(err_)); \ + } \ + } while (0) + +#define CUDA_CHECK(err) CUDA_CHECK_GEN(err, cudaSuccess, cudaGetErrorString) + + +#if CUDART_VERSION >= 12000 || defined(GGML_USE_MUSA) + static const char * cublas_get_error_str(const cublasStatus_t err) { + return cublasGetStatusString(err); + } +#else + static const char * cublas_get_error_str(const cublasStatus_t err) { + switch (err) { + case CUBLAS_STATUS_SUCCESS: return "CUBLAS_STATUS_SUCCESS"; + case CUBLAS_STATUS_NOT_INITIALIZED: return "CUBLAS_STATUS_NOT_INITIALIZED"; + case CUBLAS_STATUS_ALLOC_FAILED: return "CUBLAS_STATUS_ALLOC_FAILED"; + case CUBLAS_STATUS_INVALID_VALUE: return "CUBLAS_STATUS_INVALID_VALUE"; + case CUBLAS_STATUS_ARCH_MISMATCH: return "CUBLAS_STATUS_ARCH_MISMATCH"; + case CUBLAS_STATUS_MAPPING_ERROR: return "CUBLAS_STATUS_MAPPING_ERROR"; + case CUBLAS_STATUS_EXECUTION_FAILED: return "CUBLAS_STATUS_EXECUTION_FAILED"; + case CUBLAS_STATUS_INTERNAL_ERROR: return "CUBLAS_STATUS_INTERNAL_ERROR"; + case CUBLAS_STATUS_NOT_SUPPORTED: return "CUBLAS_STATUS_NOT_SUPPORTED"; + default: return "unknown error"; + } + } +#endif // CUDART_VERSION >= 12000 + +#define CUBLAS_CHECK(err) CUDA_CHECK_GEN(err, CUBLAS_STATUS_SUCCESS, cublas_get_error_str) + +#ifdef GGML_USE_NCCL +#define NCCL_CHECK(err) CUDA_CHECK_GEN(err, ncclSuccess, ncclGetErrorString) +#endif // GGML_USE_NCCL + +#if !defined(GGML_USE_HIP) && !defined(GGML_CUDA_NO_VMM) +static const char * cu_get_error_str(CUresult err) { + const char * err_str; + cuGetErrorString(err, &err_str); + return err_str; +} +#define CU_CHECK(err) CUDA_CHECK_GEN(err, CUDA_SUCCESS, cu_get_error_str) +#endif + +#if !defined(GGML_USE_HIP) && !defined(GGML_USE_MUSA) +# define CUDA_SET_SHARED_MEMORY_LIMIT(kernel, nbytes) \ + do { \ + static bool shared_memory_limit_raised[GGML_CUDA_MAX_DEVICES] = { false }; \ + const int id = ggml_cuda_get_device(); \ + if (!shared_memory_limit_raised[id]) { \ + CUDA_CHECK(cudaFuncSetAttribute(kernel, cudaFuncAttributeMaxDynamicSharedMemorySize, nbytes)); \ + shared_memory_limit_raised[id] = true; \ + } \ + } while (0) +#else +# define CUDA_SET_SHARED_MEMORY_LIMIT(kernel, nbytes) \ + do { \ + GGML_UNUSED(nbytes); \ + } while (0) +#endif // !(defined(GGML_USE_HIP) && !defined(GGML_USE_MUSA) + +#if CUDART_VERSION >= 11010 || defined(GGML_USE_MUSA) +#define GGML_CUDA_ASSUME(x) __builtin_assume(x) +#else +#define GGML_CUDA_ASSUME(x) +#endif // CUDART_VERSION >= 11010 + +#if (!defined(GGML_USE_HIP) && !defined(GGML_CUDA_NO_VMM)) || (defined(GGML_USE_HIP) && !defined(GGML_HIP_NO_VMM)) +#define GGML_USE_VMM +#endif // (!defined(GGML_USE_HIP) && !defined(GGML_CUDA_NO_VMM)) || (defined(GGML_USE_HIP) && !defined(GGML_HIP_NO_VMM)) + +#if defined(GGML_USE_HIP) || defined(GGML_USE_MUSA) || __CUDA_ARCH__ >= GGML_CUDA_CC_PASCAL +#define FP16_AVAILABLE +#endif // defined(GGML_USE_HIP) || defined(GGML_USE_MUSA) || __CUDA_ARCH__ >= GGML_CUDA_CC_PASCAL + +#if defined(FP16_AVAILABLE) && __CUDA_ARCH__ != 610 +#define FAST_FP16_AVAILABLE +#endif // defined(FP16_AVAILABLE) && __CUDA_ARCH__ != 610 + +#if defined(GGML_USE_HIP) && defined(CDNA) && !defined(GGML_HIP_NO_MMQ_MFMA) +#define AMD_MFMA_AVAILABLE +#endif // defined(GGML_USE_HIP) && defined(CDNA) && !defined(GGML_HIP_NO_MMQ_MFMA) + +#if defined(GGML_USE_HIP) && (defined(RDNA4) || defined(RDNA3)) +#define AMD_WMMA_AVAILABLE +#endif // defined(GGML_USE_HIP) && defined(RDNA4) + +// The Volta instructions are in principle available on Turing or newer but they are effectively unusable: +#if !defined(GGML_USE_HIP) && __CUDA_ARCH__ == GGML_CUDA_CC_VOLTA +#define VOLTA_MMA_AVAILABLE +#endif // !defined(GGML_USE_HIP) && __CUDA_ARCH__ == GGML_CUDA_CC_VOLTA + +#if !defined(GGML_USE_HIP) && __CUDA_ARCH__ >= GGML_CUDA_CC_TURING +#define TURING_MMA_AVAILABLE +#endif // !defined(GGML_USE_HIP) && __CUDA_ARCH__ >= GGML_CUDA_CC_TURING + +#if !defined(GGML_USE_HIP) && __CUDA_ARCH__ >= GGML_CUDA_CC_AMPERE +#define AMPERE_MMA_AVAILABLE +#endif // !defined(GGML_USE_HIP) && __CUDA_ARCH__ >= GGML_CUDA_CC_AMPERE + +#if !defined(GGML_USE_HIP) && __CUDA_ARCH__ >= GGML_CUDA_CC_BLACKWELL && __CUDA_ARCH__ < GGML_CUDA_CC_RUBIN +# define BLACKWELL_MMA_AVAILABLE +#endif // !defined(GGML_USE_HIP) && __CUDA_ARCH__ >= GGML_CUDA_CC_BLACKWELL + +#if !defined(GGML_USE_HIP) && __CUDA_ARCH__ >= GGML_CUDA_CC_AMPERE +#define CP_ASYNC_AVAILABLE +#endif // !defined(GGML_USE_HIP) && __CUDA_ARCH__ >= GGML_CUDA_CC_AMPERE + +#if !defined(GGML_CUDA_NO_FA) && !(defined(GGML_USE_MUSA) && __MUSA_ARCH__ < 220) +#define FLASH_ATTN_AVAILABLE +#endif // !defined(GGML_CUDA_NO_FA) && !(defined(GGML_USE_MUSA) && __MUSA_ARCH__ < 220) + +static bool fp16_available(const int cc) { + return ggml_cuda_highest_compiled_arch(cc) >= GGML_CUDA_CC_PASCAL || + (GGML_CUDA_CC_IS_MTHREADS(cc) && cc >= GGML_CUDA_CC_PH1); +} + +static bool fast_fp16_available(const int cc) { + return GGML_CUDA_CC_IS_AMD(cc) || + (GGML_CUDA_CC_IS_NVIDIA(cc) && fp16_available(cc) && ggml_cuda_highest_compiled_arch(cc) != 610) || + (GGML_CUDA_CC_IS_MTHREADS(cc) && fp16_available(cc)); +} + +// To be used for feature selection of external libraries, e.g. cuBLAS. +static bool fast_fp16_hardware_available(const int cc) { + return (GGML_CUDA_CC_IS_NVIDIA(cc) && cc >= GGML_CUDA_CC_PASCAL && cc != 610) || GGML_CUDA_CC_IS_AMD(cc) || + (GGML_CUDA_CC_IS_MTHREADS(cc) && cc >= GGML_CUDA_CC_QY2); +} + +// To be used for feature selection of external libraries, e.g. cuBLAS. +static bool fp16_mma_hardware_available(const int cc) { + return (GGML_CUDA_CC_IS_NVIDIA(cc) && cc >= GGML_CUDA_CC_VOLTA) || + GGML_CUDA_CC_IS_CDNA(cc) || GGML_CUDA_CC_IS_RDNA3(cc) || GGML_CUDA_CC_IS_RDNA4(cc) || + (GGML_CUDA_CC_IS_MTHREADS(cc) && cc >= GGML_CUDA_CC_QY2); +} + +static bool bf16_mma_hardware_available(const int cc) { + return (GGML_CUDA_CC_IS_NVIDIA(cc) && cc >= GGML_CUDA_CC_AMPERE) || + GGML_CUDA_CC_IS_CDNA(cc) || cc >= GGML_CUDA_CC_RDNA3 || + (GGML_CUDA_CC_IS_MTHREADS(cc) && cc >= GGML_CUDA_CC_PH1); +} + +static bool fp32_mma_hardware_available(const int cc) { + return GGML_CUDA_CC_IS_CDNA(cc); +} + +static bool amd_mfma_available(const int cc) { +#if !defined(GGML_HIP_NO_MMQ_MFMA) + return GGML_CUDA_CC_IS_CDNA(cc); +#else + return false; +#endif //!defined(GGML_HIP_NO_MMQ_MFMA) +} + +static bool amd_wmma_available(const int cc) { + return (GGML_CUDA_CC_IS_RDNA4(cc) || GGML_CUDA_CC_IS_RDNA3(cc)); +} + +static bool volta_mma_available(const int cc) { + return GGML_CUDA_CC_IS_NVIDIA(cc) && ggml_cuda_highest_compiled_arch(cc) == GGML_CUDA_CC_VOLTA; +} + +static bool turing_mma_available(const int cc) { + return GGML_CUDA_CC_IS_NVIDIA(cc) && ggml_cuda_highest_compiled_arch(cc) >= GGML_CUDA_CC_TURING; +} + +static bool ampere_mma_available(const int cc) { + return GGML_CUDA_CC_IS_NVIDIA(cc) && ggml_cuda_highest_compiled_arch(cc) >= GGML_CUDA_CC_AMPERE; +} + +static bool cp_async_available(const int cc) { + return GGML_CUDA_CC_IS_NVIDIA(cc) && ggml_cuda_highest_compiled_arch(cc) >= GGML_CUDA_CC_AMPERE; +} + +static bool blackwell_mma_available(const int cc) { + return GGML_CUDA_CC_IS_NVIDIA(cc) && ggml_cuda_highest_compiled_arch(cc) >= GGML_CUDA_CC_BLACKWELL && + ggml_cuda_highest_compiled_arch(cc) < GGML_CUDA_CC_RUBIN; +} + +// Checks whether the tensor's base data pointer and higher-dimensional strides are byte-aligned to `alignment` bytes. +static bool ggml_cuda_is_aligned(const ggml_tensor * tensor, const size_t alignment) { + GGML_ASSERT(tensor != nullptr); + return (reinterpret_cast(tensor->data) % alignment) == 0 && + tensor->nb[1] % alignment == 0 && + tensor->nb[2] % alignment == 0 && + tensor->nb[3] % alignment == 0; +} + +static constexpr __device__ int ggml_cuda_get_physical_warp_size() { +#if defined(GGML_USE_HIP) && (defined(__GFX9__) || defined(__GFX8__)) + return 64; +#else + return 32; +#endif // defined(GGML_USE_HIP) && (defined(__GFX9__) || defined(__GFX8__)) +} + +// Maximum number of bytes that can be copied in a single instruction. +static constexpr __device__ int ggml_cuda_get_max_cpy_bytes() { +#ifdef GGML_USE_HIP + return 16; +#else +#if __CUDA_ARCH__ >= GGML_CUDA_CC_VOLTA + return 16; +#else + return 8; +#endif // __CUDA_ARCH__ >= GGML_CUDA_CC_VOLTA +#endif // GGML_USE_HIP +} + + +[[noreturn]] +static __device__ void no_device_code( + const char * file_name, const int line, const char * function_name, const int arch, const char * arch_list) { + +#if defined(GGML_USE_HIP) + printf("%s:%d: ERROR: HIP kernel %s has no device code compatible with HIP arch %d.\n", + file_name, line, function_name, arch); + GGML_UNUSED(arch_list); +#else + printf("%s:%d: ERROR: CUDA kernel %s has no device code compatible with CUDA arch %d. ggml-cuda.cu was compiled for: %s\n", + file_name, line, function_name, arch, arch_list); +#endif // defined(GGML_USE_HIP) + __trap(); + + GGML_UNUSED(no_device_code); // suppress unused function warning + +#if defined(GGML_USE_MUSA) + __builtin_unreachable(); +#endif // defined(GGML_USE_MUSA) +} + +#ifdef __CUDA_ARCH__ +#define NO_DEVICE_CODE no_device_code(__FILE__, __LINE__, __FUNCTION__, __CUDA_ARCH__, STRINGIZE(__CUDA_ARCH_LIST__)) +#else +#define NO_DEVICE_CODE //GGML_ABORT("NO_DEVICE_CODE not valid in host code.") +#endif // __CUDA_ARCH__ + +// The compiler is always able to unroll loops if they contain continue expressions. +// In such cases loop unrolling can still be achieved via recursion: +template +struct ggml_cuda_unroll { + template + __device__ void operator()(const Func & f, Args... args) const { + f(n - 1, args...); + ggml_cuda_unroll{}(f, args...); + } +}; + +template <> +struct ggml_cuda_unroll<1> { + template + __device__ void operator()(const Func & f, Args... args) const { + f(0, args...); + } +}; + +template +static __device__ __forceinline__ int warp_reduce_sum(int x) { +#if !defined(GGML_USE_HIP) && __CUDA_ARCH__ >= GGML_CUDA_CC_AMPERE + return __reduce_add_sync(0xffffffff, x); +#else +#pragma unroll + for (int offset = width/2; offset > 0; offset >>= 1) { + x += __shfl_xor_sync(0xffffffff, x, offset, width); + } + return x; +#endif // !defined(GGML_USE_HIP) && __CUDA_ARCH__ >= GGML_CUDA_CC_AMPERE +} + +template +static __device__ __forceinline__ float warp_reduce_sum(float x) { +#pragma unroll + for (int offset = width/2; offset > 0; offset >>= 1) { + x += __shfl_xor_sync(0xffffffff, x, offset, width); + } + return x; +} + +template +static __device__ __forceinline__ float2 warp_reduce_sum(float2 a) { +#pragma unroll + for (int offset = width/2; offset > 0; offset >>= 1) { + a.x += __shfl_xor_sync(0xffffffff, a.x, offset, width); + a.y += __shfl_xor_sync(0xffffffff, a.y, offset, width); + } + return a; +} + +template +static __device__ __forceinline__ half2 warp_reduce_sum(half2 a) { +#ifdef FP16_AVAILABLE +#pragma unroll + for (int offset = width/2; offset > 0; offset >>= 1) { + a = __hadd2(a, __shfl_xor_sync(0xffffffff, a, offset, width)); + } + return a; + +#else + NO_DEVICE_CODE; + return a; +#endif // FP16_AVAILABLE +} + +template +static __device__ __forceinline__ int warp_reduce_all(int x) { + if (width == ggml_cuda_get_physical_warp_size()) { + return __all_sync(0xffffffff, x); + } else { +#pragma unroll + for (int offset = width/2; offset > 0; offset >>= 1) { + x = __shfl_xor_sync(0xffffffff, x, offset, width) && x; + } + return x; + } +} + +template +static __device__ __forceinline__ int warp_reduce_any(int x) { + if (width == ggml_cuda_get_physical_warp_size()) { + return __any_sync(0xffffffff, x); + } else { +#pragma unroll + for (int offset = width/2; offset > 0; offset >>= 1) { + x = __shfl_xor_sync(0xffffffff, x, offset, width) || x; + } + return x; + } +} + +template +static __device__ __forceinline__ float warp_reduce_max(float x) { +#pragma unroll + for (int offset = width/2; offset > 0; offset >>= 1) { + x = fmaxf(x, __shfl_xor_sync(0xffffffff, x, offset, width)); + } + return x; +} + +template +static __device__ __forceinline__ T warp_prefix_inclusive_sum(T x) { + const int lane_id = threadIdx.x % width; +#pragma unroll + for (int offset = 1; offset < width; offset <<= 1) { + const T t = __shfl_up_sync(0xffffffff, x, offset, width); + if (lane_id >= offset) { + x += t; + } + } + return x; +} + +template +static __device__ __forceinline__ float2 warp_prefix_inclusive_sum(float2 a) { + const int lane_id = threadIdx.x % width; +#pragma unroll + for (int offset = 1; offset < width; offset <<= 1) { + const float t_x = __shfl_up_sync(0xffffffff, a.x, offset, width); + const float t_y = __shfl_up_sync(0xffffffff, a.y, offset, width); + if (lane_id >= offset) { + a.x += t_x; + a.y += t_y; + } + } + return a; +} + +template +static __device__ __forceinline__ half2 warp_prefix_inclusive_sum(half2 a) { +#ifdef FP16_AVAILABLE + const int lane_id = threadIdx.x % width; +#pragma unroll + for (int offset = 1; offset < width; offset <<= 1) { + const half2 t = __shfl_up_sync(0xffffffff, a, offset, width); + if (lane_id >= offset) { + a = __hadd2(a, t); + } + } + return a; + +#else + NO_DEVICE_CODE; + return a; +#endif // FP16_AVAILABLE +} + +enum class block_reduce_method { + MAX, + SUM, +}; + +template +struct block_reduce_policy; + +template +inline constexpr bool is_any = (std::is_same_v || ...); + +template +inline constexpr bool ggml_cuda_dependent_false_v = false; + +template struct block_reduce_policy { + static __device__ T reduce(T val) { + if constexpr(is_any) { + return warp_reduce_sum(val); + } else { + static_assert(ggml_cuda_dependent_false_v, "Unsupported type for block reduce sum"); + } + } + + static __device__ T sentinel() { + if constexpr (std::is_same_v) { + return 0.0f; + } else if constexpr (std::is_same_v) { + return make_float2(0.0f, 0.0f); + } else if constexpr (std::is_same_v) { + return make_half2(0.0f, 0.0f); + } else if constexpr (std::is_same_v) { + return 0; + } else { + static_assert(ggml_cuda_dependent_false_v, "Unsupported type for block reduce sum"); + } + } +}; + +template struct block_reduce_policy { + static __device__ T reduce(T val) { + if constexpr (is_any) { + return warp_reduce_max(val); + } else { + static_assert(ggml_cuda_dependent_false_v, "Unsupported type for block reduce max"); + } + } + + static __device__ T sentinel() { + if constexpr (std::is_same_v) { + return -INFINITY; + } else if constexpr (std::is_same_v) { + return make_half2(-INFINITY, -INFINITY); + } else { + static_assert(ggml_cuda_dependent_false_v, "Unsupported type for block reduce max"); + } + } +}; + +template +static __device__ T block_reduce(T val, [[maybe_unused]] T * shared_vals) { + // for multi-warp reductions, callers must not reuse shared_vals until all reads from this invocation have completed + val = block_reduce_policy::reduce(val); + const unsigned int block_size = block_size_template == 0 ? blockDim.x : block_size_template; + if (block_size > WARP_SIZE) { + assert((block_size <= 1024) && (block_size % WARP_SIZE) == 0); + const int warp_id = threadIdx.x / WARP_SIZE; + const int lane_id = threadIdx.x % WARP_SIZE; + if (lane_id == 0) { + shared_vals[warp_id] = val; + } + __syncthreads(); + val = block_reduce_policy::sentinel(); + if (lane_id < (static_cast(block_size) / WARP_SIZE)) { + val = shared_vals[lane_id]; + } + return block_reduce_policy::reduce(val); + } + + return val; +} + +static __device__ __forceinline__ half ggml_cuda_hmax(const half a, const half b) { +#ifdef FP16_AVAILABLE + +#if !defined(GGML_USE_HIP) && CUDART_VERSION < CUDART_HMAX + return __float2half(fmaxf(__half2float(a), __half2float(b))); +#else + return __hmax(a, b); +#endif // !defined(GGML_USE_HIP) && CUDART_VERSION < CUDART_HMAX + +#else + NO_DEVICE_CODE; + GGML_UNUSED(b); + return a; +#endif // FP16_AVAILABLE +} + +static __device__ __forceinline__ half2 ggml_cuda_hmax2(const half2 a, const half2 b) { +#if defined(GGML_USE_HIP) + return half2(__hmax(a.x, b.x), __hmax(a.y, b.y)); +#elif CUDART_VERSION >= CUDART_HMAX + return __hmax2(a, b); +#else + half2 ret; + reinterpret_cast(ret.x) = __float2half(fmaxf( __low2float(a), __low2float(b))); + reinterpret_cast(ret.y) = __float2half(fmaxf(__high2float(a), __high2float(b))); + return ret; +#endif +} + +template +static __device__ __forceinline__ half2 warp_reduce_max(half2 x) { +#if !defined(GGML_USE_HIP) && __CUDA_ARCH__ >= GGML_CUDA_CC_PASCAL || defined(GGML_USE_HIP) +#pragma unroll + for (int offset = width/2; offset > 0; offset >>= 1) { + x = ggml_cuda_hmax2(x, __shfl_xor_sync(0xffffffff, x, offset, width)); + } + return x; +#else + GGML_UNUSED(x); + NO_DEVICE_CODE; +#endif // !defined(GGML_USE_HIP) && __CUDA_ARCH__ >= GGML_CUDA_CC_PASCAL || defined(GGML_USE_HIP) +} + +#if (defined(CUDART_VERSION) && CUDART_VERSION < CUDART_HMASK) || defined(GGML_USE_HIP) || \ + (defined(MUSART_VERSION) && MUSART_VERSION < MUSART_HMASK) +static __device__ __forceinline__ uint32_t __hgt2_mask(const half2 a, const half2 b) { + const uint32_t mask_low = 0x0000FFFF * (float( __low2half(a)) > float( __low2half(b))); + const uint32_t mask_high = 0xFFFF0000 * (float(__high2half(a)) > float(__high2half(b))); + return mask_low | mask_high; +} +#endif // (defined(CUDART_VERSION) && CUDART_VERSION < CUDART_HMASK) || defined(GGML_USE_HIP) || (defined(MUSART_VERSION) && MUSART_VERSION < MUSART_HMASK) + +static __device__ __forceinline__ int ggml_cuda_dp4a(const int a, const int b, int c) { +#if defined(GGML_USE_HIP) +#if defined(CDNA) || defined(RDNA2) || defined(__gfx906__) + c = __builtin_amdgcn_sdot4(a, b, c, false); +#elif defined(RDNA3) || defined(RDNA4) + c = __builtin_amdgcn_sudot4( true, a, true, b, c, false); +#elif defined(RDNA1) || defined(__gfx900__) + int tmp1; + int tmp2; + asm("\n \ + v_mul_i32_i24 %1, sext(%3), sext(%4) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:BYTE_0 \n \ + v_mul_i32_i24 %2, sext(%3), sext(%4) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_1 src1_sel:BYTE_1 \n \ + v_add3_u32 %0, %1, %2, %0 \n \ + v_mul_i32_i24 %1, sext(%3), sext(%4) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_2 src1_sel:BYTE_2 \n \ + v_mul_i32_i24 %2, sext(%3), sext(%4) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_3 src1_sel:BYTE_3 \n \ + v_add3_u32 %0, %1, %2, %0 \n \ + " + : "+v"(c), "=&v"(tmp1), "=&v"(tmp2) + : "v"(a), "v"(b) + ); +#else + const int8x4_t va = reinterpret_cast(a); + const int8x4_t vb = reinterpret_cast(b); + c += va[0] * vb[0] + va[1] * vb[1] + va[2] * vb[2] + va[3] * vb[3]; +#endif + return c; + +#else // defined(GGML_USE_HIP) + +#if __CUDA_ARCH__ >= GGML_CUDA_CC_DP4A || defined(GGML_USE_MUSA) + return __dp4a(a, b, c); +#else // __CUDA_ARCH__ >= GGML_CUDA_CC_DP4A || defined(GGML_USE_MUSA) + const int8_t * a8 = (const int8_t *) &a; + const int8_t * b8 = (const int8_t *) &b; + return c + a8[0]*b8[0] + a8[1]*b8[1] + a8[2]*b8[2] + a8[3]*b8[3]; +#endif // __CUDA_ARCH__ >= GGML_CUDA_CC_DP4A || defined(GGML_USE_MUSA) + +#endif // defined(GGML_USE_HIP) +} + +static __device__ __forceinline__ void ggml_cuda_mad(float & acc, const float v, const float u) { + acc += v*u; +} + +static __device__ __forceinline__ void ggml_cuda_mad(float & acc, const float2 v, const float2 u) { + acc += v.x*u.x; + acc += v.y*u.y; +} + +#if defined(GGML_USE_HIP) && (defined(RDNA2) || defined(RDNA3) || defined(RDNA4) || defined(__gfx906__) || defined(CDNA)) +#define V_DOT2_F32_F16_AVAILABLE +#endif // defined(GGML_USE_HIP) && (defined(RDNA2) || defined(RDNA3) || defined(RDNA4) || defined(__gfx906__) || defined(CDNA)) + +static __device__ __forceinline__ void ggml_cuda_mad(float & acc, const half2 v, const half2 u) { +#ifdef V_DOT2_F32_F16_AVAILABLE + asm volatile("v_dot2_f32_f16 %0, %1, %2, %0" : "+v"(acc) : "v"(v), "v"(u)); +#else +#ifdef FAST_FP16_AVAILABLE + const float2 tmp = __half22float2(v*u); + acc += tmp.x + tmp.y; +#else + const float2 tmpv = __half22float2(v); + const float2 tmpu = __half22float2(u); + acc += tmpv.x * tmpu.x; + acc += tmpv.y * tmpu.y; +#endif // FAST_FP16_AVAILABLE +#endif // V_DOT2_F32_F16_AVAILABLE +} + +static __device__ __forceinline__ void ggml_cuda_mad(half2 & acc, const half2 v, const half2 u) { +#ifdef FAST_FP16_AVAILABLE + acc += v*u; +#else + const float2 tmpv = __half22float2(v); + const float2 tmpu = __half22float2(u); + float2 tmpacc = __half22float2(acc); + tmpacc.x += tmpv.x * tmpu.x; + tmpacc.y += tmpv.y * tmpu.y; + acc = make_half2(tmpacc.x, tmpacc.y); +#endif // FAST_FP16_AVAILABLE +} + +// Aligned memory transfers of 8/16 bytes can be faster than 2 transfers with 4 bytes, especially on AMD. +// Important: do not use this function if dst and src both point at registers. +// Due to the strict aliasing rule the compiler can do incorrect optimizations if src and dst have different types. +// The function is intended for copies between registers and SRAM/VRAM to make the compiler emit the right instructions. +// If dst and src point at different address spaces then they are guaranteed to not be aliased. +template +static __device__ __forceinline__ void ggml_cuda_memcpy_1(void * __restrict__ dst, const void * __restrict__ src) { + static_assert( + nbytes <= ggml_cuda_get_max_cpy_bytes() || alignment == 0, + "You are misusing the alignment parameter for ggml_cuda_memcpy_1. " + "The intent is for the parameter is only as a workaround if either one of the pointers is not properly aligned. " + "If you use it to do more bytes per copy than ggml_cuda_max_cpy_bytes() the reads and writes may not be coalesced. " + "Call ggml_cuda_memcpy_1 in a loop instead."); + if constexpr (alignment != 0) { + static_assert(nbytes % alignment == 0, "bad alignment"); + } + constexpr int nb_per_cpy = alignment == 0 ? nbytes : alignment; + +#pragma unroll + for (int i = 0; i < nbytes/nb_per_cpy; ++i) { + if constexpr (nb_per_cpy == 1) { + ((char *) dst)[i] = ((const char *) src)[i]; + } else if constexpr (nb_per_cpy == 2) { + ((short *) dst)[i] = ((const short *) src)[i]; + } else if constexpr (nb_per_cpy == 4) { + ((int *) dst)[i] = ((const int *) src)[i]; + } else if constexpr (nb_per_cpy == 8) { + ((int2 *) dst)[i] = ((const int2 *) src)[i]; + } else if constexpr (nb_per_cpy == 16) { + ((int4 *) dst)[i] = ((const int4 *) src)[i]; + } else { + static_assert(nbytes == 0 && nbytes == -1, "bad nbytes"); + } + } +} + +static __device__ __forceinline__ float ggml_cuda_e8m0_to_fp32(uint8_t x) { +#if CUDART_VERSION >= 12080 + const nv_bfloat16 e = __nv_cvt_e8m0_to_bf16raw(x); + return (float) e; +#else + uint32_t bits; + if (x == 0) { + bits = 0x00400000; + } else { + bits = (uint32_t) x << 23; + } + + float result; + memcpy(&result, &bits, sizeof(float)); + return result; +#endif // CUDART_VERSION >= 12050 +} + +static __device__ __forceinline__ float ggml_cuda_ue4m3_to_fp32(uint8_t x) { +#if defined(GGML_USE_HIP) && defined(CDNA3) && defined(FP8_AVAILABLE) && HIP_VERSION >= 60200000 + // ROCm does not support fp8 in software on devices with fp8 hardware, + // but CDNA3 supports only e4m3_fnuz (no inf). + const uint32_t bits = x * (x != 0x7F && x != 0xFF); // Convert NaN to 0.0f to match CPU implementation. + const __hip_fp8_e4m3_fnuz xf = *reinterpret_cast(&bits); + return static_cast(xf) / 2; +#else +#if defined(FP8_AVAILABLE) && !defined(GGML_USE_HIP) + const uint32_t bits = x * (x != 0x7F && x != 0xFF); // Convert NaN to 0.0f to match CPU implementation. + const __nv_fp8_e4m3 xf = *reinterpret_cast(&bits); + return static_cast(xf) / 2; +#else + if (x == 0 || (x == 0x7F && x != 0xFF)) { // Convert NaN to 0.0f + return 0.0f; + } + const int exp = (x >> 3) & 0xF; + const int man = x & 0x7; + float raw; + if (exp == 0) { + raw = ldexpf((float) man, -9); + } else { + raw = ldexpf(1.0f + (float) man / 8.0f, exp - 7); + } + return static_cast(raw / 2); +#endif // defined(FP8_AVAILABLE) && !defined(GGML_USE_HIP) +#endif // defined(GGML_USE_HIP) && defined(CDNA3) && defined(FP8_AVAILABLE) && HIP_VERSION >= 60200000 +} + +static __device__ __forceinline__ uint8_t ggml_cuda_fp32_to_ue4m3(float x) { +#if defined(BLACKWELL_MMA_AVAILABLE) // This is used for NVFP4 subblock scale quantizations only + if (!(x > 0.0f)) { + return 0; + } + const __nv_fp8_e4m3 xf(x); + return xf.__x; +#else + NO_DEVICE_CODE; // Used only for NVFP4 Scales for Activations, only for Blackwell +#endif // defined(BLACKWELL_MMA_AVAILABLE) +} + +__device__ __forceinline__ uint8_t ggml_cuda_float_to_fp4_e2m1(float x, float e) { + const uint8_t sign_bit = (x < 0.0f) << 3; + float ax = fabsf(x) * e; + + // Positive LUT + static constexpr float pos_lut[8] = { 0.0f, 0.5f, 1.0f, 1.5f, 2.0f, 3.0f, 4.0f, 6.0f }; + + int best_i = 0; + float best_err = fabsf(ax - pos_lut[0]); + +#pragma unroll + for (int i = 1; i < 8; ++i) { + const float err = fabsf(ax - pos_lut[i]); + if (err < best_err) { + best_err = err; + best_i = i; + } + } + + return static_cast(best_i | sign_bit); +} + +// See https://gmplib.org/~tege/divcnst-pldi94.pdf figure 4.1. +// Precompute mp (m' in the paper) and L such that division +// can be computed using a multiply (high 32b of 64b result) +// and a shift: +// +// n/d = (mulhi(n, mp) + n) >> L; +static const uint3 init_fastdiv_values(uint64_t d_64) { + GGML_ASSERT(d_64 != 0); + GGML_ASSERT(d_64 <= std::numeric_limits::max()); + + uint32_t d = (uint32_t)d_64; + + // compute L = ceil(log2(d)); + uint32_t L = 0; + while (L < 32 && (uint32_t{ 1 } << L) < d) { + L++; + } + + uint32_t mp = (uint32_t) ((uint64_t{ 1 } << 32) * ((uint64_t{ 1 } << L) - d) / d + 1); + // pack divisor as well to reduce error surface + return make_uint3(mp, L, d); +} + +static __device__ __forceinline__ uint32_t fastdiv(uint32_t n, const uint3 fastdiv_values) { + // expects fastdiv_values to contain in + // fastdiv_values.z is unused and optimized away by the compiler. + // Compute high 32 bits of n * mp + const uint32_t hi = __umulhi(n, fastdiv_values.x); + // add n, apply bit shift + return (hi + n) >> fastdiv_values.y; +} + +static __device__ __forceinline__ uint32_t fastmodulo(uint32_t n, const uint3 fastdiv_values) { + // expects fastdiv_values to contain in (see init_fastdiv_values) + return n - fastdiv(n, fastdiv_values) * fastdiv_values.z; +} + +// Calculate both division and modulo at once, returns +static __device__ __forceinline__ uint2 fast_div_modulo(uint32_t n, const uint3 fastdiv_values) { + // expects fastdiv_values to contain in (see init_fastdiv_values) + const uint32_t div_val = fastdiv(n, fastdiv_values); + const uint32_t mod_val = n - div_val * fastdiv_values.z; + return make_uint2(div_val, mod_val); +} + +typedef void (*dequantize_kernel_t)(const void * vx, const int64_t ib, const int iqs, float2 & v); + +template +using dequantize_kq_t = void (*)(const void * vx, const int64_t ib, dst_t * y, const int tid); + +static __device__ __forceinline__ float get_alibi_slope( + const float max_bias, const uint32_t h, const uint32_t n_head_log2, const float m0, const float m1 +) { + if (max_bias <= 0.0f) { + return 1.0f; + } + const float base = h < n_head_log2 ? m0 : m1; + const int exph = h < n_head_log2 ? h + 1 : 2*(h - n_head_log2) + 1; + + return powf(base, exph); +} + +template +struct ggml_cuda_type_traits; + +template<> +struct ggml_cuda_type_traits { + static constexpr int qk = 1; + static constexpr int qr = 1; +}; + +template<> +struct ggml_cuda_type_traits { + static constexpr int qk = QK1_0; + static constexpr int qr = QR1_0; + static constexpr int qi = QI1_0; +}; + +template<> +struct ggml_cuda_type_traits { + static constexpr int qk = QK2_0; + static constexpr int qr = QR2_0; + static constexpr int qi = QI2_0; +}; + +template<> +struct ggml_cuda_type_traits { + static constexpr int qk = QK4_0; + static constexpr int qr = QR4_0; + static constexpr int qi = QI4_0; +}; + +template<> +struct ggml_cuda_type_traits { + static constexpr int qk = QK4_1; + static constexpr int qr = QR4_1; + static constexpr int qi = QI4_1; +}; + +template<> +struct ggml_cuda_type_traits { + static constexpr int qk = QK5_0; + static constexpr int qr = QR5_0; + static constexpr int qi = QI5_0; +}; + +template<> +struct ggml_cuda_type_traits { + static constexpr int qk = QK5_1; + static constexpr int qr = QR5_1; + static constexpr int qi = QI5_1; +}; + +template<> +struct ggml_cuda_type_traits { + static constexpr int qk = QK8_0; + static constexpr int qr = QR8_0; + static constexpr int qi = QI8_0; +}; + +template<> +struct ggml_cuda_type_traits { + static constexpr int qk = QK_MXFP4; + static constexpr int qr = QR_MXFP4; + static constexpr int qi = QI_MXFP4; +}; + +template<> +struct ggml_cuda_type_traits { + static constexpr int qk = QK_NVFP4; + static constexpr int qr = QR_NVFP4; + static constexpr int qi = QI_NVFP4; +}; + +template<> +struct ggml_cuda_type_traits { + static constexpr int qk = QK_K; + static constexpr int qr = QR2_K; + static constexpr int qi = QI2_K; +}; + +template<> +struct ggml_cuda_type_traits { + static constexpr int qk = QK_K; + static constexpr int qr = QR3_K; + static constexpr int qi = QI3_K; +}; + +template<> +struct ggml_cuda_type_traits { + static constexpr int qk = QK_K; + static constexpr int qr = QR4_K; + static constexpr int qi = QI4_K; +}; + +template<> +struct ggml_cuda_type_traits { + static constexpr int qk = QK_K; + static constexpr int qr = QR5_K; + static constexpr int qi = QI5_K; +}; + +template<> +struct ggml_cuda_type_traits { + static constexpr int qk = QK_K; + static constexpr int qr = QR6_K; + static constexpr int qi = QI6_K; +}; + +template<> +struct ggml_cuda_type_traits { + static constexpr int qk = QK_K; + static constexpr int qr = QR2_XXS; + static constexpr int qi = QI2_XXS; +}; + +template<> +struct ggml_cuda_type_traits { + static constexpr int qk = QK_K; + static constexpr int qr = QR2_XS; + static constexpr int qi = QI2_XS; +}; + +template<> +struct ggml_cuda_type_traits { + static constexpr int qk = QK_K; + static constexpr int qr = QR2_S; + static constexpr int qi = QI2_S; +}; + +template<> +struct ggml_cuda_type_traits { + static constexpr int qk = QK_K; + static constexpr int qr = QR3_XXS; + static constexpr int qi = QI3_XXS; +}; + +template<> +struct ggml_cuda_type_traits { + static constexpr int qk = QK_K; + static constexpr int qr = QR1_S; + static constexpr int qi = QI1_S; +}; + +template<> +struct ggml_cuda_type_traits { + static constexpr int qk = QK_K; + static constexpr int qr = QR1_M; + static constexpr int qi = QI1_M; +}; + +template<> +struct ggml_cuda_type_traits { + static constexpr int qk = QK4_NL; + static constexpr int qr = QR4_NL; + static constexpr int qi = QI4_NL; +}; + +template<> +struct ggml_cuda_type_traits { + static constexpr int qk = QK_K; + static constexpr int qr = QR4_XS; + static constexpr int qi = QI4_XS; +}; + +template<> +struct ggml_cuda_type_traits { + static constexpr int qk = QK_K; + static constexpr int qr = QR3_S; + static constexpr int qi = QI3_S; +}; + +////////////////////// + +struct ggml_cuda_device_info { + int device_count; // number of (possibly virtual) devices exposed to the rest of ggml + int physical_device_count; // number of physical CUDA devices actually present + + struct cuda_device_info { + int cc; // compute capability + int nsm; // number of streaming multiprocessors + size_t smpb; // max. shared memory per block + size_t smpbo; // max. shared memory per block (with opt-in) + bool integrated; // Device is integrated as opposed to discrete + bool vmm; // virtual memory support + size_t vmm_granularity; // granularity of virtual memory + size_t total_vram; + int warp_size; // Number of threads in a dispatch + bool supports_cooperative_launch; // whether cooperative launch is supported + int physical_device; // backing physical CUDA device for this (virtual) device + int physical_share_count; // number of (virtual) devices sharing this device's physical GPU + int virtual_index; // index of this (virtual) device among those sharing its physical GPU + }; + + cuda_device_info devices[GGML_CUDA_MAX_DEVICES] = {}; + + std::array default_tensor_split = {}; +}; + +const ggml_cuda_device_info & ggml_cuda_info(); + +void ggml_cuda_set_device(int device); +int ggml_cuda_get_device(); + +struct ggml_cuda_pool { + virtual ~ggml_cuda_pool() = default; + + virtual void * alloc(size_t size, size_t * actual_size) = 0; + virtual void free(void * ptr, size_t size) = 0; +}; + +template +struct ggml_cuda_pool_alloc { + ggml_cuda_pool * pool = nullptr; + T * ptr = nullptr; + size_t actual_size = 0; + + ggml_cuda_pool_alloc() = default; + + explicit ggml_cuda_pool_alloc(ggml_cuda_pool & pool) : pool(&pool) { + } + + ggml_cuda_pool_alloc(ggml_cuda_pool & pool, size_t size) : pool(&pool) { + alloc(size); + } + + ~ggml_cuda_pool_alloc() { + if (ptr != nullptr) { + pool->free(ptr, actual_size); + } + } + + // size is in number of elements + T * alloc(size_t size) { + GGML_ASSERT(pool != nullptr); + GGML_ASSERT(ptr == nullptr); + ptr = (T *) pool->alloc(size * sizeof(T), &this->actual_size); + return ptr; + } + + T * alloc(ggml_cuda_pool & pool, size_t size) { + this->pool = &pool; + return alloc(size); + } + + T * get() { + return ptr; + } + + ggml_cuda_pool_alloc(const ggml_cuda_pool_alloc &) = delete; + ggml_cuda_pool_alloc(ggml_cuda_pool_alloc &&) = delete; + ggml_cuda_pool_alloc& operator=(const ggml_cuda_pool_alloc &) = delete; + ggml_cuda_pool_alloc& operator=(ggml_cuda_pool_alloc &&) = delete; +}; + + +// backend interface + +struct ggml_tensor_extra_gpu { + void * data_device[GGML_CUDA_MAX_DEVICES]; // 1 pointer for each device for split tensors + cudaEvent_t events[GGML_CUDA_MAX_DEVICES][GGML_CUDA_MAX_STREAMS]; // events for synchronizing multiple GPUs +}; + + +#if (defined(GGML_CUDA_USE_GRAPHS) || defined(GGML_HIP_GRAPHS)) || defined(GGML_MUSA_GRAPHS) +#define USE_CUDA_GRAPH +#endif + +struct ggml_cuda_graph { +#ifdef USE_CUDA_GRAPH + ~ggml_cuda_graph() { + if (instance != nullptr) { + CUDA_CHECK(cudaGraphExecDestroy(instance)); + } + if (graph != nullptr) { + CUDA_CHECK(cudaGraphDestroy(graph)); + } + } + cudaGraph_t graph = nullptr; + cudaGraphExec_t instance = nullptr; + size_t num_nodes = 0; + std::vector nodes; + bool disable_due_to_gpu_arch = false; + bool warmup_complete = false; + uint64_t uid = 0; + int64_t last_used_time = 0; + struct node_properties { + ggml_tensor node; + void * node_src_data_ptrs[GGML_MAX_SRC]; + int64_t node_src_ne[GGML_MAX_SRC][GGML_MAX_DIMS]; + size_t node_src_nb[GGML_MAX_SRC][GGML_MAX_DIMS]; + }; + std::vector node_props; + + bool is_enabled() const { + static const bool disable_cuda_graphs_due_to_env = (getenv("GGML_CUDA_DISABLE_GRAPHS") != nullptr); + return !(disable_due_to_gpu_arch || disable_cuda_graphs_due_to_env); + } +#endif +}; + +struct ggml_cuda_concurrent_event { + std::vector join_events; + cudaEvent_t fork_event = nullptr; + + int n_streams = 0; + std::unordered_map stream_mapping; + + // Original order of nodes in this concurrent region (before interleaving) + // Used to restore grouping for fusion within streams + std::vector original_order; + + const ggml_tensor * join_node; + + ggml_cuda_concurrent_event() = default; + + ggml_cuda_concurrent_event(const ggml_cuda_concurrent_event &) = delete; + ggml_cuda_concurrent_event & operator=(const ggml_cuda_concurrent_event &) = delete; + + explicit ggml_cuda_concurrent_event(int n_streams) : n_streams(n_streams) { + join_events.resize(n_streams); + + for (size_t i = 0; i < join_events.size(); ++i) { + CUDA_CHECK(cudaEventCreateWithFlags(&join_events[i], cudaEventDisableTiming)); + } + + CUDA_CHECK(cudaEventCreateWithFlags(&fork_event, cudaEventDisableTiming)); + } + + ggml_cuda_concurrent_event(ggml_cuda_concurrent_event && other) noexcept + : join_events(std::move(other.join_events)) + , fork_event(other.fork_event) + , n_streams(other.n_streams) + , stream_mapping(std::move(other.stream_mapping)) + , original_order(std::move(other.original_order)) + , join_node(other.join_node) { + other.fork_event = nullptr; + } + + // 1. check if any branches write to overlapping memory ranges (except the join node) + // 2. check whether all srcs are either within the branch or outside the nodes covered by ggml_cuda_concurrent_event + // we assume all nodes have the same buffer + bool is_valid() const { + std::vector>> write_ranges; + write_ranges.resize(n_streams); + + // get join_node's memory range to exclude from overlap checking. + // multiple nodes can use join_node's buffer; we synchronize on the join node. + const ggml_tensor * join_t = join_node->view_src ? join_node->view_src : join_node; + const int64_t join_start = (int64_t) join_t->data; + const int64_t join_end = join_start + ggml_nbytes(join_t); + + for (const auto & [tensor, stream] : stream_mapping) { + const ggml_tensor * t = tensor->view_src ? tensor->view_src : tensor; + const int64_t t_start = (int64_t) t->data; + const int64_t t_end = t_start + ggml_nbytes(t); + + // skip tensors that overlap with join_node's buffer. + if ((t_start <= join_start && join_start < t_end) || (join_start <= t_start && t_start < join_end)) { + continue; + } + + // concurrent streams begin from 1 + write_ranges[stream - 1].emplace_back(t_start, t_end); + } + + for (int i = 0; i < n_streams; ++i) { + // sorts first by start then by end of write range + std::sort(write_ranges[i].begin(), write_ranges[i].end()); + } + + bool writes_overlap = false; + bool dependent_srcs = false; + for (const auto & [tensor, stream] : stream_mapping) { + const ggml_tensor * t = tensor->view_src ? tensor->view_src : tensor; + const int64_t t_start = (int64_t) t->data; + const int64_t t_end = t_start + ggml_nbytes(t); + + // skip tensors that overlap with join_node's buffer + if ((t_start <= join_start && join_start < t_end) || (join_start <= t_start && t_start < join_end)) { + continue; + } + + // check if this buffer's write data overlaps with another stream's + std::pair data_range = std::make_pair(t_start, t_end); + for (int i = 0; i < n_streams; ++i) { + if (i == stream - 1) { + continue; + } + auto it = std::lower_bound(write_ranges[i].begin(), write_ranges[i].end(), data_range); + + if (it != write_ranges[i].end()) { + const std::pair & other = *it; + + // std::lower_bound returns the first element where other >= data_range (lexicographically). + // This guarantees other.first >= data_range.first. + // Therefore, overlap occurs iff other.first < data_range.second + // (i.e., the other range starts before this range ends). + if (other.first < data_range.second) { + GGML_LOG_DEBUG("Writes overlap for %s", tensor->name); + writes_overlap = true; + break; + } + } + } + + //check if all srcs are either in branch or don't have a branch + for (int i = 0; i < GGML_MAX_SRC; ++i) { + if (!tensor->src[i]) { + continue; + } + + auto it = stream_mapping.find(tensor->src[i]); + + if (it == stream_mapping.end()) { + continue; + } + + if (it->second != stream) { + dependent_srcs = true; + break; + } + } + + if (dependent_srcs || writes_overlap) { + break; + } + } + + return !writes_overlap && !dependent_srcs; + } + + ~ggml_cuda_concurrent_event() { + if (fork_event != nullptr) { + CUDA_CHECK(cudaEventDestroy(fork_event)); + } + for (cudaEvent_t e : join_events) { + if (e != nullptr) { + CUDA_CHECK(cudaEventDestroy(e)); + } + } + } +}; + +struct ggml_cuda_stream_context { + std::unordered_map concurrent_events; + + void reset() { + concurrent_events.clear(); + } +}; + +struct ggml_backend_cuda_context { + int device; + std::string name; + cudaEvent_t copy_event = nullptr; + + cudaStream_t streams[GGML_CUDA_MAX_DEVICES][GGML_CUDA_MAX_STREAMS] = { { nullptr } }; + cublasHandle_t cublas_handles[GGML_CUDA_MAX_DEVICES][GGML_CUDA_MAX_STREAMS] = {nullptr}; + void * cublas_workspaces[GGML_CUDA_MAX_DEVICES][GGML_CUDA_MAX_STREAMS] = {nullptr}; + size_t cublas_workspace_sizes[GGML_CUDA_MAX_DEVICES] = {0}; + + int curr_stream_no = 0; + +#ifdef USE_CUDA_GRAPH + // Map from first_node_ptr to cuda_graph - allows multiple graphs per context + // when the computation is split across CPU/GPU (e.g., with --n-cpu-moe) + std::unordered_map> cuda_graphs; + + int64_t last_graph_eviction_sweep = 0; + + ggml_cuda_graph * cuda_graph(const void * first_node_ptr) { + const int64_t time_now = ggml_time_us(); + + // sweep every 5s, evicting cuda graphs unused for >=10s + if (time_now - last_graph_eviction_sweep >= 5'000'000) { + last_graph_eviction_sweep = time_now; + for (auto it = cuda_graphs.begin(); it != cuda_graphs.end(); ) { + if (time_now - it->second->last_used_time >= 10'000'000) { + it = cuda_graphs.erase(it); + } else { + ++it; + } + } + } + + auto it = cuda_graphs.find(first_node_ptr); + if (it == cuda_graphs.end()) { + it = cuda_graphs.emplace(first_node_ptr, std::make_unique()).first; + } + it->second->last_used_time = time_now; + return it->second.get(); + } + + // Check if any CUDA graph is enabled for this context (used by kernels that need to know + // if graphs are in use without having access to the specific graph key) + bool any_cuda_graph_enabled() const { + for (const auto & [key, graph] : cuda_graphs) { + if (graph && graph->is_enabled()) { + return true; + } + } + return false; + } + + // Check if any CUDA graph has an instance for this context + bool any_cuda_graph_has_instance() const { + for (const auto & [key, graph] : cuda_graphs) { + if (graph && graph->instance != nullptr) { + return true; + } + } + return false; + } +#endif // USE_CUDA_GRAPH + + explicit ggml_backend_cuda_context(int device) : + device(device), + name(GGML_CUDA_NAME + std::to_string(device)) { + } + + ggml_cuda_stream_context concurrent_stream_context; + + ~ggml_backend_cuda_context(); + + cudaStream_t stream(int device, int stream) { + if (streams[device][stream] == nullptr) { + ggml_cuda_set_device(device); + CUDA_CHECK(cudaStreamCreateWithFlags(&streams[device][stream], cudaStreamNonBlocking)); + } + return streams[device][stream]; + } + + cudaStream_t stream() { return stream(device, curr_stream_no); } + + ggml_cuda_stream_context & stream_context() { return concurrent_stream_context; } + + cublasHandle_t cublas_handle() { + if (cublas_handles[device][curr_stream_no] == nullptr) { + ggml_cuda_set_device(device); + CUBLAS_CHECK(cublasCreate(&cublas_handles[device][curr_stream_no])); + CUBLAS_CHECK(cublasSetMathMode(cublas_handles[device][curr_stream_no], CUBLAS_TF32_TENSOR_OP_MATH)); + CUBLAS_CHECK(cublasSetStream(cublas_handles[device][curr_stream_no], stream())); +#if !defined(GGML_USE_HIP) && !defined(GGML_USE_MUSA) && (CUBLAS_VER_MAJOR > 11 || (CUBLAS_VER_MAJOR == 11 && CUBLAS_VER_MINOR >= 2)) + if (cublas_workspace_sizes[device] == 0) { + const int cc = ggml_cuda_info().devices[device].cc; + cublas_workspace_sizes[device] = (cc >= GGML_CUDA_CC_HOPPER) ? 32 * 1024 * 1024 : 4 * 1024 * 1024; + } + CUDA_CHECK(cudaMalloc(&cublas_workspaces[device][curr_stream_no], cublas_workspace_sizes[device])); + CUBLAS_CHECK(cublasSetWorkspace(cublas_handles[device][curr_stream_no], cublas_workspaces[device][curr_stream_no], cublas_workspace_sizes[device])); +#endif + } + return cublas_handles[device][curr_stream_no]; + } + + // pool + std::unique_ptr pools[GGML_CUDA_MAX_DEVICES][GGML_CUDA_MAX_STREAMS]; + + static std::unique_ptr new_pool_for_device(int device, int stream_no); + + ggml_cuda_pool & pool(int device) { + if (pools[device][curr_stream_no] == nullptr) { + pools[device][curr_stream_no] = new_pool_for_device(device, curr_stream_no); + } + return *pools[device][curr_stream_no]; + } + + ggml_cuda_pool & pool() { + return pool(device); + } +}; + +struct ggml_cuda_mm_fusion_args_host { + const ggml_tensor * x_bias = nullptr; + const ggml_tensor * gate = nullptr; + const ggml_tensor * gate_bias = nullptr; + const ggml_tensor * x_scale = nullptr; + const ggml_tensor * gate_scale = nullptr; + ggml_glu_op glu_op; +}; +struct ggml_cuda_mm_fusion_args_device { + const void * x_bias = nullptr; + const void * gate = nullptr; + const void * gate_bias = nullptr; + const void * x_scale = nullptr; + const void * gate_scale = nullptr; + ggml_glu_op glu_op; +}; + +struct ggml_cuda_kernel_launch_params { + dim3 block_nums; + dim3 block_dims; + size_t shmem; + cudaStream_t stream; + + // size_t shmem + ggml_cuda_kernel_launch_params(const dim3& block_nums_, const dim3& block_dims_, const size_t shmem_, const cudaStream_t stream_) + : block_nums(block_nums_), block_dims(block_dims_), shmem(shmem_), stream(stream_) {} + + // Some call sites pass ints instead of the required size_t. This 2nd constructor casts int->size_t to avoid these -Wnarrowing warnings. + ggml_cuda_kernel_launch_params(const dim3& block_nums_, const dim3& block_dims_, const int shmem_, const cudaStream_t stream_) + : block_nums(block_nums_), block_dims(block_dims_), shmem((size_t)shmem_), stream(stream_) {} +}; + +#if defined(GGML_CUDA_USE_PDL) +struct ggml_cuda_pdl_config { + cudaLaunchAttribute attr; + cudaLaunchConfig_t cfg; + + ggml_cuda_pdl_config(const ggml_cuda_kernel_launch_params & params) { + attr.id = cudaLaunchAttributeProgrammaticStreamSerialization; + attr.val.programmaticStreamSerializationAllowed = 1; + + cfg = {}; + cfg.gridDim = params.block_nums; + cfg.blockDim = params.block_dims; + cfg.dynamicSmemBytes = params.shmem; + cfg.stream = params.stream; + cfg.attrs = &attr; + cfg.numAttrs = 1; + } + + // Delete due to &attr + ggml_cuda_pdl_config(const ggml_cuda_pdl_config&) = delete; + ggml_cuda_pdl_config& operator=(const ggml_cuda_pdl_config&) = delete; + ggml_cuda_pdl_config& operator=(ggml_cuda_pdl_config&&) = delete; + +}; + +static bool ggml_cuda_kernel_can_use_pdl(const void * kernel) { + const int device = ggml_cuda_get_device(); + + struct cache_key { + int device; + const void * kernel; + + bool operator==(const cache_key & other) const { return device == other.device && kernel == other.kernel; } + }; + + struct cache_key_hash { + // MurmurHash3 mixing function for better hash distribution (vs. just std::hash which in some implementations simply returns the identity) + static size_t hash_mix(size_t x) { + std::uint64_t y = x; + const std::uint64_t m = 0xe9846af9b1a615d; + + y ^= y >> 32; + y *= m; + y ^= y >> 32; + y *= m; + y ^= y >> 28; + + return static_cast(y); + } + + size_t operator()(const cache_key & key) const { + // Use a nonzero seed to avoid mapping all-zero keys to zero + size_t h = 42; + h = hash_mix(h + key.device); + h = hash_mix(h + reinterpret_cast(key.kernel)); + return h; + } + }; + + static std::mutex cache_mutex; + static std::unordered_map cache; + + const cache_key key = { device, kernel }; + std::lock_guard lock(cache_mutex); + const auto it = cache.find(key); + if (it != cache.end()) { + return it->second; + } + + cudaFuncAttributes attr = {}; + CUDA_CHECK(cudaFuncGetAttributes(&attr, kernel)); + + // PDL device-side primitives are emitted only for PTX versions >= 90. + // We have to guard on a loaded kernel's PTX version so a kernel forward-JIT'ed + // from pre-Hopper PTX to a Hopper-or-newer GPU does not opt into PDL. + const bool can_use_pdl = attr.ptxVersion >= 90; + cache.emplace(key, can_use_pdl); + return can_use_pdl; +} + +#endif //defined(GGML_CUDA_USE_PDL) + +// PDL and __restrict__ need to be mutually exclusive, see https://github.com/ggml-org/llama.cpp/pull/24030 +# if (defined(GGML_CUDA_USE_PDL) && defined(__CUDA_ARCH__) && __CUDA_ARCH__ >= GGML_CUDA_CC_HOPPER) +# define GGML_CUDA_RESTRICT +# else +# define GGML_CUDA_RESTRICT __restrict__ +# endif // defined(GGML_CUDA_USE_PDL) && defined(__CUDA_ARCH__) && __CUDA_ARCH__ >= GGML_CUDA_CC_HOPPER + +template +static __inline__ void ggml_cuda_kernel_launch(Kernel kernel, const ggml_cuda_kernel_launch_params & launch_params, Args&&... args) { +#if defined(GGML_CUDA_USE_PDL) + + static const bool env_pdl_enabled = []() { + const char * env = getenv("GGML_CUDA_PDL"); + return env == nullptr || std::atoi(env) != 0; + }(); + + if (env_pdl_enabled && ggml_cuda_kernel_can_use_pdl(reinterpret_cast(kernel))) { + auto pdl_cfg = ggml_cuda_pdl_config(launch_params); + + CUDA_CHECK(cudaLaunchKernelEx(&pdl_cfg.cfg, kernel, std::forward(args)... )); + return; + } +#endif //defined(GGML_CUDA_USE_PDL) + + kernel<<>>(std::forward(args)... ); + CUDA_CHECK(cudaGetLastError()); +} + diff --git a/python/freetoken/kernel/csrc/gguf_mmq/cp-async.cuh b/python/freetoken/kernel/csrc/gguf_mmq/cp-async.cuh new file mode 100644 index 00000000..63d0c482 --- /dev/null +++ b/python/freetoken/kernel/csrc/gguf_mmq/cp-async.cuh @@ -0,0 +1,57 @@ +// Simplified API for asynchronous data loading. + +#include "common.cuh" + + +static __device__ __forceinline__ unsigned int ggml_cuda_cvta_generic_to_shared(void * generic_ptr) { +#ifdef CP_ASYNC_AVAILABLE + return __cvta_generic_to_shared(generic_ptr); +#else + GGML_UNUSED(generic_ptr); + NO_DEVICE_CODE; + return 0; +#endif // CP_ASYNC_AVAILABLE +} + +// Copies data from global to shared memory, cg == cache global. +// Both the src and dst pointers must be aligned to 16 bit. +// Shared memory uses 32 bit addressing, the pointer is passed as unsigned int. +// Generic pointers can be converted to 32 bit shared memory pointers using __cvta_generic_to_shared. +// Only the 16 bit copy is exposed because 4 and 8 bit copies did not yield performance improvements. +template +static __device__ __forceinline__ void cp_async_cg_16(const unsigned int dst, const void * src) { + static_assert(preload == 0 || preload == 64 || preload == 128 || preload == 256, "bad preload"); +#ifdef CP_ASYNC_AVAILABLE +#if CUDART_VERSION >= 11040 + if (preload == 256) { + asm volatile("cp.async.cg.shared.global.L2::256B [%0], [%1], 16;" + : : "r"(dst), "l"(src)); + } else if (preload == 128) { + asm volatile("cp.async.cg.shared.global.L2::128B [%0], [%1], 16;" + : : "r"(dst), "l"(src)); + } else if (preload == 64) { + asm volatile("cp.async.cg.shared.global.L2::64B [%0], [%1], 16;" + : : "r"(dst), "l"(src)); + } else +#endif // CUDART_VERSION >= 11040 + { + asm volatile("cp.async.cg.shared.global [%0], [%1], 16;" + : : "r"(dst), "l"(src)); + } +#else + GGML_UNUSED(dst); + GGML_UNUSED(src); + NO_DEVICE_CODE; +#endif // CP_ASYNC_AVAILABLE +} + +// Makes each thread wait until its asynchronous data copies are done. +// This does NOT provide any additional synchronization. +// In particular, when copying data with multiple warps a call to __syncthreads will be needed. +static __device__ __forceinline__ void cp_async_wait_all() { +#ifdef CP_ASYNC_AVAILABLE + asm volatile("cp.async.wait_all;"); +#else + NO_DEVICE_CODE; +#endif // CP_ASYNC_AVAILABLE +} diff --git a/python/freetoken/kernel/csrc/gguf_mmq/ggml-alloc.h b/python/freetoken/kernel/csrc/gguf_mmq/ggml-alloc.h new file mode 100644 index 00000000..a7926a21 --- /dev/null +++ b/python/freetoken/kernel/csrc/gguf_mmq/ggml-alloc.h @@ -0,0 +1,86 @@ +#pragma once + +#include "ggml.h" + +#ifdef __cplusplus +extern "C" { +#endif + +typedef struct ggml_backend_buffer_type * ggml_backend_buffer_type_t; +typedef struct ggml_backend_buffer * ggml_backend_buffer_t; +typedef struct ggml_backend * ggml_backend_t; + +// Tensor allocator +struct ggml_tallocr { + ggml_backend_buffer_t buffer; + void * base; + size_t alignment; + size_t offset; +}; + +GGML_API struct ggml_tallocr ggml_tallocr_new(ggml_backend_buffer_t buffer); +GGML_API enum ggml_status ggml_tallocr_alloc(struct ggml_tallocr * talloc, struct ggml_tensor * tensor); + +// Graph allocator +/* + Example usage: + ggml_gallocr_t galloc = ggml_gallocr_new(ggml_backend_cpu_buffer_type()); + + // optional: create a worst-case graph and reserve the buffers to avoid reallocations + ggml_gallocr_reserve(galloc, build_graph(max_batch)); + + // allocate the graph + struct ggml_cgraph * graph = build_graph(batch); + ggml_gallocr_alloc_graph(galloc, graph); + + printf("compute buffer size: %zu bytes\n", ggml_gallocr_get_buffer_size(galloc, 0)); + + // evaluate the graph + ggml_backend_graph_compute(backend, graph); +*/ + +// special tensor flags for use with the graph allocator: +// ggml_set_input(): all input tensors are allocated at the beginning of the graph in non-overlapping addresses +// ggml_set_output(): output tensors are never freed and never overwritten + +typedef struct ggml_gallocr * ggml_gallocr_t; + +GGML_API ggml_gallocr_t ggml_gallocr_new(ggml_backend_buffer_type_t buft); +GGML_API ggml_gallocr_t ggml_gallocr_new_n(ggml_backend_buffer_type_t * bufts, int n_bufs); +GGML_API void ggml_gallocr_free(ggml_gallocr_t galloc); + +// pre-allocate buffers from a measure graph - does not allocate or modify the graph +// call with a worst-case graph to avoid buffer reallocations +// not strictly required for single buffer usage: ggml_gallocr_alloc_graph will reallocate the buffers automatically if needed +// returns false if the buffer allocation failed +// ggml_gallocr_resrve_n_size writes the buffer sizes per galloc buffer that would be allocated by ggml_gallocr_reserve_n to sizes +GGML_API bool ggml_gallocr_reserve(ggml_gallocr_t galloc, struct ggml_cgraph * graph); +GGML_API void ggml_gallocr_reserve_n_size( + ggml_gallocr_t galloc, + struct ggml_cgraph * graph, + const int * node_buffer_ids, + const int * leaf_buffer_ids, + size_t * sizes); +GGML_API bool ggml_gallocr_reserve_n( + ggml_gallocr_t galloc, + struct ggml_cgraph * graph, + const int * node_buffer_ids, + const int * leaf_buffer_ids); + +// automatic reallocation if the topology changes when using a single buffer +// returns false if using multiple buffers and a re-allocation is needed (call ggml_gallocr_reserve_n first to set the node buffers) +GGML_API bool ggml_gallocr_alloc_graph(ggml_gallocr_t galloc, struct ggml_cgraph * graph); + +GGML_API size_t ggml_gallocr_get_buffer_size(ggml_gallocr_t galloc, int buffer_id); + +// Utils +// Create a buffer and allocate all the tensors in a ggml_context +// ggml_backend_alloc_ctx_tensors_from_buft_size returns the size of the buffer that would be allocated by ggml_backend_alloc_ctx_tensors_from_buft +// ggml_backend_alloc_ctx_tensors_from_buft returns NULL on failure or if all tensors in ctx are already allocated or zero-sized +GGML_API size_t ggml_backend_alloc_ctx_tensors_from_buft_size(struct ggml_context * ctx, ggml_backend_buffer_type_t buft); +GGML_API struct ggml_backend_buffer * ggml_backend_alloc_ctx_tensors_from_buft(struct ggml_context * ctx, ggml_backend_buffer_type_t buft); +GGML_API struct ggml_backend_buffer * ggml_backend_alloc_ctx_tensors(struct ggml_context * ctx, ggml_backend_t backend); + +#ifdef __cplusplus +} +#endif diff --git a/python/freetoken/kernel/csrc/gguf_mmq/ggml-backend.h b/python/freetoken/kernel/csrc/gguf_mmq/ggml-backend.h new file mode 100644 index 00000000..cc3f8cd3 --- /dev/null +++ b/python/freetoken/kernel/csrc/gguf_mmq/ggml-backend.h @@ -0,0 +1,437 @@ +#pragma once + +#include "ggml.h" +#include "ggml-alloc.h" + +#ifdef GGML_BACKEND_SHARED +# if defined(_WIN32) && !defined(__MINGW32__) +# ifdef GGML_BACKEND_BUILD +# define GGML_BACKEND_API __declspec(dllexport) extern +# else +# define GGML_BACKEND_API __declspec(dllimport) extern +# endif +# else +# define GGML_BACKEND_API __attribute__ ((visibility ("default"))) extern +# endif +#else +# define GGML_BACKEND_API extern +#endif + +#ifdef __cplusplus +extern "C" { +#endif + + typedef struct ggml_backend_buffer_type * ggml_backend_buffer_type_t; + typedef struct ggml_backend_buffer * ggml_backend_buffer_t; + typedef struct ggml_backend_event * ggml_backend_event_t; + typedef struct ggml_backend * ggml_backend_t; + typedef void * ggml_backend_graph_plan_t; + typedef struct ggml_backend_reg * ggml_backend_reg_t; + typedef struct ggml_backend_device * ggml_backend_dev_t; + + + // + // Backend buffer type + // + + GGML_API const char * ggml_backend_buft_name (ggml_backend_buffer_type_t buft); + GGML_API ggml_backend_buffer_t ggml_backend_buft_alloc_buffer (ggml_backend_buffer_type_t buft, size_t size); + GGML_API size_t ggml_backend_buft_get_alignment (ggml_backend_buffer_type_t buft); + GGML_API size_t ggml_backend_buft_get_max_size (ggml_backend_buffer_type_t buft); + GGML_API size_t ggml_backend_buft_get_alloc_size(ggml_backend_buffer_type_t buft, const struct ggml_tensor * tensor); + GGML_API bool ggml_backend_buft_is_host (ggml_backend_buffer_type_t buft); + GGML_API ggml_backend_dev_t ggml_backend_buft_get_device (ggml_backend_buffer_type_t buft); + + // + // Backend buffer + // + + enum ggml_backend_buffer_usage { + GGML_BACKEND_BUFFER_USAGE_ANY = 0, + GGML_BACKEND_BUFFER_USAGE_WEIGHTS = 1, + GGML_BACKEND_BUFFER_USAGE_COMPUTE = 2, + }; + + GGML_API const char * ggml_backend_buffer_name (ggml_backend_buffer_t buffer); + GGML_API void ggml_backend_buffer_free (ggml_backend_buffer_t buffer); + GGML_API void * ggml_backend_buffer_get_base (ggml_backend_buffer_t buffer); + GGML_API size_t ggml_backend_buffer_get_size (ggml_backend_buffer_t buffer); + GGML_API enum ggml_status ggml_backend_buffer_init_tensor (ggml_backend_buffer_t buffer, struct ggml_tensor * tensor); + GGML_API size_t ggml_backend_buffer_get_alignment (ggml_backend_buffer_t buffer); + GGML_API size_t ggml_backend_buffer_get_max_size (ggml_backend_buffer_t buffer); + GGML_API size_t ggml_backend_buffer_get_alloc_size(ggml_backend_buffer_t buffer, const struct ggml_tensor * tensor); + GGML_API void ggml_backend_buffer_clear (ggml_backend_buffer_t buffer, uint8_t value); + GGML_API bool ggml_backend_buffer_is_host (ggml_backend_buffer_t buffer); + GGML_API void ggml_backend_buffer_set_usage (ggml_backend_buffer_t buffer, enum ggml_backend_buffer_usage usage); + GGML_API enum ggml_backend_buffer_usage ggml_backend_buffer_get_usage (ggml_backend_buffer_t buffer); + GGML_API ggml_backend_buffer_type_t ggml_backend_buffer_get_type (ggml_backend_buffer_t buffer); + GGML_API void ggml_backend_buffer_reset (ggml_backend_buffer_t buffer); + + // tensor copy between different backends + GGML_API void ggml_backend_tensor_copy(const struct ggml_tensor * src, struct ggml_tensor * dst); + + // + // Backend (stream) + // + + GGML_API ggml_guid_t ggml_backend_guid(ggml_backend_t backend); + GGML_API const char * ggml_backend_name(ggml_backend_t backend); + GGML_API void ggml_backend_free(ggml_backend_t backend); + + GGML_API ggml_backend_buffer_type_t ggml_backend_get_default_buffer_type(ggml_backend_t backend); + GGML_API ggml_backend_buffer_t ggml_backend_alloc_buffer(ggml_backend_t backend, size_t size); + GGML_API size_t ggml_backend_get_alignment(ggml_backend_t backend); + GGML_API size_t ggml_backend_get_max_size(ggml_backend_t backend); + + GGML_API void ggml_backend_tensor_set_async (ggml_backend_t backend, struct ggml_tensor * tensor, const void * data, size_t offset, size_t size); + GGML_API void ggml_backend_tensor_get_async (ggml_backend_t backend, const struct ggml_tensor * tensor, void * data, size_t offset, size_t size); + GGML_API void ggml_backend_tensor_set_2d_async(ggml_backend_t backend, struct ggml_tensor * tensor, const void * data, size_t offset, size_t size, size_t n_copies, size_t stride_tensor, size_t stride_data); + GGML_API void ggml_backend_tensor_get_2d_async(ggml_backend_t backend, const struct ggml_tensor * tensor, void * data, size_t offset, size_t size, size_t n_copies, size_t stride_tensor, size_t stride_data); + + // "offset" refers to the offset in tensor->data for setting/getting data + GGML_API void ggml_backend_tensor_set ( struct ggml_tensor * tensor, const void * data, size_t offset, size_t size); + GGML_API void ggml_backend_tensor_get (const struct ggml_tensor * tensor, void * data, size_t offset, size_t size); + GGML_API void ggml_backend_tensor_set_2d( struct ggml_tensor * tensor, const void * data, size_t offset, size_t size, size_t n_copies, size_t stride_tensor, size_t stride_data); + GGML_API void ggml_backend_tensor_get_2d(const struct ggml_tensor * tensor, void * data, size_t offset, size_t size, size_t n_copies, size_t stride_tensor, size_t stride_data); + GGML_API void ggml_backend_tensor_memset( struct ggml_tensor * tensor, uint8_t value, size_t offset, size_t size); + + GGML_API void ggml_backend_synchronize(ggml_backend_t backend); + + GGML_API ggml_backend_graph_plan_t ggml_backend_graph_plan_create(ggml_backend_t backend, struct ggml_cgraph * cgraph); + GGML_API void ggml_backend_graph_plan_free (ggml_backend_t backend, ggml_backend_graph_plan_t plan); + + GGML_API enum ggml_status ggml_backend_graph_plan_compute (ggml_backend_t backend, ggml_backend_graph_plan_t plan); + GGML_API enum ggml_status ggml_backend_graph_compute (ggml_backend_t backend, struct ggml_cgraph * cgraph); + GGML_API enum ggml_status ggml_backend_graph_compute_async(ggml_backend_t backend, struct ggml_cgraph * cgraph); + + // NOTE: will be removed, use device version instead + GGML_API bool ggml_backend_supports_op(ggml_backend_t backend, const struct ggml_tensor * op); + GGML_API bool ggml_backend_supports_buft(ggml_backend_t backend, ggml_backend_buffer_type_t buft); + GGML_API bool ggml_backend_offload_op(ggml_backend_t backend, const struct ggml_tensor * op); + + // asynchronous copy + // the copy is performed after all the currently queued operations in backend_src + // backend_dst will wait for the copy to complete before performing other operations + // automatic fallback to sync copy if async is not supported + GGML_API void ggml_backend_tensor_copy_async(ggml_backend_t backend_src, ggml_backend_t backend_dst, const struct ggml_tensor * src, struct ggml_tensor * dst); + + GGML_API ggml_backend_dev_t ggml_backend_get_device(ggml_backend_t backend); + + // + // Events + // + + GGML_API ggml_backend_event_t ggml_backend_event_new(ggml_backend_dev_t device); + GGML_API void ggml_backend_event_free(ggml_backend_event_t event); + GGML_API void ggml_backend_event_record(ggml_backend_event_t event, ggml_backend_t backend); + GGML_API void ggml_backend_event_synchronize(ggml_backend_event_t event); + GGML_API void ggml_backend_event_wait(ggml_backend_t backend, ggml_backend_event_t event); + + // + // Backend device + // + + enum ggml_backend_dev_type { + // CPU device using system memory + GGML_BACKEND_DEVICE_TYPE_CPU, + // GPU device using dedicated memory + GGML_BACKEND_DEVICE_TYPE_GPU, + // integrated GPU device using host memory + GGML_BACKEND_DEVICE_TYPE_IGPU, + // accelerator devices intended to be used together with the CPU backend (e.g. BLAS or AMX) + GGML_BACKEND_DEVICE_TYPE_ACCEL, + // "meta" device wrapping multiple other devices for tensor parallelism + GGML_BACKEND_DEVICE_TYPE_META, + }; + + // functionality supported by the device + struct ggml_backend_dev_caps { + // asynchronous operations + bool async; + // pinned host buffer + bool host_buffer; + // creating buffers from host ptr + bool buffer_from_host_ptr; + // event synchronization + bool events; + // mmap is supported for loading + bool mmap_support; + }; + + // all the device properties + struct ggml_backend_dev_props { + // device name + const char * name; + // device description + const char * description; + // device free memory in bytes + size_t memory_free; + // device total memory in bytes + size_t memory_total; + // device type + enum ggml_backend_dev_type type; + // device id + // for PCI devices, this should be the lower-case PCI bus id formatted as "domain:bus:device.function" (e.g. "0000:c1:00.0") + // if the id is unknown, this should be NULL + const char * device_id; + // device capabilities + struct ggml_backend_dev_caps caps; + }; + + GGML_API const char * ggml_backend_dev_name(ggml_backend_dev_t device); + GGML_API const char * ggml_backend_dev_description(ggml_backend_dev_t device); + GGML_API void ggml_backend_dev_memory(ggml_backend_dev_t device, size_t * free, size_t * total); + GGML_API enum ggml_backend_dev_type ggml_backend_dev_type(ggml_backend_dev_t device); + GGML_API void ggml_backend_dev_get_props(ggml_backend_dev_t device, struct ggml_backend_dev_props * props); + GGML_API ggml_backend_reg_t ggml_backend_dev_backend_reg(ggml_backend_dev_t device); + GGML_API ggml_backend_t ggml_backend_dev_init(ggml_backend_dev_t device, const char * params); + GGML_API ggml_backend_buffer_type_t ggml_backend_dev_buffer_type(ggml_backend_dev_t device); + GGML_API ggml_backend_buffer_type_t ggml_backend_dev_host_buffer_type(ggml_backend_dev_t device); + GGML_API ggml_backend_buffer_t ggml_backend_dev_buffer_from_host_ptr(ggml_backend_dev_t device, void * ptr, size_t size, size_t max_tensor_size); + + GGML_API bool ggml_backend_dev_supports_op(ggml_backend_dev_t device, const struct ggml_tensor * op); + GGML_API bool ggml_backend_dev_supports_buft(ggml_backend_dev_t device, ggml_backend_buffer_type_t buft); + GGML_API bool ggml_backend_dev_offload_op(ggml_backend_dev_t device, const struct ggml_tensor * op); + + // + // Backend (reg) + // + + GGML_API const char * ggml_backend_reg_name(ggml_backend_reg_t reg); + GGML_API size_t ggml_backend_reg_dev_count(ggml_backend_reg_t reg); + GGML_API ggml_backend_dev_t ggml_backend_reg_dev_get(ggml_backend_reg_t reg, size_t index); + GGML_API void * ggml_backend_reg_get_proc_address(ggml_backend_reg_t reg, const char * name); + + // Common functions that may be obtained using ggml_backend_reg_get_proc_address + + // Context management and operations for faster communication between backends, used for tensor parallelism (meta backend) + typedef void * (*ggml_backend_comm_init_t)(ggml_backend_t * backends, size_t n_backends); + typedef void (*ggml_backend_comm_free_t)(void * comm_ctx); + typedef bool (*ggml_backend_comm_allreduce_tensor_t)(void * comm_ctx, struct ggml_tensor ** tensors); + + // Split buffer type for tensor parallelism (old) + typedef ggml_backend_buffer_type_t (*ggml_backend_split_buffer_type_t)(int main_device, const float * tensor_split); + // Set the number of threads for the backend + typedef void (*ggml_backend_set_n_threads_t)(ggml_backend_t backend, int n_threads); + // Get additional buffer types provided by the device (returns a NULL-terminated array) + typedef ggml_backend_buffer_type_t * (*ggml_backend_dev_get_extra_bufts_t)(ggml_backend_dev_t device); + // Set the abort callback for the backend + typedef void (*ggml_backend_set_abort_callback_t)(ggml_backend_t backend, ggml_abort_callback abort_callback, void * abort_callback_data); + // Get a list of feature flags supported by the backend (returns a NULL-terminated array) + struct ggml_backend_feature { + const char * name; + const char * value; + }; + typedef struct ggml_backend_feature * (*ggml_backend_get_features_t)(ggml_backend_reg_t reg); + + // + // Backend registry + // + + GGML_API void ggml_backend_register(ggml_backend_reg_t reg); + + GGML_API void ggml_backend_device_register(ggml_backend_dev_t device); + + // Backend (reg) enumeration + GGML_API size_t ggml_backend_reg_count(void); + GGML_API ggml_backend_reg_t ggml_backend_reg_get(size_t index); + GGML_API ggml_backend_reg_t ggml_backend_reg_by_name(const char * name); + + // Device enumeration + GGML_API size_t ggml_backend_dev_count(void); + GGML_API ggml_backend_dev_t ggml_backend_dev_get(size_t index); + GGML_API ggml_backend_dev_t ggml_backend_dev_by_name(const char * name); + GGML_API ggml_backend_dev_t ggml_backend_dev_by_type(enum ggml_backend_dev_type type); + + // Direct backend (stream) initialization + // = ggml_backend_dev_init(ggml_backend_dev_by_name(name), params) + GGML_API ggml_backend_t ggml_backend_init_by_name(const char * name, const char * params); + // = ggml_backend_dev_init(ggml_backend_dev_by_type(type), params) + GGML_API ggml_backend_t ggml_backend_init_by_type(enum ggml_backend_dev_type type, const char * params); + // = ggml_backend_dev_init(ggml_backend_dev_by_type(GPU) OR ggml_backend_dev_by_type(CPU), NULL) + GGML_API ggml_backend_t ggml_backend_init_best(void); + + // Load a backend from a dynamic library and register it + GGML_API ggml_backend_reg_t ggml_backend_load(const char * path); + // Unload a backend if loaded dynamically and unregister it + GGML_API void ggml_backend_unload(ggml_backend_reg_t reg); + // Load all known backends from dynamic libraries + GGML_API void ggml_backend_load_all(void); + GGML_API void ggml_backend_load_all_from_path(const char * dir_path); + + // + // Backend scheduler + // + + // The backend scheduler allows for multiple backend devices to be used together + // Handles compute buffer allocation, assignment of tensors to backends, and copying of tensors between backends + // The backends are selected based on: + // - the backend that supports the operation + // - the location of the pre-allocated tensors (e.g. the weights) + /* + Example usage: + + // operations that use tensors allocated in a buffer with USAGE_WEIGHTS will be assigned + // preferably to run on the same backend as the buffer + ggml_backend_buffer_set_usage(buf_weights, GGML_BACKEND_BUFFER_USAGE_WEIGHTS); + + sched = ggml_backend_sched_new({backend_gpu, backend_gpu2, backend_cpu}, NULL, num_backends, GGML_DEFAULT_GRAPH_SIZE, false, true); + + // initialize buffers from a max size graph (optional) + reserve_graph = build_graph(sched, max_batch_size); + + // manually assign nodes to a backend (optional, should not be needed in most cases) + struct ggml_tensor * node = ggml_mul_mat(ctx, ...); + ggml_backend_sched_set_tensor_backend(sched, node, backend_gpu); + + ggml_backend_sched_reserve(sched, reserve_graph); + + // compute + graph = build_graph(sched); // the graph and its tensors are single-use in terms of allocation, multi-use in terms of computation + for (int i = 0; i < 10; ++i) { + ggml_backend_sched_graph_compute(sched, graph); // on the first iteration the graph is allocated automatically + } + + // if there are graph inputs: + graph = build_graph(sched); // get a new graph that is not allocated (the metadata for the old graph is freed once ggml_free is called) + ggml_backend_sched_reset(sched); // clear the allocation of the previous graph + ggml_backend_sched_alloc_graph(sched, graph); // explicitly allocate the new graph but do not execute it + ggml_backend_tensor_set(input_tensor, ...); // copy data to the newly allocated graph tensors + ggml_backend_sched_graph_compute(sched, graph); // execute the graph + + // as an alternative to the above it is also possible to assign the inputs to a dedicated context and + // allocate them statically via ggml_backend_alloc_ctx_tensors + } + */ + + typedef struct ggml_backend_sched * ggml_backend_sched_t; + + // Evaluation callback for each node in the graph (set with ggml_backend_sched_set_eval_callback) + // when ask == true, the scheduler wants to know if the user wants to observe this node + // this allows the scheduler to batch nodes together in order to evaluate them in a single call + // + // when ask == false, the scheduler is passing the node tensor to the user for observation + // if the user returns false, the scheduler will cancel the graph compute + // + typedef bool (*ggml_backend_sched_eval_callback)(struct ggml_tensor * t, bool ask, void * user_data); + + // Initialize a backend scheduler, backends with low index are given priority over backends with high index + GGML_API ggml_backend_sched_t ggml_backend_sched_new(ggml_backend_t * backends, ggml_backend_buffer_type_t * bufts, int n_backends, size_t graph_size, bool parallel, bool op_offload); + GGML_API void ggml_backend_sched_free(ggml_backend_sched_t sched); + + // Initialize backend buffers from a measure graph + GGML_API void ggml_backend_sched_reserve_size(ggml_backend_sched_t sched, struct ggml_cgraph * measure_graph, size_t * sizes); + GGML_API bool ggml_backend_sched_reserve(ggml_backend_sched_t sched, struct ggml_cgraph * measure_graph); // returns success + + GGML_API int ggml_backend_sched_get_n_backends(ggml_backend_sched_t sched); + GGML_API ggml_backend_t ggml_backend_sched_get_backend(ggml_backend_sched_t sched, int i); + + // Get the number of splits of the last graph + GGML_API int ggml_backend_sched_get_n_splits(ggml_backend_sched_t sched); + GGML_API int ggml_backend_sched_get_n_copies(ggml_backend_sched_t sched); + + GGML_API ggml_backend_buffer_type_t ggml_backend_sched_get_buffer_type(ggml_backend_sched_t sched, ggml_backend_t backend); + GGML_API size_t ggml_backend_sched_get_buffer_size(ggml_backend_sched_t sched, ggml_backend_t backend); + + GGML_API void ggml_backend_sched_set_tensor_backend(ggml_backend_sched_t sched, struct ggml_tensor * node, ggml_backend_t backend); + GGML_API ggml_backend_t ggml_backend_sched_get_tensor_backend(ggml_backend_sched_t sched, struct ggml_tensor * node); + + // Split graph without allocating it + GGML_API void ggml_backend_sched_split_graph(ggml_backend_sched_t sched, struct ggml_cgraph * graph); + + // Allocate and compute graph on the backend scheduler + GGML_API bool ggml_backend_sched_alloc_graph(ggml_backend_sched_t sched, struct ggml_cgraph * graph); // returns success + GGML_API enum ggml_status ggml_backend_sched_graph_compute(ggml_backend_sched_t sched, struct ggml_cgraph * graph); + GGML_API enum ggml_status ggml_backend_sched_graph_compute_async(ggml_backend_sched_t sched, struct ggml_cgraph * graph); + GGML_API void ggml_backend_sched_synchronize(ggml_backend_sched_t sched); + + // Reset all assignments and allocators - must be called before changing the node backends or allocating a new graph. + // This in effect deallocates all tensors that were previously allocated and leaves them with dangling pointers. + // The correct way to use this API is to discard the deallocated tensors and create new ones. + GGML_API void ggml_backend_sched_reset(ggml_backend_sched_t sched); + + // Set a callback to be called for each resulting node during graph compute + GGML_API void ggml_backend_sched_set_eval_callback(ggml_backend_sched_t sched, ggml_backend_sched_eval_callback callback, void * user_data); + + // + // Meta backend + // + +#define GGML_BACKEND_META_MAX_DEVICES 16 + + enum ggml_backend_meta_split_axis { + // tensor split by tensor dimensions: + GGML_BACKEND_SPLIT_AXIS_0 = 0, + GGML_BACKEND_SPLIT_AXIS_1 = 1, + GGML_BACKEND_SPLIT_AXIS_2 = 2, + GGML_BACKEND_SPLIT_AXIS_3 = 3, + + GGML_BACKEND_SPLIT_AXIS_MIRRORED = 10, // all values on all backends + GGML_BACKEND_SPLIT_AXIS_PARTIAL = 11, // each backend has a partial sum + + // for internal bookkeeping only: + GGML_BACKEND_SPLIT_AXIS_NONE = 98, + GGML_BACKEND_SPLIT_AXIS_UNKNOWN = 99, + }; + GGML_API const char * ggml_backend_meta_split_axis_name(enum ggml_backend_meta_split_axis split_axis); + + struct ggml_backend_meta_split_state { + enum ggml_backend_meta_split_axis axis; + + // for tensors with axis >= 0 && axis < GGML_MAX_DIMS: + // - each device has a slice of the tensor along the split axis + // - most tensors have n_segments == 1 and a contiguous slice of the tensor data + // - some tensors have an inhomogenenous data layout along the split axis, + // those tensors are divided into segments which are each individually split across devices + // - ne has one entry per segment and device and that segment repeats nr times, + // in total when accounting for repetitions the segments add up to ggml_tensor::ne for that axis, + // the outer/inner loops are over segments/devices like [seg0_dev0_r0, seg0_dev1_r0, seg0_dev0_r1, seg0_dev1_r1, seg1_dev0_r0, seg1_dev1_r0], + // - for example, a transformer may have a fused QKV matrix rather than 3 matrices, those would be 3 separate segments + // that each need to be split individually across devices so that each device gets a slice of Q, K, and V, + // the Q matrix can be larger than the K and V matrices so this can either be expressed as 3 segments or as 2 segments + // where the segment for K/V repeats twice + int64_t ne[16*GGML_BACKEND_META_MAX_DEVICES]; + uint32_t nr[16]; + uint32_t n_segments; + }; + + // function to assign split states for statically allocated tensors, compute tensor split states will be assigned to be compatible: + typedef struct ggml_backend_meta_split_state(*ggml_backend_meta_get_split_state_t)(const struct ggml_tensor * tensor, void * userdata); + + // create a new meta device from "simple" devices, meta buffer type/buffer/backend is then derived from this: + // TODO: this looks a bit strange - a backend API creates a device. I think we should try + // express this as a backend registry functionality instead + GGML_API ggml_backend_dev_t ggml_backend_meta_device( + ggml_backend_dev_t * devs, size_t n_devs, ggml_backend_meta_get_split_state_t get_split_state, void * get_split_state_ud); + + // + // Utils + // + + struct ggml_backend_graph_copy { + ggml_backend_buffer_t buffer; + struct ggml_context * ctx_allocated; + struct ggml_context * ctx_unallocated; + struct ggml_cgraph * graph; + }; + + // Copy a graph to a different backend + GGML_API struct ggml_backend_graph_copy ggml_backend_graph_copy(ggml_backend_t backend, struct ggml_cgraph * graph); + GGML_API void ggml_backend_graph_copy_free(struct ggml_backend_graph_copy copy); + + typedef bool (*ggml_backend_eval_callback)(int node_index, struct ggml_tensor * t1, struct ggml_tensor * t2, void * user_data); + + // Compare the output of two backends + GGML_API bool ggml_backend_compare_graph_backend(ggml_backend_t backend1, ggml_backend_t backend2, struct ggml_cgraph * graph, ggml_backend_eval_callback callback, void * user_data, struct ggml_tensor const * const * test_nodes, size_t num_test_nodes); + + // Tensor initialization + GGML_API enum ggml_status ggml_backend_tensor_alloc(ggml_backend_buffer_t buffer, struct ggml_tensor * tensor, void * addr); + GGML_API enum ggml_status ggml_backend_view_init(struct ggml_tensor * tensor); + + // CPU buffer types are always available + GGML_API ggml_backend_buffer_t ggml_backend_cpu_buffer_from_ptr(void * ptr, size_t size); + GGML_API ggml_backend_buffer_type_t ggml_backend_cpu_buffer_type(void); + +#ifdef __cplusplus +} +#endif diff --git a/python/freetoken/kernel/csrc/gguf_mmq/ggml-common.h b/python/freetoken/kernel/csrc/gguf_mmq/ggml-common.h new file mode 100644 index 00000000..83f9118d --- /dev/null +++ b/python/freetoken/kernel/csrc/gguf_mmq/ggml-common.h @@ -0,0 +1,1911 @@ +#ifndef GGML_COMMON_DECL + +#if defined(GGML_COMMON_DECL_C) +#include + +typedef uint16_t ggml_half; +typedef uint32_t ggml_half2; + +#define GGML_COMMON_AGGR_U +#define GGML_COMMON_AGGR_S + +#define GGML_COMMON_DECL +#elif defined(GGML_COMMON_DECL_CPP) +#include + +typedef uint16_t ggml_half; +typedef uint32_t ggml_half2; + +// std-c++ allow anonymous unions but some compiler warn on it +#define GGML_COMMON_AGGR_U data +// std-c++ do not allow it. +#define GGML_COMMON_AGGR_S data + +#define GGML_COMMON_DECL +#elif defined(GGML_COMMON_DECL_METAL) +#include + +typedef half ggml_half; +typedef half2 ggml_half2; + +#define GGML_COMMON_AGGR_U +#define GGML_COMMON_AGGR_S + +#define GGML_COMMON_DECL +#elif defined(GGML_COMMON_DECL_CUDA) +#if defined(GGML_COMMON_DECL_MUSA) +#include +#else +#include +#endif +#include + +typedef half ggml_half; +typedef half2 ggml_half2; + +#define GGML_COMMON_AGGR_U +#define GGML_COMMON_AGGR_S data + +#define GGML_COMMON_DECL +#elif defined(GGML_COMMON_DECL_HIP) +#include +#include + +typedef half ggml_half; +typedef half2 ggml_half2; + +#define GGML_COMMON_AGGR_U +#define GGML_COMMON_AGGR_S data + +#define GGML_COMMON_DECL +#elif defined(GGML_COMMON_DECL_SYCL) +#include +#include + +typedef sycl::half ggml_half; +typedef sycl::half2 ggml_half2; + +#define GGML_COMMON_AGGR_U +#define GGML_COMMON_AGGR_S data + +#define GGML_COMMON_DECL +#endif + +#if defined(GGML_COMMON_DECL) + +#ifndef __cplusplus +#ifndef static_assert +#if defined(__STDC_VERSION__) && (__STDC_VERSION__ >= 201100L) +#define static_assert(cond, msg) _Static_assert(cond, msg) +#else +#define static_assert(cond, msg) struct global_scope_noop_trick +#endif +#endif +#endif // __cplusplus + +// QK = number of values after dequantization +// QK_K = super-block size + +#define QK_K 256 +#define K_SCALE_SIZE 12 + +#if defined(GGML_COMMON_DECL_CUDA) || defined(GGML_COMMON_DECL_HIP) || defined(GGML_COMMON_DECL_SYCL) +// QR = QK / number of values before dequantization +// QI = number of 32 bit integers before dequantization + +#define QI1_0 (QK1_0 / 32) +#define QR1_0 1 + +#define QI2_0 (QK2_0 / 32) +#define QR2_0 1 + + +#define QI4_0 (QK4_0 / (4 * QR4_0)) +#define QR4_0 2 + +#define QI4_1 (QK4_1 / (4 * QR4_1)) +#define QR4_1 2 + +#define QI_MXFP4 (QK_MXFP4 / (4 * QR_MXFP4)) +#define QR_MXFP4 2 + +#define QI_NVFP4 (QK_NVFP4 / (4 * QR_NVFP4)) +#define QR_NVFP4 2 + +#define QI5_0 (QK5_0 / (4 * QR5_0)) +#define QR5_0 2 + +#define QI5_1 (QK5_1 / (4 * QR5_1)) +#define QR5_1 2 + +#define QI8_0 (QK8_0 / (4 * QR8_0)) +#define QR8_0 1 + +#define QI8_1 (QK8_1 / (4 * QR8_1)) +#define QR8_1 1 + +#define QI2_K (QK_K / (4*QR2_K)) +#define QR2_K 4 + +#define QI3_K (QK_K / (4*QR3_K)) +#define QR3_K 4 + +#define QI4_K (QK_K / (4*QR4_K)) +#define QR4_K 2 + +#define QI5_K (QK_K / (4*QR5_K)) +#define QR5_K 2 + +#define QI6_K (QK_K / (4*QR6_K)) +#define QR6_K 2 + +#define QI2_XXS (QK_K / (4*QR2_XXS)) +#define QR2_XXS 4 + +#define QI2_XS (QK_K / (4*QR2_XS)) +#define QR2_XS 4 + +#define QI2_S (QK_K / (4*QR2_S)) +#define QR2_S 4 + +#define QI3_XXS (QK_K / (4*QR3_XXS)) +#define QR3_XXS 4 + +#define QI3_XS (QK_K / (4*QR3_XS)) +#define QR3_XS 4 + +#define QI1_S (QK_K / (4*QR1_S)) +#define QR1_S 8 + +#define QI1_M (QK_K / (4*QR1_M)) +#define QR1_M 8 + +#define QI4_NL (QK4_NL / (4*QR4_NL)) +#define QR4_NL 2 + +#define QI4_XS (QK_K / (4*QR4_XS)) +#define QR4_XS 2 + +#define QI3_S (QK_K / (4*QR3_S)) +#define QR3_S 4 + +#endif // GGML_COMMON_DECL_CUDA || GGML_COMMON_DECL_HIP + +#ifdef _MSC_VER +#define GGML_EXTENSION +#else // _MSC_VER +#define GGML_EXTENSION __extension__ +#endif // _MSC_VER + +#define QK1_0 128 +typedef struct { + ggml_half d; // delta + uint8_t qs[QK1_0 / 8]; // bits / quants +} block_q1_0; +static_assert(sizeof(block_q1_0) == sizeof(ggml_half) + QK1_0 / 8, "wrong q1_0 block size/padding"); + +#define QK2_0 64 +typedef struct { + ggml_half d; // delta (scale) + uint8_t qs[QK2_0 / 4]; // 2 bits per element +} block_q2_0; +static_assert(sizeof(block_q2_0) == sizeof(ggml_half) + QK2_0 / 4, "wrong q2_0 block size/padding"); + +#define QK4_0 32 +typedef struct { + ggml_half d; // delta + uint8_t qs[QK4_0 / 2]; // nibbles / quants +} block_q4_0; +static_assert(sizeof(block_q4_0) == sizeof(ggml_half) + QK4_0 / 2, "wrong q4_0 block size/padding"); + +#define QK4_1 32 +typedef struct { + GGML_EXTENSION union { + struct { + ggml_half d; // delta + ggml_half m; // min + } GGML_COMMON_AGGR_S; + ggml_half2 dm; + } GGML_COMMON_AGGR_U; + uint8_t qs[QK4_1 / 2]; // nibbles / quants +} block_q4_1; +static_assert(sizeof(block_q4_1) == 2 * sizeof(ggml_half) + QK4_1 / 2, "wrong q4_1 block size/padding"); + +#define QK_MXFP4 32 +typedef struct { + uint8_t e; // E8M0 + uint8_t qs[QK_MXFP4/2]; +} block_mxfp4; +static_assert(sizeof(block_mxfp4) == sizeof(uint8_t) + QK_MXFP4/2, "wrong mxfp4 block size/padding"); + +#define QK_NVFP4 64 +#define QK_NVFP4_SUB 16 // sub-block size for per-group scales +typedef struct { + uint8_t d[QK_NVFP4/QK_NVFP4_SUB]; // UE4M3 scales (4 bytes, one per 16-element sub-block) + uint8_t qs[QK_NVFP4/2]; // packed 4-bit E2M1 values (32 bytes) +} block_nvfp4; +static_assert(sizeof(block_nvfp4) == sizeof(uint8_t)*(QK_NVFP4/QK_NVFP4_SUB) + QK_NVFP4/2, "wrong nvfp4 block size/padding"); + +#define QK5_0 32 +typedef struct { + ggml_half d; // delta + uint8_t qh[4]; // 5-th bit of quants + uint8_t qs[QK5_0 / 2]; // nibbles / quants +} block_q5_0; +static_assert(sizeof(block_q5_0) == sizeof(ggml_half) + sizeof(uint32_t) + QK5_0 / 2, "wrong q5_0 block size/padding"); + +#define QK5_1 32 +typedef struct { + GGML_EXTENSION union { + struct { + ggml_half d; // delta + ggml_half m; // min + } GGML_COMMON_AGGR_S; + ggml_half2 dm; + } GGML_COMMON_AGGR_U; + uint8_t qh[4]; // 5-th bit of quants + uint8_t qs[QK5_1 / 2]; // nibbles / quants +} block_q5_1; +static_assert(sizeof(block_q5_1) == 2 * sizeof(ggml_half) + sizeof(uint32_t) + QK5_1 / 2, "wrong q5_1 block size/padding"); + +#define QK8_0 32 +typedef struct { + ggml_half d; // delta + int8_t qs[QK8_0]; // quants +} block_q8_0; +static_assert(sizeof(block_q8_0) == sizeof(ggml_half) + QK8_0, "wrong q8_0 block size/padding"); + +#define QK8_1 32 +typedef struct { + GGML_EXTENSION union { + struct { + ggml_half d; // delta + ggml_half s; // d * sum(qs[i]) + } GGML_COMMON_AGGR_S; + ggml_half2 ds; + } GGML_COMMON_AGGR_U; + int8_t qs[QK8_1]; // quants +} block_q8_1; +static_assert(sizeof(block_q8_1) == 2*sizeof(ggml_half) + QK8_1, "wrong q8_1 block size/padding"); + +// +// Ternary quantization +// + +// 1.6875 bpw +typedef struct { + uint8_t qs[(QK_K - 4 * QK_K / 64) / 5]; // 5 elements per byte (3^5 = 243 < 256) + uint8_t qh[QK_K/64]; // 4 elements per byte + ggml_half d; +} block_tq1_0; +static_assert(sizeof(block_tq1_0) == sizeof(ggml_half) + QK_K / 64 + (QK_K - 4 * QK_K / 64) / 5, "wrong tq1_0 block size/padding"); + +// 2.0625 bpw +typedef struct { + uint8_t qs[QK_K/4]; // 2 bits per element + ggml_half d; +} block_tq2_0; +static_assert(sizeof(block_tq2_0) == sizeof(ggml_half) + QK_K / 4, "wrong tq2_0 block size/padding"); + +// +// Super-block quantization structures +// + +// 2-bit quantization +// weight is represented as x = a * q + b +// 16 blocks of 16 elements each +// Effectively 2.625 bits per weight +typedef struct { + uint8_t scales[QK_K/16]; // scales and mins, quantized with 4 bits + uint8_t qs[QK_K/4]; // quants + GGML_EXTENSION union { + struct { + ggml_half d; // super-block scale for quantized scales + ggml_half dmin; // super-block scale for quantized mins + } GGML_COMMON_AGGR_S; + ggml_half2 dm; + } GGML_COMMON_AGGR_U; +} block_q2_K; +static_assert(sizeof(block_q2_K) == 2*sizeof(ggml_half) + QK_K/16 + QK_K/4, "wrong q2_K block size/padding"); + +// 3-bit quantization +// weight is represented as x = a * q +// 16 blocks of 16 elements each +// Effectively 3.4375 bits per weight +typedef struct { + uint8_t hmask[QK_K/8]; // quants - high bit + uint8_t qs[QK_K/4]; // quants - low 2 bits + uint8_t scales[12]; // scales, quantized with 6 bits + ggml_half d; // super-block scale +} block_q3_K; +static_assert(sizeof(block_q3_K) == sizeof(ggml_half) + QK_K / 4 + QK_K / 8 + 12, "wrong q3_K block size/padding"); + +// 4-bit quantization +// 8 blocks of 32 elements each +// weight is represented as x = a * q + b +// Effectively 4.5 bits per weight +typedef struct { + GGML_EXTENSION union { + struct { + ggml_half d; // super-block scale for quantized scales + ggml_half dmin; // super-block scale for quantized mins + } GGML_COMMON_AGGR_S; + ggml_half2 dm; + } GGML_COMMON_AGGR_U; + uint8_t scales[K_SCALE_SIZE]; // scales and mins, quantized with 6 bits + uint8_t qs[QK_K/2]; // 4--bit quants +} block_q4_K; +static_assert(sizeof(block_q4_K) == 2*sizeof(ggml_half) + K_SCALE_SIZE + QK_K/2, "wrong q4_K block size/padding"); + +// 5-bit quantization +// 8 blocks of 32 elements each +// weight is represented as x = a * q + b +// Effectively 5.5 bits per weight +typedef struct { + GGML_EXTENSION union { + struct { + ggml_half d; // super-block scale for quantized scales + ggml_half dmin; // super-block scale for quantized mins + } GGML_COMMON_AGGR_S; + ggml_half2 dm; + } GGML_COMMON_AGGR_U; + uint8_t scales[K_SCALE_SIZE]; // scales and mins, quantized with 6 bits + uint8_t qh[QK_K/8]; // quants, high bit + uint8_t qs[QK_K/2]; // quants, low 4 bits +} block_q5_K; +static_assert(sizeof(block_q5_K) == 2*sizeof(ggml_half) + K_SCALE_SIZE + QK_K/2 + QK_K/8, "wrong q5_K block size/padding"); + +// 6-bit quantization +// weight is represented as x = a * q +// 16 blocks of 16 elements each +// Effectively 6.5625 bits per weight +typedef struct { + uint8_t ql[QK_K/2]; // quants, lower 4 bits + uint8_t qh[QK_K/4]; // quants, upper 2 bits + int8_t scales[QK_K/16]; // scales, quantized with 8 bits + ggml_half d; // super-block scale +} block_q6_K; +static_assert(sizeof(block_q6_K) == sizeof(ggml_half) + QK_K / 16 + 3*QK_K/4, "wrong q6_K block size/padding"); + +// This is only used for intermediate quantization and dot products +typedef struct { + float d; // delta + int8_t qs[QK_K]; // quants + int16_t bsums[QK_K/16]; // sum of quants in groups of 16 +} block_q8_K; +static_assert(sizeof(block_q8_K) == sizeof(float) + QK_K + QK_K/16*sizeof(int16_t), "wrong q8_K block size/padding"); + +// (Almost) "true" 2-bit quantization. +// Due to the need to use blocks as per ggml design, it ends up using +// 2.0625 bpw because of the 16-bit scale for each block of 256. +typedef struct { + ggml_half d; + uint16_t qs[QK_K/8]; +} block_iq2_xxs; +static_assert(sizeof(block_iq2_xxs) == sizeof(ggml_half) + QK_K/8*sizeof(uint16_t), "wrong iq2_xxs block size/padding"); + +// 2.3125 bpw quants +typedef struct { + ggml_half d; + uint16_t qs[QK_K/8]; + uint8_t scales[QK_K/32]; +} block_iq2_xs; +static_assert(sizeof(block_iq2_xs) == sizeof(ggml_half) + QK_K/8*sizeof(uint16_t) + QK_K/32, "wrong iq2_xs block size/padding"); + +// 2.5625 bpw quants +typedef struct { + ggml_half d; + uint8_t qs[QK_K/4]; + uint8_t qh[QK_K/32]; + uint8_t scales[QK_K/32]; +} block_iq2_s; +static_assert(sizeof(block_iq2_s) == sizeof(ggml_half) + QK_K/4 + QK_K/16, "wrong iq2_s block size/padding"); + +// (Almost) "true" 3-bit quantization. +// Due to the need to use blocks as per ggml design, it ends up using +// 3.0625 bpw because of the 16-bit scale for each block of 256. +typedef struct { + ggml_half d; + uint8_t qs[3*QK_K/8]; +} block_iq3_xxs; +static_assert(sizeof(block_iq3_xxs) == sizeof(ggml_half) + 3*(QK_K/8), "wrong iq3_xxs block size/padding"); + +// 3.4375 bpw +#define IQ3S_N_SCALE QK_K/64 +typedef struct { + ggml_half d; + uint8_t qs[QK_K/4]; + uint8_t qh[QK_K/32]; + uint8_t signs[QK_K/8]; + uint8_t scales[IQ3S_N_SCALE]; +} block_iq3_s; +static_assert(sizeof(block_iq3_s) == sizeof(ggml_half) + 13*(QK_K/32) + IQ3S_N_SCALE, "wrong iq3_s block size/padding"); + +// 1.5625 bpw +typedef struct { + ggml_half d; + uint8_t qs[QK_K/8]; + uint16_t qh[QK_K/32]; +} block_iq1_s; +static_assert(sizeof(block_iq1_s) == sizeof(ggml_half) + QK_K/8 + QK_K/16, "wrong iq1_s block size/padding"); + +// 1.75 bpw +typedef struct { + uint8_t qs[QK_K/8]; // grid index, low 8 bits + uint8_t qh[QK_K/16]; // grid index, high 3 bits + grid shift bit (for two groups of 8) + uint8_t scales[QK_K/32]; // 3-bit block scales (4-bit if QK_K == 64) +} block_iq1_m; +static_assert(sizeof(block_iq1_m) == QK_K/8 + QK_K/16 + QK_K/32, "wrong iq1_m block size/padding"); + +// Used by IQ1_M quants +typedef union { + ggml_half f16; + uint16_t u16; +} iq1m_scale_t; + +// Non-linear quants +#define QK4_NL 32 +typedef struct { + ggml_half d; + uint8_t qs[QK4_NL/2]; +} block_iq4_nl; +static_assert(sizeof(block_iq4_nl) == sizeof(ggml_half) + QK4_NL/2, "wrong iq4_nl block size/padding"); + +typedef struct { + ggml_half d; + uint16_t scales_h; + uint8_t scales_l[QK_K/64]; + uint8_t qs[QK_K/2]; +} block_iq4_xs; +static_assert(sizeof(block_iq4_xs) == sizeof(ggml_half) + sizeof(uint16_t) + QK_K/64 + QK_K/2, "wrong iq4_xs block size/padding"); + +#endif // GGML_COMMON_DECL +#endif // GGML_COMMON_DECL + +//////////////////////////////////////////////////////////////////////////////// + +#ifndef GGML_COMMON_IMPL + +#if defined(GGML_COMMON_IMPL_C) +#include + +#define GGML_TABLE_BEGIN(type, name, size) static const type name[size] = { +#define GGML_TABLE_END() }; + +#define GGML_COMMON_IMPL +#elif defined(GGML_COMMON_IMPL_CPP) +#include + +#define GGML_TABLE_BEGIN(type, name, size) static const type name[size] = { +#define GGML_TABLE_END() }; + +#define GGML_COMMON_IMPL +#elif defined(GGML_COMMON_IMPL_METAL) +#include + +#define GGML_TABLE_BEGIN(type, name, size) static const constant type name[size] = { +#define GGML_TABLE_END() }; + +#define GGML_COMMON_IMPL +#elif defined(GGML_COMMON_IMPL_CUDA) || defined(GGML_COMMON_IMPL_HIP) || defined(GGML_COMMON_IMPL_MUSA) +#include + +#define GGML_TABLE_BEGIN(type, name, size) static const __device__ type name[size] = { +#define GGML_TABLE_END() }; + +#define GGML_COMMON_IMPL +#elif defined(GGML_COMMON_IMPL_SYCL) + +#include + +#define GGML_TABLE_BEGIN(type, name, size) static const type name[size] = { +#define GGML_TABLE_END() }; + +#define GGML_COMMON_IMPL +#endif + +#if defined(GGML_COMMON_IMPL) + +GGML_TABLE_BEGIN(uint8_t, kmask_iq2xs, 8) + 1, 2, 4, 8, 16, 32, 64, 128 +GGML_TABLE_END() + +GGML_TABLE_BEGIN(uint8_t, ksigns_iq2xs, 128) + 0, 129, 130, 3, 132, 5, 6, 135, 136, 9, 10, 139, 12, 141, 142, 15, + 144, 17, 18, 147, 20, 149, 150, 23, 24, 153, 154, 27, 156, 29, 30, 159, + 160, 33, 34, 163, 36, 165, 166, 39, 40, 169, 170, 43, 172, 45, 46, 175, + 48, 177, 178, 51, 180, 53, 54, 183, 184, 57, 58, 187, 60, 189, 190, 63, + 192, 65, 66, 195, 68, 197, 198, 71, 72, 201, 202, 75, 204, 77, 78, 207, + 80, 209, 210, 83, 212, 85, 86, 215, 216, 89, 90, 219, 92, 221, 222, 95, + 96, 225, 226, 99, 228, 101, 102, 231, 232, 105, 106, 235, 108, 237, 238, 111, + 240, 113, 114, 243, 116, 245, 246, 119, 120, 249, 250, 123, 252, 125, 126, 255, +GGML_TABLE_END() + +GGML_TABLE_BEGIN(uint64_t, ksigns64, 128) + 0x0000000000000000, 0xff000000000000ff, 0xff0000000000ff00, 0x000000000000ffff, + 0xff00000000ff0000, 0x0000000000ff00ff, 0x0000000000ffff00, 0xff00000000ffffff, + 0xff000000ff000000, 0x00000000ff0000ff, 0x00000000ff00ff00, 0xff000000ff00ffff, + 0x00000000ffff0000, 0xff000000ffff00ff, 0xff000000ffffff00, 0x00000000ffffffff, + 0xff0000ff00000000, 0x000000ff000000ff, 0x000000ff0000ff00, 0xff0000ff0000ffff, + 0x000000ff00ff0000, 0xff0000ff00ff00ff, 0xff0000ff00ffff00, 0x000000ff00ffffff, + 0x000000ffff000000, 0xff0000ffff0000ff, 0xff0000ffff00ff00, 0x000000ffff00ffff, + 0xff0000ffffff0000, 0x000000ffffff00ff, 0x000000ffffffff00, 0xff0000ffffffffff, + 0xff00ff0000000000, 0x0000ff00000000ff, 0x0000ff000000ff00, 0xff00ff000000ffff, + 0x0000ff0000ff0000, 0xff00ff0000ff00ff, 0xff00ff0000ffff00, 0x0000ff0000ffffff, + 0x0000ff00ff000000, 0xff00ff00ff0000ff, 0xff00ff00ff00ff00, 0x0000ff00ff00ffff, + 0xff00ff00ffff0000, 0x0000ff00ffff00ff, 0x0000ff00ffffff00, 0xff00ff00ffffffff, + 0x0000ffff00000000, 0xff00ffff000000ff, 0xff00ffff0000ff00, 0x0000ffff0000ffff, + 0xff00ffff00ff0000, 0x0000ffff00ff00ff, 0x0000ffff00ffff00, 0xff00ffff00ffffff, + 0xff00ffffff000000, 0x0000ffffff0000ff, 0x0000ffffff00ff00, 0xff00ffffff00ffff, + 0x0000ffffffff0000, 0xff00ffffffff00ff, 0xff00ffffffffff00, 0x0000ffffffffffff, + 0xffff000000000000, 0x00ff0000000000ff, 0x00ff00000000ff00, 0xffff00000000ffff, + 0x00ff000000ff0000, 0xffff000000ff00ff, 0xffff000000ffff00, 0x00ff000000ffffff, + 0x00ff0000ff000000, 0xffff0000ff0000ff, 0xffff0000ff00ff00, 0x00ff0000ff00ffff, + 0xffff0000ffff0000, 0x00ff0000ffff00ff, 0x00ff0000ffffff00, 0xffff0000ffffffff, + 0x00ff00ff00000000, 0xffff00ff000000ff, 0xffff00ff0000ff00, 0x00ff00ff0000ffff, + 0xffff00ff00ff0000, 0x00ff00ff00ff00ff, 0x00ff00ff00ffff00, 0xffff00ff00ffffff, + 0xffff00ffff000000, 0x00ff00ffff0000ff, 0x00ff00ffff00ff00, 0xffff00ffff00ffff, + 0x00ff00ffffff0000, 0xffff00ffffff00ff, 0xffff00ffffffff00, 0x00ff00ffffffffff, + 0x00ffff0000000000, 0xffffff00000000ff, 0xffffff000000ff00, 0x00ffff000000ffff, + 0xffffff0000ff0000, 0x00ffff0000ff00ff, 0x00ffff0000ffff00, 0xffffff0000ffffff, + 0xffffff00ff000000, 0x00ffff00ff0000ff, 0x00ffff00ff00ff00, 0xffffff00ff00ffff, + 0x00ffff00ffff0000, 0xffffff00ffff00ff, 0xffffff00ffffff00, 0x00ffff00ffffffff, + 0xffffffff00000000, 0x00ffffff000000ff, 0x00ffffff0000ff00, 0xffffffff0000ffff, + 0x00ffffff00ff0000, 0xffffffff00ff00ff, 0xffffffff00ffff00, 0x00ffffff00ffffff, + 0x00ffffffff000000, 0xffffffffff0000ff, 0xffffffffff00ff00, 0x00ffffffff00ffff, + 0xffffffffffff0000, 0x00ffffffffff00ff, 0x00ffffffffffff00, 0xffffffffffffffff, +GGML_TABLE_END() + + +GGML_TABLE_BEGIN(uint64_t, iq2xxs_grid, 256) + 0x0808080808080808, 0x080808080808082b, 0x0808080808081919, 0x0808080808082b08, + 0x0808080808082b2b, 0x0808080808190819, 0x0808080808191908, 0x08080808082b0808, + 0x08080808082b082b, 0x08080808082b2b08, 0x08080808082b2b2b, 0x0808080819080819, + 0x0808080819081908, 0x0808080819190808, 0x0808080819192b08, 0x08080808192b0819, + 0x08080808192b1908, 0x080808082b080808, 0x080808082b08082b, 0x080808082b082b2b, + 0x080808082b2b082b, 0x0808081908080819, 0x0808081908081908, 0x0808081908190808, + 0x0808081908191919, 0x0808081919080808, 0x080808192b081908, 0x080808192b192b08, + 0x0808082b08080808, 0x0808082b0808082b, 0x0808082b082b082b, 0x0808082b2b08082b, + 0x0808190808080819, 0x0808190808081908, 0x0808190808190808, 0x08081908082b0819, + 0x08081908082b1908, 0x0808190819080808, 0x080819081908082b, 0x0808190819082b08, + 0x08081908192b0808, 0x080819082b080819, 0x080819082b081908, 0x080819082b190808, + 0x080819082b2b1908, 0x0808191908080808, 0x080819190808082b, 0x0808191908082b08, + 0x08081919082b0808, 0x080819191908192b, 0x08081919192b2b19, 0x080819192b080808, + 0x080819192b190819, 0x0808192b08082b19, 0x0808192b08190808, 0x0808192b19080808, + 0x0808192b2b081908, 0x0808192b2b2b1908, 0x08082b0808080808, 0x08082b0808081919, + 0x08082b0808082b08, 0x08082b0808191908, 0x08082b08082b2b08, 0x08082b0819080819, + 0x08082b0819081908, 0x08082b0819190808, 0x08082b081919082b, 0x08082b082b082b08, + 0x08082b1908081908, 0x08082b1919080808, 0x08082b2b0808082b, 0x08082b2b08191908, + 0x0819080808080819, 0x0819080808081908, 0x0819080808190808, 0x08190808082b0819, + 0x0819080819080808, 0x08190808192b0808, 0x081908082b081908, 0x081908082b190808, + 0x081908082b191919, 0x0819081908080808, 0x0819081908082b08, 0x08190819082b0808, + 0x0819081919190808, 0x0819081919192b2b, 0x081908192b080808, 0x0819082b082b1908, + 0x0819082b19081919, 0x0819190808080808, 0x0819190808082b08, 0x08191908082b0808, + 0x08191908082b1919, 0x0819190819082b19, 0x081919082b080808, 0x0819191908192b08, + 0x08191919192b082b, 0x0819192b08080808, 0x0819192b0819192b, 0x08192b0808080819, + 0x08192b0808081908, 0x08192b0808190808, 0x08192b0819080808, 0x08192b082b080819, + 0x08192b1908080808, 0x08192b1908081919, 0x08192b192b2b0808, 0x08192b2b19190819, + 0x082b080808080808, 0x082b08080808082b, 0x082b080808082b2b, 0x082b080819081908, + 0x082b0808192b0819, 0x082b08082b080808, 0x082b08082b08082b, 0x082b0819082b2b19, + 0x082b081919082b08, 0x082b082b08080808, 0x082b082b0808082b, 0x082b190808080819, + 0x082b190808081908, 0x082b190808190808, 0x082b190819080808, 0x082b19081919192b, + 0x082b191908080808, 0x082b191919080819, 0x082b1919192b1908, 0x082b192b2b190808, + 0x082b2b0808082b08, 0x082b2b08082b0808, 0x082b2b082b191908, 0x082b2b2b19081908, + 0x1908080808080819, 0x1908080808081908, 0x1908080808190808, 0x1908080808192b08, + 0x19080808082b0819, 0x19080808082b1908, 0x1908080819080808, 0x1908080819082b08, + 0x190808081919192b, 0x19080808192b0808, 0x190808082b080819, 0x190808082b081908, + 0x190808082b190808, 0x1908081908080808, 0x19080819082b0808, 0x19080819192b0819, + 0x190808192b080808, 0x190808192b081919, 0x1908082b08080819, 0x1908082b08190808, + 0x1908082b19082b08, 0x1908082b1919192b, 0x1908082b192b2b08, 0x1908190808080808, + 0x1908190808082b08, 0x19081908082b0808, 0x190819082b080808, 0x190819082b192b19, + 0x190819190819082b, 0x19081919082b1908, 0x1908192b08080808, 0x19082b0808080819, + 0x19082b0808081908, 0x19082b0808190808, 0x19082b0819080808, 0x19082b0819081919, + 0x19082b1908080808, 0x19082b1919192b08, 0x19082b19192b0819, 0x19082b192b08082b, + 0x19082b2b19081919, 0x19082b2b2b190808, 0x1919080808080808, 0x1919080808082b08, + 0x1919080808190819, 0x1919080808192b19, 0x19190808082b0808, 0x191908082b080808, + 0x191908082b082b08, 0x1919081908081908, 0x191908191908082b, 0x191908192b2b1908, + 0x1919082b2b190819, 0x191919082b190808, 0x191919082b19082b, 0x1919191908082b2b, + 0x1919192b08080819, 0x1919192b19191908, 0x19192b0808080808, 0x19192b0808190819, + 0x19192b0808192b19, 0x19192b08192b1908, 0x19192b1919080808, 0x19192b2b08082b08, + 0x192b080808081908, 0x192b080808190808, 0x192b080819080808, 0x192b0808192b2b08, + 0x192b081908080808, 0x192b081919191919, 0x192b082b08192b08, 0x192b082b192b0808, + 0x192b190808080808, 0x192b190808081919, 0x192b191908190808, 0x192b19190819082b, + 0x192b19192b081908, 0x192b2b081908082b, 0x2b08080808080808, 0x2b0808080808082b, + 0x2b08080808082b2b, 0x2b08080819080819, 0x2b0808082b08082b, 0x2b08081908081908, + 0x2b08081908192b08, 0x2b08081919080808, 0x2b08082b08190819, 0x2b08190808080819, + 0x2b08190808081908, 0x2b08190808190808, 0x2b08190808191919, 0x2b08190819080808, + 0x2b081908192b0808, 0x2b08191908080808, 0x2b0819191908192b, 0x2b0819192b191908, + 0x2b08192b08082b19, 0x2b08192b19080808, 0x2b08192b192b0808, 0x2b082b080808082b, + 0x2b082b1908081908, 0x2b082b2b08190819, 0x2b19080808081908, 0x2b19080808190808, + 0x2b190808082b1908, 0x2b19080819080808, 0x2b1908082b2b0819, 0x2b1908190819192b, + 0x2b1908192b080808, 0x2b19082b19081919, 0x2b19190808080808, 0x2b191908082b082b, + 0x2b19190819081908, 0x2b19191919190819, 0x2b192b082b080819, 0x2b192b19082b0808, + 0x2b2b08080808082b, 0x2b2b080819190808, 0x2b2b08082b081919, 0x2b2b081908082b19, + 0x2b2b082b08080808, 0x2b2b190808192b08, 0x2b2b2b0819190808, 0x2b2b2b1908081908, +GGML_TABLE_END() + +GGML_TABLE_BEGIN(uint64_t, iq2xs_grid, 512) + 0x0808080808080808, 0x080808080808082b, 0x0808080808081919, 0x0808080808082b08, + 0x0808080808082b2b, 0x0808080808190819, 0x0808080808191908, 0x080808080819192b, + 0x0808080808192b19, 0x08080808082b0808, 0x08080808082b082b, 0x08080808082b1919, + 0x08080808082b2b08, 0x0808080819080819, 0x0808080819081908, 0x080808081908192b, + 0x0808080819082b19, 0x0808080819190808, 0x080808081919082b, 0x0808080819191919, + 0x0808080819192b08, 0x08080808192b0819, 0x08080808192b1908, 0x080808082b080808, + 0x080808082b08082b, 0x080808082b081919, 0x080808082b082b08, 0x080808082b190819, + 0x080808082b191908, 0x080808082b192b19, 0x080808082b2b0808, 0x0808081908080819, + 0x0808081908081908, 0x080808190808192b, 0x0808081908082b19, 0x0808081908190808, + 0x080808190819082b, 0x0808081908191919, 0x0808081908192b08, 0x0808081908192b2b, + 0x08080819082b0819, 0x08080819082b1908, 0x0808081919080808, 0x080808191908082b, + 0x0808081919081919, 0x0808081919082b08, 0x0808081919190819, 0x0808081919191908, + 0x08080819192b0808, 0x08080819192b2b08, 0x080808192b080819, 0x080808192b081908, + 0x080808192b190808, 0x0808082b08080808, 0x0808082b0808082b, 0x0808082b08081919, + 0x0808082b08082b08, 0x0808082b08190819, 0x0808082b08191908, 0x0808082b082b0808, + 0x0808082b19080819, 0x0808082b19081908, 0x0808082b19190808, 0x0808082b19191919, + 0x0808082b2b080808, 0x0808082b2b082b2b, 0x0808190808080819, 0x0808190808081908, + 0x080819080808192b, 0x0808190808082b19, 0x0808190808190808, 0x080819080819082b, + 0x0808190808191919, 0x0808190808192b08, 0x08081908082b0819, 0x08081908082b1908, + 0x0808190819080808, 0x080819081908082b, 0x0808190819081919, 0x0808190819082b08, + 0x0808190819190819, 0x0808190819191908, 0x080819081919192b, 0x08081908192b0808, + 0x080819082b080819, 0x080819082b081908, 0x080819082b190808, 0x0808191908080808, + 0x080819190808082b, 0x0808191908081919, 0x0808191908082b08, 0x0808191908190819, + 0x0808191908191908, 0x08081919082b0808, 0x0808191919080819, 0x0808191919081908, + 0x0808191919190808, 0x08081919192b0819, 0x080819192b080808, 0x0808192b08080819, + 0x0808192b08081908, 0x0808192b08190808, 0x0808192b082b192b, 0x0808192b19080808, + 0x0808192b1908082b, 0x0808192b2b081908, 0x08082b0808080808, 0x08082b080808082b, + 0x08082b0808081919, 0x08082b0808082b08, 0x08082b0808082b2b, 0x08082b0808190819, + 0x08082b0808191908, 0x08082b08082b0808, 0x08082b08082b1919, 0x08082b0819080819, + 0x08082b0819081908, 0x08082b0819190808, 0x08082b0819192b08, 0x08082b082b080808, + 0x08082b082b2b0808, 0x08082b082b2b2b2b, 0x08082b1908080819, 0x08082b1908081908, + 0x08082b1908190808, 0x08082b1919080808, 0x08082b192b080819, 0x08082b192b082b19, + 0x08082b2b08080808, 0x08082b2b082b0808, 0x08082b2b082b2b08, 0x08082b2b2b19192b, + 0x08082b2b2b2b0808, 0x0819080808080819, 0x0819080808081908, 0x081908080808192b, + 0x0819080808082b19, 0x0819080808190808, 0x081908080819082b, 0x0819080808191919, + 0x0819080808192b08, 0x08190808082b0819, 0x08190808082b1908, 0x0819080819080808, + 0x081908081908082b, 0x0819080819081919, 0x0819080819082b08, 0x0819080819190819, + 0x0819080819191908, 0x08190808192b0808, 0x08190808192b2b2b, 0x081908082b080819, + 0x081908082b081908, 0x081908082b190808, 0x0819081908080808, 0x081908190808082b, + 0x0819081908081919, 0x0819081908082b08, 0x0819081908190819, 0x0819081908191908, + 0x08190819082b0808, 0x0819081919080819, 0x0819081919081908, 0x0819081919190808, + 0x081908192b080808, 0x081908192b191908, 0x081908192b19192b, 0x0819082b08080819, + 0x0819082b08081908, 0x0819082b0808192b, 0x0819082b08190808, 0x0819082b19080808, + 0x0819082b192b0808, 0x0819190808080808, 0x081919080808082b, 0x0819190808081919, + 0x0819190808082b08, 0x0819190808190819, 0x0819190808191908, 0x08191908082b0808, + 0x0819190819080819, 0x0819190819081908, 0x0819190819082b19, 0x0819190819190808, + 0x08191908192b1908, 0x081919082b080808, 0x0819191908080819, 0x0819191908081908, + 0x0819191908190808, 0x0819191919080808, 0x0819192b08080808, 0x0819192b08191908, + 0x0819192b19082b19, 0x08192b0808080819, 0x08192b0808081908, 0x08192b0808190808, + 0x08192b080819082b, 0x08192b0819080808, 0x08192b0819191908, 0x08192b082b08192b, + 0x08192b1908080808, 0x08192b1908081919, 0x08192b19192b192b, 0x08192b2b19190819, + 0x08192b2b2b2b2b19, 0x082b080808080808, 0x082b08080808082b, 0x082b080808081919, + 0x082b080808082b08, 0x082b080808082b2b, 0x082b080808190819, 0x082b080808191908, + 0x082b0808082b0808, 0x082b080819080819, 0x082b080819081908, 0x082b080819190808, + 0x082b08082b080808, 0x082b08082b2b0808, 0x082b081908080819, 0x082b081908081908, + 0x082b081908190808, 0x082b081919080808, 0x082b081919082b08, 0x082b0819192b1919, + 0x082b082b08080808, 0x082b082b082b082b, 0x082b082b2b080808, 0x082b082b2b2b2b08, + 0x082b190808080819, 0x082b190808081908, 0x082b190808190808, 0x082b1908082b2b19, + 0x082b190819080808, 0x082b191908080808, 0x082b191919080819, 0x082b19191919082b, + 0x082b19192b192b19, 0x082b192b08080819, 0x082b192b08192b2b, 0x082b192b2b2b192b, + 0x082b2b0808080808, 0x082b2b0808082b08, 0x082b2b0808082b2b, 0x082b2b08082b0808, + 0x082b2b0819191919, 0x082b2b082b082b08, 0x082b2b082b2b082b, 0x082b2b19192b2b08, + 0x082b2b192b190808, 0x082b2b2b08082b08, 0x082b2b2b082b0808, 0x082b2b2b2b08082b, + 0x082b2b2b2b082b08, 0x082b2b2b2b082b2b, 0x1908080808080819, 0x1908080808081908, + 0x190808080808192b, 0x1908080808082b19, 0x1908080808190808, 0x190808080819082b, + 0x1908080808191919, 0x1908080808192b08, 0x19080808082b0819, 0x19080808082b1908, + 0x1908080819080808, 0x190808081908082b, 0x1908080819081919, 0x1908080819082b08, + 0x1908080819082b2b, 0x1908080819190819, 0x1908080819191908, 0x19080808192b0808, + 0x19080808192b1919, 0x190808082b080819, 0x190808082b081908, 0x190808082b190808, + 0x1908081908080808, 0x190808190808082b, 0x1908081908081919, 0x1908081908082b08, + 0x1908081908190819, 0x1908081908191908, 0x19080819082b0808, 0x1908081919080819, + 0x1908081919081908, 0x1908081919190808, 0x190808192b080808, 0x190808192b081919, + 0x190808192b2b082b, 0x1908082b08080819, 0x1908082b08081908, 0x1908082b08190808, + 0x1908082b0819082b, 0x1908082b082b2b19, 0x1908082b19080808, 0x1908190808080808, + 0x190819080808082b, 0x1908190808081919, 0x1908190808082b08, 0x1908190808190819, + 0x1908190808191908, 0x1908190808192b19, 0x19081908082b0808, 0x1908190819080819, + 0x1908190819081908, 0x1908190819190808, 0x190819082b080808, 0x190819082b191908, + 0x1908191908080819, 0x1908191908081908, 0x1908191908190808, 0x19081919082b1908, + 0x1908191919080808, 0x190819192b192b2b, 0x1908192b08080808, 0x1908192b08082b2b, + 0x1908192b19081908, 0x1908192b19190808, 0x19082b0808080819, 0x19082b0808081908, + 0x19082b0808190808, 0x19082b0819080808, 0x19082b0819081919, 0x19082b0819191908, + 0x19082b08192b082b, 0x19082b1908080808, 0x19082b1908190819, 0x19082b1919081908, + 0x19082b1919190808, 0x19082b19192b2b19, 0x19082b2b08081908, 0x1919080808080808, + 0x191908080808082b, 0x1919080808081919, 0x1919080808082b08, 0x1919080808190819, + 0x1919080808191908, 0x19190808082b0808, 0x19190808082b2b08, 0x1919080819080819, + 0x1919080819081908, 0x1919080819190808, 0x191908082b080808, 0x1919081908080819, + 0x1919081908081908, 0x1919081908190808, 0x1919081908191919, 0x1919081919080808, + 0x191908191908082b, 0x1919082b08080808, 0x1919082b19081908, 0x1919082b2b2b2b2b, + 0x1919190808080819, 0x1919190808081908, 0x1919190808190808, 0x19191908082b0819, + 0x1919190819080808, 0x19191908192b0808, 0x191919082b080819, 0x191919082b2b0819, + 0x1919191908080808, 0x1919191908082b08, 0x191919192b080808, 0x191919192b082b08, + 0x1919192b082b0819, 0x1919192b192b2b08, 0x1919192b2b2b0819, 0x19192b0808080808, + 0x19192b0808191908, 0x19192b0819080819, 0x19192b0819190808, 0x19192b082b192b19, + 0x19192b1908192b2b, 0x19192b1919080808, 0x19192b191908082b, 0x19192b2b2b081919, + 0x192b080808080819, 0x192b080808081908, 0x192b080808190808, 0x192b080819080808, + 0x192b080819191908, 0x192b0808192b082b, 0x192b08082b08192b, 0x192b08082b2b2b19, + 0x192b081908080808, 0x192b082b082b1908, 0x192b082b19082b2b, 0x192b082b2b19082b, + 0x192b190808080808, 0x192b19080819192b, 0x192b191908190808, 0x192b191919080808, + 0x192b191919081919, 0x192b19192b2b1908, 0x192b2b0808080819, 0x192b2b08192b2b2b, + 0x192b2b19082b1919, 0x192b2b2b0808192b, 0x192b2b2b19191908, 0x192b2b2b192b082b, + 0x2b08080808080808, 0x2b0808080808082b, 0x2b08080808081919, 0x2b08080808082b08, + 0x2b08080808190819, 0x2b08080808191908, 0x2b080808082b0808, 0x2b080808082b2b2b, + 0x2b08080819080819, 0x2b08080819081908, 0x2b08080819190808, 0x2b0808082b080808, + 0x2b0808082b08082b, 0x2b0808082b2b2b08, 0x2b0808082b2b2b2b, 0x2b08081908080819, + 0x2b08081908081908, 0x2b0808190808192b, 0x2b08081908190808, 0x2b08081919080808, + 0x2b08081919190819, 0x2b08081919192b19, 0x2b08082b08080808, 0x2b08082b082b0808, + 0x2b08082b2b080808, 0x2b08082b2b08082b, 0x2b08082b2b2b0808, 0x2b08082b2b2b2b08, + 0x2b08190808080819, 0x2b08190808081908, 0x2b08190808190808, 0x2b0819080819082b, + 0x2b08190808191919, 0x2b08190819080808, 0x2b081908192b0808, 0x2b0819082b082b19, + 0x2b08191908080808, 0x2b08191919081908, 0x2b0819192b2b1919, 0x2b08192b08192b08, + 0x2b08192b192b2b2b, 0x2b082b0808080808, 0x2b082b0808082b08, 0x2b082b08082b1919, + 0x2b082b0819192b2b, 0x2b082b082b080808, 0x2b082b082b08082b, 0x2b082b082b2b2b08, + 0x2b082b190808192b, 0x2b082b2b082b082b, 0x2b082b2b2b080808, 0x2b082b2b2b082b08, + 0x2b082b2b2b19192b, 0x2b082b2b2b2b2b08, 0x2b19080808080819, 0x2b19080808081908, + 0x2b19080808190808, 0x2b19080819080808, 0x2b1908081919192b, 0x2b1908082b081908, + 0x2b19081908080808, 0x2b190819082b082b, 0x2b190819192b1908, 0x2b19082b1919192b, + 0x2b19082b2b082b19, 0x2b19190808080808, 0x2b19190808081919, 0x2b19190819081908, + 0x2b19190819190808, 0x2b19190819192b08, 0x2b191919082b2b19, 0x2b1919192b190808, + 0x2b1919192b19082b, 0x2b19192b19080819, 0x2b192b0819190819, 0x2b192b082b2b192b, + 0x2b192b1919082b19, 0x2b192b2b08191919, 0x2b192b2b192b0808, 0x2b2b080808080808, + 0x2b2b08080808082b, 0x2b2b080808082b08, 0x2b2b080808082b2b, 0x2b2b0808082b0808, + 0x2b2b0808082b2b2b, 0x2b2b08082b2b0808, 0x2b2b081919190819, 0x2b2b081919192b19, + 0x2b2b08192b2b192b, 0x2b2b082b08080808, 0x2b2b082b0808082b, 0x2b2b082b08082b08, + 0x2b2b082b082b2b2b, 0x2b2b082b2b080808, 0x2b2b082b2b2b0808, 0x2b2b190819080808, + 0x2b2b19082b191919, 0x2b2b192b192b1919, 0x2b2b192b2b192b08, 0x2b2b2b0808082b2b, + 0x2b2b2b08082b0808, 0x2b2b2b08082b082b, 0x2b2b2b08082b2b08, 0x2b2b2b082b2b0808, + 0x2b2b2b082b2b2b08, 0x2b2b2b1908081908, 0x2b2b2b192b081908, 0x2b2b2b192b08192b, + 0x2b2b2b2b082b2b08, 0x2b2b2b2b082b2b2b, 0x2b2b2b2b2b190819, 0x2b2b2b2b2b2b2b2b, +GGML_TABLE_END() + +GGML_TABLE_BEGIN(uint64_t, iq2s_grid, 1024) + 0x0808080808080808, 0x080808080808082b, 0x0808080808081919, 0x0808080808082b08, + 0x0808080808082b2b, 0x0808080808190819, 0x0808080808191908, 0x080808080819192b, + 0x0808080808192b19, 0x08080808082b0808, 0x08080808082b082b, 0x08080808082b1919, + 0x08080808082b2b08, 0x0808080819080819, 0x0808080819081908, 0x080808081908192b, + 0x0808080819082b19, 0x0808080819190808, 0x080808081919082b, 0x0808080819191919, + 0x0808080819192b08, 0x08080808192b0819, 0x08080808192b1908, 0x08080808192b192b, + 0x08080808192b2b19, 0x080808082b080808, 0x080808082b08082b, 0x080808082b081919, + 0x080808082b082b08, 0x080808082b190819, 0x080808082b191908, 0x080808082b2b0808, + 0x080808082b2b1919, 0x080808082b2b2b2b, 0x0808081908080819, 0x0808081908081908, + 0x080808190808192b, 0x0808081908082b19, 0x0808081908190808, 0x080808190819082b, + 0x0808081908191919, 0x0808081908192b08, 0x08080819082b0819, 0x08080819082b1908, + 0x0808081919080808, 0x080808191908082b, 0x0808081919081919, 0x0808081919082b08, + 0x0808081919190819, 0x0808081919191908, 0x080808191919192b, 0x0808081919192b19, + 0x08080819192b0808, 0x08080819192b1919, 0x08080819192b2b08, 0x080808192b080819, + 0x080808192b081908, 0x080808192b190808, 0x080808192b19082b, 0x080808192b191919, + 0x080808192b2b0819, 0x080808192b2b1908, 0x0808082b08080808, 0x0808082b0808082b, + 0x0808082b08081919, 0x0808082b08082b08, 0x0808082b08190819, 0x0808082b08191908, + 0x0808082b082b0808, 0x0808082b082b2b2b, 0x0808082b19080819, 0x0808082b19081908, + 0x0808082b1908192b, 0x0808082b19082b19, 0x0808082b19190808, 0x0808082b19191919, + 0x0808082b2b080808, 0x0808082b2b081919, 0x0808082b2b082b2b, 0x0808082b2b191908, + 0x0808082b2b2b082b, 0x0808190808080819, 0x0808190808081908, 0x080819080808192b, + 0x0808190808082b19, 0x0808190808190808, 0x080819080819082b, 0x0808190808191919, + 0x0808190808192b08, 0x08081908082b0819, 0x08081908082b1908, 0x08081908082b192b, + 0x08081908082b2b19, 0x0808190819080808, 0x080819081908082b, 0x0808190819081919, + 0x0808190819082b08, 0x0808190819082b2b, 0x0808190819190819, 0x0808190819191908, + 0x080819081919192b, 0x0808190819192b19, 0x08081908192b0808, 0x08081908192b082b, + 0x08081908192b1919, 0x080819082b080819, 0x080819082b081908, 0x080819082b08192b, + 0x080819082b082b19, 0x080819082b190808, 0x080819082b191919, 0x080819082b192b08, + 0x080819082b2b0819, 0x080819082b2b1908, 0x0808191908080808, 0x080819190808082b, + 0x0808191908081919, 0x0808191908082b08, 0x0808191908082b2b, 0x0808191908190819, + 0x0808191908191908, 0x080819190819192b, 0x0808191908192b19, 0x08081919082b0808, + 0x08081919082b1919, 0x08081919082b2b08, 0x0808191919080819, 0x0808191919081908, + 0x080819191908192b, 0x0808191919082b19, 0x0808191919190808, 0x080819191919082b, + 0x0808191919191919, 0x0808191919192b08, 0x08081919192b0819, 0x08081919192b1908, + 0x080819192b080808, 0x080819192b08082b, 0x080819192b081919, 0x080819192b082b08, + 0x080819192b190819, 0x080819192b191908, 0x080819192b2b0808, 0x0808192b08080819, + 0x0808192b08081908, 0x0808192b0808192b, 0x0808192b08082b19, 0x0808192b08190808, + 0x0808192b08191919, 0x0808192b19080808, 0x0808192b19081919, 0x0808192b19082b08, + 0x0808192b19190819, 0x0808192b19191908, 0x0808192b192b0808, 0x0808192b2b080819, + 0x0808192b2b081908, 0x0808192b2b190808, 0x08082b0808080808, 0x08082b080808082b, + 0x08082b0808081919, 0x08082b0808082b08, 0x08082b0808190819, 0x08082b0808191908, + 0x08082b080819192b, 0x08082b0808192b19, 0x08082b08082b0808, 0x08082b08082b1919, + 0x08082b08082b2b2b, 0x08082b0819080819, 0x08082b0819081908, 0x08082b081908192b, + 0x08082b0819082b19, 0x08082b0819190808, 0x08082b081919082b, 0x08082b0819191919, + 0x08082b0819192b08, 0x08082b08192b0819, 0x08082b08192b1908, 0x08082b082b080808, + 0x08082b082b081919, 0x08082b082b191908, 0x08082b082b2b2b2b, 0x08082b1908080819, + 0x08082b1908081908, 0x08082b1908190808, 0x08082b190819082b, 0x08082b1908191919, + 0x08082b1908192b08, 0x08082b19082b0819, 0x08082b1919080808, 0x08082b1919081919, + 0x08082b1919082b08, 0x08082b1919190819, 0x08082b1919191908, 0x08082b19192b0808, + 0x08082b192b080819, 0x08082b192b190808, 0x08082b2b08080808, 0x08082b2b08190819, + 0x08082b2b08191908, 0x08082b2b082b082b, 0x08082b2b082b2b08, 0x08082b2b082b2b2b, + 0x08082b2b19190808, 0x08082b2b2b192b19, 0x0819080808080819, 0x0819080808081908, + 0x081908080808192b, 0x0819080808082b19, 0x0819080808190808, 0x081908080819082b, + 0x0819080808191919, 0x0819080808192b08, 0x08190808082b0819, 0x08190808082b1908, + 0x08190808082b192b, 0x0819080819080808, 0x081908081908082b, 0x0819080819081919, + 0x0819080819082b08, 0x0819080819190819, 0x0819080819191908, 0x081908081919192b, + 0x0819080819192b19, 0x08190808192b0808, 0x08190808192b082b, 0x08190808192b1919, + 0x08190808192b2b08, 0x081908082b080819, 0x081908082b081908, 0x081908082b08192b, + 0x081908082b190808, 0x081908082b191919, 0x081908082b192b08, 0x081908082b2b0819, + 0x081908082b2b1908, 0x0819081908080808, 0x081908190808082b, 0x0819081908081919, + 0x0819081908082b08, 0x0819081908082b2b, 0x0819081908190819, 0x0819081908191908, + 0x081908190819192b, 0x0819081908192b19, 0x08190819082b0808, 0x08190819082b082b, + 0x08190819082b1919, 0x08190819082b2b08, 0x0819081919080819, 0x0819081919081908, + 0x081908191908192b, 0x0819081919082b19, 0x0819081919190808, 0x081908191919082b, + 0x0819081919191919, 0x0819081919192b08, 0x08190819192b0819, 0x08190819192b1908, + 0x081908192b080808, 0x081908192b08082b, 0x081908192b081919, 0x081908192b082b08, + 0x081908192b190819, 0x081908192b191908, 0x0819082b08080819, 0x0819082b08081908, + 0x0819082b08082b19, 0x0819082b08190808, 0x0819082b08191919, 0x0819082b082b0819, + 0x0819082b082b1908, 0x0819082b19080808, 0x0819082b19081919, 0x0819082b19190819, + 0x0819082b19191908, 0x0819082b2b080819, 0x0819082b2b081908, 0x0819082b2b190808, + 0x0819190808080808, 0x081919080808082b, 0x0819190808081919, 0x0819190808082b08, + 0x0819190808190819, 0x0819190808191908, 0x081919080819192b, 0x0819190808192b19, + 0x08191908082b0808, 0x08191908082b1919, 0x08191908082b2b08, 0x0819190819080819, + 0x0819190819081908, 0x081919081908192b, 0x0819190819082b19, 0x0819190819190808, + 0x081919081919082b, 0x0819190819191919, 0x0819190819192b08, 0x08191908192b0819, + 0x08191908192b1908, 0x081919082b080808, 0x081919082b08082b, 0x081919082b081919, + 0x081919082b082b08, 0x081919082b190819, 0x081919082b191908, 0x081919082b2b0808, + 0x0819191908080819, 0x0819191908081908, 0x081919190808192b, 0x0819191908082b19, + 0x0819191908190808, 0x081919190819082b, 0x0819191908191919, 0x0819191908192b08, + 0x08191919082b0819, 0x08191919082b1908, 0x0819191919080808, 0x081919191908082b, + 0x0819191919081919, 0x0819191919082b08, 0x0819191919190819, 0x0819191919191908, + 0x08191919192b0808, 0x081919192b080819, 0x081919192b081908, 0x081919192b190808, + 0x0819192b08080808, 0x0819192b08081919, 0x0819192b08082b08, 0x0819192b08190819, + 0x0819192b08191908, 0x0819192b082b0808, 0x0819192b19080819, 0x0819192b19081908, + 0x0819192b19190808, 0x0819192b2b080808, 0x0819192b2b2b2b2b, 0x08192b0808080819, + 0x08192b0808081908, 0x08192b080808192b, 0x08192b0808082b19, 0x08192b0808190808, + 0x08192b0808191919, 0x08192b0808192b08, 0x08192b08082b0819, 0x08192b0819080808, + 0x08192b081908082b, 0x08192b0819081919, 0x08192b0819082b08, 0x08192b0819190819, + 0x08192b0819191908, 0x08192b08192b0808, 0x08192b082b080819, 0x08192b082b081908, + 0x08192b1908080808, 0x08192b190808082b, 0x08192b1908081919, 0x08192b1908082b08, + 0x08192b1908190819, 0x08192b1908191908, 0x08192b19082b0808, 0x08192b1919080819, + 0x08192b1919081908, 0x08192b1919190808, 0x08192b19192b2b19, 0x08192b192b2b082b, + 0x08192b2b08081908, 0x08192b2b08190808, 0x08192b2b19080808, 0x08192b2b1919192b, + 0x082b080808080808, 0x082b08080808082b, 0x082b080808081919, 0x082b080808082b08, + 0x082b080808190819, 0x082b080808191908, 0x082b08080819192b, 0x082b080808192b19, + 0x082b0808082b0808, 0x082b0808082b1919, 0x082b0808082b2b2b, 0x082b080819080819, + 0x082b080819081908, 0x082b080819190808, 0x082b08081919082b, 0x082b080819191919, + 0x082b0808192b1908, 0x082b08082b080808, 0x082b08082b082b2b, 0x082b08082b191908, + 0x082b08082b2b2b2b, 0x082b081908080819, 0x082b081908081908, 0x082b081908190808, + 0x082b08190819082b, 0x082b081908191919, 0x082b0819082b0819, 0x082b081919080808, + 0x082b08191908082b, 0x082b081919081919, 0x082b081919190819, 0x082b081919191908, + 0x082b0819192b0808, 0x082b08192b080819, 0x082b08192b081908, 0x082b08192b190808, + 0x082b082b08080808, 0x082b082b08082b2b, 0x082b082b082b082b, 0x082b082b082b2b08, + 0x082b082b082b2b2b, 0x082b082b19081908, 0x082b082b19190808, 0x082b082b2b082b08, + 0x082b082b2b082b2b, 0x082b082b2b2b2b08, 0x082b190808080819, 0x082b190808081908, + 0x082b19080808192b, 0x082b190808082b19, 0x082b190808190808, 0x082b190808191919, + 0x082b190808192b08, 0x082b1908082b0819, 0x082b1908082b1908, 0x082b190819080808, + 0x082b19081908082b, 0x082b190819081919, 0x082b190819082b08, 0x082b190819190819, + 0x082b190819191908, 0x082b1908192b0808, 0x082b19082b080819, 0x082b19082b081908, + 0x082b19082b190808, 0x082b191908080808, 0x082b191908081919, 0x082b191908082b08, + 0x082b191908190819, 0x082b191908191908, 0x082b1919082b0808, 0x082b191919080819, + 0x082b191919081908, 0x082b191919190808, 0x082b1919192b192b, 0x082b19192b080808, + 0x082b192b08080819, 0x082b192b08081908, 0x082b192b08190808, 0x082b192b19080808, + 0x082b192b19192b19, 0x082b2b0808080808, 0x082b2b0808081919, 0x082b2b0808190819, + 0x082b2b0808191908, 0x082b2b0819080819, 0x082b2b0819081908, 0x082b2b0819190808, + 0x082b2b082b082b2b, 0x082b2b082b2b2b2b, 0x082b2b1908080819, 0x082b2b1908081908, + 0x082b2b1908190808, 0x082b2b192b191919, 0x082b2b2b08082b2b, 0x082b2b2b082b082b, + 0x082b2b2b192b1908, 0x082b2b2b2b082b08, 0x082b2b2b2b082b2b, 0x1908080808080819, + 0x1908080808081908, 0x190808080808192b, 0x1908080808082b19, 0x1908080808190808, + 0x190808080819082b, 0x1908080808191919, 0x1908080808192b08, 0x1908080808192b2b, + 0x19080808082b0819, 0x19080808082b1908, 0x19080808082b192b, 0x1908080819080808, + 0x190808081908082b, 0x1908080819081919, 0x1908080819082b08, 0x1908080819082b2b, + 0x1908080819190819, 0x1908080819191908, 0x190808081919192b, 0x1908080819192b19, + 0x19080808192b0808, 0x19080808192b082b, 0x19080808192b1919, 0x190808082b080819, + 0x190808082b081908, 0x190808082b190808, 0x190808082b191919, 0x190808082b192b08, + 0x190808082b2b0819, 0x190808082b2b1908, 0x1908081908080808, 0x190808190808082b, + 0x1908081908081919, 0x1908081908082b08, 0x1908081908190819, 0x1908081908191908, + 0x190808190819192b, 0x1908081908192b19, 0x19080819082b0808, 0x19080819082b082b, + 0x19080819082b1919, 0x1908081919080819, 0x1908081919081908, 0x190808191908192b, + 0x1908081919082b19, 0x1908081919190808, 0x190808191919082b, 0x1908081919191919, + 0x1908081919192b08, 0x19080819192b0819, 0x19080819192b1908, 0x190808192b080808, + 0x190808192b08082b, 0x190808192b081919, 0x190808192b082b08, 0x190808192b190819, + 0x190808192b191908, 0x190808192b2b0808, 0x1908082b08080819, 0x1908082b08081908, + 0x1908082b08190808, 0x1908082b0819082b, 0x1908082b08191919, 0x1908082b08192b08, + 0x1908082b082b1908, 0x1908082b19080808, 0x1908082b19081919, 0x1908082b19082b08, + 0x1908082b19190819, 0x1908082b19191908, 0x1908082b192b0808, 0x1908082b2b080819, + 0x1908082b2b081908, 0x1908190808080808, 0x190819080808082b, 0x1908190808081919, + 0x1908190808082b08, 0x1908190808082b2b, 0x1908190808190819, 0x1908190808191908, + 0x190819080819192b, 0x1908190808192b19, 0x19081908082b0808, 0x19081908082b082b, + 0x19081908082b1919, 0x19081908082b2b08, 0x1908190819080819, 0x1908190819081908, + 0x190819081908192b, 0x1908190819082b19, 0x1908190819190808, 0x190819081919082b, + 0x1908190819191919, 0x1908190819192b08, 0x19081908192b0819, 0x19081908192b1908, + 0x190819082b080808, 0x190819082b08082b, 0x190819082b081919, 0x190819082b082b08, + 0x190819082b190819, 0x190819082b191908, 0x190819082b2b0808, 0x1908191908080819, + 0x1908191908081908, 0x190819190808192b, 0x1908191908082b19, 0x1908191908190808, + 0x190819190819082b, 0x1908191908191919, 0x1908191908192b08, 0x19081919082b0819, + 0x19081919082b1908, 0x1908191919080808, 0x190819191908082b, 0x1908191919081919, + 0x1908191919082b08, 0x1908191919190819, 0x1908191919191908, 0x19081919192b0808, + 0x19081919192b2b2b, 0x190819192b080819, 0x190819192b081908, 0x190819192b190808, + 0x1908192b08080808, 0x1908192b0808082b, 0x1908192b08081919, 0x1908192b08082b08, + 0x1908192b08190819, 0x1908192b08191908, 0x1908192b082b0808, 0x1908192b19080819, + 0x1908192b19081908, 0x1908192b19190808, 0x1908192b2b080808, 0x1908192b2b2b1919, + 0x19082b0808080819, 0x19082b0808081908, 0x19082b0808082b19, 0x19082b0808190808, + 0x19082b080819082b, 0x19082b0808191919, 0x19082b0808192b08, 0x19082b08082b0819, + 0x19082b08082b1908, 0x19082b0819080808, 0x19082b081908082b, 0x19082b0819081919, + 0x19082b0819082b08, 0x19082b0819190819, 0x19082b0819191908, 0x19082b08192b0808, + 0x19082b082b081908, 0x19082b082b190808, 0x19082b1908080808, 0x19082b190808082b, + 0x19082b1908081919, 0x19082b1908082b08, 0x19082b1908190819, 0x19082b1908191908, + 0x19082b19082b0808, 0x19082b1919080819, 0x19082b1919081908, 0x19082b1919190808, + 0x19082b192b080808, 0x19082b192b19192b, 0x19082b2b08080819, 0x19082b2b08081908, + 0x19082b2b08190808, 0x19082b2b19080808, 0x1919080808080808, 0x191908080808082b, + 0x1919080808081919, 0x1919080808082b08, 0x1919080808190819, 0x1919080808191908, + 0x191908080819192b, 0x1919080808192b19, 0x19190808082b0808, 0x19190808082b082b, + 0x19190808082b1919, 0x19190808082b2b08, 0x1919080819080819, 0x1919080819081908, + 0x191908081908192b, 0x1919080819082b19, 0x1919080819190808, 0x191908081919082b, + 0x1919080819191919, 0x1919080819192b08, 0x19190808192b0819, 0x19190808192b1908, + 0x191908082b080808, 0x191908082b08082b, 0x191908082b081919, 0x191908082b082b08, + 0x191908082b190819, 0x191908082b191908, 0x1919081908080819, 0x1919081908081908, + 0x191908190808192b, 0x1919081908082b19, 0x1919081908190808, 0x191908190819082b, + 0x1919081908191919, 0x1919081908192b08, 0x19190819082b0819, 0x19190819082b1908, + 0x1919081919080808, 0x191908191908082b, 0x1919081919081919, 0x1919081919082b08, + 0x1919081919190819, 0x1919081919191908, 0x19190819192b0808, 0x191908192b080819, + 0x191908192b081908, 0x191908192b190808, 0x1919082b08080808, 0x1919082b08081919, + 0x1919082b08082b08, 0x1919082b08190819, 0x1919082b08191908, 0x1919082b082b0808, + 0x1919082b19080819, 0x1919082b19081908, 0x1919082b19190808, 0x1919082b192b2b19, + 0x1919082b2b080808, 0x1919190808080819, 0x1919190808081908, 0x191919080808192b, + 0x1919190808082b19, 0x1919190808190808, 0x191919080819082b, 0x1919190808191919, + 0x1919190808192b08, 0x19191908082b0819, 0x19191908082b1908, 0x1919190819080808, + 0x191919081908082b, 0x1919190819081919, 0x1919190819082b08, 0x1919190819190819, + 0x1919190819191908, 0x19191908192b0808, 0x191919082b080819, 0x191919082b081908, + 0x191919082b190808, 0x1919191908080808, 0x191919190808082b, 0x1919191908081919, + 0x1919191908082b08, 0x1919191908190819, 0x1919191908191908, 0x19191919082b0808, + 0x1919191919080819, 0x1919191919081908, 0x1919191919190808, 0x191919192b080808, + 0x1919192b08080819, 0x1919192b08081908, 0x1919192b08190808, 0x1919192b082b192b, + 0x1919192b19080808, 0x19192b0808080808, 0x19192b080808082b, 0x19192b0808081919, + 0x19192b0808082b08, 0x19192b0808190819, 0x19192b0808191908, 0x19192b08082b0808, + 0x19192b0819080819, 0x19192b0819081908, 0x19192b0819190808, 0x19192b0819192b2b, + 0x19192b082b080808, 0x19192b1908080819, 0x19192b1908081908, 0x19192b1908190808, + 0x19192b1919080808, 0x19192b2b08080808, 0x19192b2b08192b19, 0x19192b2b2b081919, + 0x19192b2b2b2b2b08, 0x192b080808080819, 0x192b080808081908, 0x192b08080808192b, + 0x192b080808190808, 0x192b08080819082b, 0x192b080808191919, 0x192b080808192b08, + 0x192b0808082b0819, 0x192b0808082b1908, 0x192b080819080808, 0x192b080819081919, + 0x192b080819082b08, 0x192b080819190819, 0x192b080819191908, 0x192b0808192b0808, + 0x192b08082b081908, 0x192b08082b190808, 0x192b081908080808, 0x192b08190808082b, + 0x192b081908081919, 0x192b081908082b08, 0x192b081908190819, 0x192b081908191908, + 0x192b0819082b0808, 0x192b081919080819, 0x192b081919081908, 0x192b081919190808, + 0x192b08192b080808, 0x192b08192b192b19, 0x192b082b08081908, 0x192b082b08190808, + 0x192b082b19080808, 0x192b082b1919192b, 0x192b082b2b2b0819, 0x192b190808080808, + 0x192b190808081919, 0x192b190808082b08, 0x192b190808190819, 0x192b190808191908, + 0x192b1908082b0808, 0x192b190819080819, 0x192b190819081908, 0x192b190819190808, + 0x192b19082b080808, 0x192b191908080819, 0x192b191908081908, 0x192b191908190808, + 0x192b191919080808, 0x192b191919082b2b, 0x192b1919192b2b08, 0x192b19192b19082b, + 0x192b192b08080808, 0x192b192b2b191908, 0x192b2b0808080819, 0x192b2b0808081908, + 0x192b2b0808190808, 0x192b2b08192b1919, 0x192b2b082b192b08, 0x192b2b1908080808, + 0x192b2b19082b2b2b, 0x192b2b2b1908082b, 0x192b2b2b2b2b0819, 0x2b08080808080808, + 0x2b0808080808082b, 0x2b08080808081919, 0x2b08080808082b08, 0x2b08080808190819, + 0x2b08080808191908, 0x2b08080808192b19, 0x2b080808082b0808, 0x2b080808082b1919, + 0x2b08080819080819, 0x2b08080819081908, 0x2b08080819190808, 0x2b0808081919082b, + 0x2b08080819191919, 0x2b08080819192b08, 0x2b080808192b0819, 0x2b0808082b080808, + 0x2b0808082b081919, 0x2b0808082b190819, 0x2b0808082b191908, 0x2b08081908080819, + 0x2b08081908081908, 0x2b08081908082b19, 0x2b08081908190808, 0x2b0808190819082b, + 0x2b08081908191919, 0x2b08081908192b08, 0x2b080819082b0819, 0x2b080819082b1908, + 0x2b08081919080808, 0x2b0808191908082b, 0x2b08081919081919, 0x2b08081919082b08, + 0x2b08081919190819, 0x2b08081919191908, 0x2b0808192b080819, 0x2b0808192b081908, + 0x2b0808192b190808, 0x2b0808192b2b2b19, 0x2b08082b08080808, 0x2b08082b08081919, + 0x2b08082b08082b2b, 0x2b08082b08190819, 0x2b08082b08191908, 0x2b08082b19080819, + 0x2b08082b19081908, 0x2b08082b19190808, 0x2b08190808080819, 0x2b08190808081908, + 0x2b0819080808192b, 0x2b08190808082b19, 0x2b08190808190808, 0x2b0819080819082b, + 0x2b08190808191919, 0x2b08190808192b08, 0x2b081908082b0819, 0x2b08190819080808, + 0x2b0819081908082b, 0x2b08190819081919, 0x2b08190819082b08, 0x2b08190819190819, + 0x2b08190819191908, 0x2b081908192b0808, 0x2b0819082b080819, 0x2b0819082b081908, + 0x2b0819082b190808, 0x2b08191908080808, 0x2b0819190808082b, 0x2b08191908081919, + 0x2b08191908082b08, 0x2b08191908190819, 0x2b08191908191908, 0x2b081919082b0808, + 0x2b08191919080819, 0x2b08191919081908, 0x2b08191919190808, 0x2b0819192b080808, + 0x2b0819192b082b2b, 0x2b08192b08080819, 0x2b08192b08081908, 0x2b08192b08190808, + 0x2b08192b082b2b19, 0x2b08192b19080808, 0x2b082b0808080808, 0x2b082b0808081919, + 0x2b082b0808190819, 0x2b082b0808191908, 0x2b082b0819080819, 0x2b082b0819081908, + 0x2b082b0819190808, 0x2b082b082b2b082b, 0x2b082b1908080819, 0x2b082b1908081908, + 0x2b082b1919080808, 0x2b082b19192b1919, 0x2b082b2b082b082b, 0x2b082b2b19192b08, + 0x2b082b2b19192b2b, 0x2b082b2b2b08082b, 0x2b082b2b2b2b082b, 0x2b19080808080819, + 0x2b19080808081908, 0x2b19080808082b19, 0x2b19080808190808, 0x2b1908080819082b, + 0x2b19080808191919, 0x2b19080808192b08, 0x2b190808082b1908, 0x2b19080819080808, + 0x2b1908081908082b, 0x2b19080819081919, 0x2b19080819082b08, 0x2b19080819190819, + 0x2b19080819191908, 0x2b190808192b0808, 0x2b1908082b080819, 0x2b1908082b081908, + 0x2b1908082b190808, 0x2b19081908080808, 0x2b19081908081919, 0x2b19081908190819, + 0x2b19081908191908, 0x2b19081919080819, 0x2b19081919081908, 0x2b19081919190808, + 0x2b19081919192b2b, 0x2b19082b08080819, 0x2b19082b08081908, 0x2b19082b08190808, + 0x2b19082b19080808, 0x2b19082b2b2b192b, 0x2b19190808080808, 0x2b1919080808082b, + 0x2b19190808081919, 0x2b19190808082b08, 0x2b19190808190819, 0x2b19190808191908, + 0x2b191908082b0808, 0x2b19190819080819, 0x2b19190819081908, 0x2b19190819190808, + 0x2b1919082b080808, 0x2b1919082b19192b, 0x2b19191908080819, 0x2b19191908081908, + 0x2b19191908190808, 0x2b19191919080808, 0x2b1919192b192b08, 0x2b1919192b2b0819, + 0x2b19192b08080808, 0x2b19192b1908192b, 0x2b19192b192b1908, 0x2b192b0808080819, + 0x2b192b0808081908, 0x2b192b0808190808, 0x2b192b08082b192b, 0x2b192b0819080808, + 0x2b192b082b2b2b19, 0x2b192b1908080808, 0x2b192b1919082b19, 0x2b192b191919082b, + 0x2b192b2b2b190808, 0x2b2b080808080808, 0x2b2b080808081919, 0x2b2b080808082b2b, + 0x2b2b080808191908, 0x2b2b0808082b082b, 0x2b2b0808082b2b2b, 0x2b2b080819080819, + 0x2b2b080819081908, 0x2b2b080819190808, 0x2b2b08082b2b082b, 0x2b2b08082b2b2b2b, + 0x2b2b081919080808, 0x2b2b0819192b1919, 0x2b2b082b0808082b, 0x2b2b082b08082b2b, + 0x2b2b082b082b082b, 0x2b2b082b082b2b08, 0x2b2b082b082b2b2b, 0x2b2b082b2b08082b, + 0x2b2b082b2b082b08, 0x2b2b082b2b082b2b, 0x2b2b082b2b2b2b08, 0x2b2b190808080819, + 0x2b2b190808081908, 0x2b2b190808190808, 0x2b2b190819080808, 0x2b2b19082b082b19, + 0x2b2b19082b2b1908, 0x2b2b191908080808, 0x2b2b191908192b19, 0x2b2b192b19190819, + 0x2b2b2b0808082b2b, 0x2b2b2b08082b2b08, 0x2b2b2b082b2b082b, 0x2b2b2b1919191908, + 0x2b2b2b192b08192b, 0x2b2b2b2b08082b08, 0x2b2b2b2b08082b2b, 0x2b2b2b2b082b0808, + 0x2b2b2b2b082b082b, 0x2b2b2b2b082b2b08, 0x2b2b2b2b2b082b08, 0x2b2b2b2b2b2b2b2b, +GGML_TABLE_END() + +GGML_TABLE_BEGIN(uint32_t, iq3xxs_grid, 256) + 0x04040404, 0x04040414, 0x04040424, 0x04040c0c, 0x04040c1c, 0x04040c3e, 0x04041404, 0x04041414, + 0x04041c0c, 0x04042414, 0x04043e1c, 0x04043e2c, 0x040c040c, 0x040c041c, 0x040c0c04, 0x040c0c14, + 0x040c140c, 0x040c142c, 0x040c1c04, 0x040c1c14, 0x040c240c, 0x040c2c24, 0x040c3e04, 0x04140404, + 0x04140414, 0x04140424, 0x04140c0c, 0x04141404, 0x04141414, 0x04141c0c, 0x04141c1c, 0x04141c3e, + 0x04142c0c, 0x04142c3e, 0x04143e2c, 0x041c040c, 0x041c043e, 0x041c0c04, 0x041c0c14, 0x041c142c, + 0x041c3e04, 0x04240c1c, 0x04241c3e, 0x04242424, 0x04242c3e, 0x04243e1c, 0x04243e2c, 0x042c040c, + 0x042c043e, 0x042c1c14, 0x042c2c14, 0x04341c2c, 0x04343424, 0x043e0c04, 0x043e0c24, 0x043e0c34, + 0x043e241c, 0x043e340c, 0x0c04040c, 0x0c04041c, 0x0c040c04, 0x0c040c14, 0x0c04140c, 0x0c04141c, + 0x0c041c04, 0x0c041c14, 0x0c041c24, 0x0c04243e, 0x0c042c04, 0x0c0c0404, 0x0c0c0414, 0x0c0c0c0c, + 0x0c0c1404, 0x0c0c1414, 0x0c14040c, 0x0c14041c, 0x0c140c04, 0x0c140c14, 0x0c14140c, 0x0c141c04, + 0x0c143e14, 0x0c1c0404, 0x0c1c0414, 0x0c1c1404, 0x0c1c1c0c, 0x0c1c2434, 0x0c1c3434, 0x0c24040c, + 0x0c24042c, 0x0c242c04, 0x0c2c1404, 0x0c2c1424, 0x0c2c2434, 0x0c2c3e0c, 0x0c34042c, 0x0c3e1414, + 0x0c3e2404, 0x14040404, 0x14040414, 0x14040c0c, 0x14040c1c, 0x14041404, 0x14041414, 0x14041434, + 0x14041c0c, 0x14042414, 0x140c040c, 0x140c041c, 0x140c042c, 0x140c0c04, 0x140c0c14, 0x140c140c, + 0x140c1c04, 0x140c341c, 0x140c343e, 0x140c3e04, 0x14140404, 0x14140414, 0x14140c0c, 0x14140c3e, + 0x14141404, 0x14141414, 0x14141c3e, 0x14142404, 0x14142c2c, 0x141c040c, 0x141c0c04, 0x141c0c24, + 0x141c3e04, 0x141c3e24, 0x14241c2c, 0x14242c1c, 0x142c041c, 0x142c143e, 0x142c240c, 0x142c3e24, + 0x143e040c, 0x143e041c, 0x143e0c34, 0x143e242c, 0x1c04040c, 0x1c040c04, 0x1c040c14, 0x1c04140c, + 0x1c04141c, 0x1c042c04, 0x1c04342c, 0x1c043e14, 0x1c0c0404, 0x1c0c0414, 0x1c0c1404, 0x1c0c1c0c, + 0x1c0c2424, 0x1c0c2434, 0x1c14040c, 0x1c14041c, 0x1c140c04, 0x1c14142c, 0x1c142c14, 0x1c143e14, + 0x1c1c0c0c, 0x1c1c1c1c, 0x1c241c04, 0x1c24243e, 0x1c243e14, 0x1c2c0404, 0x1c2c0434, 0x1c2c1414, + 0x1c2c2c2c, 0x1c340c24, 0x1c341c34, 0x1c34341c, 0x1c3e1c1c, 0x1c3e3404, 0x24040424, 0x24040c3e, + 0x24041c2c, 0x24041c3e, 0x24042c1c, 0x24042c3e, 0x240c3e24, 0x24141404, 0x24141c3e, 0x24142404, + 0x24143404, 0x24143434, 0x241c043e, 0x241c242c, 0x24240424, 0x24242c0c, 0x24243424, 0x242c142c, + 0x242c241c, 0x242c3e04, 0x243e042c, 0x243e0c04, 0x243e0c14, 0x243e1c04, 0x2c040c14, 0x2c04240c, + 0x2c043e04, 0x2c0c0404, 0x2c0c0434, 0x2c0c1434, 0x2c0c2c2c, 0x2c140c24, 0x2c141c14, 0x2c143e14, + 0x2c1c0414, 0x2c1c2c1c, 0x2c240c04, 0x2c24141c, 0x2c24143e, 0x2c243e14, 0x2c2c0414, 0x2c2c1c0c, + 0x2c342c04, 0x2c3e1424, 0x2c3e2414, 0x34041424, 0x34042424, 0x34042434, 0x34043424, 0x340c140c, + 0x340c340c, 0x34140c3e, 0x34143424, 0x341c1c04, 0x341c1c34, 0x34242424, 0x342c042c, 0x342c2c14, + 0x34341c1c, 0x343e041c, 0x343e140c, 0x3e04041c, 0x3e04042c, 0x3e04043e, 0x3e040c04, 0x3e041c14, + 0x3e042c14, 0x3e0c1434, 0x3e0c2404, 0x3e140c14, 0x3e14242c, 0x3e142c14, 0x3e1c0404, 0x3e1c0c2c, + 0x3e1c1c1c, 0x3e1c3404, 0x3e24140c, 0x3e24240c, 0x3e2c0404, 0x3e2c0414, 0x3e2c1424, 0x3e341c04, +GGML_TABLE_END() + +GGML_TABLE_BEGIN(uint32_t, iq3s_grid, 512) + 0x01010101, 0x01010103, 0x01010105, 0x0101010b, 0x0101010f, 0x01010301, 0x01010303, 0x01010305, + 0x01010309, 0x0101030d, 0x01010501, 0x01010503, 0x0101050b, 0x01010707, 0x01010901, 0x01010905, + 0x0101090b, 0x0101090f, 0x01010b03, 0x01010b07, 0x01010d01, 0x01010d05, 0x01010f03, 0x01010f09, + 0x01010f0f, 0x01030101, 0x01030103, 0x01030105, 0x01030109, 0x01030301, 0x01030303, 0x0103030b, + 0x01030501, 0x01030507, 0x0103050f, 0x01030703, 0x0103070b, 0x01030909, 0x01030d03, 0x01030d0b, + 0x01030f05, 0x01050101, 0x01050103, 0x0105010b, 0x0105010f, 0x01050301, 0x01050307, 0x0105030d, + 0x01050503, 0x0105050b, 0x01050701, 0x01050709, 0x01050905, 0x0105090b, 0x0105090f, 0x01050b03, + 0x01050b07, 0x01050f01, 0x01050f07, 0x01070107, 0x01070303, 0x0107030b, 0x01070501, 0x01070505, + 0x01070703, 0x01070707, 0x0107070d, 0x01070909, 0x01070b01, 0x01070b05, 0x01070d0f, 0x01070f03, + 0x01070f0b, 0x01090101, 0x01090307, 0x0109030f, 0x01090503, 0x01090509, 0x01090705, 0x01090901, + 0x01090907, 0x01090b03, 0x01090f01, 0x010b0105, 0x010b0109, 0x010b0501, 0x010b0505, 0x010b050d, + 0x010b0707, 0x010b0903, 0x010b090b, 0x010b090f, 0x010b0d0d, 0x010b0f07, 0x010d010d, 0x010d0303, + 0x010d0307, 0x010d0703, 0x010d0b05, 0x010d0f03, 0x010f0101, 0x010f0105, 0x010f0109, 0x010f0501, + 0x010f0505, 0x010f050d, 0x010f0707, 0x010f0b01, 0x010f0b09, 0x03010101, 0x03010103, 0x03010105, + 0x03010109, 0x03010301, 0x03010303, 0x03010307, 0x0301030b, 0x0301030f, 0x03010501, 0x03010505, + 0x03010703, 0x03010709, 0x0301070d, 0x03010b09, 0x03010b0d, 0x03010d03, 0x03010f05, 0x03030101, + 0x03030103, 0x03030107, 0x0303010d, 0x03030301, 0x03030309, 0x03030503, 0x03030701, 0x03030707, + 0x03030903, 0x03030b01, 0x03030b05, 0x03030f01, 0x03030f0d, 0x03050101, 0x03050305, 0x0305030b, + 0x0305030f, 0x03050501, 0x03050509, 0x03050705, 0x03050901, 0x03050907, 0x03050b0b, 0x03050d01, + 0x03050f05, 0x03070103, 0x03070109, 0x0307010f, 0x03070301, 0x03070307, 0x03070503, 0x0307050f, + 0x03070701, 0x03070709, 0x03070903, 0x03070d05, 0x03070f01, 0x03090107, 0x0309010b, 0x03090305, + 0x03090309, 0x03090703, 0x03090707, 0x03090905, 0x0309090d, 0x03090b01, 0x03090b09, 0x030b0103, + 0x030b0301, 0x030b0307, 0x030b0503, 0x030b0701, 0x030b0705, 0x030b0b03, 0x030d0501, 0x030d0509, + 0x030d050f, 0x030d0909, 0x030d090d, 0x030f0103, 0x030f0107, 0x030f0301, 0x030f0305, 0x030f0503, + 0x030f070b, 0x030f0903, 0x030f0d05, 0x030f0f01, 0x05010101, 0x05010103, 0x05010107, 0x0501010b, + 0x0501010f, 0x05010301, 0x05010305, 0x05010309, 0x0501030d, 0x05010503, 0x05010507, 0x0501050f, + 0x05010701, 0x05010705, 0x05010903, 0x05010907, 0x0501090b, 0x05010b01, 0x05010b05, 0x05010d0f, + 0x05010f01, 0x05010f07, 0x05010f0b, 0x05030101, 0x05030105, 0x05030301, 0x05030307, 0x0503030f, + 0x05030505, 0x0503050b, 0x05030703, 0x05030709, 0x05030905, 0x05030b03, 0x05050103, 0x05050109, + 0x0505010f, 0x05050503, 0x05050507, 0x05050701, 0x0505070f, 0x05050903, 0x05050b07, 0x05050b0f, + 0x05050f03, 0x05050f09, 0x05070101, 0x05070105, 0x0507010b, 0x05070303, 0x05070505, 0x05070509, + 0x05070703, 0x05070707, 0x05070905, 0x05070b01, 0x05070d0d, 0x05090103, 0x0509010f, 0x05090501, + 0x05090507, 0x05090705, 0x0509070b, 0x05090903, 0x05090f05, 0x05090f0b, 0x050b0109, 0x050b0303, + 0x050b0505, 0x050b070f, 0x050b0901, 0x050b0b07, 0x050b0f01, 0x050d0101, 0x050d0105, 0x050d010f, + 0x050d0503, 0x050d0b0b, 0x050d0d03, 0x050f010b, 0x050f0303, 0x050f050d, 0x050f0701, 0x050f0907, + 0x050f0b01, 0x07010105, 0x07010303, 0x07010307, 0x0701030b, 0x0701030f, 0x07010505, 0x07010703, + 0x07010707, 0x0701070b, 0x07010905, 0x07010909, 0x0701090f, 0x07010b03, 0x07010d07, 0x07010f03, + 0x07030103, 0x07030107, 0x0703010b, 0x07030309, 0x07030503, 0x07030507, 0x07030901, 0x07030d01, + 0x07030f05, 0x07030f0d, 0x07050101, 0x07050305, 0x07050501, 0x07050705, 0x07050709, 0x07050b01, + 0x07070103, 0x07070301, 0x07070309, 0x07070503, 0x07070507, 0x0707050f, 0x07070701, 0x07070903, + 0x07070907, 0x0707090f, 0x07070b0b, 0x07070f07, 0x07090107, 0x07090303, 0x0709030d, 0x07090505, + 0x07090703, 0x07090b05, 0x07090d01, 0x07090d09, 0x070b0103, 0x070b0301, 0x070b0305, 0x070b050b, + 0x070b0705, 0x070b0909, 0x070b0b0d, 0x070b0f07, 0x070d030d, 0x070d0903, 0x070f0103, 0x070f0107, + 0x070f0501, 0x070f0505, 0x070f070b, 0x09010101, 0x09010109, 0x09010305, 0x09010501, 0x09010509, + 0x0901050f, 0x09010705, 0x09010903, 0x09010b01, 0x09010f01, 0x09030105, 0x0903010f, 0x09030303, + 0x09030307, 0x09030505, 0x09030701, 0x0903070b, 0x09030907, 0x09030b03, 0x09030b0b, 0x09050103, + 0x09050107, 0x09050301, 0x0905030b, 0x09050503, 0x09050707, 0x09050901, 0x09050b0f, 0x09050d05, + 0x09050f01, 0x09070109, 0x09070303, 0x09070307, 0x09070501, 0x09070505, 0x09070703, 0x0907070b, + 0x09090101, 0x09090105, 0x09090509, 0x0909070f, 0x09090901, 0x09090f03, 0x090b010b, 0x090b010f, + 0x090b0503, 0x090b0d05, 0x090d0307, 0x090d0709, 0x090d0d01, 0x090f0301, 0x090f030b, 0x090f0701, + 0x090f0907, 0x090f0b03, 0x0b010105, 0x0b010301, 0x0b010309, 0x0b010505, 0x0b010901, 0x0b010909, + 0x0b01090f, 0x0b010b05, 0x0b010d0d, 0x0b010f09, 0x0b030103, 0x0b030107, 0x0b03010b, 0x0b030305, + 0x0b030503, 0x0b030705, 0x0b030f05, 0x0b050101, 0x0b050303, 0x0b050507, 0x0b050701, 0x0b05070d, + 0x0b050b07, 0x0b070105, 0x0b07010f, 0x0b070301, 0x0b07050f, 0x0b070909, 0x0b070b03, 0x0b070d0b, + 0x0b070f07, 0x0b090103, 0x0b090109, 0x0b090501, 0x0b090705, 0x0b09090d, 0x0b0b0305, 0x0b0b050d, + 0x0b0b0b03, 0x0b0b0b07, 0x0b0d0905, 0x0b0f0105, 0x0b0f0109, 0x0b0f0505, 0x0d010303, 0x0d010307, + 0x0d01030b, 0x0d010703, 0x0d010707, 0x0d010d01, 0x0d030101, 0x0d030501, 0x0d03050f, 0x0d030d09, + 0x0d050305, 0x0d050709, 0x0d050905, 0x0d050b0b, 0x0d050d05, 0x0d050f01, 0x0d070101, 0x0d070309, + 0x0d070503, 0x0d070901, 0x0d09050b, 0x0d090907, 0x0d090d05, 0x0d0b0101, 0x0d0b0107, 0x0d0b0709, + 0x0d0b0d01, 0x0d0d010b, 0x0d0d0901, 0x0d0f0303, 0x0d0f0307, 0x0f010101, 0x0f010109, 0x0f01010f, + 0x0f010501, 0x0f010505, 0x0f01070d, 0x0f010901, 0x0f010b09, 0x0f010d05, 0x0f030105, 0x0f030303, + 0x0f030509, 0x0f030907, 0x0f03090b, 0x0f050103, 0x0f050109, 0x0f050301, 0x0f05030d, 0x0f050503, + 0x0f050701, 0x0f050b03, 0x0f070105, 0x0f070705, 0x0f07070b, 0x0f070b07, 0x0f090103, 0x0f09010b, + 0x0f090307, 0x0f090501, 0x0f090b01, 0x0f0b0505, 0x0f0b0905, 0x0f0d0105, 0x0f0d0703, 0x0f0f0101, +GGML_TABLE_END() + +// TODO: fix name to kvalues_iq4_nl +GGML_TABLE_BEGIN(int8_t, kvalues_iq4nl, 16) + -127, -104, -83, -65, -49, -35, -22, -10, 1, 13, 25, 38, 53, 69, 89, 113, +GGML_TABLE_END() + +// e2m1 values (doubled), shared by MXFP4 and NVFP4 +// ref: https://www.opencompute.org/documents/ocp-microscaling-formats-mx-v1-0-spec-final-pdf +GGML_TABLE_BEGIN(int8_t, kvalues_fp4, 16) + 0, 1, 2, 3, 4, 6, 8, 12, 0, -1, -2, -3, -4, -6, -8, -12, +GGML_TABLE_END() +#define kvalues_mxfp4 kvalues_fp4 + +#define NGRID_IQ1S 2048 +#define IQ1S_DELTA 0.125f +#define IQ1M_DELTA 0.125f +#if defined(GGML_COMMON_IMPL_C) +GGML_TABLE_BEGIN(uint64_t, iq1s_grid, NGRID_IQ1S) + 0xffffffffffffffff, 0xffffffffffffff01, 0xffffffffffff0000, 0xffffffffffff01ff, + 0xffffffffffff0101, 0xffffffffff00ff00, 0xffffffffff000000, 0xffffffffff01ffff, + 0xffffffffff01ff01, 0xffffffffff0101ff, 0xffffffffff010101, 0xffffffff00ff0000, + 0xffffffff0000ff00, 0xffffffff000000ff, 0xffffffff00000001, 0xffffffff00010000, + 0xffffffff01ffffff, 0xffffffff01ffff01, 0xffffffff01ff01ff, 0xffffffff01ff0101, + 0xffffffff01000000, 0xffffffff0101ffff, 0xffffffff0101ff01, 0xffffffff010101ff, + 0xffffffff01010101, 0xffffff00ffff00ff, 0xffffff00ffff0000, 0xffffff00ff00ff00, + 0xffffff00ff0000ff, 0xffffff00ff000001, 0xffffff00ff000100, 0xffffff00ff000101, + 0xffffff00ff010000, 0xffffff0000ffff00, 0xffffff0000ff0001, 0xffffff0000ff0100, + 0xffffff000000ff01, 0xffffff0000000000, 0xffffff0000000101, 0xffffff000001ff00, + 0xffffff00000100ff, 0xffffff0000010001, 0xffffff00000101ff, 0xffffff0001ff0000, + 0xffffff000100ff00, 0xffffff00010000ff, 0xffffff0001000001, 0xffffff0001010000, + 0xffffff01ffffffff, 0xffffff01ffffff01, 0xffffff01ffff01ff, 0xffffff01ffff0101, + 0xffffff01ff000000, 0xffffff01ff01ffff, 0xffffff01ff01ff01, 0xffffff01ff0101ff, + 0xffffff01ff010101, 0xffffff0100ff0000, 0xffffff010000ff00, 0xffffff0100000100, + 0xffffff01000100ff, 0xffffff0100010100, 0xffffff0101ffffff, 0xffffff0101ffff01, + 0xffffff0101ff01ff, 0xffffff0101ff0101, 0xffffff010100ff00, 0xffffff0101000000, + 0xffffff0101000100, 0xffffff010101ffff, 0xffffff010101ff01, 0xffffff01010101ff, + 0xffffff0101010101, 0xffff00ffff00ff00, 0xffff00ffff0000ff, 0xffff00ffff000001, + 0xffff00ffff010000, 0xffff00ff00ffff00, 0xffff00ff00ff0100, 0xffff00ff00000000, + 0xffff00ff00000101, 0xffff00ff000100ff, 0xffff00ff00010000, 0xffff00ff0100ff00, + 0xffff00ff01000100, 0xffff00ff01010000, 0xffff0000ffffff00, 0xffff0000ffff00ff, + 0xffff0000ffff0000, 0xffff0000ffff0001, 0xffff0000ff000000, 0xffff0000ff0001ff, + 0xffff0000ff000101, 0xffff0000ff010100, 0xffff000000ffffff, 0xffff000000ff0000, + 0xffff000000ff0101, 0xffff00000000ffff, 0xffff00000000ff00, 0xffff0000000000ff, + 0xffff000000000000, 0xffff000000000001, 0xffff000000000100, 0xffff00000001ffff, + 0xffff00000001ff01, 0xffff000000010000, 0xffff0000000101ff, 0xffff000000010101, + 0xffff000001ffff00, 0xffff00000100ff00, 0xffff000001000000, 0xffff0000010001ff, + 0xffff000001000101, 0xffff00000101ff00, 0xffff0000010100ff, 0xffff000001010000, + 0xffff000001010001, 0xffff000001010100, 0xffff0001ff0000ff, 0xffff0001ff000100, + 0xffff000100ffff00, 0xffff000100ff00ff, 0xffff00010000ffff, 0xffff00010000ff01, + 0xffff000100000000, 0xffff0001000001ff, 0xffff00010001ffff, 0xffff00010001ff00, + 0xffff000100010001, 0xffff000100010100, 0xffff000101ff0000, 0xffff00010100ff00, + 0xffff0001010000ff, 0xffff000101000100, 0xffff01ffffffffff, 0xffff01ffffffff01, + 0xffff01ffffff01ff, 0xffff01ffffff0101, 0xffff01ffff000000, 0xffff01ffff01ffff, + 0xffff01ffff01ff01, 0xffff01ffff0101ff, 0xffff01ffff010101, 0xffff01ff00ff0000, + 0xffff01ff0000ff00, 0xffff01ff00000001, 0xffff01ff00010000, 0xffff01ff01ffffff, + 0xffff01ff01ffff01, 0xffff01ff01ff01ff, 0xffff01ff01ff0101, 0xffff01ff01000000, + 0xffff01ff0101ffff, 0xffff01ff0101ff01, 0xffff01ff010101ff, 0xffff01ff01010101, + 0xffff0100ffff0000, 0xffff0100ff00ff00, 0xffff0100ff0000ff, 0xffff0100ff000100, + 0xffff0100ff0100ff, 0xffff0100ff010000, 0xffff010000ffff00, 0xffff01000000ffff, + 0xffff01000000ff00, 0xffff010000000000, 0xffff01000001ff00, 0xffff0100000100ff, + 0xffff010000010100, 0xffff01000100ff00, 0xffff0100010000ff, 0xffff010001000001, + 0xffff010001000100, 0xffff010001010000, 0xffff0101ffffffff, 0xffff0101ffffff01, + 0xffff0101ffff01ff, 0xffff0101ffff0101, 0xffff0101ff000000, 0xffff0101ff01ffff, + 0xffff0101ff01ff01, 0xffff0101ff0101ff, 0xffff0101ff010101, 0xffff010100ff0000, + 0xffff01010000ff00, 0xffff010100000100, 0xffff01010001ff00, 0xffff010100010000, + 0xffff010101ffffff, 0xffff010101ffff01, 0xffff010101ff0000, 0xffff010101ff01ff, + 0xffff010101ff0101, 0xffff010101000000, 0xffff01010101ffff, 0xffff01010101ff01, + 0xffff0101010101ff, 0xffff010101010101, 0xff00ffffff00ffff, 0xff00ffffff00ff00, + 0xff00ffffff0000ff, 0xff00ffffff000100, 0xff00ffffff0100ff, 0xff00ffffff010000, + 0xff00ffff00ffff00, 0xff00ffff00ff00ff, 0xff00ffff0000ffff, 0xff00ffff00000000, + 0xff00ffff000001ff, 0xff00ffff0001ff00, 0xff00ffff000100ff, 0xff00ffff00010000, + 0xff00ffff00010100, 0xff00ffff0100ff00, 0xff00ffff010000ff, 0xff00ffff01000001, + 0xff00ffff0101ff00, 0xff00ffff01010000, 0xff00ff00ffffff00, 0xff00ff00ffff00ff, + 0xff00ff00ffff0001, 0xff00ff00ffff0100, 0xff00ff00ff00ffff, 0xff00ff00ff00ff01, + 0xff00ff00ff000000, 0xff00ff00ff0001ff, 0xff00ff00ff01ff00, 0xff00ff00ff0100ff, + 0xff00ff00ff010100, 0xff00ff0000ff0000, 0xff00ff0000ff0101, 0xff00ff000000ffff, + 0xff00ff000000ff00, 0xff00ff000000ff01, 0xff00ff00000000ff, 0xff00ff0000000000, + 0xff00ff0000000001, 0xff00ff0000000100, 0xff00ff000001ffff, 0xff00ff0000010000, + 0xff00ff0001ff00ff, 0xff00ff000100ff01, 0xff00ff0001000000, 0xff00ff000101ff00, + 0xff00ff00010100ff, 0xff00ff01ff00ff00, 0xff00ff01ff0000ff, 0xff00ff01ff000001, + 0xff00ff01ff010000, 0xff00ff0100ffffff, 0xff00ff0100ff0001, 0xff00ff0100ff0100, + 0xff00ff010000ff01, 0xff00ff0100000000, 0xff00ff01000001ff, 0xff00ff0100000101, + 0xff00ff01000100ff, 0xff00ff0100010001, 0xff00ff0101ff0000, 0xff00ff010100ff00, + 0xff00ff01010000ff, 0xff00ff0101000001, 0xff00ff0101010000, 0xff0000ffffffff00, + 0xff0000ffffff0001, 0xff0000ffffff0100, 0xff0000ffff0000ff, 0xff0000ffff000000, + 0xff0000ffff0001ff, 0xff0000ffff000100, 0xff0000ffff01ff00, 0xff0000ffff010001, + 0xff0000ff00ffff00, 0xff0000ff00ff0000, 0xff0000ff00ff0001, 0xff0000ff00ff01ff, + 0xff0000ff00ff0101, 0xff0000ff0000ff00, 0xff0000ff000000ff, 0xff0000ff00000000, + 0xff0000ff00000001, 0xff0000ff00000100, 0xff0000ff0001ff01, 0xff0000ff00010000, + 0xff0000ff000101ff, 0xff0000ff01ff00ff, 0xff0000ff01ff0100, 0xff0000ff0100ffff, + 0xff0000ff010000ff, 0xff0000ff01000000, 0xff0000ff010001ff, 0xff0000ff01000100, + 0xff0000ff01000101, 0xff0000ff0101ff00, 0xff0000ff010100ff, 0xff0000ff01010000, + 0xff0000ff01010100, 0xff000000ffffff01, 0xff000000ffff0000, 0xff000000ffff0101, + 0xff000000ff00ff00, 0xff000000ff0000ff, 0xff000000ff000000, 0xff000000ff000001, + 0xff000000ff000100, 0xff000000ff01ffff, 0xff000000ff01ff01, 0xff000000ff010000, + 0xff000000ff0101ff, 0xff000000ff010101, 0xff00000000ffff00, 0xff00000000ff00ff, + 0xff00000000ff0000, 0xff00000000ff0001, 0xff0000000000ff00, 0xff0000000000ff01, + 0xff000000000000ff, 0xff00000000000000, 0xff00000000000001, 0xff00000000000100, + 0xff00000000000101, 0xff0000000001ff00, 0xff000000000100ff, 0xff00000000010000, + 0xff00000000010001, 0xff00000000010100, 0xff00000001ffffff, 0xff00000001ffff01, + 0xff00000001ff00ff, 0xff00000001ff0000, 0xff00000001ff01ff, 0xff00000001ff0101, + 0xff0000000100ffff, 0xff0000000100ff00, 0xff000000010000ff, 0xff00000001000000, + 0xff00000001000001, 0xff00000001000100, 0xff00000001000101, 0xff0000000101ffff, + 0xff0000000101ff01, 0xff00000001010000, 0xff000001ffffff00, 0xff000001ffff00ff, + 0xff000001ffff0000, 0xff000001ffff0001, 0xff000001ff000000, 0xff000001ff000001, + 0xff000001ff0001ff, 0xff000001ff000101, 0xff000001ff01ff00, 0xff000001ff010001, + 0xff00000100ffffff, 0xff00000100ffff01, 0xff00000100ff00ff, 0xff00000100ff0000, + 0xff00000100ff01ff, 0xff00000100ff0101, 0xff0000010000ff00, 0xff00000100000000, + 0xff00000100000001, 0xff000001000001ff, 0xff00000100000100, 0xff0000010001ff00, + 0xff000001000100ff, 0xff00000100010000, 0xff000001000101ff, 0xff00000100010100, + 0xff00000100010101, 0xff00000101ff0001, 0xff00000101ff0101, 0xff0000010100ff01, + 0xff00000101000000, 0xff000001010100ff, 0xff00000101010100, 0xff0001ffff00ff00, + 0xff0001ffff000001, 0xff0001ffff010000, 0xff0001ff00ffff00, 0xff0001ff00ff00ff, + 0xff0001ff00ff0001, 0xff0001ff00ff0100, 0xff0001ff0000ffff, 0xff0001ff00000000, + 0xff0001ff000001ff, 0xff0001ff00000101, 0xff0001ff0001ffff, 0xff0001ff0001ff00, + 0xff0001ff000100ff, 0xff0001ff00010001, 0xff0001ff00010100, 0xff0001ff01ff0000, + 0xff0001ff0100ff00, 0xff0001ff010000ff, 0xff0001ff01010000, 0xff000100ff00ffff, + 0xff000100ff00ff01, 0xff000100ff000000, 0xff000100ff000101, 0xff000100ff01ff00, + 0xff000100ff010000, 0xff00010000ffff01, 0xff00010000ff00ff, 0xff00010000ff0000, + 0xff00010000ff01ff, 0xff0001000000ff00, 0xff000100000000ff, 0xff00010000000000, + 0xff00010000000001, 0xff00010000000100, 0xff00010000000101, 0xff0001000001ffff, + 0xff00010000010000, 0xff00010000010101, 0xff00010001ff0100, 0xff0001000100ff00, + 0xff0001000100ff01, 0xff00010001000000, 0xff000100010001ff, 0xff0001000101ff00, + 0xff00010001010001, 0xff00010001010100, 0xff000101ffff0100, 0xff000101ff000001, + 0xff000101ff0100ff, 0xff000101ff010001, 0xff00010100ff00ff, 0xff00010100ff0001, + 0xff00010100ff0100, 0xff0001010000ffff, 0xff0001010000ff01, 0xff00010100000000, + 0xff000101000001ff, 0xff0001010001ff00, 0xff00010100010001, 0xff00010100010100, + 0xff00010101ff0000, 0xff0001010100ff00, 0xff00010101000001, 0xff00010101000101, + 0xff01ffffffffffff, 0xff01ffffffffff01, 0xff01ffffffff01ff, 0xff01ffffffff0101, + 0xff01ffffff000000, 0xff01ffffff01ffff, 0xff01ffffff01ff01, 0xff01ffffff010000, + 0xff01ffffff0101ff, 0xff01ffffff010101, 0xff01ffff00ff0000, 0xff01ffff0000ff00, + 0xff01ffff00000100, 0xff01ffff0001ff00, 0xff01ffff00010000, 0xff01ffff01ffffff, + 0xff01ffff01ffff01, 0xff01ffff01ff01ff, 0xff01ffff01ff0101, 0xff01ffff01000000, + 0xff01ffff0101ffff, 0xff01ffff0101ff01, 0xff01ffff01010000, 0xff01ffff010101ff, + 0xff01ffff01010101, 0xff01ff00ffff0000, 0xff01ff00ff00ff00, 0xff01ff00ff0000ff, + 0xff01ff00ff000100, 0xff01ff00ff010000, 0xff01ff0000ffff01, 0xff01ff0000ff00ff, + 0xff01ff0000ff0100, 0xff01ff0000000000, 0xff01ff00000001ff, 0xff01ff0000000101, + 0xff01ff000001ff00, 0xff01ff00000100ff, 0xff01ff0000010000, 0xff01ff0000010001, + 0xff01ff0001ff0000, 0xff01ff000100ffff, 0xff01ff0001000001, 0xff01ff0001000100, + 0xff01ff0001010000, 0xff01ff01ffffff00, 0xff01ff01ffff01ff, 0xff01ff01ffff0101, + 0xff01ff01ff00ff00, 0xff01ff01ff000000, 0xff01ff01ff01ffff, 0xff01ff01ff01ff01, + 0xff01ff01ff0101ff, 0xff01ff01ff010101, 0xff01ff0100ff0000, 0xff01ff010000ff00, + 0xff01ff0100000001, 0xff01ff0100000100, 0xff01ff0100010000, 0xff01ff0101ffff00, + 0xff01ff0101ff01ff, 0xff01ff0101ff0101, 0xff01ff010100ff00, 0xff01ff0101000000, + 0xff01ff010101ffff, 0xff01ff010101ff01, 0xff01ff01010101ff, 0xff01ff0101010101, + 0xff0100ffffff0000, 0xff0100ffff0000ff, 0xff0100ffff000001, 0xff0100ffff000100, + 0xff0100ffff010000, 0xff0100ff00ff00ff, 0xff0100ff00ff0000, 0xff0100ff00ff0001, + 0xff0100ff00ff0100, 0xff0100ff0000ff01, 0xff0100ff00000000, 0xff0100ff000001ff, + 0xff0100ff00000101, 0xff0100ff00010001, 0xff0100ff01ff0000, 0xff0100ff0100ff00, + 0xff0100ff010000ff, 0xff0100ff01000100, 0xff0100ff0101ff00, 0xff0100ff01010000, + 0xff010000ffff0100, 0xff010000ff000000, 0xff010000ff01ff00, 0xff010000ff010100, + 0xff01000000ffffff, 0xff01000000ff0000, 0xff01000000ff01ff, 0xff0100000000ff00, + 0xff010000000000ff, 0xff01000000000000, 0xff01000000000100, 0xff0100000001ff01, + 0xff01000000010000, 0xff010000000101ff, 0xff01000001ff0100, 0xff0100000100ffff, + 0xff010000010000ff, 0xff01000001000000, 0xff010000010001ff, 0xff01000001000101, + 0xff0100000101ff00, 0xff010000010100ff, 0xff01000001010001, 0xff01000001010100, + 0xff010001ffff0000, 0xff010001ff00ffff, 0xff010001ff00ff01, 0xff010001ff000100, + 0xff010001ff010000, 0xff01000100ffff00, 0xff01000100ff0100, 0xff01000100000000, + 0xff0100010001ffff, 0xff0100010001ff00, 0xff01000100010100, 0xff01000101ff00ff, + 0xff01000101ff0001, 0xff0100010100ffff, 0xff01000101000101, 0xff0101ffffffffff, + 0xff0101ffffffff01, 0xff0101ffffff01ff, 0xff0101ffffff0101, 0xff0101ffff000000, + 0xff0101ffff01ffff, 0xff0101ffff01ff01, 0xff0101ffff0101ff, 0xff0101ffff010101, + 0xff0101ff00ff0000, 0xff0101ff0000ff00, 0xff0101ff000000ff, 0xff0101ff00010000, + 0xff0101ff01ffffff, 0xff0101ff01ffff01, 0xff0101ff01ff01ff, 0xff0101ff01ff0101, + 0xff0101ff0101ffff, 0xff0101ff0101ff01, 0xff0101ff010101ff, 0xff0101ff01010101, + 0xff010100ffff0100, 0xff010100ff00ff00, 0xff010100ff0000ff, 0xff010100ff000100, + 0xff010100ff010000, 0xff01010000ff0001, 0xff01010000ff0100, 0xff0101000000ff01, + 0xff01010000000000, 0xff0101000001ff00, 0xff010100000100ff, 0xff01010000010001, + 0xff01010000010100, 0xff01010001ff0000, 0xff0101000100ffff, 0xff01010001000001, + 0xff01010001000100, 0xff010100010100ff, 0xff01010001010000, 0xff010101ffffffff, + 0xff010101ffffff01, 0xff010101ffff01ff, 0xff010101ffff0101, 0xff010101ff01ffff, + 0xff010101ff01ff01, 0xff010101ff0101ff, 0xff010101ff010101, 0xff01010100ff0000, + 0xff0101010000ff00, 0xff01010100000001, 0xff01010100000100, 0xff01010100010000, + 0xff01010101ffffff, 0xff01010101ffff01, 0xff01010101ff01ff, 0xff01010101ff0101, + 0xff01010101000000, 0xff0101010101ffff, 0xff0101010101ff01, 0xff010101010101ff, + 0xff01010101010101, 0x00ffffffffff0000, 0x00ffffffff00ff00, 0x00ffffffff000001, + 0x00ffffffff010000, 0x00ffffff00ff0100, 0x00ffffff0000ff01, 0x00ffffff00000000, + 0x00ffffff000001ff, 0x00ffffff00000101, 0x00ffffff0001ff00, 0x00ffffff000100ff, + 0x00ffffff00010001, 0x00ffffff010000ff, 0x00ffffff01000100, 0x00ffffff0101ff00, + 0x00ffffff01010001, 0x00ffff00ffffffff, 0x00ffff00ffffff00, 0x00ffff00ffff00ff, + 0x00ffff00ffff0001, 0x00ffff00ffff0100, 0x00ffff00ff00ff01, 0x00ffff00ff000000, + 0x00ffff00ff000001, 0x00ffff00ff0001ff, 0x00ffff00ff000101, 0x00ffff00ff01ff00, + 0x00ffff00ff010001, 0x00ffff00ff010100, 0x00ffff0000ff0000, 0x00ffff0000ff01ff, + 0x00ffff0000ff0101, 0x00ffff000000ff00, 0x00ffff00000000ff, 0x00ffff0000000000, + 0x00ffff0000000001, 0x00ffff0000000100, 0x00ffff0000000101, 0x00ffff0000010000, + 0x00ffff00000101ff, 0x00ffff0000010101, 0x00ffff0001ffff00, 0x00ffff0001ff00ff, + 0x00ffff0001ff0001, 0x00ffff000100ffff, 0x00ffff000100ff01, 0x00ffff0001000000, + 0x00ffff000101ffff, 0x00ffff000101ff00, 0x00ffff000101ff01, 0x00ffff01ffff0000, + 0x00ffff01ff00ff00, 0x00ffff01ff0000ff, 0x00ffff01ff000001, 0x00ffff01ff010000, + 0x00ffff0100ffff00, 0x00ffff010000ff01, 0x00ffff0100000000, 0x00ffff0100000101, + 0x00ffff01000100ff, 0x00ffff0100010100, 0x00ffff0101ff0100, 0x00ffff01010000ff, + 0x00ffff0101010000, 0x00ff00ffffffff00, 0x00ff00ffff000000, 0x00ff00ffff000100, + 0x00ff00ffff010100, 0x00ff00ff00ff0000, 0x00ff00ff00ff01ff, 0x00ff00ff00ff0101, + 0x00ff00ff0000ff00, 0x00ff00ff000000ff, 0x00ff00ff00000000, 0x00ff00ff00000001, + 0x00ff00ff0001ff00, 0x00ff00ff0001ff01, 0x00ff00ff00010000, 0x00ff00ff000101ff, + 0x00ff00ff00010101, 0x00ff00ff01ffff00, 0x00ff00ff01ff0001, 0x00ff00ff01ff0100, + 0x00ff00ff0100ffff, 0x00ff00ff0100ff01, 0x00ff00ff01000000, 0x00ff00ff0101ffff, + 0x00ff00ff0101ff00, 0x00ff00ff01010100, 0x00ff0000ffffff00, 0x00ff0000ffffff01, + 0x00ff0000ffff0000, 0x00ff0000ffff0101, 0x00ff0000ff00ff00, 0x00ff0000ff0000ff, + 0x00ff0000ff000000, 0x00ff0000ff000001, 0x00ff0000ff000100, 0x00ff0000ff01ffff, + 0x00ff0000ff010000, 0x00ff0000ff010101, 0x00ff000000ffff00, 0x00ff000000ff00ff, + 0x00ff000000ff0000, 0x00ff000000ff0001, 0x00ff000000ff0100, 0x00ff00000000ffff, + 0x00ff00000000ff00, 0x00ff0000000000ff, 0x00ff000000000000, 0x00ff000000000001, + 0x00ff0000000001ff, 0x00ff000000000100, 0x00ff00000001ff00, 0x00ff0000000100ff, + 0x00ff000000010000, 0x00ff000000010001, 0x00ff000000010100, 0x00ff000001ffff01, + 0x00ff000001ff00ff, 0x00ff000001ff0000, 0x00ff000001ff01ff, 0x00ff00000100ff00, + 0x00ff0000010000ff, 0x00ff000001000000, 0x00ff000001000001, 0x00ff000001000100, + 0x00ff000001000101, 0x00ff000001010000, 0x00ff0000010101ff, 0x00ff000001010101, + 0x00ff0001ffffff00, 0x00ff0001ffff0000, 0x00ff0001ffff0100, 0x00ff0001ff0000ff, + 0x00ff0001ff000000, 0x00ff0001ff0001ff, 0x00ff0001ff000101, 0x00ff0001ff01ff00, + 0x00ff0001ff0100ff, 0x00ff0001ff010100, 0x00ff000100ffffff, 0x00ff000100ffff01, + 0x00ff000100ff0000, 0x00ff000100ff01ff, 0x00ff00010000ffff, 0x00ff00010000ff00, + 0x00ff00010000ff01, 0x00ff000100000000, 0x00ff000100000001, 0x00ff000100000100, + 0x00ff00010001ff01, 0x00ff000100010000, 0x00ff0001000101ff, 0x00ff000101ffff00, + 0x00ff000101ff0000, 0x00ff000101ff0101, 0x00ff0001010000ff, 0x00ff000101000000, + 0x00ff00010101ff00, 0x00ff0001010100ff, 0x00ff000101010001, 0x00ff01ffffff0000, + 0x00ff01ffff00ff00, 0x00ff01ffff000000, 0x00ff01ffff000101, 0x00ff01ffff010000, + 0x00ff01ff00ffff01, 0x00ff01ff00ff0100, 0x00ff01ff0000ffff, 0x00ff01ff00000000, + 0x00ff01ff000001ff, 0x00ff01ff0001ff00, 0x00ff01ff000100ff, 0x00ff01ff00010001, + 0x00ff01ff00010100, 0x00ff01ff01ff0000, 0x00ff01ff0100ff00, 0x00ff01ff010000ff, + 0x00ff01ff01000001, 0x00ff01ff01000100, 0x00ff01ff01010000, 0x00ff0100ffffff00, + 0x00ff0100ffff0000, 0x00ff0100ffff0001, 0x00ff0100ffff0101, 0x00ff0100ff00ffff, + 0x00ff0100ff0000ff, 0x00ff0100ff000000, 0x00ff0100ff0001ff, 0x00ff0100ff01ff00, + 0x00ff0100ff0100ff, 0x00ff0100ff010001, 0x00ff010000ffffff, 0x00ff010000ff0000, + 0x00ff010000ff0101, 0x00ff01000000ff00, 0x00ff01000000ff01, 0x00ff0100000000ff, + 0x00ff010000000000, 0x00ff010000000001, 0x00ff010000000100, 0x00ff01000001ffff, + 0x00ff01000001ff01, 0x00ff010000010000, 0x00ff010000010001, 0x00ff010000010101, + 0x00ff010001ff0001, 0x00ff010001ff0100, 0x00ff01000100ff01, 0x00ff010001000000, + 0x00ff010001000001, 0x00ff0100010001ff, 0x00ff01000101ff00, 0x00ff0100010100ff, + 0x00ff010001010001, 0x00ff010001010100, 0x00ff0101ff000001, 0x00ff010100ff00ff, + 0x00ff010100ff0001, 0x00ff010100ff0100, 0x00ff010100000000, 0x00ff0101000001ff, + 0x00ff010100000101, 0x00ff0101000100ff, 0x00ff010100010100, 0x00ff0101010000ff, + 0x00ff010101010000, 0x0000ffffffffff00, 0x0000ffffffff00ff, 0x0000ffffffff0000, + 0x0000ffffffff0001, 0x0000ffffffff0100, 0x0000ffffff00ff01, 0x0000ffffff000000, + 0x0000ffffff000101, 0x0000ffffff01ff00, 0x0000ffffff0100ff, 0x0000ffffff010100, + 0x0000ffff00ffffff, 0x0000ffff00ff0000, 0x0000ffff00ff01ff, 0x0000ffff0000ff00, + 0x0000ffff000000ff, 0x0000ffff00000000, 0x0000ffff00000001, 0x0000ffff00000100, + 0x0000ffff00010000, 0x0000ffff000101ff, 0x0000ffff01ff0001, 0x0000ffff01ff0100, + 0x0000ffff01000000, 0x0000ffff010001ff, 0x0000ffff0101ffff, 0x0000ffff0101ff00, + 0x0000ffff01010001, 0x0000ffff01010100, 0x0000ff00ffff0000, 0x0000ff00ffff01ff, + 0x0000ff00ffff0100, 0x0000ff00ffff0101, 0x0000ff00ff00ff00, 0x0000ff00ff0000ff, + 0x0000ff00ff000000, 0x0000ff00ff000001, 0x0000ff00ff0001ff, 0x0000ff00ff000100, + 0x0000ff00ff01ffff, 0x0000ff00ff010000, 0x0000ff00ff010001, 0x0000ff00ff0101ff, + 0x0000ff00ff010101, 0x0000ff0000ffff00, 0x0000ff0000ff00ff, 0x0000ff0000ff0000, + 0x0000ff0000ff0001, 0x0000ff0000ff0100, 0x0000ff000000ffff, 0x0000ff000000ff00, + 0x0000ff000000ff01, 0x0000ff00000000ff, 0x0000ff0000000000, 0x0000ff0000000001, + 0x0000ff00000001ff, 0x0000ff0000000100, 0x0000ff0000000101, 0x0000ff000001ff00, + 0x0000ff00000100ff, 0x0000ff0000010000, 0x0000ff0000010001, 0x0000ff0000010100, + 0x0000ff0001ffff01, 0x0000ff0001ff0000, 0x0000ff000100ff00, 0x0000ff00010000ff, + 0x0000ff0001000000, 0x0000ff0001000001, 0x0000ff0001000100, 0x0000ff000101ffff, + 0x0000ff0001010000, 0x0000ff0001010101, 0x0000ff01ffffff00, 0x0000ff01ffff0001, + 0x0000ff01ff00ff01, 0x0000ff01ff000000, 0x0000ff01ff000101, 0x0000ff01ff01ff00, + 0x0000ff01ff0100ff, 0x0000ff0100ffff01, 0x0000ff0100ff0000, 0x0000ff0100ff0101, + 0x0000ff010000ff00, 0x0000ff01000000ff, 0x0000ff0100000000, 0x0000ff0100000001, + 0x0000ff0100000100, 0x0000ff010001ff01, 0x0000ff0100010000, 0x0000ff0101ff0000, + 0x0000ff010100ffff, 0x0000ff010100ff01, 0x0000ff0101000000, 0x0000ff0101000100, + 0x0000ff0101000101, 0x0000ff01010100ff, 0x000000ffffff00ff, 0x000000ffffff0000, + 0x000000ffff00ff00, 0x000000ffff0000ff, 0x000000ffff000000, 0x000000ffff000001, + 0x000000ffff0001ff, 0x000000ffff000100, 0x000000ffff01ff00, 0x000000ffff010000, + 0x000000ffff0101ff, 0x000000ffff010101, 0x000000ff00ffff00, 0x000000ff00ff00ff, + 0x000000ff00ff0000, 0x000000ff00ff0001, 0x000000ff00ff0100, 0x000000ff00ff0101, + 0x000000ff0000ffff, 0x000000ff0000ff00, 0x000000ff000000ff, 0x000000ff00000000, + 0x000000ff00000001, 0x000000ff000001ff, 0x000000ff00000100, 0x000000ff00000101, + 0x000000ff0001ff00, 0x000000ff0001ff01, 0x000000ff000100ff, 0x000000ff00010000, + 0x000000ff00010001, 0x000000ff00010100, 0x000000ff01ffffff, 0x000000ff01ff01ff, + 0x000000ff01ff0101, 0x000000ff0100ff00, 0x000000ff010000ff, 0x000000ff01000000, + 0x000000ff01000001, 0x000000ff01000100, 0x000000ff0101ff00, 0x000000ff010100ff, + 0x000000ff01010000, 0x000000ff01010101, 0x00000000ffffff00, 0x00000000ffffff01, + 0x00000000ffff00ff, 0x00000000ffff0000, 0x00000000ffff0001, 0x00000000ffff0100, + 0x00000000ff00ffff, 0x00000000ff00ff00, 0x00000000ff00ff01, 0x00000000ff0000ff, + 0x00000000ff000000, 0x00000000ff000001, 0x00000000ff000100, 0x00000000ff000101, + 0x00000000ff01ff00, 0x00000000ff0100ff, 0x00000000ff010000, 0x00000000ff010001, + 0x00000000ff010100, 0x0000000000ffffff, 0x0000000000ffff00, 0x0000000000ffff01, + 0x0000000000ff00ff, 0x0000000000ff0000, 0x0000000000ff0001, 0x0000000000ff01ff, + 0x0000000000ff0100, 0x000000000000ffff, 0x000000000000ff00, 0x000000000000ff01, + 0x00000000000000ff, 0x0000000000000000, 0x0000000000000001, 0x00000000000001ff, + 0x0000000000000100, 0x0000000000000101, 0x000000000001ffff, 0x000000000001ff00, + 0x00000000000100ff, 0x0000000000010000, 0x0000000000010001, 0x00000000000101ff, + 0x0000000000010100, 0x0000000000010101, 0x0000000001ffff00, 0x0000000001ff00ff, + 0x0000000001ff0000, 0x0000000001ff0100, 0x0000000001ff0101, 0x000000000100ffff, + 0x000000000100ff00, 0x00000000010000ff, 0x0000000001000000, 0x0000000001000001, + 0x00000000010001ff, 0x0000000001000100, 0x000000000101ff00, 0x00000000010100ff, + 0x0000000001010000, 0x0000000001010001, 0x0000000001010100, 0x00000001ffffffff, + 0x00000001ffffff00, 0x00000001ffffff01, 0x00000001ffff00ff, 0x00000001ffff0001, + 0x00000001ffff01ff, 0x00000001ffff0100, 0x00000001ff00ff00, 0x00000001ff0000ff, + 0x00000001ff000000, 0x00000001ff0001ff, 0x00000001ff000100, 0x00000001ff01ffff, + 0x00000001ff01ff00, 0x00000001ff01ff01, 0x00000001ff0100ff, 0x00000001ff010000, + 0x00000001ff010001, 0x00000001ff0101ff, 0x00000001ff010100, 0x0000000100ffff00, + 0x0000000100ff0000, 0x0000000100ff0001, 0x0000000100ff01ff, 0x0000000100ff0100, + 0x0000000100ff0101, 0x000000010000ffff, 0x000000010000ff00, 0x000000010000ff01, + 0x00000001000000ff, 0x0000000100000000, 0x0000000100000001, 0x00000001000001ff, + 0x0000000100000100, 0x0000000100000101, 0x000000010001ff00, 0x00000001000100ff, + 0x0000000100010000, 0x0000000100010100, 0x0000000101ffff01, 0x0000000101ff0000, + 0x0000000101ff0001, 0x0000000101ff01ff, 0x0000000101ff0100, 0x0000000101ff0101, + 0x000000010100ff00, 0x0000000101000000, 0x0000000101000101, 0x000000010101ff01, + 0x0000000101010000, 0x0000000101010001, 0x00000001010101ff, 0x0000000101010100, + 0x000001ffffff00ff, 0x000001ffffff0000, 0x000001ffffff0001, 0x000001ffffff0100, + 0x000001ffff00ffff, 0x000001ffff000000, 0x000001ffff0001ff, 0x000001ffff01ff00, + 0x000001ffff010101, 0x000001ff00ff0000, 0x000001ff00ff01ff, 0x000001ff00ff0101, + 0x000001ff0000ff00, 0x000001ff000000ff, 0x000001ff00000000, 0x000001ff00000001, + 0x000001ff000001ff, 0x000001ff00000100, 0x000001ff0001ffff, 0x000001ff0001ff01, + 0x000001ff000100ff, 0x000001ff00010000, 0x000001ff01ffff01, 0x000001ff01ff0100, + 0x000001ff0100ffff, 0x000001ff0100ff01, 0x000001ff01000000, 0x000001ff010001ff, + 0x000001ff0101ff00, 0x000001ff01010100, 0x00000100ffffff00, 0x00000100ffffff01, + 0x00000100ffff0000, 0x00000100ffff0101, 0x00000100ff00ff00, 0x00000100ff0000ff, + 0x00000100ff000000, 0x00000100ff000001, 0x00000100ff000100, 0x00000100ff010000, + 0x0000010000ffff00, 0x0000010000ff00ff, 0x0000010000ff0000, 0x0000010000ff0001, + 0x0000010000ff0100, 0x000001000000ffff, 0x000001000000ff00, 0x000001000000ff01, + 0x00000100000000ff, 0x0000010000000000, 0x0000010000000001, 0x00000100000001ff, + 0x0000010000000100, 0x0000010000000101, 0x000001000001ff00, 0x00000100000100ff, + 0x0000010000010000, 0x0000010000010001, 0x0000010000010100, 0x0000010001ffff00, + 0x0000010001ff0000, 0x0000010001ff0100, 0x000001000100ff00, 0x00000100010000ff, + 0x0000010001000000, 0x0000010001000001, 0x00000100010001ff, 0x0000010001000100, + 0x0000010001010000, 0x00000101ffff00ff, 0x00000101ffff01ff, 0x00000101ff000000, + 0x00000101ff000101, 0x00000101ff01ffff, 0x00000101ff010000, 0x00000101ff010001, + 0x00000101ff010100, 0x0000010100ff0000, 0x0000010100ff01ff, 0x0000010100ff0100, + 0x000001010000ff00, 0x0000010100000000, 0x0000010100000001, 0x00000101000001ff, + 0x0000010100000100, 0x000001010001ff01, 0x0000010100010000, 0x00000101000101ff, + 0x0000010100010101, 0x0000010101ffff00, 0x0000010101ff0101, 0x000001010100ff01, + 0x0000010101000000, 0x0000010101000001, 0x00000101010001ff, 0x0000010101000101, + 0x000001010101ff00, 0x0001ffffffff0000, 0x0001ffffff0000ff, 0x0001ffffff000001, + 0x0001ffffff000100, 0x0001ffffff010000, 0x0001ffff00ff00ff, 0x0001ffff0000ffff, + 0x0001ffff00000000, 0x0001ffff00000001, 0x0001ffff000001ff, 0x0001ffff00000101, + 0x0001ffff0001ff00, 0x0001ffff000100ff, 0x0001ffff00010001, 0x0001ffff00010100, + 0x0001ffff01ffff00, 0x0001ffff01000001, 0x0001ffff01010000, 0x0001ff00ffffff00, + 0x0001ff00ffff00ff, 0x0001ff00ffff0001, 0x0001ff00ffff0100, 0x0001ff00ff00ff01, + 0x0001ff00ff000000, 0x0001ff00ff01ff00, 0x0001ff00ff01ff01, 0x0001ff00ff010001, + 0x0001ff00ff010100, 0x0001ff0000ff0000, 0x0001ff0000ff0100, 0x0001ff000000ff00, + 0x0001ff0000000000, 0x0001ff0000000001, 0x0001ff0000000100, 0x0001ff0000010000, + 0x0001ff0000010001, 0x0001ff0000010101, 0x0001ff0001ff00ff, 0x0001ff0001ff0101, + 0x0001ff000100ff01, 0x0001ff0001000000, 0x0001ff000101ff00, 0x0001ff0001010001, + 0x0001ff0001010100, 0x0001ff01ff00ff00, 0x0001ff01ff000001, 0x0001ff01ff000100, + 0x0001ff0100ffffff, 0x0001ff0100ffff00, 0x0001ff0100ff0001, 0x0001ff0100000000, + 0x0001ff0100000001, 0x0001ff01000001ff, 0x0001ff010001ffff, 0x0001ff0101ff0000, + 0x0001ff010100ff00, 0x0001ff0101000001, 0x0001ff0101010000, 0x000100ffff00ff00, + 0x000100ffff00ff01, 0x000100ffff000000, 0x000100ffff000001, 0x000100ffff000101, + 0x000100ffff01ff00, 0x000100ffff010001, 0x000100ffff010100, 0x000100ff00ffffff, + 0x000100ff00ffff01, 0x000100ff00ff0000, 0x000100ff00ff01ff, 0x000100ff00ff0101, + 0x000100ff0000ff00, 0x000100ff000000ff, 0x000100ff00000000, 0x000100ff00000001, + 0x000100ff00000100, 0x000100ff00000101, 0x000100ff0001ffff, 0x000100ff0001ff01, + 0x000100ff00010000, 0x000100ff01ff00ff, 0x000100ff01ff0000, 0x000100ff01ff0100, + 0x000100ff0100ffff, 0x000100ff0100ff01, 0x000100ff010000ff, 0x000100ff01000000, + 0x000100ff01000001, 0x000100ff010001ff, 0x000100ff01000101, 0x000100ff0101ff00, + 0x000100ff010100ff, 0x000100ff01010100, 0x00010000ffff0000, 0x00010000ffff01ff, + 0x00010000ffff0101, 0x00010000ff00ff00, 0x00010000ff000000, 0x00010000ff000001, + 0x00010000ff000100, 0x0001000000ff00ff, 0x0001000000ff0000, 0x0001000000ff0001, + 0x0001000000ff0100, 0x000100000000ffff, 0x000100000000ff00, 0x00010000000000ff, + 0x0001000000000000, 0x0001000000000001, 0x0001000000000100, 0x000100000001ff00, + 0x00010000000100ff, 0x0001000000010000, 0x0001000000010001, 0x0001000000010100, + 0x0001000001ff0001, 0x0001000001ff0100, 0x0001000001ff0101, 0x000100000100ff00, + 0x0001000001000000, 0x0001000001000001, 0x0001000001000100, 0x0001000001000101, + 0x000100000101ff01, 0x0001000001010000, 0x0001000001010001, 0x00010000010101ff, + 0x00010001ffffff01, 0x00010001ffff0100, 0x00010001ff000000, 0x00010001ff01ffff, + 0x00010001ff010001, 0x00010001ff0101ff, 0x00010001ff010100, 0x0001000100ffffff, + 0x0001000100ff0000, 0x0001000100ff01ff, 0x0001000100ff0101, 0x000100010000ff00, + 0x00010001000000ff, 0x0001000100000000, 0x0001000100000001, 0x00010001000001ff, + 0x0001000100000101, 0x000100010001ffff, 0x0001000100010000, 0x00010001000101ff, + 0x0001000101ffffff, 0x0001000101ffff01, 0x0001000101ff0000, 0x0001000101ff0101, + 0x00010001010000ff, 0x0001000101000001, 0x00010001010001ff, 0x0001000101000100, + 0x000100010101ffff, 0x00010001010100ff, 0x0001000101010001, 0x0001000101010101, + 0x000101ffff000001, 0x000101ffff000100, 0x000101ffff010000, 0x000101ff00ffff00, + 0x000101ff0000ff01, 0x000101ff00000000, 0x000101ff00000101, 0x000101ff0001ff00, + 0x000101ff00010100, 0x000101ff01ff0000, 0x000101ff0100ff00, 0x000101ff010001ff, + 0x000101ff01010001, 0x00010100ffffff00, 0x00010100ffff00ff, 0x00010100ff00ffff, + 0x00010100ff000000, 0x00010100ff01ff00, 0x00010100ff0100ff, 0x00010100ff010001, + 0x00010100ff010100, 0x0001010000ffffff, 0x0001010000ffff00, 0x0001010000ff0000, + 0x0001010000ff0001, 0x0001010000ff01ff, 0x000101000000ff00, 0x00010100000000ff, + 0x0001010000000000, 0x0001010000000001, 0x0001010000000100, 0x000101000001ffff, + 0x0001010000010000, 0x0001010000010101, 0x0001010001ffff01, 0x0001010001ff00ff, + 0x0001010001ff0101, 0x0001010001000000, 0x000101000101ff00, 0x00010100010100ff, + 0x0001010001010000, 0x0001010001010100, 0x00010101ff00ff00, 0x00010101ff000001, + 0x00010101ff0001ff, 0x0001010100ffff00, 0x0001010100ff00ff, 0x0001010100ff0100, + 0x000101010000ffff, 0x0001010100000000, 0x00010101000001ff, 0x0001010100000101, + 0x00010101000100ff, 0x0001010100010000, 0x0001010100010100, 0x0001010101ff0001, + 0x00010101010000ff, 0x00010101010001ff, 0x0001010101000101, 0x0001010101010001, + 0x01ffffffffffffff, 0x01ffffffffffff01, 0x01ffffffffff01ff, 0x01ffffffffff0101, + 0x01ffffffff01ffff, 0x01ffffffff01ff01, 0x01ffffffff0101ff, 0x01ffffffff010101, + 0x01ffffff00ff0000, 0x01ffffff0000ffff, 0x01ffffff0000ff00, 0x01ffffff000000ff, + 0x01ffffff00000001, 0x01ffffff00000100, 0x01ffffff00010000, 0x01ffffff01ffffff, + 0x01ffffff01ffff01, 0x01ffffff01ff01ff, 0x01ffffff01ff0101, 0x01ffffff01000000, + 0x01ffffff0101ffff, 0x01ffffff0101ff01, 0x01ffffff010101ff, 0x01ffffff01010101, + 0x01ffff00ffff0000, 0x01ffff00ff00ff00, 0x01ffff00ff0000ff, 0x01ffff00ff000001, + 0x01ffff00ff000100, 0x01ffff00ff010000, 0x01ffff0000ffff00, 0x01ffff0000ff00ff, + 0x01ffff0000ff0100, 0x01ffff000000ffff, 0x01ffff000000ff01, 0x01ffff0000000000, + 0x01ffff0000000001, 0x01ffff00000001ff, 0x01ffff0000000100, 0x01ffff00000100ff, + 0x01ffff0000010001, 0x01ffff0000010100, 0x01ffff0001ff0000, 0x01ffff0001ff0100, + 0x01ffff00010000ff, 0x01ffff0001000001, 0x01ffff0001000100, 0x01ffff0001010000, + 0x01ffff01ffffffff, 0x01ffff01ffffff01, 0x01ffff01ffff01ff, 0x01ffff01ffff0101, + 0x01ffff01ff000000, 0x01ffff01ff01ffff, 0x01ffff01ff01ff01, 0x01ffff01ff0101ff, + 0x01ffff01ff010101, 0x01ffff010000ff00, 0x01ffff01000000ff, 0x01ffff0100000100, + 0x01ffff0100010000, 0x01ffff0101ffffff, 0x01ffff0101ffff01, 0x01ffff0101ff01ff, + 0x01ffff0101ff0101, 0x01ffff0101000000, 0x01ffff010101ffff, 0x01ffff010101ff01, + 0x01ffff01010101ff, 0x01ffff0101010101, 0x01ff00ffff0000ff, 0x01ff00ffff000100, + 0x01ff00ff00ffff00, 0x01ff00ff00ff00ff, 0x01ff00ff0000ff00, 0x01ff00ff00000000, + 0x01ff00ff00000101, 0x01ff00ff0001ff00, 0x01ff00ff000100ff, 0x01ff00ff00010100, + 0x01ff00ff010000ff, 0x01ff00ff01000100, 0x01ff0000ffffff00, 0x01ff0000ffff0100, + 0x01ff0000ff00ff01, 0x01ff0000ff000000, 0x01ff0000ff000101, 0x01ff0000ff010001, + 0x01ff0000ff010100, 0x01ff000000ffffff, 0x01ff000000ffff00, 0x01ff000000ff0000, + 0x01ff000000ff01ff, 0x01ff00000000ff00, 0x01ff0000000000ff, 0x01ff000000000000, + 0x01ff000000000001, 0x01ff000000000100, 0x01ff000000000101, 0x01ff000000010000, + 0x01ff000000010001, 0x01ff0000000101ff, 0x01ff000000010101, 0x01ff000001ffff00, + 0x01ff000001ff00ff, 0x01ff000001ff0001, 0x01ff000001ff0100, 0x01ff00000100ffff, + 0x01ff00000100ff01, 0x01ff000001000000, 0x01ff0000010001ff, 0x01ff000001010001, + 0x01ff0001ff00ff00, 0x01ff0001ff000001, 0x01ff0001ff000100, 0x01ff0001ff010000, + 0x01ff000100ffff00, 0x01ff000100ff00ff, 0x01ff000100ff0100, 0x01ff000100ff0101, + 0x01ff00010000ffff, 0x01ff000100000000, 0x01ff000100000100, 0x01ff000100000101, + 0x01ff00010001ff00, 0x01ff000100010001, 0x01ff000100010101, 0x01ff000101ff0000, + 0x01ff00010100ff00, 0x01ff000101000101, 0x01ff0001010100ff, 0x01ff01ffffffffff, + 0x01ff01ffffffff01, 0x01ff01ffffff01ff, 0x01ff01ffffff0101, 0x01ff01ffff000000, + 0x01ff01ffff01ffff, 0x01ff01ffff01ff01, 0x01ff01ffff0101ff, 0x01ff01ffff010101, + 0x01ff01ff00ffff00, 0x01ff01ff00ff0000, 0x01ff01ff0000ff00, 0x01ff01ff000000ff, + 0x01ff01ff00000100, 0x01ff01ff00010000, 0x01ff01ff00010100, 0x01ff01ff01ffffff, + 0x01ff01ff01ffff01, 0x01ff01ff01ff01ff, 0x01ff01ff01ff0101, 0x01ff01ff01000000, + 0x01ff01ff0101ffff, 0x01ff01ff0101ff01, 0x01ff01ff010101ff, 0x01ff01ff01010101, + 0x01ff0100ffff0000, 0x01ff0100ffff0001, 0x01ff0100ff00ff00, 0x01ff0100ff0000ff, + 0x01ff0100ff000001, 0x01ff0100ff010000, 0x01ff010000ffff00, 0x01ff010000ff00ff, + 0x01ff010000ff0001, 0x01ff010000ff0100, 0x01ff01000000ffff, 0x01ff01000000ff01, + 0x01ff010000000000, 0x01ff010000000101, 0x01ff01000001ff00, 0x01ff0100000100ff, + 0x01ff010001ff0000, 0x01ff010001000001, 0x01ff010001000100, 0x01ff010001010000, + 0x01ff0101ffffffff, 0x01ff0101ffffff01, 0x01ff0101ffff01ff, 0x01ff0101ffff0101, + 0x01ff0101ff000000, 0x01ff0101ff01ffff, 0x01ff0101ff01ff01, 0x01ff0101ff0101ff, + 0x01ff0101ff010101, 0x01ff010100ff0000, 0x01ff01010000ff00, 0x01ff0101000000ff, + 0x01ff010100000001, 0x01ff010101ffffff, 0x01ff010101ffff01, 0x01ff010101ff01ff, + 0x01ff010101ff0101, 0x01ff010101000000, 0x01ff01010101ffff, 0x01ff01010101ff01, + 0x01ff0101010101ff, 0x01ff010101010101, 0x0100ffffffff0000, 0x0100ffffff00ff00, + 0x0100ffffff000001, 0x0100ffffff0001ff, 0x0100ffffff000100, 0x0100ffffff010000, + 0x0100ffff00ffff00, 0x0100ffff00ff0001, 0x0100ffff00ff0100, 0x0100ffff00000000, + 0x0100ffff000001ff, 0x0100ffff00000101, 0x0100ffff00010100, 0x0100ffff00010101, + 0x0100ffff01ff0000, 0x0100ffff0100ff00, 0x0100ffff010000ff, 0x0100ffff01000001, + 0x0100ffff01000100, 0x0100ffff01010000, 0x0100ff00ffffff00, 0x0100ff00ffff00ff, + 0x0100ff00ffff0001, 0x0100ff00ffff0100, 0x0100ff00ff00ffff, 0x0100ff00ff000000, + 0x0100ff00ff0001ff, 0x0100ff00ff000101, 0x0100ff00ff01ff00, 0x0100ff00ff0100ff, + 0x0100ff00ff010001, 0x0100ff00ff010100, 0x0100ff0000ffffff, 0x0100ff0000ff0000, + 0x0100ff000000ffff, 0x0100ff000000ff00, 0x0100ff00000000ff, 0x0100ff0000000000, + 0x0100ff0000000001, 0x0100ff0000000100, 0x0100ff000001ff01, 0x0100ff0000010000, + 0x0100ff0001ff00ff, 0x0100ff0001ff0001, 0x0100ff000100ff01, 0x0100ff0001000000, + 0x0100ff00010001ff, 0x0100ff000101ff00, 0x0100ff00010100ff, 0x0100ff0001010001, + 0x0100ff0001010100, 0x0100ff01ffff0000, 0x0100ff01ff00ff00, 0x0100ff01ff0000ff, + 0x0100ff01ff000100, 0x0100ff01ff010000, 0x0100ff0100ff00ff, 0x0100ff0100ff0001, + 0x0100ff0100ff0100, 0x0100ff010000ffff, 0x0100ff010000ff01, 0x0100ff0100000000, + 0x0100ff01000001ff, 0x0100ff0100010001, 0x0100ff0100010100, 0x0100ff0101ff0000, + 0x0100ff01010000ff, 0x0100ff0101000001, 0x0100ff0101010100, 0x010000ffffffff00, + 0x010000ffffff00ff, 0x010000ffffff0001, 0x010000ffff00ffff, 0x010000ffff000000, + 0x010000ffff0001ff, 0x010000ffff010001, 0x010000ff00ffffff, 0x010000ff00ff0101, + 0x010000ff0000ff00, 0x010000ff000000ff, 0x010000ff00000000, 0x010000ff00000001, + 0x010000ff000001ff, 0x010000ff00000100, 0x010000ff0001ffff, 0x010000ff0001ff00, + 0x010000ff0001ff01, 0x010000ff00010000, 0x010000ff01ff00ff, 0x010000ff01ff0001, + 0x010000ff0100ff01, 0x010000ff010000ff, 0x010000ff01000000, 0x010000ff010001ff, + 0x010000ff0101ff00, 0x010000ff01010100, 0x01000000ffffffff, 0x01000000ffff0000, + 0x01000000ffff01ff, 0x01000000ffff0101, 0x01000000ff00ffff, 0x01000000ff00ff00, + 0x01000000ff0000ff, 0x01000000ff000000, 0x01000000ff000001, 0x01000000ff000100, + 0x01000000ff01ff00, 0x01000000ff010000, 0x01000000ff010100, 0x01000000ff010101, + 0x0100000000ffff00, 0x0100000000ff00ff, 0x0100000000ff0000, 0x0100000000ff0001, + 0x0100000000ff0100, 0x010000000000ffff, 0x010000000000ff00, 0x010000000000ff01, + 0x01000000000000ff, 0x0100000000000000, 0x0100000000000001, 0x01000000000001ff, + 0x0100000000000100, 0x0100000000000101, 0x010000000001ff00, 0x01000000000100ff, + 0x0100000000010000, 0x0100000000010001, 0x0100000000010100, 0x0100000001ffff00, + 0x0100000001ff0000, 0x0100000001ff01ff, 0x010000000100ff00, 0x010000000100ff01, + 0x01000000010000ff, 0x0100000001000000, 0x0100000001000001, 0x0100000001000100, + 0x0100000001000101, 0x010000000101ffff, 0x010000000101ff01, 0x0100000001010000, + 0x01000000010101ff, 0x0100000001010101, 0x01000001ffffff00, 0x01000001ffff00ff, + 0x01000001ff00ffff, 0x01000001ff000000, 0x01000001ff000100, 0x01000001ff01ffff, + 0x01000001ff010001, 0x01000001ff010100, 0x0100000100ff0000, 0x0100000100ff01ff, + 0x0100000100ff0100, 0x010000010000ff00, 0x010000010000ff01, 0x0100000100000000, + 0x0100000100000001, 0x0100000100000100, 0x0100000100010000, 0x01000001000101ff, + 0x0100000101ffff01, 0x0100000101ff00ff, 0x0100000101ff0100, 0x0100000101ff0101, + 0x010000010100ff01, 0x01000001010000ff, 0x0100000101000000, 0x01000001010100ff, + 0x0100000101010001, 0x0100000101010100, 0x010001ffffff0000, 0x010001ffff000001, + 0x010001ffff000100, 0x010001ffff010000, 0x010001ff00ffff00, 0x010001ff00ff0001, + 0x010001ff0000ffff, 0x010001ff0000ff01, 0x010001ff00000000, 0x010001ff00000001, + 0x010001ff00000101, 0x010001ff000100ff, 0x010001ff00010000, 0x010001ff01ff0000, + 0x010001ff0100ff00, 0x010001ff01000001, 0x010001ff01000100, 0x010001ff01010000, + 0x01000100ffff00ff, 0x01000100ffff0001, 0x01000100ffff0100, 0x01000100ff00ffff, + 0x01000100ff00ff01, 0x01000100ff000000, 0x01000100ff0001ff, 0x01000100ff000101, + 0x01000100ff01ffff, 0x01000100ff01ff00, 0x01000100ff0100ff, 0x01000100ff010001, + 0x0100010000ffffff, 0x0100010000ffff01, 0x0100010000ff0000, 0x0100010000ff01ff, + 0x0100010000ff0101, 0x010001000000ff00, 0x01000100000000ff, 0x0100010000000000, + 0x0100010000000001, 0x0100010000000100, 0x010001000001ff01, 0x0100010000010000, + 0x0100010000010001, 0x0100010000010101, 0x0100010001ffff00, 0x0100010001ff00ff, + 0x010001000100ffff, 0x010001000100ff01, 0x0100010001000000, 0x0100010001000101, + 0x010001000101ff00, 0x0100010001010001, 0x01000101ffff0000, 0x01000101ff000000, + 0x01000101ff010000, 0x0100010100ff00ff, 0x0100010100ff0001, 0x0100010100ff0100, + 0x010001010000ffff, 0x0100010100000000, 0x01000101000001ff, 0x010001010001ff00, + 0x0100010101ff0000, 0x010001010100ff00, 0x01000101010000ff, 0x0100010101000000, + 0x0100010101000001, 0x0101ffffffffffff, 0x0101ffffffffff01, 0x0101ffffffff01ff, + 0x0101ffffffff0101, 0x0101ffffff000000, 0x0101ffffff01ffff, 0x0101ffffff01ff01, + 0x0101ffffff0101ff, 0x0101ffffff010101, 0x0101ffff00ff0000, 0x0101ffff0000ff00, + 0x0101ffff000000ff, 0x0101ffff00000001, 0x0101ffff00000100, 0x0101ffff01ffffff, + 0x0101ffff01ffff01, 0x0101ffff01ff01ff, 0x0101ffff01ff0101, 0x0101ffff01000000, + 0x0101ffff0101ffff, 0x0101ffff0101ff01, 0x0101ffff010101ff, 0x0101ffff01010101, + 0x0101ff00ffff0000, 0x0101ff00ffff0100, 0x0101ff00ff00ff00, 0x0101ff00ff0000ff, + 0x0101ff00ff000001, 0x0101ff00ff000100, 0x0101ff00ff000101, 0x0101ff0000ff0001, + 0x0101ff0000ff0100, 0x0101ff000000ff00, 0x0101ff0000000000, 0x0101ff00000001ff, + 0x0101ff0000000101, 0x0101ff000001ff00, 0x0101ff00000100ff, 0x0101ff0001ff0000, + 0x0101ff000100ffff, 0x0101ff000100ff01, 0x0101ff0001000001, 0x0101ff0001000100, + 0x0101ff01ffffff01, 0x0101ff01ffff01ff, 0x0101ff01ffff0101, 0x0101ff01ff00ffff, + 0x0101ff01ff000100, 0x0101ff01ff01ff01, 0x0101ff01ff0101ff, 0x0101ff01ff010101, + 0x0101ff0100ff0000, 0x0101ff010000ff00, 0x0101ff0100000001, 0x0101ff0100000100, + 0x0101ff0100010000, 0x0101ff0101ffffff, 0x0101ff0101ffff01, 0x0101ff0101ff01ff, + 0x0101ff0101ff0101, 0x0101ff0101000000, 0x0101ff010101ffff, 0x0101ff010101ff01, + 0x0101ff01010101ff, 0x0101ff0101010101, 0x010100ffff000100, 0x010100ffff010000, + 0x010100ff00ffff00, 0x010100ff00ff00ff, 0x010100ff0000ffff, 0x010100ff000000ff, + 0x010100ff00000000, 0x010100ff000001ff, 0x010100ff00000101, 0x010100ff0001ff00, + 0x010100ff00010000, 0x010100ff00010001, 0x010100ff000101ff, 0x010100ff00010100, + 0x010100ff01ff0000, 0x01010000ffff0001, 0x01010000ffff0100, 0x01010000ff00ffff, + 0x01010000ff00ff01, 0x01010000ff000000, 0x01010000ff0001ff, 0x01010000ff010001, + 0x01010000ff010100, 0x0101000000ffff01, 0x0101000000ff0000, 0x010100000000ff00, + 0x01010000000000ff, 0x0101000000000000, 0x0101000000000001, 0x0101000000000100, + 0x0101000000010000, 0x0101000000010101, 0x0101000001ffff00, 0x0101000001ff00ff, + 0x0101000001ff0000, 0x0101000001ff0001, 0x0101000001ff0100, 0x010100000100ff01, + 0x0101000001000000, 0x01010000010001ff, 0x01010001ffff0000, 0x01010001ff00ff00, + 0x01010001ff000001, 0x01010001ff000101, 0x01010001ff01ff00, 0x01010001ff010000, + 0x0101000100ff00ff, 0x0101000100ff0001, 0x0101000100ff0101, 0x010100010000ff01, + 0x0101000100000000, 0x0101000100000001, 0x01010001000001ff, 0x010100010001ffff, + 0x010100010001ff01, 0x0101000101ff0001, 0x010100010100ffff, 0x0101000101000000, + 0x0101000101000001, 0x0101000101000100, 0x010100010101ff00, 0x01010001010100ff, + 0x0101000101010001, 0x010101ffffffffff, 0x010101ffffffff01, 0x010101ffffff01ff, + 0x010101ffffff0101, 0x010101ffff01ffff, 0x010101ffff01ff01, 0x010101ffff0101ff, + 0x010101ffff010101, 0x010101ff0000ff00, 0x010101ff000000ff, 0x010101ff00000001, + 0x010101ff00000100, 0x010101ff01ffffff, 0x010101ff01ffff01, 0x010101ff01ff01ff, + 0x010101ff01ff0101, 0x010101ff01000000, 0x010101ff0101ffff, 0x010101ff0101ff01, + 0x010101ff010101ff, 0x010101ff01010101, 0x01010100ffff0000, 0x01010100ff0000ff, + 0x01010100ff000100, 0x01010100ff01ff00, 0x01010100ff010000, 0x0101010000ffff00, + 0x010101000000ffff, 0x0101010000000000, 0x0101010000000101, 0x010101000001ff00, + 0x0101010000010001, 0x0101010000010100, 0x010101000100ffff, 0x0101010001000001, + 0x01010101ffffffff, 0x01010101ffffff01, 0x01010101ffff01ff, 0x01010101ffff0101, + 0x01010101ff01ffff, 0x01010101ff01ff01, 0x01010101ff0101ff, 0x01010101ff010101, + 0x010101010000ff00, 0x01010101000000ff, 0x0101010100000001, 0x0101010101ffffff, + 0x0101010101ffff01, 0x0101010101ff01ff, 0x0101010101ff0101, 0x0101010101000000, + 0x010101010101ffff, 0x010101010101ff01, 0x01010101010101ff, 0x0101010101010101, +GGML_TABLE_END() +#else +GGML_TABLE_BEGIN(uint32_t, iq1s_grid_gpu, NGRID_IQ1S) + 0x00000000, 0x00000002, 0x00000101, 0x00000200, 0x00000202, 0x00010001, 0x00010101, 0x00020000, + 0x00020002, 0x00020200, 0x00020202, 0x01000101, 0x01010001, 0x01010100, 0x01010102, 0x01020101, + 0x02000000, 0x02000002, 0x02000200, 0x02000202, 0x02010101, 0x02020000, 0x02020002, 0x02020200, + 0x02020202, 0x00000110, 0x00000111, 0x00010011, 0x00010110, 0x00010112, 0x00010211, 0x00010212, + 0x00020111, 0x01000011, 0x01000112, 0x01000211, 0x01010012, 0x01010111, 0x01010212, 0x01020011, + 0x01020110, 0x01020112, 0x01020210, 0x02000111, 0x02010011, 0x02010110, 0x02010112, 0x02020111, + 0x00000020, 0x00000022, 0x00000220, 0x00000222, 0x00010121, 0x00020020, 0x00020022, 0x00020220, + 0x00020222, 0x01000121, 0x01010021, 0x01010221, 0x01020120, 0x01020221, 0x02000020, 0x02000022, + 0x02000220, 0x02000222, 0x02010021, 0x02010121, 0x02010221, 0x02020020, 0x02020022, 0x02020220, + 0x02020222, 0x00011001, 0x00011100, 0x00011102, 0x00021101, 0x01001001, 0x01001201, 0x01011101, + 0x01011202, 0x01021100, 0x01021101, 0x02011001, 0x02011201, 0x02021101, 0x00001011, 0x00001110, + 0x00001111, 0x00001112, 0x00011111, 0x00011210, 0x00011212, 0x00021211, 0x01001010, 0x01001111, + 0x01001212, 0x01011010, 0x01011011, 0x01011110, 0x01011111, 0x01011112, 0x01011211, 0x01021010, + 0x01021012, 0x01021111, 0x01021210, 0x01021212, 0x02001011, 0x02011011, 0x02011111, 0x02011210, + 0x02011212, 0x02021011, 0x02021110, 0x02021111, 0x02021112, 0x02021211, 0x00011120, 0x00011221, + 0x01001021, 0x01001120, 0x01011020, 0x01011022, 0x01011121, 0x01011220, 0x01021020, 0x01021021, + 0x01021122, 0x01021221, 0x02001121, 0x02011021, 0x02011120, 0x02011221, 0x00002000, 0x00002002, + 0x00002200, 0x00002202, 0x00012101, 0x00022000, 0x00022002, 0x00022200, 0x00022202, 0x01002101, + 0x01012001, 0x01012102, 0x01022101, 0x02002000, 0x02002002, 0x02002200, 0x02002202, 0x02012101, + 0x02022000, 0x02022002, 0x02022200, 0x02022202, 0x00002111, 0x00012011, 0x00012110, 0x00012211, + 0x00022110, 0x00022111, 0x01002011, 0x01012010, 0x01012011, 0x01012111, 0x01022011, 0x01022110, + 0x01022211, 0x02012011, 0x02012110, 0x02012112, 0x02012211, 0x02022111, 0x00002020, 0x00002022, + 0x00002220, 0x00002222, 0x00012121, 0x00022020, 0x00022022, 0x00022220, 0x00022222, 0x01002121, + 0x01012021, 0x01012221, 0x01022021, 0x01022121, 0x02002020, 0x02002022, 0x02002121, 0x02002220, + 0x02002222, 0x02012121, 0x02022020, 0x02022022, 0x02022220, 0x02022222, 0x00110000, 0x00110001, + 0x00110100, 0x00110201, 0x00120100, 0x00120101, 0x01100001, 0x01100100, 0x01110000, 0x01110101, + 0x01110200, 0x01120001, 0x01120100, 0x01120101, 0x01120201, 0x02110001, 0x02110100, 0x02110102, + 0x02120001, 0x02120101, 0x00100011, 0x00100110, 0x00100112, 0x00100211, 0x00110010, 0x00110012, + 0x00110111, 0x00110210, 0x00120011, 0x00120110, 0x00120211, 0x01100111, 0x01100212, 0x01110010, + 0x01110011, 0x01110012, 0x01110110, 0x01110111, 0x01110112, 0x01110211, 0x01120010, 0x01120111, + 0x02100110, 0x02110012, 0x02110111, 0x02120011, 0x02120110, 0x00110021, 0x00110120, 0x00110122, + 0x00120121, 0x01100020, 0x01100122, 0x01100221, 0x01110022, 0x01110121, 0x01110220, 0x01110222, + 0x01120120, 0x01120122, 0x02100121, 0x02110021, 0x02110120, 0x02110122, 0x02120121, 0x00101001, + 0x00101102, 0x00101201, 0x00111100, 0x00111101, 0x00111200, 0x00111201, 0x00121001, 0x00121102, + 0x01101001, 0x01101101, 0x01101102, 0x01101200, 0x01101202, 0x01111001, 0x01111100, 0x01111101, + 0x01111102, 0x01111201, 0x01121002, 0x01121101, 0x01121200, 0x02101100, 0x02101201, 0x02111000, + 0x02111100, 0x02111101, 0x02111200, 0x02111201, 0x02111202, 0x02121001, 0x02121100, 0x02121101, + 0x02121201, 0x00101012, 0x00101111, 0x00101212, 0x00111011, 0x00111110, 0x00111111, 0x00111112, + 0x00111211, 0x00121010, 0x00121012, 0x00121111, 0x00121210, 0x00121212, 0x01101011, 0x01101110, + 0x01101111, 0x01101112, 0x01111011, 0x01111012, 0x01111110, 0x01111111, 0x01111112, 0x01111211, + 0x01111212, 0x01121011, 0x01121110, 0x01121111, 0x01121112, 0x01121211, 0x02101010, 0x02101012, + 0x02101110, 0x02101111, 0x02101210, 0x02101212, 0x02111010, 0x02111011, 0x02111110, 0x02111111, + 0x02111112, 0x02111211, 0x02111212, 0x02121010, 0x02121012, 0x02121111, 0x00101021, 0x00101120, + 0x00101121, 0x00101122, 0x00111121, 0x00111122, 0x00111220, 0x00111222, 0x00121021, 0x00121122, + 0x01101020, 0x01101022, 0x01101120, 0x01101121, 0x01101220, 0x01101222, 0x01111021, 0x01111121, + 0x01111122, 0x01111220, 0x01111221, 0x01121021, 0x01121120, 0x01121121, 0x01121220, 0x01121221, + 0x01121222, 0x02101122, 0x02101222, 0x02111022, 0x02111121, 0x02121120, 0x02121221, 0x00112001, + 0x00112102, 0x00122101, 0x01102001, 0x01102100, 0x01102102, 0x01102201, 0x01112000, 0x01112101, + 0x01112200, 0x01112202, 0x01122000, 0x01122001, 0x01122100, 0x01122102, 0x01122201, 0x02102101, + 0x02112001, 0x02112100, 0x02122101, 0x00112010, 0x00112012, 0x00112111, 0x00112212, 0x00122011, + 0x00122111, 0x01102012, 0x01102110, 0x01102111, 0x01102210, 0x01112011, 0x01112110, 0x01112111, + 0x01112112, 0x01112211, 0x01112212, 0x01122010, 0x01122111, 0x01122212, 0x02102211, 0x02112011, + 0x02112012, 0x02112111, 0x02112210, 0x02122011, 0x02122112, 0x02122211, 0x00102221, 0x00112122, + 0x00122120, 0x00122122, 0x01102120, 0x01102122, 0x01102221, 0x01112020, 0x01112022, 0x01112121, + 0x01112220, 0x01122021, 0x01122122, 0x01122221, 0x02102121, 0x02112021, 0x02112122, 0x02112222, + 0x00200000, 0x00200002, 0x00200200, 0x00200202, 0x00210101, 0x00220000, 0x00220002, 0x00220101, + 0x00220200, 0x00220202, 0x01200101, 0x01210001, 0x01210201, 0x01220001, 0x01220101, 0x02200000, + 0x02200002, 0x02200200, 0x02200202, 0x02210101, 0x02220000, 0x02220002, 0x02220101, 0x02220200, + 0x02220202, 0x00200111, 0x00210011, 0x00210110, 0x00210211, 0x00220111, 0x01200012, 0x01200110, + 0x01200211, 0x01210111, 0x01210210, 0x01210212, 0x01220011, 0x01220110, 0x01220111, 0x01220112, + 0x02200111, 0x02210010, 0x02210112, 0x02210211, 0x02220111, 0x00200021, 0x00200220, 0x00200222, + 0x00210021, 0x00210121, 0x00220020, 0x00220022, 0x00220220, 0x00220222, 0x01200121, 0x01210021, + 0x01210122, 0x01210221, 0x01220121, 0x02200021, 0x02200220, 0x02200222, 0x02210021, 0x02210121, + 0x02220020, 0x02220022, 0x02220220, 0x02220222, 0x00201101, 0x00211100, 0x00211102, 0x00211201, + 0x00221101, 0x01201100, 0x01201101, 0x01201102, 0x01201201, 0x01211002, 0x01211101, 0x01211200, + 0x01211202, 0x01221102, 0x02201101, 0x02211001, 0x02211100, 0x02211201, 0x02221001, 0x02221101, + 0x00201211, 0x00211111, 0x00221011, 0x00221211, 0x01201010, 0x01201111, 0x01201210, 0x01211011, + 0x01211110, 0x01211111, 0x01211211, 0x01221012, 0x01221111, 0x01221210, 0x02201211, 0x02211010, + 0x02211110, 0x02211111, 0x02211210, 0x02211212, 0x02221011, 0x02221110, 0x02221112, 0x02221211, + 0x00201121, 0x00211020, 0x00211022, 0x00211221, 0x00221121, 0x01201021, 0x01201221, 0x01211121, + 0x01221020, 0x01221021, 0x01221221, 0x02201120, 0x02201122, 0x02211020, 0x02211222, 0x00202000, + 0x00202002, 0x00202200, 0x00202202, 0x00212101, 0x00222000, 0x00222002, 0x00222200, 0x00222202, + 0x01202101, 0x01212001, 0x01212100, 0x01222101, 0x02202000, 0x02202002, 0x02202200, 0x02202202, + 0x02222000, 0x02222002, 0x02222200, 0x02222202, 0x00202211, 0x00212011, 0x00212110, 0x00212211, + 0x00222111, 0x01202112, 0x01202211, 0x01212012, 0x01212111, 0x01222011, 0x01222110, 0x01222112, + 0x01222211, 0x02202111, 0x02212010, 0x02212112, 0x02212211, 0x02222110, 0x02222111, 0x00202020, + 0x00202022, 0x00202220, 0x00202222, 0x00222020, 0x00222022, 0x00222220, 0x00222222, 0x01202121, + 0x01212021, 0x01212122, 0x01212221, 0x01222121, 0x02202020, 0x02202022, 0x02202220, 0x02202222, + 0x02212121, 0x02222020, 0x02222022, 0x02222220, 0x02222222, 0x10000101, 0x10010001, 0x10010102, + 0x10020101, 0x11000201, 0x11010002, 0x11010101, 0x11010200, 0x11010202, 0x11020001, 0x11020100, + 0x11020102, 0x12010100, 0x12010201, 0x12020001, 0x12020102, 0x10000010, 0x10000011, 0x10000110, + 0x10000112, 0x10000211, 0x10010012, 0x10010111, 0x10010112, 0x10010210, 0x10010212, 0x10020011, + 0x10020112, 0x10020211, 0x11000111, 0x11000210, 0x11000212, 0x11010011, 0x11010110, 0x11010111, + 0x11010112, 0x11010211, 0x11010212, 0x11020111, 0x11020210, 0x11020212, 0x12000011, 0x12000110, + 0x12000112, 0x12010010, 0x12010012, 0x12010111, 0x12020010, 0x12020011, 0x12020012, 0x10000121, + 0x10010021, 0x10010120, 0x10010122, 0x10020121, 0x11000021, 0x11010022, 0x11010121, 0x11010222, + 0x11020120, 0x11020221, 0x12000221, 0x12010120, 0x12020121, 0x10001001, 0x10011101, 0x10011201, + 0x10021201, 0x11001101, 0x11001200, 0x11001202, 0x11011001, 0x11011100, 0x11011101, 0x11011102, + 0x11021001, 0x11021002, 0x11021101, 0x11021200, 0x11021202, 0x12001001, 0x12001102, 0x12001201, + 0x12011000, 0x12011002, 0x12011101, 0x12021000, 0x12021001, 0x12021201, 0x10001011, 0x10001012, + 0x10001111, 0x10001212, 0x10011011, 0x10011110, 0x10011111, 0x10011112, 0x10011211, 0x10021010, + 0x10021111, 0x10021212, 0x11001011, 0x11001110, 0x11001111, 0x11001112, 0x11001211, 0x11011010, + 0x11011011, 0x11011110, 0x11011111, 0x11011112, 0x11011210, 0x11011211, 0x11021011, 0x11021110, + 0x11021111, 0x11021112, 0x11021211, 0x12001012, 0x12001110, 0x12001111, 0x12001210, 0x12011011, + 0x12011110, 0x12011111, 0x12011112, 0x12011211, 0x12011212, 0x12021111, 0x12021210, 0x12021212, + 0x10001021, 0x10001121, 0x10001221, 0x10011120, 0x10011121, 0x10011220, 0x10011222, 0x10021021, + 0x10021120, 0x10021221, 0x11001020, 0x11001022, 0x11001121, 0x11001220, 0x11011020, 0x11011021, + 0x11011022, 0x11011121, 0x11011122, 0x11011221, 0x11021022, 0x11021121, 0x11021220, 0x12001021, + 0x12001121, 0x12001222, 0x12011120, 0x12011121, 0x12021021, 0x12021120, 0x12021122, 0x10002101, + 0x10012001, 0x10012101, 0x10012202, 0x10022101, 0x11002002, 0x11002201, 0x11012000, 0x11012101, + 0x11012200, 0x11022001, 0x11022100, 0x11022102, 0x11022201, 0x12002101, 0x12012001, 0x12012100, + 0x12012102, 0x12012201, 0x12022101, 0x10002011, 0x10002111, 0x10002112, 0x10002212, 0x10012010, + 0x10012110, 0x10012111, 0x10012210, 0x10022011, 0x10022110, 0x10022112, 0x11002010, 0x11002111, + 0x11002212, 0x11012011, 0x11012012, 0x11012110, 0x11012111, 0x11012112, 0x11012211, 0x11022010, + 0x11022012, 0x11022111, 0x11022112, 0x11022212, 0x12002112, 0x12002211, 0x12012012, 0x12012111, + 0x12012112, 0x12012210, 0x12022011, 0x12022110, 0x12022112, 0x12022211, 0x10012122, 0x11002120, + 0x11002122, 0x11002221, 0x11012121, 0x11012220, 0x11012222, 0x11022120, 0x11022221, 0x12012120, + 0x12022121, 0x10100001, 0x10100100, 0x10100101, 0x10100102, 0x10100201, 0x10110002, 0x10110101, + 0x10110202, 0x10120001, 0x10120100, 0x10120201, 0x11100000, 0x11100101, 0x11100200, 0x11110001, + 0x11110100, 0x11110101, 0x11110102, 0x11110201, 0x11120101, 0x11120200, 0x12100102, 0x12100201, + 0x12110101, 0x12110200, 0x12120000, 0x12120001, 0x12120102, 0x12120201, 0x10100111, 0x10100210, + 0x10100211, 0x10100212, 0x10110011, 0x10110110, 0x10110111, 0x10110112, 0x10110210, 0x10110211, + 0x10120010, 0x10120111, 0x10120112, 0x10120210, 0x10120212, 0x11100011, 0x11100110, 0x11100111, + 0x11100112, 0x11100211, 0x11110010, 0x11110011, 0x11110012, 0x11110110, 0x11110111, 0x11110112, + 0x11110210, 0x11110211, 0x11110212, 0x11120011, 0x11120110, 0x11120111, 0x11120112, 0x11120211, + 0x12100012, 0x12100111, 0x12110011, 0x12110110, 0x12110111, 0x12110112, 0x12110211, 0x12120010, + 0x12120111, 0x12120212, 0x10100021, 0x10100122, 0x10110022, 0x10110121, 0x10110222, 0x10120021, + 0x10120120, 0x11100022, 0x11100121, 0x11100222, 0x11110021, 0x11110120, 0x11110121, 0x11110122, + 0x11110221, 0x11120022, 0x11120121, 0x12100121, 0x12110020, 0x12110022, 0x12110121, 0x12110221, + 0x12110222, 0x12120120, 0x10101100, 0x10101101, 0x10111001, 0x10111100, 0x10111101, 0x10111102, + 0x10111200, 0x10111201, 0x10121001, 0x10121101, 0x10121200, 0x10121202, 0x11101001, 0x11101100, + 0x11101101, 0x11101102, 0x11101201, 0x11101202, 0x11111000, 0x11111001, 0x11111100, 0x11111101, + 0x11111102, 0x11111200, 0x11111201, 0x11111202, 0x11121001, 0x11121002, 0x11121100, 0x11121101, + 0x11121102, 0x11121201, 0x12101000, 0x12101200, 0x12101202, 0x12111001, 0x12111100, 0x12111101, + 0x12111102, 0x12111201, 0x12121001, 0x12121100, 0x12121101, 0x12121202, 0x10101011, 0x10101012, + 0x10101110, 0x10101111, 0x10101112, 0x10101211, 0x10111010, 0x10111011, 0x10111012, 0x10111110, + 0x10111111, 0x10111112, 0x10111211, 0x10111212, 0x10121011, 0x10121110, 0x10121111, 0x10121112, + 0x10121211, 0x11101010, 0x11101011, 0x11101012, 0x11101110, 0x11101111, 0x11101112, 0x11101210, + 0x11101211, 0x11111010, 0x11111011, 0x11111012, 0x11111110, 0x11111111, 0x11111112, 0x11111210, + 0x11111211, 0x11111212, 0x11121010, 0x11121011, 0x11121110, 0x11121111, 0x11121112, 0x11121210, + 0x11121211, 0x11121212, 0x12101011, 0x12101110, 0x12101111, 0x12101211, 0x12101212, 0x12111010, + 0x12111011, 0x12111110, 0x12111111, 0x12111112, 0x12111210, 0x12111211, 0x12121011, 0x12121110, + 0x12121111, 0x12121112, 0x12121211, 0x10101020, 0x10101021, 0x10101022, 0x10101120, 0x10101122, + 0x10101220, 0x10101221, 0x10111021, 0x10111120, 0x10111121, 0x10111220, 0x10111221, 0x10121020, + 0x10121021, 0x10121022, 0x10121120, 0x10121121, 0x10121122, 0x10121220, 0x10121221, 0x11101021, + 0x11101121, 0x11101122, 0x11101220, 0x11101221, 0x11101222, 0x11111020, 0x11111021, 0x11111022, + 0x11111120, 0x11111121, 0x11111122, 0x11111220, 0x11111221, 0x11111222, 0x11121021, 0x11121120, + 0x11121121, 0x11121221, 0x12101022, 0x12101121, 0x12101122, 0x12101220, 0x12101221, 0x12101222, + 0x12111021, 0x12111121, 0x12111222, 0x12121022, 0x12121121, 0x12121122, 0x12121220, 0x12121221, + 0x10102100, 0x10102101, 0x10102102, 0x10102201, 0x10112000, 0x10112101, 0x10112200, 0x10122001, + 0x10122202, 0x11102101, 0x11102200, 0x11102202, 0x11112001, 0x11112100, 0x11112101, 0x11112102, + 0x11112200, 0x11112201, 0x11122000, 0x11122002, 0x11122100, 0x11122101, 0x12102002, 0x12102201, + 0x12112000, 0x12112002, 0x12112101, 0x12112200, 0x12122001, 0x12122201, 0x10102011, 0x10102012, + 0x10102111, 0x10102212, 0x10112011, 0x10112110, 0x10112111, 0x10112112, 0x10112211, 0x10122111, + 0x11102011, 0x11102110, 0x11102111, 0x11102112, 0x11102211, 0x11112010, 0x11112011, 0x11112012, + 0x11112110, 0x11112111, 0x11112112, 0x11112210, 0x11112211, 0x11112212, 0x11122011, 0x11122110, + 0x11122111, 0x11122112, 0x11122211, 0x12102011, 0x12102111, 0x12102211, 0x12112011, 0x12112110, + 0x12112111, 0x12112112, 0x12112210, 0x12112211, 0x12122111, 0x10102120, 0x10102220, 0x10112121, + 0x10112222, 0x10122020, 0x10122121, 0x10122122, 0x10122221, 0x11102121, 0x11102220, 0x11102221, + 0x11112021, 0x11112121, 0x11112122, 0x11112220, 0x11112221, 0x11122022, 0x11122121, 0x11122220, + 0x11122222, 0x12102021, 0x12102222, 0x12112022, 0x12112121, 0x12112122, 0x12112220, 0x12112222, + 0x12122021, 0x10200101, 0x10210100, 0x10210102, 0x10210201, 0x10220101, 0x11200100, 0x11210000, + 0x11210101, 0x11210102, 0x11210200, 0x11210202, 0x11220001, 0x11220100, 0x11220102, 0x11220201, + 0x12200001, 0x12210102, 0x12220101, 0x10200011, 0x10200110, 0x10200112, 0x10200211, 0x10210012, + 0x10210111, 0x10220011, 0x10220012, 0x10220112, 0x10220211, 0x11200111, 0x11200211, 0x11210011, + 0x11210111, 0x11210112, 0x11210211, 0x11220111, 0x11220112, 0x11220212, 0x12200110, 0x12200212, + 0x12210012, 0x12210111, 0x12220011, 0x12220112, 0x12220211, 0x10210021, 0x10210122, 0x10210221, + 0x11200020, 0x11200021, 0x11200122, 0x11210121, 0x11210122, 0x11210220, 0x11220020, 0x12200121, + 0x12210021, 0x12210122, 0x12220121, 0x10211001, 0x10211002, 0x10211101, 0x10211102, 0x10211202, + 0x10221001, 0x10221102, 0x10221201, 0x11201000, 0x11201002, 0x11201101, 0x11201200, 0x11201202, + 0x11211001, 0x11211100, 0x11211101, 0x11211102, 0x11211201, 0x11211202, 0x11221000, 0x11221002, + 0x11221101, 0x12201100, 0x12201101, 0x12201201, 0x12211000, 0x12211002, 0x12211100, 0x12211101, + 0x12211102, 0x12211200, 0x12211202, 0x12221001, 0x12221100, 0x12221201, 0x10201111, 0x10201210, + 0x10201212, 0x10211011, 0x10211111, 0x10211112, 0x10211211, 0x11201110, 0x11201111, 0x11201112, + 0x11201211, 0x11211010, 0x11211011, 0x11211110, 0x11211111, 0x11211112, 0x11211211, 0x11221011, + 0x11221110, 0x11221111, 0x11221112, 0x11221211, 0x12201112, 0x12201211, 0x12201212, 0x12211011, + 0x12211111, 0x12211112, 0x12211211, 0x12211212, 0x12221012, 0x12221111, 0x12221112, 0x12221210, + 0x10201022, 0x10201221, 0x10211121, 0x10221020, 0x10221122, 0x10221220, 0x10221221, 0x11201020, + 0x11201121, 0x11201220, 0x11201222, 0x11211021, 0x11211120, 0x11211121, 0x11211122, 0x11211220, + 0x11211222, 0x11221020, 0x11221121, 0x11221220, 0x12201020, 0x12201022, 0x12201121, 0x12201222, + 0x12211120, 0x12211122, 0x12211220, 0x12211221, 0x12221020, 0x12221120, 0x12221122, 0x12221222, + 0x10212102, 0x10212201, 0x10222101, 0x11202001, 0x11212002, 0x11212101, 0x11212202, 0x11222001, + 0x11222201, 0x12202101, 0x12212001, 0x12212200, 0x12222102, 0x10202011, 0x10202110, 0x10212010, + 0x10212111, 0x10222011, 0x10222110, 0x10222112, 0x10222211, 0x11202010, 0x11202011, 0x11202111, + 0x11202112, 0x11202210, 0x11212011, 0x11212110, 0x11212111, 0x11212112, 0x11212211, 0x11222010, + 0x11222111, 0x11222212, 0x12202012, 0x12202110, 0x12202212, 0x12212111, 0x12222011, 0x12222110, + 0x12222111, 0x12222211, 0x10212021, 0x10212122, 0x10212220, 0x11202021, 0x11202120, 0x11202221, + 0x11212020, 0x11212121, 0x11212220, 0x11212222, 0x11222120, 0x11222121, 0x11222221, 0x12202122, + 0x12212120, 0x12212220, 0x12212222, 0x12222122, 0x20000000, 0x20000002, 0x20000200, 0x20000202, + 0x20020000, 0x20020002, 0x20020200, 0x20020202, 0x21000101, 0x21010000, 0x21010001, 0x21010100, + 0x21010102, 0x21010201, 0x21020101, 0x22000000, 0x22000002, 0x22000200, 0x22000202, 0x22010101, + 0x22020000, 0x22020002, 0x22020200, 0x22020202, 0x20000111, 0x20010011, 0x20010110, 0x20010112, + 0x20010211, 0x20020111, 0x21000011, 0x21000110, 0x21000211, 0x21010010, 0x21010012, 0x21010111, + 0x21010112, 0x21010210, 0x21010211, 0x21020110, 0x21020112, 0x21020211, 0x22000111, 0x22000211, + 0x22010110, 0x22010112, 0x22010211, 0x22020111, 0x20000020, 0x20000022, 0x20000220, 0x20000222, + 0x20010121, 0x20020020, 0x20020022, 0x20020220, 0x20020222, 0x21010021, 0x21010120, 0x21010221, + 0x21020121, 0x22000020, 0x22000022, 0x22000220, 0x22000222, 0x22010121, 0x22020020, 0x22020022, + 0x22020220, 0x22020222, 0x20011100, 0x20011201, 0x21001001, 0x21001100, 0x21011001, 0x21011101, + 0x21011202, 0x21021001, 0x21021100, 0x21021201, 0x22011100, 0x22011201, 0x20001011, 0x20001211, + 0x20011012, 0x20011111, 0x20011212, 0x20021112, 0x20021211, 0x21001010, 0x21001011, 0x21001111, + 0x21001210, 0x21011011, 0x21011110, 0x21011111, 0x21011112, 0x21011211, 0x21011212, 0x21021111, + 0x21021112, 0x21021210, 0x21021212, 0x22001011, 0x22001110, 0x22001112, 0x22001211, 0x22011010, + 0x22011012, 0x22011111, 0x22011210, 0x22021112, 0x20011021, 0x20011122, 0x20011221, 0x20021121, + 0x21001021, 0x21001120, 0x21001221, 0x21001222, 0x21011020, 0x21011121, 0x21011221, 0x21011222, + 0x21021021, 0x21021122, 0x21021222, 0x22001121, 0x22011021, 0x22011222, 0x22021120, 0x20002000, + 0x20002002, 0x20002200, 0x20002202, 0x20012101, 0x20022000, 0x20022002, 0x20022200, 0x20022202, + 0x21002001, 0x21002101, 0x21012001, 0x21012100, 0x21012201, 0x21022101, 0x21022201, 0x22002000, + 0x22002002, 0x22002200, 0x22002202, 0x22012101, 0x22022000, 0x22022002, 0x22022200, 0x22022202, + 0x20002111, 0x20002112, 0x20012011, 0x20012110, 0x20012112, 0x20022111, 0x21002011, 0x21002110, + 0x21002112, 0x21002211, 0x21012010, 0x21012012, 0x21012111, 0x21012212, 0x21022011, 0x21022110, + 0x22002111, 0x22012112, 0x22012211, 0x22022111, 0x20002020, 0x20002022, 0x20002220, 0x20002222, + 0x20012121, 0x20022020, 0x20022022, 0x20022220, 0x20022222, 0x21002121, 0x21012021, 0x21012120, + 0x21012122, 0x22002020, 0x22002022, 0x22002220, 0x22002222, 0x22012121, 0x22022020, 0x22022022, + 0x22022220, 0x22022222, 0x20100101, 0x20110001, 0x20110102, 0x20110200, 0x20110201, 0x20120101, + 0x21100001, 0x21100102, 0x21100201, 0x21110101, 0x21110200, 0x21110202, 0x21120201, 0x21120202, + 0x22100101, 0x22110001, 0x22110100, 0x22110102, 0x22110201, 0x22120101, 0x20100011, 0x20100110, + 0x20100112, 0x20100211, 0x20110010, 0x20110111, 0x20110210, 0x20110212, 0x20120011, 0x20120110, + 0x20120112, 0x20120211, 0x21100010, 0x21100111, 0x21110010, 0x21110011, 0x21110110, 0x21110111, + 0x21110112, 0x21110211, 0x21120012, 0x21120111, 0x22100110, 0x22100112, 0x22110012, 0x22110111, + 0x22110210, 0x22120011, 0x22120110, 0x22120112, 0x22120211, 0x20100121, 0x20110021, 0x20110120, + 0x20110221, 0x20120121, 0x21100120, 0x21100122, 0x21100221, 0x21110020, 0x21110022, 0x21110121, + 0x21110220, 0x21120122, 0x21120221, 0x22100121, 0x22110120, 0x22110122, 0x22120221, 0x20101001, + 0x20101100, 0x20101102, 0x20111000, 0x20111101, 0x20111200, 0x20121102, 0x21101000, 0x21101202, + 0x21111001, 0x21111100, 0x21111101, 0x21111102, 0x21111200, 0x21111201, 0x21121000, 0x21121001, + 0x21121002, 0x21121101, 0x22101100, 0x22101102, 0x22111002, 0x22111100, 0x22111101, 0x22111200, + 0x22121001, 0x22121201, 0x20101010, 0x20101111, 0x20101210, 0x20101212, 0x20111010, 0x20111011, + 0x20111110, 0x20111111, 0x20111112, 0x20111211, 0x20121011, 0x20121111, 0x20121211, 0x20121212, + 0x21101011, 0x21101110, 0x21101111, 0x21101112, 0x21101211, 0x21111010, 0x21111011, 0x21111012, + 0x21111110, 0x21111111, 0x21111112, 0x21111210, 0x21111211, 0x21111212, 0x21121011, 0x21121110, + 0x21121111, 0x21121112, 0x21121211, 0x22101011, 0x22101111, 0x22101210, 0x22111011, 0x22111012, + 0x22111110, 0x22111111, 0x22111112, 0x22111211, 0x22111212, 0x22121010, 0x22121012, 0x22121111, + 0x22121210, 0x22121212, 0x20101021, 0x20101120, 0x20111020, 0x20111121, 0x20111221, 0x20121020, + 0x20121122, 0x20121221, 0x21101121, 0x21101220, 0x21101221, 0x21111021, 0x21111022, 0x21111121, + 0x21111122, 0x21111221, 0x21121121, 0x21121220, 0x22101022, 0x22101120, 0x22101221, 0x22101222, + 0x22111022, 0x22111120, 0x22111121, 0x22121120, 0x22121122, 0x22121221, 0x20102101, 0x20112102, + 0x20112201, 0x20122101, 0x21102001, 0x21102102, 0x21112000, 0x21112002, 0x21112101, 0x21112102, + 0x21112202, 0x21122100, 0x21122101, 0x22102101, 0x22112001, 0x22112102, 0x22112201, 0x22122101, + 0x20102110, 0x20102112, 0x20102211, 0x20112010, 0x20112012, 0x20112111, 0x20112210, 0x20112212, + 0x20122010, 0x20122011, 0x20122110, 0x20122112, 0x21102010, 0x21102012, 0x21102111, 0x21102210, + 0x21102212, 0x21112011, 0x21112110, 0x21112111, 0x21112112, 0x21112211, 0x21122012, 0x21122111, + 0x21122112, 0x21122212, 0x22102011, 0x22102110, 0x22112010, 0x22112012, 0x22112111, 0x22112212, + 0x22122011, 0x22122112, 0x20102121, 0x20112121, 0x20122121, 0x21102120, 0x21102122, 0x21102221, + 0x21112020, 0x21112121, 0x21112220, 0x21122021, 0x22102121, 0x22112021, 0x22112120, 0x22112121, + 0x22112122, 0x20200000, 0x20200002, 0x20200200, 0x20200202, 0x20210101, 0x20220000, 0x20220002, + 0x20220200, 0x20220202, 0x21200101, 0x21210001, 0x21210100, 0x21210102, 0x21210201, 0x22200000, + 0x22200002, 0x22200200, 0x22200202, 0x22210101, 0x22220000, 0x22220002, 0x22220200, 0x22220202, + 0x20200111, 0x20200211, 0x20210011, 0x20210110, 0x20210112, 0x20210211, 0x20210212, 0x21200112, + 0x21200211, 0x21210011, 0x21210111, 0x21210210, 0x21210212, 0x21220011, 0x21220110, 0x22200111, + 0x22210010, 0x22210012, 0x22210112, 0x22210211, 0x20200022, 0x20200220, 0x20200222, 0x20210020, + 0x20210221, 0x20220022, 0x20220220, 0x20220222, 0x21200121, 0x21210021, 0x21210122, 0x21210221, + 0x21220121, 0x22200020, 0x22200022, 0x22200220, 0x22200222, 0x22210121, 0x22220020, 0x22220022, + 0x22220220, 0x22220222, 0x20211201, 0x20221101, 0x21201001, 0x21201100, 0x21211000, 0x21211100, + 0x21211101, 0x21211200, 0x21211202, 0x21221001, 0x21221101, 0x21221102, 0x21221200, 0x21221201, + 0x22201101, 0x20201112, 0x20201211, 0x20211010, 0x20211012, 0x20211111, 0x20211210, 0x20221112, + 0x20221211, 0x21201012, 0x21201111, 0x21211011, 0x21211110, 0x21211111, 0x21211112, 0x21211211, + 0x21221111, 0x21221212, 0x22201011, 0x22201110, 0x22201111, 0x22201112, 0x22201211, 0x22211012, + 0x22211111, 0x22211210, 0x20201121, 0x20211021, 0x20211122, 0x20211222, 0x20221021, 0x20221121, + 0x21201120, 0x21201122, 0x21201222, 0x21211022, 0x21211121, 0x21211122, 0x21211220, 0x21221020, + 0x21221022, 0x22201122, 0x22211020, 0x22211121, 0x22211122, 0x22211221, 0x22221021, 0x22221120, + 0x22221122, 0x20202000, 0x20202002, 0x20202200, 0x20202202, 0x20222000, 0x20222002, 0x20222200, + 0x20222202, 0x21212001, 0x21212100, 0x21212102, 0x21212201, 0x22202000, 0x22202002, 0x22202200, + 0x22202202, 0x22212101, 0x22222000, 0x22222002, 0x22222200, 0x22222202, 0x20202111, 0x20212110, + 0x20212211, 0x20222011, 0x20222111, 0x21202011, 0x21212010, 0x21212111, 0x21212212, 0x21222011, + 0x21222112, 0x21222211, 0x22212010, 0x22212112, 0x20202020, 0x20202022, 0x20202220, 0x20202222, + 0x20222020, 0x20222022, 0x20222220, 0x20222222, 0x21212021, 0x21212120, 0x21212122, 0x22202020, + 0x22202022, 0x22202220, 0x22202222, 0x22212121, 0x22222020, 0x22222022, 0x22222220, 0x22222222, +GGML_TABLE_END() +#endif + +#endif // GGML_COMMON_IMPL +#endif // GGML_COMMON_IMPL diff --git a/python/freetoken/kernel/csrc/gguf_mmq/ggml-cuda.h b/python/freetoken/kernel/csrc/gguf_mmq/ggml-cuda.h new file mode 100644 index 00000000..1cd81eea --- /dev/null +++ b/python/freetoken/kernel/csrc/gguf_mmq/ggml-cuda.h @@ -0,0 +1,47 @@ +#pragma once + +#include "ggml.h" +#include "ggml-backend.h" + +#ifdef __cplusplus +extern "C" { +#endif + +#ifdef GGML_USE_HIP +#define GGML_CUDA_NAME "ROCm" +#define GGML_CUBLAS_NAME "hipBLAS" +#elif defined(GGML_USE_MUSA) +#define GGML_CUDA_NAME "MUSA" +#define GGML_CUBLAS_NAME "muBLAS" +#else +#define GGML_CUDA_NAME "CUDA" +#define GGML_CUBLAS_NAME "cuBLAS" +#endif +#define GGML_CUDA_MAX_DEVICES 16 + +// backend API +GGML_BACKEND_API ggml_backend_t ggml_backend_cuda_init(int device); + +GGML_BACKEND_API bool ggml_backend_is_cuda(ggml_backend_t backend); + +// device buffer +GGML_BACKEND_API ggml_backend_buffer_type_t ggml_backend_cuda_buffer_type(int device); + +// conduct allreduce operation between devices +GGML_BACKEND_API bool ggml_backend_cuda_allreduce_tensor(ggml_backend_t * backends, struct ggml_tensor ** tensors, size_t n_backends); + +// pinned host buffer for use with the CPU backend for faster copies between CPU and GPU +GGML_BACKEND_API ggml_backend_buffer_type_t ggml_backend_cuda_host_buffer_type(void); + +GGML_BACKEND_API int ggml_backend_cuda_get_device_count(void); +GGML_BACKEND_API void ggml_backend_cuda_get_device_description(int device, char * description, size_t description_size); +GGML_BACKEND_API void ggml_backend_cuda_get_device_memory(int device, size_t * free, size_t * total); + +GGML_BACKEND_API bool ggml_backend_cuda_register_host_buffer(void * buffer, size_t size); +GGML_BACKEND_API void ggml_backend_cuda_unregister_host_buffer(void * buffer); + +GGML_BACKEND_API ggml_backend_reg_t ggml_backend_cuda_reg(void); + +#ifdef __cplusplus +} +#endif diff --git a/python/freetoken/kernel/csrc/gguf_mmq/ggml-impl.h b/python/freetoken/kernel/csrc/gguf_mmq/ggml-impl.h new file mode 100644 index 00000000..62b76abb --- /dev/null +++ b/python/freetoken/kernel/csrc/gguf_mmq/ggml-impl.h @@ -0,0 +1,783 @@ +#pragma once + +// GGML internal header + +#include "ggml.h" +#include "gguf.h" + +#include +#include +#include // load `stdlib.h` before other headers to work around MinGW bug: https://sourceforge.net/p/mingw-w64/bugs/192/ +#include +#include +#include + +#ifdef __ARM_FEATURE_SVE +#include +#endif // __ARM_FEATURE_SVE + +#if defined(__ARM_NEON) && !defined(__CUDACC__) && !defined(__MUSACC__) +// if YCM cannot find , make a symbolic link to it, for example: +// +// $ ln -sfn /Library/Developer/CommandLineTools/usr/lib/clang/13.1.6/include/arm_neon.h ./src/ +// +#include +#endif + +#ifdef __cplusplus +extern "C" { +#endif + +void ggml_print_backtrace(void); + +uint64_t ggml_graph_next_uid(void); + +#ifndef MIN +# define MIN(a, b) ((a) < (b) ? (a) : (b)) +#endif + +#ifndef MAX +# define MAX(a, b) ((a) > (b) ? (a) : (b)) +#endif + +// required for mmap as gguf only guarantees 32-byte alignment +#define TENSOR_ALIGNMENT 32 + +// static_assert should be a #define, but if it's not, +// fall back to the _Static_assert C11 keyword. +// if C99 - static_assert is noop +// ref: https://stackoverflow.com/a/53923785/4039976 +#ifndef __cplusplus + #ifndef static_assert + #if defined(__STDC_VERSION__) && (__STDC_VERSION__ >= 201100L) + #define static_assert(cond, msg) _Static_assert(cond, msg) + #else + #define static_assert(cond, msg) struct global_scope_noop_trick + #endif + #endif +#endif + +static inline int ggml_up32(int n) { + return (n + 31) & ~31; +} + +//static inline int ggml_up64(int n) { +// return (n + 63) & ~63; +//} + +static inline int ggml_up(int n, int m) { + // assert m is a power of 2 + GGML_ASSERT((m & (m - 1)) == 0); + return (n + m - 1) & ~(m - 1); +} + +// TODO: move to ggml.h? (won't be able to inline) +static bool ggml_are_same_layout(const struct ggml_tensor * a, const struct ggml_tensor * b) { + if (a->type != b->type) { + return false; + } + for (int i = 0; i < GGML_MAX_DIMS; i++) { + if (a->ne[i] != b->ne[i]) { + return false; + } + if (a->nb[i] != b->nb[i]) { + return false; + } + } + return true; +} + +static bool ggml_op_is_empty(enum ggml_op op) { + switch (op) { + case GGML_OP_NONE: + case GGML_OP_RESHAPE: + case GGML_OP_TRANSPOSE: + case GGML_OP_VIEW: + case GGML_OP_PERMUTE: + return true; + default: + return false; + } +} + +static inline bool ggml_impl_is_view(const struct ggml_tensor * t) { + return t->view_src != NULL; +} + +static inline float ggml_compute_softplus_f32(float input) { + return (input > 20.0f) ? input : logf(1 + expf(input)); +} +// +// logging +// + +GGML_ATTRIBUTE_FORMAT(2, 3) +GGML_API void ggml_log_internal (enum ggml_log_level level, const char * format, ...); +GGML_API void ggml_log_callback_default(enum ggml_log_level level, const char * text, void * user_data); + +#define GGML_LOG(...) ggml_log_internal(GGML_LOG_LEVEL_NONE , __VA_ARGS__) +#define GGML_LOG_INFO(...) ggml_log_internal(GGML_LOG_LEVEL_INFO , __VA_ARGS__) +#define GGML_LOG_WARN(...) ggml_log_internal(GGML_LOG_LEVEL_WARN , __VA_ARGS__) +#define GGML_LOG_ERROR(...) ggml_log_internal(GGML_LOG_LEVEL_ERROR, __VA_ARGS__) +#define GGML_LOG_DEBUG(...) ggml_log_internal(GGML_LOG_LEVEL_DEBUG, __VA_ARGS__) +#define GGML_LOG_CONT(...) ggml_log_internal(GGML_LOG_LEVEL_CONT , __VA_ARGS__) + +#define GGML_DEBUG 0 + +#if (GGML_DEBUG >= 1) +#define GGML_PRINT_DEBUG(...) GGML_LOG_DEBUG(__VA_ARGS__) +#else +#define GGML_PRINT_DEBUG(...) +#endif + +#if (GGML_DEBUG >= 5) +#define GGML_PRINT_DEBUG_5(...) GGML_LOG_DEBUG(__VA_ARGS__) +#else +#define GGML_PRINT_DEBUG_5(...) +#endif + +#if (GGML_DEBUG >= 10) +#define GGML_PRINT_DEBUG_10(...) GGML_LOG_DEBUG(__VA_ARGS__) +#else +#define GGML_PRINT_DEBUG_10(...) +#endif + +// tensor params + +static void ggml_set_op_params(struct ggml_tensor * tensor, const void * params, size_t params_size) { + GGML_ASSERT(tensor != NULL); // silence -Warray-bounds warnings + assert(params_size <= GGML_MAX_OP_PARAMS); + memcpy(tensor->op_params, params, params_size); +} + +static int32_t ggml_get_op_params_i32(const struct ggml_tensor * tensor, uint32_t i) { + assert(i < GGML_MAX_OP_PARAMS / sizeof(int32_t)); + return ((const int32_t *)(tensor->op_params))[i]; +} + +static float ggml_get_op_params_f32(const struct ggml_tensor * tensor, uint32_t i) { + assert(i < GGML_MAX_OP_PARAMS / sizeof(float)); + return ((const float *)(tensor->op_params))[i]; +} + +static void ggml_set_op_params_i32(struct ggml_tensor * tensor, uint32_t i, int32_t value) { + assert(i < GGML_MAX_OP_PARAMS / sizeof(int32_t)); + ((int32_t *)(tensor->op_params))[i] = value; +} + +static void ggml_set_op_params_f32(struct ggml_tensor * tensor, uint32_t i, float value) { + assert(i < GGML_MAX_OP_PARAMS / sizeof(float)); + ((float *)(tensor->op_params))[i] = value; +} + +struct ggml_map_custom1_op_params { + ggml_custom1_op_t fun; + int n_tasks; + void * userdata; +}; + +struct ggml_map_custom2_op_params { + ggml_custom2_op_t fun; + int n_tasks; + void * userdata; +}; + +struct ggml_map_custom3_op_params { + ggml_custom3_op_t fun; + int n_tasks; + void * userdata; +}; + +struct ggml_custom_op_params { + ggml_custom_op_t fun; + int n_tasks; + void * userdata; +}; + +// bitset + +typedef uint32_t ggml_bitset_t; + +static_assert(sizeof(ggml_bitset_t) == 4, "bitset_t constants must be updated"); +#define BITSET_SHR 5 // log2(sizeof(ggml_bitset_t)*8) +#define BITSET_MASK (sizeof(ggml_bitset_t)*8 - 1) + +static size_t ggml_bitset_size(size_t n) { + return (n + BITSET_MASK) >> BITSET_SHR; +} + +static inline bool ggml_bitset_get(const ggml_bitset_t * bitset, size_t i) { + return !!(bitset[i >> BITSET_SHR] & (1u << (i & BITSET_MASK))); +} + +static inline void ggml_bitset_set(ggml_bitset_t * bitset, size_t i) { + bitset[i >> BITSET_SHR] |= (1u << (i & BITSET_MASK)); +} + +static inline void ggml_bitset_clear(ggml_bitset_t * bitset, size_t i) { + bitset[i >> BITSET_SHR] &= ~(1u << (i & BITSET_MASK)); +} + +// hash set + +#define GGML_HASHSET_FULL ((size_t)-1) +#define GGML_HASHSET_ALREADY_EXISTS ((size_t)-2) + +struct ggml_hash_set { + size_t size; + ggml_bitset_t * used; // whether or not the keys are in use i.e. set + struct ggml_tensor ** keys; // actual tensors in the set, keys[i] is only defined if ggml_bitset_get(used, i) +}; + +struct ggml_hash_set ggml_hash_set_new(size_t size); +void ggml_hash_set_free(struct ggml_hash_set * hash_set); + +// returns the minimum size for a hash set that can hold min_sz elements +size_t ggml_hash_size(size_t min_sz); + +// remove all elements from the hash set +void ggml_hash_set_reset(struct ggml_hash_set * hash_set); + +// returns true if key is in the hash set +static bool ggml_hash_contains(const struct ggml_hash_set * hash_set, struct ggml_tensor * key); + +// returns GGML_HASHSET_FULL if table is full, otherwise the current index of the key or where it should be inserted +static size_t ggml_hash_find(const struct ggml_hash_set * hash_set, const struct ggml_tensor * key); + +// returns GGML_HASHSET_ALREADY_EXISTS if key already exists, index otherwise, asserts if table is full +static size_t ggml_hash_insert(struct ggml_hash_set * hash_set, struct ggml_tensor * key); + +// return index, asserts if table is full +static size_t ggml_hash_find_or_insert(struct ggml_hash_set * hash_set, struct ggml_tensor * key); + +// hash function for ggml_tensor +static inline size_t ggml_hash(const struct ggml_tensor * p) { + // the last 4 bits are always zero due to alignment + return (size_t)(uintptr_t)p >> 4; +} + +static size_t ggml_hash_find(const struct ggml_hash_set * hash_set, const struct ggml_tensor * key) { + size_t h = ggml_hash(key) % hash_set->size; + + // linear probing + size_t i = h; + while (ggml_bitset_get(hash_set->used, i) && hash_set->keys[i] != key) { + i = (i + 1) % hash_set->size; + if (i == h) { + // visited all hash table entries -> not found + return GGML_HASHSET_FULL; + } + } + return i; +} + +static bool ggml_hash_contains(const struct ggml_hash_set * hash_set, struct ggml_tensor * key) { + size_t i = ggml_hash_find(hash_set, key); + return i != GGML_HASHSET_FULL && ggml_bitset_get(hash_set->used, i); +} + +static size_t ggml_hash_insert(struct ggml_hash_set * hash_set, struct ggml_tensor * key) { + size_t h = ggml_hash(key) % hash_set->size; + + // linear probing + size_t i = h; + do { + if (!ggml_bitset_get(hash_set->used, i)) { + ggml_bitset_set(hash_set->used, i); + hash_set->keys[i] = key; + return i; + } + if (hash_set->keys[i] == key) { + return GGML_HASHSET_ALREADY_EXISTS; + } + i = (i + 1) % hash_set->size; + } while (i != h); + + // visited all hash table entries -> not found + GGML_ABORT("fatal error"); +} + +static size_t ggml_hash_find_or_insert(struct ggml_hash_set * hash_set, struct ggml_tensor * key) { + size_t h = ggml_hash(key) % hash_set->size; + + // linear probing + size_t i = h; + do { + if (!ggml_bitset_get(hash_set->used, i)) { + ggml_bitset_set(hash_set->used, i); + hash_set->keys[i] = key; + return i; + } + if (hash_set->keys[i] == key) { + return i; + } + i = (i + 1) % hash_set->size; + } while (i != h); + + // visited all hash table entries -> not found + GGML_ABORT("fatal error"); +} + +// computation graph + +enum ggml_cgraph_eval_order { + GGML_CGRAPH_EVAL_ORDER_LEFT_TO_RIGHT = 0, + GGML_CGRAPH_EVAL_ORDER_RIGHT_TO_LEFT, + GGML_CGRAPH_EVAL_ORDER_COUNT +}; + +struct ggml_cgraph { + int size; // maximum number of nodes/leafs/grads/grad_accs + int n_nodes; // number of nodes currently in use + int n_leafs; // number of leafs currently in use + + struct ggml_tensor ** nodes; // tensors with data that can change if the graph is evaluated + struct ggml_tensor ** grads; // the outputs of these tensors are the gradients of the nodes + struct ggml_tensor ** grad_accs; // accumulators for node gradients + struct ggml_tensor ** leafs; // tensors with constant data + int32_t * use_counts;// number of uses of each tensor, indexed by hash table slot + + struct ggml_hash_set visited_hash_set; + + enum ggml_cgraph_eval_order order; + + // an optional identifier that can be utilized to recognize same graphs if two non-zero values match + // a value of 0 means it is not set and should be ignored + uint64_t uid; +}; + +// returns a slice of cgraph with nodes [i0, i1) +// the slice does not have leafs or gradients +// if you need the gradients, get them from the original graph +struct ggml_cgraph ggml_graph_view(struct ggml_cgraph * cgraph, int i0, int i1); + +// ggml-alloc.c: true if the operation can reuse memory from its sources +GGML_API bool ggml_op_can_inplace(enum ggml_op op); + + +// Memory allocation + +GGML_API void * ggml_aligned_malloc(size_t size); +GGML_API void ggml_aligned_free(void * ptr, size_t size); + +// FP16 <-> FP32 +// ref: https://github.com/Maratyszcza/FP16 + +static inline float fp32_from_bits(uint32_t w) { + union { + uint32_t as_bits; + float as_value; + } fp32; + fp32.as_bits = w; + return fp32.as_value; +} + +static inline uint32_t fp32_to_bits(float f) { + union { + float as_value; + uint32_t as_bits; + } fp32; + fp32.as_value = f; + return fp32.as_bits; +} + +static inline float ggml_compute_fp16_to_fp32(ggml_fp16_t h) { + const uint32_t w = (uint32_t) h << 16; + const uint32_t sign = w & UINT32_C(0x80000000); + const uint32_t two_w = w + w; + + const uint32_t exp_offset = UINT32_C(0xE0) << 23; +#if (defined(__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) || defined(__GNUC__) && !defined(__STRICT_ANSI__)) && (!defined(__cplusplus) || __cplusplus >= 201703L) + const float exp_scale = 0x1.0p-112f; +#else + const float exp_scale = fp32_from_bits(UINT32_C(0x7800000)); +#endif + const float normalized_value = fp32_from_bits((two_w >> 4) + exp_offset) * exp_scale; + + const uint32_t magic_mask = UINT32_C(126) << 23; + const float magic_bias = 0.5f; + const float denormalized_value = fp32_from_bits((two_w >> 17) | magic_mask) - magic_bias; + + const uint32_t denormalized_cutoff = UINT32_C(1) << 27; + const uint32_t result = sign | + (two_w < denormalized_cutoff ? fp32_to_bits(denormalized_value) : fp32_to_bits(normalized_value)); + return fp32_from_bits(result); +} + +static inline ggml_fp16_t ggml_compute_fp32_to_fp16(float f) { +#if (defined(__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) || defined(__GNUC__) && !defined(__STRICT_ANSI__)) && (!defined(__cplusplus) || __cplusplus >= 201703L) + const float scale_to_inf = 0x1.0p+112f; + const float scale_to_zero = 0x1.0p-110f; +#else + const float scale_to_inf = fp32_from_bits(UINT32_C(0x77800000)); + const float scale_to_zero = fp32_from_bits(UINT32_C(0x08800000)); +#endif + float base = (fabsf(f) * scale_to_inf) * scale_to_zero; + + const uint32_t w = fp32_to_bits(f); + const uint32_t shl1_w = w + w; + const uint32_t sign = w & UINT32_C(0x80000000); + uint32_t bias = shl1_w & UINT32_C(0xFF000000); + if (bias < UINT32_C(0x71000000)) { + bias = UINT32_C(0x71000000); + } + + base = fp32_from_bits((bias >> 1) + UINT32_C(0x07800000)) + base; + const uint32_t bits = fp32_to_bits(base); + const uint32_t exp_bits = (bits >> 13) & UINT32_C(0x00007C00); + const uint32_t mantissa_bits = bits & UINT32_C(0x00000FFF); + const uint32_t nonsign = exp_bits + mantissa_bits; + return (sign >> 16) | (shl1_w > UINT32_C(0xFF000000) ? UINT16_C(0x7E00) : nonsign); +} + +#define GGML_COMPUTE_FP16_TO_FP32(x) ggml_compute_fp16_to_fp32(x) +#define GGML_COMPUTE_FP32_TO_FP16(x) ggml_compute_fp32_to_fp16(x) + +#define GGML_FP16_TO_FP32(x) GGML_COMPUTE_FP16_TO_FP32(x) +#define GGML_FP32_TO_FP16(x) GGML_COMPUTE_FP32_TO_FP16(x) + +static inline float ggml_e8m0_to_fp32(uint8_t x) { + uint32_t bits; // Stores the raw bit representation of the float + + // Handle special case for minimum exponent (denormalized float) + if (x == 0) { + // Bit pattern for 2^(-127): + // - Sign bit: 0 (positive) + // - Exponent: 0 (denormalized number) + // - Mantissa: 0x400000 (0.5 in fractional form) + // Value = 0.5 * 2^(-126) = 2^(-127) + bits = 0x00400000; + } + // note: disabled as we don't need to handle NaNs + //// Handle special case for NaN (all bits set) + //else if (x == 0xFF) { + // // Standard quiet NaN pattern: + // // - Sign bit: 0 + // // - Exponent: all 1s (0xFF) + // // - Mantissa: 0x400000 (quiet NaN flag) + // bits = 0x7FC00000; + //} + // Normalized values (most common case) + else { + // Construct normalized float by shifting exponent into position: + // - Exponent field: 8 bits (positions 30-23) + // - Mantissa: 0 (implicit leading 1) + // Value = 2^(x - 127) + bits = (uint32_t) x << 23; + } + + float result; // Final float value + // Safely reinterpret bit pattern as float without type-punning issues + memcpy(&result, &bits, sizeof(float)); + return result; +} + +// Equal to ggml_e8m0_to_fp32/2 +// Useful with MXFP4 quantization since the E0M2 values are doubled +static inline float ggml_e8m0_to_fp32_half(uint8_t x) { + uint32_t bits; + + // For x < 2: use precomputed denormal patterns + if (x < 2) { + // 0x00200000 = 2^(-128), 0x00400000 = 2^(-127) + bits = 0x00200000 << x; + } + // For x >= 2: normalized exponent adjustment + else { + // 0.5 * 2^(x-127) = 2^(x-128) = normalized with exponent (x-1) + bits = (uint32_t)(x - 1) << 23; + } + // Note: NaNs are not handled here + + float result; + memcpy(&result, &bits, sizeof(float)); + return result; +} + +#define GGML_E8M0_TO_FP32(x) ggml_e8m0_to_fp32(x) +#define GGML_E8M0_TO_FP32_HALF(x) ggml_e8m0_to_fp32_half(x) + +// UE4M3: unsigned, 4 exp bits (bias=7), 3 mantissa bits +// Returns value * 0.5 to match kvalues_mxfp4 convention (kvalues = 2 * E2M1_float) +static inline float ggml_ue4m3_to_fp32(uint8_t x) { + if (x == 0 || x == 0x7F) { + return 0.0f; + } + int exp = (x >> 3) & 0xF; + int man = x & 0x7; + float raw; + if (exp == 0) { + raw = ldexpf((float) man, -9); + } else { + raw = ldexpf(1.0f + (float) man / 8.0f, exp - 7); + } + return raw * 0.5f; +} + +static inline uint8_t ggml_fp32_to_ue4m3(float x) { + if (!(x > 0.0f)) { + return 0; + } + if (x > 448.0f) { + x = 448.0f; + } + uint32_t bits; + memcpy(&bits, &x, 4); + int fp32_exp = ((bits >> 23) & 0xFF) - 127; + int fp32_man = (bits >> 20) & 0x7; + int ue4m3_exp = fp32_exp + 7; + if (ue4m3_exp <= 0) { + // subnormal: value = man * 2^-9, man = round(x * 2^9) + int man = (int) (x * 512.0f + 0.5f); + if (man > 7) { + man = 7; + } + if (man < 1) { + return 0; + } + return (uint8_t) man; + } + if (ue4m3_exp >= 15) { + return 0x7E; + } + int round_bit = (bits >> 19) & 1; + int ue4m3_man = fp32_man + round_bit; + if (ue4m3_man > 7) { + ue4m3_man = 0; + ue4m3_exp++; + if (ue4m3_exp >= 15) { + return 0x7E; + } + } + return (uint8_t) ((ue4m3_exp << 3) | ue4m3_man); +} + +/** + * Converts brain16 to float32. + * + * The bfloat16 floating point format has the following structure: + * + * ┌sign + * │ + * │ ┌exponent + * │ │ + * │ │ ┌mantissa + * │ │ │ + * │┌──┴───┐┌─┴───┐ + * 0b0000000000000000 brain16 + * + * Since bf16 has the same number of exponent bits as a 32bit float, + * encoding and decoding numbers becomes relatively straightforward. + * + * ┌sign + * │ + * │ ┌exponent + * │ │ + * │ │ ┌mantissa + * │ │ │ + * │┌──┴───┐┌─┴───────────────────┐ + * 0b00000000000000000000000000000000 IEEE binary32 + * + * For comparison, the standard fp16 format has fewer exponent bits. + * + * ┌sign + * │ + * │ ┌exponent + * │ │ + * │ │ ┌mantissa + * │ │ │ + * │┌─┴─┐┌─┴──────┐ + * 0b0000000000000000 IEEE binary16 + * + * @see IEEE 754-2008 + */ +static inline float ggml_compute_bf16_to_fp32(ggml_bf16_t h) { + union { + float f; + uint32_t i; + } u; + u.i = (uint32_t)h.bits << 16; + return u.f; +} + +/** + * Converts float32 to brain16. + * + * This is binary identical with Google Brain float conversion. + * Floats shall round to nearest even, and NANs shall be quiet. + * Subnormals aren't flushed to zero, except perhaps when used. + * This code should vectorize nicely if using modern compilers. + */ +static inline ggml_bf16_t ggml_compute_fp32_to_bf16(float s) { + ggml_bf16_t h; + union { + float f; + uint32_t i; + } u; + u.f = s; + if ((u.i & 0x7fffffff) > 0x7f800000) { /* nan */ + h.bits = (u.i >> 16) | 64; /* force to quiet */ + return h; + } + h.bits = (u.i + (0x7fff + ((u.i >> 16) & 1))) >> 16; + return h; +} + +#define GGML_FP32_TO_BF16(x) ggml_compute_fp32_to_bf16(x) +#define GGML_BF16_TO_FP32(x) ggml_compute_bf16_to_fp32(x) + +static inline int32_t ggml_node_get_use_count(const struct ggml_cgraph * cgraph, int node_idx) { + const struct ggml_tensor * node = cgraph->nodes[node_idx]; + + size_t hash_pos = ggml_hash_find(&cgraph->visited_hash_set, node); + if (!ggml_bitset_get(cgraph->visited_hash_set.used, hash_pos)) { + return 0; + } + return cgraph->use_counts[hash_pos]; +} + +// return true if the node's results are only used by N other nodes +// and can be fused into their calculations. +static inline bool ggml_node_has_n_uses(const struct ggml_cgraph * cgraph, int node_idx, int32_t n_uses) { + const struct ggml_tensor * node = cgraph->nodes[node_idx]; + + // check the use count against how many we're replacing + if (ggml_node_get_use_count(cgraph, node_idx) != n_uses) { + return false; + } + + // if node is a view, some other node might be using the intermediate result + // via the view source. + if (node->view_src) { + return false; + } + + // If the user requested output for the node, can't fuse + if (node->flags & GGML_TENSOR_FLAG_OUTPUT) { + return false; + } + + return true; +} + +// Returns true if nodes with indices { node_idxs } are the sequence of ggml_ops in ops[] +// and are fusable. Nodes are considered fusable according to this function if: +// - all nodes except the last have only one use and are not views/outputs (see ggml_node_has_N_uses). +// - all nodes except the last are a src of the following node. +// - all nodes are the same shape. +// TODO: Consider allowing GGML_OP_NONE nodes in between +static inline bool ggml_can_fuse_ext(const struct ggml_cgraph * cgraph, const int * node_idxs, const enum ggml_op * ops, int num_ops) { + for (int i = 0; i < num_ops; ++i) { + if (node_idxs[i] >= cgraph->n_nodes) { + return false; + } + + struct ggml_tensor * node = cgraph->nodes[node_idxs[i]]; + if (node->op != ops[i]) { + return false; + } + if ((node->flags & GGML_TENSOR_FLAG_COMPUTE) == 0) { + return false; + } + if (i < num_ops - 1 && !ggml_node_has_n_uses(cgraph, node_idxs[i], 1)) { + return false; + } + if (i > 0) { + struct ggml_tensor * prev = cgraph->nodes[node_idxs[i - 1]]; + if (node->src[0] != prev && node->src[1] != prev) { + return false; + } + if (!ggml_are_same_shape(node, prev)) { + return false; + } + } + } + return true; +} + +// same as above, for sequential indices starting at node_idx +static inline bool ggml_can_fuse(const struct ggml_cgraph * cgraph, int node_idx, const enum ggml_op * ops, int num_ops) { + assert(num_ops < 32); + + if (node_idx + num_ops > cgraph->n_nodes) { + return false; + } + + int idxs[32]; + for (int i = 0; i < num_ops; ++i) { + idxs[i] = node_idx + i; + } + + return ggml_can_fuse_ext(cgraph, idxs, ops, num_ops); +} + +GGML_API bool ggml_can_fuse_subgraph_ext(const struct ggml_cgraph * cgraph, + const int * node_idxs, + int count, + const enum ggml_op * ops, + const int * outputs, + int num_outputs); + +// Returns true if the subgraph formed by {node_idxs} can be fused +// checks whethers all nodes which are not part of outputs can be elided +// by checking if their num_uses are confined to the subgraph +static inline bool ggml_can_fuse_subgraph(const struct ggml_cgraph * cgraph, + int node_idx, + int count, + const enum ggml_op * ops, + const int * outputs, + int num_outputs) { + GGML_ASSERT(count < 32); + if (node_idx + count > cgraph->n_nodes) { + return false; + } + + int idxs[32]; + + for (int i = 0; i < count; ++i) { + idxs[i] = node_idx + i; + } + + return ggml_can_fuse_subgraph_ext(cgraph, idxs, count, ops, outputs, num_outputs); +} + +#ifdef __cplusplus +} +#endif + +#ifdef __cplusplus +#include +#include +#include + +// nicer C++ syntax for ggml_can_fuse +inline bool ggml_can_fuse(const struct ggml_cgraph * cgraph, int node_idx, std::initializer_list ops) { + return ggml_can_fuse(cgraph, node_idx, ops.begin(), (int)ops.size()); +} + +inline bool ggml_can_fuse_subgraph(const struct ggml_cgraph * cgraph, + int start_idx, + std::initializer_list ops, + std::initializer_list outputs = {}) { + return ggml_can_fuse_subgraph(cgraph, start_idx, ops.size(), ops.begin(), outputs.begin(), outputs.size()); +} + +// Return true if the edges in the graph match expectations. +inline bool ggml_check_edges(const struct ggml_cgraph * cgraph, + int start_idx, + std::initializer_list> edges) { + for (const auto & edge : edges) { + int dst_node = edge[0]; + int src_idx = edge[1]; + int src_node = edge[2]; + if (cgraph->nodes[start_idx + dst_node]->src[src_idx] != cgraph->nodes[start_idx + src_node]) { + return false; + } + } + return true; +} + +// expose GGUF internals for test code +GGML_API size_t gguf_type_size(enum gguf_type type); +GGML_API void gguf_write_to_buf(const struct gguf_context * ctx, std::vector & buf, bool only_meta); +#endif // __cplusplus diff --git a/python/freetoken/kernel/csrc/gguf_mmq/ggml.h b/python/freetoken/kernel/csrc/gguf_mmq/ggml.h new file mode 100644 index 00000000..5f6774a6 --- /dev/null +++ b/python/freetoken/kernel/csrc/gguf_mmq/ggml.h @@ -0,0 +1,2950 @@ +#pragma once + +// +// GGML Tensor Library +// +// This documentation is still a work in progress. +// If you wish some specific topics to be covered, feel free to drop a comment: +// +// https://github.com/ggml-org/whisper.cpp/issues/40 +// +// ## Overview +// +// This library implements: +// +// - a set of tensor operations +// - automatic differentiation +// - basic optimization algorithms +// +// The aim of this library is to provide a minimalistic approach for various machine learning tasks. This includes, +// but is not limited to, the following: +// +// - linear regression +// - support vector machines +// - neural networks +// +// The library allows the user to define a certain function using the available tensor operations. This function +// definition is represented internally via a computation graph. Each tensor operation in the function definition +// corresponds to a node in the graph. Having the computation graph defined, the user can choose to compute the +// function's value and/or its gradient with respect to the input variables. Optionally, the function can be optimized +// using one of the available optimization algorithms. +// +// For example, here we define the function: f(x) = a*x^2 + b +// +// { +// struct ggml_init_params params = { +// .mem_size = 16*1024*1024, +// .mem_buffer = NULL, +// }; +// +// // memory allocation happens here +// struct ggml_context * ctx = ggml_init(params); +// +// struct ggml_tensor * x = ggml_new_tensor_1d(ctx, GGML_TYPE_F32, 1); +// +// ggml_set_param(ctx, x); // x is an input variable +// +// struct ggml_tensor * a = ggml_new_tensor_1d(ctx, GGML_TYPE_F32, 1); +// struct ggml_tensor * b = ggml_new_tensor_1d(ctx, GGML_TYPE_F32, 1); +// struct ggml_tensor * x2 = ggml_mul(ctx, x, x); +// struct ggml_tensor * f = ggml_add(ctx, ggml_mul(ctx, a, x2), b); +// +// ... +// } +// +// Notice that the function definition above does not involve any actual computation. The computation is performed only +// when the user explicitly requests it. For example, to compute the function's value at x = 2.0: +// +// { +// ... +// +// struct ggml_cgraph * gf = ggml_new_graph(ctx); +// ggml_build_forward_expand(gf, f); +// +// // set the input variable and parameter values +// ggml_set_f32(x, 2.0f); +// ggml_set_f32(a, 3.0f); +// ggml_set_f32(b, 4.0f); +// +// ggml_graph_compute_with_ctx(ctx, &gf, n_threads); +// +// printf("f = %f\n", ggml_get_f32_1d(f, 0)); +// +// ... +// } +// +// The actual computation is performed in the ggml_graph_compute() function. +// +// The ggml_new_tensor_...() functions create new tensors. They are allocated in the memory buffer provided to the +// ggml_init() function. You have to be careful not to exceed the memory buffer size. Therefore, you have to know +// in advance how much memory you need for your computation. Alternatively, you can allocate a large enough memory +// and after defining the computation graph, call the ggml_used_mem() function to find out how much memory was +// actually needed. +// +// The ggml_set_param() function marks a tensor as an input variable. This is used by the automatic +// differentiation and optimization algorithms. +// +// The described approach allows to define the function graph once and then compute its forward or backward graphs +// multiple times. All computations will use the same memory buffer allocated in the ggml_init() function. This way +// the user can avoid the memory allocation overhead at runtime. +// +// The library supports multi-dimensional tensors - up to 4 dimensions. The FP16 and FP32 data types are first class +// citizens, but in theory the library can be extended to support FP8 and integer data types. +// +// Each tensor operation produces a new tensor. Initially the library was envisioned to support only the use of unary +// and binary operations. Most of the available operations fall into one of these two categories. With time, it became +// clear that the library needs to support more complex operations. The way to support these operations is not clear +// yet, but a few examples are demonstrated in the following operations: +// +// - ggml_permute() +// - ggml_conv_1d_1s() +// - ggml_conv_1d_2s() +// +// For each tensor operator, the library implements a forward and backward computation function. The forward function +// computes the output tensor value given the input tensor values. The backward function computes the adjoint of the +// input tensors given the adjoint of the output tensor. For a detailed explanation of what this means, take a +// calculus class, or watch the following video: +// +// What is Automatic Differentiation? +// https://www.youtube.com/watch?v=wG_nF1awSSY +// +// +// ## Tensor data (struct ggml_tensor) +// +// The tensors are stored in memory via the ggml_tensor struct. The structure provides information about the size of +// the tensor, the data type, and the memory buffer where the tensor data is stored. Additionally, it contains +// pointers to the "source" tensors - i.e. the tensors that were used to compute the current tensor. For example: +// +// { +// struct ggml_tensor * c = ggml_add(ctx, a, b); +// +// assert(c->src[0] == a); +// assert(c->src[1] == b); +// } +// +// The multi-dimensional tensors are stored in row-major order. The ggml_tensor struct contains fields for the +// number of elements in each dimension ("ne") as well as the number of bytes ("nb", a.k.a. stride). This allows +// to store tensors that are not contiguous in memory, which is useful for operations such as transposition and +// permutation. All tensor operations have to take the stride into account and not assume that the tensor is +// contiguous in memory. +// +// The data of the tensor is accessed via the "data" pointer. For example: +// +// { +// const int nx = 2; +// const int ny = 3; +// +// struct ggml_tensor * a = ggml_new_tensor_2d(ctx, GGML_TYPE_F32, nx, ny); +// +// for (int y = 0; y < ny; y++) { +// for (int x = 0; x < nx; x++) { +// *(float *) ((char *) a->data + y*a->nb[1] + x*a->nb[0]) = x + y; +// } +// } +// +// ... +// } +// +// Alternatively, there are helper functions, such as ggml_get_f32_1d() and ggml_set_f32_1d() that can be used. +// +// ## The matrix multiplication operator (ggml_mul_mat) +// +// TODO +// +// +// ## Multi-threading +// +// TODO +// +// +// ## Overview of ggml.c +// +// TODO +// +// +// ## SIMD optimizations +// +// TODO +// +// +// ## Debugging ggml +// +// TODO +// +// + +#ifdef GGML_SHARED +# if defined(_WIN32) && !defined(__MINGW32__) +# ifdef GGML_BUILD +# define GGML_API __declspec(dllexport) extern +# else +# define GGML_API __declspec(dllimport) extern +# endif +# else +# define GGML_API __attribute__ ((visibility ("default"))) extern +# endif +#else +# define GGML_API extern +#endif + +// TODO: support for clang +#ifdef __GNUC__ +# define GGML_DEPRECATED(func, hint) func __attribute__((deprecated(hint))) +#elif defined(_MSC_VER) +# define GGML_DEPRECATED(func, hint) __declspec(deprecated(hint)) func +#else +# define GGML_DEPRECATED(func, hint) func +#endif + +#ifndef __GNUC__ +# define GGML_ATTRIBUTE_FORMAT(...) +#elif defined(__MINGW32__) && !defined(__clang__) +# define GGML_ATTRIBUTE_FORMAT(...) __attribute__((format(gnu_printf, __VA_ARGS__))) +#else +# define GGML_ATTRIBUTE_FORMAT(...) __attribute__((format(printf, __VA_ARGS__))) +#endif + +#if defined(_WIN32) && !defined(_WIN32_WINNT) +# define _WIN32_WINNT 0x0A00 +#endif + +#include +#include +#include +#include + +#define GGML_FILE_MAGIC 0x67676d6c // "ggml" +#define GGML_FILE_VERSION 2 + +#define GGML_QNT_VERSION 2 // bump this on quantization format changes +#define GGML_QNT_VERSION_FACTOR 1000 // do not change this + +#define GGML_MAX_DIMS 4 +#define GGML_MAX_PARAMS 2048 +#define GGML_MAX_SRC 10 +#define GGML_MAX_N_THREADS 512 +#define GGML_MAX_OP_PARAMS 64 + +#ifndef GGML_MAX_NAME +# define GGML_MAX_NAME 64 +#endif + +#define GGML_DEFAULT_N_THREADS 4 +#define GGML_DEFAULT_GRAPH_SIZE 2048 + +#if UINTPTR_MAX == 0xFFFFFFFF + #define GGML_MEM_ALIGN 4 +#elif defined(__EMSCRIPTEN__) +// emscripten uses max_align_t == 8, so we need GGML_MEM_ALIGN == 8 for 64-bit wasm. +// (for 32-bit wasm, the first conditional is true and GGML_MEM_ALIGN stays 4.) +// ref: https://github.com/ggml-org/llama.cpp/pull/18628 + #define GGML_MEM_ALIGN 8 +#else + #define GGML_MEM_ALIGN 16 +#endif + +#define GGML_EXIT_SUCCESS 0 +#define GGML_EXIT_ABORTED 1 + +// TODO: convert to enum https://github.com/ggml-org/llama.cpp/pull/16187#discussion_r2388538726 +#define GGML_ROPE_TYPE_NORMAL 0 +#define GGML_ROPE_TYPE_NEOX 2 +#define GGML_ROPE_TYPE_MROPE 8 +#define GGML_ROPE_TYPE_VISION 24 +#define GGML_ROPE_TYPE_IMROPE 40 // binary: 101000 + +#define GGML_MROPE_SECTIONS 4 + +#define GGML_UNUSED(x) (void)(x) +#ifdef __CUDACC__ +template +__host__ __device__ constexpr inline void ggml_unused_vars_impl(Args&&...) noexcept {} +#define GGML_UNUSED_VARS(...) ggml_unused_vars_impl(__VA_ARGS__) +#else +#define GGML_UNUSED_VARS(...) do { (void)sizeof((__VA_ARGS__, 0)); } while(0) +#endif // __CUDACC__ + +#define GGML_PAD(x, n) (((x) + (n) - 1) & ~((n) - 1)) + +#ifndef NDEBUG +# define GGML_UNREACHABLE() do { fprintf(stderr, "statement should be unreachable\n"); abort(); } while(0) +#elif defined(__GNUC__) +# define GGML_UNREACHABLE() __builtin_unreachable() +#elif defined(_MSC_VER) +# define GGML_UNREACHABLE() __assume(0) +#else +# define GGML_UNREACHABLE() ((void) 0) +#endif + +#ifdef __cplusplus +# define GGML_NORETURN [[noreturn]] +#elif defined(_MSC_VER) +# define GGML_NORETURN __declspec(noreturn) +#else +# define GGML_NORETURN _Noreturn +#endif + +#define GGML_ABORT(...) ggml_abort(__FILE__, __LINE__, __VA_ARGS__) +#define GGML_ASSERT(x) if (!(x)) GGML_ABORT("GGML_ASSERT(%s) failed", #x) + +// used to copy the number of elements and stride in bytes of tensors into local variables. +// main purpose is to reduce code duplication and improve readability. +// +// example: +// +// GGML_TENSOR_LOCALS(int64_t, ne1, src1, ne); +// GGML_TENSOR_LOCALS(size_t, nb1, src1, nb); +// +#define GGML_TENSOR_LOCALS_1(type, prefix, pointer, array) \ + const type prefix##0 = (pointer) ? (pointer)->array[0] : 0; \ + GGML_UNUSED(prefix##0); +#define GGML_TENSOR_LOCALS_2(type, prefix, pointer, array) \ + GGML_TENSOR_LOCALS_1 (type, prefix, pointer, array) \ + const type prefix##1 = (pointer) ? (pointer)->array[1] : 0; \ + GGML_UNUSED(prefix##1); +#define GGML_TENSOR_LOCALS_3(type, prefix, pointer, array) \ + GGML_TENSOR_LOCALS_2 (type, prefix, pointer, array) \ + const type prefix##2 = (pointer) ? (pointer)->array[2] : 0; \ + GGML_UNUSED(prefix##2); +#define GGML_TENSOR_LOCALS(type, prefix, pointer, array) \ + GGML_TENSOR_LOCALS_3 (type, prefix, pointer, array) \ + const type prefix##3 = (pointer) ? (pointer)->array[3] : 0; \ + GGML_UNUSED(prefix##3); + +#define GGML_TENSOR_UNARY_OP_LOCALS \ + GGML_TENSOR_LOCALS(int64_t, ne0, src0, ne) \ + GGML_TENSOR_LOCALS(size_t, nb0, src0, nb) \ + GGML_TENSOR_LOCALS(int64_t, ne, dst, ne) \ + GGML_TENSOR_LOCALS(size_t, nb, dst, nb) + +#define GGML_TENSOR_BINARY_OP_LOCALS \ + GGML_TENSOR_LOCALS(int64_t, ne0, src0, ne) \ + GGML_TENSOR_LOCALS(size_t, nb0, src0, nb) \ + GGML_TENSOR_LOCALS(int64_t, ne1, src1, ne) \ + GGML_TENSOR_LOCALS(size_t, nb1, src1, nb) \ + GGML_TENSOR_LOCALS(int64_t, ne, dst, ne) \ + GGML_TENSOR_LOCALS(size_t, nb, dst, nb) + +#define GGML_TENSOR_TERNARY_OP_LOCALS \ + GGML_TENSOR_LOCALS(int64_t, ne0, src0, ne) \ + GGML_TENSOR_LOCALS(size_t, nb0, src0, nb) \ + GGML_TENSOR_LOCALS(int64_t, ne1, src1, ne) \ + GGML_TENSOR_LOCALS(size_t, nb1, src1, nb) \ + GGML_TENSOR_LOCALS(int64_t, ne2, src2, ne) \ + GGML_TENSOR_LOCALS(size_t, nb2, src2, nb) \ + GGML_TENSOR_LOCALS(int64_t, ne, dst, ne) \ + GGML_TENSOR_LOCALS(size_t, nb, dst, nb) + +#define GGML_TENSOR_BINARY_OP_LOCALS01 \ + GGML_TENSOR_LOCALS(int64_t, ne0, src0, ne) \ + GGML_TENSOR_LOCALS(size_t, nb0, src0, nb) \ + GGML_TENSOR_LOCALS(int64_t, ne1, src1, ne) \ + GGML_TENSOR_LOCALS(size_t, nb1, src1, nb) + +#ifdef __cplusplus +extern "C" { +#endif + + // Function type used in fatal error callbacks + typedef void (*ggml_abort_callback_t)(const char * error_message); + + // Set the abort callback (passing null will restore original abort functionality: printing a message to stdout) + // Returns the old callback for chaining + GGML_API ggml_abort_callback_t ggml_set_abort_callback(ggml_abort_callback_t callback); + + GGML_NORETURN GGML_ATTRIBUTE_FORMAT(3, 4) + GGML_API void ggml_abort(const char * file, int line, const char * fmt, ...); + + enum ggml_status { + GGML_STATUS_ALLOC_FAILED = -2, + GGML_STATUS_FAILED = -1, + GGML_STATUS_SUCCESS = 0, + GGML_STATUS_ABORTED = 1, + }; + + // get ggml_status name string + GGML_API const char * ggml_status_to_string(enum ggml_status status); + + // ieee 754-2008 half-precision float16 + // todo: make this not an integral type + typedef uint16_t ggml_fp16_t; + GGML_API float ggml_fp16_to_fp32(ggml_fp16_t); + GGML_API ggml_fp16_t ggml_fp32_to_fp16(float); + GGML_API void ggml_fp16_to_fp32_row(const ggml_fp16_t *, float *, int64_t); + GGML_API void ggml_fp32_to_fp16_row(const float *, ggml_fp16_t *, int64_t); + + // google brain half-precision bfloat16 + typedef struct { uint16_t bits; } ggml_bf16_t; + GGML_API ggml_bf16_t ggml_fp32_to_bf16(float); + GGML_API float ggml_bf16_to_fp32(ggml_bf16_t); // consider just doing << 16 + GGML_API void ggml_bf16_to_fp32_row(const ggml_bf16_t *, float *, int64_t); + GGML_API void ggml_fp32_to_bf16_row_ref(const float *, ggml_bf16_t *, int64_t); + GGML_API void ggml_fp32_to_bf16_row(const float *, ggml_bf16_t *, int64_t); + + struct ggml_object; + struct ggml_context; + struct ggml_cgraph; + + // NOTE: always add types at the end of the enum to keep backward compatibility + enum ggml_type { + GGML_TYPE_F32 = 0, + GGML_TYPE_F16 = 1, + GGML_TYPE_Q4_0 = 2, + GGML_TYPE_Q4_1 = 3, + // GGML_TYPE_Q4_2 = 4, support has been removed + // GGML_TYPE_Q4_3 = 5, support has been removed + GGML_TYPE_Q5_0 = 6, + GGML_TYPE_Q5_1 = 7, + GGML_TYPE_Q8_0 = 8, + GGML_TYPE_Q8_1 = 9, + GGML_TYPE_Q2_K = 10, + GGML_TYPE_Q3_K = 11, + GGML_TYPE_Q4_K = 12, + GGML_TYPE_Q5_K = 13, + GGML_TYPE_Q6_K = 14, + GGML_TYPE_Q8_K = 15, + GGML_TYPE_IQ2_XXS = 16, + GGML_TYPE_IQ2_XS = 17, + GGML_TYPE_IQ3_XXS = 18, + GGML_TYPE_IQ1_S = 19, + GGML_TYPE_IQ4_NL = 20, + GGML_TYPE_IQ3_S = 21, + GGML_TYPE_IQ2_S = 22, + GGML_TYPE_IQ4_XS = 23, + GGML_TYPE_I8 = 24, + GGML_TYPE_I16 = 25, + GGML_TYPE_I32 = 26, + GGML_TYPE_I64 = 27, + GGML_TYPE_F64 = 28, + GGML_TYPE_IQ1_M = 29, + GGML_TYPE_BF16 = 30, + // GGML_TYPE_Q4_0_4_4 = 31, support has been removed from gguf files + // GGML_TYPE_Q4_0_4_8 = 32, + // GGML_TYPE_Q4_0_8_8 = 33, + GGML_TYPE_TQ1_0 = 34, + GGML_TYPE_TQ2_0 = 35, + // GGML_TYPE_IQ4_NL_4_4 = 36, + // GGML_TYPE_IQ4_NL_4_8 = 37, + // GGML_TYPE_IQ4_NL_8_8 = 38, + GGML_TYPE_MXFP4 = 39, // MXFP4 (1 block) + GGML_TYPE_NVFP4 = 40, // NVFP4 (4 blocks, E4M3 scale) + GGML_TYPE_Q1_0 = 41, + GGML_TYPE_Q2_0 = 42, + GGML_TYPE_COUNT = 43, + }; + + // precision + enum ggml_prec { + GGML_PREC_DEFAULT = 0, // stored as ggml_tensor.op_params, 0 by default + GGML_PREC_F32 = 10, + }; + + // op hint + enum ggml_op_hint { + GGML_HINT_NONE = 0, + GGML_HINT_SRC0_IS_HADAMARD = 1, + }; + + // model file types + enum ggml_ftype { + GGML_FTYPE_UNKNOWN = -1, + GGML_FTYPE_ALL_F32 = 0, + GGML_FTYPE_MOSTLY_F16 = 1, // except 1d tensors + GGML_FTYPE_MOSTLY_Q4_0 = 2, // except 1d tensors + GGML_FTYPE_MOSTLY_Q4_1 = 3, // except 1d tensors + GGML_FTYPE_MOSTLY_Q4_1_SOME_F16 = 4, // tok_embeddings.weight and output.weight are F16 + GGML_FTYPE_MOSTLY_Q8_0 = 7, // except 1d tensors + GGML_FTYPE_MOSTLY_Q5_0 = 8, // except 1d tensors + GGML_FTYPE_MOSTLY_Q5_1 = 9, // except 1d tensors + GGML_FTYPE_MOSTLY_Q2_K = 10, // except 1d tensors + GGML_FTYPE_MOSTLY_Q3_K = 11, // except 1d tensors + GGML_FTYPE_MOSTLY_Q4_K = 12, // except 1d tensors + GGML_FTYPE_MOSTLY_Q5_K = 13, // except 1d tensors + GGML_FTYPE_MOSTLY_Q6_K = 14, // except 1d tensors + GGML_FTYPE_MOSTLY_IQ2_XXS = 15, // except 1d tensors + GGML_FTYPE_MOSTLY_IQ2_XS = 16, // except 1d tensors + GGML_FTYPE_MOSTLY_IQ3_XXS = 17, // except 1d tensors + GGML_FTYPE_MOSTLY_IQ1_S = 18, // except 1d tensors + GGML_FTYPE_MOSTLY_IQ4_NL = 19, // except 1d tensors + GGML_FTYPE_MOSTLY_IQ3_S = 20, // except 1d tensors + GGML_FTYPE_MOSTLY_IQ2_S = 21, // except 1d tensors + GGML_FTYPE_MOSTLY_IQ4_XS = 22, // except 1d tensors + GGML_FTYPE_MOSTLY_IQ1_M = 23, // except 1d tensors + GGML_FTYPE_MOSTLY_BF16 = 24, // except 1d tensors + GGML_FTYPE_MOSTLY_MXFP4 = 25, // except 1d tensors + GGML_FTYPE_MOSTLY_NVFP4 = 26, // except 1d tensors + GGML_FTYPE_MOSTLY_Q1_0 = 27, // except 1d tensors + GGML_FTYPE_MOSTLY_Q2_0 = 28, // except 1d tensors + }; + + // available tensor operations: + enum ggml_op { + GGML_OP_NONE = 0, + + GGML_OP_DUP, + GGML_OP_ADD, + GGML_OP_ADD_ID, + GGML_OP_ADD1, + GGML_OP_ACC, + GGML_OP_SUB, + GGML_OP_MUL, + GGML_OP_DIV, + GGML_OP_SQR, + GGML_OP_SQRT, + GGML_OP_LOG, + GGML_OP_SIN, + GGML_OP_COS, + GGML_OP_SUM, + GGML_OP_SUM_ROWS, + GGML_OP_CUMSUM, + GGML_OP_MEAN, + GGML_OP_ARGMAX, + GGML_OP_COUNT_EQUAL, + GGML_OP_REPEAT, + GGML_OP_REPEAT_BACK, + GGML_OP_CONCAT, + GGML_OP_SILU_BACK, + GGML_OP_NORM, // normalize + GGML_OP_RMS_NORM, + GGML_OP_RMS_NORM_BACK, + GGML_OP_GROUP_NORM, + GGML_OP_L2_NORM, + + GGML_OP_MUL_MAT, + GGML_OP_MUL_MAT_ID, + GGML_OP_OUT_PROD, + + GGML_OP_SCALE, + GGML_OP_SET, + GGML_OP_CPY, + GGML_OP_CONT, + GGML_OP_RESHAPE, + GGML_OP_VIEW, + GGML_OP_PERMUTE, + GGML_OP_TRANSPOSE, + GGML_OP_GET_ROWS, + GGML_OP_GET_ROWS_BACK, + GGML_OP_SET_ROWS, + GGML_OP_DIAG, + GGML_OP_DIAG_MASK_INF, + GGML_OP_DIAG_MASK_ZERO, + GGML_OP_SOFT_MAX, + GGML_OP_SOFT_MAX_BACK, + GGML_OP_ROPE, + GGML_OP_ROPE_BACK, + GGML_OP_CLAMP, + GGML_OP_CONV_TRANSPOSE_1D, + GGML_OP_IM2COL, + GGML_OP_IM2COL_BACK, + GGML_OP_IM2COL_3D, + GGML_OP_COL2IM_1D, + GGML_OP_CONV_2D, + GGML_OP_CONV_3D, + GGML_OP_CONV_2D_DW, + GGML_OP_CONV_TRANSPOSE_2D, + GGML_OP_POOL_1D, + GGML_OP_POOL_2D, + GGML_OP_POOL_2D_BACK, + GGML_OP_UPSCALE, + GGML_OP_PAD, + GGML_OP_PAD_REFLECT_1D, + GGML_OP_ROLL, + GGML_OP_ARANGE, + GGML_OP_TIMESTEP_EMBEDDING, + GGML_OP_ARGSORT, + GGML_OP_TOP_K, + GGML_OP_LEAKY_RELU, + GGML_OP_TRI, + GGML_OP_FILL, + + GGML_OP_FLASH_ATTN_EXT, + GGML_OP_FLASH_ATTN_BACK, + GGML_OP_SSM_CONV, + GGML_OP_SSM_SCAN, + GGML_OP_WIN_PART, + GGML_OP_WIN_UNPART, + GGML_OP_GET_REL_POS, + GGML_OP_ADD_REL_POS, + GGML_OP_RWKV_WKV6, + GGML_OP_GATED_LINEAR_ATTN, + GGML_OP_RWKV_WKV7, + GGML_OP_SOLVE_TRI, + GGML_OP_GATED_DELTA_NET, + GGML_OP_LIGHTNING_INDEXER, + GGML_OP_DSV4_HC_COMB, + GGML_OP_DSV4_HC_PRE, + GGML_OP_DSV4_HC_POST, + + GGML_OP_UNARY, + + GGML_OP_MAP_CUSTOM1, + GGML_OP_MAP_CUSTOM2, + GGML_OP_MAP_CUSTOM3, + + GGML_OP_CUSTOM, + + GGML_OP_CROSS_ENTROPY_LOSS, + GGML_OP_CROSS_ENTROPY_LOSS_BACK, + GGML_OP_OPT_STEP_ADAMW, + GGML_OP_OPT_STEP_SGD, + + GGML_OP_GLU, + + GGML_OP_COUNT, + }; + + enum ggml_unary_op { + GGML_UNARY_OP_ABS, + GGML_UNARY_OP_SGN, + GGML_UNARY_OP_NEG, + GGML_UNARY_OP_STEP, + GGML_UNARY_OP_TANH, + GGML_UNARY_OP_ELU, + GGML_UNARY_OP_RELU, + GGML_UNARY_OP_SIGMOID, + GGML_UNARY_OP_GELU, + GGML_UNARY_OP_GELU_QUICK, + GGML_UNARY_OP_SILU, + GGML_UNARY_OP_HARDSWISH, + GGML_UNARY_OP_HARDSIGMOID, + GGML_UNARY_OP_EXP, + GGML_UNARY_OP_EXPM1, + GGML_UNARY_OP_SOFTPLUS, + GGML_UNARY_OP_GELU_ERF, + GGML_UNARY_OP_XIELU, + GGML_UNARY_OP_FLOOR, + GGML_UNARY_OP_CEIL, + GGML_UNARY_OP_ROUND, + GGML_UNARY_OP_TRUNC, + + GGML_UNARY_OP_COUNT, + }; + + enum ggml_glu_op { + GGML_GLU_OP_REGLU, + GGML_GLU_OP_GEGLU, + GGML_GLU_OP_SWIGLU, + GGML_GLU_OP_SWIGLU_OAI, + GGML_GLU_OP_GEGLU_ERF, + GGML_GLU_OP_GEGLU_QUICK, + + GGML_GLU_OP_COUNT, + }; + + enum ggml_object_type { + GGML_OBJECT_TYPE_TENSOR, + GGML_OBJECT_TYPE_GRAPH, + GGML_OBJECT_TYPE_WORK_BUFFER + }; + + enum ggml_log_level { + GGML_LOG_LEVEL_NONE = 0, + GGML_LOG_LEVEL_DEBUG = 1, + GGML_LOG_LEVEL_INFO = 2, + GGML_LOG_LEVEL_WARN = 3, + GGML_LOG_LEVEL_ERROR = 4, + GGML_LOG_LEVEL_CONT = 5, // continue previous log + }; + + // this tensor... + enum ggml_tensor_flag { + GGML_TENSOR_FLAG_INPUT = 1, // ...is an input for the GGML compute graph + GGML_TENSOR_FLAG_OUTPUT = 2, // ...is an output for the GGML compute graph + GGML_TENSOR_FLAG_PARAM = 4, // ...contains trainable parameters + GGML_TENSOR_FLAG_LOSS = 8, // ...defines loss for numerical optimization (multiple loss tensors add up) + GGML_TENSOR_FLAG_COMPUTE = 16, // ...must be computed + }; + + enum ggml_tri_type { + GGML_TRI_TYPE_UPPER_DIAG = 0, + GGML_TRI_TYPE_UPPER = 1, + GGML_TRI_TYPE_LOWER_DIAG = 2, + GGML_TRI_TYPE_LOWER = 3 + }; + + struct ggml_init_params { + // memory pool + size_t mem_size; // bytes + void * mem_buffer; // if NULL, memory will be allocated internally + bool no_alloc; // don't allocate memory for the tensor data + }; + + // n-dimensional tensor + struct ggml_tensor { + enum ggml_type type; + + struct ggml_backend_buffer * buffer; + + int64_t ne[GGML_MAX_DIMS]; // number of elements + size_t nb[GGML_MAX_DIMS]; // stride in bytes: + // nb[0] = ggml_type_size(type) + // nb[1] = nb[0] * (ne[0] / ggml_blck_size(type)) + padding + // nb[i] = nb[i-1] * ne[i-1] + + // compute data + enum ggml_op op; + + // op params - allocated as int32_t for alignment + int32_t op_params[GGML_MAX_OP_PARAMS / sizeof(int32_t)]; + + int32_t flags; + + struct ggml_tensor * src[GGML_MAX_SRC]; + + // source tensor and offset for views + struct ggml_tensor * view_src; + size_t view_offs; + + void * data; + + char name[GGML_MAX_NAME]; + + void * extra; // extra things e.g. for ggml-cuda.cu + + char padding[8]; + }; + + static const size_t GGML_TENSOR_SIZE = sizeof(struct ggml_tensor); + + // Abort callback + // If not NULL, called before ggml computation + // If it returns true, the computation is aborted + typedef bool (*ggml_abort_callback)(void * data); + + + // + // GUID + // + + // GUID types + typedef uint8_t ggml_guid[16]; + typedef ggml_guid * ggml_guid_t; + + GGML_API bool ggml_guid_matches(ggml_guid_t guid_a, ggml_guid_t guid_b); + + // misc + + GGML_API const char * ggml_version(void); + GGML_API const char * ggml_commit(void); + + GGML_API void ggml_time_init(void); // call this once at the beginning of the program + GGML_API int64_t ggml_time_ms(void); + GGML_API int64_t ggml_time_us(void); + GGML_API int64_t ggml_cycles(void); + GGML_API int64_t ggml_cycles_per_ms(void); + + // accepts a UTF-8 path, even on Windows + GGML_API FILE * ggml_fopen(const char * fname, const char * mode); + + GGML_API void ggml_print_object (const struct ggml_object * obj); + GGML_API void ggml_print_objects(const struct ggml_context * ctx); + + GGML_API int64_t ggml_nelements (const struct ggml_tensor * tensor); + GGML_API int64_t ggml_nrows (const struct ggml_tensor * tensor); + GGML_API size_t ggml_nbytes (const struct ggml_tensor * tensor); + GGML_API size_t ggml_nbytes_pad(const struct ggml_tensor * tensor); // same as ggml_nbytes() but padded to GGML_MEM_ALIGN + + GGML_API int64_t ggml_blck_size(enum ggml_type type); + GGML_API size_t ggml_type_size(enum ggml_type type); // size in bytes for all elements in a block + GGML_API size_t ggml_row_size (enum ggml_type type, int64_t ne); // size in bytes for all elements in a row + + GGML_DEPRECATED( + GGML_API double ggml_type_sizef(enum ggml_type type), // ggml_type_size()/ggml_blck_size() as float + "use ggml_row_size() instead"); + + GGML_API const char * ggml_type_name(enum ggml_type type); + GGML_API const char * ggml_op_name (enum ggml_op op); + GGML_API const char * ggml_op_symbol(enum ggml_op op); + + GGML_API const char * ggml_unary_op_name(enum ggml_unary_op op); + GGML_API const char * ggml_glu_op_name(enum ggml_glu_op op); + GGML_API const char * ggml_op_desc(const struct ggml_tensor * t); // unary or op name + + GGML_API size_t ggml_element_size(const struct ggml_tensor * tensor); + + GGML_API bool ggml_is_quantized(enum ggml_type type); + + // TODO: temporary until model loading of ggml examples is refactored + GGML_API enum ggml_type ggml_ftype_to_ggml_type(enum ggml_ftype ftype); + + GGML_API bool ggml_is_transposed(const struct ggml_tensor * tensor); + GGML_API bool ggml_is_permuted (const struct ggml_tensor * tensor); + GGML_API bool ggml_is_empty (const struct ggml_tensor * tensor); + GGML_API bool ggml_is_view (const struct ggml_tensor * tensor); + GGML_API bool ggml_is_scalar (const struct ggml_tensor * tensor); + GGML_API bool ggml_is_vector (const struct ggml_tensor * tensor); + GGML_API bool ggml_is_matrix (const struct ggml_tensor * tensor); + GGML_API bool ggml_is_3d (const struct ggml_tensor * tensor); + GGML_API int ggml_n_dims (const struct ggml_tensor * tensor); // returns 1 for scalars + + // returns whether the tensor elements can be iterated over with a flattened index (no gaps, no permutation) + GGML_API bool ggml_is_contiguous (const struct ggml_tensor * tensor); + GGML_API bool ggml_is_contiguous_0(const struct ggml_tensor * tensor); // same as ggml_is_contiguous() + GGML_API bool ggml_is_contiguous_1(const struct ggml_tensor * tensor); // contiguous for dims >= 1 + GGML_API bool ggml_is_contiguous_2(const struct ggml_tensor * tensor); // contiguous for dims >= 2 + + GGML_API bool ggml_is_contiguous_to_1(const struct ggml_tensor * tensor); // contiguous for dims < 1 + GGML_API bool ggml_is_contiguous_to_2(const struct ggml_tensor * tensor); // contiguous for dims < 2 + GGML_API bool ggml_is_contiguous_to_3(const struct ggml_tensor * tensor); // contiguous for dims < 3 + + // returns whether the tensor elements are allocated as one contiguous block of memory (no gaps, but permutation ok) + GGML_API bool ggml_is_contiguously_allocated(const struct ggml_tensor * tensor); + + // true for tensor that is stored in memory as CxWxHxN and has been permuted to WxHxCxN + GGML_API bool ggml_is_contiguous_channels(const struct ggml_tensor * tensor); + + // true if the elements in dimension 0 are contiguous, or there is just 1 block of elements + GGML_API bool ggml_is_contiguous_rows(const struct ggml_tensor * tensor); + + GGML_API bool ggml_are_same_shape (const struct ggml_tensor * t0, const struct ggml_tensor * t1); + GGML_API bool ggml_are_same_stride(const struct ggml_tensor * t0, const struct ggml_tensor * t1); + + GGML_API bool ggml_can_repeat(const struct ggml_tensor * t0, const struct ggml_tensor * t1); + + // use this to compute the memory overhead of a tensor + GGML_API size_t ggml_tensor_overhead(void); + + GGML_API bool ggml_validate_row_data(enum ggml_type type, const void * data, size_t nbytes); + + // main + + GGML_API struct ggml_context * ggml_init (struct ggml_init_params params); + GGML_API void ggml_reset(struct ggml_context * ctx); + GGML_API void ggml_free (struct ggml_context * ctx); + + GGML_API size_t ggml_used_mem(const struct ggml_context * ctx); + + GGML_API bool ggml_get_no_alloc(struct ggml_context * ctx); + GGML_API void ggml_set_no_alloc(struct ggml_context * ctx, bool no_alloc); + + GGML_API void * ggml_get_mem_buffer (const struct ggml_context * ctx); + GGML_API size_t ggml_get_mem_size (const struct ggml_context * ctx); + GGML_API size_t ggml_get_max_tensor_size(const struct ggml_context * ctx); + + GGML_API struct ggml_tensor * ggml_new_tensor( + struct ggml_context * ctx, + enum ggml_type type, + int n_dims, + const int64_t *ne); + + GGML_API struct ggml_tensor * ggml_new_tensor_1d( + struct ggml_context * ctx, + enum ggml_type type, + int64_t ne0); + + GGML_API struct ggml_tensor * ggml_new_tensor_2d( + struct ggml_context * ctx, + enum ggml_type type, + int64_t ne0, + int64_t ne1); + + GGML_API struct ggml_tensor * ggml_new_tensor_3d( + struct ggml_context * ctx, + enum ggml_type type, + int64_t ne0, + int64_t ne1, + int64_t ne2); + + GGML_API struct ggml_tensor * ggml_new_tensor_4d( + struct ggml_context * ctx, + enum ggml_type type, + int64_t ne0, + int64_t ne1, + int64_t ne2, + int64_t ne3); + + GGML_API void * ggml_new_buffer(struct ggml_context * ctx, size_t nbytes); + + GGML_API struct ggml_tensor * ggml_dup_tensor (struct ggml_context * ctx, const struct ggml_tensor * src); + GGML_API struct ggml_tensor * ggml_view_tensor(struct ggml_context * ctx, struct ggml_tensor * src); + + // Context tensor enumeration and lookup + GGML_API struct ggml_tensor * ggml_get_first_tensor(const struct ggml_context * ctx); + GGML_API struct ggml_tensor * ggml_get_next_tensor (const struct ggml_context * ctx, struct ggml_tensor * tensor); + GGML_API struct ggml_tensor * ggml_get_tensor(struct ggml_context * ctx, const char * name); + + // Converts a flat index into coordinates + GGML_API void ggml_unravel_index(const struct ggml_tensor * tensor, int64_t i, int64_t * i0, int64_t * i1, int64_t * i2, int64_t * i3); + + GGML_API enum ggml_unary_op ggml_get_unary_op(const struct ggml_tensor * tensor); + GGML_API enum ggml_glu_op ggml_get_glu_op(const struct ggml_tensor * tensor); + + GGML_API void * ggml_get_data (const struct ggml_tensor * tensor); + GGML_API float * ggml_get_data_f32(const struct ggml_tensor * tensor); + + GGML_API const char * ggml_get_name (const struct ggml_tensor * tensor); + GGML_API struct ggml_tensor * ggml_set_name ( struct ggml_tensor * tensor, const char * name); + GGML_ATTRIBUTE_FORMAT(2, 3) + GGML_API struct ggml_tensor * ggml_format_name( struct ggml_tensor * tensor, const char * fmt, ...); + + // Tensor flags + GGML_API void ggml_set_input(struct ggml_tensor * tensor); + GGML_API void ggml_set_output(struct ggml_tensor * tensor); + GGML_API void ggml_set_param(struct ggml_tensor * tensor); + GGML_API void ggml_set_loss(struct ggml_tensor * tensor); + + // + // operations on tensors with backpropagation + // + + GGML_API struct ggml_tensor * ggml_dup( + struct ggml_context * ctx, + struct ggml_tensor * a); + + // in-place, returns view(a) + GGML_API struct ggml_tensor * ggml_dup_inplace( + struct ggml_context * ctx, + struct ggml_tensor * a); + + GGML_API struct ggml_tensor * ggml_add( + struct ggml_context * ctx, + struct ggml_tensor * a, + struct ggml_tensor * b); + + GGML_API struct ggml_tensor * ggml_add_inplace( + struct ggml_context * ctx, + struct ggml_tensor * a, + struct ggml_tensor * b); + + GGML_API struct ggml_tensor * ggml_add_cast( + struct ggml_context * ctx, + struct ggml_tensor * a, + struct ggml_tensor * b, + enum ggml_type type); + + // dst[i0, i1, i2] = a[i0, i1, i2] + b[i0, ids[i1, i2]] + GGML_API struct ggml_tensor * ggml_add_id( + struct ggml_context * ctx, + struct ggml_tensor * a, + struct ggml_tensor * b, + struct ggml_tensor * ids); + + GGML_DEPRECATED(GGML_API struct ggml_tensor * ggml_add1( + struct ggml_context * ctx, + struct ggml_tensor * a, + struct ggml_tensor * b), + "use ggml_add instead"); + + GGML_DEPRECATED(GGML_API struct ggml_tensor * ggml_add1_inplace( + struct ggml_context * ctx, + struct ggml_tensor * a, + struct ggml_tensor * b), + "use ggml_add_inplace instead"); + + // dst = a + // view(dst, nb1, nb2, nb3, offset) += b + // return dst + GGML_API struct ggml_tensor * ggml_acc( + struct ggml_context * ctx, + struct ggml_tensor * a, + struct ggml_tensor * b, + size_t nb1, + size_t nb2, + size_t nb3, + size_t offset); + + GGML_API struct ggml_tensor * ggml_acc_inplace( + struct ggml_context * ctx, + struct ggml_tensor * a, + struct ggml_tensor * b, + size_t nb1, + size_t nb2, + size_t nb3, + size_t offset); + + GGML_API struct ggml_tensor * ggml_sub( + struct ggml_context * ctx, + struct ggml_tensor * a, + struct ggml_tensor * b); + + GGML_API struct ggml_tensor * ggml_sub_inplace( + struct ggml_context * ctx, + struct ggml_tensor * a, + struct ggml_tensor * b); + + GGML_API struct ggml_tensor * ggml_mul( + struct ggml_context * ctx, + struct ggml_tensor * a, + struct ggml_tensor * b); + + GGML_API struct ggml_tensor * ggml_mul_inplace( + struct ggml_context * ctx, + struct ggml_tensor * a, + struct ggml_tensor * b); + + GGML_API struct ggml_tensor * ggml_div( + struct ggml_context * ctx, + struct ggml_tensor * a, + struct ggml_tensor * b); + + GGML_API struct ggml_tensor * ggml_div_inplace( + struct ggml_context * ctx, + struct ggml_tensor * a, + struct ggml_tensor * b); + + GGML_API struct ggml_tensor * ggml_sqr( + struct ggml_context * ctx, + struct ggml_tensor * a); + + GGML_API struct ggml_tensor * ggml_sqr_inplace( + struct ggml_context * ctx, + struct ggml_tensor * a); + + GGML_API struct ggml_tensor * ggml_sqrt( + struct ggml_context * ctx, + struct ggml_tensor * a); + + GGML_API struct ggml_tensor * ggml_sqrt_inplace( + struct ggml_context * ctx, + struct ggml_tensor * a); + + GGML_API struct ggml_tensor * ggml_log( + struct ggml_context * ctx, + struct ggml_tensor * a); + + GGML_API struct ggml_tensor * ggml_log_inplace( + struct ggml_context * ctx, + struct ggml_tensor * a); + + GGML_API struct ggml_tensor * ggml_expm1( + struct ggml_context * ctx, + struct ggml_tensor * a); + + GGML_API struct ggml_tensor * ggml_expm1_inplace( + struct ggml_context * ctx, + struct ggml_tensor * a); + + GGML_API struct ggml_tensor * ggml_softplus( + struct ggml_context * ctx, + struct ggml_tensor * a); + + GGML_API struct ggml_tensor * ggml_softplus_inplace( + struct ggml_context * ctx, + struct ggml_tensor * a); + + GGML_API struct ggml_tensor * ggml_sin( + struct ggml_context * ctx, + struct ggml_tensor * a); + + GGML_API struct ggml_tensor * ggml_sin_inplace( + struct ggml_context * ctx, + struct ggml_tensor * a); + + GGML_API struct ggml_tensor * ggml_cos( + struct ggml_context * ctx, + struct ggml_tensor * a); + + GGML_API struct ggml_tensor * ggml_cos_inplace( + struct ggml_context * ctx, + struct ggml_tensor * a); + + // return scalar + GGML_API struct ggml_tensor * ggml_sum( + struct ggml_context * ctx, + struct ggml_tensor * a); + + // sums along rows, with input shape [a,b,c,d] return shape [1,b,c,d] + GGML_API struct ggml_tensor * ggml_sum_rows( + struct ggml_context * ctx, + struct ggml_tensor * a); + + GGML_API struct ggml_tensor * ggml_cumsum( + struct ggml_context * ctx, + struct ggml_tensor * a); + + // mean along rows + GGML_API struct ggml_tensor * ggml_mean( + struct ggml_context * ctx, + struct ggml_tensor * a); + + // argmax along rows + GGML_API struct ggml_tensor * ggml_argmax( + struct ggml_context * ctx, + struct ggml_tensor * a); + + // count number of equal elements in a and b + GGML_API struct ggml_tensor * ggml_count_equal( + struct ggml_context * ctx, + struct ggml_tensor * a, + struct ggml_tensor * b); + + // if a is the same shape as b, and a is not parameter, return a + // otherwise, return a new tensor: repeat(a) to fit in b + GGML_API struct ggml_tensor * ggml_repeat( + struct ggml_context * ctx, + struct ggml_tensor * a, + struct ggml_tensor * b); + + // repeat a to the specified shape + GGML_API struct ggml_tensor * ggml_repeat_4d( + struct ggml_context * ctx, + struct ggml_tensor * a, + int64_t ne0, + int64_t ne1, + int64_t ne2, + int64_t ne3); + + // sums repetitions in a into shape of b + GGML_API struct ggml_tensor * ggml_repeat_back( + struct ggml_context * ctx, + struct ggml_tensor * a, + struct ggml_tensor * b); // sum up values that are adjacent in dims > 0 instead of repeated with same stride + + // concat a and b along dim + // used in stable-diffusion + GGML_API struct ggml_tensor * ggml_concat( + struct ggml_context * ctx, + struct ggml_tensor * a, + struct ggml_tensor * b, + int dim); + + GGML_API struct ggml_tensor * ggml_abs( + struct ggml_context * ctx, + struct ggml_tensor * a); + + GGML_API struct ggml_tensor * ggml_abs_inplace( + struct ggml_context * ctx, + struct ggml_tensor * a); + + GGML_API struct ggml_tensor * ggml_sgn( + struct ggml_context * ctx, + struct ggml_tensor * a); + + GGML_API struct ggml_tensor * ggml_sgn_inplace( + struct ggml_context * ctx, + struct ggml_tensor * a); + + GGML_API struct ggml_tensor * ggml_neg( + struct ggml_context * ctx, + struct ggml_tensor * a); + + GGML_API struct ggml_tensor * ggml_neg_inplace( + struct ggml_context * ctx, + struct ggml_tensor * a); + + GGML_API struct ggml_tensor * ggml_step( + struct ggml_context * ctx, + struct ggml_tensor * a); + + GGML_API struct ggml_tensor * ggml_step_inplace( + struct ggml_context * ctx, + struct ggml_tensor * a); + + GGML_API struct ggml_tensor * ggml_tanh( + struct ggml_context * ctx, + struct ggml_tensor * a); + + GGML_API struct ggml_tensor * ggml_tanh_inplace( + struct ggml_context * ctx, + struct ggml_tensor * a); + + GGML_API struct ggml_tensor * ggml_elu( + struct ggml_context * ctx, + struct ggml_tensor * a); + + GGML_API struct ggml_tensor * ggml_elu_inplace( + struct ggml_context * ctx, + struct ggml_tensor * a); + + GGML_API struct ggml_tensor * ggml_relu( + struct ggml_context * ctx, + struct ggml_tensor * a); + + GGML_API struct ggml_tensor * ggml_leaky_relu( + struct ggml_context * ctx, + struct ggml_tensor * a, float negative_slope, bool inplace); + + GGML_API struct ggml_tensor * ggml_relu_inplace( + struct ggml_context * ctx, + struct ggml_tensor * a); + + GGML_API struct ggml_tensor * ggml_sigmoid( + struct ggml_context * ctx, + struct ggml_tensor * a); + + GGML_API struct ggml_tensor * ggml_sigmoid_inplace( + struct ggml_context * ctx, + struct ggml_tensor * a); + + GGML_API struct ggml_tensor * ggml_gelu( + struct ggml_context * ctx, + struct ggml_tensor * a); + + GGML_API struct ggml_tensor * ggml_gelu_inplace( + struct ggml_context * ctx, + struct ggml_tensor * a); + + // GELU using erf (error function) when possible + // some backends may fallback to approximation based on Abramowitz and Stegun formula + GGML_API struct ggml_tensor * ggml_gelu_erf( + struct ggml_context * ctx, + struct ggml_tensor * a); + + GGML_API struct ggml_tensor * ggml_gelu_erf_inplace( + struct ggml_context * ctx, + struct ggml_tensor * a); + + GGML_API struct ggml_tensor * ggml_gelu_quick( + struct ggml_context * ctx, + struct ggml_tensor * a); + + GGML_API struct ggml_tensor * ggml_gelu_quick_inplace( + struct ggml_context * ctx, + struct ggml_tensor * a); + + GGML_API struct ggml_tensor * ggml_silu( + struct ggml_context * ctx, + struct ggml_tensor * a); + + GGML_API struct ggml_tensor * ggml_silu_inplace( + struct ggml_context * ctx, + struct ggml_tensor * a); + + // a - dy + // b - x + GGML_API struct ggml_tensor * ggml_silu_back( + struct ggml_context * ctx, + struct ggml_tensor * a, + struct ggml_tensor * b); + + // hardswish(x) = x * relu6(x + 3) / 6 + GGML_API struct ggml_tensor * ggml_hardswish( + struct ggml_context * ctx, + struct ggml_tensor * a); + + // hardsigmoid(x) = relu6(x + 3) / 6 + GGML_API struct ggml_tensor * ggml_hardsigmoid( + struct ggml_context * ctx, + struct ggml_tensor * a); + + GGML_API struct ggml_tensor * ggml_exp( + struct ggml_context * ctx, + struct ggml_tensor * a); + + GGML_API struct ggml_tensor * ggml_exp_inplace( + struct ggml_context * ctx, + struct ggml_tensor * a); + + GGML_API struct ggml_tensor * ggml_floor( + struct ggml_context * ctx, + struct ggml_tensor * a); + + GGML_API struct ggml_tensor * ggml_floor_inplace( + struct ggml_context * ctx, + struct ggml_tensor * a); + + GGML_API struct ggml_tensor * ggml_ceil( + struct ggml_context * ctx, + struct ggml_tensor * a); + + GGML_API struct ggml_tensor * ggml_ceil_inplace( + struct ggml_context * ctx, + struct ggml_tensor * a); + + GGML_API struct ggml_tensor * ggml_round( + struct ggml_context * ctx, + struct ggml_tensor * a); + + GGML_API struct ggml_tensor * ggml_round_inplace( + struct ggml_context * ctx, + struct ggml_tensor * a); + + /** + * Truncates the fractional part of each element in the tensor (towards zero). + * For example: trunc(3.7) = 3.0, trunc(-2.9) = -2.0 + * Similar to std::trunc in C/C++. + */ + + GGML_API struct ggml_tensor * ggml_trunc( + struct ggml_context * ctx, + struct ggml_tensor * a); + + GGML_API struct ggml_tensor * ggml_trunc_inplace( + struct ggml_context * ctx, + struct ggml_tensor * a); + + + + // xIELU activation function + // x = x * (c_a(alpha_n) + c_b(alpha_p, beta) * sigmoid(beta * x)) + eps * (x > 0) + // where c_a = softplus and c_b(a, b) = softplus(a) + b are constraining functions + // that constrain the positive and negative source alpha values respectively + GGML_API struct ggml_tensor * ggml_xielu( + struct ggml_context * ctx, + struct ggml_tensor * a, + float alpha_n, + float alpha_p, + float beta, + float eps); + + // gated linear unit ops + // A: n columns, r rows, + // result is n / 2 columns, r rows, + // expects gate in second half of row, unless swapped is true + GGML_API struct ggml_tensor * ggml_glu( + struct ggml_context * ctx, + struct ggml_tensor * a, + enum ggml_glu_op op, + bool swapped); + + GGML_API struct ggml_tensor * ggml_reglu( + struct ggml_context * ctx, + struct ggml_tensor * a); + + GGML_API struct ggml_tensor * ggml_reglu_swapped( + struct ggml_context * ctx, + struct ggml_tensor * a); + + GGML_API struct ggml_tensor * ggml_geglu( + struct ggml_context * ctx, + struct ggml_tensor * a); + + GGML_API struct ggml_tensor * ggml_geglu_swapped( + struct ggml_context * ctx, + struct ggml_tensor * a); + + GGML_API struct ggml_tensor * ggml_swiglu( + struct ggml_context * ctx, + struct ggml_tensor * a); + + GGML_API struct ggml_tensor * ggml_swiglu_swapped( + struct ggml_context * ctx, + struct ggml_tensor * a); + + GGML_API struct ggml_tensor * ggml_geglu_erf( + struct ggml_context * ctx, + struct ggml_tensor * a); + + GGML_API struct ggml_tensor * ggml_geglu_erf_swapped( + struct ggml_context * ctx, + struct ggml_tensor * a); + + GGML_API struct ggml_tensor * ggml_geglu_quick( + struct ggml_context * ctx, + struct ggml_tensor * a); + + GGML_API struct ggml_tensor * ggml_geglu_quick_swapped( + struct ggml_context * ctx, + struct ggml_tensor * a); + + // A: n columns, r rows, + // B: n columns, r rows, + GGML_API struct ggml_tensor * ggml_glu_split( + struct ggml_context * ctx, + struct ggml_tensor * a, + struct ggml_tensor * b, + enum ggml_glu_op op); + + GGML_API struct ggml_tensor * ggml_reglu_split( + struct ggml_context * ctx, + struct ggml_tensor * a, + struct ggml_tensor * b); + + GGML_API struct ggml_tensor * ggml_geglu_split( + struct ggml_context * ctx, + struct ggml_tensor * a, + struct ggml_tensor * b); + + GGML_API struct ggml_tensor * ggml_swiglu_split( + struct ggml_context * ctx, + struct ggml_tensor * a, + struct ggml_tensor * b); + + GGML_API struct ggml_tensor * ggml_geglu_erf_split( + struct ggml_context * ctx, + struct ggml_tensor * a, + struct ggml_tensor * b); + + GGML_API struct ggml_tensor * ggml_geglu_quick_split( + struct ggml_context * ctx, + struct ggml_tensor * a, + struct ggml_tensor * b); + + GGML_API struct ggml_tensor * ggml_swiglu_oai( + struct ggml_context * ctx, + struct ggml_tensor * a, + struct ggml_tensor * b, + float alpha, + float limit); + + // normalize along rows + GGML_API struct ggml_tensor * ggml_norm( + struct ggml_context * ctx, + struct ggml_tensor * a, + float eps); + + GGML_API struct ggml_tensor * ggml_norm_inplace( + struct ggml_context * ctx, + struct ggml_tensor * a, + float eps); + + GGML_API struct ggml_tensor * ggml_rms_norm( + struct ggml_context * ctx, + struct ggml_tensor * a, + float eps); + + GGML_API struct ggml_tensor * ggml_rms_norm_inplace( + struct ggml_context * ctx, + struct ggml_tensor * a, + float eps); + + // group normalize along ne0*ne1*n_groups + // used in stable-diffusion + GGML_API struct ggml_tensor * ggml_group_norm( + struct ggml_context * ctx, + struct ggml_tensor * a, + int n_groups, + float eps); + + GGML_API struct ggml_tensor * ggml_group_norm_inplace( + struct ggml_context * ctx, + struct ggml_tensor * a, + int n_groups, + float eps); + + // l2 normalize along rows + // used in rwkv v7 + GGML_API struct ggml_tensor * ggml_l2_norm( + struct ggml_context * ctx, + struct ggml_tensor * a, + float eps); + + GGML_API struct ggml_tensor * ggml_l2_norm_inplace( + struct ggml_context * ctx, + struct ggml_tensor * a, + float eps); + + // a - x + // b - dy + GGML_API struct ggml_tensor * ggml_rms_norm_back( + struct ggml_context * ctx, + struct ggml_tensor * a, + struct ggml_tensor * b, + float eps); + + // A: k columns, n rows => [ne03, ne02, n, k] + // B: k columns, m rows (i.e. we transpose it internally) => [ne03 * x, ne02 * y, m, k] + // result is n columns, m rows => [ne03 * x, ne02 * y, m, n] + GGML_API struct ggml_tensor * ggml_mul_mat( + struct ggml_context * ctx, + struct ggml_tensor * a, + struct ggml_tensor * b); + + // change the precision of a matrix multiplication + // set to GGML_PREC_F32 for higher precision (useful for phi-2) + GGML_API void ggml_mul_mat_set_prec( + struct ggml_tensor * a, + enum ggml_prec prec); + + // change the hint of a matrix multiplication + GGML_API void ggml_mul_mat_set_hint( + struct ggml_tensor * a, + enum ggml_op_hint hint); + + // indirect matrix multiplication + GGML_API struct ggml_tensor * ggml_mul_mat_id( + struct ggml_context * ctx, + struct ggml_tensor * as, + struct ggml_tensor * b, + struct ggml_tensor * ids); + + // A: m columns, n rows, + // B: p columns, n rows, + // result is m columns, p rows + GGML_API struct ggml_tensor * ggml_out_prod( + struct ggml_context * ctx, + struct ggml_tensor * a, + struct ggml_tensor * b); + + // + // operations on tensors without backpropagation + // + + GGML_API struct ggml_tensor * ggml_scale( + struct ggml_context * ctx, + struct ggml_tensor * a, + float s); + + // in-place, returns view(a) + GGML_API struct ggml_tensor * ggml_scale_inplace( + struct ggml_context * ctx, + struct ggml_tensor * a, + float s); + + // x = s * a + b + GGML_API struct ggml_tensor * ggml_scale_bias( + struct ggml_context * ctx, + struct ggml_tensor * a, + float s, + float b); + + GGML_API struct ggml_tensor * ggml_scale_bias_inplace( + struct ggml_context * ctx, + struct ggml_tensor * a, + float s, + float b); + + // b -> view(a,offset,nb1,nb2,3), return modified a + GGML_API struct ggml_tensor * ggml_set( + struct ggml_context * ctx, + struct ggml_tensor * a, + struct ggml_tensor * b, + size_t nb1, + size_t nb2, + size_t nb3, + size_t offset); // in bytes + + // b -> view(a,offset,nb1,nb2,3), return view(a) + GGML_API struct ggml_tensor * ggml_set_inplace( + struct ggml_context * ctx, + struct ggml_tensor * a, + struct ggml_tensor * b, + size_t nb1, + size_t nb2, + size_t nb3, + size_t offset); // in bytes + + GGML_API struct ggml_tensor * ggml_set_1d( + struct ggml_context * ctx, + struct ggml_tensor * a, + struct ggml_tensor * b, + size_t offset); // in bytes + + GGML_API struct ggml_tensor * ggml_set_1d_inplace( + struct ggml_context * ctx, + struct ggml_tensor * a, + struct ggml_tensor * b, + size_t offset); // in bytes + + // b -> view(a,offset,nb1,nb2,3), return modified a + GGML_API struct ggml_tensor * ggml_set_2d( + struct ggml_context * ctx, + struct ggml_tensor * a, + struct ggml_tensor * b, + size_t nb1, + size_t offset); // in bytes + + // b -> view(a,offset,nb1,nb2,3), return view(a) + GGML_API struct ggml_tensor * ggml_set_2d_inplace( + struct ggml_context * ctx, + struct ggml_tensor * a, + struct ggml_tensor * b, + size_t nb1, + size_t offset); // in bytes + + // a -> b, return view(b) + GGML_API struct ggml_tensor * ggml_cpy( + struct ggml_context * ctx, + struct ggml_tensor * a, + struct ggml_tensor * b); + + // note: casting from f32 to i32 will discard the fractional part + GGML_API struct ggml_tensor * ggml_cast( + struct ggml_context * ctx, + struct ggml_tensor * a, + enum ggml_type type); + + // make contiguous + GGML_API struct ggml_tensor * ggml_cont( + struct ggml_context * ctx, + struct ggml_tensor * a); + + // make contiguous, with new shape + GGML_API struct ggml_tensor * ggml_cont_1d( + struct ggml_context * ctx, + struct ggml_tensor * a, + int64_t ne0); + + GGML_API struct ggml_tensor * ggml_cont_2d( + struct ggml_context * ctx, + struct ggml_tensor * a, + int64_t ne0, + int64_t ne1); + + GGML_API struct ggml_tensor * ggml_cont_3d( + struct ggml_context * ctx, + struct ggml_tensor * a, + int64_t ne0, + int64_t ne1, + int64_t ne2); + + GGML_API struct ggml_tensor * ggml_cont_4d( + struct ggml_context * ctx, + struct ggml_tensor * a, + int64_t ne0, + int64_t ne1, + int64_t ne2, + int64_t ne3); + + // return view(a), b specifies the new shape + // TODO: when we start computing gradient, make a copy instead of view + GGML_API struct ggml_tensor * ggml_reshape( + struct ggml_context * ctx, + struct ggml_tensor * a, + struct ggml_tensor * b); + + // return view(a) + // TODO: when we start computing gradient, make a copy instead of view + GGML_API struct ggml_tensor * ggml_reshape_1d( + struct ggml_context * ctx, + struct ggml_tensor * a, + int64_t ne0); + + GGML_API struct ggml_tensor * ggml_reshape_2d( + struct ggml_context * ctx, + struct ggml_tensor * a, + int64_t ne0, + int64_t ne1); + + // return view(a) + // TODO: when we start computing gradient, make a copy instead of view + GGML_API struct ggml_tensor * ggml_reshape_3d( + struct ggml_context * ctx, + struct ggml_tensor * a, + int64_t ne0, + int64_t ne1, + int64_t ne2); + + GGML_API struct ggml_tensor * ggml_reshape_4d( + struct ggml_context * ctx, + struct ggml_tensor * a, + int64_t ne0, + int64_t ne1, + int64_t ne2, + int64_t ne3); + + // offset in bytes + GGML_API struct ggml_tensor * ggml_view_1d( + struct ggml_context * ctx, + struct ggml_tensor * a, + int64_t ne0, + size_t offset); + + GGML_API struct ggml_tensor * ggml_view_2d( + struct ggml_context * ctx, + struct ggml_tensor * a, + int64_t ne0, + int64_t ne1, + size_t nb1, // row stride in bytes + size_t offset); + + GGML_API struct ggml_tensor * ggml_view_3d( + struct ggml_context * ctx, + struct ggml_tensor * a, + int64_t ne0, + int64_t ne1, + int64_t ne2, + size_t nb1, // row stride in bytes + size_t nb2, // slice stride in bytes + size_t offset); + + GGML_API struct ggml_tensor * ggml_view_4d( + struct ggml_context * ctx, + struct ggml_tensor * a, + int64_t ne0, + int64_t ne1, + int64_t ne2, + int64_t ne3, + size_t nb1, // row stride in bytes + size_t nb2, // slice stride in bytes + size_t nb3, + size_t offset); + + GGML_API struct ggml_tensor * ggml_permute( + struct ggml_context * ctx, + struct ggml_tensor * a, + int axis0, + int axis1, + int axis2, + int axis3); + + // alias for ggml_permute(ctx, a, 1, 0, 2, 3) + GGML_API struct ggml_tensor * ggml_transpose( + struct ggml_context * ctx, + struct ggml_tensor * a); + + // supports 4D a: + // a [n_embd, ne1, ne2, ne3] + // b I32 [n_rows, ne2, ne3, 1] + // + // return [n_embd, n_rows, ne2, ne3] + GGML_API struct ggml_tensor * ggml_get_rows( + struct ggml_context * ctx, + struct ggml_tensor * a, // data + struct ggml_tensor * b); // row indices + + GGML_API struct ggml_tensor * ggml_get_rows_back( + struct ggml_context * ctx, + struct ggml_tensor * a, // gradients of ggml_get_rows result + struct ggml_tensor * b, // row indices + struct ggml_tensor * c); // data for ggml_get_rows, only used for its shape + + // a TD [n_embd, ne1, ne2, ne3] + // b TS [n_embd, n_rows, ne02, ne03] | ne02 == ne2, ne03 == ne3 + // c I64 [n_rows, ne11, ne12, 1] | c[i] in [0, ne1) + // + // undefined behavior if destination rows overlap + // + // broadcast: + // ne2 % ne11 == 0 + // ne3 % ne12 == 0 + // + // return view(a) + GGML_API struct ggml_tensor * ggml_set_rows( + struct ggml_context * ctx, + struct ggml_tensor * a, // destination + struct ggml_tensor * b, // source + struct ggml_tensor * c); // row indices + + GGML_API struct ggml_tensor * ggml_diag( + struct ggml_context * ctx, + struct ggml_tensor * a); + + // set elements above the diagonal to -INF + GGML_API struct ggml_tensor * ggml_diag_mask_inf( + struct ggml_context * ctx, + struct ggml_tensor * a, + int n_past); + + // in-place, returns view(a) + GGML_API struct ggml_tensor * ggml_diag_mask_inf_inplace( + struct ggml_context * ctx, + struct ggml_tensor * a, + int n_past); + + // set elements above the diagonal to 0 + GGML_API struct ggml_tensor * ggml_diag_mask_zero( + struct ggml_context * ctx, + struct ggml_tensor * a, + int n_past); + + // in-place, returns view(a) + GGML_API struct ggml_tensor * ggml_diag_mask_zero_inplace( + struct ggml_context * ctx, + struct ggml_tensor * a, + int n_past); + + GGML_API struct ggml_tensor * ggml_clamp( + struct ggml_context * ctx, + struct ggml_tensor * a, + float min, + float max); + + // in-place, returns view(a) + GGML_API struct ggml_tensor * ggml_clamp_inplace( + struct ggml_context * ctx, + struct ggml_tensor * a, + float min, + float max); + + GGML_API struct ggml_tensor * ggml_soft_max( + struct ggml_context * ctx, + struct ggml_tensor * a); + + // in-place, returns view(a) + GGML_API struct ggml_tensor * ggml_soft_max_inplace( + struct ggml_context * ctx, + struct ggml_tensor * a); + + // a [ne0, ne01, ne02, ne03] + // mask [ne0, ne11, ne12, ne13] | ne11 >= ne01, F16 or F32, optional + // + // broadcast: + // ne02 % ne12 == 0 + // ne03 % ne13 == 0 + // + // fused soft_max(a*scale + mask*(ALiBi slope)) + // max_bias = 0.0f for no ALiBi + GGML_API struct ggml_tensor * ggml_soft_max_ext( + struct ggml_context * ctx, + struct ggml_tensor * a, + struct ggml_tensor * mask, + float scale, + float max_bias); + + GGML_API struct ggml_tensor * ggml_soft_max_ext_inplace( + struct ggml_context * ctx, + struct ggml_tensor * a, + struct ggml_tensor * mask, + float scale, + float max_bias); + + GGML_API void ggml_soft_max_add_sinks( + struct ggml_tensor * a, + struct ggml_tensor * sinks); + + GGML_API struct ggml_tensor * ggml_soft_max_ext_back( + struct ggml_context * ctx, + struct ggml_tensor * a, + struct ggml_tensor * b, + float scale, + float max_bias); + + // in-place, returns view(a) + GGML_API struct ggml_tensor * ggml_soft_max_ext_back_inplace( + struct ggml_context * ctx, + struct ggml_tensor * a, + struct ggml_tensor * b, + float scale, + float max_bias); + + // rotary position embedding + // if (mode & 1) - skip n_past elements (NOT SUPPORTED) + // if (mode & GGML_ROPE_TYPE_NEOX) - GPT-NeoX style + // + // b is an int32 vector with size a->ne[2], it contains the positions + GGML_API struct ggml_tensor * ggml_rope( + struct ggml_context * ctx, + struct ggml_tensor * a, + struct ggml_tensor * b, + int n_dims, + int mode); + + // in-place, returns view(a) + GGML_API struct ggml_tensor * ggml_rope_inplace( + struct ggml_context * ctx, + struct ggml_tensor * a, + struct ggml_tensor * b, + int n_dims, + int mode); + + // RoPE operations with extended options + // a is the input tensor to apply RoPE to, shape [n_embd, n_head, n_token] + // b is an int32 vector with size n_token + // c is freq factors (e.g. phi3-128k), (optional) + // mode can be GGML_ROPE_TYPE_NORMAL or NEOX; for MROPE and VISION mode, use ggml_rope_multi + // + // pseudo-code for computing theta: + // for i in [0, n_dims/2): + // theta[i] = b[i] * powf(freq_base, -2.0 * i / n_dims); + // theta[i] = theta[i] / c[i]; # if c is provided, divide theta by c + // theta[i] = rope_yarn(theta[i], ...); # note: theta = theta * freq_scale is applied here + // + // other params are used by YaRN RoPE scaling, these default values will disable YaRN: + // freq_scale = 1.0f + // ext_factor = 0.0f + // attn_factor = 1.0f + // beta_fast = 0.0f + // beta_slow = 0.0f + // + // example: + // (marking: c = cos, s = sin, 0 = unrotated) + // given a single head with size = 8 --> [00000000] + // GGML_ROPE_TYPE_NORMAL n_dims = 4 --> [cscs0000] + // GGML_ROPE_TYPE_NORMAL n_dims = 8 --> [cscscscs] + // GGML_ROPE_TYPE_NEOX n_dims = 4 --> [ccss0000] + // GGML_ROPE_TYPE_NEOX n_dims = 8 --> [ccccssss] + GGML_API struct ggml_tensor * ggml_rope_ext( + struct ggml_context * ctx, + struct ggml_tensor * a, + struct ggml_tensor * b, + struct ggml_tensor * c, + int n_dims, + int mode, + int n_ctx_orig, + float freq_base, + float freq_scale, + float ext_factor, + float attn_factor, + float beta_fast, + float beta_slow); + + // multi-dimensional RoPE, for Qwen-VL and similar vision models + // mode can be either VISION, MROPE, IMROPE, cannot be combined with NORMAL or NEOX + // sections specify how many dimensions to rotate in each section: + // section length is equivalent to number of cos/sin pairs, NOT the number of dims + // (i.e. sum of 4 sections are expected to be n_dims/2) + // last sections can be 0, means ignored + // all other options are identical to ggml_rope_ext + // + // important note: + // - NEOX ordering is automatically applied and cannot be disabled for MROPE and VISION + // if you need normal ordering, there are 2 methods: + // (1) split the tensor manually using ggml_view + // (2) permute the weight upon conversion + // - for VISION, n_dims must be head_size/2 + // + // example M-RoPE: + // given sections = [t=4, y=2, x=2, 0] + // given a single head with size = 18 --> [000000000000000000] + // GGML_ROPE_TYPE_MROPE n_dims = 16 --> [ttttyyxxttttyyxx00] (cos/sin are applied in NEOX ordering) + // GGML_ROPE_TYPE_IMROPE n_dims = 16 --> [ttyxttyxttyxttyx00] (interleaved M-RoPE, still NEOX ordering) + // note: the theta for each dim is computed the same way as ggml_rope_ext, no matter the section + // in other words, idx used for theta: [0123456789... until n_dims/2], not reset for each section + // + // example vision RoPE: + // given sections = [y=4, x=4, 0, 0] (last 2 sections are ignored) + // given a single head with size = 8 --> [00000000] + // GGML_ROPE_TYPE_VISION n_dims = 4 --> [yyyyxxxx] + // other values of n_dims are untested and is undefined behavior + // note: unlike MROPE, the theta for each dim is computed differently for each section + // in other words, idx used for theta: [0123] for y section, then [0123] for x section + GGML_API struct ggml_tensor * ggml_rope_multi( + struct ggml_context * ctx, + struct ggml_tensor * a, + struct ggml_tensor * b, + struct ggml_tensor * c, + int n_dims, + int sections[GGML_MROPE_SECTIONS], + int mode, + int n_ctx_orig, + float freq_base, + float freq_scale, + float ext_factor, + float attn_factor, + float beta_fast, + float beta_slow); + + // in-place, returns view(a) + GGML_API struct ggml_tensor * ggml_rope_ext_inplace( + struct ggml_context * ctx, + struct ggml_tensor * a, + struct ggml_tensor * b, + struct ggml_tensor * c, + int n_dims, + int mode, + int n_ctx_orig, + float freq_base, + float freq_scale, + float ext_factor, + float attn_factor, + float beta_fast, + float beta_slow); + + GGML_API struct ggml_tensor * ggml_rope_multi_inplace( + struct ggml_context * ctx, + struct ggml_tensor * a, + struct ggml_tensor * b, + struct ggml_tensor * c, + int n_dims, + int sections[GGML_MROPE_SECTIONS], + int mode, + int n_ctx_orig, + float freq_base, + float freq_scale, + float ext_factor, + float attn_factor, + float beta_fast, + float beta_slow); + + GGML_DEPRECATED(GGML_API struct ggml_tensor * ggml_rope_custom( + struct ggml_context * ctx, + struct ggml_tensor * a, + struct ggml_tensor * b, + int n_dims, + int mode, + int n_ctx_orig, + float freq_base, + float freq_scale, + float ext_factor, + float attn_factor, + float beta_fast, + float beta_slow), + "use ggml_rope_ext instead"); + + GGML_DEPRECATED(GGML_API struct ggml_tensor * ggml_rope_custom_inplace( + struct ggml_context * ctx, + struct ggml_tensor * a, + struct ggml_tensor * b, + int n_dims, + int mode, + int n_ctx_orig, + float freq_base, + float freq_scale, + float ext_factor, + float attn_factor, + float beta_fast, + float beta_slow), + "use ggml_rope_ext_inplace instead"); + + // compute correction dims for YaRN RoPE scaling + GGML_API void ggml_rope_yarn_corr_dims( + int n_dims, int n_ctx_orig, float freq_base, float beta_fast, float beta_slow, float dims[2]); + + // rotary position embedding backward, i.e compute dx from dy + // a - dy + GGML_API struct ggml_tensor * ggml_rope_ext_back( + struct ggml_context * ctx, + struct ggml_tensor * a, // gradients of ggml_rope result + struct ggml_tensor * b, // positions + struct ggml_tensor * c, // freq factors + int n_dims, + int mode, + int n_ctx_orig, + float freq_base, + float freq_scale, + float ext_factor, + float attn_factor, + float beta_fast, + float beta_slow); + + GGML_API struct ggml_tensor * ggml_rope_multi_back( + struct ggml_context * ctx, + struct ggml_tensor * a, + struct ggml_tensor * b, + struct ggml_tensor * c, + int n_dims, + int sections[4], + int mode, + int n_ctx_orig, + float freq_base, + float freq_scale, + float ext_factor, + float attn_factor, + float beta_fast, + float beta_slow); + + // set the offset dims for RoPE + // a must be GGML_OP_ROPE or GGML_OP_ROPE_BACK + // vision RoPE is not supported + // example: (marking: x = rotated, 0 = unrotated) + // n_embd = 10, n_dims = 4, offset = 2 --> [00xxxx0000] + GGML_API struct ggml_tensor * ggml_rope_set_offset( + struct ggml_tensor * a, + int n_offs); + + // im2col + // converts data into a format that effectively results in a convolution when combined with matrix multiplication + GGML_API struct ggml_tensor * ggml_im2col( + struct ggml_context * ctx, + struct ggml_tensor * a, // convolution kernel + struct ggml_tensor * b, // data + int s0, // stride dimension 0 + int s1, // stride dimension 1 + int p0, // padding dimension 0 + int p1, // padding dimension 1 + int d0, // dilation dimension 0 + int d1, // dilation dimension 1 + bool is_2D, + enum ggml_type dst_type); + + GGML_API struct ggml_tensor * ggml_im2col_back( + struct ggml_context * ctx, + struct ggml_tensor * a, // convolution kernel + struct ggml_tensor * b, // gradient of im2col output + int64_t * ne, // shape of im2col input + int s0, // stride dimension 0 + int s1, // stride dimension 1 + int p0, // padding dimension 0 + int p1, // padding dimension 1 + int d0, // dilation dimension 0 + int d1, // dilation dimension 1 + bool is_2D); + + // col2im_1d: scatter-add GEMM columns back to 1D signal + // a: [K*OC, T_in] (columns from matmul, K = a->ne[0]/OC) + // result: [T_out, OC] where T_out = (T_in - 1)*s0 + K - 2*p0 + GGML_API struct ggml_tensor * ggml_col2im_1d( + struct ggml_context * ctx, + struct ggml_tensor * a, // columns [K*OC, T_in] + int s0, // stride + int oc, // output channels + int p0); // padding to crop from both sides + + GGML_API struct ggml_tensor * ggml_conv_1d( + struct ggml_context * ctx, + struct ggml_tensor * a, // convolution kernel + struct ggml_tensor * b, // data + int s0, // stride + int p0, // padding + int d0); // dilation + + // conv_1d with padding = half + // alias for ggml_conv_1d(a, b, s, a->ne[0]/2, d) + GGML_API struct ggml_tensor* ggml_conv_1d_ph( + struct ggml_context * ctx, + struct ggml_tensor * a, // convolution kernel + struct ggml_tensor * b, // data + int s, // stride + int d); // dilation + + // depthwise + // TODO: this is very likely wrong for some cases! - needs more testing + GGML_API struct ggml_tensor * ggml_conv_1d_dw( + struct ggml_context * ctx, + struct ggml_tensor * a, // convolution kernel + struct ggml_tensor * b, // data + int s0, // stride + int p0, // padding + int d0); // dilation + + GGML_API struct ggml_tensor * ggml_conv_1d_dw_ph( + struct ggml_context * ctx, + struct ggml_tensor * a, // convolution kernel + struct ggml_tensor * b, // data + int s0, // stride + int d0); // dilation + + GGML_API struct ggml_tensor * ggml_conv_transpose_1d( + struct ggml_context * ctx, + struct ggml_tensor * a, // convolution kernel + struct ggml_tensor * b, // data + int s0, // stride + int p0, // padding + int d0); // dilation + + GGML_API struct ggml_tensor * ggml_conv_2d( + struct ggml_context * ctx, + struct ggml_tensor * a, // convolution kernel + struct ggml_tensor * b, // data + int s0, // stride dimension 0 + int s1, // stride dimension 1 + int p0, // padding dimension 0 + int p1, // padding dimension 1 + int d0, // dilation dimension 0 + int d1); // dilation dimension 1 + + GGML_API struct ggml_tensor * ggml_im2col_3d( + struct ggml_context * ctx, + struct ggml_tensor * a, + struct ggml_tensor * b, + int64_t IC, + int s0, // stride width + int s1, // stride height + int s2, // stride depth + int p0, // padding width + int p1, // padding height + int p2, // padding depth + int d0, // dilation width + int d1, // dilation height + int d2, // dilation depth + enum ggml_type dst_type); + + // a: [OC*IC, KD, KH, KW] + // b: [N*IC, ID, IH, IW] + // result: [N*OC, OD, OH, OW] + GGML_API struct ggml_tensor * ggml_conv_3d( + struct ggml_context * ctx, + struct ggml_tensor * a, + struct ggml_tensor * b, + int64_t IC, + int s0, // stride width + int s1, // stride height + int s2, // stride depth + int p0, // padding width + int p1, // padding height + int p2, // padding depth + int d0, // dilation width + int d1, // dilation height + int d2 // dilation depth + ); + + // kernel size is a->ne[0] x a->ne[1] + // stride is equal to kernel size + // padding is zero + // example: + // a: 16 16 3 768 + // b: 1024 1024 3 1 + // res: 64 64 768 1 + // used in sam + GGML_API struct ggml_tensor * ggml_conv_2d_sk_p0( + struct ggml_context * ctx, + struct ggml_tensor * a, + struct ggml_tensor * b); + + // kernel size is a->ne[0] x a->ne[1] + // stride is 1 + // padding is half + // example: + // a: 3 3 256 256 + // b: 64 64 256 1 + // res: 64 64 256 1 + // used in sam + GGML_API struct ggml_tensor * ggml_conv_2d_s1_ph( + struct ggml_context * ctx, + struct ggml_tensor * a, + struct ggml_tensor * b); + + // depthwise (via im2col and mul_mat) + GGML_API struct ggml_tensor * ggml_conv_2d_dw( + struct ggml_context * ctx, + struct ggml_tensor * a, // convolution kernel + struct ggml_tensor * b, // data + int s0, // stride dimension 0 + int s1, // stride dimension 1 + int p0, // padding dimension 0 + int p1, // padding dimension 1 + int d0, // dilation dimension 0 + int d1); // dilation dimension 1 + + // Depthwise 2D convolution + // may be faster than ggml_conv_2d_dw, but not available in all backends + // a: KW KH 1 C convolution kernel + // b: W H C N input data + // res: W_out H_out C N + GGML_API struct ggml_tensor * ggml_conv_2d_dw_direct( + struct ggml_context * ctx, + struct ggml_tensor * a, + struct ggml_tensor * b, + int stride0, + int stride1, + int pad0, + int pad1, + int dilation0, + int dilation1); + + GGML_API struct ggml_tensor * ggml_conv_transpose_2d_p0( + struct ggml_context * ctx, + struct ggml_tensor * a, + struct ggml_tensor * b, + int stride); + + GGML_API struct ggml_tensor * ggml_conv_2d_direct( + struct ggml_context * ctx, + struct ggml_tensor * a, // convolution kernel [KW, KH, IC, OC] + struct ggml_tensor * b, // input data [W, H, C, N] + int s0, // stride dimension 0 + int s1, // stride dimension 1 + int p0, // padding dimension 0 + int p1, // padding dimension 1 + int d0, // dilation dimension 0 + int d1); // dilation dimension 1 + + GGML_API struct ggml_tensor * ggml_conv_3d_direct( + struct ggml_context * ctx, + struct ggml_tensor * a, // kernel [KW, KH, KD, IC * OC] + struct ggml_tensor * b, // input [W, H, D, C * N] + int s0, // stride + int s1, + int s2, + int p0, // padding + int p1, + int p2, + int d0, // dilation + int d1, + int d2, + int n_channels, + int n_batch, + int n_channels_out); + + enum ggml_op_pool { + GGML_OP_POOL_MAX, + GGML_OP_POOL_AVG, + GGML_OP_POOL_COUNT, + }; + + GGML_API struct ggml_tensor * ggml_pool_1d( + struct ggml_context * ctx, + struct ggml_tensor * a, + enum ggml_op_pool op, + int k0, // kernel size + int s0, // stride + int p0); // padding + + // the result will have 2*p0 padding for the first dimension + // and 2*p1 padding for the second dimension + GGML_API struct ggml_tensor * ggml_pool_2d( + struct ggml_context * ctx, + struct ggml_tensor * a, + enum ggml_op_pool op, + int k0, + int k1, + int s0, + int s1, + float p0, + float p1); + + GGML_API struct ggml_tensor * ggml_pool_2d_back( + struct ggml_context * ctx, + struct ggml_tensor * a, + struct ggml_tensor * af, // "a"/input used in forward pass + enum ggml_op_pool op, + int k0, + int k1, + int s0, + int s1, + float p0, + float p1); + + enum ggml_scale_mode { + GGML_SCALE_MODE_NEAREST = 0, + GGML_SCALE_MODE_BILINEAR = 1, + GGML_SCALE_MODE_BICUBIC = 2, + + GGML_SCALE_MODE_COUNT + }; + + enum ggml_scale_flag { + GGML_SCALE_FLAG_ALIGN_CORNERS = (1 << 8), + GGML_SCALE_FLAG_ANTIALIAS = (1 << 9), + }; + + // interpolate + // multiplies ne0 and ne1 by scale factor + GGML_API struct ggml_tensor * ggml_upscale( + struct ggml_context * ctx, + struct ggml_tensor * a, + int scale_factor, + enum ggml_scale_mode mode); + + // interpolate + // interpolate scale to specified dimensions + GGML_DEPRECATED(GGML_API struct ggml_tensor * ggml_upscale_ext( + struct ggml_context * ctx, + struct ggml_tensor * a, + int ne0, + int ne1, + int ne2, + int ne3, + enum ggml_scale_mode mode), + "use ggml_interpolate instead"); + + // Up- or downsamples the input to the specified size. + // 2D scale modes (eg. bilinear) are applied to the first two dimensions. + GGML_API struct ggml_tensor * ggml_interpolate( + struct ggml_context * ctx, + struct ggml_tensor * a, + int64_t ne0, + int64_t ne1, + int64_t ne2, + int64_t ne3, + uint32_t mode); // ggml_scale_mode [ | ggml_scale_flag...] + + // pad each dimension with zeros: [x, ..., x] -> [x, ..., x, 0, ..., 0] + GGML_API struct ggml_tensor * ggml_pad( + struct ggml_context * ctx, + struct ggml_tensor * a, + int p0, + int p1, + int p2, + int p3); + + // pad each dimension with values on the other side of the torus (looping around) + GGML_API struct ggml_tensor * ggml_pad_circular( + struct ggml_context * ctx, + struct ggml_tensor * a, + int p0, + int p1, + int p2, + int p3); + + GGML_API struct ggml_tensor * ggml_pad_ext( + struct ggml_context * ctx, + struct ggml_tensor * a, + int lp0, + int rp0, + int lp1, + int rp1, + int lp2, + int rp2, + int lp3, + int rp3 + ); + + // pad each dimension with values on the other side of the torus (looping around) + GGML_API struct ggml_tensor * ggml_pad_ext_circular( + struct ggml_context * ctx, + struct ggml_tensor * a, + int lp0, + int rp0, + int lp1, + int rp1, + int lp2, + int rp2, + int lp3, + int rp3); + + // pad each dimension with reflection: [a, b, c, d] -> [b, a, b, c, d, c] + GGML_API struct ggml_tensor * ggml_pad_reflect_1d( + struct ggml_context * ctx, + struct ggml_tensor * a, + int p0, + int p1); + + // Move tensor elements by an offset given for each dimension. Elements that + // are shifted beyond the last position are wrapped around to the beginning. + GGML_API struct ggml_tensor * ggml_roll( + struct ggml_context * ctx, + struct ggml_tensor * a, + int shift0, + int shift1, + int shift2, + int shift3); + + // Convert matrix into a triangular one (upper, strict upper, lower or strict lower) by writing + // zeroes everywhere outside the masked area + GGML_API struct ggml_tensor * ggml_tri( + struct ggml_context * ctx, + struct ggml_tensor * a, + enum ggml_tri_type type); + + // Fill tensor a with constant c + GGML_API struct ggml_tensor * ggml_fill( + struct ggml_context * ctx, + struct ggml_tensor * a, + float c); + + GGML_API struct ggml_tensor * ggml_fill_inplace( + struct ggml_context * ctx, + struct ggml_tensor * a, + float c); + + // Ref: https://github.com/CompVis/stable-diffusion/blob/main/ldm/modules/diffusionmodules/util.py#L151 + // timesteps: [N,] + // return: [N, dim] + GGML_API struct ggml_tensor * ggml_timestep_embedding( + struct ggml_context * ctx, + struct ggml_tensor * timesteps, + int dim, + int max_period); + + // sort rows + enum ggml_sort_order { + GGML_SORT_ORDER_ASC, + GGML_SORT_ORDER_DESC, + }; + + GGML_API struct ggml_tensor * ggml_argsort( + struct ggml_context * ctx, + struct ggml_tensor * a, + enum ggml_sort_order order); + + // similar to ggml_top_k but implemented as `argsort` + `view` + GGML_API struct ggml_tensor * ggml_argsort_top_k( + struct ggml_context * ctx, + struct ggml_tensor * a, + int k); + + // top k elements per row + // note: the resulting top k indices are in no particular order + GGML_API struct ggml_tensor * ggml_top_k( + struct ggml_context * ctx, + struct ggml_tensor * a, + int k); + + GGML_API struct ggml_tensor * ggml_arange( + struct ggml_context * ctx, + float start, + float stop, + float step); + + // q: [n_embd_k, n_batch, n_head, ne3 ] + // k: [n_embd_k, n_kv, n_head_kv, ne3 ] + // v: [n_embd_v, n_kv, n_head_kv, ne3 ] !! not transposed !! + // mask: [n_kv, n_batch, ne32, ne33] + // res: [n_embd_v, n_head, n_batch, ne3 ] !! permuted !! + // + // broadcast: + // n_head % n_head_kv == 0 + // n_head % ne32 == 0 + // ne3 % ne33 == 0 + // + GGML_API struct ggml_tensor * ggml_flash_attn_ext( + struct ggml_context * ctx, + struct ggml_tensor * q, + struct ggml_tensor * k, + struct ggml_tensor * v, + struct ggml_tensor * mask, + float scale, + float max_bias, + float logit_softcap); + + GGML_API void ggml_flash_attn_ext_set_prec( + struct ggml_tensor * a, + enum ggml_prec prec); + + GGML_API enum ggml_prec ggml_flash_attn_ext_get_prec( + const struct ggml_tensor * a); + + GGML_API void ggml_flash_attn_ext_add_sinks( + struct ggml_tensor * a, + struct ggml_tensor * sinks); + + // TODO: needs to be adapted to ggml_flash_attn_ext + GGML_API struct ggml_tensor * ggml_flash_attn_back( + struct ggml_context * ctx, + struct ggml_tensor * q, + struct ggml_tensor * k, + struct ggml_tensor * v, + struct ggml_tensor * d, + bool masked); + + GGML_API struct ggml_tensor * ggml_ssm_conv( + struct ggml_context * ctx, + struct ggml_tensor * sx, + struct ggml_tensor * c); + + GGML_API struct ggml_tensor * ggml_ssm_scan( + struct ggml_context * ctx, + struct ggml_tensor * s, + struct ggml_tensor * x, + struct ggml_tensor * dt, + struct ggml_tensor * A, + struct ggml_tensor * B, + struct ggml_tensor * C, + struct ggml_tensor * ids, + int64_t K); + + // partition into non-overlapping windows with padding if needed + // example: + // a: 768 64 64 1 + // w: 14 + // res: 768 14 14 25 + // used in sam + GGML_API struct ggml_tensor * ggml_win_part( + struct ggml_context * ctx, + struct ggml_tensor * a, + int w); + + // reverse of ggml_win_part + // used in sam + GGML_API struct ggml_tensor * ggml_win_unpart( + struct ggml_context * ctx, + struct ggml_tensor * a, + int w0, + int h0, + int w); + + GGML_API struct ggml_tensor * ggml_unary( + struct ggml_context * ctx, + struct ggml_tensor * a, + enum ggml_unary_op op); + + GGML_API struct ggml_tensor * ggml_unary_inplace( + struct ggml_context * ctx, + struct ggml_tensor * a, + enum ggml_unary_op op); + + // used in sam + GGML_API struct ggml_tensor * ggml_get_rel_pos( + struct ggml_context * ctx, + struct ggml_tensor * a, + int qh, + int kh); + + // used in sam + GGML_API struct ggml_tensor * ggml_add_rel_pos( + struct ggml_context * ctx, + struct ggml_tensor * a, + struct ggml_tensor * pw, + struct ggml_tensor * ph); + + GGML_API struct ggml_tensor * ggml_add_rel_pos_inplace( + struct ggml_context * ctx, + struct ggml_tensor * a, + struct ggml_tensor * pw, + struct ggml_tensor * ph); + + GGML_API struct ggml_tensor * ggml_rwkv_wkv6( + struct ggml_context * ctx, + struct ggml_tensor * k, + struct ggml_tensor * v, + struct ggml_tensor * r, + struct ggml_tensor * tf, + struct ggml_tensor * td, + struct ggml_tensor * state); + + GGML_API struct ggml_tensor * ggml_gated_linear_attn( + struct ggml_context * ctx, + struct ggml_tensor * k, + struct ggml_tensor * v, + struct ggml_tensor * q, + struct ggml_tensor * g, + struct ggml_tensor * state, + float scale); + + GGML_API struct ggml_tensor * ggml_rwkv_wkv7( + struct ggml_context * ctx, + struct ggml_tensor * r, + struct ggml_tensor * w, + struct ggml_tensor * k, + struct ggml_tensor * v, + struct ggml_tensor * a, + struct ggml_tensor * b, + struct ggml_tensor * state); + + /* Solves a specific equation of the form Ax=B, where A is a triangular matrix + * without zeroes on the diagonal (i.e. invertible). + * B can have any number of columns, but must have the same number of rows as A + * If A is [n, n] and B is [n, m], then the result will be [n, m] as well + * Has O(n^3) complexity (unlike most matrix ops out there), so use on cases + * where n > 100 sparingly, pre-chunk if necessary. + * + * If left = false, solves xA=B instead + * If lower = false, assumes upper triangular instead + * If uni = true, assumes diagonal of A to be all ones (will override actual values) + * + * TODO: currently only lower, right, non-unitriangular variant is implemented + */ + GGML_API struct ggml_tensor * ggml_solve_tri( + struct ggml_context * ctx, + struct ggml_tensor * a, + struct ggml_tensor * b, + bool left, + bool lower, + bool uni); + + // TODO: add ggml_gated_delta_net_set_bcast() to be able to configure Q, K broadcast type: tiled vs interleaved [TAG_GGML_GDN_BCAST] + // ref: https://github.com/ggml-org/llama.cpp/pull/19468#discussion_r2786394306 + // + // tensor shapes (S_k == S_v, H_v % H_k == 0): + // q, k : [S_k, H_k, n_tokens, n_seqs] + // v : [S_v, H_v, n_tokens, n_seqs] + // g : [1, H_v, n_tokens, n_seqs] (scalar gate) or [S_v, H_v, n_tokens, n_seqs] (KDA) + // beta : [1, H_v, n_tokens, n_seqs] + // state : [S_v, S_v, H_v, n_seqs] -- initial recurrent state s0 + // + // the output packs the attention scores [S_v, H_v, n_tokens, n_seqs] followed by K state + // snapshots, most-recent first (slot 0 = final state, slot s = state s tokens back). K == 1 + // keeps only the final state; when n_tokens < K only slots 0..n_tokens-1 are written. + GGML_API struct ggml_tensor * ggml_gated_delta_net( + struct ggml_context * ctx, + struct ggml_tensor * q, + struct ggml_tensor * k, + struct ggml_tensor * v, + struct ggml_tensor * g, + struct ggml_tensor * beta, + struct ggml_tensor * state, + int64_t K); + + // DSA lightning indexer + // + // q: [n_embd_idx, n_head_idx, n_batch, ne3 ] + // k: [n_embd_idx, 1, n_kv, ne3 ] + // weights: [n_head_idx, n_batch, 1, ne3 ] !! prescaled !! + // mask: [n_kv, n_batch, 1, ne33] !! f16 !! + // res: [n_kv, n_batch, 1, ne3 ] + // + // broadcast: + // ne3 % ne33 == 0 + // + GGML_API struct ggml_tensor * ggml_lightning_indexer( + struct ggml_context * ctx, + struct ggml_tensor * q, + struct ggml_tensor * k, + struct ggml_tensor * weights, + struct ggml_tensor * mask); + + // DeepSeek V4 hyper-connections (ref. https://arxiv.org/pdf/2512.24880) + // In short these operations are replacements for the original residual connection (x = transformer(x) + x) + // using a richer representation through streams. + // + // hc_comb: mixes [(2 + hc)*hc, n_tokens], scale [3], base [(2 + hc)*hc] + // -> [dst_hc, src_hc, n_tokens] + // logits[dst, src, t] = mixes[2*hc + dst + hc*src, t]*scale[2] + // + base[2*hc + dst + hc*src] + // Softmax over dst, add eps, normalize over src, then repeat normalization + // over dst followed by src for iterations 1 through n_iter - 1. + GGML_API struct ggml_tensor * ggml_dsv4_hc_comb( + struct ggml_context * ctx, + struct ggml_tensor * mixes, + struct ggml_tensor * scale, + struct ggml_tensor * base, + float eps, + int32_t n_iter); + + // hc_pre: x [n_embd, hc, n_tokens], weights [hc, n_tokens] -> [n_embd, n_tokens] + // result[i, t] = sum_h x[i, h, t]*weights[h, t] + // + GGML_API struct ggml_tensor * ggml_dsv4_hc_pre( + struct ggml_context * ctx, + struct ggml_tensor * x, + struct ggml_tensor * weights); + + // hc_post: x [n_embd, n_tokens], residual [n_embd, hc, n_tokens], + // post [hc, n_tokens], comb [dst_hc, src_hc, n_tokens] + // -> [n_embd, hc, n_tokens] + // result[i, dst, t] = x[i, t]*post[dst, t] + // + sum_src residual[i, src, t]*comb[dst, src, t] + // + GGML_API struct ggml_tensor * ggml_dsv4_hc_post( + struct ggml_context * ctx, + struct ggml_tensor * x, + struct ggml_tensor * residual, + struct ggml_tensor * post, + struct ggml_tensor * comb); + + // custom operators + + typedef void (*ggml_custom1_op_t)(struct ggml_tensor * dst , const struct ggml_tensor * a, int ith, int nth, void * userdata); + typedef void (*ggml_custom2_op_t)(struct ggml_tensor * dst , const struct ggml_tensor * a, const struct ggml_tensor * b, int ith, int nth, void * userdata); + typedef void (*ggml_custom3_op_t)(struct ggml_tensor * dst , const struct ggml_tensor * a, const struct ggml_tensor * b, const struct ggml_tensor * c, int ith, int nth, void * userdata); + +#define GGML_N_TASKS_MAX (-1) + // n_tasks == GGML_N_TASKS_MAX means to use max number of tasks + + GGML_API struct ggml_tensor * ggml_map_custom1( + struct ggml_context * ctx, + struct ggml_tensor * a, + ggml_custom1_op_t fun, + int n_tasks, + void * userdata); + + GGML_API struct ggml_tensor * ggml_map_custom1_inplace( + struct ggml_context * ctx, + struct ggml_tensor * a, + ggml_custom1_op_t fun, + int n_tasks, + void * userdata); + + GGML_API struct ggml_tensor * ggml_map_custom2( + struct ggml_context * ctx, + struct ggml_tensor * a, + struct ggml_tensor * b, + ggml_custom2_op_t fun, + int n_tasks, + void * userdata); + + GGML_API struct ggml_tensor * ggml_map_custom2_inplace( + struct ggml_context * ctx, + struct ggml_tensor * a, + struct ggml_tensor * b, + ggml_custom2_op_t fun, + int n_tasks, + void * userdata); + + GGML_API struct ggml_tensor * ggml_map_custom3( + struct ggml_context * ctx, + struct ggml_tensor * a, + struct ggml_tensor * b, + struct ggml_tensor * c, + ggml_custom3_op_t fun, + int n_tasks, + void * userdata); + + GGML_API struct ggml_tensor * ggml_map_custom3_inplace( + struct ggml_context * ctx, + struct ggml_tensor * a, + struct ggml_tensor * b, + struct ggml_tensor * c, + ggml_custom3_op_t fun, + int n_tasks, + void * userdata); + + typedef void (*ggml_custom_op_t)(struct ggml_tensor * dst , int ith, int nth, void * userdata); + + GGML_API struct ggml_tensor * ggml_custom_4d( + struct ggml_context * ctx, + enum ggml_type type, + int64_t ne0, + int64_t ne1, + int64_t ne2, + int64_t ne3, + struct ggml_tensor ** args, + int n_args, + ggml_custom_op_t fun, + int n_tasks, + void * userdata); + + GGML_API struct ggml_tensor * ggml_custom_inplace( + struct ggml_context * ctx, + struct ggml_tensor * a, + struct ggml_tensor ** args, + int n_args, + ggml_custom_op_t fun, + int n_tasks, + void * userdata); + + // loss function + + GGML_API struct ggml_tensor * ggml_cross_entropy_loss( + struct ggml_context * ctx, + struct ggml_tensor * a, // logits + struct ggml_tensor * b); // labels + + GGML_API struct ggml_tensor * ggml_cross_entropy_loss_back( + struct ggml_context * ctx, + struct ggml_tensor * a, // logits + struct ggml_tensor * b, // labels + struct ggml_tensor * c); // gradients of cross_entropy_loss result + + // AdamW optimizer step + // Paper: https://arxiv.org/pdf/1711.05101v3.pdf + // PyTorch: https://pytorch.org/docs/stable/generated/torch.optim.AdamW.html + GGML_API struct ggml_tensor * ggml_opt_step_adamw( + struct ggml_context * ctx, + struct ggml_tensor * a, + struct ggml_tensor * grad, + struct ggml_tensor * m, + struct ggml_tensor * v, + struct ggml_tensor * adamw_params); // parameters such as the learning rate + + // stochastic gradient descent step (with weight decay) + GGML_API struct ggml_tensor * ggml_opt_step_sgd( + struct ggml_context * ctx, + struct ggml_tensor * a, + struct ggml_tensor * grad, + struct ggml_tensor * sgd_params); // alpha, weight decay + + // build forward multiple tensors and select one of them for computing + // this is useful for creating graphs that have constant topology but compute different things based on the input + // ref: https://github.com/ggml-org/llama.cpp/pull/18550 + // + // nodes: + // | - build forward into the graph but do not compute + // c - build forward into the graph and compute + // + // | | ... c ... | + // | | ... c ... | + // | | ... c ... | + // [0 1 ... idx ... n-1] <-- ggml_build_forward_select(..., n, idx) + // c + // c + // + // example: + // struct ggml_tensor * curs[3]; + // + // curs[0] = compute0(...); + // curs[1] = compute1(...); + // curs[2] = compute2(...); + // + // int idx = select_branch(some_input); + // + // struct ggml_tensor * out = ggml_build_forward_select(cgraph, curs, 3, idx); + // + GGML_API struct ggml_tensor * ggml_build_forward_select( + struct ggml_cgraph * cgraph, + struct ggml_tensor ** tensors, + int n_tensors, + int idx); + + GGML_API void ggml_build_forward_expand( + struct ggml_cgraph * cgraph, + struct ggml_tensor * tensor); + + // add the tensor and its parents to the graph without marking them for compute + // the flag is set later, when the tensor is reached from a node that computes + GGML_API void ggml_build_forward_order( + struct ggml_cgraph * cgraph, + struct ggml_tensor * tensor); + + GGML_API void ggml_build_backward_expand( + struct ggml_context * ctx, // context for gradient computation + struct ggml_cgraph * cgraph, + struct ggml_tensor ** grad_accs); + + // graph allocation in a context + GGML_API struct ggml_cgraph * ggml_new_graph (struct ggml_context * ctx); // size = GGML_DEFAULT_GRAPH_SIZE, grads = false + GGML_API struct ggml_cgraph * ggml_new_graph_custom(struct ggml_context * ctx, size_t size, bool grads); + GGML_API struct ggml_cgraph * ggml_graph_dup (struct ggml_context * ctx, struct ggml_cgraph * cgraph, bool force_grads); + GGML_API void ggml_graph_cpy (struct ggml_cgraph * src, struct ggml_cgraph * dst); + GGML_API void ggml_graph_reset (struct ggml_cgraph * cgraph); // set regular grads + optimizer momenta to 0, set loss grad to 1 + GGML_API void ggml_graph_clear (struct ggml_cgraph * cgraph); + + GGML_API int ggml_graph_size (struct ggml_cgraph * cgraph); + GGML_API struct ggml_tensor * ggml_graph_node (struct ggml_cgraph * cgraph, int i); // if i < 0, returns nodes[n_nodes + i] + GGML_API struct ggml_tensor ** ggml_graph_nodes (struct ggml_cgraph * cgraph); + GGML_API int ggml_graph_n_nodes(struct ggml_cgraph * cgraph); + + GGML_API void ggml_graph_add_node(struct ggml_cgraph * cgraph, struct ggml_tensor * tensor); + + GGML_API size_t ggml_graph_overhead(void); + GGML_API size_t ggml_graph_overhead_custom(size_t size, bool grads); + + GGML_API struct ggml_tensor * ggml_graph_get_tensor (const struct ggml_cgraph * cgraph, const char * name); + GGML_API struct ggml_tensor * ggml_graph_get_grad (const struct ggml_cgraph * cgraph, const struct ggml_tensor * node); + GGML_API struct ggml_tensor * ggml_graph_get_grad_acc(const struct ggml_cgraph * cgraph, const struct ggml_tensor * node); + + // print info and performance information for the graph + GGML_API void ggml_graph_print(const struct ggml_cgraph * cgraph); + + // dump the graph into a file using the dot format + GGML_API void ggml_graph_dump_dot(const struct ggml_cgraph * gb, const struct ggml_cgraph * cgraph, const char * filename); + + // TODO these functions were sandwiched in the old optimization interface, is there a better place for them? + typedef void (*ggml_log_callback)(enum ggml_log_level level, const char * text, void * user_data); + + // Set callback for all future logging events. + // If this is not called, or NULL is supplied, everything is output on stderr. + GGML_API void ggml_log_get(ggml_log_callback * log_callback, void ** user_data); + GGML_API void ggml_log_set(ggml_log_callback log_callback, void * user_data); + + GGML_API struct ggml_tensor * ggml_set_zero(struct ggml_tensor * tensor); + + // + // quantization + // + + // - ggml_quantize_init can be called multiple times with the same type + // it will only initialize the quantization tables for the first call or after ggml_quantize_free + // automatically called by ggml_quantize_chunk for convenience + // + // - ggml_quantize_free will free any memory allocated by ggml_quantize_init + // call this at the end of the program to avoid memory leaks + // + // note: these are thread-safe + // + GGML_API void ggml_quantize_init(enum ggml_type type); + GGML_API void ggml_quantize_free(void); + + // some quantization type cannot be used without an importance matrix + GGML_API bool ggml_quantize_requires_imatrix(enum ggml_type type); + + // calls ggml_quantize_init internally (i.e. can allocate memory) + GGML_API size_t ggml_quantize_chunk( + enum ggml_type type, + const float * src, + void * dst, + int64_t start, + int64_t nrows, + int64_t n_per_row, + const float * imatrix); + +#ifdef __cplusplus + // restrict not standard in C++ +# if defined(__GNUC__) +# define GGML_RESTRICT __restrict__ +# elif defined(__clang__) +# define GGML_RESTRICT __restrict +# elif defined(_MSC_VER) +# define GGML_RESTRICT __restrict +# else +# define GGML_RESTRICT +# endif +#else +# if defined (_MSC_VER) && (__STDC_VERSION__ < 201112L) +# define GGML_RESTRICT __restrict +# else +# define GGML_RESTRICT restrict +# endif +#endif + typedef void (*ggml_to_float_t) (const void * GGML_RESTRICT x, float * GGML_RESTRICT y, int64_t k); + typedef void (*ggml_from_float_t)(const float * GGML_RESTRICT x, void * GGML_RESTRICT y, int64_t k); + + struct ggml_type_traits { + const char * type_name; + int64_t blck_size; + int64_t blck_size_interleave; // interleave elements in blocks + size_t type_size; + bool is_quantized; + ggml_to_float_t to_float; + ggml_from_float_t from_float_ref; + }; + + GGML_API const struct ggml_type_traits * ggml_get_type_traits(enum ggml_type type); + + // ggml threadpool + // TODO: currently, only a few functions are in the base ggml API, while the rest are in the CPU backend + // the goal should be to create an API that other backends can use move everything to the ggml base + + // scheduling priorities + enum ggml_sched_priority { + GGML_SCHED_PRIO_LOW = -1, + GGML_SCHED_PRIO_NORMAL, + GGML_SCHED_PRIO_MEDIUM, + GGML_SCHED_PRIO_HIGH, + GGML_SCHED_PRIO_REALTIME + }; + + // threadpool params + // Use ggml_threadpool_params_default() or ggml_threadpool_params_init() to populate the defaults + struct ggml_threadpool_params { + bool cpumask[GGML_MAX_N_THREADS]; // mask of cpu cores (all-zeros means use default affinity settings) + int n_threads; // number of threads + enum ggml_sched_priority prio; // thread priority + uint32_t poll; // polling level (0 - no polling, 100 - aggressive polling) + bool strict_cpu; // strict cpu placement + bool paused; // start in paused state + }; + + struct ggml_threadpool; // forward declaration, see ggml.c + + typedef struct ggml_threadpool * ggml_threadpool_t; + + GGML_API struct ggml_threadpool_params ggml_threadpool_params_default(int n_threads); + GGML_API void ggml_threadpool_params_init (struct ggml_threadpool_params * p, int n_threads); + GGML_API bool ggml_threadpool_params_match (const struct ggml_threadpool_params * p0, const struct ggml_threadpool_params * p1); + +#ifdef __cplusplus +} +#endif diff --git a/python/freetoken/kernel/csrc/gguf_mmq/gguf.h b/python/freetoken/kernel/csrc/gguf_mmq/gguf.h new file mode 100644 index 00000000..b3a1e123 --- /dev/null +++ b/python/freetoken/kernel/csrc/gguf_mmq/gguf.h @@ -0,0 +1,211 @@ +// This file contains functionality related to "GGUF" files, the binary file format used by ggml. +// GGUF files have the following structure: +// +// 1. File magic "GGUF" (4 bytes). +// 2. File version (uint32_t). +// 3. Number of ggml tensors in file (int64_t). +// 4. Number of key-value-pairs in file (int64_t). +// 5. For each KV pair: +// 1. The key (string). +// 2. The value type (gguf_type). +// 3a. If the value type is GGUF_TYPE_ARRAY: +// 1. The type of the array (gguf_type). +// 2. The number of elements in the array (uint64_t). +// 3. The binary representation of each element in the array. +// 3b. Otherwise: +// 1. The binary representation of the value. +// 6. For each ggml tensor: +// 1. The tensor name (string). +// 2. The number of dimensions of the tensor (uint32_t). +// 3. For each dimension: +// 1. The size of the tensor in the dimension (int64_t). +// 4. The tensor data type (ggml_type). +// 5. The tensor data offset in the tensor data binary blob (uint64_t). +// 7. The tensor data binary blob (optional, aligned). +// +// Strings are serialized as the string length (uint64_t) followed by the C string without the null terminator. +// All enums are stored as int32_t. +// All bool values are stored as int8_t. +// If the special key "general.alignment" (uint32_t) is defined it is used for alignment, +// otherwise GGUF_DEFAULT_ALIGNMENT is used. +// +// Module maintainer: Johannes Gäßler (@JohannesGaessler, johannesg@5d6.de) + +#pragma once + +#include "ggml.h" + +#include +#include + +#define GGUF_MAGIC "GGUF" +#define GGUF_VERSION 3 + +#define GGUF_KEY_GENERAL_ALIGNMENT "general.alignment" + +#define GGUF_DEFAULT_ALIGNMENT 32 + +#ifdef __cplusplus +extern "C" { +#endif + + // types that can be stored as GGUF KV data + enum gguf_type { + GGUF_TYPE_UINT8 = 0, + GGUF_TYPE_INT8 = 1, + GGUF_TYPE_UINT16 = 2, + GGUF_TYPE_INT16 = 3, + GGUF_TYPE_UINT32 = 4, + GGUF_TYPE_INT32 = 5, + GGUF_TYPE_FLOAT32 = 6, + GGUF_TYPE_BOOL = 7, + GGUF_TYPE_STRING = 8, + GGUF_TYPE_ARRAY = 9, + GGUF_TYPE_UINT64 = 10, + GGUF_TYPE_INT64 = 11, + GGUF_TYPE_FLOAT64 = 12, + GGUF_TYPE_COUNT, // marks the end of the enum + }; + + struct gguf_context; + + struct gguf_init_params { + bool no_alloc; + + // if not NULL, create a ggml_context and allocate the tensor data in it + struct ggml_context ** ctx; + }; + + // callback to simulate or wrap a FILE pointer - read up to `len` bytes at `offset` into `output` and return the number of bytes read + typedef size_t (*gguf_reader_callback_t)(void * userdata, void * output, uint64_t offset, size_t len); + + GGML_API struct gguf_context * gguf_init_empty(void); + GGML_API struct gguf_context * gguf_init_from_file_ptr(FILE * file, struct gguf_init_params params); + GGML_API struct gguf_context * gguf_init_from_file(const char * fname, struct gguf_init_params params); + GGML_API struct gguf_context * gguf_init_from_buffer(const void * data, size_t size, struct gguf_init_params params); + + // max_chunk_read is the maximum number of bytes that the GGUF code will read at once from the callback, a value of 0 means no limit + GGML_API struct gguf_context * gguf_init_from_callback(gguf_reader_callback_t callback, void * userdata, size_t max_chunk_read, uint64_t max_expected_size, struct gguf_init_params params); + + GGML_API void gguf_free(struct gguf_context * ctx); + + GGML_API const char * gguf_type_name(enum gguf_type type); + + GGML_API uint32_t gguf_get_version (const struct gguf_context * ctx); + GGML_API size_t gguf_get_alignment (const struct gguf_context * ctx); + GGML_API size_t gguf_get_data_offset(const struct gguf_context * ctx); // padded to gguf_get_alignment if and only if the gguf_context contains at least one tensor + + GGML_API int64_t gguf_get_n_kv(const struct gguf_context * ctx); + GGML_API int64_t gguf_find_key(const struct gguf_context * ctx, const char * key); // returns -1 if key is not found + GGML_API const char * gguf_get_key (const struct gguf_context * ctx, int64_t key_id); + + GGML_API enum gguf_type gguf_get_kv_type (const struct gguf_context * ctx, int64_t key_id); + GGML_API enum gguf_type gguf_get_arr_type(const struct gguf_context * ctx, int64_t key_id); + + // will abort if the wrong type is used for the key + GGML_API uint8_t gguf_get_val_u8 (const struct gguf_context * ctx, int64_t key_id); + GGML_API int8_t gguf_get_val_i8 (const struct gguf_context * ctx, int64_t key_id); + GGML_API uint16_t gguf_get_val_u16 (const struct gguf_context * ctx, int64_t key_id); + GGML_API int16_t gguf_get_val_i16 (const struct gguf_context * ctx, int64_t key_id); + GGML_API uint32_t gguf_get_val_u32 (const struct gguf_context * ctx, int64_t key_id); + GGML_API int32_t gguf_get_val_i32 (const struct gguf_context * ctx, int64_t key_id); + GGML_API float gguf_get_val_f32 (const struct gguf_context * ctx, int64_t key_id); + GGML_API uint64_t gguf_get_val_u64 (const struct gguf_context * ctx, int64_t key_id); + GGML_API int64_t gguf_get_val_i64 (const struct gguf_context * ctx, int64_t key_id); + GGML_API double gguf_get_val_f64 (const struct gguf_context * ctx, int64_t key_id); + GGML_API bool gguf_get_val_bool(const struct gguf_context * ctx, int64_t key_id); + GGML_API const char * gguf_get_val_str (const struct gguf_context * ctx, int64_t key_id); + GGML_API const void * gguf_get_val_data(const struct gguf_context * ctx, int64_t key_id); + GGML_API size_t gguf_get_arr_n (const struct gguf_context * ctx, int64_t key_id); + + // get raw pointer to the first element of the array with the given key_id + // for bool arrays, note that they are always stored as int8 on all platforms (usually this makes no difference) + GGML_API const void * gguf_get_arr_data(const struct gguf_context * ctx, int64_t key_id); + + // get ith C string from array with given key_id + GGML_API const char * gguf_get_arr_str (const struct gguf_context * ctx, int64_t key_id, size_t i); + + GGML_API int64_t gguf_get_n_tensors (const struct gguf_context * ctx); + GGML_API int64_t gguf_find_tensor (const struct gguf_context * ctx, const char * name); // returns -1 if the tensor is not found + GGML_API size_t gguf_get_tensor_offset(const struct gguf_context * ctx, int64_t tensor_id); + GGML_API const char * gguf_get_tensor_name (const struct gguf_context * ctx, int64_t tensor_id); + GGML_API const int64_t * gguf_get_tensor_ne (const struct gguf_context * ctx, int64_t tensor_id); // returns ne, an array of GGML_MAX_DIMS elements; ne[dim] is 1 for dim >= n_dims + GGML_API enum ggml_type gguf_get_tensor_type (const struct gguf_context * ctx, int64_t tensor_id); + GGML_API size_t gguf_get_tensor_size (const struct gguf_context * ctx, int64_t tensor_id); + + // removes key if it exists, returns id that the key had prior to removal (-1 if it didn't exist) + GGML_API int64_t gguf_remove_key(struct gguf_context * ctx, const char * key); + + // overrides an existing KV pair or adds a new one, the new KV pair is always at the back + GGML_API void gguf_set_val_u8 (struct gguf_context * ctx, const char * key, uint8_t val); + GGML_API void gguf_set_val_i8 (struct gguf_context * ctx, const char * key, int8_t val); + GGML_API void gguf_set_val_u16 (struct gguf_context * ctx, const char * key, uint16_t val); + GGML_API void gguf_set_val_i16 (struct gguf_context * ctx, const char * key, int16_t val); + GGML_API void gguf_set_val_u32 (struct gguf_context * ctx, const char * key, uint32_t val); + GGML_API void gguf_set_val_i32 (struct gguf_context * ctx, const char * key, int32_t val); + GGML_API void gguf_set_val_f32 (struct gguf_context * ctx, const char * key, float val); + GGML_API void gguf_set_val_u64 (struct gguf_context * ctx, const char * key, uint64_t val); + GGML_API void gguf_set_val_i64 (struct gguf_context * ctx, const char * key, int64_t val); + GGML_API void gguf_set_val_f64 (struct gguf_context * ctx, const char * key, double val); + GGML_API void gguf_set_val_bool(struct gguf_context * ctx, const char * key, bool val); + GGML_API void gguf_set_val_str (struct gguf_context * ctx, const char * key, const char * val); + + // creates a new array with n elements of the given type and copies the corresponding number of bytes from data + GGML_API void gguf_set_arr_data(struct gguf_context * ctx, const char * key, enum gguf_type type, const void * data, size_t n); + + // creates a new array with n strings and copies the corresponding strings from data + GGML_API void gguf_set_arr_str (struct gguf_context * ctx, const char * key, const char ** data, size_t n); + + // set or add KV pairs from another context + GGML_API void gguf_set_kv(struct gguf_context * ctx, const struct gguf_context * src); + + // add tensor to GGUF context, tensor name must be unique + GGML_API void gguf_add_tensor(struct gguf_context * ctx, const struct ggml_tensor * tensor); + + // after changing a tensor's type, the offsets of all tensors with higher indices are immediately recalculated + // in such a way that the tensor data remains as one contiguous block (except for padding) + GGML_API void gguf_set_tensor_type(struct gguf_context * ctx, const char * name, enum ggml_type type); + + // assumes that at least gguf_get_tensor_size bytes can be read from data + GGML_API void gguf_set_tensor_data(struct gguf_context * ctx, const char * name, const void * data); + + // writing gguf files can be done in 3 ways: + // + // - write the entire gguf_context to a binary file in a single pass: + // + // gguf_write_to_file(ctx, fname, /*only_meta =*/ false); + // + // - write only the meta data to a file, then re-open the file and append the tensor data: + // + // gguf_write_to_file(ctx, fname, /*only_meta =*/ true); + // FILE * f = fopen(fname, "ab"); + // fwrite(f, ...); // write tensor data + // fclose(f); + // + // - first prepare a file with a placeholder for the meta data, write the tensor data, then write the meta data: + // + // FILE * f = fopen(fname, "wb"); + // const size_t size_meta = gguf_get_meta_size(ctx); + // fseek(f, size_meta, SEEK_SET); + // fwrite(f, ...); // write tensor data + // void * data = malloc(size_meta); + // gguf_get_meta_data(ctx, data); + // rewind(f); + // fwrite(data, 1, data, f); + // free(data); + // fclose(f); + // + + // write the entire context to a binary file + GGML_API bool gguf_write_to_file_ptr(const struct gguf_context * ctx, FILE * file, bool only_meta); + GGML_API bool gguf_write_to_file(const struct gguf_context * ctx, const char * fname, bool only_meta); + + // get the size in bytes of the meta data (header, kv pairs, tensor info) including padding + GGML_API size_t gguf_get_meta_size(const struct gguf_context * ctx); + + // writes the meta data to pointer "data" + GGML_API void gguf_get_meta_data(const struct gguf_context * ctx, void * data); + +#ifdef __cplusplus +} +#endif diff --git a/python/freetoken/kernel/csrc/gguf_mmq/mma.cuh b/python/freetoken/kernel/csrc/gguf_mmq/mma.cuh new file mode 100644 index 00000000..8d7c69dc --- /dev/null +++ b/python/freetoken/kernel/csrc/gguf_mmq/mma.cuh @@ -0,0 +1,1456 @@ +#pragma once +// This file contains primitives that expose the tensor core PTX instructions for CUDA code. +// The primitives can be used in a similar way as the nvcuda::wmma interface but with a well-defined memory layout. +// The documentation for the PTX instructions can be found under: +// https://docs.nvidia.com/cuda/parallel-thread-execution/index.html#matrix-multiply-accumulate-operation-using-mma-instruction +// +// Like with nvcuda::wmma there are three types of matrix tiles: A, B, and C with A @ B = C. +// A is a row-major matrix with shape M x K. +// B is a column-major matrix with shape K x N. +// C is a column-major matrix with shape M x N. +// A, B, and C are represented using the same fundamental data type: a row-major matrix with I rows and J columns. +// Note that J is measured in physical 32 bit elements instead of logical elements. +// The methods get_i and get_j can be used to get the physical 32 bit index of the lth element of a thread within a tile. +// All matrix tiles have ne physical 32 bit elements per warp. +// +// As described in the PTX documentation, all pointers for load_ldmatrix must be to shared memory and aligned to 16 bytes. +// The API in this file also assumes that the pointers for load_generic are aligned to 16 bytes, unaligned pointers are considered undefined behavior. + +#include "common.cuh" + +// On Volta each warp is doing 4 8x8 mma operations in parallel. +// The basic memory layout for a 32x8 output tile is to stack 4 input tiles in I direction and to mirror the B tile. +// However, the i indices in this file are by default permuted to simplify the index calculations. +// #define GGML_CUDA_MMA_NO_VOLTA_PERM + +#if CUDART_VERSION >= 11080 + +static __device__ __forceinline__ int ggml_cuda_movmatrix(const int x) { + int ret = 0; + +#ifdef TURING_MMA_AVAILABLE + asm("movmatrix.sync.aligned.m8n8.trans.b16 %0, %1;" + : "=r"(ret) : "r"(x)); +#else + GGML_UNUSED(x); + NO_DEVICE_CODE; +#endif // defined(TURING_MMA_AVAILABLE) + return ret; +} + +#else + +static __device__ __forceinline__ int ggml_cuda_movmatrix(const int x) { + // Imagine transposing row-major matrix to column-major matrix. + const int src_i_low = 2 * (threadIdx.x % 4); + const int src_i_high = src_i_low + 1; + const int src_j = threadIdx.x / 4; + + const int src_laneid_low = src_i_low * 4 + src_j / 2; + const int src_laneid_high = src_i_high * 4 + src_j / 2; + + const int shift_low = ((src_j + 0) % 2) * 16; + const int shift_high = ((src_j + 1) % 2) * 16; + + const int ret_low = (__shfl_sync(0xFFFFFFFF, x, src_laneid_low, WARP_SIZE) >> shift_low) & 0x0000FFFF; + const int ret_high = (__shfl_sync(0xFFFFFFFF, x, src_laneid_high, WARP_SIZE) << shift_high) & 0xFFFF0000; + + return ret_low | ret_high; +} + +#endif // CUDART_VERSION >= 11080 + +static __device__ __forceinline__ half2 ggml_cuda_movmatrix(const half2 x) { + half2 ret; + *((int *) &ret) = ggml_cuda_movmatrix(*((const int *) &x)); + return ret; +} + +namespace ggml_cuda_mma { + + // Some architectures like Volta or CDNA3 perform multiple matrix multiplications per warp in parallel, + // effectively the warp is being split into subgroups of threads that each perform a single mma instruction. + // In those cases the data can be split in different ways across the warp. + enum data_layout { + // By default the data uses the I direction as its major dimension and the J direction as its minor dimension. + // For the A/C matrices this means I major == row major, J major == column major. + // For the B matrix this means I major == column major, J major == row major. + // MIRRORED == Each data value is held exactly once per thread subgroup. + DATA_LAYOUT_I_MAJOR = 0, // Always used for Turing, Ampere, Ada Lovelace, consumer Blackwell, matrix A&B for RDNA4 and CDNA. + DATA_LAYOUT_J_MAJOR = 10, // Matrix C for CDNA and RDNA4, int and float matrix C for RDNA3. + DATA_LAYOUT_I_MAJOR_MIRRORED = 20, // Volta, matrix A&B for RDNA3. + DATA_LAYOUT_J_MAJOR_MIRRORED = 30, + DATA_LAYOUT_I_MAJOR_SCRAMBLED = 40, // Scrambled matrix C for faster transposition (RDNA4/CDNA), convert to float to unscramble. + }; + // Implemented mma combinations are: + // - (I_MAJOR, I_MAJOR) -> I_MAJOR + // - (I_MAJOR, I_MAJOR_MIRRORED) -> I_MAJOR + // - (I_MAJOR, J_MAJOR_MIRRORED) -> I_MAJOR + + static constexpr __device__ data_layout get_input_data_layout() { +#if defined(RDNA3) || defined(VOLTA_MMA_AVAILABLE) + return DATA_LAYOUT_I_MAJOR_MIRRORED; +#else + return DATA_LAYOUT_I_MAJOR; +#endif // defined(RDNA3) || defined(VOLTA_MMA_AVAILABLE) + } + + template + struct tile {}; + + template + struct tile { + static constexpr int I = I_; + static constexpr int J = J_; + static constexpr data_layout dl = DATA_LAYOUT_I_MAJOR; + +#if defined(AMD_MFMA_AVAILABLE) + static constexpr int ne = I * J / 64; + T x[ne] = {0}; + + static constexpr __device__ bool supported() { + if (I == 16 && J == 8) return true; + if (I == 32 && J == 4) return true; + if (I == 16 && J == 16) return true; + if (I == 32 && J == 32) return true; + return false; + } + + static __device__ __forceinline__ int get_i(const int l) { + if constexpr (I == 16 && J == 4) { + return threadIdx.x % 16; + } else if constexpr (I == 16 && J == 8) { + return threadIdx.x % 16; + } else if constexpr (I == 32 && J == 4) { + return threadIdx.x % 32; + } else if constexpr (I == 16 && J == 16) { + return threadIdx.x % 16; + } else if constexpr (I == 32 && J == 32) { + return threadIdx.x % 32; + } else { + NO_DEVICE_CODE; + return -1; + } + } + + static __device__ __forceinline__ int get_j(const int l) { + if constexpr (I == 16 && J == 4) { + return threadIdx.x / 16; + } else if constexpr (I == 16 && J == 8) { + return 2 * (threadIdx.x / 16) + l; + } else if constexpr (I == 32 && J == 4) { + return 2 * (threadIdx.x / 32) + l; + } else if constexpr (I == 16 && J == 16) { + return 4 * (threadIdx.x / 16) + l; + } else if constexpr (I == 32 && J == 32) { + return 4 * (threadIdx.x / 32) + 8 * (l / 4) + (l % 4); + } else { + NO_DEVICE_CODE; + return -1; + } + } +#elif defined(VOLTA_MMA_AVAILABLE) + static constexpr int ne = I * J / 32; + T x[ne] = {0}; + + static constexpr __device__ bool supported() { + if (I == 32 && J == 8) return true; + return false; + } + + static __device__ __forceinline__ int get_i(const int l) { + if constexpr (I == 32 && J == 8) { +#ifdef GGML_CUDA_MMA_NO_VOLTA_PERM + return (((threadIdx.x % 16) / 4) * 8) + ((threadIdx.x / 16) * 4) + (l & 2) + (threadIdx.x % 2); +#else + return (l & 2) + (threadIdx.x & ~2); +#endif // GGML_CUDA_MMA_NO_VOLTA_PERM + } else { + NO_DEVICE_CODE; + return -1; + } + } + + static __device__ __forceinline__ int get_j(const int l) { + if constexpr (I == 32 && J == 8) { + return (threadIdx.x & 2) + (l & (4 + 1)); + } else { + NO_DEVICE_CODE; + return -1; + } + } +#elif defined(AMD_WMMA_AVAILABLE) + static constexpr int ne = I * J / 32; + T x[ne] = {0}; + + static constexpr __device__ bool supported() { + if (I == 16 && J == 16) return true; + if (I == 16 && J == 8) return true; + if (I == 16 && J == 4) return true; + return false; + } + + static __device__ __forceinline__ int get_i(const int l) { + if constexpr (supported()) { + return threadIdx.x % 16; + } else { + NO_DEVICE_CODE; + return -1; + } + } + + static __device__ __forceinline__ int get_j(const int l) { + if constexpr (I == 16 && J == 16) { +#if defined(RDNA3) + if constexpr (std::is_same_v || std::is_same_v) { + // matrix C + return 2 * l + (threadIdx.x / 16); + } else { + // matrix A&B + return l; + } +#else + // matrix C is the transposed matrix A&B on RDNA4 + return ne * (threadIdx.x / 16) + l; +#endif // defined(RDNA3) + } else if constexpr (I == 16 && J == 8) { + // mmq input for RDNA4 + return ne * (threadIdx.x / 16) + l; + } else if constexpr (I == 16 && J == 4) { + return ne * (threadIdx.x / 16) + l; + } else { + NO_DEVICE_CODE; + return -1; + } + } +#else + static constexpr int ne = I * J / 32; + T x[ne] = {0}; + + static constexpr __device__ bool supported() { + if (I == 8 && J == 4) return true; + if (I == 8 && J == 8) return true; + if (I == 16 && J == 8) return true; + if (I == 16 && J == 16) return true; + if (I == 32 && J == 8) return true; + return false; + } + + static __device__ __forceinline__ int get_i(const int l) { + if constexpr (I == 8 && J == 4) { + return threadIdx.x / 4; + } else if constexpr (I == 8 && J == 8) { + return threadIdx.x / 4; + } else if constexpr (I == 16 && J == 8) { + return ((l / 2) * 8) + (threadIdx.x / 4); + } else if constexpr (I == 16 && J == 16) { + return (((l / 2) % 2) * 8) + (threadIdx.x / 4); + } else if constexpr (I == 32 && J == 8) { + return tile<16, 8, T>::get_i(l); // Memory layout simply repeated with same pattern in i direction. + } else { + NO_DEVICE_CODE; + return -1; + } + } + + static __device__ __forceinline__ int get_j(const int l) { + if constexpr (I == 8 && J == 4) { + return threadIdx.x % 4; + } else if constexpr (I == 8 && J == 8) { + return (l * 4) + (threadIdx.x % 4); + } else if constexpr (I == 16 && J == 8) { + return ((threadIdx.x % 4) * 2) + (l % 2); + } else if constexpr (I == 16 && J == 16) { + return ((l / 4) * 8) + ((threadIdx.x % 4) * 2) + (l % 2); + } else if constexpr (I == 32 && J == 8) { + return tile<16, 8, T>::get_j(l); // Memory layout simply repeated with same pattern in i direction. + } else { + NO_DEVICE_CODE; + return -1; + } + } +#endif // defined(GGML_USE_HIP) + }; + + template + struct tile { + static constexpr int I = I_; + static constexpr int J = J_; + static constexpr data_layout dl = DATA_LAYOUT_I_MAJOR; + +#if defined(VOLTA_MMA_AVAILABLE) + static constexpr int ne = I * J / WARP_SIZE; + half2 x[ne] = {{0.0f, 0.0f}}; + + static constexpr __device__ bool supported() { + if (I == 32 && J == 4) return true; + return false; + } + + static __device__ __forceinline__ int get_i(const int l) { + if constexpr (I == 32 && J == 4) { +#ifdef GGML_CUDA_MMA_NO_VOLTA_PERM + return (((threadIdx.x % 16) / 4) * 8) + ((threadIdx.x / 16) * 4) + (threadIdx.x % 4); +#else + return threadIdx.x; +#endif // GGML_CUDA_MMA_NO_VOLTA_PERM + } else { + NO_DEVICE_CODE; + return -1; + } + } + + static __device__ __forceinline__ int get_j(const int l) { + if constexpr (I == 32 && J == 4) { + return l; + } else { + NO_DEVICE_CODE; + return -1; + } + } +#elif defined(AMD_WMMA_AVAILABLE) + static constexpr int ne = I * J / 32; + half2 x[ne] = {{0.0f, 0.0f}}; + + static constexpr __device__ bool supported() { + if (I == 16 && J == 8) return true; + if (I == 16 && J == 16) return true; + if (I == 32 && J == 8) return true; + return false; + } + + static __device__ __forceinline__ int get_i(const int l) { + if constexpr (I == 16 && J == 8) { + return threadIdx.x % 16; + } else if constexpr (I == 16 && J == 16) { + return threadIdx.x % 16; + } else if constexpr (I == 32 && J == 8) { + return (threadIdx.x % 16) * 2 + l / (ne/2); + } else { + NO_DEVICE_CODE; + return -1; + } + } + + static __device__ __forceinline__ int get_j(const int l) { + if constexpr (I == 16 && J == 8) { + return (threadIdx.x / 16) * ne + l; + } else if constexpr (I == 16 && J == 16) { +#ifdef RDNA3 + return l*2 + (threadIdx.x / 16); +#else + return (threadIdx.x / 16) * ne + l; +#endif // RDNA3 + } else if constexpr (I == 32 && J == 8) { + return (threadIdx.x / 16) * (ne/2) + l % (ne/2); + } else { + NO_DEVICE_CODE; + return -1; + } + } +#elif defined(AMD_MFMA_AVAILABLE) + static constexpr int ne = I * J / 64; + half2 x[ne] = {{0.0f, 0.0f}}; + + static constexpr __device__ bool supported() { + if (I == 16 && J == 8) return true; + if (I == 16 && J == 16) return true; + if (I == 32 && J == 8) return true; + return false; + } + + static __device__ __forceinline__ int get_i(const int l) { + if constexpr (I == 16 && J == 8) { + return threadIdx.x % 16; + } else if constexpr (I == 16 && J == 16) { + return threadIdx.x % 16; + } else if constexpr (I == 32 && J == 8) { + return (threadIdx.x % 16) * 2 + l / (ne/2); + } else { + NO_DEVICE_CODE; + return -1; + } + } + + static __device__ __forceinline__ int get_j(const int l) { + if constexpr (I == 16 && J == 8) { + return (threadIdx.x / 16) * ne + l; + } else if constexpr (I == 16 && J == 16) { + return (threadIdx.x / 16) * ne + l; + } else if constexpr (I == 32 && J == 8) { + return (threadIdx.x / 16) * (ne/2) + l % (ne/2); + } else { + NO_DEVICE_CODE; + return -1; + } + } +#else + static constexpr int ne = I * J / WARP_SIZE; + half2 x[ne] = {{0.0f, 0.0f}}; + + static constexpr __device__ bool supported() { + if (I == 8 && J == 4) return true; + if (I == 8 && J == 8) return true; + if (I == 16 && J == 8) return true; + if (I == 16 && J == 16) return true; + if (I == 32 && J == 8) return true; + return false; + } + + static __device__ __forceinline__ int get_i(const int l) { + if constexpr (I == 8 && J == 8) { + return threadIdx.x / 4; + } else if constexpr (I == 16 && J == 4) { + return (l * 8) + (threadIdx.x / 4); + } else if constexpr (I == 16 && J == 8) { + return ((l % 2) * 8) + (threadIdx.x / 4); + } else if constexpr (I == 32 && J == 8) { + return ((l / 4) * 16) + ((l % 2) * 8) + (threadIdx.x / 4); + } else { + NO_DEVICE_CODE; + return -1; + } + } + + static __device__ __forceinline__ int get_j(const int l) { + if constexpr (I == 8 && J == 8) { + return (l * 4) + (threadIdx.x % 4); + } else if constexpr (I == 16 && J == 4) { + return threadIdx.x % 4; + } else if constexpr (I == 16 && J == 8) { + return ((l / 2) * 4) + (threadIdx.x % 4); + } else if constexpr (I == 32 && J == 8) { + return ((l & 2) * 2) + (threadIdx.x % 4); + } else { + NO_DEVICE_CODE; + return -1; + } + } +#endif // defined(VOLTA_MMA_AVAILABLE) + }; + + template + struct tile { + static constexpr int I = I_; + static constexpr int J = J_; + static constexpr data_layout dl = DATA_LAYOUT_I_MAJOR; + +#if defined(AMD_WMMA_AVAILABLE) + static constexpr int ne = tile::ne; + nv_bfloat162 x[ne] = {{0.0f, 0.0f}}; + + static constexpr __device__ bool supported() { + return tile::supported(); + } + + static __device__ __forceinline__ int get_i(const int l) { + return tile::get_i(l); + } + + static __device__ __forceinline__ int get_j(const int l) { + return tile::get_j(l); + } +#elif defined(AMD_MFMA_AVAILABLE) + static constexpr int ne = tile::ne; + nv_bfloat162 x[ne] = {{0.0f, 0.0f}}; + + static constexpr __device__ bool supported() { + return tile::supported(); + } + + static __device__ __forceinline__ int get_i(const int l) { + return tile::get_i(l); + } + + static __device__ __forceinline__ int get_j(const int l) { + return tile::get_j(l); + } +#else + static constexpr int ne = I * J / WARP_SIZE; + nv_bfloat162 x[ne] = {{0.0f, 0.0f}}; + + static constexpr __device__ bool supported() { + if (I == 8 && J == 8) return true; + if (I == 16 && J == 4) return true; + if (I == 16 && J == 8) return true; + return false; + } + + static __device__ __forceinline__ int get_i(const int l) { + if constexpr (I == 8 && J == 8) { + return threadIdx.x / 4; + } else if constexpr (I == 16 && J == 4) { + return (l * 8) + (threadIdx.x / 4); + } else if constexpr (I == 16 && J == 8) { + return ((l % 2) * 8) + (threadIdx.x / 4); + } else { + NO_DEVICE_CODE; + return -1; + } + } + + static __device__ __forceinline__ int get_j(const int l) { + if constexpr (I == 8 && J == 8) { + return (l * 4) + (threadIdx.x % 4); + } else if constexpr (I == 16 && J == 4) { + return threadIdx.x % 4; + } else if constexpr (I == 16 && J == 8) { + return ((l / 2) * 4) + (threadIdx.x % 4); + } else { + NO_DEVICE_CODE; + return -1; + } + } +#endif // defined(AMD_WMMA_AVAILABLE) + }; + + template + struct tile { + static constexpr int I = I_; + static constexpr int J = J_; + static constexpr data_layout dl = DATA_LAYOUT_J_MAJOR; + + static constexpr int ne = tile::ne; + T x[ne] = {0}; + + static constexpr __device__ bool supported() { + return tile::supported(); + } + + static __device__ __forceinline__ int get_i(const int l) { + return tile::get_j(l); + } + + static __device__ __forceinline__ int get_j(const int l) { + return tile::get_i(l); + } + }; + + template + struct tile { + static constexpr int I = I_; + static constexpr int J = J_; + static constexpr data_layout dl = DATA_LAYOUT_I_MAJOR_MIRRORED; + + // RDNA3 + static constexpr int ne = I * J / 32 * 2; + + T x[ne] = {0}; + + static constexpr __device__ bool supported() { + if (I == 16 && J == 16) return true; + if (I == 16 && J == 8) return true; + if (I == 16 && J == 4) return true; + if (I == 32 && J == 8) return true; + return false; + } + + static __device__ __forceinline__ int get_i(const int l) { + if constexpr (I == 16) { + return threadIdx.x % 16; + } else if constexpr (I == 32) { + return (threadIdx.x % 16) * 2 + l / (ne/2); + } else { + NO_DEVICE_CODE; + return -1; + } + } + + static __device__ __forceinline__ int get_j(const int l) { + if constexpr (I == 16) { + return l; + } else if constexpr (I == 32) { + return l % (ne/2); + } else { + NO_DEVICE_CODE; + return -1; + } + } + }; + + template + struct tile { + static constexpr int I = I_; + static constexpr int J = J_; + static constexpr data_layout dl = DATA_LAYOUT_I_MAJOR_MIRRORED; +#if defined(RDNA3) + static constexpr int ne = tile::ne; + + half2 x[ne] = {{0.0f, 0.0f}}; + + static constexpr __device__ bool supported() { + return tile::supported(); + } + + static __device__ __forceinline__ int get_i(const int l) { + return tile::get_i(l); + } + + static __device__ __forceinline__ int get_j(const int l) { + return tile::get_j(l); + } +#else // Volta + static constexpr int ne = I * J / (WARP_SIZE/4); + + half2 x[ne] = {{0.0f, 0.0f}}; + + static constexpr __device__ bool supported() { + if (I == 8 && J == 4) return true; + return false; + } + + static __device__ __forceinline__ int get_i(const int /*l*/) { + if constexpr (I == 8 && J == 4) { + return ((threadIdx.x / 16) * 4) + (threadIdx.x % 4); + } else { + NO_DEVICE_CODE; + return -1; + } + } + + static __device__ __forceinline__ int get_j(const int l) { + if constexpr (I == 8 && J == 4) { + return l; + } else { + NO_DEVICE_CODE; + return -1; + } + } +#endif // defined(RDNA3) + }; + + template + struct tile { + static constexpr int I = I_; + static constexpr int J = J_; + static constexpr data_layout dl = DATA_LAYOUT_I_MAJOR_MIRRORED; + static constexpr int ne = tile::ne; + + nv_bfloat162 x[ne] = {{0.0f, 0.0f}}; + + static constexpr __device__ bool supported() { + return tile::supported(); + } + + static __device__ __forceinline__ int get_i(const int l) { + return tile::get_i(l); + } + + static __device__ __forceinline__ int get_j(const int l) { + return tile::get_j(l); + } + }; + + template + struct tile { + static constexpr int I = I_; + static constexpr int J = J_; + static constexpr data_layout dl = DATA_LAYOUT_J_MAJOR_MIRRORED; + static constexpr int ne = I * J / (WARP_SIZE/4); + + half2 x[ne] = {{0.0f, 0.0f}}; + + static constexpr __device__ bool supported() { + if (I == 8 && J == 4) return true; + return false; + } + + static __device__ __forceinline__ int get_i(const int l) { + if constexpr (I == 8 && J == 4) { + return ((l / 2) * 4) + (threadIdx.x % 4); + } else { + NO_DEVICE_CODE; + return -1; + } + } + + static __device__ __forceinline__ int get_j(const int l) { + if constexpr (I == 8 && J == 4) { + return ((threadIdx.x / 16) * 2) + (l % 2); + } else { + NO_DEVICE_CODE; + return -1; + } + } + }; + + template + struct tile { + static constexpr int I = I_; + static constexpr int J = J_; + static constexpr data_layout dl = DATA_LAYOUT_I_MAJOR_SCRAMBLED; + + static constexpr int ne = I * J / ggml_cuda_get_physical_warp_size(); + half2 x[ne] = {{0.0f, 0.0f}}; + + static constexpr __device__ bool supported() { + if (I == 16 && J == 16) return true; + return false; + } + + static __device__ __forceinline__ int get_i(const int l) { + return tile::get_i(l); + } + }; + + static __device__ __forceinline__ tile<16, 16, half2, DATA_LAYOUT_I_MAJOR> unscramble(const tile<16, 16, half2, DATA_LAYOUT_I_MAJOR_SCRAMBLED> & t) { +#if defined(AMD_MFMA_AVAILABLE) || (defined(AMD_WMMA_AVAILABLE) && defined(RDNA4)) + tile<16, 16, half2, DATA_LAYOUT_I_MAJOR> ret; +#pragma unroll + for (int l0 = 0; l0 < t.ne/2; ++l0) { + ret.x[2*l0 + 0] = __lows2half2(t.x[l0], t.x[l0 + t.ne/2]); + ret.x[2*l0 + 1] = __highs2half2(t.x[l0], t.x[l0 + t.ne/2]); + } + return ret; +#else + NO_DEVICE_CODE; + GGML_UNUSED(t); +#endif // defined(AMD_MFMA_AVAILABLE) || (defined(AMD_WMMA_AVAILABLE) && defined(RDNA4)) + } + +#if defined(TURING_MMA_AVAILABLE) + template + static __device__ __forceinline__ tile get_half2(const tile & tile_float) { + tile ret; +#pragma unroll + for (int l0 = 0; l0 < tile_float.ne; l0 += 2) { + ret.x[l0/2] = make_half2(tile_float.x[l0 + 0], tile_float.x[l0 + 1]); + } + return ret; + } + + static __device__ __forceinline__ tile<8, 8, half2> get_transposed(const tile<16, 4, half2> & t) { + tile<8, 8, half2> ret; + ret.x[0] = ggml_cuda_movmatrix(t.x[0]); + ret.x[1] = ggml_cuda_movmatrix(t.x[1]); + + return ret; + } +#elif defined(AMD_WMMA_AVAILABLE) && defined(RDNA3) + static __device__ __forceinline__ tile<16, 8, half2, DATA_LAYOUT_I_MAJOR_MIRRORED> get_half2( + const tile<16, 16, float, DATA_LAYOUT_I_MAJOR> & tile_float) { + tile<16, 8, half2, DATA_LAYOUT_I_MAJOR_MIRRORED> ret; +#pragma unroll + for (int l = 0; l < tile_float.ne; ++l) { + float tmp[2]; + int i = threadIdx.x / 16; + tmp[i] = tile_float.x[l]; + i ^= 1; + tmp[i] = __shfl_xor_sync(0xFFFFFFFF, tile_float.x[l], 16, WARP_SIZE); + ret.x[l] = make_half2(tmp[0], tmp[1]); + } + return ret; + } +#elif defined(AMD_WMMA_AVAILABLE) || defined(AMD_MFMA_AVAILABLE) + template + static __device__ __forceinline__ tile get_half2(const tile & tile_float) { + tile ret; +#pragma unroll + for (int l0 = 0; l0 < tile_float.ne; l0 += 2) { + ret.x[l0/2] = make_half2(tile_float.x[l0 + 0], tile_float.x[l0 + 1]); + } + return ret; + } + + static __device__ __forceinline__ tile<8, 8, half2> get_transposed(const tile<16, 4, half2> & t) { + NO_DEVICE_CODE; + return tile<8, 8, half2>{}; + } +#else // Volta + template + static __device__ __forceinline__ tile get_half2(const tile & tile_float) { + tile ret; +#pragma unroll + for (int l0 = 0; l0 < tile_float.ne; l0 += 4) { + ret.x[l0/2 + 0] = make_half2(tile_float.x[l0 + 0], tile_float.x[l0 + 1]); + ret.x[l0/2 + 1] = make_half2(tile_float.x[l0 + 2], tile_float.x[l0 + 3]); + + // On Volta FP16 and FP32 tiles have a different memory layout, + // for the conversion threads with an offset of 2 need to exchange half their values: + ret.x[l0/2 + (((threadIdx.x % 4) / 2) ^ 1)] = __shfl_xor_sync( + 0xFFFFFFFF, ret.x[l0/2 + (((threadIdx.x % 4) / 2) ^ 1)], 2, WARP_SIZE); + } + return ret; + } +#endif // defined(TURING_MMA_AVAILABLE) + + template + static __device__ __forceinline__ void load_generic(tile & t, const T * __restrict__ xs0, const int stride) { +#pragma unroll + for (int l = 0; l < t.ne; ++l) { + t.x[l] = xs0[t.get_i(l)*stride + t.get_j(l)]; + } + } + + template + static __device__ __forceinline__ void load_ldmatrix( + tile<8, 8, T> & t, const T * __restrict__ xs0, const int stride) { +#ifdef TURING_MMA_AVAILABLE + int * xi = (int *) t.x; + const int * xs = (const int *) xs0 + (threadIdx.x % t.I) * stride + ((threadIdx.x / t.I) * (t.J / 2)) % t.J; + asm volatile("ldmatrix.sync.aligned.m8n8.x2.b16 {%0, %1}, [%2];" + : "=r"(xi[0]), "=r"(xi[1]) + : "l"(xs)); +#else + GGML_UNUSED_VARS(t, xs0, stride); + NO_DEVICE_CODE; +#endif // TURING_MMA_AVAILABLE + } + + template + static __device__ __forceinline__ void load_ldmatrix( + tile<16, 4, T, dl> & t, const T * __restrict__ xs0, const int stride) { +#ifdef TURING_MMA_AVAILABLE + int * xi = (int *) t.x; + const int * xs = (const int *) xs0 + (threadIdx.x % t.I) * stride; + asm volatile("ldmatrix.sync.aligned.m8n8.x2.b16 {%0, %1}, [%2];" + : "=r"(xi[0]), "=r"(xi[1]) + : "l"(xs)); +#elif defined(AMD_WMMA_AVAILABLE) +#ifdef RDNA3 + static_assert(dl == DATA_LAYOUT_I_MAJOR_MIRRORED, "bad data layout"); + static_assert(sizeof(t.x) == 16, "bad ne"); + ggml_cuda_memcpy_1<8>(t.x + 0, xs0 + t.get_i(0)*stride + 0); + ggml_cuda_memcpy_1<8>(t.x + 2, xs0 + t.get_i(0)*stride + 2); +#else + static_assert(dl == DATA_LAYOUT_I_MAJOR, "bad data layout"); + static_assert(sizeof(t.x) == 8, "bad ne"); + ggml_cuda_memcpy_1<8>(t.x, xs0 + t.get_i(0)*stride + t.get_j(0)); +#endif // RDNA3 +#elif defined(AMD_MFMA_AVAILABLE) + static_assert(sizeof(t.x) == 4, "bad ne"); + ggml_cuda_memcpy_1<4>(t.x, xs0 + t.get_i(0)*stride + t.get_j(0)); +#else + GGML_UNUSED_VARS(t, xs0, stride); + NO_DEVICE_CODE; +#endif // TURING_MMA_AVAILABLE + } + + template + static __device__ __forceinline__ void load_ldmatrix( + tile<16, 8, T, dl> & t, const T * __restrict__ xs0, const int stride) { +#if defined(TURING_MMA_AVAILABLE) + int * xi = (int * ) t.x; + const int * xs = (const int *) xs0 + (threadIdx.x % t.I) * stride + (threadIdx.x / t.I) * (t.J / 2); + asm volatile("ldmatrix.sync.aligned.m8n8.x4.b16 {%0, %1, %2, %3}, [%4];" + : "=r"(xi[0]), "=r"(xi[1]), "=r"(xi[2]), "=r"(xi[3]) + : "l"(xs)); +#elif defined(VOLTA_MMA_AVAILABLE) + ggml_cuda_memcpy_1<4*sizeof(T)>(t.x + 0, xs0 + t.get_i(0)*stride + 0); + ggml_cuda_memcpy_1<4*sizeof(T)>(t.x + 4, xs0 + t.get_i(4)*stride + 4); +#elif defined(AMD_WMMA_AVAILABLE) +#ifdef RDNA3 + static_assert(dl == DATA_LAYOUT_I_MAJOR_MIRRORED, "bad data layout"); + static_assert(sizeof(t.x) == 32, "bad ne"); + ggml_cuda_memcpy_1<16>(t.x + 0, xs0 + t.get_i(0)*stride + 0); + ggml_cuda_memcpy_1<16>(t.x + 4, xs0 + t.get_i(0)*stride + 4); +#else + static_assert(dl == DATA_LAYOUT_I_MAJOR, "bad data layout"); + static_assert(sizeof(t.x) == 16, "bad ne"); + ggml_cuda_memcpy_1<16>(t.x, xs0 + t.get_i(0)*stride + t.get_j(0)); +#endif // RDNA3 +#elif defined(AMD_MFMA_AVAILABLE) + static_assert(sizeof(t.x) == 8, "bad ne"); + ggml_cuda_memcpy_1<8>(t.x, xs0 + t.get_i(0)*stride + t.get_j(0)); +#else + GGML_UNUSED_VARS(t, xs0, stride); + NO_DEVICE_CODE; +#endif // TURING_MMA_AVAILABLE + } + + static __device__ __forceinline__ void load_ldmatrix( + tile<8, 4, half2, DATA_LAYOUT_I_MAJOR_MIRRORED> & t, const half2 * __restrict__ xs0, const int stride) { + ggml_cuda_memcpy_1<4*sizeof(half2)>(t.x, xs0 + t.get_i(0)*stride); + } + + static __device__ __forceinline__ void load_ldmatrix( + tile<8, 4, half2, DATA_LAYOUT_J_MAJOR_MIRRORED> & t, const half2 * __restrict__ xs0, const int stride) { +#pragma unroll + for (int l0 = 0; l0 < t.ne; l0 += 2) { + ggml_cuda_memcpy_1<2*sizeof(half2)>(t.x + l0, xs0 + t.get_i(l0)*stride + t.get_j(l0)); + } + } + + static __device__ __forceinline__ void load_ldmatrix( + tile<32, 4, half2> & t, const half2 * __restrict__ xs0, const int stride) { +#if defined(VOLTA_MMA_AVAILABLE) + ggml_cuda_memcpy_1<4*sizeof(half2)>(t.x, xs0 + t.get_i(0)*stride); +#else + GGML_UNUSED_VARS(t, xs0, stride); + NO_DEVICE_CODE; +#endif // defined(VOLTA_MMA_AVAILABLE) + } + + template + static __device__ __forceinline__ void load_ldmatrix_trans( + tile & t, const T * __restrict__ xs0, const int stride) { +#ifdef TURING_MMA_AVAILABLE + static_assert(I == 16, "bad tile width"); + static_assert(dl == DATA_LAYOUT_I_MAJOR, "bad data layout"); + int * xi = (int *) t.x; + const int * xs = (const int *) xs0 + (threadIdx.x % t.I) * stride + (threadIdx.x / t.I) * (t.J / 2); + asm volatile("ldmatrix.sync.aligned.m8n8.x4.trans.b16 {%0, %1, %2, %3}, [%4];" + : "=r"(xi[0]), "=r"(xi[2]), "=r"(xi[1]), "=r"(xi[3]) + : "l"(xs)); +#elif defined(AMD_MFMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE) + static_assert(dl == DATA_LAYOUT_I_MAJOR || dl == DATA_LAYOUT_I_MAJOR_MIRRORED, "bad data layout"); + if constexpr (I == 32) { +#pragma unroll + for (int l0 = 0; l0 < t.ne/2; ++l0) { + const half2 tmp0 = xs0[(2*t.get_j(l0) + 0)*stride + t.get_i(l0)/2]; + const half2 tmp1 = xs0[(2*t.get_j(l0) + 1)*stride + t.get_i(l0)/2]; + + t.x[l0] = __lows2half2(tmp0, tmp1); + t.x[l0 + t.ne/2] = __highs2half2(tmp0, tmp1); + } + } else { + half * xh = (half *) t.x; +#pragma unroll + for (int l = 0; l < t.ne; ++l) { + xh[2*l + 0] = ((const half *) xs0)[(2*t.get_j(l) + 0)*(2*stride) + t.get_i(l)]; + xh[2*l + 1] = ((const half *) xs0)[(2*t.get_j(l) + 1)*(2*stride) + t.get_i(l)]; + } + } +#else + GGML_UNUSED_VARS(t, xs0, stride); + NO_DEVICE_CODE; +#endif // TURING_MMA_AVAILABLE + } + + static __device__ __forceinline__ void mma( + tile<16, 8, int> & D, const tile<16, 4, int> & A, const tile<8, 4, int> & B) { +#ifdef TURING_MMA_AVAILABLE +#if __CUDA_ARCH__ >= GGML_CUDA_CC_AMPERE + asm("mma.sync.aligned.m16n8k16.row.col.s32.s8.s8.s32 {%0, %1, %2, %3}, {%4, %5}, {%6}, {%0, %1, %2, %3};" + : "+r"(D.x[0]), "+r"(D.x[1]), "+r"(D.x[2]), "+r"(D.x[3]) + : "r"(A.x[0]), "r"(A.x[1]), "r"(B.x[0])); +#else + // On Turing m16n8k16 mma is not available, use 2x m8n8k16 mma instead: + asm("mma.sync.aligned.m8n8k16.row.col.s32.s8.s8.s32 {%0, %1}, {%2}, {%3}, {%0, %1};" + : "+r"(D.x[0]), "+r"(D.x[1]) + : "r"(A.x[0]), "r"(B.x[0])); + asm("mma.sync.aligned.m8n8k16.row.col.s32.s8.s8.s32 {%0, %1}, {%2}, {%3}, {%0, %1};" + : "+r"(D.x[2]), "+r"(D.x[3]) + : "r"(A.x[1]), "r"(B.x[0])); +#endif // __CUDA_ARCH__ >= GGML_CUDA_CC_AMPERE +#else + GGML_UNUSED_VARS(D, A, B); + NO_DEVICE_CODE; +#endif // TURING_MMA_AVAILABLE + } + + static __device__ __forceinline__ void mma( + tile<16, 8, int> & D, const tile<16, 8, int> & A, const tile<8, 8, int> & B) { +#ifdef TURING_MMA_AVAILABLE +#if __CUDA_ARCH__ >= GGML_CUDA_CC_AMPERE + asm("mma.sync.aligned.m16n8k32.row.col.s32.s8.s8.s32 {%0, %1, %2, %3}, {%4, %5, %6, %7}, {%8, %9}, {%0, %1, %2, %3};" + : "+r"(D.x[0]), "+r"(D.x[1]), "+r"(D.x[2]), "+r"(D.x[3]) + : "r"(A.x[0]), "r"(A.x[1]), "r"(A.x[2]), "r"(A.x[3]), "r"(B.x[0]), "r"(B.x[1])); +#else + // On Turing m16n8k32 mma is not available, use 4x m8n8k16 mma instead: + asm("mma.sync.aligned.m8n8k16.row.col.s32.s8.s8.s32 {%0, %1}, {%2}, {%3}, {%0, %1};" + : "+r"(D.x[0]), "+r"(D.x[1]) + : "r"(A.x[0]), "r"(B.x[0])); + asm("mma.sync.aligned.m8n8k16.row.col.s32.s8.s8.s32 {%0, %1}, {%2}, {%3}, {%0, %1};" + : "+r"(D.x[2]), "+r"(D.x[3]) + : "r"(A.x[1]), "r"(B.x[0])); + asm("mma.sync.aligned.m8n8k16.row.col.s32.s8.s8.s32 {%0, %1}, {%2}, {%3}, {%0, %1};" + : "+r"(D.x[0]), "+r"(D.x[1]) + : "r"(A.x[2]), "r"(B.x[1])); + asm("mma.sync.aligned.m8n8k16.row.col.s32.s8.s8.s32 {%0, %1}, {%2}, {%3}, {%0, %1};" + : "+r"(D.x[2]), "+r"(D.x[3]) + : "r"(A.x[3]), "r"(B.x[1])); +#endif // __CUDA_ARCH__ >= GGML_CUDA_CC_AMPERE +#else + GGML_UNUSED_VARS(D, A, B); + NO_DEVICE_CODE; +#endif // TURING_MMA_AVAILABLE + } + + static __device__ __forceinline__ void mma( + tile<16, 4, half2> & D, const tile<16, 8, half2> & A, const tile<8, 8, half2> & B) { +#ifdef TURING_MMA_AVAILABLE + const int * Axi = (const int *) A.x; + const int * Bxi = (const int *) B.x; + int * Dxi = (int *) D.x; +#if __CUDA_ARCH__ >= GGML_CUDA_CC_AMPERE + asm("mma.sync.aligned.m16n8k16.row.col.f16.f16.f16.f16 {%0, %1}, {%2, %3, %4, %5}, {%6, %7}, {%0, %1};" + : "+r"(Dxi[0]), "+r"(Dxi[1]) + : "r"(Axi[0]), "r"(Axi[1]), "r"(Axi[2]), "r"(Axi[3]), "r"(Bxi[0]), "r"(Bxi[1])); +#else + // On Turing m16n8k16 mma is not available, use 2x m8n8k8 mma instead: + asm("mma.sync.aligned.m16n8k8.row.col.f16.f16.f16.f16 {%0, %1}, {%2, %3}, {%4}, {%0, %1};" + : "+r"(Dxi[0]), "+r"(Dxi[1]) + : "r"(Axi[0]), "r"(Axi[1]), "r"(Bxi[0])); + asm("mma.sync.aligned.m16n8k8.row.col.f16.f16.f16.f16 {%0, %1}, {%2, %3}, {%4}, {%0, %1};" + : "+r"(Dxi[0]), "+r"(Dxi[1]) + : "r"(Axi[2]), "r"(Axi[3]), "r"(Bxi[1])); +#endif // __CUDA_ARCH__ >= GGML_CUDA_CC_AMPERE +#else + GGML_UNUSED_VARS(D, A, B); + NO_DEVICE_CODE; +#endif // TURING_MMA_AVAILABLE + } + + static __device__ __forceinline__ void mma( + tile<16, 8, half2> & D, const tile<16, 8, half2> & A, const tile<16, 8, half2> & B) { +#ifdef TURING_MMA_AVAILABLE + const int * Axi = (const int *) A.x; + const int * Bxi = (const int *) B.x; + int * Dxi = (int *) D.x; +#if __CUDA_ARCH__ >= GGML_CUDA_CC_AMPERE + asm("mma.sync.aligned.m16n8k16.row.col.f16.f16.f16.f16 {%0, %1}, {%2, %3, %4, %5}, {%6, %7}, {%0, %1};" + : "+r"(Dxi[0]), "+r"(Dxi[1]) + : "r"(Axi[0]), "r"(Axi[1]), "r"(Axi[2]), "r"(Axi[3]), "r"(Bxi[0]), "r"(Bxi[2])); + asm("mma.sync.aligned.m16n8k16.row.col.f16.f16.f16.f16 {%0, %1}, {%2, %3, %4, %5}, {%6, %7}, {%0, %1};" + : "+r"(Dxi[2]), "+r"(Dxi[3]) + : "r"(Axi[0]), "r"(Axi[1]), "r"(Axi[2]), "r"(Axi[3]), "r"(Bxi[1]), "r"(Bxi[3])); +#else + // On Turing m16n8k16 mma is not available, use 4x m8n8k8 mma instead: + asm("mma.sync.aligned.m16n8k8.row.col.f16.f16.f16.f16 {%0, %1}, {%2, %3}, {%4}, {%0, %1};" + : "+r"(Dxi[0]), "+r"(Dxi[1]) + : "r"(Axi[0]), "r"(Axi[1]), "r"(Bxi[0])); + asm("mma.sync.aligned.m16n8k8.row.col.f16.f16.f16.f16 {%0, %1}, {%2, %3}, {%4}, {%0, %1};" + : "+r"(Dxi[0]), "+r"(Dxi[1]) + : "r"(Axi[2]), "r"(Axi[3]), "r"(Bxi[2])); + asm("mma.sync.aligned.m16n8k8.row.col.f16.f16.f16.f16 {%0, %1}, {%2, %3}, {%4}, {%0, %1};" + : "+r"(Dxi[2]), "+r"(Dxi[3]) + : "r"(Axi[0]), "r"(Axi[1]), "r"(Bxi[1])); + asm("mma.sync.aligned.m16n8k8.row.col.f16.f16.f16.f16 {%0, %1}, {%2, %3}, {%4}, {%0, %1};" + : "+r"(Dxi[2]), "+r"(Dxi[3]) + : "r"(Axi[2]), "r"(Axi[3]), "r"(Bxi[3])); +#endif // __CUDA_ARCH__ >= GGML_CUDA_CC_AMPERE +#elif defined(AMD_WMMA_AVAILABLE) +#if defined(RDNA4) + using halfx8_t = __attribute__((ext_vector_type(8))) _Float16; + halfx8_t& acc_frag = reinterpret_cast(D.x[0]); + const halfx8_t& a_frag = reinterpret_cast(A.x[0]); + const halfx8_t& b_frag = reinterpret_cast(B.x[0]); + acc_frag = __builtin_amdgcn_wmma_f16_16x16x16_f16_w32_gfx12(a_frag, b_frag, acc_frag); +#else + GGML_UNUSED_VARS(D, A, B); + NO_DEVICE_CODE; +#endif // defined(RDNA4) +#elif defined(AMD_MFMA_AVAILABLE) + // MFMA: FP16 input, FP32 accumulate, convert back to half2. + using halfx4_t = __attribute__((ext_vector_type(4))) _Float16; + using floatx4_t = __attribute__((ext_vector_type(4))) float; + + // Convert existing half2 accumulator to float for MFMA: + floatx4_t acc_f32; + { + const halfx4_t acc_h = reinterpret_cast(D.x[0]); +#pragma unroll + for (int i = 0; i < 4; ++i) { + acc_f32[i] = (float)acc_h[i]; + } + } + + const halfx4_t& a_frag = reinterpret_cast(A.x[0]); + const halfx4_t& b_frag = reinterpret_cast(B.x[0]); + acc_f32 = __builtin_amdgcn_mfma_f32_16x16x16f16(a_frag, b_frag, acc_f32, 0, 0, 0); + + // Convert back to half2: + { + halfx4_t result_h; +#pragma unroll + for (int i = 0; i < 4; ++i) { + result_h[i] = (_Float16)acc_f32[i]; + } + reinterpret_cast(D.x[0]) = result_h; + } +#else + GGML_UNUSED_VARS(D, A, B); + NO_DEVICE_CODE; +#endif // TURING_MMA_AVAILABLE + } + + static __device__ __forceinline__ void mma( + tile<16, 16, half2, DATA_LAYOUT_I_MAJOR_SCRAMBLED> & D, const tile<32, 8, half2, DATA_LAYOUT_I_MAJOR> & A, + const tile<16, 8, half2, DATA_LAYOUT_I_MAJOR> & B) { +#if defined(AMD_MFMA_AVAILABLE) || (defined(AMD_WMMA_AVAILABLE) && defined(RDNA4)) + tile<16, 8, half2> * D16 = (tile<16, 8, half2> *) &D; + const tile<16, 8, half2> * A16 = (const tile<16, 8, half2> *) &A; + mma(D16[0], A16[0], B); + mma(D16[1], A16[1], B); +#else + GGML_UNUSED_VARS(D, A, B); + NO_DEVICE_CODE; +#endif // defined(AMD_MFMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE) && defined(RDNA4) + } + + template + static __device__ __forceinline__ void mma( + tile<16, 8, float, dl_d> & D, const tile<16, 8, float, dl_ab> & A, const tile<8, 8, float, dl_ab> & B) { +#ifdef AMPERE_MMA_AVAILABLE + const int * Axi = (const int *) A.x; + const int * Bxi = (const int *) B.x; + int * Dxi = (int *) D.x; + asm("mma.sync.aligned.m16n8k8.row.col.f32.tf32.tf32.f32 {%0, %1, %2, %3}, {%4, %5, %6, %7}, {%8, %9}, {%0, %1, %2, %3};" + : "+r"(Dxi[0]), "+r"(Dxi[1]), "+r"(Dxi[2]), "+r"(Dxi[3]) + : "r"(Axi[0]), "r"(Axi[1]), "r"(Axi[2]), "r"(Axi[3]), "r"(Bxi[0]), "r"(Bxi[1])); +#else + GGML_UNUSED_VARS(D, A, B); + NO_DEVICE_CODE; +#endif // AMPERE_MMA_AVAILABLE + } + + template + static __device__ __forceinline__ void mma( + tile<16, 16, float, dl_d> & D, const tile<16, 8, float, dl_ab> & A, const tile<16, 8, float, dl_ab> & B) { +#ifdef AMD_MFMA_AVAILABLE + using floatx4_t = __attribute__((ext_vector_type(4))) float; + floatx4_t& acc_frag = reinterpret_cast(D.x[0]); +#if defined(CDNA3) + using floatx2_t = __attribute__((ext_vector_type(2))) float; + const floatx2_t& a_frag = reinterpret_cast(A.x[0]); + const floatx2_t& b_frag = reinterpret_cast(B.x[0]); + acc_frag = __builtin_amdgcn_mfma_f32_16x16x8_xf32(a_frag, b_frag, acc_frag, 0, 0, 0); +#elif defined(CDNA4) || defined(CDNA2) || defined(CDNA1) + // CDNA4 (gfx950) does not support xf32 MFMA, use f32 path like CDNA2/CDNA1 +#pragma unroll + for (int i = 0; i < 2; ++i) { + acc_frag = __builtin_amdgcn_mfma_f32_16x16x4f32(A.x[i], B.x[i], acc_frag, 0, 0, 0); + } +#else + GGML_UNUSED_VARS(D, A, B); + NO_DEVICE_CODE; +#endif // defined(CDNA3) +#else + GGML_UNUSED_VARS(D, A, B); + NO_DEVICE_CODE; +#endif // AMD_MFMA_AVAILABLE + } + + template + static __device__ __forceinline__ void mma_block_scaled_fp4(tile<16, 8, float> & D, + const tile<16, 8, int> & A, + const tile<8, 8, int> & B, + uint32_t a_scale, + uint32_t b_scale) { +#ifdef BLACKWELL_MMA_AVAILABLE + const int * Axi = (const int *) A.x; + const int * Bxi = (const int *) B.x; + float * Dxi = (float *) D.x; + + if constexpr (type == GGML_TYPE_MXFP4) { + asm volatile( + "mma.sync.aligned.kind::mxf4.block_scale.scale_vec::2X.m16n8k64.row.col.f32.e2m1.e2m1.f32.ue8m0 " + "{%0, %1, %2, %3}, {%4, %5, %6, %7}, {%8, %9}, {%0, %1, %2, %3}, " + "%10, {0, 0}, %11, {0, 0};" + : "+f"(Dxi[0]), "+f"(Dxi[1]), "+f"(Dxi[2]), "+f"(Dxi[3]) + : "r"(Axi[0]), "r"(Axi[1]), "r"(Axi[2]), "r"(Axi[3]), "r"(Bxi[0]), "r"(Bxi[1]), "r"(a_scale), "r"(b_scale)); + } else { + asm volatile( + "mma.sync.aligned.kind::mxf4nvf4.block_scale.scale_vec::4X.m16n8k64.row.col.f32.e2m1.e2m1.f32.ue4m3 " + "{%0, %1, %2, %3}, {%4, %5, %6, %7}, {%8, %9}, {%0, %1, %2, %3}, " + "%10, {0, 0}, %11, {0, 0};" + : "+f"(Dxi[0]), "+f"(Dxi[1]), "+f"(Dxi[2]), "+f"(Dxi[3]) + : "r"(Axi[0]), "r"(Axi[1]), "r"(Axi[2]), "r"(Axi[3]), "r"(Bxi[0]), "r"(Bxi[1]), "r"(a_scale), "r"(b_scale)); + } +#else + GGML_UNUSED_VARS(D, A, B, a_scale, b_scale); +#endif // BLACKWELL_MMA_AVAILABLE + } + + static __device__ __forceinline__ void mma( + tile<16, 8, float> & D, const tile<16, 8, half2> & A, const tile<8, 8, half2> & B) { +#ifdef TURING_MMA_AVAILABLE + const int * Axi = (const int *) A.x; + const int * Bxi = (const int *) B.x; + int * Dxi = (int *) D.x; +#if __CUDA_ARCH__ >= GGML_CUDA_CC_AMPERE + asm("mma.sync.aligned.m16n8k16.row.col.f32.f16.f16.f32 {%0, %1, %2, %3}, {%4, %5, %6, %7}, {%8, %9}, {%0, %1, %2, %3};" + : "+r"(Dxi[0]), "+r"(Dxi[1]), "+r"(Dxi[2]), "+r"(Dxi[3]) + : "r"(Axi[0]), "r"(Axi[1]), "r"(Axi[2]), "r"(Axi[3]), "r"(Bxi[0]), "r"(Bxi[1])); +#else + // On Turing m16n8k16 mma is not available, use 2x m8n8k8 mma instead: + asm("mma.sync.aligned.m16n8k8.row.col.f32.f16.f16.f32 {%0, %1, %2, %3}, {%4, %5}, {%6}, {%0, %1, %2, %3};" + : "+r"(Dxi[0]), "+r"(Dxi[1]), "+r"(Dxi[2]), "+r"(Dxi[3]) + : "r"(Axi[0]), "r"(Axi[1]), "r"(Bxi[0])); + asm("mma.sync.aligned.m16n8k8.row.col.f32.f16.f16.f32 {%0, %1, %2, %3}, {%4, %5}, {%6}, {%0, %1, %2, %3};" + : "+r"(Dxi[0]), "+r"(Dxi[1]), "+r"(Dxi[2]), "+r"(Dxi[3]) + : "r"(Axi[2]), "r"(Axi[3]), "r"(Bxi[1])); +#endif // __CUDA_ARCH__ >= GGML_CUDA_CC_AMPERE +#else + GGML_UNUSED_VARS(D, A, B); + NO_DEVICE_CODE; +#endif // TURING_MMA_AVAILABLE + } + + static __device__ __forceinline__ void mma( + tile<16, 8, float> & D, const tile<16, 8, nv_bfloat162> & A, const tile<8, 8, nv_bfloat162> & B) { +#ifdef AMPERE_MMA_AVAILABLE + const int * Axi = (const int *) A.x; + const int * Bxi = (const int *) B.x; + int * Dxi = (int *) D.x; + asm("mma.sync.aligned.m16n8k16.row.col.f32.bf16.bf16.f32 {%0, %1, %2, %3}, {%4, %5, %6, %7}, {%8, %9}, {%0, %1, %2, %3};" + : "+r"(Dxi[0]), "+r"(Dxi[1]), "+r"(Dxi[2]), "+r"(Dxi[3]) + : "r"(Axi[0]), "r"(Axi[1]), "r"(Axi[2]), "r"(Axi[3]), "r"(Bxi[0]), "r"(Bxi[1])); +#else + GGML_UNUSED_VARS(D, A, B); + NO_DEVICE_CODE; +#endif // AMPERE_MMA_AVAILABLE + } + + template + static __device__ __forceinline__ void mma( + tile<16, 16, float, dl_d> & D, const tile<16, 8, half2, dl_ab> & A, const tile<16, 8, half2, dl_ab> & B) { +#ifdef TURING_MMA_AVAILABLE + const int * Axi = (const int *) A.x; + const int * Bxi = (const int *) B.x; + int * Dxi = (int *) D.x; +#if __CUDA_ARCH__ >= GGML_CUDA_CC_AMPERE + asm("mma.sync.aligned.m16n8k16.row.col.f32.f16.f16.f32 {%0, %1, %2, %3}, {%4, %5, %6, %7}, {%8, %9}, {%0, %1, %2, %3};" + : "+r"(Dxi[0]), "+r"(Dxi[1]), "+r"(Dxi[2]), "+r"(Dxi[3]) + : "r"(Axi[0]), "r"(Axi[1]), "r"(Axi[2]), "r"(Axi[3]), "r"(Bxi[0]), "r"(Bxi[2])); + asm("mma.sync.aligned.m16n8k16.row.col.f32.f16.f16.f32 {%0, %1, %2, %3}, {%4, %5, %6, %7}, {%8, %9}, {%0, %1, %2, %3};" + : "+r"(Dxi[4]), "+r"(Dxi[5]), "+r"(Dxi[6]), "+r"(Dxi[7]) + : "r"(Axi[0]), "r"(Axi[1]), "r"(Axi[2]), "r"(Axi[3]), "r"(Bxi[1]), "r"(Bxi[3])); +#else + // On Turing m16n8k16 mma is not available, use 4x m8n8k8 mma instead: + asm("mma.sync.aligned.m16n8k8.row.col.f32.f16.f16.f32 {%0, %1, %2, %3}, {%4, %5}, {%6}, {%0, %1, %2, %3};" + : "+r"(Dxi[0]), "+r"(Dxi[1]), "+r"(Dxi[2]), "+r"(Dxi[3]) + : "r"(Axi[0]), "r"(Axi[1]), "r"(Bxi[0])); + asm("mma.sync.aligned.m16n8k8.row.col.f32.f16.f16.f32 {%0, %1, %2, %3}, {%4, %5}, {%6}, {%0, %1, %2, %3};" + : "+r"(Dxi[0]), "+r"(Dxi[1]), "+r"(Dxi[2]), "+r"(Dxi[3]) + : "r"(Axi[2]), "r"(Axi[3]), "r"(Bxi[2])); + asm("mma.sync.aligned.m16n8k8.row.col.f32.f16.f16.f32 {%0, %1, %2, %3}, {%4, %5}, {%6}, {%0, %1, %2, %3};" + : "+r"(Dxi[4]), "+r"(Dxi[5]), "+r"(Dxi[6]), "+r"(Dxi[7]) + : "r"(Axi[0]), "r"(Axi[1]), "r"(Bxi[1])); + asm("mma.sync.aligned.m16n8k8.row.col.f32.f16.f16.f32 {%0, %1, %2, %3}, {%4, %5}, {%6}, {%0, %1, %2, %3};" + : "+r"(Dxi[4]), "+r"(Dxi[5]), "+r"(Dxi[6]), "+r"(Dxi[7]) + : "r"(Axi[2]), "r"(Axi[3]), "r"(Bxi[3])); +#endif // __CUDA_ARCH__ >= GGML_CUDA_CC_AMPERE +#elif defined(AMD_WMMA_AVAILABLE) +#if defined(RDNA4) + using halfx8_t = __attribute__((ext_vector_type(8))) _Float16; + using floatx8_t = __attribute__((ext_vector_type(8))) float; + floatx8_t& acc_frag = reinterpret_cast(D.x[0]); + const halfx8_t& a_frag = reinterpret_cast(A.x[0]); + const halfx8_t& b_frag = reinterpret_cast(B.x[0]); + acc_frag = __builtin_amdgcn_wmma_f32_16x16x16_f16_w32_gfx12(a_frag, b_frag, acc_frag); +#elif defined(RDNA3) + using halfx16_t = __attribute__((ext_vector_type(16))) _Float16; + using floatx8_t = __attribute__((ext_vector_type(8))) float; + floatx8_t& acc_frag = reinterpret_cast(D.x[0]); + const halfx16_t& a_frag = reinterpret_cast(A.x[0]); + const halfx16_t& b_frag = reinterpret_cast(B.x[0]); + acc_frag = __builtin_amdgcn_wmma_f32_16x16x16_f16_w32(a_frag, b_frag, acc_frag); +#else + GGML_UNUSED_VARS(D, A, B); + NO_DEVICE_CODE; +#endif // RDNA4 +#elif defined(AMD_MFMA_AVAILABLE) + using halfx4_t = __attribute__((ext_vector_type(4))) _Float16; + using floatx4_t = __attribute__((ext_vector_type(4))) float; + floatx4_t& acc_frag = reinterpret_cast(D.x[0]); + const halfx4_t& a_frag = reinterpret_cast(A.x[0]); + const halfx4_t& b_frag = reinterpret_cast(B.x[0]); + acc_frag = __builtin_amdgcn_mfma_f32_16x16x16f16(a_frag, b_frag, acc_frag, 0, 0, 0); +#else + GGML_UNUSED_VARS(D, A, B); + NO_DEVICE_CODE; +#endif // TURING_MMA_AVAILABLE + } + + template + static __device__ __forceinline__ void mma( + tile<16, 16, float, dl_d> & D, const tile<16, 8, nv_bfloat162, dl_ab> & A, const tile<16, 8, nv_bfloat162, dl_ab> & B) { +#if defined(AMD_WMMA_AVAILABLE) +#if defined(RDNA4) + using bf16x8_t = __attribute__((ext_vector_type(8))) __bf16; + using floatx8_t = __attribute__((ext_vector_type(8))) float; + floatx8_t& acc_frag = reinterpret_cast(D.x[0]); + const bf16x8_t& a_frag = reinterpret_cast(A.x[0]); + const bf16x8_t& b_frag = reinterpret_cast(B.x[0]); + acc_frag = __builtin_amdgcn_wmma_f32_16x16x16_bf16_w32_gfx12(a_frag, b_frag, acc_frag); +#elif defined(RDNA3) + using bf16x16_t = __attribute__((ext_vector_type(16))) __bf16; + using floatx8_t = __attribute__((ext_vector_type(8))) float; + floatx8_t& acc_frag = reinterpret_cast(D.x[0]); + const bf16x16_t& a_frag = reinterpret_cast(A.x[0]); + const bf16x16_t& b_frag = reinterpret_cast(B.x[0]); + acc_frag = __builtin_amdgcn_wmma_f32_16x16x16_bf16_w32(a_frag, b_frag, acc_frag); +#else + GGML_UNUSED_VARS(D, A, B); + NO_DEVICE_CODE; +#endif // defined(RDNA4) +#elif defined(AMD_MFMA_AVAILABLE) + using floatx4_t = __attribute__((ext_vector_type(4))) float; + floatx4_t& acc_frag = reinterpret_cast(D.x[0]); +#if defined(CDNA4) || defined(CDNA3) || defined(CDNA2) + using bf16x4_t = __attribute__((ext_vector_type(4))) __bf16; + const bf16x4_t& a_frag = reinterpret_cast(A.x[0]); + const bf16x4_t& b_frag = reinterpret_cast(B.x[0]); + acc_frag = __builtin_amdgcn_mfma_f32_16x16x16bf16_1k(a_frag, b_frag, acc_frag, 0, 0, 0); +#elif defined(CDNA1) +#pragma unroll + for (int i = 0; i < 2; ++i) { + using bf16x2_t = __attribute__((ext_vector_type(2))) __bf16; + const bf16x2_t& a_frag = reinterpret_cast(A.x[i]); + const bf16x2_t& b_frag = reinterpret_cast(B.x[i]); + acc_frag = __builtin_amdgcn_mfma_f32_16x16x8bf16(a_frag, b_frag, acc_frag, 0, 0, 0); + } +#else + GGML_UNUSED_VARS(D, A, B); + NO_DEVICE_CODE; +#endif // defined(CDNA3) || defined(CDNA2) +#else + GGML_UNUSED_VARS(D, A, B); + NO_DEVICE_CODE; +#endif // defined(AMD_WMMA_AVAILABLE) + } + + template + static __device__ __forceinline__ void mma( + tile<16, 16, int, dl_d> & D, const tile<16, 8, int, dl_ab> & A, const tile<16, 8, int, dl_ab> & B) { +#if defined(AMD_MFMA_AVAILABLE) + using int32x4_t = __attribute__((__vector_size__(4 * sizeof(int)))) int; + int32x4_t * acc = (int32x4_t *) D.x; +#if defined(CDNA4) || defined(CDNA3) + acc[0] = __builtin_amdgcn_mfma_i32_16x16x32_i8(((int64_t *) A.x)[0], ((int64_t *) B.x)[0], acc[0], 0, 0, 0); +#elif defined(CDNA2) || defined(CDNA1) + acc[0] = __builtin_amdgcn_mfma_i32_16x16x16i8(A.x[0], B.x[0], acc[0], 0, 0, 0); + acc[0] = __builtin_amdgcn_mfma_i32_16x16x16i8(A.x[1], B.x[1], acc[0], 0, 0, 0); +#endif // defined(CDNA4) || defined(CDNA3) +#elif defined(AMD_WMMA_AVAILABLE) + using int32x8_t = __attribute__((__vector_size__(8 * sizeof(int)))) int; + int32x8_t * acc = (int32x8_t *) D.x; +#if defined(RDNA4) + using int32x2_t = __attribute__((__vector_size__(2 * sizeof(int)))) int; + int32x2_t * a_vec = (int32x2_t *) A.x; + int32x2_t * b_vec = (int32x2_t *) B.x; + acc[0] = __builtin_amdgcn_wmma_i32_16x16x16_iu8_w32_gfx12(true, a_vec[0], true, b_vec[0], acc[0], true); + acc[0] = __builtin_amdgcn_wmma_i32_16x16x16_iu8_w32_gfx12(true, a_vec[1], true, b_vec[1], acc[0], true); +#elif defined(RDNA3) + using int32x4_t = __attribute__((__vector_size__(4 * sizeof(int)))) int; + int32x4_t * a_vec = (int32x4_t *) A.x; + int32x4_t * b_vec = (int32x4_t *) B.x; + acc[0] = __builtin_amdgcn_wmma_i32_16x16x16_iu8_w32(true, a_vec[0], true, b_vec[0], acc[0], true); + acc[0] = __builtin_amdgcn_wmma_i32_16x16x16_iu8_w32(true, a_vec[1], true, b_vec[1], acc[0], true); +#endif // RDNA4 +#else + GGML_UNUSED_VARS(D, A, B); + NO_DEVICE_CODE; +#endif // AMD_MFMA_AVAILABLE + } + + static __device__ __forceinline__ void mma( + tile<32, 32, int> & D, const tile<32, 4, int> & A, const tile<32, 4, int> & B) { +#if defined(AMD_MFMA_AVAILABLE) + using int32x16_t = __attribute__((__vector_size__(16 * sizeof(int)))) int; + int32x16_t * acc = (int32x16_t *) D.x; +#if defined(CDNA4) || defined(CDNA3) + acc[0] = __builtin_amdgcn_mfma_i32_32x32x16_i8(((int64_t *) A.x)[0], ((int64_t *) B.x)[0], acc[0], 0, 0, 0); +#elif defined(CDNA2) || defined(CDNA1) + acc[0] = __builtin_amdgcn_mfma_i32_32x32x8i8(A.x[0], B.x[0], acc[0], 0, 0, 0); + acc[0] = __builtin_amdgcn_mfma_i32_32x32x8i8(A.x[1], B.x[1], acc[0], 0, 0, 0); +#endif // defined(CDNA4) || defined(CDNA3) + +#else + GGML_UNUSED_VARS(D, A, B); + NO_DEVICE_CODE; +#endif // AMD_MFMA_AVAILABLE + } + + template + static __device__ __forceinline__ void mma( + tile<32, J, T1> & D, const tile<32, K, T2> & A, const tile & B) { + tile <16, J, T1> * D16 = reinterpret_cast< tile<16, J, T1> *>(&D); + const tile<16, K, T2> * A16 = reinterpret_cast *>(&A); + mma(D16[0], A16[0], B); + mma(D16[1], A16[1], B); + } + + static __device__ __forceinline__ void mma( + tile<32, 8, float> & D, const tile<32, 4, half2> & A, const tile<8, 4, half2, DATA_LAYOUT_I_MAJOR_MIRRORED> & B) { +#if defined(VOLTA_MMA_AVAILABLE) + const int * Axi = (const int *) A.x; + const int * Bxi = (const int *) B.x; + int * Dxi = (int *) D.x; + asm("mma.sync.aligned.m8n8k4.row.col.f32.f16.f16.f32 " + "{%0, %1, %2, %3, %4, %5, %6, %7}, {%8, %9}, {%10, %11}, {%0, %1, %2, %3, %4, %5, %6, %7};" + : "+r"(Dxi[0]), "+r"(Dxi[1]), "+r"(Dxi[2]), "+r"(Dxi[3]), "+r"(Dxi[4]), "+r"(Dxi[5]), "+r"(Dxi[6]), "+r"(Dxi[7]) + : "r"(Axi[0]), "r"(Axi[1]), "r"(Bxi[0]), "r"(Bxi[1])); + asm("mma.sync.aligned.m8n8k4.row.col.f32.f16.f16.f32 " + "{%0, %1, %2, %3, %4, %5, %6, %7}, {%8, %9}, {%10, %11}, {%0, %1, %2, %3, %4, %5, %6, %7};" + : "+r"(Dxi[0]), "+r"(Dxi[1]), "+r"(Dxi[2]), "+r"(Dxi[3]), "+r"(Dxi[4]), "+r"(Dxi[5]), "+r"(Dxi[6]), "+r"(Dxi[7]) + : "r"(Axi[2]), "r"(Axi[3]), "r"(Bxi[2]), "r"(Bxi[3])); +#else + GGML_UNUSED_VARS(D, A, B); + NO_DEVICE_CODE; +#endif // defined(VOLTA_MMA_AVAILABLE) + } + + static __device__ __forceinline__ void mma( + tile<32, 4, half2> & D, const tile<32, 4, half2> & A, const tile<8, 4, half2, DATA_LAYOUT_J_MAJOR_MIRRORED> & B) { +#if defined(VOLTA_MMA_AVAILABLE) + const int * Axi = (const int *) A.x; + const int * Bxi = (const int *) B.x; + int * Dxi = (int *) D.x; + asm("mma.sync.aligned.m8n8k4.row.row.f16.f16.f16.f16 " + "{%0, %1, %2, %3}, {%4, %5}, {%6, %7}, {%0, %1, %2, %3};" + : "+r"(Dxi[0]), "+r"(Dxi[1]), "+r"(Dxi[2]), "+r"(Dxi[3]) + : "r"(Axi[0]), "r"(Axi[1]), "r"(Bxi[0]), "r"(Bxi[1])); + asm("mma.sync.aligned.m8n8k4.row.row.f16.f16.f16.f16 " + "{%0, %1, %2, %3}, {%4, %5}, {%6, %7}, {%0, %1, %2, %3};" + : "+r"(Dxi[0]), "+r"(Dxi[1]), "+r"(Dxi[2]), "+r"(Dxi[3]) + : "r"(Axi[2]), "r"(Axi[3]), "r"(Bxi[2]), "r"(Bxi[3])); +#else + GGML_UNUSED_VARS(D, A, B); + NO_DEVICE_CODE; +#endif // defined(VOLTA_MMA_AVAILABLE) + } + + static __device__ __forceinline__ void mma( + tile<16, 16, half2, DATA_LAYOUT_I_MAJOR> & D, const tile<32, 8, half2, DATA_LAYOUT_I_MAJOR_MIRRORED> & A, + const tile<16, 8, half2, DATA_LAYOUT_I_MAJOR_MIRRORED> & B) { +#if defined(AMD_WMMA_AVAILABLE) && defined(RDNA3) + using halfx16_t = __attribute__((ext_vector_type(16))) _Float16; + halfx16_t * xD = (halfx16_t *) D.x; + const halfx16_t * xA = (const halfx16_t *) A.x; + const halfx16_t * xB = (const halfx16_t *) B.x; + xD[0] = __builtin_amdgcn_wmma_f16_16x16x16_f16_w32(xA[0], xB[0], xD[0], /*opsel =*/ 0); + xD[0] = __builtin_amdgcn_wmma_f16_16x16x16_f16_w32(xA[1], xB[0], xD[0], /*opsel =*/ 1); +#else + GGML_UNUSED_VARS(D, A, B); + NO_DEVICE_CODE; +#endif // TURING_MMA_AVAILABLE + } + + template + static __device__ __forceinline__ void mma( + tile<16, 16, int, dl_d> & D, const tile<16, 4, int, dl_ab> & A, const tile<16, 4, int, dl_ab> & B) { +#if defined(AMD_MFMA_AVAILABLE) + using int32x4_t = __attribute__((__vector_size__(4 * sizeof(int)))) int; + int32x4_t * acc = (int32x4_t *) D.x; +#if defined(CDNA4) || defined(CDNA3) + const int64_t xA = uint32_t(A.x[0]); + const int64_t xB = uint32_t(B.x[0]); + acc[0] = __builtin_amdgcn_mfma_i32_16x16x32_i8(xA, xB, acc[0], 0, 0, 0); +#elif defined(CDNA2) || defined(CDNA1) + acc[0] = __builtin_amdgcn_mfma_i32_16x16x16i8(A.x[0], B.x[0], acc[0], 0, 0, 0); +#endif // defined(CDNA4) || defined(CDNA3) +#elif defined(AMD_WMMA_AVAILABLE) + using int32x8_t = __attribute__((__vector_size__(8 * sizeof(int)))) int; + int32x8_t * acc = (int32x8_t *) D.x; +#if defined(RDNA4) + using int32x2_t = __attribute__((__vector_size__(2 * sizeof(int)))) int; + int32x2_t * a_vec = (int32x2_t *) A.x; + int32x2_t * b_vec = (int32x2_t *) B.x; + acc[0] = __builtin_amdgcn_wmma_i32_16x16x16_iu8_w32_gfx12(true, a_vec[0], true, b_vec[0], acc[0], false); +#elif defined(RDNA3) + using int32x4_t = __attribute__((__vector_size__(4 * sizeof(int)))) int; + int32x4_t * a_vec = (int32x4_t *) A.x; + int32x4_t * b_vec = (int32x4_t *) B.x; + acc[0] = __builtin_amdgcn_wmma_i32_16x16x16_iu8_w32(true, a_vec[0], true, b_vec[0], acc[0], false); +#endif // RDNA4 +#else + GGML_UNUSED(D); + GGML_UNUSED(A); + GGML_UNUSED(B); + NO_DEVICE_CODE; +#endif // AMD_WMMA_AVAILABLE + } +} diff --git a/python/freetoken/kernel/csrc/gguf_mmq/mmid.cu b/python/freetoken/kernel/csrc/gguf_mmq/mmid.cu new file mode 100644 index 00000000..f80442fb --- /dev/null +++ b/python/freetoken/kernel/csrc/gguf_mmq/mmid.cu @@ -0,0 +1,169 @@ +#include "common.cuh" +#include "mmid.cuh" + +// To reduce shared memory use, store "it" and "iex_used" with 22/10 bits each. +struct mm_ids_helper_store { + uint32_t data; + + __device__ mm_ids_helper_store(const uint32_t it, const uint32_t iex_used) { + data = (it & 0x003FFFFF) | (iex_used << 22); + } + + __device__ uint32_t it() const { + return data & 0x003FFFFF; + } + + __device__ uint32_t iex_used() const { + return data >> 22; + } +}; +static_assert(sizeof(mm_ids_helper_store) == 4, "unexpected size for mm_ids_helper_store"); + +// Helper function for mul_mat_id, converts ids to a more convenient format. +// ids_src1 describes how to permute the flattened column indices of src1 in order to get a compact src1 tensor sorted by expert. +// ids_dst describes the same mapping but for the dst tensor. +// The upper and lower bounds for the ith expert in the compact src1 tensor are stored in expert_bounds[i:i+1]. +template +__launch_bounds__(ggml_cuda_get_physical_warp_size(), 1) +static __global__ void mm_ids_helper( + const int32_t * __restrict__ ids, int32_t * __restrict__ ids_src1, int32_t * __restrict__ ids_dst, int32_t * __restrict__ expert_bounds, + const int n_tokens, const int n_expert_used_var, const int nchannels_y, const int si1, const int sis1, const bool write_inverse) { + constexpr int warp_size = ggml_cuda_get_physical_warp_size(); + const int n_expert_used = n_expert_used_template == 0 ? n_expert_used_var : n_expert_used_template; + const int expert = blockIdx.x; + + extern __shared__ char data_mm_ids_helper[]; + mm_ids_helper_store * store = (mm_ids_helper_store *) data_mm_ids_helper; + + int nex_prev = 0; // Number of columns for experts with a lower index. + int it_compact = 0; // Running index for the compact slice of this expert. + + if constexpr (n_expert_used_template == 0) { + // Generic implementation: + for (int it = 0; it < n_tokens; ++it) { + int iex_used = -1; // The index at which the expert is used, if any. + for (int iex = threadIdx.x; iex < n_expert_used; iex += warp_size) { + const int expert_used = ids[it*si1 + iex]; + nex_prev += expert_used < expert; + if (expert_used == expert) { + iex_used = iex; + } + } + + if (iex_used != -1) { + store[it_compact] = mm_ids_helper_store(it, iex_used); + } + + if (warp_reduce_any(iex_used != -1)) { + it_compact++; + } + } + } else { + // Implementation optimized for specific numbers of experts used: + static_assert(n_expert_used == 6 || warp_size % n_expert_used == 0, "bad n_expert_used"); + const int neu_padded = n_expert_used == 6 ? 8 : n_expert_used; // Padded to next higher power of 2. + for (int it0 = 0; it0 < n_tokens; it0 += warp_size/neu_padded) { + const int it = it0 + threadIdx.x / neu_padded; + + const int iex = threadIdx.x % neu_padded; // The index at which the expert is used, if any. + const int expert_used = (neu_padded == n_expert_used || iex < n_expert_used) && it < n_tokens ? + ids[it*si1 + iex] : INT_MAX; + const int iex_used = expert_used == expert ? iex : -1; + nex_prev += expert_used < expert; + + // Whether the threads at this token position have used the expert: + const int it_compact_add_self = warp_reduce_any(iex_used != -1); + + // Do a scan over threads at lower token positions in warp to get the correct index for writing data: + int it_compact_add_lower = 0; +#pragma unroll + for (int offset = neu_padded; offset < warp_size; offset += neu_padded) { + const int tmp = __shfl_up_sync(0xFFFFFFFF, it_compact_add_self, offset, warp_size); + if (threadIdx.x >= static_cast(offset)) { + it_compact_add_lower += tmp; + } + } + + if (iex_used != -1) { + store[it_compact + it_compact_add_lower] = mm_ids_helper_store(it, iex_used); + } + + // The thread with the highest index in the warp always has the sum over the whole warp, use it to increment all threads: + it_compact += __shfl_sync(0xFFFFFFFF, it_compact_add_lower + it_compact_add_self, warp_size - 1, warp_size); + } + } + nex_prev = warp_reduce_sum(nex_prev); + + for (int itc = threadIdx.x; itc < it_compact; itc += warp_size) { + const mm_ids_helper_store store_it = store[itc]; + const int it = store_it.it(); + const int iex_used = store_it.iex_used(); + ids_dst[nex_prev + itc] = it*n_expert_used + iex_used; + // ids_src1 holds the forward map, or the inverse map (token slot -> compact row) for quant dedup + if (write_inverse) { + ids_src1[it*n_expert_used + iex_used] = nex_prev + itc; + } else { + ids_src1[nex_prev + itc] = it*sis1 + iex_used % nchannels_y; + } + } + + if (threadIdx.x != 0) { + return; + } + + expert_bounds[expert] = nex_prev; + + if (expert < static_cast(gridDim.x) - 1) { + return; + } + + expert_bounds[gridDim.x] = nex_prev + it_compact; +} + +template +static void launch_mm_ids_helper( + const int32_t * __restrict__ ids, int32_t * __restrict__ ids_src1, int32_t * __restrict__ ids_dst, int32_t * __restrict__ expert_bounds, + const int n_experts, const int n_tokens, const int n_expert_used_var, const int nchannels_y, const int si1, const int sis1, const bool write_inverse, cudaStream_t stream) { + GGML_ASSERT(n_tokens < (1 << 22) && "too few bits in mm_ids_helper_store"); + GGML_ASSERT(n_expert_used_var < (1 << 10) && "too few bits in mm_ids_helper_store"); + + const int id = ggml_cuda_get_device(); + const int warp_size = ggml_cuda_info().devices[id].warp_size; + const size_t smpbo = ggml_cuda_info().devices[id].smpbo; + CUDA_SET_SHARED_MEMORY_LIMIT(mm_ids_helper, smpbo); + + const dim3 num_blocks(n_experts, 1, 1); + const dim3 block_size(warp_size, 1, 1); + const size_t nbytes_shared = n_tokens*sizeof(mm_ids_helper_store); + GGML_ASSERT(nbytes_shared <= smpbo); + mm_ids_helper<<>> + (ids, ids_src1, ids_dst, expert_bounds, n_tokens, n_expert_used_var, nchannels_y, si1, sis1, write_inverse); +} + +void ggml_cuda_launch_mm_ids_helper( + const int32_t * __restrict__ ids, int32_t * __restrict__ ids_src1, int32_t * __restrict__ ids_dst, int32_t * __restrict__ expert_bounds, + const int n_experts, const int n_tokens, const int n_expert_used, const int nchannels_y, const int si1, const int sis1, const bool write_inverse, cudaStream_t stream) { + switch (n_expert_used) { + case 2: + launch_mm_ids_helper< 2>(ids, ids_src1, ids_dst, expert_bounds, n_experts, n_tokens, n_expert_used, nchannels_y, si1, sis1, write_inverse, stream); + break; + case 4: + launch_mm_ids_helper< 4>(ids, ids_src1, ids_dst, expert_bounds, n_experts, n_tokens, n_expert_used, nchannels_y, si1, sis1, write_inverse, stream); + break; + case 6: + launch_mm_ids_helper< 6>(ids, ids_src1, ids_dst, expert_bounds, n_experts, n_tokens, n_expert_used, nchannels_y, si1, sis1, write_inverse, stream); + break; + case 8: + launch_mm_ids_helper< 8>(ids, ids_src1, ids_dst, expert_bounds, n_experts, n_tokens, n_expert_used, nchannels_y, si1, sis1, write_inverse, stream); + break; + case 16: + launch_mm_ids_helper<16>(ids, ids_src1, ids_dst, expert_bounds, n_experts, n_tokens, n_expert_used, nchannels_y, si1, sis1, write_inverse, stream); + break; + case 32: + launch_mm_ids_helper<32>(ids, ids_src1, ids_dst, expert_bounds, n_experts, n_tokens, n_expert_used, nchannels_y, si1, sis1, write_inverse, stream); + break; + default: + launch_mm_ids_helper< 0>(ids, ids_src1, ids_dst, expert_bounds, n_experts, n_tokens, n_expert_used, nchannels_y, si1, sis1, write_inverse, stream); + break; + } +} diff --git a/python/freetoken/kernel/csrc/gguf_mmq/mmid.cuh b/python/freetoken/kernel/csrc/gguf_mmq/mmid.cuh new file mode 100644 index 00000000..74c2db43 --- /dev/null +++ b/python/freetoken/kernel/csrc/gguf_mmq/mmid.cuh @@ -0,0 +1,5 @@ +#pragma once + +void ggml_cuda_launch_mm_ids_helper( + const int32_t * ids, int32_t * ids_src1, int32_t * ids_dst, int32_t * expert_bounds, + int n_experts, int n_tokens, int n_expert_used, int nchannels_y, int si1, int sis1, bool write_inverse, cudaStream_t stream); diff --git a/python/freetoken/kernel/csrc/gguf_mmq/mmq-config-ampere.cuh b/python/freetoken/kernel/csrc/gguf_mmq/mmq-config-ampere.cuh new file mode 100644 index 00000000..9f9fd197 --- /dev/null +++ b/python/freetoken/kernel/csrc/gguf_mmq/mmq-config-ampere.cuh @@ -0,0 +1,383 @@ +static constexpr __host__ __device__ ggml_cuda_mmq_config ggml_cuda_mmq_get_config_ampere(ggml_type type, int J, bool fallback) { + CASE(GGML_TYPE_Q1_0, 256, 1, 128, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, true); + CASE(GGML_TYPE_Q1_0, 256, 1, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, true); + CASE(GGML_TYPE_Q1_0, 256, 1, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, true); + CASE(GGML_TYPE_Q1_0, 256, 1, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, true); + CASE(GGML_TYPE_Q1_0, 256, 1, 128, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, true); + CASE(GGML_TYPE_Q1_0, 256, 1, 128, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, false); + CASE(GGML_TYPE_Q1_0, 256, 1, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, false); + CASE(GGML_TYPE_Q1_0, 256, 1, 128, 24, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, false); + CASE(GGML_TYPE_Q1_0, 256, 1, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, false); + CASE(GGML_TYPE_Q1_0, 256, 1, 128, 40, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, false); + CASE(GGML_TYPE_Q1_0, 256, 1, 128, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, false); + CASE(GGML_TYPE_Q1_0, 256, 1, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, false); + CASE(GGML_TYPE_Q1_0, 256, 1, 128, 80, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, false); + CASE(GGML_TYPE_Q1_0, 256, 1, 128, 96, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, false); + CASE(GGML_TYPE_Q1_0, 256, 1, 128, 112, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, false); + CASE(GGML_TYPE_Q1_0, 256, 1, 128, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, false); + + CASE(GGML_TYPE_Q2_0, 256, 1, 128, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, true); + CASE(GGML_TYPE_Q2_0, 256, 1, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, true); + CASE(GGML_TYPE_Q2_0, 256, 1, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, true); + CASE(GGML_TYPE_Q2_0, 256, 1, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, true); + CASE(GGML_TYPE_Q2_0, 256, 1, 128, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, true); + CASE(GGML_TYPE_Q2_0, 256, 1, 128, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, false); + CASE(GGML_TYPE_Q2_0, 256, 1, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, false); + CASE(GGML_TYPE_Q2_0, 256, 1, 128, 24, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, false); + CASE(GGML_TYPE_Q2_0, 256, 1, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, false); + CASE(GGML_TYPE_Q2_0, 256, 1, 128, 40, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, false); + CASE(GGML_TYPE_Q2_0, 256, 1, 128, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, false); + CASE(GGML_TYPE_Q2_0, 256, 1, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, false); + CASE(GGML_TYPE_Q2_0, 256, 1, 128, 80, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, false); + CASE(GGML_TYPE_Q2_0, 256, 1, 128, 96, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, false); + CASE(GGML_TYPE_Q2_0, 256, 1, 128, 112, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, false); + CASE(GGML_TYPE_Q2_0, 256, 1, 128, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, false); + + CASE(GGML_TYPE_Q4_0, 256, 1, 128, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, true); + CASE(GGML_TYPE_Q4_0, 256, 1, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, true); + CASE(GGML_TYPE_Q4_0, 256, 1, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, true); + CASE(GGML_TYPE_Q4_0, 256, 1, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, true); + CASE(GGML_TYPE_Q4_0, 256, 1, 128, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, true); + CASE(GGML_TYPE_Q4_0, 256, 1, 128, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, false); + CASE(GGML_TYPE_Q4_0, 256, 1, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, false); + CASE(GGML_TYPE_Q4_0, 256, 1, 128, 24, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, false); + CASE(GGML_TYPE_Q4_0, 256, 1, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, false); + CASE(GGML_TYPE_Q4_0, 256, 1, 128, 40, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, false); + CASE(GGML_TYPE_Q4_0, 256, 1, 128, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, false); + CASE(GGML_TYPE_Q4_0, 256, 1, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, false); + CASE(GGML_TYPE_Q4_0, 256, 1, 128, 80, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, false); + CASE(GGML_TYPE_Q4_0, 256, 1, 128, 96, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, false); + CASE(GGML_TYPE_Q4_0, 256, 1, 128, 112, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, false); + CASE(GGML_TYPE_Q4_0, 256, 1, 128, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, false); + + CASE(GGML_TYPE_Q4_1, 256, 1, 128, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, true, true); + CASE(GGML_TYPE_Q4_1, 256, 1, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, true, true); + CASE(GGML_TYPE_Q4_1, 256, 1, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, true, true); + CASE(GGML_TYPE_Q4_1, 256, 1, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, true, true); + CASE(GGML_TYPE_Q4_1, 256, 1, 128, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, true, true); + CASE(GGML_TYPE_Q4_1, 256, 1, 128, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, true, false); + CASE(GGML_TYPE_Q4_1, 256, 1, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, true, false); + CASE(GGML_TYPE_Q4_1, 256, 1, 128, 24, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, true, false); + CASE(GGML_TYPE_Q4_1, 256, 1, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, true, false); + CASE(GGML_TYPE_Q4_1, 256, 1, 128, 40, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, true, false); + CASE(GGML_TYPE_Q4_1, 256, 1, 128, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, true, false); + CASE(GGML_TYPE_Q4_1, 256, 1, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, true, false); + CASE(GGML_TYPE_Q4_1, 256, 1, 128, 80, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, true, false); + CASE(GGML_TYPE_Q4_1, 256, 1, 128, 96, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, true, false); + CASE(GGML_TYPE_Q4_1, 256, 1, 128, 112, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, true, false); + CASE(GGML_TYPE_Q4_1, 256, 1, 128, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, true, false); + + CASE(GGML_TYPE_Q5_0, 256, 1, 128, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, true); + CASE(GGML_TYPE_Q5_0, 256, 1, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, true); + CASE(GGML_TYPE_Q5_0, 256, 1, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, true); + CASE(GGML_TYPE_Q5_0, 256, 1, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, true); + CASE(GGML_TYPE_Q5_0, 256, 1, 128, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, true); + CASE(GGML_TYPE_Q5_0, 256, 1, 128, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, false); + CASE(GGML_TYPE_Q5_0, 256, 1, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, false); + CASE(GGML_TYPE_Q5_0, 256, 1, 128, 24, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, false); + CASE(GGML_TYPE_Q5_0, 256, 1, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, false); + CASE(GGML_TYPE_Q5_0, 256, 1, 128, 40, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, false); + CASE(GGML_TYPE_Q5_0, 256, 1, 128, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, false); + CASE(GGML_TYPE_Q5_0, 256, 1, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, false); + CASE(GGML_TYPE_Q5_0, 256, 1, 128, 80, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, false); + CASE(GGML_TYPE_Q5_0, 256, 1, 128, 96, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, false); + CASE(GGML_TYPE_Q5_0, 256, 1, 128, 112, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, false); + CASE(GGML_TYPE_Q5_0, 256, 1, 128, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, false); + + CASE(GGML_TYPE_Q5_1, 256, 1, 128, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, true, true); + CASE(GGML_TYPE_Q5_1, 256, 1, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, true, true); + CASE(GGML_TYPE_Q5_1, 256, 1, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, true, true); + CASE(GGML_TYPE_Q5_1, 256, 1, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, true, true); + CASE(GGML_TYPE_Q5_1, 256, 1, 128, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, true, true); + CASE(GGML_TYPE_Q5_1, 256, 1, 128, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, true, false); + CASE(GGML_TYPE_Q5_1, 256, 1, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, true, false); + CASE(GGML_TYPE_Q5_1, 256, 1, 128, 24, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, true, false); + CASE(GGML_TYPE_Q5_1, 256, 1, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, true, false); + CASE(GGML_TYPE_Q5_1, 256, 1, 128, 40, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, true, false); + CASE(GGML_TYPE_Q5_1, 256, 1, 128, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, true, false); + CASE(GGML_TYPE_Q5_1, 256, 1, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, true, false); + CASE(GGML_TYPE_Q5_1, 256, 1, 128, 80, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, true, false); + CASE(GGML_TYPE_Q5_1, 256, 1, 128, 96, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, true, false); + CASE(GGML_TYPE_Q5_1, 256, 1, 128, 112, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, true, false); + CASE(GGML_TYPE_Q5_1, 256, 1, 128, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, true, false); + + CASE(GGML_TYPE_Q8_0, 256, 1, 128, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, true); + CASE(GGML_TYPE_Q8_0, 256, 1, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, true); + CASE(GGML_TYPE_Q8_0, 256, 1, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, true); + CASE(GGML_TYPE_Q8_0, 256, 1, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, true); + CASE(GGML_TYPE_Q8_0, 256, 1, 128, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, true); + CASE(GGML_TYPE_Q8_0, 256, 1, 128, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, false); + CASE(GGML_TYPE_Q8_0, 256, 1, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, false); + CASE(GGML_TYPE_Q8_0, 256, 1, 128, 24, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, false); + CASE(GGML_TYPE_Q8_0, 256, 1, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, false); + CASE(GGML_TYPE_Q8_0, 256, 1, 128, 40, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, false); + CASE(GGML_TYPE_Q8_0, 256, 1, 128, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, false); + CASE(GGML_TYPE_Q8_0, 256, 1, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, false); + CASE(GGML_TYPE_Q8_0, 256, 1, 128, 80, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, false); + CASE(GGML_TYPE_Q8_0, 256, 1, 128, 96, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, false); + CASE(GGML_TYPE_Q8_0, 256, 1, 128, 112, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, false); + CASE(GGML_TYPE_Q8_0, 256, 1, 128, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, false); + +// --------------------------------------------------------------------------------------------- + + CASE(GGML_TYPE_Q2_K, 256, 1, 128, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q2_K, MMQ_ITER_K, true, true); + CASE(GGML_TYPE_Q2_K, 256, 1, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q2_K, MMQ_ITER_K, true, true); + CASE(GGML_TYPE_Q2_K, 256, 1, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q2_K, MMQ_ITER_K, true, true); + CASE(GGML_TYPE_Q2_K, 256, 1, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q2_K, MMQ_ITER_K, true, true); + CASE(GGML_TYPE_Q2_K, 256, 1, 128, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q2_K, MMQ_ITER_K, true, true); + CASE(GGML_TYPE_Q2_K, 256, 1, 128, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q2_K, MMQ_ITER_K, true, false); + CASE(GGML_TYPE_Q2_K, 256, 1, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q2_K, MMQ_ITER_K, true, false); + CASE(GGML_TYPE_Q2_K, 256, 1, 128, 24, GGML_CUDA_MMQ_SRAM_LAYOUT_Q2_K, MMQ_ITER_K, true, false); + CASE(GGML_TYPE_Q2_K, 256, 1, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q2_K, MMQ_ITER_K, true, false); + CASE(GGML_TYPE_Q2_K, 256, 1, 128, 40, GGML_CUDA_MMQ_SRAM_LAYOUT_Q2_K, MMQ_ITER_K, true, false); + CASE(GGML_TYPE_Q2_K, 256, 1, 128, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q2_K, MMQ_ITER_K, true, false); + CASE(GGML_TYPE_Q2_K, 256, 1, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q2_K, MMQ_ITER_K, true, false); + CASE(GGML_TYPE_Q2_K, 256, 1, 128, 80, GGML_CUDA_MMQ_SRAM_LAYOUT_Q2_K, MMQ_ITER_K, true, false); + CASE(GGML_TYPE_Q2_K, 256, 1, 128, 96, GGML_CUDA_MMQ_SRAM_LAYOUT_Q2_K, MMQ_ITER_K, true, false); + CASE(GGML_TYPE_Q2_K, 256, 1, 128, 112, GGML_CUDA_MMQ_SRAM_LAYOUT_Q2_K, MMQ_ITER_K, true, false); + CASE(GGML_TYPE_Q2_K, 256, 1, 128, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q2_K, MMQ_ITER_K, true, false); + + CASE(GGML_TYPE_Q3_K, 256, 1, 128, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, true, true); + CASE(GGML_TYPE_Q3_K, 256, 1, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, true, true); + CASE(GGML_TYPE_Q3_K, 256, 1, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, true, true); + CASE(GGML_TYPE_Q3_K, 256, 1, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, true, true); + CASE(GGML_TYPE_Q3_K, 256, 1, 128, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, true, true); + CASE(GGML_TYPE_Q3_K, 256, 1, 128, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, true, false); + CASE(GGML_TYPE_Q3_K, 256, 1, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, true, false); + CASE(GGML_TYPE_Q3_K, 256, 1, 128, 24, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, true, false); + CASE(GGML_TYPE_Q3_K, 256, 1, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, true, false); + CASE(GGML_TYPE_Q3_K, 256, 1, 128, 40, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, true, false); + CASE(GGML_TYPE_Q3_K, 256, 1, 128, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, true, false); + CASE(GGML_TYPE_Q3_K, 256, 1, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, true, false); + CASE(GGML_TYPE_Q3_K, 256, 1, 128, 80, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, true, false); + CASE(GGML_TYPE_Q3_K, 256, 1, 128, 96, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, true, false); + CASE(GGML_TYPE_Q3_K, 256, 1, 128, 112, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, true, false); + CASE(GGML_TYPE_Q3_K, 256, 1, 128, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, true, false); + + CASE(GGML_TYPE_Q4_K, 256, 1, 128, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, true, true); + CASE(GGML_TYPE_Q4_K, 256, 1, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, true, true); + CASE(GGML_TYPE_Q4_K, 256, 1, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, true, true); + CASE(GGML_TYPE_Q4_K, 256, 1, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, true, true); + CASE(GGML_TYPE_Q4_K, 256, 1, 128, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, true, true); + CASE(GGML_TYPE_Q4_K, 256, 1, 128, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, true, false); + CASE(GGML_TYPE_Q4_K, 256, 1, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, true, false); + CASE(GGML_TYPE_Q4_K, 256, 1, 128, 24, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, true, false); + CASE(GGML_TYPE_Q4_K, 256, 1, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, true, false); + CASE(GGML_TYPE_Q4_K, 256, 1, 128, 40, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, true, false); + CASE(GGML_TYPE_Q4_K, 256, 1, 128, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, true, false); + CASE(GGML_TYPE_Q4_K, 256, 1, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, true, false); + CASE(GGML_TYPE_Q4_K, 256, 1, 128, 80, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, true, false); + CASE(GGML_TYPE_Q4_K, 256, 1, 128, 96, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, true, false); + CASE(GGML_TYPE_Q4_K, 256, 1, 128, 112, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, true, false); + CASE(GGML_TYPE_Q4_K, 256, 1, 128, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, true, false); + + CASE(GGML_TYPE_Q5_K, 256, 1, 128, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, true, true); + CASE(GGML_TYPE_Q5_K, 256, 1, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, true, true); + CASE(GGML_TYPE_Q5_K, 256, 1, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, true, true); + CASE(GGML_TYPE_Q5_K, 256, 1, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, true, true); + CASE(GGML_TYPE_Q5_K, 256, 1, 128, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, true, true); + CASE(GGML_TYPE_Q5_K, 256, 1, 128, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, true, false); + CASE(GGML_TYPE_Q5_K, 256, 1, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, true, false); + CASE(GGML_TYPE_Q5_K, 256, 1, 128, 24, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, true, false); + CASE(GGML_TYPE_Q5_K, 256, 1, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, true, false); + CASE(GGML_TYPE_Q5_K, 256, 1, 128, 40, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, true, false); + CASE(GGML_TYPE_Q5_K, 256, 1, 128, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, true, false); + CASE(GGML_TYPE_Q5_K, 256, 1, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, true, false); + CASE(GGML_TYPE_Q5_K, 256, 1, 128, 80, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, true, false); + CASE(GGML_TYPE_Q5_K, 256, 1, 128, 96, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, true, false); + CASE(GGML_TYPE_Q5_K, 256, 1, 128, 112, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, true, false); + CASE(GGML_TYPE_Q5_K, 256, 1, 128, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, true, false); + + CASE(GGML_TYPE_Q6_K, 256, 1, 128, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q6_K, MMQ_ITER_K, true, true); + CASE(GGML_TYPE_Q6_K, 256, 1, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q6_K, MMQ_ITER_K, true, true); + CASE(GGML_TYPE_Q6_K, 256, 1, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q6_K, MMQ_ITER_K, true, true); + CASE(GGML_TYPE_Q6_K, 256, 1, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q6_K, MMQ_ITER_K, true, true); + CASE(GGML_TYPE_Q6_K, 256, 1, 128, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q6_K, MMQ_ITER_K, true, true); + CASE(GGML_TYPE_Q6_K, 256, 1, 128, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q6_K, MMQ_ITER_K, true, false); + CASE(GGML_TYPE_Q6_K, 256, 1, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q6_K, MMQ_ITER_K, true, false); + CASE(GGML_TYPE_Q6_K, 256, 1, 128, 24, GGML_CUDA_MMQ_SRAM_LAYOUT_Q6_K, MMQ_ITER_K, true, false); + CASE(GGML_TYPE_Q6_K, 256, 1, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q6_K, MMQ_ITER_K, true, false); + CASE(GGML_TYPE_Q6_K, 256, 1, 128, 40, GGML_CUDA_MMQ_SRAM_LAYOUT_Q6_K, MMQ_ITER_K, true, false); + CASE(GGML_TYPE_Q6_K, 256, 1, 128, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q6_K, MMQ_ITER_K, true, false); + CASE(GGML_TYPE_Q6_K, 256, 1, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q6_K, MMQ_ITER_K, true, false); + CASE(GGML_TYPE_Q6_K, 256, 1, 128, 80, GGML_CUDA_MMQ_SRAM_LAYOUT_Q6_K, MMQ_ITER_K, true, false); + CASE(GGML_TYPE_Q6_K, 256, 1, 128, 96, GGML_CUDA_MMQ_SRAM_LAYOUT_Q6_K, MMQ_ITER_K, true, false); + CASE(GGML_TYPE_Q6_K, 256, 1, 128, 112, GGML_CUDA_MMQ_SRAM_LAYOUT_Q6_K, MMQ_ITER_K, true, false); + CASE(GGML_TYPE_Q6_K, 256, 1, 128, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q6_K, MMQ_ITER_K, true, false); + +// --------------------------------------------------------------------------------------------- + + CASE(GGML_TYPE_IQ1_S, 256, 1, 128, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, true); + CASE(GGML_TYPE_IQ1_S, 256, 1, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, true); + CASE(GGML_TYPE_IQ1_S, 256, 1, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, true); + CASE(GGML_TYPE_IQ1_S, 256, 1, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, true); + CASE(GGML_TYPE_IQ1_S, 256, 1, 128, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, true); + CASE(GGML_TYPE_IQ1_S, 256, 1, 128, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, false); + CASE(GGML_TYPE_IQ1_S, 256, 1, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, false); + CASE(GGML_TYPE_IQ1_S, 256, 1, 128, 24, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, false); + CASE(GGML_TYPE_IQ1_S, 256, 1, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, false); + CASE(GGML_TYPE_IQ1_S, 256, 1, 128, 40, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, false); + CASE(GGML_TYPE_IQ1_S, 256, 1, 128, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, false); + CASE(GGML_TYPE_IQ1_S, 256, 1, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, false); + CASE(GGML_TYPE_IQ1_S, 256, 1, 128, 80, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, false); + CASE(GGML_TYPE_IQ1_S, 256, 1, 128, 96, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, false); + CASE(GGML_TYPE_IQ1_S, 256, 1, 128, 112, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, false); + CASE(GGML_TYPE_IQ1_S, 256, 1, 128, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, false); + + CASE(GGML_TYPE_IQ2_XXS, 256, 1, 128, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, true); + CASE(GGML_TYPE_IQ2_XXS, 256, 1, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, true); + CASE(GGML_TYPE_IQ2_XXS, 256, 1, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, true); + CASE(GGML_TYPE_IQ2_XXS, 256, 1, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, true); + CASE(GGML_TYPE_IQ2_XXS, 256, 1, 128, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, true); + CASE(GGML_TYPE_IQ2_XXS, 256, 1, 128, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, false); + CASE(GGML_TYPE_IQ2_XXS, 256, 1, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, false); + CASE(GGML_TYPE_IQ2_XXS, 256, 1, 128, 24, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, false); + CASE(GGML_TYPE_IQ2_XXS, 256, 1, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, false); + CASE(GGML_TYPE_IQ2_XXS, 256, 1, 128, 40, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, false); + CASE(GGML_TYPE_IQ2_XXS, 256, 1, 128, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, false); + CASE(GGML_TYPE_IQ2_XXS, 256, 1, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, false); + CASE(GGML_TYPE_IQ2_XXS, 256, 1, 128, 80, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, false); + CASE(GGML_TYPE_IQ2_XXS, 256, 1, 128, 96, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, false); + CASE(GGML_TYPE_IQ2_XXS, 256, 1, 128, 112, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, false); + CASE(GGML_TYPE_IQ2_XXS, 256, 1, 128, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, false); + + CASE(GGML_TYPE_IQ2_XS, 256, 1, 128, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, true, true); + CASE(GGML_TYPE_IQ2_XS, 256, 1, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, true, true); + CASE(GGML_TYPE_IQ2_XS, 256, 1, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, true, true); + CASE(GGML_TYPE_IQ2_XS, 256, 1, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, true, true); + CASE(GGML_TYPE_IQ2_XS, 256, 1, 128, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, true, true); + CASE(GGML_TYPE_IQ2_XS, 256, 1, 128, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, true, false); + CASE(GGML_TYPE_IQ2_XS, 256, 1, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, true, false); + CASE(GGML_TYPE_IQ2_XS, 256, 1, 128, 24, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, true, false); + CASE(GGML_TYPE_IQ2_XS, 256, 1, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, true, false); + CASE(GGML_TYPE_IQ2_XS, 256, 1, 128, 40, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, true, false); + CASE(GGML_TYPE_IQ2_XS, 256, 1, 128, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, true, false); + CASE(GGML_TYPE_IQ2_XS, 256, 1, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, true, false); + CASE(GGML_TYPE_IQ2_XS, 256, 1, 128, 80, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, true, false); + CASE(GGML_TYPE_IQ2_XS, 256, 1, 128, 96, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, true, false); + CASE(GGML_TYPE_IQ2_XS, 256, 1, 128, 112, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, true, false); + CASE(GGML_TYPE_IQ2_XS, 256, 1, 128, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, true, false); + + CASE(GGML_TYPE_IQ2_S, 256, 1, 128, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, true, true); + CASE(GGML_TYPE_IQ2_S, 256, 1, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, true, true); + CASE(GGML_TYPE_IQ2_S, 256, 1, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, true, true); + CASE(GGML_TYPE_IQ2_S, 256, 1, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, true, true); + CASE(GGML_TYPE_IQ2_S, 256, 1, 128, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, true, true); + CASE(GGML_TYPE_IQ2_S, 256, 1, 128, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, true, false); + CASE(GGML_TYPE_IQ2_S, 256, 1, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, true, false); + CASE(GGML_TYPE_IQ2_S, 256, 1, 128, 24, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, true, false); + CASE(GGML_TYPE_IQ2_S, 256, 1, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, true, false); + CASE(GGML_TYPE_IQ2_S, 256, 1, 128, 40, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, true, false); + CASE(GGML_TYPE_IQ2_S, 256, 1, 128, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, true, false); + CASE(GGML_TYPE_IQ2_S, 256, 1, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, true, false); + CASE(GGML_TYPE_IQ2_S, 256, 1, 128, 80, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, true, false); + CASE(GGML_TYPE_IQ2_S, 256, 1, 128, 96, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, true, false); + CASE(GGML_TYPE_IQ2_S, 256, 1, 128, 112, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, true, false); + CASE(GGML_TYPE_IQ2_S, 256, 1, 128, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, true, false); + + CASE(GGML_TYPE_IQ3_XXS, 256, 1, 128, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, true); + CASE(GGML_TYPE_IQ3_XXS, 256, 1, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, true); + CASE(GGML_TYPE_IQ3_XXS, 256, 1, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, true); + CASE(GGML_TYPE_IQ3_XXS, 256, 1, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, true); + CASE(GGML_TYPE_IQ3_XXS, 256, 1, 128, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, true); + CASE(GGML_TYPE_IQ3_XXS, 256, 1, 128, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, false); + CASE(GGML_TYPE_IQ3_XXS, 256, 1, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, false); + CASE(GGML_TYPE_IQ3_XXS, 256, 1, 128, 24, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, false); + CASE(GGML_TYPE_IQ3_XXS, 256, 1, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, false); + CASE(GGML_TYPE_IQ3_XXS, 256, 1, 128, 40, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, false); + CASE(GGML_TYPE_IQ3_XXS, 256, 1, 128, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, false); + CASE(GGML_TYPE_IQ3_XXS, 256, 1, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, false); + CASE(GGML_TYPE_IQ3_XXS, 256, 1, 128, 80, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, false); + CASE(GGML_TYPE_IQ3_XXS, 256, 1, 128, 96, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, false); + CASE(GGML_TYPE_IQ3_XXS, 256, 1, 128, 112, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, false); + CASE(GGML_TYPE_IQ3_XXS, 256, 1, 128, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, false); + + CASE(GGML_TYPE_IQ3_S, 256, 1, 128, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, true); + CASE(GGML_TYPE_IQ3_S, 256, 1, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, true); + CASE(GGML_TYPE_IQ3_S, 256, 1, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, true); + CASE(GGML_TYPE_IQ3_S, 256, 1, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, true); + CASE(GGML_TYPE_IQ3_S, 256, 1, 128, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, true); + CASE(GGML_TYPE_IQ3_S, 256, 1, 128, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, false); + CASE(GGML_TYPE_IQ3_S, 256, 1, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, false); + CASE(GGML_TYPE_IQ3_S, 256, 1, 128, 24, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, false); + CASE(GGML_TYPE_IQ3_S, 256, 1, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, false); + CASE(GGML_TYPE_IQ3_S, 256, 1, 128, 40, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, false); + CASE(GGML_TYPE_IQ3_S, 256, 1, 128, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, false); + CASE(GGML_TYPE_IQ3_S, 256, 1, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, false); + CASE(GGML_TYPE_IQ3_S, 256, 1, 128, 80, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, false); + CASE(GGML_TYPE_IQ3_S, 256, 1, 128, 96, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, false); + CASE(GGML_TYPE_IQ3_S, 256, 1, 128, 112, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, false); + CASE(GGML_TYPE_IQ3_S, 256, 1, 128, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, false); + + CASE(GGML_TYPE_IQ4_XS, 256, 1, 128, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, true); + CASE(GGML_TYPE_IQ4_XS, 256, 1, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, true); + CASE(GGML_TYPE_IQ4_XS, 256, 1, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, true); + CASE(GGML_TYPE_IQ4_XS, 256, 1, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, true); + CASE(GGML_TYPE_IQ4_XS, 256, 1, 128, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, true); + CASE(GGML_TYPE_IQ4_XS, 256, 1, 128, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, false); + CASE(GGML_TYPE_IQ4_XS, 256, 1, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, false); + CASE(GGML_TYPE_IQ4_XS, 256, 1, 128, 24, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, false); + CASE(GGML_TYPE_IQ4_XS, 256, 1, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, false); + CASE(GGML_TYPE_IQ4_XS, 256, 1, 128, 40, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, false); + CASE(GGML_TYPE_IQ4_XS, 256, 1, 128, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, false); + CASE(GGML_TYPE_IQ4_XS, 256, 1, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, false); + CASE(GGML_TYPE_IQ4_XS, 256, 1, 128, 80, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, false); + CASE(GGML_TYPE_IQ4_XS, 256, 1, 128, 96, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, false); + CASE(GGML_TYPE_IQ4_XS, 256, 1, 128, 112, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, false); + CASE(GGML_TYPE_IQ4_XS, 256, 1, 128, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, false); + + CASE(GGML_TYPE_IQ4_NL, 256, 1, 128, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, true); + CASE(GGML_TYPE_IQ4_NL, 256, 1, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, true); + CASE(GGML_TYPE_IQ4_NL, 256, 1, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, true); + CASE(GGML_TYPE_IQ4_NL, 256, 1, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, true); + CASE(GGML_TYPE_IQ4_NL, 256, 1, 128, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, true); + CASE(GGML_TYPE_IQ4_NL, 256, 1, 128, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, false); + CASE(GGML_TYPE_IQ4_NL, 256, 1, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, false); + CASE(GGML_TYPE_IQ4_NL, 256, 1, 128, 24, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, false); + CASE(GGML_TYPE_IQ4_NL, 256, 1, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, false); + CASE(GGML_TYPE_IQ4_NL, 256, 1, 128, 40, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, false); + CASE(GGML_TYPE_IQ4_NL, 256, 1, 128, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, false); + CASE(GGML_TYPE_IQ4_NL, 256, 1, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, false); + CASE(GGML_TYPE_IQ4_NL, 256, 1, 128, 80, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, false); + CASE(GGML_TYPE_IQ4_NL, 256, 1, 128, 96, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, false); + CASE(GGML_TYPE_IQ4_NL, 256, 1, 128, 112, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, false); + CASE(GGML_TYPE_IQ4_NL, 256, 1, 128, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, false); + +// --------------------------------------------------------------------------------------------- + + CASE(GGML_TYPE_MXFP4, 256, 1, 128, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, true, true); + CASE(GGML_TYPE_MXFP4, 256, 1, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, true, true); + CASE(GGML_TYPE_MXFP4, 256, 1, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, true, true); + CASE(GGML_TYPE_MXFP4, 256, 1, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, true, true); + CASE(GGML_TYPE_MXFP4, 256, 1, 128, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, true, true); + CASE(GGML_TYPE_MXFP4, 256, 1, 128, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, true, false); + CASE(GGML_TYPE_MXFP4, 256, 1, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, true, false); + CASE(GGML_TYPE_MXFP4, 256, 1, 128, 24, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, true, false); + CASE(GGML_TYPE_MXFP4, 256, 1, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, true, false); + CASE(GGML_TYPE_MXFP4, 256, 1, 128, 40, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, true, false); + CASE(GGML_TYPE_MXFP4, 256, 1, 128, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, true, false); + CASE(GGML_TYPE_MXFP4, 256, 1, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, true, false); + CASE(GGML_TYPE_MXFP4, 256, 1, 128, 80, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, true, false); + CASE(GGML_TYPE_MXFP4, 256, 1, 128, 96, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, true, false); + CASE(GGML_TYPE_MXFP4, 256, 1, 128, 112, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, true, false); + CASE(GGML_TYPE_MXFP4, 256, 1, 128, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, true, false); + + CASE(GGML_TYPE_NVFP4, 256, 1, 128, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_NVFP4, MMQ_ITER_K, true, true); + CASE(GGML_TYPE_NVFP4, 256, 1, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_NVFP4, MMQ_ITER_K, true, true); + CASE(GGML_TYPE_NVFP4, 256, 1, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_NVFP4, MMQ_ITER_K, true, true); + CASE(GGML_TYPE_NVFP4, 256, 1, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_NVFP4, MMQ_ITER_K, true, true); + CASE(GGML_TYPE_NVFP4, 256, 1, 128, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_NVFP4, MMQ_ITER_K, true, true); + CASE(GGML_TYPE_NVFP4, 256, 1, 128, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_NVFP4, MMQ_ITER_K, true, false); + CASE(GGML_TYPE_NVFP4, 256, 1, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_NVFP4, MMQ_ITER_K, true, false); + CASE(GGML_TYPE_NVFP4, 256, 1, 128, 24, GGML_CUDA_MMQ_SRAM_LAYOUT_NVFP4, MMQ_ITER_K, true, false); + CASE(GGML_TYPE_NVFP4, 256, 1, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_NVFP4, MMQ_ITER_K, true, false); + CASE(GGML_TYPE_NVFP4, 256, 1, 128, 40, GGML_CUDA_MMQ_SRAM_LAYOUT_NVFP4, MMQ_ITER_K, true, false); + CASE(GGML_TYPE_NVFP4, 256, 1, 128, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_NVFP4, MMQ_ITER_K, true, false); + CASE(GGML_TYPE_NVFP4, 256, 1, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_NVFP4, MMQ_ITER_K, true, false); + CASE(GGML_TYPE_NVFP4, 256, 1, 128, 80, GGML_CUDA_MMQ_SRAM_LAYOUT_NVFP4, MMQ_ITER_K, true, false); + CASE(GGML_TYPE_NVFP4, 256, 1, 128, 96, GGML_CUDA_MMQ_SRAM_LAYOUT_NVFP4, MMQ_ITER_K, true, false); + CASE(GGML_TYPE_NVFP4, 256, 1, 128, 112, GGML_CUDA_MMQ_SRAM_LAYOUT_NVFP4, MMQ_ITER_K, true, false); + CASE(GGML_TYPE_NVFP4, 256, 1, 128, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_NVFP4, MMQ_ITER_K, true, false); + + return ggml_cuda_mmq_config(GGML_TYPE_COUNT, 256, 1, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, 256, false, true); +} diff --git a/python/freetoken/kernel/csrc/gguf_mmq/mmq-config-blackwell.cuh b/python/freetoken/kernel/csrc/gguf_mmq/mmq-config-blackwell.cuh new file mode 100644 index 00000000..9fbe32b6 --- /dev/null +++ b/python/freetoken/kernel/csrc/gguf_mmq/mmq-config-blackwell.cuh @@ -0,0 +1,37 @@ +static constexpr __host__ __device__ ggml_cuda_mmq_config ggml_cuda_mmq_get_config_blackwell(ggml_type type, int J, bool fallback) { + CASE(GGML_TYPE_MXFP4, 256, 1, 128, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_FP4, MMQ_ITER_K_FP4, true, true); + CASE(GGML_TYPE_MXFP4, 256, 1, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_FP4, MMQ_ITER_K_FP4, true, true); + CASE(GGML_TYPE_MXFP4, 256, 1, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_FP4, MMQ_ITER_K_FP4, true, true); + CASE(GGML_TYPE_MXFP4, 256, 1, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_FP4, MMQ_ITER_K_FP4, true, true); + CASE(GGML_TYPE_MXFP4, 256, 1, 128, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_FP4, MMQ_ITER_K_FP4, true, true); + CASE(GGML_TYPE_MXFP4, 256, 1, 128, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_FP4, MMQ_ITER_K_FP4, true, false); + CASE(GGML_TYPE_MXFP4, 256, 1, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_FP4, MMQ_ITER_K_FP4, true, false); + CASE(GGML_TYPE_MXFP4, 256, 1, 128, 24, GGML_CUDA_MMQ_SRAM_LAYOUT_FP4, MMQ_ITER_K_FP4, true, false); + CASE(GGML_TYPE_MXFP4, 256, 1, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_FP4, MMQ_ITER_K_FP4, true, false); + CASE(GGML_TYPE_MXFP4, 256, 1, 128, 40, GGML_CUDA_MMQ_SRAM_LAYOUT_FP4, MMQ_ITER_K_FP4, true, false); + CASE(GGML_TYPE_MXFP4, 256, 1, 128, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_FP4, MMQ_ITER_K_FP4, true, false); + CASE(GGML_TYPE_MXFP4, 256, 1, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_FP4, MMQ_ITER_K_FP4, true, false); + CASE(GGML_TYPE_MXFP4, 256, 1, 128, 80, GGML_CUDA_MMQ_SRAM_LAYOUT_FP4, MMQ_ITER_K_FP4, true, false); + CASE(GGML_TYPE_MXFP4, 256, 1, 128, 96, GGML_CUDA_MMQ_SRAM_LAYOUT_FP4, MMQ_ITER_K_FP4, true, false); + CASE(GGML_TYPE_MXFP4, 256, 1, 128, 112, GGML_CUDA_MMQ_SRAM_LAYOUT_FP4, MMQ_ITER_K_FP4, true, false); + CASE(GGML_TYPE_MXFP4, 256, 1, 128, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_FP4, MMQ_ITER_K_FP4, true, false); + + CASE(GGML_TYPE_NVFP4, 256, 1, 128, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_FP4, MMQ_ITER_K_FP4, true, true); + CASE(GGML_TYPE_NVFP4, 256, 1, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_FP4, MMQ_ITER_K_FP4, true, true); + CASE(GGML_TYPE_NVFP4, 256, 1, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_FP4, MMQ_ITER_K_FP4, true, true); + CASE(GGML_TYPE_NVFP4, 256, 1, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_FP4, MMQ_ITER_K_FP4, true, true); + CASE(GGML_TYPE_NVFP4, 256, 1, 128, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_FP4, MMQ_ITER_K_FP4, true, true); + CASE(GGML_TYPE_NVFP4, 256, 1, 128, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_FP4, MMQ_ITER_K_FP4, true, false); + CASE(GGML_TYPE_NVFP4, 256, 1, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_FP4, MMQ_ITER_K_FP4, true, false); + CASE(GGML_TYPE_NVFP4, 256, 1, 128, 24, GGML_CUDA_MMQ_SRAM_LAYOUT_FP4, MMQ_ITER_K_FP4, true, false); + CASE(GGML_TYPE_NVFP4, 256, 1, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_FP4, MMQ_ITER_K_FP4, true, false); + CASE(GGML_TYPE_NVFP4, 256, 1, 128, 40, GGML_CUDA_MMQ_SRAM_LAYOUT_FP4, MMQ_ITER_K_FP4, true, false); + CASE(GGML_TYPE_NVFP4, 256, 1, 128, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_FP4, MMQ_ITER_K_FP4, true, false); + CASE(GGML_TYPE_NVFP4, 256, 1, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_FP4, MMQ_ITER_K_FP4, true, false); + CASE(GGML_TYPE_NVFP4, 256, 1, 128, 80, GGML_CUDA_MMQ_SRAM_LAYOUT_FP4, MMQ_ITER_K_FP4, true, false); + CASE(GGML_TYPE_NVFP4, 256, 1, 128, 96, GGML_CUDA_MMQ_SRAM_LAYOUT_FP4, MMQ_ITER_K_FP4, true, false); + CASE(GGML_TYPE_NVFP4, 256, 1, 128, 112, GGML_CUDA_MMQ_SRAM_LAYOUT_FP4, MMQ_ITER_K_FP4, true, false); + CASE(GGML_TYPE_NVFP4, 256, 1, 128, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_FP4, MMQ_ITER_K_FP4, true, false); + + return ggml_cuda_mmq_get_config_ampere(type, J, fallback); +} diff --git a/python/freetoken/kernel/csrc/gguf_mmq/mmq-config-cdna.cuh b/python/freetoken/kernel/csrc/gguf_mmq/mmq-config-cdna.cuh new file mode 100644 index 00000000..4a8d89f7 --- /dev/null +++ b/python/freetoken/kernel/csrc/gguf_mmq/mmq-config-cdna.cuh @@ -0,0 +1,185 @@ +static constexpr __host__ __device__ ggml_cuda_mmq_config ggml_cuda_mmq_get_config_cdna(ggml_type type, int J, bool fallback) { + CASE(GGML_TYPE_Q1_0, 512, 1, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, true); + CASE(GGML_TYPE_Q1_0, 512, 1, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, true); + CASE(GGML_TYPE_Q1_0, 512, 1, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, true); + CASE(GGML_TYPE_Q1_0, 512, 1, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, false); + CASE(GGML_TYPE_Q1_0, 512, 1, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, false); + CASE(GGML_TYPE_Q1_0, 512, 1, 128, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, false); + CASE(GGML_TYPE_Q1_0, 512, 1, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, false); + + CASE(GGML_TYPE_Q2_0, 512, 1, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, true); + CASE(GGML_TYPE_Q2_0, 512, 1, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, true); + CASE(GGML_TYPE_Q2_0, 512, 1, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, true); + CASE(GGML_TYPE_Q2_0, 512, 1, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, false); + CASE(GGML_TYPE_Q2_0, 512, 1, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, false); + CASE(GGML_TYPE_Q2_0, 512, 1, 128, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, false); + CASE(GGML_TYPE_Q2_0, 512, 1, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, false); + + CASE(GGML_TYPE_Q4_0, 512, 1, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, true); + CASE(GGML_TYPE_Q4_0, 512, 1, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, true); + CASE(GGML_TYPE_Q4_0, 512, 1, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, true); + CASE(GGML_TYPE_Q4_0, 512, 1, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, false); + CASE(GGML_TYPE_Q4_0, 512, 1, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, false); + CASE(GGML_TYPE_Q4_0, 512, 1, 128, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, false); + CASE(GGML_TYPE_Q4_0, 512, 1, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, false); + + CASE(GGML_TYPE_Q4_1, 512, 1, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, true, true); + CASE(GGML_TYPE_Q4_1, 512, 1, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, true, true); + CASE(GGML_TYPE_Q4_1, 512, 1, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, true, true); + CASE(GGML_TYPE_Q4_1, 512, 1, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, true, false); + CASE(GGML_TYPE_Q4_1, 512, 1, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, true, false); + CASE(GGML_TYPE_Q4_1, 512, 1, 128, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, true, false); + CASE(GGML_TYPE_Q4_1, 512, 1, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, true, false); + + CASE(GGML_TYPE_Q5_0, 512, 1, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, true); + CASE(GGML_TYPE_Q5_0, 512, 1, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, true); + CASE(GGML_TYPE_Q5_0, 512, 1, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, true); + CASE(GGML_TYPE_Q5_0, 512, 1, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, false); + CASE(GGML_TYPE_Q5_0, 512, 1, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, false); + CASE(GGML_TYPE_Q5_0, 512, 1, 128, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, false); + CASE(GGML_TYPE_Q5_0, 512, 1, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, false); + + CASE(GGML_TYPE_Q5_1, 512, 1, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, true, true); + CASE(GGML_TYPE_Q5_1, 512, 1, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, true, true); + CASE(GGML_TYPE_Q5_1, 512, 1, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, true, true); + CASE(GGML_TYPE_Q5_1, 512, 1, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, true, false); + CASE(GGML_TYPE_Q5_1, 512, 1, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, true, false); + CASE(GGML_TYPE_Q5_1, 512, 1, 128, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, true, false); + CASE(GGML_TYPE_Q5_1, 512, 1, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, true, false); + + CASE(GGML_TYPE_Q8_0, 512, 1, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, true); + CASE(GGML_TYPE_Q8_0, 512, 1, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, true); + CASE(GGML_TYPE_Q8_0, 512, 1, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, true); + CASE(GGML_TYPE_Q8_0, 512, 1, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, false); + CASE(GGML_TYPE_Q8_0, 512, 1, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, false); + CASE(GGML_TYPE_Q8_0, 512, 1, 128, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, false); + CASE(GGML_TYPE_Q8_0, 512, 1, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, false); + +// --------------------------------------------------------------------------------------------- + + CASE(GGML_TYPE_Q2_K, 512, 1, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q2_K, MMQ_ITER_K, true, true); + CASE(GGML_TYPE_Q2_K, 512, 1, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q2_K, MMQ_ITER_K, true, true); + CASE(GGML_TYPE_Q2_K, 512, 1, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q2_K, MMQ_ITER_K, true, true); + CASE(GGML_TYPE_Q2_K, 512, 1, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q2_K, MMQ_ITER_K, true, false); + CASE(GGML_TYPE_Q2_K, 512, 1, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q2_K, MMQ_ITER_K, true, false); + CASE(GGML_TYPE_Q2_K, 512, 1, 128, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q2_K, MMQ_ITER_K, true, false); + CASE(GGML_TYPE_Q2_K, 512, 1, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q2_K, MMQ_ITER_K, true, false); + + CASE(GGML_TYPE_Q3_K, 512, 1, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, true, true); + CASE(GGML_TYPE_Q3_K, 512, 1, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, true, true); + CASE(GGML_TYPE_Q3_K, 512, 1, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, true, true); + CASE(GGML_TYPE_Q3_K, 512, 1, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, true, false); + CASE(GGML_TYPE_Q3_K, 512, 1, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, true, false); + CASE(GGML_TYPE_Q3_K, 512, 1, 128, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, true, false); + CASE(GGML_TYPE_Q3_K, 512, 1, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, true, false); + + CASE(GGML_TYPE_Q4_K, 512, 1, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, true, true); + CASE(GGML_TYPE_Q4_K, 512, 1, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, true, true); + CASE(GGML_TYPE_Q4_K, 512, 1, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, true, true); + CASE(GGML_TYPE_Q4_K, 512, 1, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, true, false); + CASE(GGML_TYPE_Q4_K, 512, 1, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, true, false); + CASE(GGML_TYPE_Q4_K, 512, 1, 128, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, true, false); + CASE(GGML_TYPE_Q4_K, 512, 1, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, true, false); + + CASE(GGML_TYPE_Q5_K, 512, 1, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, true, true); + CASE(GGML_TYPE_Q5_K, 512, 1, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, true, true); + CASE(GGML_TYPE_Q5_K, 512, 1, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, true, true); + CASE(GGML_TYPE_Q5_K, 512, 1, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, true, false); + CASE(GGML_TYPE_Q5_K, 512, 1, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, true, false); + CASE(GGML_TYPE_Q5_K, 512, 1, 128, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, true, false); + CASE(GGML_TYPE_Q5_K, 512, 1, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, true, false); + + CASE(GGML_TYPE_Q6_K, 512, 1, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q6_K, MMQ_ITER_K, true, true); + CASE(GGML_TYPE_Q6_K, 512, 1, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q6_K, MMQ_ITER_K, true, true); + CASE(GGML_TYPE_Q6_K, 512, 1, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q6_K, MMQ_ITER_K, true, true); + CASE(GGML_TYPE_Q6_K, 512, 1, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q6_K, MMQ_ITER_K, true, false); + CASE(GGML_TYPE_Q6_K, 512, 1, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q6_K, MMQ_ITER_K, true, false); + CASE(GGML_TYPE_Q6_K, 512, 1, 128, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q6_K, MMQ_ITER_K, true, false); + CASE(GGML_TYPE_Q6_K, 512, 1, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q6_K, MMQ_ITER_K, true, false); + +// --------------------------------------------------------------------------------------------- + + CASE(GGML_TYPE_IQ1_S, 512, 1, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, true); + CASE(GGML_TYPE_IQ1_S, 512, 1, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, true); + CASE(GGML_TYPE_IQ1_S, 512, 1, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, true); + CASE(GGML_TYPE_IQ1_S, 512, 1, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, false); + CASE(GGML_TYPE_IQ1_S, 512, 1, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, false); + CASE(GGML_TYPE_IQ1_S, 512, 1, 128, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, false); + CASE(GGML_TYPE_IQ1_S, 512, 1, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, false); + + CASE(GGML_TYPE_IQ2_XXS, 512, 1, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, true); + CASE(GGML_TYPE_IQ2_XXS, 512, 1, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, true); + CASE(GGML_TYPE_IQ2_XXS, 512, 1, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, true); + CASE(GGML_TYPE_IQ2_XXS, 512, 1, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, false); + CASE(GGML_TYPE_IQ2_XXS, 512, 1, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, false); + CASE(GGML_TYPE_IQ2_XXS, 512, 1, 128, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, false); + CASE(GGML_TYPE_IQ2_XXS, 512, 1, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, false); + + CASE(GGML_TYPE_IQ2_XS, 512, 1, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, true, true); + CASE(GGML_TYPE_IQ2_XS, 512, 1, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, true, true); + CASE(GGML_TYPE_IQ2_XS, 512, 1, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, true, true); + CASE(GGML_TYPE_IQ2_XS, 512, 1, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, true, false); + CASE(GGML_TYPE_IQ2_XS, 512, 1, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, true, false); + CASE(GGML_TYPE_IQ2_XS, 512, 1, 128, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, true, false); + CASE(GGML_TYPE_IQ2_XS, 512, 1, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, true, false); + + CASE(GGML_TYPE_IQ2_S, 512, 1, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, true, true); + CASE(GGML_TYPE_IQ2_S, 512, 1, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, true, true); + CASE(GGML_TYPE_IQ2_S, 512, 1, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, true, true); + CASE(GGML_TYPE_IQ2_S, 512, 1, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, true, false); + CASE(GGML_TYPE_IQ2_S, 512, 1, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, true, false); + CASE(GGML_TYPE_IQ2_S, 512, 1, 128, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, true, false); + CASE(GGML_TYPE_IQ2_S, 512, 1, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, true, false); + + CASE(GGML_TYPE_IQ3_XXS, 512, 1, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, true); + CASE(GGML_TYPE_IQ3_XXS, 512, 1, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, true); + CASE(GGML_TYPE_IQ3_XXS, 512, 1, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, true); + CASE(GGML_TYPE_IQ3_XXS, 512, 1, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, false); + CASE(GGML_TYPE_IQ3_XXS, 512, 1, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, false); + CASE(GGML_TYPE_IQ3_XXS, 512, 1, 128, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, false); + CASE(GGML_TYPE_IQ3_XXS, 512, 1, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, false); + + CASE(GGML_TYPE_IQ3_S, 512, 1, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, true); + CASE(GGML_TYPE_IQ3_S, 512, 1, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, true); + CASE(GGML_TYPE_IQ3_S, 512, 1, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, true); + CASE(GGML_TYPE_IQ3_S, 512, 1, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, false); + CASE(GGML_TYPE_IQ3_S, 512, 1, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, false); + CASE(GGML_TYPE_IQ3_S, 512, 1, 128, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, false); + CASE(GGML_TYPE_IQ3_S, 512, 1, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, false); + + CASE(GGML_TYPE_IQ4_XS, 512, 1, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, true); + CASE(GGML_TYPE_IQ4_XS, 512, 1, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, true); + CASE(GGML_TYPE_IQ4_XS, 512, 1, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, true); + CASE(GGML_TYPE_IQ4_XS, 512, 1, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, false); + CASE(GGML_TYPE_IQ4_XS, 512, 1, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, false); + CASE(GGML_TYPE_IQ4_XS, 512, 1, 128, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, false); + CASE(GGML_TYPE_IQ4_XS, 512, 1, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, false); + + CASE(GGML_TYPE_IQ4_NL, 512, 1, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, true); + CASE(GGML_TYPE_IQ4_NL, 512, 1, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, true); + CASE(GGML_TYPE_IQ4_NL, 512, 1, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, true); + CASE(GGML_TYPE_IQ4_NL, 512, 1, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, false); + CASE(GGML_TYPE_IQ4_NL, 512, 1, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, false); + CASE(GGML_TYPE_IQ4_NL, 512, 1, 128, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, false); + CASE(GGML_TYPE_IQ4_NL, 512, 1, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, false); + +// --------------------------------------------------------------------------------------------- + + CASE(GGML_TYPE_MXFP4, 512, 1, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, true, true); + CASE(GGML_TYPE_MXFP4, 512, 1, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, true, true); + CASE(GGML_TYPE_MXFP4, 512, 1, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, true, true); + CASE(GGML_TYPE_MXFP4, 512, 1, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, true, false); + CASE(GGML_TYPE_MXFP4, 512, 1, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, true, false); + CASE(GGML_TYPE_MXFP4, 512, 1, 128, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, true, false); + CASE(GGML_TYPE_MXFP4, 512, 1, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, true, false); + + CASE(GGML_TYPE_NVFP4, 512, 1, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_NVFP4, MMQ_ITER_K, true, true); + CASE(GGML_TYPE_NVFP4, 512, 1, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_NVFP4, MMQ_ITER_K, true, true); + CASE(GGML_TYPE_NVFP4, 512, 1, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_NVFP4, MMQ_ITER_K, true, true); + CASE(GGML_TYPE_NVFP4, 512, 1, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_NVFP4, MMQ_ITER_K, true, false); + CASE(GGML_TYPE_NVFP4, 512, 1, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_NVFP4, MMQ_ITER_K, true, false); + CASE(GGML_TYPE_NVFP4, 512, 1, 128, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_NVFP4, MMQ_ITER_K, true, false); + CASE(GGML_TYPE_NVFP4, 512, 1, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_NVFP4, MMQ_ITER_K, true, false); + + return ggml_cuda_mmq_config(GGML_TYPE_COUNT, 512, 1, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, 256, false, true); +} diff --git a/python/freetoken/kernel/csrc/gguf_mmq/mmq-config-pascal.cuh b/python/freetoken/kernel/csrc/gguf_mmq/mmq-config-pascal.cuh new file mode 100644 index 00000000..e7d4a9a3 --- /dev/null +++ b/python/freetoken/kernel/csrc/gguf_mmq/mmq-config-pascal.cuh @@ -0,0 +1,273 @@ +static constexpr __host__ __device__ ggml_cuda_mmq_config ggml_cuda_mmq_get_config_pascal(ggml_type type, int J, bool fallback) { + CASE(GGML_TYPE_Q1_0, 256, 2, 64, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_Q1_0, 256, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_Q1_0, 256, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_Q1_0, 256, 2, 64, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_Q1_0, 256, 2, 64, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q1_0, 256, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q1_0, 256, 2, 64, 24, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q1_0, 256, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q1_0, 256, 2, 64, 40, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q1_0, 256, 2, 64, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q1_0, 256, 2, 64, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + + CASE(GGML_TYPE_Q2_0, 256, 2, 64, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_Q2_0, 256, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_Q2_0, 256, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_Q2_0, 256, 2, 64, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_Q2_0, 256, 2, 64, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q2_0, 256, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q2_0, 256, 2, 64, 24, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q2_0, 256, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q2_0, 256, 2, 64, 40, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q2_0, 256, 2, 64, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q2_0, 256, 2, 64, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + + CASE(GGML_TYPE_Q4_0, 256, 2, 64, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_Q4_0, 256, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_Q4_0, 256, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_Q4_0, 256, 2, 64, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_Q4_0, 256, 2, 64, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q4_0, 256, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q4_0, 256, 2, 64, 24, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q4_0, 256, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q4_0, 256, 2, 64, 40, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q4_0, 256, 2, 64, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q4_0, 256, 2, 64, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + + CASE(GGML_TYPE_Q4_1, 256, 2, 64, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_Q4_1, 256, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_Q4_1, 256, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_Q4_1, 256, 2, 64, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_Q4_1, 256, 2, 64, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q4_1, 256, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q4_1, 256, 2, 64, 24, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q4_1, 256, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q4_1, 256, 2, 64, 40, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q4_1, 256, 2, 64, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q4_1, 256, 2, 64, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false); + + CASE(GGML_TYPE_Q5_0, 256, 2, 64, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_Q5_0, 256, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_Q5_0, 256, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_Q5_0, 256, 2, 64, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_Q5_0, 256, 2, 64, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q5_0, 256, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q5_0, 256, 2, 64, 24, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q5_0, 256, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q5_0, 256, 2, 64, 40, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q5_0, 256, 2, 64, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q5_0, 256, 2, 64, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + + CASE(GGML_TYPE_Q5_1, 256, 2, 64, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_Q5_1, 256, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_Q5_1, 256, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_Q5_1, 256, 2, 64, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_Q5_1, 256, 2, 64, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q5_1, 256, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q5_1, 256, 2, 64, 24, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q5_1, 256, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q5_1, 256, 2, 64, 40, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q5_1, 256, 2, 64, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q5_1, 256, 2, 64, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false); + + CASE(GGML_TYPE_Q8_0, 256, 2, 64, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_Q8_0, 256, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_Q8_0, 256, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_Q8_0, 256, 2, 64, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_Q8_0, 256, 2, 64, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q8_0, 256, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q8_0, 256, 2, 64, 24, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q8_0, 256, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q8_0, 256, 2, 64, 40, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q8_0, 256, 2, 64, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q8_0, 256, 2, 64, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + +// --------------------------------------------------------------------------------------------- + + CASE(GGML_TYPE_Q2_K, 256, 2, 64, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q2_K, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_Q2_K, 256, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q2_K, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_Q2_K, 256, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q2_K, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_Q2_K, 256, 2, 64, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q2_K, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_Q2_K, 256, 2, 64, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q2_K, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q2_K, 256, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q2_K, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q2_K, 256, 2, 64, 24, GGML_CUDA_MMQ_SRAM_LAYOUT_Q2_K, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q2_K, 256, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q2_K, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q2_K, 256, 2, 64, 40, GGML_CUDA_MMQ_SRAM_LAYOUT_Q2_K, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q2_K, 256, 2, 64, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q2_K, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q2_K, 256, 2, 64, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q2_K, MMQ_ITER_K, false, false); + + CASE(GGML_TYPE_Q3_K, 256, 2, 64, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_Q3_K, 256, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_Q3_K, 256, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_Q3_K, 256, 2, 64, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_Q3_K, 256, 2, 64, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q3_K, 256, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q3_K, 256, 2, 64, 24, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q3_K, 256, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q3_K, 256, 2, 64, 40, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q3_K, 256, 2, 64, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q3_K, 256, 2, 64, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, false); + + CASE(GGML_TYPE_Q4_K, 256, 2, 64, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_Q4_K, 256, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_Q4_K, 256, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_Q4_K, 256, 2, 64, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_Q4_K, 256, 2, 64, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q4_K, 256, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q4_K, 256, 2, 64, 24, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q4_K, 256, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q4_K, 256, 2, 64, 40, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q4_K, 256, 2, 64, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q4_K, 256, 2, 64, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false); + + CASE(GGML_TYPE_Q5_K, 256, 2, 64, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_Q5_K, 256, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_Q5_K, 256, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_Q5_K, 256, 2, 64, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_Q5_K, 256, 2, 64, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q5_K, 256, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q5_K, 256, 2, 64, 24, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q5_K, 256, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q5_K, 256, 2, 64, 40, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q5_K, 256, 2, 64, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q5_K, 256, 2, 64, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false); + + CASE(GGML_TYPE_Q6_K, 256, 2, 64, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q6_K, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_Q6_K, 256, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q6_K, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_Q6_K, 256, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q6_K, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_Q6_K, 256, 2, 64, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q6_K, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_Q6_K, 256, 2, 64, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q6_K, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q6_K, 256, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q6_K, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q6_K, 256, 2, 64, 24, GGML_CUDA_MMQ_SRAM_LAYOUT_Q6_K, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q6_K, 256, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q6_K, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q6_K, 256, 2, 64, 40, GGML_CUDA_MMQ_SRAM_LAYOUT_Q6_K, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q6_K, 256, 2, 64, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q6_K, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q6_K, 256, 2, 64, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q6_K, MMQ_ITER_K, false, false); + +// --------------------------------------------------------------------------------------------- + + CASE(GGML_TYPE_IQ1_S, 256, 2, 64, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_IQ1_S, 256, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_IQ1_S, 256, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_IQ1_S, 256, 2, 64, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_IQ1_S, 256, 2, 64, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ1_S, 256, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ1_S, 256, 2, 64, 24, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ1_S, 256, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ1_S, 256, 2, 64, 40, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ1_S, 256, 2, 64, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ1_S, 256, 2, 64, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + + CASE(GGML_TYPE_IQ2_XXS, 256, 2, 64, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_IQ2_XXS, 256, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_IQ2_XXS, 256, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_IQ2_XXS, 256, 2, 64, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_IQ2_XXS, 256, 2, 64, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ2_XXS, 256, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ2_XXS, 256, 2, 64, 24, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ2_XXS, 256, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ2_XXS, 256, 2, 64, 40, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ2_XXS, 256, 2, 64, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ2_XXS, 256, 2, 64, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + + CASE(GGML_TYPE_IQ2_XS, 256, 2, 64, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_IQ2_XS, 256, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_IQ2_XS, 256, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_IQ2_XS, 256, 2, 64, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_IQ2_XS, 256, 2, 64, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ2_XS, 256, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ2_XS, 256, 2, 64, 24, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ2_XS, 256, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ2_XS, 256, 2, 64, 40, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ2_XS, 256, 2, 64, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ2_XS, 256, 2, 64, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, false); + + CASE(GGML_TYPE_IQ2_S, 256, 2, 64, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_IQ2_S, 256, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_IQ2_S, 256, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_IQ2_S, 256, 2, 64, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_IQ2_S, 256, 2, 64, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ2_S, 256, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ2_S, 256, 2, 64, 24, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ2_S, 256, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ2_S, 256, 2, 64, 40, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ2_S, 256, 2, 64, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ2_S, 256, 2, 64, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, false); + + CASE(GGML_TYPE_IQ3_XXS, 256, 2, 64, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_IQ3_XXS, 256, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_IQ3_XXS, 256, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_IQ3_XXS, 256, 2, 64, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_IQ3_XXS, 256, 2, 64, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ3_XXS, 256, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ3_XXS, 256, 2, 64, 24, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ3_XXS, 256, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ3_XXS, 256, 2, 64, 40, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ3_XXS, 256, 2, 64, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ3_XXS, 256, 2, 64, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + + CASE(GGML_TYPE_IQ3_S, 256, 2, 64, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_IQ3_S, 256, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_IQ3_S, 256, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_IQ3_S, 256, 2, 64, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_IQ3_S, 256, 2, 64, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ3_S, 256, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ3_S, 256, 2, 64, 24, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ3_S, 256, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ3_S, 256, 2, 64, 40, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ3_S, 256, 2, 64, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ3_S, 256, 2, 64, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + + CASE(GGML_TYPE_IQ4_XS, 256, 2, 64, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_IQ4_XS, 256, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_IQ4_XS, 256, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_IQ4_XS, 256, 2, 64, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_IQ4_XS, 256, 2, 64, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ4_XS, 256, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ4_XS, 256, 2, 64, 24, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ4_XS, 256, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ4_XS, 256, 2, 64, 40, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ4_XS, 256, 2, 64, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ4_XS, 256, 2, 64, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + + CASE(GGML_TYPE_IQ4_NL, 256, 2, 64, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_IQ4_NL, 256, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_IQ4_NL, 256, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_IQ4_NL, 256, 2, 64, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_IQ4_NL, 256, 2, 64, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ4_NL, 256, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ4_NL, 256, 2, 64, 24, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ4_NL, 256, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ4_NL, 256, 2, 64, 40, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ4_NL, 256, 2, 64, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ4_NL, 256, 2, 64, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + +// --------------------------------------------------------------------------------------------- + + CASE(GGML_TYPE_MXFP4, 256, 2, 64, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_MXFP4, 256, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_MXFP4, 256, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_MXFP4, 256, 2, 64, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_MXFP4, 256, 2, 64, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_MXFP4, 256, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_MXFP4, 256, 2, 64, 24, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_MXFP4, 256, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_MXFP4, 256, 2, 64, 40, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_MXFP4, 256, 2, 64, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_MXFP4, 256, 2, 64, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false); + + CASE(GGML_TYPE_NVFP4, 256, 2, 64, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_NVFP4, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_NVFP4, 256, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_NVFP4, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_NVFP4, 256, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_NVFP4, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_NVFP4, 256, 2, 64, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_NVFP4, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_NVFP4, 256, 2, 64, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_NVFP4, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_NVFP4, 256, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_NVFP4, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_NVFP4, 256, 2, 64, 24, GGML_CUDA_MMQ_SRAM_LAYOUT_NVFP4, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_NVFP4, 256, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_NVFP4, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_NVFP4, 256, 2, 64, 40, GGML_CUDA_MMQ_SRAM_LAYOUT_NVFP4, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_NVFP4, 256, 2, 64, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_NVFP4, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_NVFP4, 256, 2, 64, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_NVFP4, MMQ_ITER_K, false, false); + + return ggml_cuda_mmq_config(GGML_TYPE_COUNT, 256, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, 256, false, true); +} diff --git a/python/freetoken/kernel/csrc/gguf_mmq/mmq-config-rdna2.cuh b/python/freetoken/kernel/csrc/gguf_mmq/mmq-config-rdna2.cuh new file mode 100644 index 00000000..8324d9e1 --- /dev/null +++ b/python/freetoken/kernel/csrc/gguf_mmq/mmq-config-rdna2.cuh @@ -0,0 +1,273 @@ +static constexpr __host__ __device__ ggml_cuda_mmq_config ggml_cuda_mmq_get_config_rdna2(ggml_type type, int J, bool fallback) { + CASE(GGML_TYPE_Q1_0, 256, 2, 128, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_Q1_0, 256, 2, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_Q1_0, 256, 2, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_Q1_0, 256, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_Q1_0, 256, 2, 128, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q1_0, 256, 2, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q1_0, 256, 2, 128, 24, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q1_0, 256, 2, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q1_0, 256, 2, 128, 40, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q1_0, 256, 2, 128, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q1_0, 256, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + + CASE(GGML_TYPE_Q2_0, 256, 2, 128, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_Q2_0, 256, 2, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_Q2_0, 256, 2, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_Q2_0, 256, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_Q2_0, 256, 2, 128, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q2_0, 256, 2, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q2_0, 256, 2, 128, 24, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q2_0, 256, 2, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q2_0, 256, 2, 128, 40, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q2_0, 256, 2, 128, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q2_0, 256, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + + CASE(GGML_TYPE_Q4_0, 256, 2, 128, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_Q4_0, 256, 2, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_Q4_0, 256, 2, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_Q4_0, 256, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_Q4_0, 256, 2, 128, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q4_0, 256, 2, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q4_0, 256, 2, 128, 24, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q4_0, 256, 2, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q4_0, 256, 2, 128, 40, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q4_0, 256, 2, 128, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q4_0, 256, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + + CASE(GGML_TYPE_Q4_1, 256, 2, 128, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_Q4_1, 256, 2, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_Q4_1, 256, 2, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_Q4_1, 256, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_Q4_1, 256, 2, 128, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q4_1, 256, 2, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q4_1, 256, 2, 128, 24, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q4_1, 256, 2, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q4_1, 256, 2, 128, 40, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q4_1, 256, 2, 128, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q4_1, 256, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false); + + CASE(GGML_TYPE_Q5_0, 256, 2, 128, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_Q5_0, 256, 2, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_Q5_0, 256, 2, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_Q5_0, 256, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_Q5_0, 256, 2, 128, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q5_0, 256, 2, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q5_0, 256, 2, 128, 24, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q5_0, 256, 2, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q5_0, 256, 2, 128, 40, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q5_0, 256, 2, 128, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q5_0, 256, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + + CASE(GGML_TYPE_Q5_1, 256, 2, 128, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_Q5_1, 256, 2, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_Q5_1, 256, 2, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_Q5_1, 256, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_Q5_1, 256, 2, 128, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q5_1, 256, 2, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q5_1, 256, 2, 128, 24, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q5_1, 256, 2, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q5_1, 256, 2, 128, 40, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q5_1, 256, 2, 128, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q5_1, 256, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false); + + CASE(GGML_TYPE_Q8_0, 256, 2, 128, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_Q8_0, 256, 2, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_Q8_0, 256, 2, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_Q8_0, 256, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_Q8_0, 256, 2, 128, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q8_0, 256, 2, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q8_0, 256, 2, 128, 24, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q8_0, 256, 2, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q8_0, 256, 2, 128, 40, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q8_0, 256, 2, 128, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q8_0, 256, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + +// --------------------------------------------------------------------------------------------- + + CASE(GGML_TYPE_Q2_K, 256, 2, 128, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q2_K, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_Q2_K, 256, 2, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q2_K, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_Q2_K, 256, 2, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q2_K, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_Q2_K, 256, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q2_K, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_Q2_K, 256, 2, 128, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q2_K, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q2_K, 256, 2, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q2_K, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q2_K, 256, 2, 128, 24, GGML_CUDA_MMQ_SRAM_LAYOUT_Q2_K, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q2_K, 256, 2, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q2_K, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q2_K, 256, 2, 128, 40, GGML_CUDA_MMQ_SRAM_LAYOUT_Q2_K, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q2_K, 256, 2, 128, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q2_K, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q2_K, 256, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q2_K, MMQ_ITER_K, false, false); + + CASE(GGML_TYPE_Q3_K, 256, 2, 128, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_Q3_K, 256, 2, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_Q3_K, 256, 2, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_Q3_K, 256, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_Q3_K, 256, 2, 128, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q3_K, 256, 2, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q3_K, 256, 2, 128, 24, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q3_K, 256, 2, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q3_K, 256, 2, 128, 40, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q3_K, 256, 2, 128, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q3_K, 256, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, false); + + CASE(GGML_TYPE_Q4_K, 256, 2, 128, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_Q4_K, 256, 2, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_Q4_K, 256, 2, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_Q4_K, 256, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_Q4_K, 256, 2, 128, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q4_K, 256, 2, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q4_K, 256, 2, 128, 24, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q4_K, 256, 2, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q4_K, 256, 2, 128, 40, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q4_K, 256, 2, 128, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q4_K, 256, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false); + + CASE(GGML_TYPE_Q5_K, 256, 2, 128, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_Q5_K, 256, 2, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_Q5_K, 256, 2, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_Q5_K, 256, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_Q5_K, 256, 2, 128, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q5_K, 256, 2, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q5_K, 256, 2, 128, 24, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q5_K, 256, 2, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q5_K, 256, 2, 128, 40, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q5_K, 256, 2, 128, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q5_K, 256, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false); + + CASE(GGML_TYPE_Q6_K, 256, 2, 128, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q6_K, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_Q6_K, 256, 2, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q6_K, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_Q6_K, 256, 2, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q6_K, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_Q6_K, 256, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q6_K, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_Q6_K, 256, 2, 128, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q6_K, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q6_K, 256, 2, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q6_K, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q6_K, 256, 2, 128, 24, GGML_CUDA_MMQ_SRAM_LAYOUT_Q6_K, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q6_K, 256, 2, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q6_K, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q6_K, 256, 2, 128, 40, GGML_CUDA_MMQ_SRAM_LAYOUT_Q6_K, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q6_K, 256, 2, 128, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q6_K, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q6_K, 256, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q6_K, MMQ_ITER_K, false, false); + +// --------------------------------------------------------------------------------------------- + + CASE(GGML_TYPE_IQ1_S, 256, 2, 128, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_IQ1_S, 256, 2, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_IQ1_S, 256, 2, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_IQ1_S, 256, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_IQ1_S, 256, 2, 128, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ1_S, 256, 2, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ1_S, 256, 2, 128, 24, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ1_S, 256, 2, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ1_S, 256, 2, 128, 40, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ1_S, 256, 2, 128, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ1_S, 256, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + + CASE(GGML_TYPE_IQ2_XXS, 256, 2, 128, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_IQ2_XXS, 256, 2, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_IQ2_XXS, 256, 2, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_IQ2_XXS, 256, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_IQ2_XXS, 256, 2, 128, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ2_XXS, 256, 2, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ2_XXS, 256, 2, 128, 24, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ2_XXS, 256, 2, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ2_XXS, 256, 2, 128, 40, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ2_XXS, 256, 2, 128, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ2_XXS, 256, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + + CASE(GGML_TYPE_IQ2_XS, 256, 2, 128, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_IQ2_XS, 256, 2, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_IQ2_XS, 256, 2, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_IQ2_XS, 256, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_IQ2_XS, 256, 2, 128, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ2_XS, 256, 2, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ2_XS, 256, 2, 128, 24, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ2_XS, 256, 2, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ2_XS, 256, 2, 128, 40, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ2_XS, 256, 2, 128, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ2_XS, 256, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, false); + + CASE(GGML_TYPE_IQ2_S, 256, 2, 128, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_IQ2_S, 256, 2, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_IQ2_S, 256, 2, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_IQ2_S, 256, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_IQ2_S, 256, 2, 128, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ2_S, 256, 2, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ2_S, 256, 2, 128, 24, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ2_S, 256, 2, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ2_S, 256, 2, 128, 40, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ2_S, 256, 2, 128, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ2_S, 256, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, false); + + CASE(GGML_TYPE_IQ3_XXS, 256, 2, 128, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_IQ3_XXS, 256, 2, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_IQ3_XXS, 256, 2, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_IQ3_XXS, 256, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_IQ3_XXS, 256, 2, 128, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ3_XXS, 256, 2, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ3_XXS, 256, 2, 128, 24, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ3_XXS, 256, 2, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ3_XXS, 256, 2, 128, 40, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ3_XXS, 256, 2, 128, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ3_XXS, 256, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + + CASE(GGML_TYPE_IQ3_S, 256, 2, 128, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_IQ3_S, 256, 2, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_IQ3_S, 256, 2, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_IQ3_S, 256, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_IQ3_S, 256, 2, 128, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ3_S, 256, 2, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ3_S, 256, 2, 128, 24, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ3_S, 256, 2, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ3_S, 256, 2, 128, 40, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ3_S, 256, 2, 128, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ3_S, 256, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + + CASE(GGML_TYPE_IQ4_XS, 256, 2, 128, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_IQ4_XS, 256, 2, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_IQ4_XS, 256, 2, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_IQ4_XS, 256, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_IQ4_XS, 256, 2, 128, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ4_XS, 256, 2, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ4_XS, 256, 2, 128, 24, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ4_XS, 256, 2, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ4_XS, 256, 2, 128, 40, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ4_XS, 256, 2, 128, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ4_XS, 256, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + + CASE(GGML_TYPE_IQ4_NL, 256, 2, 128, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_IQ4_NL, 256, 2, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_IQ4_NL, 256, 2, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_IQ4_NL, 256, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_IQ4_NL, 256, 2, 128, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ4_NL, 256, 2, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ4_NL, 256, 2, 128, 24, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ4_NL, 256, 2, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ4_NL, 256, 2, 128, 40, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ4_NL, 256, 2, 128, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ4_NL, 256, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + +// --------------------------------------------------------------------------------------------- + + CASE(GGML_TYPE_MXFP4, 256, 2, 128, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_MXFP4, 256, 2, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_MXFP4, 256, 2, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_MXFP4, 256, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_MXFP4, 256, 2, 128, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_MXFP4, 256, 2, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_MXFP4, 256, 2, 128, 24, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_MXFP4, 256, 2, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_MXFP4, 256, 2, 128, 40, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_MXFP4, 256, 2, 128, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_MXFP4, 256, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false); + + CASE(GGML_TYPE_NVFP4, 256, 2, 128, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_NVFP4, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_NVFP4, 256, 2, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_NVFP4, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_NVFP4, 256, 2, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_NVFP4, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_NVFP4, 256, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_NVFP4, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_NVFP4, 256, 2, 128, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_NVFP4, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_NVFP4, 256, 2, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_NVFP4, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_NVFP4, 256, 2, 128, 24, GGML_CUDA_MMQ_SRAM_LAYOUT_NVFP4, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_NVFP4, 256, 2, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_NVFP4, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_NVFP4, 256, 2, 128, 40, GGML_CUDA_MMQ_SRAM_LAYOUT_NVFP4, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_NVFP4, 256, 2, 128, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_NVFP4, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_NVFP4, 256, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_NVFP4, MMQ_ITER_K, false, false); + + return ggml_cuda_mmq_config(GGML_TYPE_COUNT, 256, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, 256, false, true); +} diff --git a/python/freetoken/kernel/csrc/gguf_mmq/mmq-config-rdna3-5.cuh b/python/freetoken/kernel/csrc/gguf_mmq/mmq-config-rdna3-5.cuh new file mode 100644 index 00000000..180b2d93 --- /dev/null +++ b/python/freetoken/kernel/csrc/gguf_mmq/mmq-config-rdna3-5.cuh @@ -0,0 +1,290 @@ +static constexpr __host__ __device__ ggml_cuda_mmq_config ggml_cuda_mmq_get_config_rdna3_5(ggml_type type, int J, bool fallback) { + CASE(GGML_TYPE_Q1_0, 128, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_Q1_0, 128, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_Q1_0, 256, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_Q1_0, 256, 2, 128, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_Q1_0, 128, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q1_0, 128, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q1_0, 256, 2, 128, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q1_0, 256, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q1_0, 256, 2, 128, 80, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q1_0, 256, 2, 128, 96, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q1_0, 256, 2, 128, 112, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q1_0, 256, 2, 128, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q2_0, 128, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_Q2_0, 128, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_Q2_0, 256, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_Q2_0, 256, 2, 128, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_Q2_0, 128, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q2_0, 128, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q2_0, 256, 2, 128, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q2_0, 256, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q2_0, 256, 2, 128, 80, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q2_0, 256, 2, 128, 96, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q2_0, 256, 2, 128, 112, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q2_0, 256, 2, 128, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + + CASE(GGML_TYPE_Q4_0, 128, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_Q4_0, 128, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_Q4_0, 256, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_Q4_0, 256, 2, 128, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_Q4_0, 128, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q4_0, 128, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q4_0, 256, 2, 128, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q4_0, 256, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q4_0, 256, 2, 128, 80, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q4_0, 256, 2, 128, 96, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q4_0, 256, 2, 128, 112, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q4_0, 256, 2, 128, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + + CASE(GGML_TYPE_Q4_1, 128, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_Q4_1, 128, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_Q4_1, 256, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_Q4_1, 256, 2, 128, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_Q4_1, 128, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q4_1, 128, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q4_1, 256, 2, 128, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q4_1, 256, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q4_1, 256, 2, 128, 80, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q4_1, 256, 2, 128, 96, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q4_1, 256, 2, 128, 112, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q4_1, 256, 2, 128, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false); + + CASE(GGML_TYPE_Q5_0, 128, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_Q5_0, 128, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_Q5_0, 256, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_Q5_0, 256, 2, 128, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_Q5_0, 128, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q5_0, 128, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q5_0, 256, 2, 128, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q5_0, 256, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q5_0, 256, 2, 128, 80, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q5_0, 256, 2, 128, 96, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q5_0, 256, 2, 128, 112, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q5_0, 256, 2, 128, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + + CASE(GGML_TYPE_Q5_1, 128, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_Q5_1, 128, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_Q5_1, 256, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_Q5_1, 256, 2, 128, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_Q5_1, 128, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q5_1, 128, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q5_1, 256, 2, 128, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q5_1, 256, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q5_1, 256, 2, 128, 80, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q5_1, 256, 2, 128, 96, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q5_1, 256, 2, 128, 112, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q5_1, 256, 2, 128, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false); + + CASE(GGML_TYPE_Q8_0, 128, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_Q8_0, 128, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_Q8_0, 256, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_Q8_0, 256, 2, 128, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_Q8_0, 128, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q8_0, 128, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q8_0, 256, 2, 128, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q8_0, 256, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q8_0, 256, 2, 128, 80, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q8_0, 256, 2, 128, 96, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q8_0, 256, 2, 128, 112, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q8_0, 256, 2, 128, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + +// --------------------------------------------------------------------------------------------- + + CASE(GGML_TYPE_Q2_K, 128, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q2_K, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_Q2_K, 128, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q2_K, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_Q2_K, 256, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q2_K, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_Q2_K, 128, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q2_K, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q2_K, 128, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q2_K, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q2_K, 256, 2, 128, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q2_K, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q2_K, 256, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q2_K, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q2_K, 256, 2, 128, 80, GGML_CUDA_MMQ_SRAM_LAYOUT_Q2_K, MMQ_ITER_K, false, false); + + CASE(GGML_TYPE_Q3_K, 128, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_Q3_K, 128, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_Q3_K, 256, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_Q3_K, 256, 2, 128, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_Q3_K, 128, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q3_K, 128, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q3_K, 256, 2, 128, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q3_K, 256, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q3_K, 256, 2, 128, 80, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q3_K, 256, 2, 128, 96, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q3_K, 256, 2, 128, 112, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q3_K, 256, 2, 128, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, false); + + CASE(GGML_TYPE_Q4_K, 128, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_Q4_K, 128, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_Q4_K, 256, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_Q4_K, 256, 2, 128, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_Q4_K, 128, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q4_K, 128, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q4_K, 256, 2, 128, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q4_K, 256, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q4_K, 256, 2, 128, 80, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q4_K, 256, 2, 128, 96, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q4_K, 256, 2, 128, 112, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q4_K, 256, 2, 128, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false); + + CASE(GGML_TYPE_Q5_K, 128, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_Q5_K, 128, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_Q5_K, 256, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_Q5_K, 256, 2, 128, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_Q5_K, 128, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q5_K, 128, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q5_K, 256, 2, 128, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q5_K, 256, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q5_K, 256, 2, 128, 80, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q5_K, 256, 2, 128, 96, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q5_K, 256, 2, 128, 112, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q5_K, 256, 2, 128, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false); + + CASE(GGML_TYPE_Q6_K, 128, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q6_K, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_Q6_K, 128, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q6_K, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_Q6_K, 256, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q6_K, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_Q6_K, 256, 2, 128, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q6_K, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_Q6_K, 128, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q6_K, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q6_K, 128, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q6_K, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q6_K, 256, 2, 128, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q6_K, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q6_K, 256, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q6_K, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q6_K, 256, 2, 128, 80, GGML_CUDA_MMQ_SRAM_LAYOUT_Q6_K, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q6_K, 256, 2, 128, 96, GGML_CUDA_MMQ_SRAM_LAYOUT_Q6_K, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q6_K, 256, 2, 128, 112, GGML_CUDA_MMQ_SRAM_LAYOUT_Q6_K, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q6_K, 256, 2, 128, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q6_K, MMQ_ITER_K, false, false); + +// --------------------------------------------------------------------------------------------- + + CASE(GGML_TYPE_IQ1_S, 128, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_IQ1_S, 128, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_IQ1_S, 256, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_IQ1_S, 256, 2, 128, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_IQ1_S, 128, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ1_S, 128, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ1_S, 256, 2, 128, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ1_S, 256, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ1_S, 256, 2, 128, 80, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ1_S, 256, 2, 128, 96, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ1_S, 256, 2, 128, 112, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ1_S, 256, 2, 128, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + + CASE(GGML_TYPE_IQ2_XXS, 128, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_IQ2_XXS, 128, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_IQ2_XXS, 256, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_IQ2_XXS, 256, 2, 128, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_IQ2_XXS, 128, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ2_XXS, 128, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ2_XXS, 256, 2, 128, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ2_XXS, 256, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ2_XXS, 256, 2, 128, 80, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ2_XXS, 256, 2, 128, 96, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ2_XXS, 256, 2, 128, 112, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ2_XXS, 256, 2, 128, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + + CASE(GGML_TYPE_IQ2_XS, 128, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_IQ2_XS, 128, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_IQ2_XS, 256, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_IQ2_XS, 256, 2, 128, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_IQ2_XS, 128, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ2_XS, 128, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ2_XS, 256, 2, 128, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ2_XS, 256, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ2_XS, 256, 2, 128, 80, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ2_XS, 256, 2, 128, 96, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ2_XS, 256, 2, 128, 112, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ2_XS, 256, 2, 128, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, false); + + CASE(GGML_TYPE_IQ2_S, 128, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_IQ2_S, 128, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_IQ2_S, 256, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_IQ2_S, 256, 2, 128, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_IQ2_S, 128, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ2_S, 128, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ2_S, 256, 2, 128, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ2_S, 256, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ2_S, 256, 2, 128, 80, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ2_S, 256, 2, 128, 96, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ2_S, 256, 2, 128, 112, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ2_S, 256, 2, 128, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, false); + + CASE(GGML_TYPE_IQ3_XXS, 128, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_IQ3_XXS, 128, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_IQ3_XXS, 256, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_IQ3_XXS, 256, 2, 128, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_IQ3_XXS, 128, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ3_XXS, 128, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ3_XXS, 256, 2, 128, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ3_XXS, 256, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ3_XXS, 256, 2, 128, 80, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ3_XXS, 256, 2, 128, 96, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ3_XXS, 256, 2, 128, 112, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ3_XXS, 256, 2, 128, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + + CASE(GGML_TYPE_IQ3_S, 128, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_IQ3_S, 128, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_IQ3_S, 256, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_IQ3_S, 256, 2, 128, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_IQ3_S, 128, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ3_S, 128, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ3_S, 256, 2, 128, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ3_S, 256, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ3_S, 256, 2, 128, 80, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ3_S, 256, 2, 128, 96, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ3_S, 256, 2, 128, 112, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ3_S, 256, 2, 128, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + + CASE(GGML_TYPE_IQ4_XS, 128, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_IQ4_XS, 128, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_IQ4_XS, 256, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_IQ4_XS, 256, 2, 128, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_IQ4_XS, 128, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ4_XS, 128, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ4_XS, 256, 2, 128, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ4_XS, 256, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ4_XS, 256, 2, 128, 80, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ4_XS, 256, 2, 128, 96, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ4_XS, 256, 2, 128, 112, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ4_XS, 256, 2, 128, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + + CASE(GGML_TYPE_IQ4_NL, 128, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_IQ4_NL, 128, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_IQ4_NL, 256, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_IQ4_NL, 256, 2, 128, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_IQ4_NL, 128, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ4_NL, 128, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ4_NL, 256, 2, 128, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ4_NL, 256, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ4_NL, 256, 2, 128, 80, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ4_NL, 256, 2, 128, 96, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ4_NL, 256, 2, 128, 112, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ4_NL, 256, 2, 128, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + +// --------------------------------------------------------------------------------------------- + + CASE(GGML_TYPE_MXFP4, 128, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_MXFP4, 128, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_MXFP4, 256, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_MXFP4, 256, 2, 128, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_MXFP4, 128, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_MXFP4, 128, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_MXFP4, 256, 2, 128, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_MXFP4, 256, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_MXFP4, 256, 2, 128, 80, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_MXFP4, 256, 2, 128, 96, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_MXFP4, 256, 2, 128, 112, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_MXFP4, 256, 2, 128, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false); + + CASE(GGML_TYPE_NVFP4, 128, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_NVFP4, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_NVFP4, 128, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_NVFP4, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_NVFP4, 256, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_NVFP4, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_NVFP4, 256, 2, 128, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_NVFP4, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_NVFP4, 128, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_NVFP4, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_NVFP4, 128, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_NVFP4, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_NVFP4, 256, 2, 128, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_NVFP4, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_NVFP4, 256, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_NVFP4, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_NVFP4, 256, 2, 128, 80, GGML_CUDA_MMQ_SRAM_LAYOUT_NVFP4, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_NVFP4, 256, 2, 128, 96, GGML_CUDA_MMQ_SRAM_LAYOUT_NVFP4, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_NVFP4, 256, 2, 128, 112, GGML_CUDA_MMQ_SRAM_LAYOUT_NVFP4, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_NVFP4, 256, 2, 128, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_NVFP4, MMQ_ITER_K, false, false); + + return ggml_cuda_mmq_config(GGML_TYPE_COUNT, 256, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, 256, false, true); +} diff --git a/python/freetoken/kernel/csrc/gguf_mmq/mmq-config-rdna3.cuh b/python/freetoken/kernel/csrc/gguf_mmq/mmq-config-rdna3.cuh new file mode 100644 index 00000000..676f27fe --- /dev/null +++ b/python/freetoken/kernel/csrc/gguf_mmq/mmq-config-rdna3.cuh @@ -0,0 +1,290 @@ +static constexpr __host__ __device__ ggml_cuda_mmq_config ggml_cuda_mmq_get_config_rdna3(ggml_type type, int J, bool fallback) { + CASE(GGML_TYPE_Q1_0, 128, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_Q1_0, 128, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_Q1_0, 256, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_Q1_0, 256, 2, 128, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_Q1_0, 128, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q1_0, 128, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q1_0, 256, 2, 128, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q1_0, 256, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q1_0, 256, 2, 128, 80, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q1_0, 256, 2, 128, 96, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q1_0, 256, 2, 128, 112, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q1_0, 256, 2, 128, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q2_0, 128, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_Q2_0, 128, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_Q2_0, 256, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_Q2_0, 256, 2, 128, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_Q2_0, 128, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q2_0, 128, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q2_0, 256, 2, 128, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q2_0, 256, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q2_0, 256, 2, 128, 80, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q2_0, 256, 2, 128, 96, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q2_0, 256, 2, 128, 112, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q2_0, 256, 2, 128, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + + CASE(GGML_TYPE_Q4_0, 128, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_Q4_0, 128, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_Q4_0, 256, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_Q4_0, 256, 2, 128, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_Q4_0, 128, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q4_0, 128, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q4_0, 256, 2, 128, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q4_0, 256, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q4_0, 256, 2, 128, 80, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q4_0, 256, 2, 128, 96, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q4_0, 256, 2, 128, 112, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q4_0, 256, 2, 128, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + + CASE(GGML_TYPE_Q4_1, 128, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_Q4_1, 128, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_Q4_1, 256, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_Q4_1, 256, 2, 128, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_Q4_1, 128, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q4_1, 128, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q4_1, 256, 2, 128, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q4_1, 256, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q4_1, 256, 2, 128, 80, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q4_1, 256, 2, 128, 96, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q4_1, 256, 2, 128, 112, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q4_1, 256, 2, 128, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false); + + CASE(GGML_TYPE_Q5_0, 128, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_Q5_0, 128, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_Q5_0, 256, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_Q5_0, 256, 2, 128, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_Q5_0, 128, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q5_0, 128, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q5_0, 256, 2, 128, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q5_0, 256, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q5_0, 256, 2, 128, 80, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q5_0, 256, 2, 128, 96, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q5_0, 256, 2, 128, 112, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q5_0, 256, 2, 128, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + + CASE(GGML_TYPE_Q5_1, 128, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_Q5_1, 128, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_Q5_1, 256, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_Q5_1, 256, 2, 128, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_Q5_1, 128, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q5_1, 128, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q5_1, 256, 2, 128, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q5_1, 256, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q5_1, 256, 2, 128, 80, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q5_1, 256, 2, 128, 96, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q5_1, 256, 2, 128, 112, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q5_1, 256, 2, 128, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false); + + CASE(GGML_TYPE_Q8_0, 128, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_Q8_0, 128, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_Q8_0, 256, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_Q8_0, 256, 2, 128, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_Q8_0, 128, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q8_0, 128, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q8_0, 256, 2, 128, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q8_0, 256, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q8_0, 256, 2, 128, 80, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q8_0, 256, 2, 128, 96, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q8_0, 256, 2, 128, 112, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q8_0, 256, 2, 128, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + +// --------------------------------------------------------------------------------------------- + + CASE(GGML_TYPE_Q2_K, 128, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q2_K, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_Q2_K, 128, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q2_K, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_Q2_K, 256, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q2_K, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_Q2_K, 128, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q2_K, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q2_K, 128, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q2_K, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q2_K, 256, 2, 128, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q2_K, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q2_K, 256, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q2_K, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q2_K, 256, 2, 128, 80, GGML_CUDA_MMQ_SRAM_LAYOUT_Q2_K, MMQ_ITER_K, false, false); + + CASE(GGML_TYPE_Q3_K, 128, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_Q3_K, 128, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_Q3_K, 256, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_Q3_K, 256, 2, 128, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_Q3_K, 128, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q3_K, 128, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q3_K, 256, 2, 128, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q3_K, 256, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q3_K, 256, 2, 128, 80, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q3_K, 256, 2, 128, 96, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q3_K, 256, 2, 128, 112, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q3_K, 256, 2, 128, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, false); + + CASE(GGML_TYPE_Q4_K, 128, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_Q4_K, 128, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_Q4_K, 256, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_Q4_K, 256, 2, 128, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_Q4_K, 128, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q4_K, 128, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q4_K, 256, 2, 128, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q4_K, 256, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q4_K, 256, 2, 128, 80, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q4_K, 256, 2, 128, 96, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q4_K, 256, 2, 128, 112, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q4_K, 256, 2, 128, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false); + + CASE(GGML_TYPE_Q5_K, 128, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_Q5_K, 128, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_Q5_K, 256, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_Q5_K, 256, 2, 128, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_Q5_K, 128, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q5_K, 128, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q5_K, 256, 2, 128, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q5_K, 256, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q5_K, 256, 2, 128, 80, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q5_K, 256, 2, 128, 96, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q5_K, 256, 2, 128, 112, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q5_K, 256, 2, 128, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false); + + CASE(GGML_TYPE_Q6_K, 128, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q6_K, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_Q6_K, 128, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q6_K, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_Q6_K, 256, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q6_K, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_Q6_K, 256, 2, 128, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q6_K, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_Q6_K, 128, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q6_K, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q6_K, 128, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q6_K, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q6_K, 256, 2, 128, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q6_K, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q6_K, 256, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q6_K, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q6_K, 256, 2, 128, 80, GGML_CUDA_MMQ_SRAM_LAYOUT_Q6_K, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q6_K, 256, 2, 128, 96, GGML_CUDA_MMQ_SRAM_LAYOUT_Q6_K, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q6_K, 256, 2, 128, 112, GGML_CUDA_MMQ_SRAM_LAYOUT_Q6_K, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q6_K, 256, 2, 128, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q6_K, MMQ_ITER_K, false, false); + +// --------------------------------------------------------------------------------------------- + + CASE(GGML_TYPE_IQ1_S, 128, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_IQ1_S, 128, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_IQ1_S, 256, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_IQ1_S, 256, 2, 128, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_IQ1_S, 128, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ1_S, 128, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ1_S, 256, 2, 128, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ1_S, 256, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ1_S, 256, 2, 128, 80, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ1_S, 256, 2, 128, 96, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ1_S, 256, 2, 128, 112, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ1_S, 256, 2, 128, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + + CASE(GGML_TYPE_IQ2_XXS, 128, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_IQ2_XXS, 128, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_IQ2_XXS, 256, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_IQ2_XXS, 256, 2, 128, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_IQ2_XXS, 128, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ2_XXS, 128, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ2_XXS, 256, 2, 128, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ2_XXS, 256, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ2_XXS, 256, 2, 128, 80, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ2_XXS, 256, 2, 128, 96, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ2_XXS, 256, 2, 128, 112, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ2_XXS, 256, 2, 128, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + + CASE(GGML_TYPE_IQ2_XS, 128, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_IQ2_XS, 128, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_IQ2_XS, 256, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_IQ2_XS, 256, 2, 128, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_IQ2_XS, 128, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ2_XS, 128, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ2_XS, 256, 2, 128, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ2_XS, 256, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ2_XS, 256, 2, 128, 80, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ2_XS, 256, 2, 128, 96, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ2_XS, 256, 2, 128, 112, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ2_XS, 256, 2, 128, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, false); + + CASE(GGML_TYPE_IQ2_S, 128, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_IQ2_S, 128, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_IQ2_S, 256, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_IQ2_S, 256, 2, 128, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_IQ2_S, 128, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ2_S, 128, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ2_S, 256, 2, 128, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ2_S, 256, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ2_S, 256, 2, 128, 80, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ2_S, 256, 2, 128, 96, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ2_S, 256, 2, 128, 112, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ2_S, 256, 2, 128, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, false); + + CASE(GGML_TYPE_IQ3_XXS, 128, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_IQ3_XXS, 128, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_IQ3_XXS, 256, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_IQ3_XXS, 256, 2, 128, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_IQ3_XXS, 128, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ3_XXS, 128, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ3_XXS, 256, 2, 128, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ3_XXS, 256, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ3_XXS, 256, 2, 128, 80, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ3_XXS, 256, 2, 128, 96, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ3_XXS, 256, 2, 128, 112, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ3_XXS, 256, 2, 128, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + + CASE(GGML_TYPE_IQ3_S, 128, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_IQ3_S, 128, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_IQ3_S, 256, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_IQ3_S, 256, 2, 128, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_IQ3_S, 128, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ3_S, 128, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ3_S, 256, 2, 128, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ3_S, 256, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ3_S, 256, 2, 128, 80, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ3_S, 256, 2, 128, 96, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ3_S, 256, 2, 128, 112, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ3_S, 256, 2, 128, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + + CASE(GGML_TYPE_IQ4_XS, 128, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_IQ4_XS, 128, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_IQ4_XS, 256, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_IQ4_XS, 256, 2, 128, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_IQ4_XS, 128, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ4_XS, 128, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ4_XS, 256, 2, 128, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ4_XS, 256, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ4_XS, 256, 2, 128, 80, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ4_XS, 256, 2, 128, 96, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ4_XS, 256, 2, 128, 112, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ4_XS, 256, 2, 128, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + + CASE(GGML_TYPE_IQ4_NL, 128, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_IQ4_NL, 128, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_IQ4_NL, 256, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_IQ4_NL, 256, 2, 128, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_IQ4_NL, 128, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ4_NL, 128, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ4_NL, 256, 2, 128, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ4_NL, 256, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ4_NL, 256, 2, 128, 80, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ4_NL, 256, 2, 128, 96, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ4_NL, 256, 2, 128, 112, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ4_NL, 256, 2, 128, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + +// --------------------------------------------------------------------------------------------- + + CASE(GGML_TYPE_MXFP4, 128, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_MXFP4, 128, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_MXFP4, 256, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_MXFP4, 256, 2, 128, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_MXFP4, 128, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_MXFP4, 128, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_MXFP4, 256, 2, 128, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_MXFP4, 256, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_MXFP4, 256, 2, 128, 80, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_MXFP4, 256, 2, 128, 96, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_MXFP4, 256, 2, 128, 112, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_MXFP4, 256, 2, 128, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false); + + CASE(GGML_TYPE_NVFP4, 128, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_NVFP4, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_NVFP4, 128, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_NVFP4, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_NVFP4, 256, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_NVFP4, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_NVFP4, 256, 2, 128, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_NVFP4, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_NVFP4, 128, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_NVFP4, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_NVFP4, 128, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_NVFP4, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_NVFP4, 256, 2, 128, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_NVFP4, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_NVFP4, 256, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_NVFP4, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_NVFP4, 256, 2, 128, 80, GGML_CUDA_MMQ_SRAM_LAYOUT_NVFP4, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_NVFP4, 256, 2, 128, 96, GGML_CUDA_MMQ_SRAM_LAYOUT_NVFP4, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_NVFP4, 256, 2, 128, 112, GGML_CUDA_MMQ_SRAM_LAYOUT_NVFP4, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_NVFP4, 256, 2, 128, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_NVFP4, MMQ_ITER_K, false, false); + + return ggml_cuda_mmq_config(GGML_TYPE_COUNT, 256, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, 256, false, true); +} diff --git a/python/freetoken/kernel/csrc/gguf_mmq/mmq-config-rdna4.cuh b/python/freetoken/kernel/csrc/gguf_mmq/mmq-config-rdna4.cuh new file mode 100644 index 00000000..9293d9d5 --- /dev/null +++ b/python/freetoken/kernel/csrc/gguf_mmq/mmq-config-rdna4.cuh @@ -0,0 +1,290 @@ +static constexpr __host__ __device__ ggml_cuda_mmq_config ggml_cuda_mmq_get_config_rdna4(ggml_type type, int J, bool fallback) { + CASE(GGML_TYPE_Q1_0, 128, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_Q1_0, 128, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_Q1_0, 128, 2, 64, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_Q1_0, 256, 2, 128, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_Q1_0, 128, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q1_0, 128, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q1_0, 128, 2, 64, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q1_0, 128, 2, 64, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q1_0, 256, 2, 128, 80, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q1_0, 256, 2, 128, 96, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q1_0, 256, 2, 128, 112, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q1_0, 256, 2, 128, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q2_0, 128, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_Q2_0, 128, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_Q2_0, 128, 2, 64, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_Q2_0, 256, 2, 128, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_Q2_0, 128, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q2_0, 128, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q2_0, 128, 2, 64, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q2_0, 128, 2, 64, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q2_0, 256, 2, 128, 80, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q2_0, 256, 2, 128, 96, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q2_0, 256, 2, 128, 112, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q2_0, 256, 2, 128, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + + CASE(GGML_TYPE_Q4_0, 128, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_Q4_0, 128, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_Q4_0, 128, 2, 64, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_Q4_0, 256, 2, 128, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_Q4_0, 128, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q4_0, 128, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q4_0, 128, 2, 64, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q4_0, 128, 2, 64, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q4_0, 256, 2, 128, 80, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q4_0, 256, 2, 128, 96, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q4_0, 256, 2, 128, 112, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q4_0, 256, 2, 128, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + + CASE(GGML_TYPE_Q4_1, 128, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_Q4_1, 128, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_Q4_1, 128, 2, 64, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_Q4_1, 256, 2, 128, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_Q4_1, 128, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q4_1, 128, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q4_1, 128, 2, 64, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q4_1, 128, 2, 64, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q4_1, 256, 2, 128, 80, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q4_1, 256, 2, 128, 96, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q4_1, 256, 2, 128, 112, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q4_1, 256, 2, 128, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false); + + CASE(GGML_TYPE_Q5_0, 128, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_Q5_0, 128, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_Q5_0, 128, 2, 64, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_Q5_0, 256, 2, 128, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_Q5_0, 128, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q5_0, 128, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q5_0, 128, 2, 64, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q5_0, 128, 2, 64, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q5_0, 256, 2, 128, 80, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q5_0, 256, 2, 128, 96, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q5_0, 256, 2, 128, 112, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q5_0, 256, 2, 128, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + + CASE(GGML_TYPE_Q5_1, 128, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_Q5_1, 128, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_Q5_1, 128, 2, 64, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_Q5_1, 256, 2, 128, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_Q5_1, 128, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q5_1, 128, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q5_1, 128, 2, 64, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q5_1, 128, 2, 64, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q5_1, 256, 2, 128, 80, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q5_1, 256, 2, 128, 96, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q5_1, 256, 2, 128, 112, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q5_1, 256, 2, 128, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false); + + CASE(GGML_TYPE_Q8_0, 128, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_Q8_0, 128, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_Q8_0, 128, 2, 64, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_Q8_0, 256, 2, 128, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_Q8_0, 128, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q8_0, 128, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q8_0, 128, 2, 64, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q8_0, 128, 2, 64, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q8_0, 256, 2, 128, 80, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q8_0, 256, 2, 128, 96, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q8_0, 256, 2, 128, 112, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q8_0, 256, 2, 128, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + +// --------------------------------------------------------------------------------------------- + + CASE(GGML_TYPE_Q2_K, 128, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q2_K, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_Q2_K, 128, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q2_K, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_Q2_K, 128, 2, 64, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q2_K, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_Q2_K, 128, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q2_K, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q2_K, 128, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q2_K, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q2_K, 128, 2, 64, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q2_K, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q2_K, 128, 2, 64, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q2_K, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q2_K, 256, 2, 128, 80, GGML_CUDA_MMQ_SRAM_LAYOUT_Q2_K, MMQ_ITER_K, false, false); + + CASE(GGML_TYPE_Q3_K, 128, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_Q3_K, 128, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_Q3_K, 128, 2, 64, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_Q3_K, 256, 2, 128, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_Q3_K, 128, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q3_K, 128, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q3_K, 128, 2, 64, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q3_K, 128, 2, 64, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q3_K, 256, 2, 128, 80, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q3_K, 256, 2, 128, 96, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q3_K, 256, 2, 128, 112, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q3_K, 256, 2, 128, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, false); + + CASE(GGML_TYPE_Q4_K, 128, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_Q4_K, 128, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_Q4_K, 128, 2, 64, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_Q4_K, 256, 2, 128, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_Q4_K, 128, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q4_K, 128, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q4_K, 128, 2, 64, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q4_K, 128, 2, 64, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q4_K, 256, 2, 128, 80, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q4_K, 256, 2, 128, 96, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q4_K, 256, 2, 128, 112, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q4_K, 256, 2, 128, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false); + + CASE(GGML_TYPE_Q5_K, 128, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_Q5_K, 128, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_Q5_K, 128, 2, 64, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_Q5_K, 256, 2, 128, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_Q5_K, 128, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q5_K, 128, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q5_K, 128, 2, 64, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q5_K, 128, 2, 64, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q5_K, 256, 2, 128, 80, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q5_K, 256, 2, 128, 96, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q5_K, 256, 2, 128, 112, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q5_K, 256, 2, 128, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false); + + CASE(GGML_TYPE_Q6_K, 128, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q6_K, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_Q6_K, 128, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q6_K, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_Q6_K, 128, 2, 64, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q6_K, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_Q6_K, 256, 2, 128, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q6_K, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_Q6_K, 128, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q6_K, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q6_K, 128, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q6_K, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q6_K, 128, 2, 64, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q6_K, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q6_K, 128, 2, 64, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q6_K, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q6_K, 256, 2, 128, 80, GGML_CUDA_MMQ_SRAM_LAYOUT_Q6_K, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q6_K, 256, 2, 128, 96, GGML_CUDA_MMQ_SRAM_LAYOUT_Q6_K, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q6_K, 256, 2, 128, 112, GGML_CUDA_MMQ_SRAM_LAYOUT_Q6_K, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q6_K, 256, 2, 128, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q6_K, MMQ_ITER_K, false, false); + +// --------------------------------------------------------------------------------------------- + + CASE(GGML_TYPE_IQ1_S, 128, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_IQ1_S, 128, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_IQ1_S, 128, 2, 64, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_IQ1_S, 256, 2, 128, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_IQ1_S, 128, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ1_S, 128, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ1_S, 128, 2, 64, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ1_S, 128, 2, 64, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ1_S, 256, 2, 128, 80, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ1_S, 256, 2, 128, 96, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ1_S, 256, 2, 128, 112, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ1_S, 256, 2, 128, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + + CASE(GGML_TYPE_IQ2_XXS, 128, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_IQ2_XXS, 128, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_IQ2_XXS, 128, 2, 64, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_IQ2_XXS, 256, 2, 128, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_IQ2_XXS, 128, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ2_XXS, 128, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ2_XXS, 128, 2, 64, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ2_XXS, 128, 2, 64, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ2_XXS, 256, 2, 128, 80, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ2_XXS, 256, 2, 128, 96, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ2_XXS, 256, 2, 128, 112, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ2_XXS, 256, 2, 128, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + + CASE(GGML_TYPE_IQ2_XS, 128, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_IQ2_XS, 128, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_IQ2_XS, 128, 2, 64, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_IQ2_XS, 256, 2, 128, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_IQ2_XS, 128, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ2_XS, 128, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ2_XS, 128, 2, 64, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ2_XS, 128, 2, 64, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ2_XS, 256, 2, 128, 80, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ2_XS, 256, 2, 128, 96, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ2_XS, 256, 2, 128, 112, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ2_XS, 256, 2, 128, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, false); + + CASE(GGML_TYPE_IQ2_S, 128, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_IQ2_S, 128, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_IQ2_S, 128, 2, 64, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_IQ2_S, 256, 2, 128, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_IQ2_S, 128, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ2_S, 128, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ2_S, 128, 2, 64, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ2_S, 128, 2, 64, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ2_S, 256, 2, 128, 80, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ2_S, 256, 2, 128, 96, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ2_S, 256, 2, 128, 112, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ2_S, 256, 2, 128, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, false); + + CASE(GGML_TYPE_IQ3_XXS, 128, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_IQ3_XXS, 128, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_IQ3_XXS, 128, 2, 64, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_IQ3_XXS, 256, 2, 128, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_IQ3_XXS, 128, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ3_XXS, 128, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ3_XXS, 128, 2, 64, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ3_XXS, 128, 2, 64, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ3_XXS, 256, 2, 128, 80, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ3_XXS, 256, 2, 128, 96, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ3_XXS, 256, 2, 128, 112, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ3_XXS, 256, 2, 128, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + + CASE(GGML_TYPE_IQ3_S, 128, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_IQ3_S, 128, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_IQ3_S, 128, 2, 64, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_IQ3_S, 256, 2, 128, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_IQ3_S, 128, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ3_S, 128, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ3_S, 128, 2, 64, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ3_S, 128, 2, 64, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ3_S, 256, 2, 128, 80, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ3_S, 256, 2, 128, 96, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ3_S, 256, 2, 128, 112, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ3_S, 256, 2, 128, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + + CASE(GGML_TYPE_IQ4_XS, 128, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_IQ4_XS, 128, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_IQ4_XS, 128, 2, 64, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_IQ4_XS, 256, 2, 128, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_IQ4_XS, 128, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ4_XS, 128, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ4_XS, 128, 2, 64, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ4_XS, 128, 2, 64, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ4_XS, 256, 2, 128, 80, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ4_XS, 256, 2, 128, 96, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ4_XS, 256, 2, 128, 112, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ4_XS, 256, 2, 128, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + + CASE(GGML_TYPE_IQ4_NL, 128, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_IQ4_NL, 128, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_IQ4_NL, 128, 2, 64, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_IQ4_NL, 256, 2, 128, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_IQ4_NL, 128, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ4_NL, 128, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ4_NL, 128, 2, 64, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ4_NL, 128, 2, 64, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ4_NL, 256, 2, 128, 80, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ4_NL, 256, 2, 128, 96, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ4_NL, 256, 2, 128, 112, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ4_NL, 256, 2, 128, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + +// --------------------------------------------------------------------------------------------- + + CASE(GGML_TYPE_MXFP4, 128, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_MXFP4, 128, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_MXFP4, 128, 2, 64, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_MXFP4, 256, 2, 128, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_MXFP4, 128, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_MXFP4, 128, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_MXFP4, 128, 2, 64, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_MXFP4, 128, 2, 64, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_MXFP4, 256, 2, 128, 80, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_MXFP4, 256, 2, 128, 96, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_MXFP4, 256, 2, 128, 112, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_MXFP4, 256, 2, 128, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false); + + CASE(GGML_TYPE_NVFP4, 128, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_NVFP4, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_NVFP4, 128, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_NVFP4, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_NVFP4, 128, 2, 64, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_NVFP4, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_NVFP4, 256, 2, 128, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_NVFP4, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_NVFP4, 128, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_NVFP4, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_NVFP4, 128, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_NVFP4, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_NVFP4, 128, 2, 64, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_NVFP4, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_NVFP4, 128, 2, 64, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_NVFP4, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_NVFP4, 256, 2, 128, 80, GGML_CUDA_MMQ_SRAM_LAYOUT_NVFP4, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_NVFP4, 256, 2, 128, 96, GGML_CUDA_MMQ_SRAM_LAYOUT_NVFP4, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_NVFP4, 256, 2, 128, 112, GGML_CUDA_MMQ_SRAM_LAYOUT_NVFP4, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_NVFP4, 256, 2, 128, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_NVFP4, MMQ_ITER_K, false, false); + + return ggml_cuda_mmq_config(GGML_TYPE_COUNT, 256, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, 256, false, true); +} diff --git a/python/freetoken/kernel/csrc/gguf_mmq/mmq-load-tiles.cuh b/python/freetoken/kernel/csrc/gguf_mmq/mmq-load-tiles.cuh new file mode 100644 index 00000000..8ed704c2 --- /dev/null +++ b/python/freetoken/kernel/csrc/gguf_mmq/mmq-load-tiles.cuh @@ -0,0 +1,1760 @@ +#pragma once + +#include "vecdotq.cuh" + +#include "mmq.cuh" + +template static __device__ __forceinline__ void ggml_cuda_mmq_load_tiles_q1_0( + const char * __restrict__ x, int * __restrict__ x_tile, const int kbx0, const int i_max, const int stride) { + constexpr int warp_size = ggml_cuda_get_physical_warp_size(); + constexpr int nwarps = ggml_cuda_mmq_get_nthreads(type, J, fallback) / warp_size; + constexpr int I = ggml_cuda_mmq_get_I(type, J, fallback); + constexpr int sram_stride = ggml_cuda_mmq_get_sram_stride(type, J, fallback); + +#if defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE) + int * x_qs = (int *) x_tile; + float * x_df = (float *) (x_qs + 2*MMQ_TILE_NE_K); +#else + constexpr tile_x_sizes txs = mmq_get_dp4a_tile_x_sizes(GGML_TYPE_Q8_0, I); + int * x_qs = (int *) x_tile; + float * x_df = (float *) (x_qs + txs.qs); +#endif // defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE) + + constexpr int blocks_per_iter = MMQ_ITER_K / QK1_0; + constexpr int threads_per_row = blocks_per_iter * QI1_0; + constexpr int nrows = warp_size / threads_per_row; + constexpr int scale_entries_per_block = QK1_0 / QK8_1; + constexpr int scale_entries_per_row = blocks_per_iter * scale_entries_per_block; + + const int txi = threadIdx.x % threads_per_row; + const int kbx = txi / QI1_0; + const int kqsx = txi % QI1_0; + +#pragma unroll + for (int i0 = 0; i0 < I; i0 += nrows*nwarps) { + int i = i0 + threadIdx.y*nrows + threadIdx.x/threads_per_row; + + if (fallback) { + i = min(i, i_max); + } + + const block_q1_0 * bxi = (const block_q1_0 *) x + kbx0 + i*stride + kbx; + const int16_t * qxi = (const int16_t *) bxi->qs + kqsx * 2; + + const int dst_offset = kbx*(scale_entries_per_block*QI8_0) + kqsx*QI8_0; +#pragma unroll + for (int j = 0; j < 2; ++j) { + const int q = qxi[j]; + + // unpack crumbs into nibble indices + const int n0 = __byte_perm(0x11100100, 0x11100100, q >> 0); // [0, 1, 4, 5] [ 8, 9, 12, 13] + const int n1 = __byte_perm(0x11100100, 0x11100100, q >> 2); // [2, 3, 6, 7] [10, 11, 14, 15] + // unpack nibbles into byte values + const int s0 = __byte_perm(0x01FF, 0x01FF, n0 >> 0); + const int s1 = __byte_perm(0x01FF, 0x01FF, n1 >> 0); + const int s2 = __byte_perm(0x01FF, 0x01FF, n0 >> 16); + const int s3 = __byte_perm(0x01FF, 0x01FF, n1 >> 16); + // unshuffle values + const int v0 = __byte_perm(s0, s1, 0x5410); + const int v1 = __byte_perm(s0, s1, 0x7632); + const int v2 = __byte_perm(s2, s3, 0x5410); + const int v3 = __byte_perm(s2, s3, 0x7632); + +#if defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE) + x_qs[i*sram_stride + dst_offset + j*4+0] = v0; + x_qs[i*sram_stride + dst_offset + j*4+1] = v1; + x_qs[i*sram_stride + dst_offset + j*4+2] = v2; + x_qs[i*sram_stride + dst_offset + j*4+3] = v3; +#else + x_qs[i*(2*MMQ_TILE_NE_K + 1) + dst_offset + j*4+0] = v0; + x_qs[i*(2*MMQ_TILE_NE_K + 1) + dst_offset + j*4+1] = v1; + x_qs[i*(2*MMQ_TILE_NE_K + 1) + dst_offset + j*4+2] = v2; + x_qs[i*(2*MMQ_TILE_NE_K + 1) + dst_offset + j*4+3] = v3; +#endif // defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE) + } + } + + const int ksx = threadIdx.x % scale_entries_per_row; + const int scale_block = ksx / scale_entries_per_block; + +#pragma unroll + for (int i0 = 0; i0 < I; i0 += nwarps) { + int i = i0 + threadIdx.y; + + if (fallback) { + i = min(i, i_max); + } + + const block_q1_0 * bxi = (const block_q1_0 *) x + kbx0 + i*stride + scale_block; + +#if defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE) + x_df[i*sram_stride + ksx] = bxi->d; +#else + x_df[i*(2*MMQ_TILE_NE_K/QI8_0) + i/(QI8_0/2) + ksx] = bxi->d; +#endif // defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE) + } +} + +template static __device__ __forceinline__ void ggml_cuda_mmq_load_tiles_q2_0( + const char * __restrict__ x, int * __restrict__ x_tile, const int kbx0, const int i_max, const int stride) { + constexpr int warp_size = ggml_cuda_get_physical_warp_size(); + constexpr int nwarps = ggml_cuda_mmq_get_nthreads(type, J, fallback) / warp_size; + constexpr int I = ggml_cuda_mmq_get_I(type, J, fallback); + constexpr int sram_stride = ggml_cuda_mmq_get_sram_stride(type, J, fallback); + +#if defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE) + int * x_qs = (int *) x_tile; + float * x_df = (float *) (x_qs + 2*MMQ_TILE_NE_K); +#else + constexpr tile_x_sizes txs = mmq_get_dp4a_tile_x_sizes(GGML_TYPE_Q8_0, I); + int * x_qs = (int *) x_tile; + float * x_df = (float *) (x_qs + txs.qs); +#endif // defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE) + + constexpr int blocks_per_iter = MMQ_ITER_K / QK2_0; + constexpr int threads_per_row = blocks_per_iter * QI2_0; + constexpr int nrows = warp_size / threads_per_row; + constexpr int scale_entries_per_block = QK2_0 / QK8_1; + constexpr int scale_entries_per_row = blocks_per_iter * scale_entries_per_block; + + const int txi = threadIdx.x % threads_per_row; + const int kbx = txi / QI2_0; + const int kqsx = txi % QI2_0; + +#pragma unroll + for (int i0 = 0; i0 < I; i0 += nrows*nwarps) { + int i = i0 + threadIdx.y*nrows + threadIdx.x/threads_per_row; + + if (fallback) { + i = min(i, i_max); + } + + const block_q2_0 * bxi = (const block_q2_0 *) x + kbx0 + i*stride + kbx; + const int16_t * qxi = (const int16_t *) bxi->qs + kqsx * 4; + + const int dst_offset = kbx*(scale_entries_per_block*QI8_0) + kqsx*QI8_0; + +#pragma unroll + for (int j = 0; j < 4; ++j) { + const int q = qxi[j]; + + // unpack even and odd crumbs into byte values + const int qe = __byte_perm(0x020100FF, 0x020100FF, q >> 0); + const int qo = __byte_perm(0x020100FF, 0x020100FF, q >> 2); + // unshuffle values + const int qx = __byte_perm(qe, qo, 0x5140); + const int qy = __byte_perm(qe, qo, 0x7362); + +#if defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE) + x_qs[i*sram_stride + dst_offset + j*2+0] = qx; + x_qs[i*sram_stride + dst_offset + j*2+1] = qy; +#else + x_qs[i*(2*MMQ_TILE_NE_K + 1) + dst_offset + j*2+0] = qx; + x_qs[i*(2*MMQ_TILE_NE_K + 1) + dst_offset + j*2+1] = qy; +#endif // defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE) + } + } + + const int ksx = threadIdx.x % scale_entries_per_row; + const int scale_block = ksx / scale_entries_per_block; + +#pragma unroll + for (int i0 = 0; i0 < I; i0 += nwarps) { + int i = i0 + threadIdx.y; + + if (fallback) { + i = min(i, i_max); + } + + const block_q2_0 * bxi = (const block_q2_0 *) x + kbx0 + i*stride + scale_block; + +#if defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE) + x_df[i*sram_stride + ksx] = bxi->d; +#else + x_df[i*(2*MMQ_TILE_NE_K/QI8_0) + i/(QI8_0/2) + ksx] = bxi->d; +#endif // defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE) + } +} + +template static __device__ __forceinline__ void ggml_cuda_mmq_load_tiles_q4_0( + const char * __restrict__ x, int * __restrict__ x_tile, const int kbx0, const int i_max, const int stride) { + constexpr int warp_size = ggml_cuda_get_physical_warp_size(); + constexpr int nwarps = ggml_cuda_mmq_get_nthreads(type, J, fallback) / warp_size; + constexpr int I = ggml_cuda_mmq_get_I(type, J, fallback); + constexpr int sram_stride = ggml_cuda_mmq_get_sram_stride(type, J, fallback); + +#if defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE) + int * x_qs = (int *) x_tile; + float * x_df = (float *) (x_qs + 2*MMQ_TILE_NE_K); +#else + constexpr tile_x_sizes txs = mmq_get_dp4a_tile_x_sizes(GGML_TYPE_Q4_0, I); + int * x_qs = (int *) x_tile; + float * x_df = (float *) (x_qs + txs.qs); +#endif // defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE) + + constexpr int threads_per_row = MMQ_ITER_K / (4 * QR4_0); + constexpr int nrows = warp_size / threads_per_row; + const int txi = warp_size > threads_per_row ? threadIdx.x % threads_per_row : threadIdx.x; + const int kbx = txi / QI4_0; + const int kqsx = txi % QI4_0; + +#pragma unroll + for (int i0 = 0; i0 < I; i0 += nrows*nwarps) { + int i = i0 + (nrows == 1 ? threadIdx.y : threadIdx.y*nrows + threadIdx.x/threads_per_row); + + if (fallback) { + i = min(i, i_max); + } + + const block_q4_0 * bxi = (const block_q4_0 *) x + kbx0 + i*stride + kbx; + const int qs0 = get_int_b2(bxi->qs, kqsx); + +#if defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE) + x_qs[i*sram_stride + kbx*(2*QI4_0) + kqsx + 0] = __vsubss4((qs0 >> 0) & 0x0F0F0F0F, 0x08080808); + x_qs[i*sram_stride + kbx*(2*QI4_0) + kqsx + QI4_0] = __vsubss4((qs0 >> 4) & 0x0F0F0F0F, 0x08080808); +#else + x_qs[i*(MMQ_TILE_NE_K + 1) + txi] = qs0; +#endif // defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) + } + + constexpr int blocks_per_tile_x_row = MMQ_TILE_NE_K / QI4_0; + constexpr int rows_per_warp = warp_size / blocks_per_tile_x_row; + const int kbxd = threadIdx.x % blocks_per_tile_x_row; + +#pragma unroll + for (int i0 = 0; i0 < I; i0 += nwarps * rows_per_warp) { + int i = i0 + threadIdx.y * rows_per_warp + threadIdx.x / blocks_per_tile_x_row; + + if (fallback) { + i = min(i, i_max); + } + + const block_q4_0 * bxi = (const block_q4_0 *) x + kbx0 + i*stride + kbxd; + +#if defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE) + x_df[i*sram_stride + kbxd] = bxi->d; +#else + x_df[i*(MMQ_TILE_NE_K/QI4_0) + i/QI4_0 + kbxd] = bxi->d; +#endif // defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE) + } +} + +template static __device__ __forceinline__ void ggml_cuda_mmq_load_tiles_q4_1( + const char * __restrict__ x, int * __restrict__ x_tile, const int kbx0, const int i_max, const int stride) { + constexpr int warp_size = ggml_cuda_get_physical_warp_size(); + constexpr int nwarps = ggml_cuda_mmq_get_nthreads(type, J, fallback) / warp_size; + constexpr int I = ggml_cuda_mmq_get_I(type, J, fallback); + constexpr int sram_stride = ggml_cuda_mmq_get_sram_stride(type, J, fallback); + +#if defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE) + int * x_qs = (int *) x_tile; + half2 * x_dm = (half2 *) (x_qs + 2*MMQ_TILE_NE_K); +#else + constexpr tile_x_sizes txs = mmq_get_dp4a_tile_x_sizes(GGML_TYPE_Q4_1, I); + int * x_qs = (int *) x_tile; + half2 * x_dm = (half2 *) (x_qs + txs.qs); +#endif // defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE) + + constexpr int threads_per_row = MMQ_ITER_K / (4 * QR4_1); + constexpr int nrows = warp_size / threads_per_row; + const int txi = warp_size > threads_per_row ? threadIdx.x % threads_per_row : threadIdx.x; + const int kbx = txi / QI4_1; + const int kqsx = txi % QI4_1; + +#pragma unroll + for (int i0 = 0; i0 < I; i0 += nrows*nwarps) { + int i = i0 + (nrows == 1 ? threadIdx.y : threadIdx.y*nrows + threadIdx.x/threads_per_row); + + if (fallback) { + i = min(i, i_max); + } + + const block_q4_1 * bxi = (const block_q4_1 *) x + kbx0 + i*stride + kbx; + const int qs0 = get_int_b4(bxi->qs, kqsx); + +#if defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE) + x_qs[i*sram_stride + kbx*(2*QI4_1) + kqsx + 0] = (qs0 >> 0) & 0x0F0F0F0F; + x_qs[i*sram_stride + kbx*(2*QI4_1) + kqsx + QI4_1] = (qs0 >> 4) & 0x0F0F0F0F; +#else + x_qs[i*(MMQ_TILE_NE_K + 1) + txi] = qs0; +#endif // defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE) + } + + constexpr int blocks_per_tile_x_row = MMQ_TILE_NE_K / QI4_1; + constexpr int rows_per_warp = warp_size / blocks_per_tile_x_row; + const int kbxd = threadIdx.x % blocks_per_tile_x_row; + +#pragma unroll + for (int i0 = 0; i0 < I; i0 += nwarps * rows_per_warp) { + int i = i0 + threadIdx.y * rows_per_warp + threadIdx.x / blocks_per_tile_x_row; + + if (fallback) { + i = min(i, i_max); + } + + const block_q4_1 * bxi = (const block_q4_1 *) x + kbx0 + i*stride + kbxd; + +#if defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE) + x_dm[i*sram_stride + kbxd] = bxi->dm; +#else + x_dm[i*(MMQ_TILE_NE_K/QI4_1) + i/QI4_1 + kbxd] = bxi->dm; +#endif // defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE) + } +} + +template static __device__ __forceinline__ void ggml_cuda_mmq_load_tiles_q5_0( + const char * __restrict__ x, int * __restrict__ x_tile, const int kbx0, const int i_max, const int stride) { + constexpr int warp_size = ggml_cuda_get_physical_warp_size(); + constexpr int nwarps = ggml_cuda_mmq_get_nthreads(type, J, fallback) / warp_size; + constexpr int I = ggml_cuda_mmq_get_I(type, J, fallback); + constexpr int sram_stride = ggml_cuda_mmq_get_sram_stride(type, J, fallback); + +#if defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE) + int * x_qs = (int *) x_tile; + float * x_df = (float *) (x_qs + MMQ_TILE_NE_K*2); +#else + constexpr tile_x_sizes txs = mmq_get_dp4a_tile_x_sizes(GGML_TYPE_Q5_0, I); + int * x_qs = (int *) x_tile; + float * x_df = (float *) (x_qs + txs.qs); +#endif // defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE) + + constexpr int threads_per_row = MMQ_ITER_K / (4 * QR5_0); + constexpr int nrows = warp_size / threads_per_row; + const int txi = warp_size > threads_per_row ? threadIdx.x % threads_per_row : threadIdx.x; + const int kbx = txi / QI5_0; + const int kqsx = txi % QI5_0; + +#pragma unroll + for (int i0 = 0; i0 < I; i0 += nrows*nwarps) { + int i = i0 + (nrows == 1 ? threadIdx.y : threadIdx.y*nrows + threadIdx.x/threads_per_row); + + if (fallback) { + i = min(i, i_max); + } + + const block_q5_0 * bxi = (const block_q5_0 *) x + kbx0 + i*stride + kbx; + + const int ql = get_int_b2(bxi->qs, kqsx); + const int qh = get_int_b2(bxi->qh, 0) >> (4 * kqsx); + + int qs0 = (ql >> 0) & 0x0F0F0F0F; + qs0 |= (qh << 4) & 0x00000010; // 0 -> 4 + qs0 |= (qh << 11) & 0x00001000; // 1 -> 12 + qs0 |= (qh << 18) & 0x00100000; // 2 -> 20 + qs0 |= (qh << 25) & 0x10000000; // 3 -> 28 + qs0 = __vsubss4(qs0, 0x10101010); // subtract 16 + + int qs1 = (ql >> 4) & 0x0F0F0F0F; + qs1 |= (qh >> 12) & 0x00000010; // 16 -> 4 + qs1 |= (qh >> 5) & 0x00001000; // 17 -> 12 + qs1 |= (qh << 2) & 0x00100000; // 18 -> 20 + qs1 |= (qh << 9) & 0x10000000; // 19 -> 28 + qs1 = __vsubss4(qs1, 0x10101010); // subtract 16 + +#if defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE) + x_qs[i*sram_stride + kbx*(2*QI5_0) + kqsx + 0] = qs0; + x_qs[i*sram_stride + kbx*(2*QI5_0) + kqsx + QI5_0] = qs1; +#else + x_qs[i*(2*MMQ_TILE_NE_K + 1) + kbx*(2*QI5_0) + kqsx + 0] = qs0; + x_qs[i*(2*MMQ_TILE_NE_K + 1) + kbx*(2*QI5_0) + kqsx + QI5_0] = qs1; +#endif // defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE) + } + + constexpr int blocks_per_tile_x_row = MMQ_TILE_NE_K / QI5_0; + constexpr int rows_per_warp = warp_size / blocks_per_tile_x_row; + const int kbxd = threadIdx.x % blocks_per_tile_x_row; + +#pragma unroll + for (int i0 = 0; i0 < I; i0 += nwarps * rows_per_warp) { + int i = i0 + threadIdx.y * rows_per_warp + threadIdx.x / blocks_per_tile_x_row; + + if (fallback) { + i = min(i, i_max); + } + + const block_q5_0 * bxi = (const block_q5_0 *) x + kbx0 + i*stride + kbxd; + +#if defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE) + x_df[i*sram_stride + kbxd] = bxi->d; +#else + x_df[i*(MMQ_TILE_NE_K/QI5_0) + i/QI5_0 + kbxd] = bxi->d; +#endif // defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE) + } +} + +template static __device__ __forceinline__ void ggml_cuda_mmq_load_tiles_q5_1( + const char * __restrict__ x, int * __restrict__ x_tile, const int kbx0, const int i_max, const int stride) { + constexpr int warp_size = ggml_cuda_get_physical_warp_size(); + constexpr int nwarps = ggml_cuda_mmq_get_nthreads(type, J, fallback) / warp_size; + constexpr int I = ggml_cuda_mmq_get_I(type, J, fallback); + constexpr int sram_stride = ggml_cuda_mmq_get_sram_stride(type, J, fallback); + +#if defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE) + int * x_qs = (int *) x_tile; + half2 * x_dm = (half2 *) (x_qs + 2*MMQ_TILE_NE_K); +#else + constexpr tile_x_sizes txs = mmq_get_dp4a_tile_x_sizes(GGML_TYPE_Q5_1, I); + int * x_qs = (int *) x_tile; + half2 * x_dm = (half2 *) (x_qs + txs.qs); +#endif // defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE) + + constexpr int threads_per_row = MMQ_ITER_K / (4 * QR5_1); + constexpr int nrows = warp_size / threads_per_row; + const int txi = warp_size > threads_per_row ? threadIdx.x % threads_per_row : threadIdx.x; + const int kbx = txi / QI5_1; + const int kqsx = txi % QI5_1; + +#pragma unroll + for (int i0 = 0; i0 < I; i0 += nrows*nwarps) { + int i = i0 + (nrows == 1 ? threadIdx.y : threadIdx.y*nrows + threadIdx.x/threads_per_row); + + if (fallback) { + i = min(i, i_max); + } + + const block_q5_1 * bxi = (const block_q5_1 *) x + kbx0 + i*stride + kbx; + + const int ql = get_int_b4(bxi->qs, kqsx); + const int qh = get_int_b4(bxi->qh, 0) >> (4 * kqsx); + + int qs0 = (ql >> 0) & 0x0F0F0F0F; + qs0 |= (qh << 4) & 0x00000010; // 0 -> 4 + qs0 |= (qh << 11) & 0x00001000; // 1 -> 12 + qs0 |= (qh << 18) & 0x00100000; // 2 -> 20 + qs0 |= (qh << 25) & 0x10000000; // 3 -> 28 + + int qs1 = (ql >> 4) & 0x0F0F0F0F; + qs1 |= (qh >> 12) & 0x00000010; // 16 -> 4 + qs1 |= (qh >> 5) & 0x00001000; // 17 -> 12 + qs1 |= (qh << 2) & 0x00100000; // 18 -> 20 + qs1 |= (qh << 9) & 0x10000000; // 19 -> 28 + +#if defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE) + x_qs[i*sram_stride + kbx*(2*QI5_1) + kqsx + 0] = qs0; + x_qs[i*sram_stride + kbx*(2*QI5_1) + kqsx + QI5_1] = qs1; +#else + x_qs[i*(2*MMQ_TILE_NE_K + 1) + kbx*(2*QI5_1) + kqsx + 0] = qs0; + x_qs[i*(2*MMQ_TILE_NE_K + 1) + kbx*(2*QI5_1) + kqsx + QI5_1] = qs1; +#endif // defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE) + } + + constexpr int blocks_per_tile_x_row = MMQ_TILE_NE_K / QI5_1; + constexpr int rows_per_warp = warp_size / blocks_per_tile_x_row; + const int kbxd = threadIdx.x % blocks_per_tile_x_row; + +#pragma unroll + for (int i0 = 0; i0 < I; i0 += nwarps * rows_per_warp) { + int i = i0 + threadIdx.y * rows_per_warp + threadIdx.x / blocks_per_tile_x_row; + + if (fallback) { + i = min(i, i_max); + } + + const block_q5_1 * bxi = (const block_q5_1 *) x + kbx0 + i*stride + kbxd; + +#if defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE) + x_dm[i*sram_stride + kbxd] = bxi->dm; +#else + x_dm[i*(MMQ_TILE_NE_K/QI5_1) + i/QI5_1 + kbxd] = bxi->dm; +#endif // defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE) + } +} + +template static __device__ __forceinline__ void ggml_cuda_mmq_load_tiles_q8_0( + const char * __restrict__ x, int * __restrict__ x_tile, const int kbx0, const int i_max, const int stride) { + constexpr int warp_size = ggml_cuda_get_physical_warp_size(); + constexpr int nwarps = ggml_cuda_mmq_get_nthreads(type, J, fallback) / warp_size; + constexpr int I = ggml_cuda_mmq_get_I(type, J, fallback); + constexpr int sram_stride = ggml_cuda_mmq_get_sram_stride(type, J, fallback); + +#if defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE) + int * x_qs = (int *) x_tile; + float * x_df = (float *) (x_tile + 2*MMQ_TILE_NE_K); +#else + constexpr tile_x_sizes txs = mmq_get_dp4a_tile_x_sizes(GGML_TYPE_Q8_0, I); + int * x_qs = (int *) x_tile; + float * x_df = (float *) (x_qs + txs.qs); +#endif // defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE) + + // MMQ_ITER_K / (4 * QR8_0) == 64 required. but NV has only 32 threads per warp + constexpr int threads_per_row = 32; + constexpr int nrows = warp_size / threads_per_row; + const int txi = warp_size > threads_per_row ? threadIdx.x % threads_per_row : threadIdx.x; + const int kbx = txi / QI8_0; + const int kqsx = txi % QI8_0; + +#pragma unroll + for (int i0 = 0; i0 < I; i0 += nrows*nwarps) { + int i = i0 + (nrows == 1 ? threadIdx.y : threadIdx.y*nrows + threadIdx.x/threads_per_row); + + if (fallback) { + i = min(i, i_max); + } + + const block_q8_0 * bxi = (const block_q8_0 *) x + kbx0 + i*stride + kbx; + +#if defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE) + x_qs[i*sram_stride + 0 + txi] = get_int_b2(bxi[0].qs, kqsx); + x_qs[i*sram_stride + MMQ_TILE_NE_K + txi] = get_int_b2(bxi[MMQ_TILE_NE_K/QI8_0].qs, kqsx); +#else + x_qs[i*(2*MMQ_TILE_NE_K + 1) + 0 + txi] = get_int_b2(bxi[0].qs, kqsx); + x_qs[i*(2*MMQ_TILE_NE_K + 1) + MMQ_TILE_NE_K + txi] = get_int_b2(bxi[MMQ_TILE_NE_K/QI8_0].qs, kqsx); +#endif // defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE) + } + + constexpr int blocks_per_tile_x_row = 2*MMQ_TILE_NE_K / QI8_0; + constexpr int rows_per_warp = warp_size / blocks_per_tile_x_row; + const int kbxd = threadIdx.x % blocks_per_tile_x_row; + +#pragma unroll + for (int i0 = 0; i0 < I; i0 += nwarps * rows_per_warp) { + int i = i0 + threadIdx.y * rows_per_warp + threadIdx.x / blocks_per_tile_x_row; + + if (fallback) { + i = min(i, i_max); + } + + const block_q8_0 * bxi = (const block_q8_0 *) x + kbx0 + i*stride + kbxd; + +#if defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE) + x_df[i*sram_stride + kbxd] = bxi->d; +#else + x_df[i*(2*MMQ_TILE_NE_K/QI8_0) + i/(QI8_0/2) + kbxd] = bxi->d; +#endif // defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE) + } +} + +// --------------------------------------------------------------------------------------------- + +template static __device__ __forceinline__ void ggml_cuda_mmq_load_tiles_q2_K( + const char * __restrict__ x, int * __restrict__ x_tile, const int kbx0, const int i_max, const int stride) { + constexpr int warp_size = ggml_cuda_get_physical_warp_size(); + constexpr int nwarps = ggml_cuda_mmq_get_nthreads(type, J, fallback) / warp_size; + constexpr int I = ggml_cuda_mmq_get_I(type, J, fallback); + constexpr int sram_stride = ggml_cuda_mmq_get_sram_stride(type, J, fallback); + +#if defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE) + int * x_qs = (int *) x_tile; + half2 * x_dm = (half2 *) (x_qs + 2*MMQ_TILE_NE_K); +#else + constexpr tile_x_sizes txs = mmq_get_dp4a_tile_x_sizes(GGML_TYPE_Q2_K, I); + int * x_qs = (int *) x_tile; + half2 * x_dm = (half2 *) (x_qs + txs.qs); +#endif // defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE) + + constexpr int threads_per_row = MMQ_ITER_K / (4 * QR2_K); + constexpr int nrows = ggml_cuda_get_physical_warp_size() / threads_per_row; + const int kqsx = threadIdx.x % threads_per_row; + +#pragma unroll + for (int i0 = 0; i0 < I; i0 += nrows*nwarps) { + int i = i0 + threadIdx.y*nrows + threadIdx.x/threads_per_row; + + if (fallback) { + i = min(i, i_max); + } + + const block_q2_K * bxi = (const block_q2_K *) x + kbx0 + i*stride; + + const int x_ql_0 = get_int_b2(bxi->qs, kqsx); + +#pragma unroll + for (int l = 0; l < QR2_K; ++l) { + const int k = (kqsx/8)*32 + l*8 + kqsx % 8; + + const int x_qs_k = (x_ql_0 >> (2*l)) & 0x03030303; + +#if defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE) + x_qs[i*sram_stride + k] = x_qs_k; +#else + x_qs[i*(2*MMQ_TILE_NE_K + 1) + k] = x_qs_k; +#endif // defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE) + } + + const int sc_m = bxi->scales[kqsx]; +#ifdef FAST_FP16_AVAILABLE + const half2 x_dm_ik = __hmul2(bxi->dm, make_half2(sc_m & 0x0F, sc_m >> 4)); +#else + const float2 bxi_dmf = __half22float2(bxi->dm); + const half2 x_dm_ik = make_half2(bxi_dmf.x*(sc_m & 0x0F), bxi_dmf.y*(sc_m >> 4)); +#endif // FAST_FP16_AVAILABLE + +#if defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE) + x_dm[i*sram_stride + kqsx] = x_dm_ik; +#else + x_dm[i*(MMQ_TILE_NE_K + 1) + kqsx] = x_dm_ik; +#endif // defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE) + } +} + +template static __device__ __forceinline__ void ggml_cuda_mmq_load_tiles_q3_K( + const char * __restrict__ x, int * __restrict__ x_tile, const int kbx0, const int i_max, const int stride) { + constexpr int warp_size = ggml_cuda_get_physical_warp_size(); + constexpr int nwarps = ggml_cuda_mmq_get_nthreads(type, J, fallback) / warp_size; + constexpr int I = ggml_cuda_mmq_get_I(type, J, fallback); + constexpr int sram_stride = ggml_cuda_mmq_get_sram_stride(type, J, fallback); + +#if defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE) + int * x_qs = (int *) x_tile; + float * x_df = (float *) (x_qs + MMQ_TILE_NE_K*2); +#else + constexpr tile_x_sizes txs = mmq_get_dp4a_tile_x_sizes(GGML_TYPE_Q3_K, I); + int * x_qs = (int *) x_tile; + float * x_df = (float *) (x_qs + txs.qs); + int * x_sc = (int *) (x_df + txs.dm); +#endif // defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) + + constexpr int threads_per_row = MMQ_ITER_K / (4 * QR3_K); + constexpr int nrows = warp_size / threads_per_row; + const int kqsx = threadIdx.x % threads_per_row; + +#pragma unroll + for (int i0 = 0; i0 < I; i0 += nrows*nwarps) { + int i = i0 + threadIdx.y*nrows + threadIdx.x/threads_per_row; + + if (fallback) { + i = min(i, i_max); + } + + const block_q3_K * bxi = (const block_q3_K *) x + kbx0 + i*stride; + + const int x_ql_0 = get_int_b2(bxi->qs, kqsx); + const int x_qh_0 = get_int_b2(bxi->hmask, kqsx % (QI3_K/2)) >> (4 * (kqsx / (QI3_K/2))); + +#pragma unroll + for (int l = 0; l < QR3_K; ++l) { + const int k = (kqsx/8)*32 + l*8 + kqsx % 8; + + const int x_ql_k = (x_ql_0 >> (2*l)) & 0x03030303; + const int x_qh_k = ((x_qh_0 >> l) << 2) & 0x04040404; + + const int x_qs_k = __vsubss4(x_ql_k | x_qh_k, 0x04040404); + +#if defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE) + x_qs[i*sram_stride + k] = x_qs_k; +#else + x_qs[i*(2*MMQ_TILE_NE_K + 1) + k] = x_qs_k; +#endif // defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE) + } + } + + constexpr int rows_per_warp = warp_size / 4; +#pragma unroll + for (int i0 = 0; i0 < I; i0 += nwarps*rows_per_warp) { + int i = i0 + threadIdx.y*rows_per_warp + threadIdx.x/4; + + if (fallback) { + i = min(i, i_max); + } + + const block_q3_K * bxi = (const block_q3_K *) x + kbx0 + i*stride; + + const int ksc = threadIdx.x % 4; + + const int ksc_low = ksc % (QI3_K/8); + const int shift_low = 4 * (ksc / (QI3_K/8)); + const int sc_low = (get_int_b2(bxi->scales, ksc_low) >> shift_low) & 0x0F0F0F0F; + + const int ksc_high = QI3_K/8; + const int shift_high = 2 * ksc; + const int sc_high = ((get_int_b2(bxi->scales, ksc_high) >> shift_high) << 4) & 0x30303030; + + const int sc = __vsubss4(sc_low | sc_high, 0x20202020); + +#if defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE) + const int8_t * sc8 = (const int8_t *) ≻ + const float d = bxi->d; + +#pragma unroll + for (int l = 0; l < int(sizeof(int)); ++l) { + x_df[i*sram_stride + sizeof(int)*ksc + l] = d*sc8[l]; + } +#else + x_sc[i*(MMQ_TILE_NE_K/8) + i/8 + ksc] = sc; +#endif // defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE) + } + +#if !(defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE)) +#pragma unroll + for (int i0 = 0; i0 < I; i0 += nwarps*warp_size) { + int i = (i0 + threadIdx.y*warp_size + threadIdx.x) % I; + + if (fallback) { + i = min(i, i_max); + } + + const block_q3_K * bxi = (const block_q3_K *) x + kbx0 + i*stride; + + x_df[i] = bxi->d; + } +#endif // !(defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE)) || defined(AMD_WMMA_AVAILABLE) +} + +static __device__ __forceinline__ int unpack_scales_q45_K(const int * scales, const int ksc) { + // scale arrangement after the following two lines: + // - ksc == 0: sc0, sc1, sc2, sc3 + // - ksc == 1: sc4, sc5, sc6, sc7 + // - ksc == 2: m0, m1, m2, m3 + // - ksc == 3: m4, m5, m6, m7 + return ((scales[(ksc%2) + (ksc!=0)] >> (4 * (ksc & (ksc/2)))) & 0x0F0F0F0F) | // lower 4 bits + ((scales[ksc/2] >> (2 * (ksc % 2))) & 0x30303030); // upper 2 bits +} + +template static __device__ __forceinline__ void ggml_cuda_mmq_load_tiles_q4_K( + const char * __restrict__ x, int * __restrict__ x_tile, const int kbx0, const int i_max, const int stride) { + constexpr int warp_size = ggml_cuda_get_physical_warp_size(); + constexpr int nwarps = ggml_cuda_mmq_get_nthreads(type, J, fallback) / warp_size; + constexpr int I = ggml_cuda_mmq_get_I(type, J, fallback); + constexpr int sram_stride = ggml_cuda_mmq_get_sram_stride(type, J, fallback); + +#if defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE) + int * x_qs = (int *) x_tile; + half2 * x_dm = (half2 *) (x_qs + 2*MMQ_TILE_NE_K); +#else + constexpr tile_x_sizes txs = mmq_get_dp4a_tile_x_sizes(GGML_TYPE_Q4_K, I); + int * x_qs = (int *) x_tile; + half2 * x_dm = (half2 *) (x_qs + txs.qs); + int * x_sc = (int *) (x_dm + txs.dm); +#endif // defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE) + + constexpr int threads_per_row = MMQ_ITER_K / (4 * QR4_K); + constexpr int nrows = warp_size / threads_per_row; + const int txi = warp_size > threads_per_row ? threadIdx.x % threads_per_row : threadIdx.x; + +#pragma unroll + for (int i0 = 0; i0 < I; i0 += nrows*nwarps) { + int i = i0 + (nrows == 1 ? threadIdx.y : threadIdx.y*nrows + threadIdx.x/threads_per_row); + + if (fallback) { + i = min(i, i_max); + } + + const block_q4_K * bxi = (const block_q4_K *) x + kbx0 + i*stride; + const int qs0 = get_int_b4(bxi->qs, txi); + +#if defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE) + x_qs[i*sram_stride + 16*(txi/8) + txi % 8 + 0] = (qs0 >> 0) & 0x0F0F0F0F; + x_qs[i*sram_stride + 16*(txi/8) + txi % 8 + 8] = (qs0 >> 4) & 0x0F0F0F0F; +#else + x_qs[i*(MMQ_TILE_NE_K + 1) + txi] = qs0; +#endif // defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE) + } + +#if defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE) + constexpr int rows_per_warp = warp_size / 2; +#pragma unroll + for (int i0 = 0; i0 < I; i0 += nwarps*rows_per_warp) { +#if defined(AMD_MFMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE) + // Need if on AMD instead of % because warp_size == 64 + // This causes double work and throughput loss (MI300X) + // H100 loses about 100 t/s with 'if' condition over '%' + int i = i0 + threadIdx.y*rows_per_warp + threadIdx.x/2; + if (i < I) { +#else + int i = (i0 + threadIdx.y*rows_per_warp + threadIdx.x/2) % I; + { +#endif // defined(AMD_MFMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE) + if (fallback) { + i = min(i, i_max); + } + + const block_q4_K * bxi = (const block_q4_K *) x + kbx0 + i*stride; + + const int * scales = (const int *) bxi->scales; + const int ksc = threadIdx.x % 2; + + const int sc32 = unpack_scales_q45_K(scales, ksc + 0); + const int m32 = unpack_scales_q45_K(scales, ksc + 2); + + const uint8_t * sc8 = (const uint8_t *) &sc32; + const uint8_t * m8 = (const uint8_t *) &m32; + + const half2 dm = bxi->dm * make_half2(1.0f, -1.0f); + + #pragma unroll + for (int l = 0; l < sizeof(int); ++l) { + x_dm[i*sram_stride + sizeof(int)*ksc + l] = dm*make_half2(sc8[l], m8[l]); + } + } + } +#else +#pragma unroll + for (int i0 = 0; i0 < I; i0 += nwarps*warp_size) { + int i = (i0 + threadIdx.y*warp_size + threadIdx.x) % I; + + if (fallback) { + i = min(i, i_max); + } + + const block_q4_K * bxi = (const block_q4_K *) x + kbx0 + i*stride; + + x_dm[i] = bxi->dm; + } + constexpr int rows_per_warp = warp_size / 4; +#pragma unroll + for (int i0 = 0; i0 < I; i0 += nwarps*rows_per_warp) { + int i = (i0 + threadIdx.y*rows_per_warp + threadIdx.x/(MMQ_TILE_NE_K/8)) % I; + + if (fallback) { + i = min(i, i_max); + } + + const block_q4_K * bxi = (const block_q4_K *) x + kbx0 + i*stride + (threadIdx.x % (MMQ_TILE_NE_K/8)) / (QI4_K/8); + + const int * scales = (const int *) bxi->scales; + + const int ksc = threadIdx.x % (MMQ_TILE_NE_K/8); + const int scales8 = unpack_scales_q45_K(scales, ksc); + + x_sc[i*(MMQ_TILE_NE_K/8) + i/8 + ksc] = scales8; + } +#endif // defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE) +} + +template static __device__ __forceinline__ void ggml_cuda_mmq_load_tiles_q5_K( + const char * __restrict__ x, int * __restrict__ x_tile, const int kbx0, const int i_max, const int stride) { + constexpr int warp_size = ggml_cuda_get_physical_warp_size(); + constexpr int nwarps = ggml_cuda_mmq_get_nthreads(type, J, fallback) / warp_size; + constexpr int I = ggml_cuda_mmq_get_I(type, J, fallback); + constexpr int sram_stride = ggml_cuda_mmq_get_sram_stride(type, J, fallback); + +#if defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE) + int * x_qs = (int *) x_tile; + half2 * x_dm = (half2 *) (x_qs + MMQ_TILE_NE_K*2); +#else + constexpr tile_x_sizes txs = mmq_get_dp4a_tile_x_sizes(GGML_TYPE_Q5_K, I); + int * x_qs = (int *) x_tile; + half2 * x_dm = (half2 *) (x_qs + txs.qs); + int * x_sc = (int *) (x_dm + txs.dm); +#endif // defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) + + constexpr int threads_per_row = MMQ_ITER_K / (4 * QR5_K); + constexpr int nrows = warp_size / threads_per_row; + const int txi = warp_size > threads_per_row ? threadIdx.x % threads_per_row : threadIdx.x; + +#pragma unroll + for (int i0 = 0; i0 < I; i0 += nrows*nwarps) { + int i = i0 + (nrows == 1 ? threadIdx.y : threadIdx.y*nrows + threadIdx.x/threads_per_row); + + if (fallback) { + i = min(i, i_max); + } + + const block_q5_K * bxi = (const block_q5_K *) x + kbx0 + i*stride; + const int ky = QR5_K*txi; + + const int ql = get_int_b4(bxi->qs, txi); + const int ql0 = (ql >> 0) & 0x0F0F0F0F; + const int ql1 = (ql >> 4) & 0x0F0F0F0F; + + const int qh = get_int_b4(bxi->qh, txi % (QI5_K/4)); + const int qh0 = ((qh >> (2 * (txi / (QI5_K/4)) + 0)) << 4) & 0x10101010; + const int qh1 = ((qh >> (2 * (txi / (QI5_K/4)) + 1)) << 4) & 0x10101010; + + const int kq0 = ky - ky % (QI5_K/2) + txi % (QI5_K/4) + 0; + const int kq1 = ky - ky % (QI5_K/2) + txi % (QI5_K/4) + QI5_K/4; + +#if defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE) + x_qs[i*sram_stride + kq0] = ql0 | qh0; + x_qs[i*sram_stride + kq1] = ql1 | qh1; +#else + x_qs[i*(2*MMQ_TILE_NE_K + 1) + kq0] = ql0 | qh0; + x_qs[i*(2*MMQ_TILE_NE_K + 1) + kq1] = ql1 | qh1; +#endif // defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE) + } + +#if defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE) + constexpr int rows_per_warp = warp_size / 2; +#pragma unroll + for (int i0 = 0; i0 < I; i0 += nwarps*rows_per_warp) { +#if defined(AMD_MFMA_AVAILABLE) + // Need if on AMD instead of % because warp_size == 64 + // This causes double work and throughput loss (MI300X) + // H100 loses about 100 t/s with 'if' condition over '%' + int i = i0 + threadIdx.y*rows_per_warp + threadIdx.x/2; + if (i < I) { +#else + int i = (i0 + threadIdx.y*rows_per_warp + threadIdx.x/2) % I; + { +#endif // defined(AMD_MFMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE) + if (fallback) { + i = min(i, i_max); + } + + const block_q5_K * bxi = (const block_q5_K *) x + kbx0 + i*stride; + + const int * scales = (const int *) bxi->scales; + const int ksc = threadIdx.x % 2; + + const int sc32 = unpack_scales_q45_K(scales, ksc + 0); + const int m32 = unpack_scales_q45_K(scales, ksc + 2); + + const uint8_t * sc8 = (const uint8_t *) &sc32; + const uint8_t * m8 = (const uint8_t *) &m32; + + const half2 dm = bxi->dm * make_half2(1.0f, -1.0f); + +#pragma unroll + for (int l = 0; l < int(sizeof(int)); ++l) { + x_dm[i*sram_stride + sizeof(int)*ksc + l] = dm*make_half2(sc8[l], m8[l]); + } + } + } +#else +#pragma unroll + for (int i0 = 0; i0 < I; i0 += nwarps*warp_size) { + int i = (i0 + threadIdx.y*warp_size + threadIdx.x) % I; + + if (fallback) { + i = min(i, i_max); + } + + const block_q5_K * bxi = (const block_q5_K *) x + kbx0 + i*stride; + + x_dm[i] = bxi->dm; + } + + constexpr int rows_per_warp = warp_size / 4; +#pragma unroll + for (int i0 = 0; i0 < I; i0 += nwarps*rows_per_warp) { + int i = (i0 + threadIdx.y*rows_per_warp + threadIdx.x/(MMQ_TILE_NE_K/8)) % I; + + if (fallback) { + i = min(i, i_max); + } + + const block_q5_K * bxi = (const block_q5_K *) x + kbx0 + i*stride; + + const int * scales = (const int *) bxi->scales; + + const int ksc = threadIdx.x % (MMQ_TILE_NE_K/8); + const int scales8 = unpack_scales_q45_K(scales, ksc); + + x_sc[i*(MMQ_TILE_NE_K/8) + i/8 + ksc] = scales8; + } +#endif // defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE) +} + +template static __device__ __forceinline__ void ggml_cuda_mmq_load_tiles_q6_K( + const char * __restrict__ x, int * __restrict__ x_tile, const int kbx0, const int i_max, const int stride) { + constexpr int warp_size = ggml_cuda_get_physical_warp_size(); + constexpr int nwarps = ggml_cuda_mmq_get_nthreads(type, J, fallback) / warp_size; + constexpr int I = ggml_cuda_mmq_get_I(type, J, fallback); + constexpr int sram_stride = ggml_cuda_mmq_get_sram_stride(type, J, fallback); + +#if defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE) + int * x_qs = (int *) x_tile; + float * x_df = (float *) (x_qs + MMQ_TILE_NE_K*2); + int * x_sc = (int *) (x_df + MMQ_TILE_NE_K/QI6_K); +#else + constexpr tile_x_sizes txs = mmq_get_dp4a_tile_x_sizes(GGML_TYPE_Q6_K, I); + int * x_qs = (int *) x_tile; + float * x_df = (float *) (x_qs + txs.qs); + int * x_sc = (int *) (x_df + txs.dm); +#endif // defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE) + + constexpr int threads_per_row = MMQ_ITER_K / (4 * QR6_K); + constexpr int nrows = warp_size / threads_per_row; + const int txi = warp_size > threads_per_row ? threadIdx.x % threads_per_row : threadIdx.x; + +#pragma unroll + for (int i0 = 0; i0 < I; i0 += nrows*nwarps) { + int i = i0 + (nrows == 1 ? threadIdx.y : threadIdx.y*nrows + threadIdx.x/threads_per_row); + + if (fallback) { + i = min(i, i_max); + } + + const block_q6_K * bxi = (const block_q6_K *) x + kbx0 + i*stride; + + const int ql = get_int_b2(bxi->ql, txi); + const int ql0 = (ql >> 0) & 0x0F0F0F0F; + const int ql1 = (ql >> 4) & 0x0F0F0F0F; + + const int qh = get_int_b2(bxi->qh, (QI6_K/4) * (txi / (QI6_K/2)) + txi % (QI6_K/4)); + const int qh0 = ((qh >> ((txi & 0x08) >> 2)) << 4) & 0x30303030; + const int qh1 = (qh >> ((txi & 0x08) >> 2)) & 0x30303030; + + const int kq0 = 2*txi - txi % (QI6_K/2) + 0; + const int kq1 = 2*txi - txi % (QI6_K/2) + QI6_K/2; + +#if defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE) + x_qs[i*sram_stride + kq0] = __vsubss4(ql0 | qh0, 0x20202020); + x_qs[i*sram_stride + kq1] = __vsubss4(ql1 | qh1, 0x20202020); +#else + x_qs[i*(2*MMQ_TILE_NE_K + 1) + kq0] = __vsubss4(ql0 | qh0, 0x20202020); + x_qs[i*(2*MMQ_TILE_NE_K + 1) + kq1] = __vsubss4(ql1 | qh1, 0x20202020); +#endif // defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE) + } + +#pragma unroll + for (int i0 = 0; i0 < I; i0 += nwarps*warp_size) { + int i = (i0 + threadIdx.y*warp_size + threadIdx.x) % I; + + if (fallback) { + i = min(i, i_max); + } + + const block_q6_K * bxi = (const block_q6_K *) x + kbx0 + i*stride; + +#if defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE) + x_df[i*sram_stride] = bxi->d; +#else + x_df[i*(MMQ_TILE_NE_K/QI6_K) + i/QI6_K] = bxi->d; +#endif // defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE) + } + + constexpr int rows_per_warp = warp_size / 4; +#pragma unroll + for (int i0 = 0; i0 < I; i0 += nwarps*rows_per_warp) { + int i = (i0 + threadIdx.y*rows_per_warp + threadIdx.x/(MMQ_TILE_NE_K/8)) % I; + + if (fallback) { + i = min(i, i_max); + } + + const block_q6_K * bxi = (const block_q6_K *) x + kbx0 + i*stride + (threadIdx.x % (MMQ_TILE_NE_K/8)) / 4; + +#if defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE) + x_sc[i*sram_stride + threadIdx.x%4] = get_int_b2(bxi->scales, threadIdx.x % (MMQ_TILE_NE_K/8)); +#else + x_sc[i*(MMQ_TILE_NE_K/8) + i/8 + threadIdx.x%(MMQ_TILE_NE_K/8)] = get_int_b2(bxi->scales, threadIdx.x%(QI6_K/8)); +#endif // defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE) + } +} + +// --------------------------------------------------------------------------------------------- + +template static __device__ __forceinline__ void ggml_cuda_mmq_load_tiles_iq1_s( + const char * __restrict__ x, int * __restrict__ x_tile, const int kbx0, const int i_max, const int stride) { + constexpr int warp_size = ggml_cuda_get_physical_warp_size(); + constexpr int nwarps = ggml_cuda_mmq_get_nthreads(type, J, fallback) / warp_size; + constexpr int I = ggml_cuda_mmq_get_I(type, J, fallback); + constexpr int sram_stride = ggml_cuda_mmq_get_sram_stride(type, J, fallback); + +#if defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE) + int * x_qs = (int *) x_tile; + half2 * x_ds = (half2 *) (x_qs + MMQ_TILE_NE_K*2); +#else + constexpr tile_x_sizes txs = mmq_get_dp4a_tile_x_sizes(GGML_TYPE_IQ3_S, I); + int * x_qs = (int *) x_tile; + half2 * x_ds = (half2 *) (x_qs + txs.qs); +#endif // defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE) + + constexpr int threads_per_row = MMQ_ITER_K / (4 * QR1_S); + constexpr int nrows = warp_size / threads_per_row; + const int kqsx = threadIdx.x % threads_per_row; + +#pragma unroll + for (int i0 = 0; i0 < I; i0 += nwarps * nrows) { + int i = i0 + threadIdx.y*nrows + threadIdx.x/threads_per_row; + + if (fallback) { + i = min(i, i_max); + } + + const block_iq1_s * bxi = (const block_iq1_s *) x + kbx0 + i*stride; + + const int qs_packed = get_int_b2(bxi->qs, kqsx); + const uint8_t * qs = (const uint8_t *) &qs_packed; + + const int qh = bxi->qh[kqsx]; + + #pragma unroll + for (int l = 0; l < QR1_S/2; ++l) { + const int grid = iq1s_grid_gpu[qs[l] | (((qh >> (3*l)) & 0x07) << 8)]; + + const int grid0 = (grid >> 0) & 0x0F0F0F0F; + const int grid1 = (grid >> 4) & 0x0F0F0F0F; + +#if defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE) + x_qs[i*sram_stride + 8*kqsx + (2*l+0)] = grid0; + x_qs[i*sram_stride + 8*kqsx + (2*l+1)] = grid1; +#else + x_qs[i*(2*MMQ_TILE_NE_K + 1) + 8*kqsx + (2*l+0)] = grid0; + x_qs[i*(2*MMQ_TILE_NE_K + 1) + 8*kqsx + (2*l+1)] = grid1; +#endif // defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE) + } + + const float d1q = __half2float(bxi->d) * (((qh >> 11) & 0x0E) + 1); + const float delta = -1.0f + IQ1S_DELTA - (qh & 0x8000) * (2.0f*IQ1S_DELTA/0x8000); + +#if defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE) + x_ds[i*sram_stride + kqsx] = make_half2(d1q, d1q*delta); +#else + x_ds[i*(MMQ_TILE_NE_K/4) + i/4 + kqsx] = make_half2(d1q, d1q*delta); +#endif // defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE) + } +} + +template static __device__ __forceinline__ void ggml_cuda_mmq_load_tiles_iq2_xxs( + const char * __restrict__ x, int * __restrict__ x_tile, const int kbx0, const int i_max, const int stride) { + constexpr int warp_size = ggml_cuda_get_physical_warp_size(); + constexpr int nwarps = ggml_cuda_mmq_get_nthreads(type, J, fallback) / warp_size; + constexpr int I = ggml_cuda_mmq_get_I(type, J, fallback); + constexpr int sram_stride = ggml_cuda_mmq_get_sram_stride(type, J, fallback); + +#if defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE) + int * x_qs = (int *) x_tile; + float * x_df = (float *) (x_qs + MMQ_TILE_NE_K*2); +#else + constexpr tile_x_sizes txs = mmq_get_dp4a_tile_x_sizes(GGML_TYPE_IQ2_XXS, I); + int * x_qs = (int *) x_tile; + float * x_df = (float *) (x_qs + txs.qs); +#endif // defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE) + + constexpr int threads_per_row = (MMQ_ITER_K / (4 * QR2_XXS)) / 2; + constexpr int nrows = warp_size / threads_per_row; + const int kqsx = warp_size > threads_per_row ? threadIdx.x % threads_per_row : threadIdx.x; + +#pragma unroll + for (int i0 = 0; i0 < I; i0 += nwarps * nrows) { + int i = i0 + threadIdx.y*nrows + threadIdx.x/threads_per_row; + + if (fallback) { + i = min(i, i_max); + } + + const block_iq2_xxs * bxi = (const block_iq2_xxs *) x + kbx0 + i*stride; + + const int q2 = get_int_b2(bxi->qs, 2*kqsx+0); + const uint8_t * aux8 = (const uint8_t *) &q2; + const uint32_t aux32 = get_int_b2(bxi->qs, 2*kqsx+1); + +#pragma unroll + for (int l = 0; l < QR2_XXS; ++l) { + const uint2 grid_pos = ((const uint2*)iq2xxs_grid)[aux8[l]]; + const uint32_t signs = unpack_ksigns(aux32 >> (7 * l)); + + const int signs0 = __vcmpne4(signs & 0x08040201, 0); + const int grid0 = __vsub4(grid_pos.x ^ signs0, signs0); + + const int signs1 = __vcmpne4(signs & 0x80402010, 0); + const int grid1 = __vsub4(grid_pos.y ^ signs1, signs1); + +#if defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE) + x_qs[i*sram_stride + 8*kqsx + (2*l + 0)] = grid0; + x_qs[i*sram_stride + 8*kqsx + (2*l + 1)] = grid1; +#else + x_qs[i*(2*MMQ_TILE_NE_K + 1) + 8*kqsx + (2*l + 0)] = grid0; + x_qs[i*(2*MMQ_TILE_NE_K + 1) + 8*kqsx + (2*l + 1)] = grid1; +#endif // defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE) + } + + const int ls = aux32 >> 27 | 1; // (scale * 2 + 1) + const float d = bxi->d; +#if defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE) + x_df[i*sram_stride + kqsx] = d * ls / 8; // (d * scale + d / 2) / 4 +#else + x_df[i*(MMQ_TILE_NE_K/4) + i/4 + kqsx] = d * ls / 8; // (d * scale + d / 2) / 4 +#endif // defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE) + } +} + +template static __device__ __forceinline__ void ggml_cuda_mmq_load_tiles_iq2_xs( + const char * __restrict__ x, int * __restrict__ x_tile, const int kbx0, const int i_max, const int stride) { + constexpr int warp_size = ggml_cuda_get_physical_warp_size(); + constexpr int nwarps = ggml_cuda_mmq_get_nthreads(type, J, fallback) / warp_size; + constexpr int I = ggml_cuda_mmq_get_I(type, J, fallback); + constexpr int sram_stride = ggml_cuda_mmq_get_sram_stride(type, J, fallback); + +#if defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE) + int * x_qs = (int *) x_tile; + float * x_df = (float *) (x_qs + MMQ_TILE_NE_K*2); +#else + constexpr tile_x_sizes txs = mmq_get_dp4a_tile_x_sizes(GGML_TYPE_IQ2_XS, I); + int * x_qs = (int *) x_tile; + float * x_df = (float *) (x_qs + txs.qs); +#endif // defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE) + + constexpr int threads_per_row = (MMQ_ITER_K / (4 * QR2_XS)) / 2; + constexpr int nrows = warp_size / threads_per_row; + const int kqsx = threadIdx.x % threads_per_row; + +#pragma unroll + for (int i0 = 0; i0 < I; i0 += nwarps * nrows) { + int i = i0 + threadIdx.y*nrows + threadIdx.x/threads_per_row; + + if (fallback) { + i = min(i, i_max); + } + + const block_iq2_xs * bxi = (const block_iq2_xs *) x + kbx0 + i*stride; + + const int2 q2_packed = make_int2(get_int_b2(bxi->qs, 2*kqsx+0), get_int_b2(bxi->qs, 2*kqsx+1)); + const uint16_t * q2 = (const uint16_t *) &q2_packed; + + #pragma unroll + for (int l = 0; l < QR2_XS; ++l) { + const uint2 grid_pos = ((const uint2*)iq2xs_grid)[q2[l] & 0x1FF]; + const uint32_t signs = unpack_ksigns(q2[l] >> 9); + + const int signs0 = __vcmpne4(signs & 0x08040201, 0); + const int grid_l = __vsub4(grid_pos.x ^ signs0, signs0); + + const int signs1 = __vcmpne4(signs & 0x80402010, 0); + const int grid_h = __vsub4(grid_pos.y ^ signs1, signs1); + +#if defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE) + x_qs[i*sram_stride + 8*kqsx + (2*l + 0)] = grid_l; + x_qs[i*sram_stride + 8*kqsx + (2*l + 1)] = grid_h; +#else + x_qs[i*(2*MMQ_TILE_NE_K + 1) + 8*kqsx + (2*l + 0)] = grid_l; + x_qs[i*(2*MMQ_TILE_NE_K + 1) + 8*kqsx + (2*l + 1)] = grid_h; +#endif // defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE) + } + + const int ls = bxi->scales[kqsx]; + const float d = bxi->d; +#if defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE) + x_df[i*sram_stride + 2*kqsx+0] = ((ls & 0x0F)*d + d/2)/4; + x_df[i*sram_stride + 2*kqsx+1] = ((ls >> 4)*d + d/2)/4; +#else + x_df[i*(2*MMQ_TILE_NE_K*2/QI8_0) + i/(QI8_0/4) + 2*kqsx+0] = ((ls & 0x0F)*d + d/2)/4; + x_df[i*(2*MMQ_TILE_NE_K*2/QI8_0) + i/(QI8_0/4) + 2*kqsx+1] = ((ls >> 4)*d + d/2)/4; +#endif // defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE) + } +} + +template static __device__ __forceinline__ void ggml_cuda_mmq_load_tiles_iq2_s( + const char * __restrict__ x, int * __restrict__ x_tile, const int kbx0, const int i_max, const int stride) { + constexpr int warp_size = ggml_cuda_get_physical_warp_size(); + constexpr int nwarps = ggml_cuda_mmq_get_nthreads(type, J, fallback) / warp_size; + constexpr int I = ggml_cuda_mmq_get_I(type, J, fallback); + constexpr int sram_stride = ggml_cuda_mmq_get_sram_stride(type, J, fallback); + +#if defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE) + int * x_qs = (int *) x_tile; + float * x_df = (float *) (x_qs + MMQ_TILE_NE_K*2); +#else + constexpr tile_x_sizes txs = mmq_get_dp4a_tile_x_sizes(GGML_TYPE_IQ2_S, I); + int * x_qs = (int *) x_tile; + float * x_df = (float *) (x_qs + txs.qs); +#endif // defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE) + constexpr int threads_per_row = (MMQ_ITER_K / (4 * QR2_S)) / 2; + constexpr int nrows = warp_size / threads_per_row; + const int kqsx = threadIdx.x % threads_per_row; + +#pragma unroll + for (int i0 = 0; i0 < I; i0 += nwarps * nrows) { + int i = i0 + threadIdx.y*nrows + threadIdx.x/threads_per_row; + + if (fallback) { + i = min(i, i_max); + } + + const block_iq2_s * bxi = (const block_iq2_s *) x + kbx0 + i*stride; + + const int qs_packed = get_int_b2(bxi->qs, kqsx); + const uint8_t * qs = (const uint8_t *) &qs_packed; + + const int qh = bxi->qh[kqsx]; + + const int signs_packed_32 = get_int_b2(bxi->qs, QK_K/32 + kqsx); + const uint8_t * signs_packed_8 = (const uint8_t *) &signs_packed_32; + +#pragma unroll + for (int l = 0; l < QR2_S; ++l) { + const int * grid_pos = (const int *)(iq2s_grid + (qs[l] | ((qh << (8-2*l)) & 0x300))); + + const int signs0 = __vcmpne4(((signs_packed_8[l] & 0x03) << 7) | ((signs_packed_8[l] & 0x0C) << 21), 0x00000000); + const int signs1 = __vcmpne4(((signs_packed_8[l] & 0x30) << 3) | ((signs_packed_8[l] & 0xC0) << 17), 0x00000000); + + const int grid_l = __vsub4(grid_pos[0] ^ signs0, signs0); + const int grid_h = __vsub4(grid_pos[1] ^ signs1, signs1); + +#if defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE) + x_qs[i*sram_stride + 8*kqsx + (2*l + 0)] = grid_l; + x_qs[i*sram_stride + 8*kqsx + (2*l + 1)] = grid_h; +#else + x_qs[i*(2*MMQ_TILE_NE_K + 1) + 8*kqsx + (2*l + 0)] = grid_l; + x_qs[i*(2*MMQ_TILE_NE_K + 1) + 8*kqsx + (2*l + 1)] = grid_h; +#endif // defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE) + } + + const int ls = bxi->scales[kqsx]; + const float d = bxi->d; +#if defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE) + x_df[i*sram_stride + 2*kqsx+0] = ((ls & 0x0F)*d + d/2)/4; + x_df[i*sram_stride + 2*kqsx+1] = ((ls >> 4)*d + d/2)/4; +#else + x_df[i*(2*MMQ_TILE_NE_K*2/QI8_0) + i/(QI8_0/4) + 2*kqsx+0] = ((ls & 0x0F)*d + d/2)/4; + x_df[i*(2*MMQ_TILE_NE_K*2/QI8_0) + i/(QI8_0/4) + 2*kqsx+1] = ((ls >> 4)*d + d/2)/4; +#endif // defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE) + } +} + +template static __device__ __forceinline__ void ggml_cuda_mmq_load_tiles_iq3_xxs( + const char * __restrict__ x, int * __restrict__ x_tile, const int kbx0, const int i_max, const int stride) { + constexpr int warp_size = ggml_cuda_get_physical_warp_size(); + constexpr int nwarps = ggml_cuda_mmq_get_nthreads(type, J, fallback) / warp_size; + constexpr int I = ggml_cuda_mmq_get_I(type, J, fallback); + constexpr int sram_stride = ggml_cuda_mmq_get_sram_stride(type, J, fallback); + +#if defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE) + int * x_qs = (int *) x_tile; + float * x_df = (float *) (x_qs + MMQ_TILE_NE_K*2); +#else + constexpr tile_x_sizes txs = mmq_get_dp4a_tile_x_sizes(GGML_TYPE_IQ3_XXS, I); + int * x_qs = (int *) x_tile; + float * x_df = (float *) (x_qs + txs.qs); +#endif // defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE) + + constexpr int threads_per_row = (MMQ_ITER_K / (4 * QR3_XXS)) / 2; + constexpr int nrows = warp_size / threads_per_row; + const int kqsx = threadIdx.x % threads_per_row; + +#pragma unroll + for (int i0 = 0; i0 < I; i0 += nwarps * nrows) { + int i = i0 + threadIdx.y*nrows + threadIdx.x/threads_per_row; + + if (fallback) { + i = min(i, i_max); + } + + const block_iq3_xxs * bxi = (const block_iq3_xxs *) x + kbx0 + i*stride; + + const int2 q3_packed = make_int2(get_int_b2(bxi->qs, 2*kqsx+0), get_int_b2(bxi->qs, 2*kqsx+1)); + const uint8_t * q3 = (const uint8_t *) &q3_packed; + const uint32_t aux32 = get_int_b2(bxi->qs, QK_K/16 + kqsx); + +#pragma unroll + for (int l = 0; l < QR3_XXS; ++l) { + const int2 grid_pos = make_int2(iq3xxs_grid[q3[2*l+0]], iq3xxs_grid[q3[2*l+1]]); + const uint32_t signs = unpack_ksigns(aux32 >> (7*l)); + + const int signs0 = __vcmpne4(signs & 0x08040201, 0); + const int grid_l = __vsub4(grid_pos.x ^ signs0, signs0); + + const int signs1 = __vcmpne4(signs & 0x80402010, 0); + const int grid_h = __vsub4(grid_pos.y ^ signs1, signs1); + +#if defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE) + x_qs[i*sram_stride + 8*kqsx + (2*l + 0)] = grid_l; + x_qs[i*sram_stride + 8*kqsx + (2*l + 1)] = grid_h; +#else + x_qs[i*(2*MMQ_TILE_NE_K + 1) + 8*kqsx + (2*l + 0)] = grid_l; + x_qs[i*(2*MMQ_TILE_NE_K + 1) + 8*kqsx + (2*l + 1)] = grid_h; +#endif // defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE) + } + + const int ls = aux32 >> 28; + const float d = bxi->d; +#if defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE) + x_df[i*sram_stride + kqsx] = (ls*d + d/2)/2; +#else + x_df[i*(MMQ_TILE_NE_K/4) + i/4 + kqsx] = (ls*d + d/2)/2; +#endif // defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE) + } +} + +template static __device__ __forceinline__ void ggml_cuda_mmq_load_tiles_iq3_s( + const char * __restrict__ x, int * __restrict__ x_tile, const int kbx0, const int i_max, const int stride) { + constexpr int warp_size = ggml_cuda_get_physical_warp_size(); + constexpr int nwarps = ggml_cuda_mmq_get_nthreads(type, J, fallback) / warp_size; + constexpr int I = ggml_cuda_mmq_get_I(type, J, fallback); + constexpr int sram_stride = ggml_cuda_mmq_get_sram_stride(type, J, fallback); + +#if defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE) + int * x_qs = (int *) x_tile; + float * x_df = (float *) (x_qs + MMQ_TILE_NE_K*2); +#else + constexpr tile_x_sizes txs = mmq_get_dp4a_tile_x_sizes(GGML_TYPE_IQ3_S, I); + int * x_qs = (int *) x_tile; + float * x_df = (float *) (x_qs + txs.qs); +#endif // defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE) + + constexpr int threads_per_row = (MMQ_ITER_K / (4 * QR3_S)) / 2; + constexpr int nrows = warp_size / threads_per_row; + const int kqsx = threadIdx.x % threads_per_row; + +#pragma unroll + for (int i0 = 0; i0 < I; i0 += nwarps * nrows) { + int i = i0 + threadIdx.y*nrows + threadIdx.x/threads_per_row; + + if (fallback) { + i = min(i, i_max); + } + + const block_iq3_s * bxi = (const block_iq3_s *) x + kbx0 + i*stride; + + const int2 qs_packed = make_int2(get_int_b2(bxi->qs, 2*kqsx+0), get_int_b2(bxi->qs, 2*kqsx+1)); + const uint8_t * qs = (const uint8_t *) &qs_packed; + + const int qh = bxi->qh[kqsx]; + + const int signs_packed_32 = get_int_b2(bxi->signs, kqsx); + const uint8_t * signs_packed_8 = (const uint8_t *) &signs_packed_32; + +#pragma unroll + for (int l = 0; l < QR3_S; ++l) { + const int2 grid_pos = make_int2( + iq3s_grid[qs[2*l+0] | ((qh << (8 - 2*l)) & 0x100)], + iq3s_grid[qs[2*l+1] | ((qh << (7 - 2*l)) & 0x100)]); + + const int signs0 = __vcmpne4(((signs_packed_8[l] & 0x03) << 7) | ((signs_packed_8[l] & 0x0C) << 21), 0x00000000); + const int signs1 = __vcmpne4(((signs_packed_8[l] & 0x30) << 3) | ((signs_packed_8[l] & 0xC0) << 17), 0x00000000); + + const int grid_l = __vsub4(grid_pos.x ^ signs0, signs0); + const int grid_h = __vsub4(grid_pos.y ^ signs1, signs1); + +#if defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE) + x_qs[i*sram_stride + 8*kqsx + (2*l+0)] = grid_l; + x_qs[i*sram_stride + 8*kqsx + (2*l+1)] = grid_h; +#else + x_qs[i*(2*MMQ_TILE_NE_K + 1) + 8*kqsx + (2*l+0)] = grid_l; + x_qs[i*(2*MMQ_TILE_NE_K + 1) + 8*kqsx + (2*l+1)] = grid_h; +#endif // defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE) + } + + const int ls = 1 + 2*((bxi->scales[kqsx/2] >> (((2*kqsx) << 1) & 0x04)) & 0x0F); + const float d = bxi->d; +#if defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE) + x_df[i*sram_stride + kqsx] = ls*d; +#else + x_df[i*(MMQ_TILE_NE_K/4) + i/4 + kqsx] = ls*d; +#endif // defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE) + } +} + +template static __device__ __forceinline__ void ggml_cuda_mmq_load_tiles_iq4_xs( + const char * __restrict__ x, int * __restrict__ x_tile, const int kbx0, const int i_max, const int stride) { + constexpr int warp_size = ggml_cuda_get_physical_warp_size(); + constexpr int nwarps = ggml_cuda_mmq_get_nthreads(type, J, fallback) / warp_size; + constexpr int I = ggml_cuda_mmq_get_I(type, J, fallback); + constexpr int sram_stride = ggml_cuda_mmq_get_sram_stride(type, J, fallback); + +#if defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE) + int * x_qs = (int *) x_tile; + float * x_df = (float *) (x_qs + MMQ_TILE_NE_K*2); +#else + constexpr tile_x_sizes txs = mmq_get_dp4a_tile_x_sizes(GGML_TYPE_IQ4_XS, I); + int * x_qs = (int *) x_tile; + float * x_df = (float *) (x_qs + txs.qs); +#endif // defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE) + + constexpr int threads_per_row = MMQ_ITER_K / (4 * QR4_XS); + constexpr int nrows = warp_size / threads_per_row; + const int kqsx = threadIdx.x % threads_per_row; + +#pragma unroll + for (int i0 = 0; i0 < I; i0 += nrows*nwarps) { + int i = i0 + (nrows == 1 ? threadIdx.y : threadIdx.y*nrows + threadIdx.x/threads_per_row); + + if (fallback) { + i = min(i, i_max); + } + + const block_iq4_xs * bxi = (const block_iq4_xs *) x + kbx0 + i*stride; + + const int aux_q4 = get_int_b4(bxi->qs, kqsx); + const int2 v = get_int_from_table_16(aux_q4, kvalues_iq4nl); + const int k0 = 8 * (kqsx / 4) + kqsx % 4; + +#if defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE) + x_qs[i*sram_stride + k0 + 0] = v.x; + x_qs[i*sram_stride + k0 + 4] = v.y; +#else + x_qs[i*(2*MMQ_TILE_NE_K + 1) + k0 + 0] = v.x; + x_qs[i*(2*MMQ_TILE_NE_K + 1) + k0 + 4] = v.y; +#endif // defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE) + } + + constexpr int rows_per_warp = warp_size / 8; +#pragma unroll + for (int i0 = 0; i0 < I; i0 += nwarps * rows_per_warp) { + int i = i0 + threadIdx.y * rows_per_warp + threadIdx.x / (MMQ_TILE_NE_K/4); + + if (fallback) { + i = min(i, i_max); + } + + const block_iq4_xs * bxi = (const block_iq4_xs *) x + kbx0 + i*stride; + + const float d = __half2float(bxi->d); + + const int ls = ((bxi->scales_l[(threadIdx.x % 8)/2] >> (4*(threadIdx.x % 2))) & 0x0F) + | (((bxi->scales_h >> (2*(threadIdx.x % 8))) & 0x03) << 4); + +#if defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE) + x_df[i*sram_stride + threadIdx.x % 8] = d * (ls - 32); +#else + x_df[i*(MMQ_TILE_NE_K/4) + i/4 + threadIdx.x % 8] = d * (ls - 32); +#endif // defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE) + } +} + +template static __device__ __forceinline__ void ggml_cuda_mmq_load_tiles_iq4_nl( + const char * __restrict__ x, int * __restrict__ x_tile, const int kbx0, const int i_max, const int stride) { + constexpr int warp_size = ggml_cuda_get_physical_warp_size(); + constexpr int nwarps = ggml_cuda_mmq_get_nthreads(type, J, fallback) / warp_size; + constexpr int I = ggml_cuda_mmq_get_I(type, J, fallback); + constexpr int sram_stride = ggml_cuda_mmq_get_sram_stride(type, J, fallback); + +#if defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE) + int * x_qs = (int *) x_tile; + float * x_df = (float *) (x_qs + MMQ_TILE_NE_K*2); +#else + constexpr tile_x_sizes txs = mmq_get_dp4a_tile_x_sizes(GGML_TYPE_IQ4_NL, I); + int * x_qs = (int *) x_tile; + float * x_df = (float *) (x_qs + txs.qs); +#endif // defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE) + + constexpr int threads_per_row = MMQ_ITER_K / (4 * QR4_NL); + constexpr int nrows = warp_size / threads_per_row; + const int txi = warp_size > threads_per_row ? threadIdx.x % threads_per_row : threadIdx.x; + const int kbx = txi / QI4_NL; + const int kqsx = txi % QI4_NL; + +#pragma unroll + for (int i0 = 0; i0 < I; i0 += nrows*nwarps) { + int i = i0 + (nrows == 1 ? threadIdx.y : threadIdx.y*nrows + threadIdx.x/threads_per_row); + + if (fallback) { + i = min(i, i_max); + } + + const block_iq4_nl * bxi = (const block_iq4_nl *) x + kbx0 + i*stride + kbx; + + const int aux_q4 = get_int_b2(bxi->qs, kqsx); + const int2 v = get_int_from_table_16(aux_q4, kvalues_iq4nl); + const int k0 = kbx * (2 * QI4_NL) + kqsx; + +#if defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE) + x_qs[i*sram_stride + k0 + 0] = v.x; + x_qs[i*sram_stride + k0 + QI4_NL] = v.y; +#else + x_qs[i*(2*MMQ_TILE_NE_K + 1) + k0 + 0] = v.x; + x_qs[i*(2*MMQ_TILE_NE_K + 1) + k0 + QI4_NL] = v.y; +#endif // defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE) + } + + constexpr int blocks_per_tile_x_row = MMQ_TILE_NE_K / QI4_NL; + constexpr int rows_per_warp = warp_size / blocks_per_tile_x_row; + const int kbxd = threadIdx.x % blocks_per_tile_x_row; + +#pragma unroll + for (int i0 = 0; i0 < I; i0 += nwarps * rows_per_warp) { + int i = i0 + threadIdx.y * rows_per_warp + threadIdx.x / blocks_per_tile_x_row; + + if (fallback) { + i = min(i, i_max); + } + + const block_iq4_nl * bxi = (const block_iq4_nl *) x + kbx0 + i*stride + kbxd; + +#if defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE) + x_df[i*sram_stride + kbxd] = __half2float(bxi->d); +#else + x_df[i*(MMQ_TILE_NE_K/QI4_NL) + i/QI4_NL + kbxd] = __half2float(bxi->d); +#endif // defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE) + } +} + +// --------------------------------------------------------------------------------------------- + +template static __device__ __forceinline__ void ggml_cuda_mmq_load_tiles_mxfp4( + const char * __restrict__ x, int * __restrict__ x_tile, const int kbx0, const int i_max, const int stride) { + constexpr int warp_size = ggml_cuda_get_physical_warp_size(); + constexpr int nwarps = ggml_cuda_mmq_get_nthreads(type, J, fallback) / warp_size; + constexpr int I = ggml_cuda_mmq_get_I(type, J, fallback); + constexpr int sram_stride = ggml_cuda_mmq_get_sram_stride(type, J, fallback); + +#if defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE) + int * x_qs = (int *) x_tile; + float * x_df = (float *) (x_qs + MMQ_TILE_NE_K*2); +#else + constexpr tile_x_sizes txs = mmq_get_dp4a_tile_x_sizes(GGML_TYPE_MXFP4, I); + int * x_qs = (int *) x_tile; + float * x_df = (float *) (x_qs + txs.qs); +#endif // defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE) + + constexpr int threads_per_row = MMQ_ITER_K / (4 * QR_MXFP4); + constexpr int nrows = warp_size / threads_per_row; + const int txi = warp_size > threads_per_row ? threadIdx.x % threads_per_row : threadIdx.x; + const int kbx = txi / QI_MXFP4; + const int kqsx = txi % QI_MXFP4; + +#pragma unroll + for (int i0 = 0; i0 < I; i0 += nrows*nwarps) { + int i = i0 + (nrows == 1 ? threadIdx.y : threadIdx.y*nrows + threadIdx.x/threads_per_row); + + if (fallback) { + i = min(i, i_max); + } + + const block_mxfp4 * bxi = (const block_mxfp4 *) x + kbx0 + i*stride + kbx; + + const int aux_q4 = get_int_b1(bxi->qs, kqsx); + const int2 v = get_int_from_table_16(aux_q4, kvalues_mxfp4); + const int k0 = kbx * (2 * QI_MXFP4) + kqsx; + +#if defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE) + x_qs[i*sram_stride + k0 + 0] = v.x; + x_qs[i*sram_stride + k0 + QI_MXFP4] = v.y; +#else + x_qs[i*(2*MMQ_TILE_NE_K + 1) + k0 + 0] = v.x; + x_qs[i*(2*MMQ_TILE_NE_K + 1) + k0 + QI_MXFP4] = v.y; +#endif // defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE) + } + + constexpr int blocks_per_tile_x_row = MMQ_TILE_NE_K / QI_MXFP4; + constexpr int rows_per_warp = warp_size / blocks_per_tile_x_row; + const int kbxd = threadIdx.x % blocks_per_tile_x_row; + +#pragma unroll + for (int i0 = 0; i0 < I; i0 += nwarps * rows_per_warp) { + int i = i0 + threadIdx.y * rows_per_warp + threadIdx.x / blocks_per_tile_x_row; + + if (fallback) { + i = min(i, i_max); + } + + const block_mxfp4 * bxi = (const block_mxfp4 *) x + kbx0 + i*stride + kbxd; + +#if defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE) + x_df[i*sram_stride + kbxd] = ggml_cuda_e8m0_to_fp32(bxi->e)*0.5f; +#else + x_df[i*(MMQ_TILE_NE_K/QI_MXFP4) + i/QI_MXFP4 + kbxd] = ggml_cuda_e8m0_to_fp32(bxi->e)*0.5f; +#endif // defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE) + } +} + +template static __device__ __forceinline__ void ggml_cuda_mmq_load_tiles_mxfp4_fp4( + const char * __restrict__ x, int * __restrict__ x_tile, const int kbx0, const int i_max, const int stride) { + constexpr int warp_size = ggml_cuda_get_physical_warp_size(); + constexpr int nwarps = ggml_cuda_mmq_get_nthreads(type, J, fallback) / warp_size; + constexpr int I = ggml_cuda_mmq_get_I(type, J, fallback); + constexpr int sram_stride = ggml_cuda_mmq_get_sram_stride(type, J, fallback); + + int * x_qs = (int *) x_tile; + uint32_t * x_sc = (uint32_t *) (x_qs + 2 * MMQ_TILE_NE_K); + + const int txi = threadIdx.x; + + constexpr int iter_k = ggml_cuda_mmq_get_K_vram(type, J, fallback); + + constexpr int threads_per_row = iter_k / QK_MXFP4; // each thread processes 1 block + constexpr int rows_per_warp = warp_size / threads_per_row; + const int kbx = txi % threads_per_row; + const int row_in_warp = txi / threads_per_row; + +#pragma unroll + for (int i0 = 0; i0 < I; i0 += rows_per_warp * nwarps) { + int i = i0 + threadIdx.y * rows_per_warp + row_in_warp; + + if constexpr (fallback) { + i = min(i, i_max); + } + + const block_mxfp4 * bxi = (const block_mxfp4 *) x + kbx0 + i * stride + kbx; + + // quantize_mxfp4_mmq permutes nibbles to match the quantized format + const int k0 = kbx * 4; + memcpy(x_qs + i*sram_stride + k0, bxi->qs, 16); + + // Load E8M0 scales: pack 2 consecutive scales into one uint32 + if (kbx % 2 == 0) { + uint32_t e = bxi->e; + e |= ((bxi + 1)->e << 8); + x_sc[i*sram_stride + kbx / 2] = e; + } + } +} + +template static __device__ __forceinline__ void ggml_cuda_mmq_load_tiles_nvfp4( + const char * __restrict__ x, int * __restrict__ x_tile, const int kb0, const int i_max, const int stride) { + constexpr int warp_size = ggml_cuda_get_physical_warp_size(); + constexpr int nwarps = ggml_cuda_mmq_get_nthreads(type, J, fallback) / warp_size; + constexpr int I = ggml_cuda_mmq_get_I(type, J, fallback); + constexpr int sram_stride = ggml_cuda_mmq_get_sram_stride(type, J, fallback); + +#if defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE) + int * x_qs = (int *) x_tile; + float * x_df = (float *) (x_qs + MMQ_TILE_NE_K*2); +#else + constexpr tile_x_sizes txs = mmq_get_dp4a_tile_x_sizes(GGML_TYPE_NVFP4, I); + int * x_qs = (int *) x_tile; + float * x_df = (float *) (x_qs + txs.qs); +#endif // defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE) + + constexpr int threads_per_row = MMQ_ITER_K / QK_NVFP4; + constexpr int rows_per_warp = warp_size / threads_per_row; + const int kbx = threadIdx.x % threads_per_row; + const int row_in_warp = threadIdx.x / threads_per_row; + +#pragma unroll + for (int i0 = 0; i0 < I; i0 += rows_per_warp * nwarps) { + int i = i0 + threadIdx.y * rows_per_warp + row_in_warp; + + if constexpr (fallback) { + i = min(i, i_max); + } + + const block_nvfp4 * bxi = (const block_nvfp4 *) x + kb0 + i * stride + kbx; + const uint32_t * __restrict__ src_qs = reinterpret_cast(bxi->qs); + const int kqs = 16 * kbx; + const int ksc = 4 * kbx; + +#pragma unroll + for (int sub = 0; sub < QK_NVFP4 / QK_NVFP4_SUB; ++sub) { + const int2 q0 = get_int_from_table_16(src_qs[2 * sub + 0], kvalues_mxfp4); + const int2 q1 = get_int_from_table_16(src_qs[2 * sub + 1], kvalues_mxfp4); + +#if defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE) + x_qs[i*sram_stride + kqs + 4 * sub + 0] = q0.x; + x_qs[i*sram_stride + kqs + 4 * sub + 1] = q1.x; + x_qs[i*sram_stride + kqs + 4 * sub + 2] = q0.y; + x_qs[i*sram_stride + kqs + 4 * sub + 3] = q1.y; + x_df[i*sram_stride + ksc + sub] = ggml_cuda_ue4m3_to_fp32(bxi->d[sub]); +#else + x_qs[i * (2 * MMQ_TILE_NE_K + 1) + kqs + 4 * sub + 0] = q0.x; + x_qs[i * (2 * MMQ_TILE_NE_K + 1) + kqs + 4 * sub + 1] = q1.x; + x_qs[i * (2 * MMQ_TILE_NE_K + 1) + kqs + 4 * sub + 2] = q0.y; + x_qs[i * (2 * MMQ_TILE_NE_K + 1) + kqs + 4 * sub + 3] = q1.y; + x_df[i * (2 * MMQ_TILE_NE_K * 2 / QI_NVFP4) + i / (QK_NVFP4_SUB / QI_NVFP4) + ksc + sub] = ggml_cuda_ue4m3_to_fp32(bxi->d[sub]); +#endif // defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE) + } + } +} + +template static __device__ __forceinline__ void ggml_cuda_mmq_load_tiles_nvfp4_nvfp4( + const char * __restrict__ x, int * __restrict__ x_tile, const int kbx0, const int i_max, const int stride) { + constexpr int warp_size = ggml_cuda_get_physical_warp_size(); + constexpr int nwarps = ggml_cuda_mmq_get_nthreads(type, J, fallback) / warp_size; + constexpr int I = ggml_cuda_mmq_get_I(type, J, fallback); + constexpr int iter_k = ggml_cuda_mmq_get_K_vram(type, J, fallback); + constexpr int threads_per_row = iter_k / QK_NVFP4; // each thread processes 1 block + constexpr int rows_per_warp = warp_size / threads_per_row; + constexpr int sram_stride = ggml_cuda_mmq_get_sram_stride(type, J, fallback); + + uint32_t * x_u32 = (uint32_t *) x_tile; + + const int txi = threadIdx.x; + const int kbx = txi % threads_per_row; + const int row_in_warp = txi / threads_per_row; + + const block_nvfp4 * bxi_base = (const block_nvfp4 *) x + kbx0 + kbx; + uint32_t * x_u32_scale = x_u32 + 64 + kbx; + +#pragma unroll + for (int i0 = 0; i0 < I; i0 += rows_per_warp * nwarps) { + int i = i0 + threadIdx.y * rows_per_warp + row_in_warp; + + if constexpr (fallback) { + i = min(i, i_max); + } + + const block_nvfp4 * bxi = bxi_base + i * stride; + + const uint32_t * src_qs = reinterpret_cast(bxi->qs); + +#pragma unroll + for (int sub = 0; sub < QK_NVFP4 / QK_NVFP4_SUB; ++sub) { + x_u32[i*sram_stride + 8*kbx + 2 * sub + 0] = src_qs[2 * sub + 0]; + x_u32[i*sram_stride + 8*kbx + 2 * sub + 1] = src_qs[2 * sub + 1]; + } + + x_u32_scale[i*sram_stride] = get_int_b4(bxi->d, 0); + } +} diff --git a/python/freetoken/kernel/csrc/gguf_mmq/mmq-vec-dot.cuh b/python/freetoken/kernel/csrc/gguf_mmq/mmq-vec-dot.cuh new file mode 100644 index 00000000..d5734338 --- /dev/null +++ b/python/freetoken/kernel/csrc/gguf_mmq/mmq-vec-dot.cuh @@ -0,0 +1,1251 @@ +#pragma once + +#include "vecdotq.cuh" +#include "mma.cuh" + +using namespace ggml_cuda_mma; + +#include "mmq.cuh" + +template static __device__ __forceinline__ void ggml_cuda_mmq_vec_dot_q4_0_q8_1_dp4a( + const int * __restrict__ x, const int * __restrict__ y, float * __restrict__ sum, const int k00) { + constexpr int warp_size = ggml_cuda_get_physical_warp_size(); + constexpr int nwarps = ggml_cuda_mmq_get_nthreads(type, J, fallback) / warp_size; + constexpr int I = ggml_cuda_mmq_get_I(type, J, fallback); + + constexpr tile_x_sizes txs = mmq_get_dp4a_tile_x_sizes(GGML_TYPE_Q4_0, I); + const int * x_qs = (const int *) x; + const float * x_df = (const float *) x_qs + txs.qs; + const int * y_qs = (const int *) y + 4; + const half2 * y_ds = (const half2 *) y; + +// #pragma unroll + for (int k01 = 0; k01 < MMQ_TILE_NE_K; k01 += QR4_0*VDR_Q4_0_Q8_1_MMQ) { + const int k0 = k00 + k01; + +#pragma unroll + for (int j0 = 0; j0 < J; j0 += nwarps) { + const int j = j0 + threadIdx.y; + +#pragma unroll + for (int i0 = 0; i0 < I; i0 += warp_size) { + const int i = i0 + threadIdx.x; + const int kyqs = QI8_1 * ((k01/2) / (QI8_1/2)) + (k01/2) % (QI8_1/2); + + int u[2*VDR_Q4_0_Q8_1_MMQ]; + + constexpr int max_cpy = ggml_cuda_get_max_cpy_bytes(); + constexpr int mcpy_int = max_cpy / sizeof(int); + static_assert(VDR_Q4_0_Q8_1_MMQ == 4, "bad VDR_Q4_0_Q8_1_MMQ"); + + int tmp0[4], tmp1[4]; + + #pragma unroll + for (int l0 = 0; l0 < 4 / mcpy_int; ++l0) { + ggml_cuda_memcpy_1(tmp0 + l0 * mcpy_int, &y_qs[j*MMQ_TILE_Y_K + kyqs + l0 * mcpy_int] ); + ggml_cuda_memcpy_1(tmp1 + l0 * mcpy_int, &y_qs[j*MMQ_TILE_Y_K + kyqs + QI4_0 + l0 * mcpy_int]); + } + + u[0]=tmp0[0]; u[2]=tmp0[1]; u[4]=tmp0[2]; u[6]=tmp0[3]; + u[1]=tmp1[0]; u[3]=tmp1[1]; u[5]=tmp1[2]; u[7]=tmp1[3]; + + sum[j0/nwarps*I/warp_size + i0/warp_size] += vec_dot_q4_0_q8_1_impl + (&x_qs[i*(MMQ_TILE_NE_K + 1) + k0/QR4_0], u, + x_df[i*(MMQ_TILE_NE_K/QI4_0) + i/QI4_0 + k0/(QR4_0*QI4_0)], y_ds[j*MMQ_TILE_Y_K + k01/QI8_1]); + } + } + } +} + +template static __device__ __forceinline__ void ggml_cuda_mmq_vec_dot_q4_1_q8_1_dp4a( + const int * __restrict__ x, const int * __restrict__ y, float * __restrict__ sum, const int k00) { + constexpr int warp_size = ggml_cuda_get_physical_warp_size(); + constexpr int nwarps = ggml_cuda_mmq_get_nthreads(type, J, fallback) / warp_size; + constexpr int I = ggml_cuda_mmq_get_I(type, J, fallback); + + constexpr tile_x_sizes txs = mmq_get_dp4a_tile_x_sizes(GGML_TYPE_Q4_1, I); + const int * x_qs = (const int *) x; + const half2 * x_dm = (const half2 *) x_qs + txs.qs; + const int * y_qs = (const int *) y + 4; + const half2 * y_ds = (const half2 *) y; + +// #pragma unroll + for (int k01 = 0; k01 < MMQ_TILE_NE_K; k01 += QR4_1*VDR_Q4_1_Q8_1_MMQ) { + const int k0 = k00 + k01; + +#pragma unroll + for (int j0 = 0; j0 < J; j0 += nwarps) { + const int j = j0 + threadIdx.y; + +#pragma unroll + for (int i0 = 0; i0 < I; i0 += warp_size) { + const int i = i0 + threadIdx.x; + const int kyqs = QI8_1 * ((k01/2) / (QI8_1/2)) + (k01/2) % (QI8_1/2); + + int u[2*VDR_Q4_1_Q8_1_MMQ]; + + constexpr int max_cpy = ggml_cuda_get_max_cpy_bytes(); + constexpr int mcpy_int = max_cpy / sizeof(int); + static_assert(VDR_Q4_0_Q8_1_MMQ == 4, "bad VDR_Q4_0_Q8_1_MMQ"); + + int tmp0[4], tmp1[4]; + + #pragma unroll + for (int l0 = 0; l0 < 4 / mcpy_int; ++l0) { + ggml_cuda_memcpy_1(tmp0 + l0 * mcpy_int, &y_qs[j*MMQ_TILE_Y_K + kyqs + l0 * mcpy_int] ); + ggml_cuda_memcpy_1(tmp1 + l0 * mcpy_int, &y_qs[j*MMQ_TILE_Y_K + kyqs + QI4_1 + l0 * mcpy_int]); + } + + u[0]=tmp0[0]; u[2]=tmp0[1]; u[4]=tmp0[2]; u[6]=tmp0[3]; + u[1]=tmp1[0]; u[3]=tmp1[1]; u[5]=tmp1[2]; u[7]=tmp1[3]; + + sum[j0/nwarps*I/warp_size + i0/warp_size] += vec_dot_q4_1_q8_1_impl + (&x_qs[i*(MMQ_TILE_NE_K + 1) + k0/QR4_1], u, + x_dm[i*(MMQ_TILE_NE_K/QI4_1) + i/QI4_1 + k0/(QR4_1*QI4_1)], y_ds[j*MMQ_TILE_Y_K + k01/QI8_1]); + } + } + } +} + +template static __device__ __forceinline__ void ggml_cuda_mmq_vec_dot_q8_0_q8_1_dp4a( + const int * __restrict__ x, const int * __restrict__ y, float * __restrict__ sum, const int k00) { + constexpr int warp_size = ggml_cuda_get_physical_warp_size(); + constexpr int nwarps = ggml_cuda_mmq_get_nthreads(type, J, fallback) / warp_size; + constexpr int I = ggml_cuda_mmq_get_I(type, J, fallback); + + constexpr tile_x_sizes txs = mmq_get_dp4a_tile_x_sizes(GGML_TYPE_Q8_0, I); + const int * x_qs = (const int *) x; + const float * x_df = (const float *) x_qs + txs.qs; + const int * y_qs = (const int *) y + 4; + const float * y_df = (const float *) y; + +// #pragma unroll + for (int k01 = 0; k01 < MMQ_TILE_NE_K; k01 += VDR_Q8_0_Q8_1_MMQ) { + const int k0 = k00 + k01; + +#pragma unroll + for (int j0 = 0; j0 < J; j0 += nwarps) { + const int j = j0 + threadIdx.y; + +#pragma unroll + for (int i0 = 0; i0 < I; i0 += warp_size) { + const int i = i0 + threadIdx.x; + + sum[j0/nwarps*I/warp_size + i0/warp_size] += vec_dot_q8_0_q8_1_impl + (&x_qs[i*(2*MMQ_TILE_NE_K + 1) + k0], &y_qs[j*MMQ_TILE_Y_K + k0 % MMQ_TILE_NE_K], + x_df[i*(2*MMQ_TILE_NE_K/QI8_0) + i/(QI8_0/2) + k0/QI8_0], y_df[j*MMQ_TILE_Y_K + (k0/QI8_1) % (MMQ_TILE_NE_K/QI8_1)]); + } + } + } +} + +template +static __device__ __forceinline__ void ggml_cuda_mmq_vec_dot_q8_0_q8_1_mma( + const int * __restrict__ x, const int * __restrict__ y, float * __restrict__ sum, const int k00) { +#if defined(AMD_MFMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE) + constexpr data_layout input_layout = get_input_data_layout(); + typedef tile<16, 8, int, input_layout> tile_A; + typedef tile<16, 8, int, input_layout> tile_B; + typedef tile<16, 16, int, DATA_LAYOUT_J_MAJOR> tile_C; + + constexpr int I = ggml_cuda_mmq_get_I(type, J, fallback); + constexpr int sram_stride = ggml_cuda_mmq_get_sram_stride(type, J, fallback); + constexpr int rows_per_warp = ggml_cuda_mmq_get_rows_per_warp(type, J, fallback); + constexpr int ntx = rows_per_warp/tile_C::I; // Number of x minitiles per warp. + + y += (threadIdx.y % ntx) * (tile_C::J*MMQ_TILE_Y_K); + + const int * x_qs = (const int *) x; + const float * x_df = (const float *) x_qs + 2*MMQ_TILE_NE_K; + const int * y_qs = (const int *) y + 4; + const float * y_df = (const float *) y; + const half2 * y_ds = (const half2 *) y; + + const int i0 = (threadIdx.y / ntx) * rows_per_warp; + + for (int k01 = 0; k01 < MMQ_TILE_NE_K; k01 += QI8_0) { + const int k0 = k00 + k01; + + tile_A A[ntx]; +#pragma unroll + for (int n = 0; n < ntx; ++n) { + load_ldmatrix(A[n], x_qs + (i0 + n*tile_A::I)*sram_stride + k0, sram_stride); + } + +#pragma unroll + for (int j0 = 0; j0 < J; j0 += ntx*tile_C::J) { + tile_B B; + load_ldmatrix(B, y_qs + j0*MMQ_TILE_Y_K + k01, MMQ_TILE_Y_K); + + float dB; + const int j = j0 + tile_C::get_j(0); + if (ds_layout == MMQ_Q8_1_DS_LAYOUT_D4) { + dB = y_df[j*MMQ_TILE_Y_K + k01/QI8_1]; + } else { + dB = __low2float(y_ds[j*MMQ_TILE_Y_K + k01/QI8_1]); + } + +#pragma unroll + for (int n = 0; n < ntx; ++n) { + tile_C C; + mma(C, A[n], B); + +#pragma unroll + for (int l = 0; l < tile_C::ne; ++l) { + const int i = i0 + n*tile_A::I + tile_C::get_i(l); + const float dA = x_df[i*sram_stride + k0/QI8_0]; + sum[(j0/tile_C::J + n)*tile_C::ne + l] += C.x[l]*dA*dB; + } + } + } + } +#else + typedef tile<16, 8, int> tile_A; + typedef tile< 8, 8, int> tile_B; + typedef tile<16, 8, int> tile_C; + + constexpr int I = ggml_cuda_mmq_get_I(type, J, fallback); + constexpr int sram_stride = ggml_cuda_mmq_get_sram_stride(type, J, fallback); + constexpr int rows_per_warp = ggml_cuda_mmq_get_rows_per_warp(type, J, fallback); + constexpr int ntx = rows_per_warp/tile_C::I; // Number of x minitiles per warp. + + y += (threadIdx.y % ntx) * (tile_C::J*MMQ_TILE_Y_K); + + const int * x_qs = (const int *) x; + const float * x_df = (const float *) x_qs + 2*MMQ_TILE_NE_K; + const int * y_qs = (const int *) y + 4; + const float * y_df = (const float *) y; + const half2 * y_ds = (const half2 *) y; + + tile_A A[ntx][MMQ_TILE_NE_K/QI8_0]; + float dA[ntx][tile_C::ne/2][MMQ_TILE_NE_K/QI8_0]; + + const int i0 = (threadIdx.y/ntx)*rows_per_warp; + +#pragma unroll + for (int n = 0; n < ntx; ++n) { +#pragma unroll + for (int k01 = 0; k01 < MMQ_TILE_NE_K; k01 += QI8_0) { + const int k0 = k00 + k01; + + load_ldmatrix(A[n][k01/QI8_0], x_qs + (i0 + n*tile_A::I)*sram_stride + k0, sram_stride); + } + +#pragma unroll + for (int l = 0; l < tile_C::ne/2; ++l) { + const int i = i0 + n*tile_A::I + tile_C::get_i(2*l); + +#pragma unroll + for (int k01 = 0; k01 < MMQ_TILE_NE_K; k01 += QI8_0) { + const int k0 = k00 + k01; + + dA[n][l][k01/QI8_0] = x_df[i*sram_stride + k0/QI8_0]; + } + } + } + +#pragma unroll + for (int j0 = 0; j0 < J; j0 += ntx*tile_C::J) { +#pragma unroll + for (int k01 = 0; k01 < MMQ_TILE_NE_K; k01 += QI8_0) { + tile_B B; + float dB[tile_C::ne/2]; + + load_generic(B, y_qs + j0*MMQ_TILE_Y_K + k01, MMQ_TILE_Y_K); // faster than load_ldmatrix + +#pragma unroll + for (int l = 0; l < tile_C::ne/2; ++l) { + const int j = j0 + tile_C::get_j(l); + + if (ds_layout == MMQ_Q8_1_DS_LAYOUT_D4) { + dB[l] = y_df[j*MMQ_TILE_Y_K + k01/QI8_1]; + } else { + dB[l] = __low2float(y_ds[j*MMQ_TILE_Y_K + k01/QI8_1]); + } + } + +#pragma unroll + for (int n = 0; n < ntx; ++n) { + tile_C C; + mma(C, A[n][k01/QI8_0], B); + +#pragma unroll + for (int l = 0; l < tile_C::ne; ++l) { + sum[(j0/tile_C::J + n)*tile_C::ne + l] += C.x[l]*dA[n][l/2][k01/QI8_0]*dB[l%2]; + } + } + } + } +#endif // defined(AMD_MFMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE) +} + + +template static __device__ __forceinline__ void ggml_cuda_mmq_vec_dot_q8_1_q8_1_dp4a( + const int * __restrict__ x, const int * __restrict__ y, float * __restrict__ sum, const int k00) { + constexpr int warp_size = ggml_cuda_get_physical_warp_size(); + constexpr int nwarps = ggml_cuda_mmq_get_nthreads(type, J, fallback) / warp_size; + constexpr int I = ggml_cuda_mmq_get_I(type, J, fallback); + + constexpr tile_x_sizes txs = mmq_get_dp4a_tile_x_sizes(GGML_TYPE_Q5_1, I); + const int * x_qs = (const int *) x; + const half2 * x_dm = (const half2 *) x_qs + txs.qs; + const int * y_qs = (const int *) y + 4; + const half2 * y_ds = (const half2 *) y; + +// #pragma unroll + for (int k01 = 0; k01 < MMQ_TILE_NE_K; k01 += VDR_Q8_0_Q8_1_MMQ) { + const int k0 = k00 + k01; + +#pragma unroll + for (int j0 = 0; j0 < J; j0 += nwarps) { + const int j = j0 + threadIdx.y; + +#pragma unroll + for (int i0 = 0; i0 < I; i0 += warp_size) { + const int i = i0 + threadIdx.x; + + sum[j0/nwarps*I/warp_size + i0/warp_size] += vec_dot_q8_1_q8_1_impl + (&x_qs[i*(2*MMQ_TILE_NE_K + 1) + k0], &y_qs[j*MMQ_TILE_Y_K + k01], + x_dm[i*(MMQ_TILE_NE_K/QI5_1) + i/QI5_1 + k0/QI8_1], y_ds[j*MMQ_TILE_Y_K + k01/QI8_1]); + } + } + } +} + +template static __device__ __forceinline__ void ggml_cuda_mmq_vec_dot_q8_1_q8_1_mma( + const int * __restrict__ x, const int * __restrict__ y, float * __restrict__ sum, const int k00) { +#if defined(AMD_MFMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE) + constexpr data_layout input_layout = get_input_data_layout(); + typedef tile<16, 8, int, input_layout> tile_A; + typedef tile<16, 8, int, input_layout> tile_B; + typedef tile<16, 16, int, DATA_LAYOUT_J_MAJOR> tile_C; + + constexpr int I = ggml_cuda_mmq_get_I(type, J, fallback); + constexpr int sram_stride = ggml_cuda_mmq_get_sram_stride(type, J, fallback); + constexpr int rows_per_warp = ggml_cuda_mmq_get_rows_per_warp(type, J, fallback); + constexpr int ntx = rows_per_warp/tile_C::I; // Number of x minitiles per warp. + + y += (threadIdx.y % ntx) * (tile_C::J*MMQ_TILE_Y_K); + + const int * x_qs = (const int *) x; + const half2 * x_dm = (const half2 *) x_qs + 2*MMQ_TILE_NE_K; + const int * y_qs = (const int *) y + 4; + const half2 * y_dm = (const half2 *) y; + + const int i0 = (threadIdx.y / ntx) * rows_per_warp; + + for (int k01 = 0; k01 < MMQ_TILE_NE_K; k01 += QI8_1) { + const int k0 = k00 + k01; + + tile_A A[ntx]; +#pragma unroll + for (int n = 0; n < ntx; ++n) { + load_ldmatrix(A[n], x_qs + (i0 + n*tile_A::I)*sram_stride + k0, sram_stride); + } + +#pragma unroll + for (int j0 = 0; j0 < J; j0 += ntx*tile_C::J) { + tile_B B; + load_ldmatrix(B, y_qs + j0*MMQ_TILE_Y_K + k01, MMQ_TILE_Y_K); + + const int j = j0 + tile_C::get_j(0); + const float2 dsB = __half22float2(y_dm[j*MMQ_TILE_Y_K + k01/QI8_1]); + +#pragma unroll + for (int n = 0; n < ntx; ++n) { + tile_C C; + mma(C, A[n], B); + +#pragma unroll + for (int l = 0; l < tile_C::ne; ++l) { + const int i = i0 + n*tile_A::I + tile_C::get_i(l); + float2 dmA = __half22float2(x_dm[i*sram_stride + k0/QI8_1]); + sum[(j0/tile_C::J + n)*tile_C::ne + l] += dmA.x*dsB.x*C.x[l]; + sum[(j0/tile_C::J + n)*tile_C::ne + l] += dmA.y*dsB.y; + } + } + } + } +#else + typedef tile<16, 8, int> tile_A; + typedef tile< 8, 8, int> tile_B; + typedef tile<16, 8, int> tile_C; + + constexpr int I = ggml_cuda_mmq_get_I(type, J, fallback); + constexpr int sram_stride = ggml_cuda_mmq_get_sram_stride(type, J, fallback); + constexpr int rows_per_warp = ggml_cuda_mmq_get_rows_per_warp(type, J, fallback); + constexpr int ntx = rows_per_warp/tile_C::I; // Number of x minitiles per warp. + + y += (threadIdx.y % ntx) * (tile_C::J*MMQ_TILE_Y_K); + + const int * x_qs = (const int *) x; + const half2 * x_dm = (const half2 *) x_qs + 2*MMQ_TILE_NE_K; + const int * y_qs = (const int *) y + 4; + const half2 * y_dm = (const half2 *) y; + + tile_A A[ntx][MMQ_TILE_NE_K/QI8_1]; + float2 dmA[ntx][tile_C::ne/2][MMQ_TILE_NE_K/QI8_1]; + + const int i0 = (threadIdx.y/ntx)*rows_per_warp; + +#pragma unroll + for (int n = 0; n < ntx; ++n) { +#pragma unroll + for (int k01 = 0; k01 < MMQ_TILE_NE_K; k01 += QI8_1) { + const int k0 = k00 + k01; + + load_ldmatrix(A[n][k01/QI8_1], x_qs + (i0 + n*tile_A::I)*sram_stride + k0, sram_stride); + } + +#pragma unroll + for (int l = 0; l < tile_C::ne/2; ++l) { + const int i = i0 + n*tile_A::I + tile_C::get_i(2*l); + +#pragma unroll + for (int k01 = 0; k01 < MMQ_TILE_NE_K; k01 += QI8_1) { + const int k0 = k00 + k01; + + dmA[n][l][k01/QI8_1] = __half22float2(x_dm[i*sram_stride + k0/QI8_1]); + } + } + } + +#pragma unroll + for (int j0 = 0; j0 < J; j0 += ntx*tile_C::J) { +#pragma unroll + for (int k01 = 0; k01 < MMQ_TILE_NE_K; k01 += QI8_1) { + tile_B B; + float2 dsB[tile_C::ne/2]; + + load_generic(B, y_qs + j0*MMQ_TILE_Y_K + k01, MMQ_TILE_Y_K); // faster than load_ldmatrix + +#pragma unroll + for (int l = 0; l < tile_C::ne/2; ++l) { + const int j = j0 + tile_C::get_j(l); + + dsB[l] = __half22float2(y_dm[j*MMQ_TILE_Y_K + k01/QI8_1]); + } + +#pragma unroll + for (int n = 0; n < ntx; ++n) { + tile_C C; + mma(C, A[n][k01/QI8_1], B); + +#pragma unroll + for (int l = 0; l < tile_C::ne; ++l) { + sum[(j0/tile_C::J + n)*tile_C::ne + l] += dmA[n][l/2][k01/QI8_1].x*dsB[l%2].x*C.x[l]; + sum[(j0/tile_C::J + n)*tile_C::ne + l] += dmA[n][l/2][k01/QI8_1].y*dsB[l%2].y; + } + } + } + } +#endif // defined(AMD_MFMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE) +} + +// Used for NVFP4, Q3_K, IQ2_S, and IQ2_XS +template static __device__ __forceinline__ void ggml_cuda_mmq_vec_dot_q8_0_16_q8_1_dp4a( + const int * __restrict__ x, const int * __restrict__ y, float * __restrict__ sum, const int k00) { + constexpr int warp_size = ggml_cuda_get_physical_warp_size(); + constexpr int nwarps = ggml_cuda_mmq_get_nthreads(type, J, fallback) / warp_size; + constexpr int I = ggml_cuda_mmq_get_I(type, J, fallback); + + constexpr tile_x_sizes txs = mmq_get_dp4a_tile_x_sizes(type, I); + const int * x_qs = (const int *) x; + const float * x_df = (const float *) x_qs + txs.qs; + const int * y_qs = (const int *) y + 4; + const float * y_df = (const float *) y; + +// #pragma unroll + for (int k01 = 0; k01 < MMQ_TILE_NE_K; k01 += QI8_0) { + const int k0 = k00 + k01; + +#pragma unroll + for (int j0 = 0; j0 < J; j0 += nwarps) { + const int j = j0 + threadIdx.y; + +#pragma unroll + for (int i0 = 0; i0 < I; i0 += warp_size) { + const int i = i0 + threadIdx.x; + + sum[j0/nwarps*I/warp_size + i0/warp_size] += vec_dot_q8_0_16_q8_1_impl( + &x_qs[i*(2*MMQ_TILE_NE_K + 1) + k0], + &y_qs[j*MMQ_TILE_Y_K + k01], + &x_df[i*(2*MMQ_TILE_NE_K*2/QI8_0) + i/(QI8_0/4) + k0/(QI8_0/2)], + y_df[j*MMQ_TILE_Y_K + k01/QI8_1]); + } + } + } +} + +// Used for Q3_K, IQ2_S, and IQ2_XS: +template static __device__ __forceinline__ void ggml_cuda_mmq_vec_dot_q8_0_16_q8_1_mma( + const int * __restrict__ x, const int * __restrict__ y, float * __restrict__ sum, const int k00) { +#if defined(AMD_MFMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE) + constexpr data_layout input_layout = get_input_data_layout(); + typedef tile<16, 4, int, input_layout> tile_A; + typedef tile<16, 4, int, input_layout> tile_B; + typedef tile<16, 16, int, DATA_LAYOUT_J_MAJOR> tile_C; + + constexpr int I = ggml_cuda_mmq_get_I(type, J, fallback); + constexpr int sram_stride = ggml_cuda_mmq_get_sram_stride(type, J, fallback); + constexpr int rows_per_warp = ggml_cuda_mmq_get_rows_per_warp(type, J, fallback); + constexpr int ntx = rows_per_warp/tile_C::I; // Number of x minitiles per warp. + + y += (threadIdx.y % ntx) * (tile_C::J*MMQ_TILE_Y_K); + + const int * x_qs = (const int *) x; + const float * x_df = (const float *) x_qs + MMQ_TILE_NE_K*2; + const int * y_qs = (const int *) y + 4; + const float * y_df = (const float *) y; + + const int i0 = (threadIdx.y / ntx) * rows_per_warp; + + for (int k01 = 0; k01 < MMQ_TILE_NE_K; k01 += 4) { + const int k0 = k00 + k01; + + tile_A A[ntx]; +#pragma unroll + for (int n = 0; n < ntx; ++n) { + load_ldmatrix(A[n], x_qs + (i0 + n*tile_A::I)*sram_stride + k0, sram_stride); + } + +#pragma unroll + for (int j0 = 0; j0 < J; j0 += ntx*tile_C::J) { + tile_B B; + load_ldmatrix(B, y_qs + j0*MMQ_TILE_Y_K + k01, MMQ_TILE_Y_K); + + const int j = j0 + tile_C::get_j(0); + const float dB = y_df[j*MMQ_TILE_Y_K + k01/QI8_1]; + +#pragma unroll + for (int n = 0; n < ntx; ++n) { + tile_C C; + mma(C, A[n], B); + +#pragma unroll + for (int l = 0; l < tile_C::ne; ++l) { + const int i = i0 + n*tile_C::I + tile_C::get_i(l); + sum[(j0/tile_C::J + n)*tile_C::ne + l] += C.x[l] * x_df[i*sram_stride + k0/4] * dB; + } + } + } + } +#elif defined(TURING_MMA_AVAILABLE) + + typedef tile<16, 4, int> tile_A; + typedef tile<16, 8, int> tile_A_8; + typedef tile< 8, 4, int> tile_B; + typedef tile<16, 8, int> tile_C; + + constexpr int I = ggml_cuda_mmq_get_I(type, J, fallback); + constexpr int sram_stride = ggml_cuda_mmq_get_sram_stride(type, J, fallback); + constexpr int rows_per_warp = ggml_cuda_mmq_get_rows_per_warp(type, J, fallback); + constexpr int ntx = rows_per_warp/tile_C::I; // Number of x minitiles per warp. + + y += (threadIdx.y % ntx) * (tile_C::J*MMQ_TILE_Y_K); + + const int * x_qs = (const int *) x; + const float * x_df = (const float *) x_qs + MMQ_TILE_NE_K*2; + const int * y_qs = (const int *) y + 4; + const float * y_df = (const float *) y; + + const int i0 = (threadIdx.y / ntx) * (ntx*tile_A::I); + + tile_A A[ntx][8]; + float dA[ntx][tile_C::ne/2][8]; + +#pragma unroll + for (int n = 0; n < ntx; ++n) { +#pragma unroll + for (int k01 = 0; k01 < MMQ_TILE_NE_K; k01 += 8) { + const int k0 = k00 + k01; + + load_ldmatrix(((tile_A_8 *) A[n])[k01/8], x_qs + (i0 + n*tile_A::I)*sram_stride + k0, sram_stride); + } + +#pragma unroll + for (int l = 0; l < tile_C::ne/2; ++l) { + const int i = i0 + n*tile_C::I + tile_C::get_i(2*l); + +#pragma unroll + for (int k01 = 0; k01 < MMQ_TILE_NE_K; k01 += 4) { + const int k0 = k00 + k01; + + dA[n][l][k01/4] = x_df[i*sram_stride + k0/4]; + } + } + } + +#pragma unroll + for (int j0 = 0; j0 < J; j0 += ntx*tile_C::J) { +#pragma unroll + for (int k01 = 0; k01 < MMQ_TILE_NE_K; k01 += QR3_K*VDR_Q3_K_Q8_1_MMQ) { + tile_B B[2]; + float dB[tile_C::ne/2]; + + // Here load_generic is faster than load_ldmatrix. + load_generic(B[0], y_qs + j0*MMQ_TILE_Y_K + (k01 + 0), MMQ_TILE_Y_K); + load_generic(B[1], y_qs + j0*MMQ_TILE_Y_K + (k01 + tile_B::J), MMQ_TILE_Y_K); + +#pragma unroll + for (int l = 0; l < tile_C::ne/2; ++l) { + const int j = j0 + tile_C::get_j(l); + + dB[l] = y_df[j*MMQ_TILE_Y_K + k01/QI8_1]; + } + +#pragma unroll + for (int n = 0; n < ntx; ++n) { + tile_C C[2]; + mma(C[0], A[n][k01/4 + 0], B[0]); + mma(C[1], A[n][k01/4 + 1], B[1]); + +#pragma unroll + for (int l = 0; l < tile_C::ne; ++l) { + sum[(j0/tile_C::J + n)*tile_C::ne + l] += dB[l%2]*(C[0].x[l]*dA[n][l/2][k01/4 + 0] + C[1].x[l]*dA[n][l/2][k01/4 + 1]); + } + } + } + } +#else + GGML_UNUSED_VARS(x, y, sum, k00); + NO_DEVICE_CODE; +#endif // AMD_MFMA_AVAILABLE || AMD_WMMA_AVAILABLE +} + +template static __device__ __forceinline__ void ggml_cuda_mmq_vec_dot_q2_K_q8_1_dp4a( + const int * __restrict__ x, const int * __restrict__ y, float * __restrict__ sum, const int k00) { + constexpr int warp_size = ggml_cuda_get_physical_warp_size(); + constexpr int nwarps = ggml_cuda_mmq_get_nthreads(type, J, fallback) / warp_size; + constexpr int I = ggml_cuda_mmq_get_I(type, J, fallback); + + constexpr tile_x_sizes txs = mmq_get_dp4a_tile_x_sizes(GGML_TYPE_Q2_K, I); + const int * x_qs = (const int *) x; + const half2 * x_dm = (const half2 *) x_qs + txs.qs; + const int * y_qs = (const int *) y + 4; + const half2 * y_ds = (const half2 *) y; + + float2 y_df[J/nwarps]; +#pragma unroll + for (int j0 = 0; j0 < J; j0 += nwarps) { + const int j = j0 + threadIdx.y; + + y_df[j0/nwarps] = __half22float2(y_ds[j*MMQ_TILE_Y_K]); + } + +#pragma unroll + for (int k01 = 0; k01 < MMQ_TILE_NE_K/2; k01 += QR2_K*VDR_Q2_K_Q8_1_MMQ) { + const int k0 = k00 + k01; + +#pragma unroll + for (int j0 = 0; j0 < J; j0 += nwarps) { + const int j = j0 + threadIdx.y; + +#pragma unroll + for (int i0 = 0; i0 < I; i0 += warp_size) { + const int i = i0 + threadIdx.x; + + constexpr int ns = 2; + sum[j0/nwarps*I/warp_size + i0/warp_size] += vec_dot_q2_K_q8_1_impl_mmq( + &x_qs[i*(2*MMQ_TILE_NE_K + 1) + k0], &y_qs[j*MMQ_TILE_Y_K + k01], + &x_dm[i*(MMQ_TILE_NE_K + 1) + k0/4], k01 < MMQ_TILE_NE_K/2 ? y_df[j0/nwarps].x : y_df[j0/nwarps].y, + &y_ds[j*MMQ_TILE_Y_K + (1 + k01/QI8_1)]); + } + } + } + + // Some compilers fail to unroll the loop over k01 if there is a conditional statement for ns in the inner loop. + // As a workaround 2 separate loops are used instead. +#pragma unroll + for (int k01 = MMQ_TILE_NE_K/2; k01 < MMQ_TILE_NE_K; k01 += QR2_K*VDR_Q2_K_Q8_1_MMQ) { + const int k0 = k00 + k01; + +#pragma unroll + for (int j0 = 0; j0 < J; j0 += nwarps) { + const int j = j0 + threadIdx.y; + +#pragma unroll + for (int i0 = 0; i0 < I; i0 += warp_size) { + const int i = i0 + threadIdx.x; + + constexpr int ns = 1; + sum[j0/nwarps*I/warp_size + i0/warp_size] += vec_dot_q2_K_q8_1_impl_mmq( + &x_qs[i*(2*MMQ_TILE_NE_K + 1) + k0], &y_qs[j*MMQ_TILE_Y_K + k01], + &x_dm[i*(MMQ_TILE_NE_K + 1) + k0/4], k01 < MMQ_TILE_NE_K/2 ? y_df[j0/nwarps].x : y_df[j0/nwarps].y, + &y_ds[j*MMQ_TILE_Y_K + (1 + k01/QI8_1)]); + } + } + } +} + +template static __device__ __forceinline__ void ggml_cuda_mmq_vec_dot_q2_K_q8_1_mma( + const int * __restrict__ x, const int * __restrict__ y, float * __restrict__ sum, const int k00) { +#if defined(AMD_MFMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE) + constexpr data_layout input_layout = get_input_data_layout(); + typedef tile<16, 4, int, input_layout> tile_A; + typedef tile<16, 4, int, input_layout> tile_B; + typedef tile<16, 16, int, DATA_LAYOUT_J_MAJOR> tile_C; + + constexpr int I = ggml_cuda_mmq_get_I(type, J, fallback); + constexpr int sram_stride = ggml_cuda_mmq_get_sram_stride(type, J, fallback); + constexpr int rows_per_warp = ggml_cuda_mmq_get_rows_per_warp(type, J, fallback); + constexpr int ntx = rows_per_warp/tile_C::I; // Number of x minitiles per warp. + + y += (threadIdx.y % ntx) * (tile_C::J*MMQ_TILE_Y_K); + + const int * x_qs = (const int *) x; + const half2 * x_dm = (const half2 *) x_qs + MMQ_TILE_NE_K*2; + const int * y_qs = (const int *) y + 4; + const half2 * y_ds = (const half2 *) y; + + const int i0 = (threadIdx.y / ntx) * rows_per_warp; + + for (int k01 = 0; k01 < MMQ_TILE_NE_K; k01 += 4) { + const int k0 = k00 + k01; + + tile_A A[ntx]; +#pragma unroll + for (int n = 0; n < ntx; ++n) { + load_ldmatrix(A[n], x_qs + (i0 + n*tile_A::I)*sram_stride + k0, sram_stride); + } + +#pragma unroll + for (int j0 = 0; j0 < J; j0 += ntx*tile_C::J) { + tile_B B; + load_ldmatrix(B, y_qs + j0*MMQ_TILE_Y_K + k01, MMQ_TILE_Y_K); + + const int j = j0 + tile_C::get_j(0); + const float dB = (k01 < MMQ_TILE_NE_K/2) ? __half22float2(y_ds[j*MMQ_TILE_Y_K]).x : __half22float2(y_ds[j*MMQ_TILE_Y_K]).y; + const float sB = (k01 >= MMQ_TILE_NE_K * 3/4) ? 0 + : (((k01/4)%2) ? __half22float2(y_ds[j*MMQ_TILE_Y_K + (1 + k01/QI8_1)]).y + : __half22float2(y_ds[j*MMQ_TILE_Y_K + (1 + k01/QI8_1)]).x); + + tile_C Cm; + if (k01 >= MMQ_TILE_NE_K * 3/4) { + tile_A A1; +#pragma unroll + for (int l = 0; l < tile_A::ne; ++l) { + A1.x[l] = 0x01010101; + } + mma(Cm, A1, B); + } + +#pragma unroll + for (int n = 0; n < ntx; ++n) { + tile_C Cd; + mma(Cd, A[n], B); + +#pragma unroll + for (int l = 0; l < tile_C::ne; ++l) { + const int i = i0 + n*tile_C::I + tile_C::get_i(l); + const float2 dm = __half22float2(x_dm[i*sram_stride + k0/4]); + float tmp = Cd.x[l]*dm.x; + if (k01 >= MMQ_TILE_NE_K * 3/4) { + tmp -= Cm.x[l]*dm.y; + } + sum[(j0/tile_C::J + n)*tile_C::ne + l] += tmp*dB; + sum[(j0/tile_C::J + n)*tile_C::ne + l] -= dm.y*sB; + } + } + } + } +#elif defined(TURING_MMA_AVAILABLE) + + typedef tile<16, 4, int> tile_A; + typedef tile<16, 8, int> tile_A_8; + typedef tile< 8, 4, int> tile_B; + typedef tile<16, 8, int> tile_C; + + constexpr int I = ggml_cuda_mmq_get_I(type, J, fallback); + constexpr int sram_stride = ggml_cuda_mmq_get_sram_stride(type, J, fallback); + constexpr int rows_per_warp = ggml_cuda_mmq_get_rows_per_warp(type, J, fallback); + constexpr int ntx = rows_per_warp/tile_C::I; // Number of x minitiles per warp. + + y += (threadIdx.y % ntx) * (tile_C::J*MMQ_TILE_Y_K); + + const int * x_qs = (const int *) x; + const half2 * x_dm = (const half2 *) x_qs + MMQ_TILE_NE_K*2; + const int * y_qs = (const int *) y + 4; + const half2 * y_ds = (const half2 *) y; + + const int i0 = (threadIdx.y / ntx) * (ntx*tile_A::I); + + tile_A A[ntx][8]; + float dA[ntx][tile_C::ne/2][8]; + float mA[ntx][tile_C::ne/2][8]; + +#pragma unroll + for (int n = 0; n < ntx; ++n) { +#pragma unroll + for (int k01 = 0; k01 < MMQ_TILE_NE_K; k01 += QI8_1) { + const int k0 = k00 + k01; + + load_ldmatrix(((tile_A_8 *) A[n])[k01/QI8_1], x_qs + (i0 + n*tile_A::I)*sram_stride + k0, sram_stride); + } + } + +#pragma unroll + for (int n = 0; n < ntx; ++n) { +#pragma unroll + for (int l = 0; l < tile_C::ne/2; ++l) { + const int i = i0 + n*tile_C::I + tile_C::get_i(2*l); + +#pragma unroll + for (int k01 = 0; k01 < MMQ_TILE_NE_K; k01 += QI8_1/2) { + const int k0 = k00 + k01; + + const float2 dm = __half22float2(x_dm[i*sram_stride + k0/(QI8_1/2)]); + + dA[n][l][k01/(QI8_1/2)] = dm.x; + mA[n][l][k01/(QI8_1/2)] = dm.y; + } + } + } + +#pragma unroll + for (int j0 = 0; j0 < J; j0 += ntx*tile_C::J) { + float2 dB[tile_C::ne/2]; + +#pragma unroll + for (int l = 0; l < tile_C::ne/2; ++l) { + const int j = j0 + tile_C::get_j(l); + + dB[l] = __half22float2(y_ds[j*MMQ_TILE_Y_K]); + } + +#pragma unroll + for (int k01 = 0; k01 < MMQ_TILE_NE_K; k01 += QI8_1) { + tile_B B[2]; + + // Here load_generic is faster than load_ldmatrix. + load_generic(B[0], y_qs + j0*MMQ_TILE_Y_K + (k01 + 0), MMQ_TILE_Y_K); + load_generic(B[1], y_qs + j0*MMQ_TILE_Y_K + (k01 + tile_B::J), MMQ_TILE_Y_K); + + tile_C Cm[2]; + if (k01 >= MMQ_TILE_NE_K * 3/4) { + tile_A A1; + A1.x[0] = 0x01010101; + A1.x[1] = 0x01010101; + mma(Cm[0], A1, B[0]); + mma(Cm[1], A1, B[1]); + } + +#pragma unroll + for (int n = 0; n < ntx; ++n) { + tile_C Cd[2]; + + mma(Cd[0], A[n][k01/4 + 0], B[0]); + mma(Cd[1], A[n][k01/4 + 1], B[1]); + +#pragma unroll + for (int l = 0; l < tile_C::ne; ++l) { + float tmp = Cd[0].x[l]*dA[n][l/2][k01/4 + 0] + Cd[1].x[l]*dA[n][l/2][k01/4 + 1]; + if (k01 >= MMQ_TILE_NE_K * 3/4) { + tmp -= Cm[0].x[l]*mA[n][l/2][k01/4 + 0] + Cm[1].x[l]*mA[n][l/2][k01/4 + 1]; + } + sum[(j0/tile_C::J + n)*tile_C::ne + l] += tmp*(k01 < MMQ_TILE_NE_K/2 ? dB[l%2].x : dB[l%2].y); + } + } + } + +#pragma unroll + for (int k01 = 0; k01 < MMQ_TILE_NE_K * 3/4; k01 += QI8_1) { + float2 sB[tile_C::ne/2]; + +#pragma unroll + for (int l = 0; l < tile_C::ne/2; ++l) { + const int j = j0 + tile_C::get_j(l); + + sB[l] = __half22float2(y_ds[j*MMQ_TILE_Y_K + (1 + k01/QI8_1)]); + } + +#pragma unroll + for (int n = 0; n < ntx; ++n) { +#pragma unroll + for (int l = 0; l < tile_C::ne; ++l) { + sum[(j0/tile_C::J + n)*tile_C::ne + l] -= mA[n][l/2][k01/4 + 0]*sB[l%2].x; + sum[(j0/tile_C::J + n)*tile_C::ne + l] -= mA[n][l/2][k01/4 + 1]*sB[l%2].y; + } + } + } + } +#else + GGML_UNUSED_VARS(x, y, sum, k00); + NO_DEVICE_CODE; +#endif // AMD_MFMA_AVAILABLE || AMD_WMMA_AVAILABLE +} + +template static __device__ __forceinline__ void ggml_cuda_mmq_vec_dot_q3_K_q8_1_dp4a( + const int * __restrict__ x, const int * __restrict__ y, float * __restrict__ sum, const int k00) { + constexpr int warp_size = ggml_cuda_get_physical_warp_size(); + constexpr int nwarps = ggml_cuda_mmq_get_nthreads(type, J, fallback) / warp_size; + constexpr int I = ggml_cuda_mmq_get_I(type, J, fallback); + + constexpr tile_x_sizes txs = mmq_get_dp4a_tile_x_sizes(GGML_TYPE_Q3_K, I); + const int * x_qs = (const int *) x; + const float * x_df = (const float *) x_qs + txs.qs; + const int * x_sc = (const int *) x_df + txs.dm; + const int * y_qs = (const int *) y + 4; + const float * y_df = (const float *) y; + +// #pragma unroll + for (int k01 = 0; k01 < MMQ_TILE_NE_K; k01 += QR3_K*VDR_Q3_K_Q8_1_MMQ) { + const int k0 = k00 + k01; + +#pragma unroll + for (int j0 = 0; j0 < J; j0 += nwarps) { + const int j = j0 + threadIdx.y; + +#pragma unroll + for (int i0 = 0; i0 < I; i0 += warp_size) { + const int i = i0 + threadIdx.x; + + const int8_t * scales = ((const int8_t *) (x_sc + i*(MMQ_TILE_NE_K/8) + i/8)) + k0/4; + + sum[j0/nwarps*I/warp_size + i0/warp_size] += vec_dot_q3_K_q8_1_impl_mmq( + &x_qs[i*(2*MMQ_TILE_NE_K + 1) + k0], &y_qs[j*MMQ_TILE_Y_K + k01], scales, + x_df[i], y_df[j*MMQ_TILE_Y_K + k01/QI8_1]); + } + } + } +} + +template static __device__ __forceinline__ void ggml_cuda_mmq_vec_dot_q4_K_q8_1_dp4a( + const int * __restrict__ x, const int * __restrict__ y, float * __restrict__ sum, const int k00) { + constexpr int warp_size = ggml_cuda_get_physical_warp_size(); + constexpr int nwarps = ggml_cuda_mmq_get_nthreads(type, J, fallback) / warp_size; + constexpr int I = ggml_cuda_mmq_get_I(type, J, fallback); + + constexpr tile_x_sizes txs = mmq_get_dp4a_tile_x_sizes(GGML_TYPE_Q4_K, I); + const int * x_qs = (const int *) x; + const half2 * x_dm = (const half2 *) x_qs + txs.qs; + const int * x_sc = (const int *) x_dm + txs.dm; + const int * y_qs = (const int *) y + 4; + const half2 * y_ds = (const half2 *) y; + +// #pragma unroll + for (int k01 = 0; k01 < MMQ_TILE_NE_K; k01 += QR4_K*VDR_Q4_K_Q8_1_MMQ) { + const int k0 = k00 + k01; + +#pragma unroll + for (int j0 = 0; j0 < J; j0 += nwarps) { + const int j = j0 + threadIdx.y; + +#pragma unroll + for (int i0 = 0; i0 < I; i0 += warp_size) { + const int i = i0 + threadIdx.x; + + const uint8_t * sc = (const uint8_t *) &x_sc[i * (MMQ_TILE_NE_K/8) + i/8 + k0/32] + 2*(k01/16); + + sum[j0/nwarps*I/warp_size + i0/warp_size] += vec_dot_q4_K_q8_1_impl_mmq( + &x_qs[i*(MMQ_TILE_NE_K + 1) + k0/2], &y_qs[j*MMQ_TILE_Y_K + k01], sc, sc+8, + x_dm[i], &y_ds[j*MMQ_TILE_Y_K + k01/QI8_1]); + } + } + } +} + +template static __device__ __forceinline__ void ggml_cuda_mmq_vec_dot_q5_K_q8_1_dp4a( + const int * __restrict__ x, const int * __restrict__ y, float * __restrict__ sum, const int k00) { + constexpr int warp_size = ggml_cuda_get_physical_warp_size(); + constexpr int nwarps = ggml_cuda_mmq_get_nthreads(type, J, fallback) / warp_size; + constexpr int I = ggml_cuda_mmq_get_I(type, J, fallback); + + constexpr tile_x_sizes txs = mmq_get_dp4a_tile_x_sizes(GGML_TYPE_Q5_K, I); + const int * x_qs = (const int *) x; + const half2 * x_dm = (const half2 *) x_qs + txs.qs; + const int * x_sc = (const int *) x_dm + txs.dm; + const int * y_qs = (const int *) y + 4; + const half2 * y_ds = (const half2 *) y; + +// #pragma unroll + for (int k01 = 0; k01 < MMQ_TILE_NE_K; k01 += QR5_K*VDR_Q5_K_Q8_1_MMQ) { + const int k0 = k00 + k01; + +#pragma unroll + for (int j0 = 0; j0 < J; j0 += nwarps) { + const int j = j0 + threadIdx.y; + +#pragma unroll + for (int i0 = 0; i0 < I; i0 += warp_size) { + const int i = i0 + threadIdx.x; + + const uint8_t * sc = ((const uint8_t *) &x_sc[i * (MMQ_TILE_NE_K/8) + i/8 + k00/32]) + 2*(k01/16); + + sum[j0/nwarps*I/warp_size + i0/warp_size] += vec_dot_q5_K_q8_1_impl_mmq( + &x_qs[i*(QR5_K*MMQ_TILE_NE_K + 1) + k0], &y_qs[j*MMQ_TILE_Y_K + k01], sc, sc+8, + x_dm[i], &y_ds[j*MMQ_TILE_Y_K + k01/QI8_1]); + } + } + } +} + +template static __device__ __forceinline__ void ggml_cuda_mmq_vec_dot_q6_K_q8_1_dp4a( + const int * __restrict__ x, const int * __restrict__ y, float * __restrict__ sum, const int k00) { + constexpr int warp_size = ggml_cuda_get_physical_warp_size(); + constexpr int nwarps = ggml_cuda_mmq_get_nthreads(type, J, fallback) / warp_size; + constexpr int I = ggml_cuda_mmq_get_I(type, J, fallback); + + constexpr tile_x_sizes txs = mmq_get_dp4a_tile_x_sizes(GGML_TYPE_Q6_K, I); + const int * x_qs = (const int *) x; + const float * x_df = (const float *) x_qs + txs.qs; + const int * x_sc = (const int *) x_df + txs.dm; + const int * y_qs = (const int *) y + 4; + const float * y_df = (const float *) y; + +// #pragma unroll + for (int k01 = 0; k01 < MMQ_TILE_NE_K; k01 += QR6_K*VDR_Q6_K_Q8_1_MMQ) { + const int k0 = k00 + k01; + +#pragma unroll + for (int j0 = 0; j0 < J; j0 += nwarps) { + const int j = j0 + threadIdx.y; + +#pragma unroll + for (int i0 = 0; i0 < I; i0 += warp_size) { + const int i = i0 + threadIdx.x; + + const int8_t * sc = ((const int8_t *) &x_sc[i * (MMQ_TILE_NE_K/8) + i/8 + k0/16]); + + sum[j0/nwarps*I/warp_size + i0/warp_size] += vec_dot_q6_K_q8_1_impl_mmq( + &x_qs[i*(QR6_K*MMQ_TILE_NE_K + 1) + k0], &y_qs[j*MMQ_TILE_Y_K + k01], sc, + x_df[i*(MMQ_TILE_NE_K/QI6_K) + i/QI6_K], &y_df[j*MMQ_TILE_Y_K + k01/QI8_1]); + } + } + } +} + +template static __device__ __forceinline__ void ggml_cuda_mmq_vec_dot_q6_K_q8_1_mma( + const int * __restrict__ x, const int * __restrict__ y, float * __restrict__ sum, const int k00) { +#if defined(AMD_MFMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE) + constexpr data_layout input_layout = get_input_data_layout(); + typedef tile<16, 4, int, input_layout> tile_A; + typedef tile<16, 4, int, input_layout> tile_B; + typedef tile<16, 16, int, DATA_LAYOUT_J_MAJOR> tile_C; + + constexpr int I = ggml_cuda_mmq_get_I(type, J, fallback); + constexpr int sram_stride = ggml_cuda_mmq_get_sram_stride(type, J, fallback); + constexpr int rows_per_warp = ggml_cuda_mmq_get_rows_per_warp(type, J, fallback); + constexpr int ntx = rows_per_warp/tile_C::I; // Number of x minitiles per warp. + + y += (threadIdx.y % ntx) * (tile_C::J*MMQ_TILE_Y_K); + + const int * x_qs = (const int *) x; + const float * x_df = (const float *) x_qs + MMQ_TILE_NE_K*2; + const int * x_sc = (const int *) x_df + MMQ_TILE_NE_K/QI6_K; + const int * y_qs = (const int *) y + 4; + const float * y_df = (const float *) y; + + const int i0 = (threadIdx.y / ntx) * rows_per_warp; + + for (int k01 = 0; k01 < MMQ_TILE_NE_K; k01 += 4) { + const int k0 = k00 + k01; + + tile_A A[ntx]; +#pragma unroll + for (int n = 0; n < ntx; ++n) { + load_ldmatrix(A[n], x_qs + (i0 + n*tile_A::I)*sram_stride + k0, sram_stride); + } + +#pragma unroll + for (int j0 = 0; j0 < J; j0 += ntx*tile_C::J) { + tile_B B; + load_ldmatrix(B, y_qs + j0*MMQ_TILE_Y_K + k01, MMQ_TILE_Y_K); + + const int j = j0 + tile_C::get_j(0); + const float dB = y_df[j*MMQ_TILE_Y_K + k01/QI8_1]; + +#pragma unroll + for (int n = 0; n < ntx; ++n) { + tile_C C; + mma(C, A[n], B); + +#pragma unroll + for (int l = 0; l < tile_C::ne; ++l) { + const int i = i0 + n*tile_C::I + tile_C::get_i(l); + const int8_t * sc = (const int8_t *) (x_sc + i*sram_stride + k00/16); + sum[(j0/tile_C::J + n)*tile_C::ne + l] += C.x[l] * sc[k01/4] * x_df[i*sram_stride] * dB; + } + } + } + } +#elif defined(TURING_MMA_AVAILABLE) + + typedef tile<16, 4, int> tile_A; + typedef tile< 8, 4, int> tile_B; + typedef tile<16, 8, int> tile_C; + + constexpr int I = ggml_cuda_mmq_get_I(type, J, fallback); + constexpr int sram_stride = ggml_cuda_mmq_get_sram_stride(type, J, fallback); + constexpr int rows_per_warp = ggml_cuda_mmq_get_rows_per_warp(type, J, fallback); + constexpr int ntx = rows_per_warp/tile_C::I; // Number of x minitiles per warp. + + y += (threadIdx.y % ntx) * (tile_C::J*MMQ_TILE_Y_K); + + const int * x_qs = (const int *) x; + const float * x_df = (const float *) x_qs + MMQ_TILE_NE_K*2; + const int * x_sc = (const int *) x_df + MMQ_TILE_NE_K/QI6_K; + const int * y_qs = (const int *) y + 4; + const float * y_df = (const float *) y; + + const int i0 = (threadIdx.y / ntx) * (ntx*tile_A::I); + + tile_A A[ntx][8]; + int scA[ntx][tile_C::ne/2][8]; + float dA[ntx][tile_C::ne/2]; + +#pragma unroll + for (int n = 0; n < ntx; ++n) { +#pragma unroll + for (int k01 = 0; k01 < MMQ_TILE_NE_K; k01 += 8) { + const int k0 = k00 + k01; + + load_ldmatrix(A[n][k01/4 + 0], x_qs + (i0 + n*tile_A::I)*sram_stride + (k0 + 0), sram_stride); + load_ldmatrix(A[n][k01/4 + 1], x_qs + (i0 + n*tile_A::I)*sram_stride + (k0 + tile_A::J), sram_stride); + } + +#pragma unroll + for (int k01 = 0; k01 < MMQ_TILE_NE_K; k01 += 16) { + const int k0 = k00 + k01; + +#pragma unroll + for (int l = 0; l < tile_C::ne/2; ++l) { + const int i = i0 + n*tile_C::I + tile_C::get_i(2*l); + + const int sc_packed = x_sc[i*sram_stride + k0/16]; + const int8_t * sc = (const int8_t *) &sc_packed; + +#pragma unroll + for (int ksc = 0; ksc < sizeof(int); ++ksc) { + scA[n][l][k01/4 + ksc] = sc[ksc]; + } + } + } + +#pragma unroll + for (int l = 0; l < tile_C::ne/2; ++l) { + const int i = i0 + n*tile_C::I + tile_C::get_i(2*l); + + dA[n][l] = x_df[i*sram_stride]; + } + } + +#pragma unroll + for (int j0 = 0; j0 < J; j0 += ntx*tile_C::J) { + float tmp[ntx][tile_C::ne] = {{0.0f}}; + +#pragma unroll + for (int k01 = 0; k01 < MMQ_TILE_NE_K; k01 += 8) { + tile_B B[2]; + float dB[tile_C::ne/2]; + + // Here load_generic is faster than load_ldmatrix. + load_generic(B[0], y_qs + j0*MMQ_TILE_Y_K + 0 + k01, MMQ_TILE_Y_K); + load_generic(B[1], y_qs + j0*MMQ_TILE_Y_K + tile_B::J + k01, MMQ_TILE_Y_K); + +#pragma unroll + for (int l = 0; l < tile_C::ne/2; ++l) { + const int j = j0 + tile_C::get_j(l); + + dB[l] = y_df[j*MMQ_TILE_Y_K + k01/QI8_1]; + } + +#pragma unroll + for (int n = 0; n < ntx; ++n) { + tile_C C[2]; + mma(C[0], A[n][k01/4 + 0], B[0]); + mma(C[1], A[n][k01/4 + 1], B[1]); + +#pragma unroll + for (int l = 0; l < tile_C::ne; ++l) { + tmp[n][l] += (C[0].x[l]*scA[n][l/2][k01/4 + 0] + C[1].x[l]*scA[n][l/2][k01/4 + 1])*dB[l%2]; + } + } + } + +#pragma unroll + for (int n = 0; n < ntx; ++n) { +#pragma unroll + for (int l = 0; l < tile_C::ne; ++l) { + sum[(j0/tile_C::J + n)*tile_C::ne + l] += tmp[n][l]*dA[n][l/2]; + } + } + } +#else + GGML_UNUSED_VARS(x, y, sum, k00); + NO_DEVICE_CODE; +#endif // AMD_MFMA_AVAILABLE || AMD_WMMA_AVAILABLE +} + +// --------------------------------------------------------------------------------------------- + +// Shared MMA kernel for MXFP4 and NVFP4 on Blackwell. +// Both quantizations encode values as e2m1 (FP4) and produce one uint32 scale per +// m16n8k64 MMA call; only the PTX kind (scale_vec::2X ue8m0 vs scale_vec::4X ue4m3) +// and the per-type stride constant differ. +template static __device__ __forceinline__ void ggml_cuda_mmq_vec_dot_fp4_fp4_mma( + const int * __restrict__ x, const int * __restrict__ y, float * __restrict__ sum, const int k00) { + + typedef tile<16, 8, int> tile_A; + typedef tile<8, 8, int> tile_B; + typedef tile<16, 8, float> tile_C; + + constexpr int I = ggml_cuda_mmq_get_I(type, J, fallback); + constexpr int sram_stride = ggml_cuda_mmq_get_sram_stride(type, J, fallback); + constexpr int rows_per_warp = ggml_cuda_mmq_get_rows_per_warp(type, J, fallback); + constexpr int ntx = rows_per_warp / tile_C::I; + constexpr int nfrags = MMQ_TILE_NE_K / tile_A::J; + + y += (threadIdx.y % ntx) * (tile_C::J * MMQ_TILE_Y_K); + + const int * x_qs = (const int *) x; + const uint32_t * x_sc = (const uint32_t *) (x_qs + 2 * MMQ_TILE_NE_K); + const int * y_qs = (const int *) y + 4; + const uint32_t * y_sc = (const uint32_t *) y; + + // 2 threads per quad supply the packed scale register to the block_scale MMA, + // see https://docs.nvidia.com/cuda/parallel-thread-execution/#warp-level-block-scaling + const int tidx_A = threadIdx.x / 4 + (threadIdx.x % 2) * 8; + const int tidx_B = threadIdx.x / 4; + const int i0 = (threadIdx.y / ntx) * rows_per_warp; + + tile_A A[ntx][nfrags]; + uint32_t scaleA[ntx][nfrags]; + +#pragma unroll + for (int n = 0; n < ntx; ++n) { +#pragma unroll + for (int frag = 0; frag < nfrags; ++frag) { + const int k0 = k00 + frag * tile_A::J; + load_ldmatrix(A[n][frag], x_qs + (i0 + n * tile_A::I) * sram_stride + k0, sram_stride); + scaleA[n][frag] = x_sc[(i0 + n * tile_A::I + tidx_A) * sram_stride + k0 / tile_A::J]; + } + } + +#pragma unroll + for (int j0 = 0; j0 < J; j0 += ntx * tile_C::J) { + tile_B B[nfrags]; + uint32_t scaleB[nfrags]; + +#pragma unroll + for (int frag = 0; frag < nfrags; ++frag) { + const int k0 = frag * tile_B::J; + load_generic(B[frag], y_qs + j0 * MMQ_TILE_Y_K + k0, MMQ_TILE_Y_K); + scaleB[frag] = y_sc[(j0 + tidx_B) * MMQ_TILE_Y_K + frag]; + } + +#pragma unroll + for (int n = 0; n < ntx; ++n) { +#pragma unroll + for (int frag = 0; frag < nfrags; ++frag) { + tile_C C = {}; + mma_block_scaled_fp4(C, A[n][frag], B[frag], scaleA[n][frag], scaleB[frag]); +#pragma unroll + for (int l = 0; l < tile_C::ne; ++l) { + sum[(j0 / tile_C::J + n) * tile_C::ne + l] += C.x[l]; + } + } + } + } +} + diff --git a/python/freetoken/kernel/csrc/gguf_mmq/mmq.cuh b/python/freetoken/kernel/csrc/gguf_mmq/mmq.cuh new file mode 100644 index 00000000..2eb15fdf --- /dev/null +++ b/python/freetoken/kernel/csrc/gguf_mmq/mmq.cuh @@ -0,0 +1,1597 @@ +#pragma once + +#include "common.cuh" + +#include +#include + +#define MMQ_DP4A_MAX_BATCH_SIZE 64 // Max. batch size to use for dp4a MMQ kernels when FP16 tensor cores are available. +#define MMQ_ITER_K 256 +#define MMQ_ITER_K_FP4 512 +#define MMQ_NWARPS 8 + +typedef void (*ggml_cuda_mmq_load_tiles_t)(const char * __restrict__ x, int * x_tile, const int kbx0, const int i_max, const int stride); +typedef void (*ggml_cuda_mmq_vec_dot_t)(const int * __restrict__ x, const int * __restrict__ y, float * __restrict__ sum, const int k00); +typedef void (*ggml_cuda_mmq_write_back_t)(const float * __restrict__ sum, const int32_t * __restrict__ get_rows_to_sorted, + float * __restrict__ dst, const float * __restrict__ y_scale, const int stride, const int i_max, const int j_max); + +enum mmq_q8_1_ds_layout { + MMQ_Q8_1_DS_LAYOUT_D4, + MMQ_Q8_1_DS_LAYOUT_DS4, + MMQ_Q8_1_DS_LAYOUT_D2S6, +}; + +static constexpr int QK8_1_MMQ = 4*QK8_1; +static constexpr int QK_FP4_MMQ = 2*QK8_1_MMQ; + +struct block_q8_1_mmq { + // The y float data is converted to a data layout that can simply be copied to shared memory as a contiguous block. + // The y float data is first grouped as blocks of 128 values. + // These blocks are then treated as individual data values and transposed. + // + // To avoid shared memory bank conflicts each block is padded with 16 bytes. + // This padding is also used to store block scales/partial sums. + // The scales multiplied with the quantized data are equal to the unquantized values. + // The partial sums are obtained by summing up a subgroup of the contained values (prior to quantization) + // and are only needed for performance reasons. + // + // The exact data stored depends on the x data type. + union { + float d4[4]; // 1 32 bit scale per 32 values, stored as d0,d1,d2,d3 + half2 ds4[4]; // 1 16 bit scale + 1 16 bit partial sum per 32 values, stored as d0,s0,d1,s1,d2,s2,d3,s3 + half d2s6[8]; // 1 16 bit scale per 64 values + 1 16 bit partial sum per 16 values for the first 96 values, + // stored as d0,d1,s1,s2,s3,s4,s5 + }; + int8_t qs[QK8_1_MMQ]; +}; + +// this struct is used for fp4 data types (currently only used for Blackwell) +// mxfp4 has block size 32, each int32 of d4 contains 2 e8m0 scales in the lower 16 bits +// nvfp4 has block size 16, each int32 of d4 contains 4 ue4m3 scales +struct block_fp4_mmq { + uint32_t d4[4]; + int8_t qs[QK_FP4_MMQ / 2]; +}; + +static_assert(sizeof(block_q8_1_mmq) == QK8_1_MMQ + 4*sizeof(half2), "Unexpected block_q8_1_mmq size"); +static_assert(sizeof(block_q8_1_mmq) == 4*sizeof(block_q8_1), "Unexpected block_q8_1_mmq size"); +static_assert(sizeof(block_fp4_mmq) == sizeof(block_q8_1_mmq), "Unexpected block_fp4_mmq size"); + +static mmq_q8_1_ds_layout mmq_get_q8_1_ds_layout(const ggml_type type_x) { + switch (type_x) { + case GGML_TYPE_Q1_0: + case GGML_TYPE_Q2_0: + return MMQ_Q8_1_DS_LAYOUT_D4; + case GGML_TYPE_Q4_0: + case GGML_TYPE_Q4_1: + return MMQ_Q8_1_DS_LAYOUT_DS4; + case GGML_TYPE_Q5_0: + return MMQ_Q8_1_DS_LAYOUT_D4; + case GGML_TYPE_Q5_1: + return MMQ_Q8_1_DS_LAYOUT_DS4; + case GGML_TYPE_Q8_0: + return MMQ_Q8_1_DS_LAYOUT_D4; + case GGML_TYPE_MXFP4: + return MMQ_Q8_1_DS_LAYOUT_D4; + case GGML_TYPE_NVFP4: + return MMQ_Q8_1_DS_LAYOUT_D4; + case GGML_TYPE_Q2_K: + return MMQ_Q8_1_DS_LAYOUT_D2S6; + case GGML_TYPE_Q3_K: + return MMQ_Q8_1_DS_LAYOUT_D4; + case GGML_TYPE_Q4_K: + case GGML_TYPE_Q5_K: + return MMQ_Q8_1_DS_LAYOUT_DS4; + case GGML_TYPE_Q6_K: + case GGML_TYPE_IQ2_XXS: + case GGML_TYPE_IQ2_XS: + case GGML_TYPE_IQ2_S: + case GGML_TYPE_IQ3_XXS: + case GGML_TYPE_IQ3_S: + return MMQ_Q8_1_DS_LAYOUT_D4; + case GGML_TYPE_IQ1_S: + return MMQ_Q8_1_DS_LAYOUT_DS4; + case GGML_TYPE_IQ4_XS: + case GGML_TYPE_IQ4_NL: + return MMQ_Q8_1_DS_LAYOUT_D4; + default: + GGML_ABORT("fatal error"); + break; + } +} + +struct tile_x_sizes { + int qs; + int dm; + int sc; +}; + +// Decouple shared memory tile sizes from WARP_SIZE to allow for different warp sizes. +// The K dimension of the tiles has either, +// 1*MMQ_TILE_NE_K==32 (always for TILE_Y_K) or 2*MMQ_TILE_NE_K==64 (typically for TILE_X_K), +// 32 bit elements for the quantized data (does not include scales). +// In other words, the size of the quantized data in the K dimension is a multiple of MMQ_TILE_NE_K. +// The final tile size in K direction is padded to avoid shared memory bank conflicts, +// in terms of 32 bit elements that means K % 2 == 1 for dp4a or K % 8 == 4 for mma. +#define MMQ_TILE_NE_K 32 + +// block_q8_1_mmq has (128 8-bit ints == 32 32-bit ints + 4 32-bit scales) +#define MMQ_TILE_Y_K (MMQ_TILE_NE_K + MMQ_TILE_NE_K / QI8_1) +#define MMQ_TILE_Y_FP4_K MMQ_TILE_Y_K + +enum ggml_cuda_mmq_sram_layout { + GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, + GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, + GGML_CUDA_MMQ_SRAM_LAYOUT_Q2_K, + GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, + GGML_CUDA_MMQ_SRAM_LAYOUT_Q6_K, + GGML_CUDA_MMQ_SRAM_LAYOUT_FP4, // MXFP4 and NVFP4 on Blackwell. + GGML_CUDA_MMQ_SRAM_LAYOUT_NVFP4, // Generic NVFP4 +}; + +static constexpr __host__ __device__ int ggml_cuda_mmq_get_sram_stride(ggml_cuda_mmq_sram_layout sram_layout) { + switch (sram_layout) { + case GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0: + return 2*MMQ_TILE_NE_K + 2*MMQ_TILE_NE_K/QI8_0 + 4; + case GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1: + return 2*MMQ_TILE_NE_K + 2*MMQ_TILE_NE_K/QI8_1 + 4; + case GGML_CUDA_MMQ_SRAM_LAYOUT_Q2_K: + return 2*MMQ_TILE_NE_K + MMQ_TILE_NE_K + 4; + case GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K: + return 2*MMQ_TILE_NE_K + MMQ_TILE_NE_K/2 + 4; + case GGML_CUDA_MMQ_SRAM_LAYOUT_Q6_K: + return 2*MMQ_TILE_NE_K + MMQ_TILE_NE_K/QI6_K + MMQ_TILE_NE_K/8 + 7; + case GGML_CUDA_MMQ_SRAM_LAYOUT_FP4: + return 2*MMQ_TILE_NE_K + 8 + 4; + case GGML_CUDA_MMQ_SRAM_LAYOUT_NVFP4: + return 2*MMQ_TILE_NE_K + MMQ_TILE_NE_K/2 + 4; + default: + return -1; + } +} + +static_assert(ggml_cuda_mmq_get_sram_stride(GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0) % 8 == 4, "Wrong padding."); +static_assert(ggml_cuda_mmq_get_sram_stride(GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1) % 8 == 4, "Wrong padding."); +static_assert(ggml_cuda_mmq_get_sram_stride(GGML_CUDA_MMQ_SRAM_LAYOUT_Q2_K) % 8 == 4, "Wrong padding."); +static_assert(ggml_cuda_mmq_get_sram_stride(GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K) % 8 == 4, "Wrong padding."); +static_assert(ggml_cuda_mmq_get_sram_stride(GGML_CUDA_MMQ_SRAM_LAYOUT_Q6_K) % 8 == 4, "Wrong padding."); +static_assert(ggml_cuda_mmq_get_sram_stride(GGML_CUDA_MMQ_SRAM_LAYOUT_FP4) % 8 == 4, "Wrong padding."); +static_assert(ggml_cuda_mmq_get_sram_stride(GGML_CUDA_MMQ_SRAM_LAYOUT_NVFP4) % 8 == 4, "Wrong padding."); + +static_assert(ggml_cuda_mmq_get_sram_stride(GGML_CUDA_MMQ_SRAM_LAYOUT_FP4) == ggml_cuda_mmq_get_sram_stride(GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1), "Wrong tile size for MXFP4"); + +// Config options for the MMQ kernel. +// Should not affect results, only speed/register pressure/shared memory use. +struct ggml_cuda_mmq_config { + ggml_type type; // src0->type + int nthreads; // Number of threads per CUDA block. + int occupancy; // Targeted occupancy for the MMA kernel. + int I; // SRAM tile width in src0->ne[1]/dst->ne[0] direction. + int J; // SRAM tile width in src1->ne[1]/dst->ne[1] direction. + ggml_cuda_mmq_sram_layout sram_layout; // SRAM tile length in src0->ne[0]/src1->ne[0] direction (physical 32 bit elements). + int K_vram; // VRAM tile length in src0->ne[0]/src1->ne[0] direction (logical elements). + bool stream_k; // Whether or not to use stream-k decomposition. + bool fallback; // Whether a fallback for out-of-bounds check in src0->ne[1] direction is needed. + + constexpr __host__ __device__ ggml_cuda_mmq_config( + ggml_type type, int nthreads, int occupancy, int I, int J, ggml_cuda_mmq_sram_layout sram_layout, int K_vram, bool stream_k, bool fallback) : + type(type), nthreads(nthreads), occupancy(occupancy), I(I), J(J), sram_layout(sram_layout), K_vram(K_vram), stream_k(stream_k), fallback(fallback) {} + + constexpr __device__ int rows_per_warp() const { +#if defined(AMD_MFMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE) + return 16; +#else + return J >= 48 && J % 16 == 0 ? 32 : 16; +#endif // defined(AMD_MFMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE) + } + + // TODO transition all combinations of GPUs and quantizations to the MMA data layout. + __host__ int use_mma_data_layout(const int cc) const { + if (amd_mfma_available(cc) || amd_wmma_available(cc) || turing_mma_available(cc)) { + return true; + } + return false; + } + + constexpr __device__ bool use_mma_data_layout() const { +#if defined(AMD_MFMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) + return true; +#else + return false; +#endif // defined(AMD_MFMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) + } + +}; + +#define CASE(type_, nthreads_, occupancy_, I_, J_, sram_layout_, K_vram_, stream_k_, fallback_) \ + if (type == (type_) && J == (J_) && fallback == (fallback_)) { \ + static_assert((nthreads_) % 32 == 0 && (nthreads_) <= 512, "bad nthreads"); \ + static_assert( (occupancy_) <= 8, "bad occupancy"); \ + static_assert((I_) % 32 == 0, "bad I"); \ + static_assert((J_) % 8 == 0, "bad J"); \ + static_assert((K_vram_) % 256 == 0, "bad K_vram"); \ + return ggml_cuda_mmq_config((type_), (nthreads_), (occupancy_), (I_), (J_), (sram_layout_), (K_vram_), (stream_k_), (fallback_)); \ + } \ + +#include "mmq-config-pascal.cuh" +#include "mmq-config-ampere.cuh" +#include "mmq-config-blackwell.cuh" + +#include "mmq-config-cdna.cuh" +#include "mmq-config-rdna2.cuh" +#include "mmq-config-rdna3.cuh" +#include "mmq-config-rdna3-5.cuh" +#include "mmq-config-rdna4.cuh" + +#undef CASE + +static __host__ ggml_cuda_mmq_config ggml_cuda_mmq_get_config(const ggml_type type, const int J, const bool fallback, const int cc) { + if (GGML_CUDA_CC_IS_AMD(cc)) { + if (GGML_CUDA_CC_IS_CDNA(cc)) { + return ggml_cuda_mmq_get_config_cdna(type, J, fallback); + } + if (GGML_CUDA_CC_IS_RDNA4(cc)) { + return ggml_cuda_mmq_get_config_rdna4(type, J, fallback); + } + if (GGML_CUDA_CC_IS_RDNA3_5(cc)) { + return ggml_cuda_mmq_get_config_rdna3_5(type, J, fallback); + } + if (GGML_CUDA_CC_IS_RDNA3(cc)) { // covers RDNA 3.0 + return ggml_cuda_mmq_get_config_rdna3(type, J, fallback); + } + return ggml_cuda_mmq_get_config_rdna2(type, J, fallback); + } + if (blackwell_mma_available(cc)) { + return ggml_cuda_mmq_get_config_blackwell(type, J, fallback); + } + if (ggml_cuda_highest_compiled_arch(cc) >= GGML_CUDA_CC_VOLTA) { + return ggml_cuda_mmq_get_config_ampere(type, J, fallback); + } + return ggml_cuda_mmq_get_config_pascal(type, J, fallback); +} + +static constexpr __device__ ggml_cuda_mmq_config ggml_cuda_mmq_get_config(ggml_type type, int J, bool fallback) { +#ifdef GGML_USE_HIP +#ifdef CDNA + return ggml_cuda_mmq_get_config_cdna(type, J, fallback); +#elif defined(RDNA4) + return ggml_cuda_mmq_get_config_rdna4(type, J, fallback); +#elif defined(RDNA3_5) + return ggml_cuda_mmq_get_config_rdna3_5(type, J, fallback); +#elif defined(RDNA3) + return ggml_cuda_mmq_get_config_rdna3(type, J, fallback); +#else + return ggml_cuda_mmq_get_config_rdna2(type, J, fallback); +#endif // CDNA +#else +#ifdef BLACKWELL_MMA_AVAILABLE + return ggml_cuda_mmq_get_config_blackwell(type, J, fallback); +#elif __CUDA_ARCH__ >= GGML_CUDA_CC_VOLTA + return ggml_cuda_mmq_get_config_ampere(type, J, fallback); +#else + return ggml_cuda_mmq_get_config_pascal(type, J, fallback); +#endif // BLACKWELL_MMA_AVAILABLE +#endif // GGML_USE_HIP + GGML_UNUSED_VARS(type, J, fallback); +} + +static __host__ int ggml_cuda_mmq_get_type(const ggml_type type, const int J, const bool fallback, const int cc) { + return ggml_cuda_mmq_get_config(type, J, fallback, cc).type; +} + +static constexpr __device__ int ggml_cuda_mmq_get_type(ggml_type type, int J, bool fallback) { + return ggml_cuda_mmq_get_config(type, J, fallback).type; +} + +static __host__ int ggml_cuda_mmq_get_nthreads(const ggml_type type, const int J, const bool fallback, const int cc) { + return ggml_cuda_mmq_get_config(type, J, fallback, cc).nthreads; +} + +static constexpr __device__ int ggml_cuda_mmq_get_nthreads(ggml_type type, int J, bool fallback) { + return ggml_cuda_mmq_get_config(type, J, fallback).nthreads; +} + +static __host__ int ggml_cuda_mmq_get_occupancy(const ggml_type type, const int J, const bool fallback, const int cc) { + return ggml_cuda_mmq_get_config(type, J, fallback, cc).occupancy; +} + +static constexpr __device__ int ggml_cuda_mmq_get_occupancy(ggml_type type, int J, bool fallback) { + return ggml_cuda_mmq_get_config(type, J, fallback).occupancy; +} + +static __host__ int ggml_cuda_mmq_get_I(const ggml_type type, const int J, const bool fallback, const int cc) { + return ggml_cuda_mmq_get_config(type, J, fallback, cc).I; +} + +static constexpr __device__ int ggml_cuda_mmq_get_I(ggml_type type, int J, bool fallback) { + return ggml_cuda_mmq_get_config(type, J, fallback).I; +} + +static __host__ int ggml_cuda_mmq_get_J(const ggml_type type, const int J, const bool fallback, const int cc) { + return ggml_cuda_mmq_get_config(type, J, fallback, cc).J; +} + +static constexpr __device__ int ggml_cuda_mmq_get_J(ggml_type type, int J, bool fallback) { + return ggml_cuda_mmq_get_config(type, J, fallback).J; +} + +static __host__ ggml_cuda_mmq_sram_layout ggml_cuda_mmq_get_sram_layout(const ggml_type type, const int J, const bool fallback, const int cc) { + return ggml_cuda_mmq_get_config(type, J, fallback, cc).sram_layout; +} + +static constexpr __device__ ggml_cuda_mmq_sram_layout ggml_cuda_mmq_get_sram_layout(ggml_type type, int J, bool fallback) { + return ggml_cuda_mmq_get_config(type, J, fallback).sram_layout; +} + +static __host__ int ggml_cuda_mmq_get_K_vram(const ggml_type type, const int J, const bool fallback, const int cc) { + return ggml_cuda_mmq_get_config(type, J, fallback, cc).K_vram; +} + +static constexpr __device__ int ggml_cuda_mmq_get_K_vram(ggml_type type, int J, bool fallback) { + return ggml_cuda_mmq_get_config(type, J, fallback).K_vram; +} + +static __host__ bool ggml_cuda_mmq_get_stream_k(const ggml_type type, const int J, const bool fallback, const int cc) { + return ggml_cuda_mmq_get_config(type, J, fallback, cc).stream_k; +} + +static constexpr __device__ bool ggml_cuda_mmq_get_stream_k(ggml_type type, int J, bool fallback) { + return ggml_cuda_mmq_get_config(type, J, fallback).stream_k; +} + +static __host__ int ggml_cuda_mmq_get_fallback(const ggml_type type, const int J, const bool fallback, const int cc) { + return ggml_cuda_mmq_get_config(type, J, fallback, cc).fallback; +} + +static constexpr __device__ int ggml_cuda_mmq_get_fallback(ggml_type type, int J, bool fallback) { + return ggml_cuda_mmq_get_config(type, J, fallback).fallback; +} + +// --------------------------------------------------------------------------------------------- + +static __host__ int ggml_cuda_mmq_get_sram_stride(const ggml_type type, const int J, const bool fallback, const int cc) { + return ggml_cuda_mmq_get_sram_stride(ggml_cuda_mmq_get_sram_layout(type, J, fallback, cc)); +} + +static constexpr __device__ int ggml_cuda_mmq_get_sram_stride(ggml_type type, int J, bool fallback) { + return ggml_cuda_mmq_get_sram_stride(ggml_cuda_mmq_get_sram_layout(type, J, fallback)); +} + +static __host__ int ggml_cuda_mmq_get_J_max(const ggml_type type, const bool fallback, const int cc, const int64_t ne11) { + int ret = std::min(ne11, int64_t(512)); + ret -= ret % 8; + for (;ret > 0; ret -= 8) { + if (ggml_cuda_mmq_get_config(type, ret, fallback, cc).type != GGML_TYPE_COUNT) { + return ret; + } + } + return ret; +} + +static constexpr __device__ int ggml_cuda_mmq_get_rows_per_warp(ggml_type type, int J, bool fallback) { + return ggml_cuda_mmq_get_config(type, J, fallback).rows_per_warp(); +} + +#define MMQ_DP4A_TXS_Q4_0 tile_x_sizes{I*MMQ_TILE_NE_K + I, I*MMQ_TILE_NE_K/QI4_0 + I/QI4_0, 0} +#define MMQ_DP4A_TXS_Q4_1 tile_x_sizes{I*MMQ_TILE_NE_K + I, I*MMQ_TILE_NE_K/QI4_1 + I/QI4_1, 0} +#define MMQ_DP4A_TXS_Q8_0 tile_x_sizes{I*MMQ_TILE_NE_K*2 + I, I*MMQ_TILE_NE_K*2/QI8_0 + I/(QI8_0/2), 0} +#define MMQ_DP4A_TXS_Q8_0_16 tile_x_sizes{I*MMQ_TILE_NE_K*2 + I, I*MMQ_TILE_NE_K*4/QI8_0 + I/(QI8_0/4), 0} +#define MMQ_DP4A_TXS_Q8_1 tile_x_sizes{I*MMQ_TILE_NE_K*2 + I, I*MMQ_TILE_NE_K*2/QI8_1 + I/(QI8_1/2), 0} +#define MMQ_DP4A_TXS_Q2_K tile_x_sizes{I*MMQ_TILE_NE_K*2 + I, I*MMQ_TILE_NE_K + I, 0} +#define MMQ_DP4A_TXS_Q3_K tile_x_sizes{I*MMQ_TILE_NE_K*2 + I, I, I*MMQ_TILE_NE_K/8 + I/8} +#define MMQ_DP4A_TXS_Q4_K tile_x_sizes{I*MMQ_TILE_NE_K + I, I*MMQ_TILE_NE_K/QI4_K, I*MMQ_TILE_NE_K/8 + I/8} +#define MMQ_DP4A_TXS_Q5_K tile_x_sizes{I*MMQ_TILE_NE_K*2 + I, I*MMQ_TILE_NE_K/QI5_K + I/QI5_K, I*MMQ_TILE_NE_K/8 + I/8} +#define MMQ_DP4A_TXS_Q6_K tile_x_sizes{I*MMQ_TILE_NE_K*2 + I, I*MMQ_TILE_NE_K/QI6_K + I/QI6_K, I*MMQ_TILE_NE_K/8 + I/8} + +static constexpr __host__ __device__ tile_x_sizes mmq_get_dp4a_tile_x_sizes(ggml_type type, int I) { + switch (type) { + case GGML_TYPE_Q1_0: return MMQ_DP4A_TXS_Q8_0; + case GGML_TYPE_Q2_0: return MMQ_DP4A_TXS_Q8_0; + case GGML_TYPE_Q4_0: return MMQ_DP4A_TXS_Q4_0; + case GGML_TYPE_Q4_1: return MMQ_DP4A_TXS_Q4_1; + case GGML_TYPE_Q5_0: return MMQ_DP4A_TXS_Q8_0; + case GGML_TYPE_Q5_1: return MMQ_DP4A_TXS_Q8_1; + case GGML_TYPE_Q8_0: return MMQ_DP4A_TXS_Q8_0; + case GGML_TYPE_MXFP4: return MMQ_DP4A_TXS_Q8_1; + case GGML_TYPE_NVFP4: return MMQ_DP4A_TXS_Q8_0_16; + case GGML_TYPE_Q2_K: return MMQ_DP4A_TXS_Q2_K; + case GGML_TYPE_Q3_K: return MMQ_DP4A_TXS_Q3_K; + case GGML_TYPE_Q4_K: return MMQ_DP4A_TXS_Q4_K; + case GGML_TYPE_Q5_K: return MMQ_DP4A_TXS_Q5_K; + case GGML_TYPE_Q6_K: return MMQ_DP4A_TXS_Q6_K; + case GGML_TYPE_IQ2_XXS: return MMQ_DP4A_TXS_Q8_0; + case GGML_TYPE_IQ2_XS: return MMQ_DP4A_TXS_Q8_0_16; + case GGML_TYPE_IQ2_S: return MMQ_DP4A_TXS_Q8_0_16; + case GGML_TYPE_IQ3_XXS: return MMQ_DP4A_TXS_Q8_0; + case GGML_TYPE_IQ3_S: return MMQ_DP4A_TXS_Q8_0; + case GGML_TYPE_IQ1_S: return MMQ_DP4A_TXS_Q8_0; + case GGML_TYPE_IQ4_XS: return MMQ_DP4A_TXS_Q8_0; + case GGML_TYPE_IQ4_NL: return MMQ_DP4A_TXS_Q8_0; + default: return tile_x_sizes{0, 0, 0}; + } +} + +// FIXME temporary until all combinations of data types and GPUs can use the MMA data layout +static __host__ int ggml_cuda_mmq_get_nbytes_shared_x(const ggml_cuda_mmq_config & config, const int cc) { + if (config.use_mma_data_layout(cc)) { + return config.I * ggml_cuda_mmq_get_sram_stride(config.sram_layout) * 4; + } + const tile_x_sizes txs = mmq_get_dp4a_tile_x_sizes(config.type, config.I); + return (txs.qs + txs.dm + txs.sc) * 4; +} + +// ------------------------------------------------------------ + +#include "mmq-load-tiles.cuh" +#include "mmq-vec-dot.cuh" + +template static __device__ __forceinline__ void ggml_cuda_mmq_write_back_dp4a( + const float * __restrict__ sum, const int32_t * __restrict__ ids_dst, float * __restrict__ dst, + const float * __restrict__ y_scale, const int stride, const int i_max, const int j_max) { + constexpr int warp_size = ggml_cuda_get_physical_warp_size(); + constexpr int nwarps = ggml_cuda_mmq_get_nthreads(type, J, fallback) / warp_size; + constexpr int I = ggml_cuda_mmq_get_I(type, J, fallback); + + const bool y_scale_used = y_scale != nullptr; + +#pragma unroll + for (int j0 = 0; j0 < J; j0 += nwarps) { + const int j = j0 + threadIdx.y; + + if (j > j_max) { + return; + } + +#pragma unroll + for (int i0 = 0; i0 < I; i0 += warp_size) { + const int i = i0 + threadIdx.x; + + if (fallback && i > i_max) { + continue; + } + + if constexpr (type == GGML_TYPE_NVFP4) { + if (y_scale_used) { + dst[ids_dst[j]*stride + i] = y_scale[j] * sum[(j0/nwarps) * (I/warp_size) + i0/warp_size]; + } else { + dst[ids_dst[j]*stride + i] = sum[(j0/nwarps) * (I/warp_size) + i0/warp_size]; + } + } else { + dst[ids_dst[j]*stride + i] = sum[(j0/nwarps) * (I/warp_size) + i0/warp_size]; + GGML_UNUSED(y_scale_used); + } + } + } +} + +template +static __device__ __forceinline__ void ggml_cuda_mmq_write_back_mma( + const float * __restrict__ sum, const int * __restrict__ ids_dst, float * __restrict__ dst, + const float * __restrict__ y_scale, const int stride, const int i_max, const int j_max) { + +#if defined(AMD_MFMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE) + typedef tile<16, 16, int, DATA_LAYOUT_J_MAJOR> tile_C; +#else + typedef tile<16, 8, int> tile_C; +#endif // defined(AMD_MFMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE) + + constexpr int warp_size = ggml_cuda_get_physical_warp_size(); + constexpr int nwarps = ggml_cuda_mmq_get_nthreads(type, J, fallback) / warp_size; + constexpr int I = ggml_cuda_mmq_get_I(type, J, fallback); + constexpr int rows_per_warp = ggml_cuda_mmq_get_rows_per_warp(type, J, fallback); + constexpr int ntx = rows_per_warp/tile_C::I; // Number of x minitiles per warp. + + const int i0 = (threadIdx.y / ntx) * (ntx*tile_C::I); + + const bool y_scale_used = y_scale != nullptr; + +#pragma unroll + for (int j0 = 0; j0 < J; j0 += ntx*tile_C::J) { +#pragma unroll + for (int n = 0; n < ntx; ++n) { +#pragma unroll + for (int l = 0; l < tile_C::ne; ++l) { + const int j = j0 + (threadIdx.y % ntx) * tile_C::J + tile_C::get_j(l); + + if (j > j_max) { + continue; + } + + const int i = i0 + n*tile_C::I + tile_C::get_i(l); + + if (fallback && i > i_max) { + continue; + } + + if constexpr (type == GGML_TYPE_NVFP4) { + if (y_scale_used) { + dst[ids_dst[j]*stride + i] = y_scale[j] * sum[(j0/tile_C::J + n)*tile_C::ne + l]; + } else { + dst[ids_dst[j]*stride + i] = sum[(j0/tile_C::J + n)*tile_C::ne + l]; + } + } else { + dst[ids_dst[j]*stride + i] = sum[(j0/tile_C::J + n)*tile_C::ne + l]; + GGML_UNUSED(y_scale_used); + } + } + } + } +} + +// ------------------------------------------------------------------------------------------------------------------------------------- + +// TODO remove this struct and use ggml_cuda_mmq_sram_layout instead. +struct ggml_cuda_mmq_util_funcs { + int vdr; + ggml_cuda_mmq_load_tiles_t load_tiles; + ggml_cuda_mmq_vec_dot_t vec_dot; + ggml_cuda_mmq_write_back_t write_back; + + constexpr __host__ __device__ ggml_cuda_mmq_util_funcs( + int vdr, ggml_cuda_mmq_load_tiles_t load_tiles, ggml_cuda_mmq_vec_dot_t vec_dot, ggml_cuda_mmq_write_back_t write_back) : + vdr(vdr), load_tiles(load_tiles), vec_dot(vec_dot), write_back(write_back) {} +}; + +template +static constexpr __device__ ggml_cuda_mmq_util_funcs ggml_cuda_mmq_get_util_funcs() { + constexpr int I = ggml_cuda_mmq_get_I(type, J, fallback); + + if (!ggml_cuda_mmq_get_config(type, J, fallback).use_mma_data_layout()) { + switch (type) { + case GGML_TYPE_Q1_0: + return ggml_cuda_mmq_util_funcs( + VDR_Q1_0_Q8_1_MMQ, + ggml_cuda_mmq_load_tiles_q1_0, + ggml_cuda_mmq_vec_dot_q8_0_q8_1_dp4a, + ggml_cuda_mmq_write_back_dp4a); + case GGML_TYPE_Q2_0: + return ggml_cuda_mmq_util_funcs( + VDR_Q2_0_Q8_1_MMQ, + ggml_cuda_mmq_load_tiles_q2_0, + ggml_cuda_mmq_vec_dot_q8_0_q8_1_dp4a, + ggml_cuda_mmq_write_back_dp4a); + case GGML_TYPE_Q4_0: + return ggml_cuda_mmq_util_funcs( + VDR_Q4_0_Q8_1_MMQ, + ggml_cuda_mmq_load_tiles_q4_0, + ggml_cuda_mmq_vec_dot_q4_0_q8_1_dp4a, + ggml_cuda_mmq_write_back_dp4a); + case GGML_TYPE_Q4_1: + return ggml_cuda_mmq_util_funcs( + VDR_Q4_1_Q8_1_MMQ, + ggml_cuda_mmq_load_tiles_q4_1, + ggml_cuda_mmq_vec_dot_q4_1_q8_1_dp4a, + ggml_cuda_mmq_write_back_dp4a); + case GGML_TYPE_Q5_0: + return ggml_cuda_mmq_util_funcs( + VDR_Q5_0_Q8_1_MMQ, + ggml_cuda_mmq_load_tiles_q5_0, + ggml_cuda_mmq_vec_dot_q8_0_q8_1_dp4a, + ggml_cuda_mmq_write_back_dp4a); + case GGML_TYPE_Q5_1: + return ggml_cuda_mmq_util_funcs( + VDR_Q5_1_Q8_1_MMQ, + ggml_cuda_mmq_load_tiles_q5_1, + ggml_cuda_mmq_vec_dot_q8_1_q8_1_dp4a, + ggml_cuda_mmq_write_back_dp4a); + case GGML_TYPE_Q8_0: + return ggml_cuda_mmq_util_funcs( + VDR_Q8_0_Q8_1_MMQ, + ggml_cuda_mmq_load_tiles_q8_0, + ggml_cuda_mmq_vec_dot_q8_0_q8_1_dp4a, + ggml_cuda_mmq_write_back_dp4a); +// --------------------------------------------------------------------------------------------- + case GGML_TYPE_Q2_K: + return ggml_cuda_mmq_util_funcs( + VDR_Q2_K_Q8_1_MMQ, + ggml_cuda_mmq_load_tiles_q2_K, + ggml_cuda_mmq_vec_dot_q2_K_q8_1_dp4a, + ggml_cuda_mmq_write_back_dp4a); + case GGML_TYPE_Q3_K: + return ggml_cuda_mmq_util_funcs( + VDR_Q3_K_Q8_1_MMQ, + ggml_cuda_mmq_load_tiles_q3_K, + ggml_cuda_mmq_vec_dot_q3_K_q8_1_dp4a, + ggml_cuda_mmq_write_back_dp4a); + case GGML_TYPE_Q4_K: + return ggml_cuda_mmq_util_funcs( + VDR_Q4_K_Q8_1_MMQ, + ggml_cuda_mmq_load_tiles_q4_K, + ggml_cuda_mmq_vec_dot_q4_K_q8_1_dp4a, + ggml_cuda_mmq_write_back_dp4a); + case GGML_TYPE_Q5_K: + return ggml_cuda_mmq_util_funcs( + VDR_Q5_K_Q8_1_MMQ, + ggml_cuda_mmq_load_tiles_q5_K, + ggml_cuda_mmq_vec_dot_q5_K_q8_1_dp4a, + ggml_cuda_mmq_write_back_dp4a); + case GGML_TYPE_Q6_K: + return ggml_cuda_mmq_util_funcs( + VDR_Q6_K_Q8_1_MMQ, + ggml_cuda_mmq_load_tiles_q6_K, + ggml_cuda_mmq_vec_dot_q6_K_q8_1_dp4a, + ggml_cuda_mmq_write_back_dp4a); +// --------------------------------------------------------------------------------------------- + case GGML_TYPE_IQ1_S: + return ggml_cuda_mmq_util_funcs( + VDR_IQ1_S_Q8_1_MMQ, + ggml_cuda_mmq_load_tiles_iq1_s, + ggml_cuda_mmq_vec_dot_q8_1_q8_1_dp4a, + ggml_cuda_mmq_write_back_dp4a); + case GGML_TYPE_IQ2_XXS: + return ggml_cuda_mmq_util_funcs( + VDR_IQ2_XXS_Q8_1_MMQ, + ggml_cuda_mmq_load_tiles_iq2_xxs, + ggml_cuda_mmq_vec_dot_q8_0_q8_1_dp4a, + ggml_cuda_mmq_write_back_dp4a); + case GGML_TYPE_IQ2_XS: + return ggml_cuda_mmq_util_funcs( + VDR_IQ2_XS_Q8_1_MMQ, + ggml_cuda_mmq_load_tiles_iq2_xs, + ggml_cuda_mmq_vec_dot_q8_0_16_q8_1_dp4a, + ggml_cuda_mmq_write_back_dp4a); + case GGML_TYPE_IQ2_S: + return ggml_cuda_mmq_util_funcs( + VDR_IQ2_S_Q8_1_MMQ, + ggml_cuda_mmq_load_tiles_iq2_s, + ggml_cuda_mmq_vec_dot_q8_0_16_q8_1_dp4a, + ggml_cuda_mmq_write_back_dp4a); + case GGML_TYPE_IQ3_XXS: + return ggml_cuda_mmq_util_funcs( + VDR_IQ3_XXS_Q8_1_MMQ, + ggml_cuda_mmq_load_tiles_iq3_xxs, + ggml_cuda_mmq_vec_dot_q8_0_q8_1_dp4a, + ggml_cuda_mmq_write_back_dp4a); + case GGML_TYPE_IQ3_S: + return ggml_cuda_mmq_util_funcs( + VDR_IQ3_S_Q8_1_MMQ, + ggml_cuda_mmq_load_tiles_iq3_s, + ggml_cuda_mmq_vec_dot_q8_0_q8_1_dp4a, + ggml_cuda_mmq_write_back_dp4a); + case GGML_TYPE_IQ4_XS: + return ggml_cuda_mmq_util_funcs( + VDR_IQ4_XS_Q8_1_MMQ, + ggml_cuda_mmq_load_tiles_iq4_xs, + ggml_cuda_mmq_vec_dot_q8_0_q8_1_dp4a, + ggml_cuda_mmq_write_back_dp4a); + case GGML_TYPE_IQ4_NL: + return ggml_cuda_mmq_util_funcs( + VDR_IQ4_NL_Q8_1_MMQ, + ggml_cuda_mmq_load_tiles_iq4_nl, + ggml_cuda_mmq_vec_dot_q8_0_q8_1_dp4a, + ggml_cuda_mmq_write_back_dp4a); +// --------------------------------------------------------------------------------------------- + case GGML_TYPE_MXFP4: + return ggml_cuda_mmq_util_funcs( + VDR_MXFP4_Q8_1_MMQ, + ggml_cuda_mmq_load_tiles_mxfp4, + ggml_cuda_mmq_vec_dot_q8_0_q8_1_dp4a, + ggml_cuda_mmq_write_back_dp4a); + case GGML_TYPE_NVFP4: + return ggml_cuda_mmq_util_funcs( + VDR_NVFP4_Q8_1_MMQ, + ggml_cuda_mmq_load_tiles_nvfp4, + ggml_cuda_mmq_vec_dot_q8_0_16_q8_1_dp4a, + ggml_cuda_mmq_write_back_dp4a); + default: + return ggml_cuda_mmq_util_funcs(1, nullptr, nullptr, nullptr); + } + } + +// --------------------------------------------------------------------------------------------- + +#ifdef BLACKWELL_MMA_AVAILABLE + switch (type) { + case GGML_TYPE_MXFP4: + return ggml_cuda_mmq_util_funcs( + -1, + ggml_cuda_mmq_load_tiles_mxfp4_fp4, + ggml_cuda_mmq_vec_dot_fp4_fp4_mma, + ggml_cuda_mmq_write_back_mma); + case GGML_TYPE_NVFP4: + return ggml_cuda_mmq_util_funcs( + -1, + ggml_cuda_mmq_load_tiles_nvfp4_nvfp4, + ggml_cuda_mmq_vec_dot_fp4_fp4_mma, + ggml_cuda_mmq_write_back_mma); + default: + break; + } +#endif // BLACKWELL_MMA_AVAILABLE + +// --------------------------------------------------------------------------------------------- + + switch (type) { + case GGML_TYPE_Q1_0: + return ggml_cuda_mmq_util_funcs( + -1, + ggml_cuda_mmq_load_tiles_q1_0, + ggml_cuda_mmq_vec_dot_q8_0_q8_1_mma, + ggml_cuda_mmq_write_back_mma); + case GGML_TYPE_Q2_0: + return ggml_cuda_mmq_util_funcs( + -1, + ggml_cuda_mmq_load_tiles_q2_0, + ggml_cuda_mmq_vec_dot_q8_0_q8_1_mma, + ggml_cuda_mmq_write_back_mma); + case GGML_TYPE_Q4_0: + return ggml_cuda_mmq_util_funcs( + -1, + ggml_cuda_mmq_load_tiles_q4_0, + ggml_cuda_mmq_vec_dot_q8_0_q8_1_mma, + ggml_cuda_mmq_write_back_mma); + case GGML_TYPE_Q4_1: + return ggml_cuda_mmq_util_funcs( + -1, + ggml_cuda_mmq_load_tiles_q4_1, + ggml_cuda_mmq_vec_dot_q8_1_q8_1_mma, + ggml_cuda_mmq_write_back_mma); + case GGML_TYPE_Q5_0: + return ggml_cuda_mmq_util_funcs( + -1, + ggml_cuda_mmq_load_tiles_q5_0, + ggml_cuda_mmq_vec_dot_q8_0_q8_1_mma, + ggml_cuda_mmq_write_back_mma); + case GGML_TYPE_Q5_1: + return ggml_cuda_mmq_util_funcs( + -1, + ggml_cuda_mmq_load_tiles_q5_1, + ggml_cuda_mmq_vec_dot_q8_1_q8_1_mma, + ggml_cuda_mmq_write_back_mma); + case GGML_TYPE_Q8_0: + return ggml_cuda_mmq_util_funcs( + -1, + ggml_cuda_mmq_load_tiles_q8_0, + ggml_cuda_mmq_vec_dot_q8_0_q8_1_mma, + ggml_cuda_mmq_write_back_mma); +// --------------------------------------------------------------------------------------------- + case GGML_TYPE_Q2_K: + return ggml_cuda_mmq_util_funcs( + -1, + ggml_cuda_mmq_load_tiles_q2_K, + ggml_cuda_mmq_vec_dot_q2_K_q8_1_mma, + ggml_cuda_mmq_write_back_mma); + case GGML_TYPE_Q3_K: + return ggml_cuda_mmq_util_funcs( + -1, + ggml_cuda_mmq_load_tiles_q3_K, + ggml_cuda_mmq_vec_dot_q8_0_16_q8_1_mma, + ggml_cuda_mmq_write_back_mma); + case GGML_TYPE_Q4_K: + return ggml_cuda_mmq_util_funcs( + -1, + ggml_cuda_mmq_load_tiles_q4_K, + ggml_cuda_mmq_vec_dot_q8_1_q8_1_mma, + ggml_cuda_mmq_write_back_mma); + case GGML_TYPE_Q5_K: + return ggml_cuda_mmq_util_funcs( + -1, + ggml_cuda_mmq_load_tiles_q5_K, + ggml_cuda_mmq_vec_dot_q8_1_q8_1_mma, + ggml_cuda_mmq_write_back_mma); + case GGML_TYPE_Q6_K: + return ggml_cuda_mmq_util_funcs( + -1, + ggml_cuda_mmq_load_tiles_q6_K, + ggml_cuda_mmq_vec_dot_q6_K_q8_1_mma, + ggml_cuda_mmq_write_back_mma); +// --------------------------------------------------------------------------------------------- + case GGML_TYPE_IQ1_S: + return ggml_cuda_mmq_util_funcs( + -1, + ggml_cuda_mmq_load_tiles_iq1_s, + ggml_cuda_mmq_vec_dot_q8_1_q8_1_mma, + ggml_cuda_mmq_write_back_mma); + case GGML_TYPE_IQ2_XXS: + return ggml_cuda_mmq_util_funcs( + -1, + ggml_cuda_mmq_load_tiles_iq2_xxs, + ggml_cuda_mmq_vec_dot_q8_0_q8_1_mma, + ggml_cuda_mmq_write_back_mma); + case GGML_TYPE_IQ2_XS: + return ggml_cuda_mmq_util_funcs( + -1, + ggml_cuda_mmq_load_tiles_iq2_xs, + ggml_cuda_mmq_vec_dot_q8_0_16_q8_1_mma, + ggml_cuda_mmq_write_back_mma); + case GGML_TYPE_IQ2_S: + return ggml_cuda_mmq_util_funcs( + -1, + ggml_cuda_mmq_load_tiles_iq2_s, + ggml_cuda_mmq_vec_dot_q8_0_16_q8_1_mma, + ggml_cuda_mmq_write_back_mma); + case GGML_TYPE_IQ3_XXS: + return ggml_cuda_mmq_util_funcs( + -1, + ggml_cuda_mmq_load_tiles_iq3_xxs, + ggml_cuda_mmq_vec_dot_q8_0_q8_1_mma, + ggml_cuda_mmq_write_back_mma); + case GGML_TYPE_IQ3_S: + return ggml_cuda_mmq_util_funcs( + -1, + ggml_cuda_mmq_load_tiles_iq3_s, + ggml_cuda_mmq_vec_dot_q8_0_q8_1_mma, + ggml_cuda_mmq_write_back_mma); + case GGML_TYPE_IQ4_XS: + return ggml_cuda_mmq_util_funcs( + -1, + ggml_cuda_mmq_load_tiles_iq4_xs, + ggml_cuda_mmq_vec_dot_q8_0_q8_1_mma, + ggml_cuda_mmq_write_back_mma); + case GGML_TYPE_IQ4_NL: + return ggml_cuda_mmq_util_funcs( + -1, + ggml_cuda_mmq_load_tiles_iq4_nl, + ggml_cuda_mmq_vec_dot_q8_0_q8_1_mma, + ggml_cuda_mmq_write_back_mma); +// --------------------------------------------------------------------------------------------- + case GGML_TYPE_MXFP4: + return ggml_cuda_mmq_util_funcs( + -1, + ggml_cuda_mmq_load_tiles_mxfp4, + ggml_cuda_mmq_vec_dot_q8_0_q8_1_mma, + ggml_cuda_mmq_write_back_mma); + case GGML_TYPE_NVFP4: + return ggml_cuda_mmq_util_funcs( + -1, + ggml_cuda_mmq_load_tiles_nvfp4, + ggml_cuda_mmq_vec_dot_q8_0_16_q8_1_mma, + ggml_cuda_mmq_write_back_mma); + default: + return ggml_cuda_mmq_util_funcs(1, nullptr, nullptr, nullptr); + } +} + +template +static constexpr __device__ int ggml_cuda_mmq_get_vdr() { + return ggml_cuda_mmq_get_util_funcs().vdr; +} + +template +static constexpr __device__ ggml_cuda_mmq_load_tiles_t ggml_cuda_mmq_get_load_tiles() { + return ggml_cuda_mmq_get_util_funcs().load_tiles; +} + +template +static constexpr __device__ ggml_cuda_mmq_vec_dot_t ggml_cuda_mmq_get_vec_dot() { + return ggml_cuda_mmq_get_util_funcs().vec_dot; +} + +template +static constexpr __device__ ggml_cuda_mmq_write_back_t ggml_cuda_mmq_get_write_back() { + return ggml_cuda_mmq_get_util_funcs().write_back; +} + +// --------------------------------------------------------------------------------------------- + +template +static __device__ __forceinline__ void mul_mat_q_process_tile( + const char * __restrict__ x, const int offset_x, const int * __restrict__ y, + const int * __restrict__ ids_dst, float * __restrict__ dst, float * __restrict__ tmp_fixup, + const float * __restrict__ y_scale, + const int stride_row_x, const int ncols_y, const int stride_col_dst, + const int tile_x_max_i, const int tile_y_max_j, const int kb0_start, const int kb0_stop) { + + constexpr int warp_size = ggml_cuda_get_physical_warp_size(); + constexpr int nwarps = ggml_cuda_mmq_get_nthreads(type, J, fallback) / warp_size; + constexpr int qk = ggml_cuda_type_traits::qk; + constexpr int I = ggml_cuda_mmq_get_I(type, J, fallback); + constexpr ggml_cuda_mmq_load_tiles_t load_tiles = ggml_cuda_mmq_get_load_tiles(); + constexpr ggml_cuda_mmq_vec_dot_t vec_dot = ggml_cuda_mmq_get_vec_dot(); + constexpr ggml_cuda_mmq_write_back_t write_back = ggml_cuda_mmq_get_write_back(); + + extern __shared__ int data_mul_mat_q[]; + int * tile_y = data_mul_mat_q + J; + int * tile_x = tile_y + GGML_PAD(J*MMQ_TILE_Y_K, nwarps*warp_size); + +#if defined(BLACKWELL_MMA_AVAILABLE) + // FP4 tile stores 8 blocks + constexpr int ne_block = (type == GGML_TYPE_MXFP4 || type == GGML_TYPE_NVFP4) ? QK_FP4_MMQ : QK8_1_MMQ; +#else + constexpr int ne_block = QK8_1_MMQ; +#endif // defined(BLACKWELL_MMA_AVAILABLE) + + constexpr int ITER_K = ggml_cuda_mmq_get_K_vram(type, J, fallback); + constexpr int blocks_per_iter = ITER_K / qk; + + float sum[J*I / (nwarps*warp_size)] = {0.0f}; + + constexpr int sz = sizeof(block_q8_1_mmq) / sizeof(int); + + for (int kb0 = kb0_start; kb0 < kb0_stop; kb0 += blocks_per_iter) { + load_tiles(x, tile_x, offset_x + kb0, tile_x_max_i, stride_row_x); + { + const int * by0 = y + ncols_y * (kb0 * qk / ne_block) * sz; +#pragma unroll + for (int l0 = 0; l0 < J * MMQ_TILE_Y_K; l0 += nwarps * warp_size) { + int l = l0 + threadIdx.y*warp_size + threadIdx.x; + + tile_y[l] = by0[l]; + } + } + + __syncthreads(); + + vec_dot(tile_x, tile_y, sum, 0); + + __syncthreads(); + + { + const int * by0 = y + ncols_y * ((kb0 * qk / ne_block) * sz + sz); +#pragma unroll + for (int l0 = 0; l0 < J * MMQ_TILE_Y_K; l0 += nwarps * warp_size) { + int l = l0 + threadIdx.y*warp_size + threadIdx.x; + + tile_y[l] = by0[l]; + } + } + + __syncthreads(); + + vec_dot(tile_x, tile_y, sum, MMQ_TILE_NE_K); + + __syncthreads(); + } + + if (fixup) { + write_back(sum, ids_dst, tmp_fixup + blockIdx.x*(J*I), y_scale, I, I, J); + } else { + write_back(sum, ids_dst, dst, y_scale, stride_col_dst, tile_x_max_i, tile_y_max_j); + } +} + + +// The mul_mat_q kernel implements "stream-k" work partitioning as described in https://arxiv.org/abs/2301.03598 + +template +__launch_bounds__(ggml_cuda_mmq_get_nthreads(type, J, fallback), ggml_cuda_mmq_get_occupancy(type, J, fallback)) +static __global__ void mul_mat_q( + const char * __restrict__ x, const int * __restrict__ y, const int32_t * __restrict__ ids_dst, + const int32_t * __restrict__ expert_bounds, float * __restrict__ dst, float * __restrict__ tmp_fixup, + const float * __restrict__ y_scale, + const uint3 blocks_per_ne00, const int nrows_x, const int ncols_dst, const int stride_row_x, const int ncols_y, const int stride_col_dst, + const uint3 channel_ratio, const uint3 nchannels_y, const int stride_channel_x, const int stride_channel_y, const int stride_channel_dst, + const uint3 sample_ratio, const uint3 nsamples_y, const int stride_sample_x, const int stride_sample_y, const int stride_sample_dst, + const uint3 ntx) { + + // Skip unused template specializations for faster compilation: + if (ggml_cuda_mmq_get_config(type, J, fallback).type == GGML_TYPE_COUNT) { + NO_DEVICE_CODE; + return; + } + + constexpr int warp_size = ggml_cuda_get_physical_warp_size(); + constexpr int nwarps = ggml_cuda_mmq_get_nthreads(type, J, fallback) / warp_size; + constexpr int qk = ggml_cuda_type_traits::qk; + constexpr int I = ggml_cuda_mmq_get_I(type, J, fallback); + + const uint32_t nty = (nrows_x + I - 1) / I; // Number of tiles y + + // Initialize the ids for writing back data with just the index. + // For regular matrix multiplications this is never changed. + // For MoE the correct indices are loaded from ids_dst. + extern __shared__ int ids_dst_shared[]; // Stored at beginning of shared memory. +#pragma unroll + for (int j0 = 0; j0 < J; j0 += nwarps*warp_size) { + const int j = j0 + threadIdx.y*warp_size + threadIdx.x; + + if (j0 + nwarps*warp_size > J && j >= J) { + break; + } + + ids_dst_shared[j] = j; + } + __syncthreads(); + + if constexpr (!ggml_cuda_mmq_get_stream_k(type, J, fallback)) { + const uint2 tmp2 = fast_div_modulo(blockIdx.z, nchannels_y); + const int wt = tmp2.x; + const int zt = tmp2.y; + const int jt = blockIdx.y; + const int it = blockIdx.x; + + // Defaults for regular matrix multiplication: + int col_low = 0; + int col_high = ncols_dst; + int col_diff = ncols_dst; + int offset_y = wt*stride_sample_y + zt*stride_channel_y; + int offset_dst = wt*stride_sample_dst + zt*stride_channel_dst + jt*J*stride_col_dst; + int offset_y_scale; + if constexpr (type == GGML_TYPE_NVFP4) { + offset_y_scale = wt*nchannels_y.z*ncols_y + zt*ncols_y; + } else { + GGML_UNUSED(offset_y_scale); + } + + if (ids_dst) { + col_low = expert_bounds[zt + 0]; + col_high = expert_bounds[zt + 1]; + col_diff = col_high - col_low; + + offset_y = 0; + offset_dst = 0; + if constexpr (type == GGML_TYPE_NVFP4) { + offset_y_scale = 0; + } + + if (jt*J >= col_diff) { + return; + } + + // __syncthreads(); // There is no previous tile that could cause a race condition. +#pragma unroll + for (int j0 = 0; j0 < J; j0 += nwarps*warp_size) { + const int j = j0 + threadIdx.y*warp_size + threadIdx.x; + + if (j0 + nwarps*warp_size > J && j >= J) { + break; + } + + ids_dst_shared[j] = ids_dst[col_low + jt*J + j]; + } + __syncthreads(); + } + + offset_y += (col_low + jt*J)*(sizeof(block_q8_1_mmq)/sizeof(int)); + offset_dst += it*I; + const float * y_scale_tile = nullptr; + if constexpr (type == GGML_TYPE_NVFP4) { + offset_y_scale += col_low + jt*J; + y_scale_tile = y_scale ? y_scale + offset_y_scale : nullptr; + } + + const int tile_x_max_i = nrows_x - it*I - 1; + const int tile_y_max_j = col_diff - jt*J - 1; + + const int offset_x = fastdiv(wt, sample_ratio)*stride_sample_x + fastdiv(zt, channel_ratio)*stride_channel_x + it*I*stride_row_x; + + constexpr bool fixup = false; + mul_mat_q_process_tile + (x, offset_x, y + offset_y, ids_dst_shared, dst + offset_dst, tmp_fixup, y_scale_tile, + stride_row_x, ncols_y, stride_col_dst, + tile_x_max_i, tile_y_max_j, 0, blocks_per_ne00.z); + return; + } + + constexpr int ITER_K = ggml_cuda_mmq_get_K_vram(type, J, fallback); + constexpr int blocks_per_iter = ITER_K / qk; + + // kbc == k block continuous, current index in continuous ijk space. + int kbc = int64_t(blockIdx.x) *(nsamples_y.z*nchannels_y.z*ntx.z*nty*blocks_per_ne00.z) / gridDim.x; + int kbc_stop = int64_t(blockIdx.x + 1)*(nsamples_y.z*nchannels_y.z*ntx.z*nty*blocks_per_ne00.z) / gridDim.x; + + kbc -= fastmodulo(kbc, blocks_per_ne00) % blocks_per_iter; + kbc_stop -= fastmodulo(kbc_stop, blocks_per_ne00) % blocks_per_iter; + + // kb0 == k index when doing the matrix multiplication for an output tile. + int kb0_start = fastmodulo(kbc, blocks_per_ne00); + int kb0_stop = min(blocks_per_ne00.z, uint32_t(kb0_start + kbc_stop - kbc)); + while (kbc < kbc_stop && kb0_stop == int(blocks_per_ne00.z)) { + int tmp = fastdiv(kbc, blocks_per_ne00); + uint2 tmp2 = fast_div_modulo(tmp, ntx); + const int jt = tmp2.y; + tmp = tmp2.x; + tmp2 = fast_div_modulo(tmp, nchannels_y); + const int zt = tmp2.y; + tmp = tmp2.x; + tmp2 = fast_div_modulo(tmp, nsamples_y); + const int wt = tmp2.y; + const int it = tmp2.x; + + // Defaults for regular matrix multiplication: + int col_low = 0; + int col_high = ncols_dst; + int col_diff = ncols_dst; + int offset_y = wt*stride_sample_y + zt*stride_channel_y; + int offset_dst = wt*stride_sample_dst + zt*stride_channel_dst + jt*J*stride_col_dst; + int offset_y_scale; + if constexpr (type == GGML_TYPE_NVFP4) { + offset_y_scale = wt*nchannels_y.z*ncols_y + zt*ncols_y; + } else { + GGML_UNUSED(offset_y_scale); + } + + if (ids_dst) { + col_low = expert_bounds[zt + 0]; + col_high = expert_bounds[zt + 1]; + col_diff = col_high - col_low; + + offset_y = 0; + offset_dst = 0; + if constexpr (type == GGML_TYPE_NVFP4) { + offset_y_scale = 0; + } + + if (jt*J >= col_diff) { + kbc += blocks_per_ne00.z; + kbc -= fastmodulo(kbc, blocks_per_ne00); + + kb0_start = 0; + kb0_stop = min(blocks_per_ne00.z, uint32_t(kbc_stop - kbc)); + + continue; + } + + __syncthreads(); +#pragma unroll + for (int j0 = 0; j0 < J; j0 += nwarps*warp_size) { + const int j = j0 + threadIdx.y*warp_size + threadIdx.x; + + if (j0 + nwarps*warp_size > J && j >= J) { + break; + } + + ids_dst_shared[j] = ids_dst[col_low + jt*J + j]; + } + __syncthreads(); + } + + offset_y += (col_low + jt * J) * (sizeof(block_q8_1_mmq) / sizeof(int)); + offset_dst += it*I; + const float * y_scale_tile = nullptr; + if constexpr (type == GGML_TYPE_NVFP4) { + offset_y_scale += col_low + jt * J; + y_scale_tile = y_scale ? y_scale + offset_y_scale : nullptr; + } + + const int tile_x_max_i = nrows_x - it*I - 1; + const int tile_y_max_j = col_diff - jt*J - 1; + + const int offset_x = fastdiv(wt, sample_ratio)*stride_sample_x + fastdiv(zt, channel_ratio)*stride_channel_x + it*I*stride_row_x; + + constexpr bool fixup = false; // All but (potentially) the last iterations write their data to dst rather than the fixup buffer. + mul_mat_q_process_tile + (x, offset_x, y + offset_y, ids_dst_shared, dst + offset_dst, tmp_fixup, y_scale_tile, + stride_row_x, ncols_y, stride_col_dst, + tile_x_max_i, tile_y_max_j, kb0_start, kb0_stop); + + kbc += blocks_per_ne00.z; + kbc -= fastmodulo(kbc, blocks_per_ne00); + + kb0_start = 0; + kb0_stop = min(blocks_per_ne00.z, uint32_t(kbc_stop - kbc)); + } + + if (kbc >= kbc_stop) { + return; + } + + int tmp = fastdiv(kbc, blocks_per_ne00); + uint2 tmp2 = fast_div_modulo(tmp, ntx); + const int jt = tmp2.y; + tmp = tmp2.x; + tmp2 = fast_div_modulo(tmp, nchannels_y); + const int zt = tmp2.y; + tmp = tmp2.x; + tmp2 = fast_div_modulo(tmp, nsamples_y); + const int wt = tmp2.y; + const int it = tmp2.x; + + // Defaults for regular matrix multiplication: + int col_low = 0; + int col_high = ncols_dst; + int col_diff = ncols_dst; + int offset_y = wt*stride_sample_y + zt*stride_channel_y; + int offset_dst = wt*stride_sample_dst + zt*stride_channel_dst + jt*J*stride_col_dst; + int offset_y_scale; + if constexpr (type == GGML_TYPE_NVFP4) { + offset_y_scale = wt*nchannels_y.z*ncols_y + zt*ncols_y; + } else { + GGML_UNUSED(offset_y_scale); + } + + if (ids_dst) { + col_low = expert_bounds[zt + 0]; + col_high = expert_bounds[zt + 1]; + col_diff = col_high - col_low; + + offset_y = 0; + offset_dst = 0; + if constexpr (type == GGML_TYPE_NVFP4) { + offset_y_scale = 0; + } + + if (jt*J >= col_diff) { + return; + } + + // The memory layout for the fixup buffer is always contiguous, therefore reset ids: + __syncthreads(); +#pragma unroll + for (int j0 = 0; j0 < J; j0 += nwarps*warp_size) { + const int j = j0 + threadIdx.y*warp_size + threadIdx.x; + + if (j0 + nwarps*warp_size > J && j >= J) { + break; + } + + ids_dst_shared[j] = j; + } + __syncthreads(); + } + + offset_y += (col_low + jt * J) * (sizeof(block_q8_1_mmq) / sizeof(int)); + offset_dst += it*I; + const float * y_scale_tile = nullptr; + if constexpr (type == GGML_TYPE_NVFP4) { + offset_y_scale += col_low + jt * J; + y_scale_tile = y_scale ? y_scale + offset_y_scale : nullptr; + } + + const int tile_x_max_i = nrows_x - it*I - 1; + const int tile_y_max_j = col_diff - jt*J - 1; + + const int offset_x = fastdiv(wt, sample_ratio)*stride_sample_x + fastdiv(zt, channel_ratio)*stride_channel_x + it*I*stride_row_x; + + constexpr bool fixup = true; // Last index writes its data to fixup buffer to avoid data races with other blocks. + mul_mat_q_process_tile + (x, offset_x, y + offset_y, ids_dst_shared, dst + offset_dst, tmp_fixup, y_scale_tile, + stride_row_x, ncols_y, stride_col_dst, + tile_x_max_i, tile_y_max_j, kb0_start, kb0_stop); +} + +template +__launch_bounds__(ggml_cuda_mmq_get_nthreads(type, J, fallback)/2, 1) +static __global__ void mul_mat_q_stream_k_fixup( + const int32_t * __restrict__ ids_dst, const int32_t * __restrict__ expert_bounds, float * __restrict__ dst, + float * __restrict__ tmp_last_tile, const uint3 blocks_per_ne00, const int nrows_x, const int ncols_dst, + const int stride_col_dst, const uint3 nchannels_y, const int stride_channel_dst, const uint3 nsamples_y, + const int stride_sample_dst, const uint3 ntx) { + constexpr int warp_size = ggml_cuda_get_physical_warp_size(); + constexpr int nwarps = (ggml_cuda_mmq_get_nthreads(type, J, fallback) / 2) / warp_size; + constexpr int I = ggml_cuda_mmq_get_I(type, J, fallback); + constexpr int qk = ggml_cuda_type_traits::qk; + constexpr int ITER_K = ggml_cuda_mmq_get_K_vram(type, J, fallback); + constexpr int blocks_per_iter = ITER_K / qk; + + float sum[J / nwarps] = {0.0f}; + const int i = blockIdx.y*warp_size + threadIdx.x; + + const int nty = (nrows_x + I - 1) / I; + + const int bidx0 = blockIdx.x; + + // kbc == k block continuous, current index in continuous ijk space. + int kbc0 = int64_t(blockIdx.x) *(nsamples_y.z*nchannels_y.z*ntx.z*nty*blocks_per_ne00.z) / gridDim.x; + int kbc0_stop = int64_t(blockIdx.x + 1)*(nsamples_y.z*nchannels_y.z*ntx.z*nty*blocks_per_ne00.z) / gridDim.x; + + kbc0 -= fastmodulo(kbc0, blocks_per_ne00) % blocks_per_iter; + kbc0_stop -= fastmodulo(kbc0_stop, blocks_per_ne00) % blocks_per_iter; + + const bool did_not_have_any_data = kbc0 == kbc0_stop; + const bool wrote_beginning_of_tile = fastmodulo(kbc0, blocks_per_ne00) == 0; + const bool did_not_write_last = fastdiv(kbc0, blocks_per_ne00) == fastdiv(kbc0_stop, blocks_per_ne00) && fastmodulo(kbc0_stop, blocks_per_ne00) != 0; + if (did_not_have_any_data || wrote_beginning_of_tile || did_not_write_last) { + return; + } + + bool any_fixup = false; + + // Iterate over previous blocks and sum up partial sums written to fixup buffer. + // All CUDA blocks that get here must have a previous block that needs a fixup. + int bidx = bidx0 - 1; + int kbc_stop = kbc0; + while(true) { + int kbc = int64_t(bidx)*(nsamples_y.z*nchannels_y.z*ntx.z*nty*blocks_per_ne00.z) / gridDim.x; + kbc -= fastmodulo(kbc, blocks_per_ne00) % blocks_per_iter; + + if (kbc == kbc_stop) { // Did not have any data. + bidx--; + kbc_stop = kbc; + continue; + } + + any_fixup = true; + + +#pragma unroll + for (int j0 = 0; j0 < J; j0 += nwarps) { + const int j = j0 + threadIdx.y; + + sum[j0/nwarps] += tmp_last_tile[bidx*(J*I) + j*I + i]; + } + + // If this block started in a previous tile we are done and don't need to combine additional partial results. + if (fastmodulo(kbc, blocks_per_ne00) == 0 || fastdiv(kbc, blocks_per_ne00) < fastdiv(kbc0, blocks_per_ne00)) { + break; + } + bidx--; + kbc_stop = kbc; + } + + if (!any_fixup) { + return; + } + + int tmp = fastdiv(kbc0, blocks_per_ne00); + uint2 tmp2 = fast_div_modulo(tmp, ntx); + const int jt = tmp2.y; + tmp = tmp2.x; + tmp2 = fast_div_modulo(tmp, nchannels_y); + const int zt = tmp2.y; + tmp = tmp2.x; + tmp2 = fast_div_modulo(tmp, nsamples_y); + const int wt = tmp2.y; + const int it = tmp2.x; + + if (!ids_dst) { + const int offset_dst = wt*stride_sample_dst + zt*stride_channel_dst + jt*J*stride_col_dst + it*I; + dst += offset_dst; + + const int i_max = nrows_x - it*I - 1; + const int j_max = ncols_dst - jt*J - 1; + if (fallback && i > i_max) { + return; + } + +#pragma unroll + for (int j0 = 0; j0 < J; j0 += nwarps) { + const int j = j0 + threadIdx.y; + + if (j > j_max) { + return; + } + + dst[j*stride_col_dst + i] += sum[j0/nwarps]; + } + return; + } + + __shared__ int ids_dst_shared[J]; + const int col_low = expert_bounds[zt + 0]; + const int col_high = expert_bounds[zt + 1]; + const int col_diff = col_high - col_low; + + for (int j = threadIdx.y*warp_size + threadIdx.x; j < J; j += nwarps*warp_size) { + ids_dst_shared[j] = ids_dst[col_low + jt*J + j]; + } + __syncthreads(); + + const int offset_dst = it*I; + dst += offset_dst; + + const int i_max = nrows_x - it*I - 1; + const int j_max = col_diff - jt*J - 1; + if (fallback && i > i_max) { + return; + } + +#pragma unroll + for (int j0 = 0; j0 < J; j0 += nwarps) { + const int j = j0 + threadIdx.y; + + if (j > j_max) { + return; + } + + dst[ids_dst_shared[j]*stride_col_dst + i] += sum[j0/nwarps]; + } +} + +struct mmq_args { + const char * x; ggml_type type_x; const int * y; const int32_t * ids_dst; const int32_t * expert_bounds; float * dst; + const float * y_scale; + int64_t ncols_x; int64_t nrows_x; int64_t ncols_dst; int64_t stride_row_x; int64_t ncols_y; int64_t nrows_dst; + int64_t nchannels_x; int64_t nchannels_y; int64_t stride_channel_x; int64_t stride_channel_y; int64_t stride_channel_dst; + int64_t nsamples_x; int64_t nsamples_y; int64_t stride_sample_x; int64_t stride_sample_y; int64_t stride_sample_dst; + int64_t ncols_max; +}; + +static size_t mmq_get_nbytes_shared(const ggml_cuda_mmq_config & config, const int cc) { + const size_t nbs_ids = config.J*sizeof(int); + const size_t nbs_x = ggml_cuda_mmq_get_nbytes_shared_x(config, cc); + const size_t nbs_y = config.J * (sizeof(block_q8_1_mmq)); + return nbs_ids + nbs_x + GGML_PAD(nbs_y, config.nthreads*sizeof(int)); +} + +template +static void launch_mul_mat_q(ggml_backend_cuda_context & ctx, const mmq_args & args, cudaStream_t stream) { + const int id = ggml_cuda_get_device(); + const int cc = ggml_cuda_info().devices[id].cc; + const int nsm = ggml_cuda_info().devices[id].nsm; + const int warp_size = ggml_cuda_info().devices[id].warp_size; + + const ggml_cuda_mmq_config config = ggml_cuda_mmq_get_config(type, J, fallback, cc); + GGML_ASSERT(config.nthreads % warp_size == 0); + const int nwarps = config.nthreads / warp_size; + const int nbytes_shared = mmq_get_nbytes_shared(config, cc); + + const dim3 block_dims(warp_size, nwarps, 1); + + CUDA_SET_SHARED_MEMORY_LIMIT((mul_mat_q), nbytes_shared); + CUDA_SET_SHARED_MEMORY_LIMIT((mul_mat_q), nbytes_shared); + + const int nty = (args.nrows_x + config.I - 1) / config.I; + const int ntx = (args.ncols_max + config.J - 1) / config.J; + const int ntzw = args.nchannels_y * args.nsamples_y; + const dim3 block_nums_xy_tiling(nty, ntx, ntzw); + + GGML_ASSERT(args.nchannels_y % args.nchannels_x == 0); + GGML_ASSERT(args.nsamples_y % args.nsamples_x == 0); + const int channel_ratio = args.nchannels_y / args.nchannels_x; + const int sample_ratio = args.nsamples_y / args.nsamples_x; + + const uint3 blocks_per_ne00_fd = init_fastdiv_values(args.ncols_x / ggml_cuda_type_traits::qk); + const uint3 ntx_fd = init_fastdiv_values(ntx); + const uint3 nchannels_y_fd = init_fastdiv_values(args.nchannels_y); + const uint3 nsamples_y_fd = init_fastdiv_values(args.nsamples_y); + const uint3 channel_ratio_fd = init_fastdiv_values(channel_ratio); + const uint3 sample_ratio_fd = init_fastdiv_values(sample_ratio); + + if (!ggml_cuda_mmq_get_stream_k(type, J, fallback, cc)) { + mul_mat_q<<>> + (args.x, args.y, args.ids_dst, args.expert_bounds, args.dst, nullptr, args.y_scale, + blocks_per_ne00_fd, args.nrows_x, args.ncols_dst, args.stride_row_x, args.ncols_y, args.nrows_dst, + channel_ratio_fd, nchannels_y_fd, args.stride_channel_x, args.stride_channel_y, args.stride_channel_dst, + sample_ratio_fd, nsamples_y_fd, args.stride_sample_x, args.stride_sample_y, args.stride_sample_dst, + ntx_fd); + return; + } + + // For the stream-k kernel it is possible to run it with tiling by setting the number of CUDA blocks equal to the number of tiles. + // This is worthwhile if the efficiency of tiling is high and skipping the fixup kernel is more important. + const int ntiles_dst = ntx * nty * ntzw; + const int tiles_nwaves = (ntiles_dst + nsm - 1) / nsm; + const int tiles_efficiency_percent = 100 * ntiles_dst / (nsm*tiles_nwaves); + const dim3 block_nums_stream_k(GGML_CUDA_CC_IS_NVIDIA(cc) && tiles_efficiency_percent >= 90 ? ntiles_dst : nsm, 1, 1); + + GGML_ASSERT(ntiles_dst * blocks_per_ne00_fd.z < (1 << 30)); // Assert that variable kbc will not overflow. + + const bool fixup_needed = ntiles_dst % block_nums_stream_k.x != 0; + + ggml_cuda_pool & pool = ctx.pool(id); + ggml_cuda_pool_alloc tmp_fixup(pool); + if (fixup_needed) { + tmp_fixup.alloc(block_nums_stream_k.x * config.J*config.I); + } + + const dim3 block_nums_fixup(block_nums_stream_k.x, config.I/warp_size, 1); + const dim3 block_dims_fixup(block_dims.x, block_dims.y/2, block_dims.z); + + mul_mat_q<<>> + (args.x, args.y, args.ids_dst, args.expert_bounds, args.dst, tmp_fixup.ptr, args.y_scale, + blocks_per_ne00_fd, args.nrows_x, args.ncols_dst, args.stride_row_x, args.ncols_y, args.nrows_dst, + channel_ratio_fd, nchannels_y_fd, args.stride_channel_x, args.stride_channel_y, args.stride_channel_dst, + sample_ratio_fd, nsamples_y_fd, args.stride_sample_x, args.stride_sample_y, args.stride_sample_dst, + ntx_fd); + + if (!fixup_needed) { + return; + } + + CUDA_CHECK(cudaGetLastError()); + mul_mat_q_stream_k_fixup<<>> + (args.ids_dst, args.expert_bounds, args.dst, tmp_fixup.ptr, blocks_per_ne00_fd, args.nrows_x, args.ncols_dst, + args.nrows_dst, nchannels_y_fd, args.stride_channel_dst, nsamples_y_fd, args.stride_sample_dst, + ntx_fd); +} + +template +void mul_mat_q_switch_J(ggml_backend_cuda_context & ctx, const mmq_args & args, cudaStream_t stream) { + const int id = ggml_cuda_get_device(); + const int cc = ggml_cuda_info().devices[id].cc; + const size_t smpbo = ggml_cuda_info().devices[id].smpbo; + + int J_best = 0; + int ntiles_J_best = INT_MAX; + + for (int J = 8; J <= 128 && ntiles_J_best > 1; J += 8) { + const ggml_cuda_mmq_config config = ggml_cuda_mmq_get_config(type, J, fallback, cc); + if (config.type == GGML_TYPE_COUNT) { + continue; + } + + if (mmq_get_nbytes_shared(config, cc) > smpbo) { + continue; + } + + const int ntiles_x = (args.ncols_max + config.J - 1) / config.J; + + if (ntiles_x < ntiles_J_best) { + J_best = J; + ntiles_J_best = ntiles_x; + } + } + + switch (J_best) { + case 8: + launch_mul_mat_q(ctx, args, stream); + break; + case 16: + launch_mul_mat_q(ctx, args, stream); + break; + case 24: + launch_mul_mat_q(ctx, args, stream); + break; + case 32: + launch_mul_mat_q(ctx, args, stream); + break; + case 40: + launch_mul_mat_q(ctx, args, stream); + break; + case 48: + launch_mul_mat_q(ctx, args, stream); + break; + case 56: + launch_mul_mat_q(ctx, args, stream); + break; + case 64: + launch_mul_mat_q(ctx, args, stream); + break; + case 72: + launch_mul_mat_q(ctx, args, stream); + break; + case 80: + launch_mul_mat_q(ctx, args, stream); + break; + case 88: + launch_mul_mat_q(ctx, args, stream); + break; + case 96: + launch_mul_mat_q(ctx, args, stream); + break; + case 104: + launch_mul_mat_q(ctx, args, stream); + break; + case 112: + launch_mul_mat_q(ctx, args, stream); + break; + case 120: + launch_mul_mat_q(ctx, args, stream); + break; + case 128: + launch_mul_mat_q(ctx, args, stream); + break; + default: + fprintf(stderr, "J_best=%d\n", J_best); + GGML_ABORT("fatal error"); + break; + } +} + +template +void mul_mat_q_case(ggml_backend_cuda_context & ctx, const mmq_args & args, cudaStream_t stream) { + if (args.nrows_x % 128 == 0) { + constexpr bool fallback = false; + mul_mat_q_switch_J(ctx, args, stream); + } else { + constexpr bool fallback = true; + mul_mat_q_switch_J(ctx, args, stream); + } +} + +#define DECL_MMQ_CASE(type) \ + template void mul_mat_q_case(ggml_backend_cuda_context & ctx, const mmq_args & args, cudaStream_t stream) \ + +extern DECL_MMQ_CASE(GGML_TYPE_Q1_0); +extern DECL_MMQ_CASE(GGML_TYPE_Q2_0); +extern DECL_MMQ_CASE(GGML_TYPE_Q4_0); +extern DECL_MMQ_CASE(GGML_TYPE_Q4_1); +extern DECL_MMQ_CASE(GGML_TYPE_Q5_0); +extern DECL_MMQ_CASE(GGML_TYPE_Q5_1); +extern DECL_MMQ_CASE(GGML_TYPE_Q8_0); +// ----------------------------------------- +extern DECL_MMQ_CASE(GGML_TYPE_Q2_K); +extern DECL_MMQ_CASE(GGML_TYPE_Q3_K); +extern DECL_MMQ_CASE(GGML_TYPE_Q4_K); +extern DECL_MMQ_CASE(GGML_TYPE_Q5_K); +extern DECL_MMQ_CASE(GGML_TYPE_Q6_K); +// ----------------------------------------- +extern DECL_MMQ_CASE(GGML_TYPE_IQ1_S); +extern DECL_MMQ_CASE(GGML_TYPE_IQ2_XXS); +extern DECL_MMQ_CASE(GGML_TYPE_IQ2_XS); +extern DECL_MMQ_CASE(GGML_TYPE_IQ2_S); +extern DECL_MMQ_CASE(GGML_TYPE_IQ3_XXS); +extern DECL_MMQ_CASE(GGML_TYPE_IQ3_S); +extern DECL_MMQ_CASE(GGML_TYPE_IQ4_NL); +extern DECL_MMQ_CASE(GGML_TYPE_IQ4_XS); +// ----------------------------------------- +extern DECL_MMQ_CASE(GGML_TYPE_MXFP4); +extern DECL_MMQ_CASE(GGML_TYPE_NVFP4); + +// ------------------------------------------------------------------------------------------------------------------------- + +void ggml_cuda_mul_mat_q( + ggml_backend_cuda_context & ctx, const ggml_tensor * src0, const ggml_tensor * src1, const ggml_tensor * ids, ggml_tensor * dst); + +bool ggml_cuda_should_use_mmq(enum ggml_type type, int cc, int64_t ne11, int64_t n_experts); diff --git a/python/freetoken/kernel/csrc/gguf_mmq/mmq_ext.cu b/python/freetoken/kernel/csrc/gguf_mmq/mmq_ext.cu new file mode 100644 index 00000000..5c141b43 --- /dev/null +++ b/python/freetoken/kernel/csrc/gguf_mmq/mmq_ext.cu @@ -0,0 +1,321 @@ +// Upstream llama.cpp int8-tensor-core MMQ (mul_mat_q) for Q4_K/Q6_K. +// +// The sibling csrc/gguf tree carries llama.cpp b2899 DP4A kernels (via +// vLLM/sgl-kernel); upstream has since rewritten MMQ around int8 MMA tiles +// (turing_mma, sm_75+), which is ~13x faster at prefill row counts on sm_120 +// and also beats transient dequant+cuBLAS at every batch size. Files in this +// directory other than this one are vendored VERBATIM from llama.cpp master +// eab8ee41f889ef7823af517e8098fb8a9b3cf601 (ggml/src/ggml-cuda + the ggml +// headers they include); this file supplies the small backend shims that +// ggml-cuda.cu would normally provide (device info, pool, error/abort) plus +// the torch bindings. Only Q4_K/Q6_K mul_mat_q cases are instantiated to keep +// JIT compile time down; the GEMV (batch<=6) and MoE paths stay on the +// existing csrc/gguf kernels. + +#include "common.cuh" +#include "mmq.cuh" +#include "quantize.cuh" +#include "mmid.cuh" + +#include +#include +#include +#include + +#include +#include + +// --------------------------------------------------------------------------- +// ggml backend shims +// --------------------------------------------------------------------------- + +extern "C" size_t ggml_type_size(enum ggml_type type) { + switch (type) { + case GGML_TYPE_Q4_K: return sizeof(block_q4_K); + case GGML_TYPE_Q6_K: return sizeof(block_q6_K); + case GGML_TYPE_F32: return sizeof(float); + default: GGML_ABORT("ggml_type_size: unsupported type %d", (int) type); + } +} + +extern "C" int64_t ggml_blck_size(enum ggml_type type) { + switch (type) { + case GGML_TYPE_Q4_K: return QK_K; + case GGML_TYPE_Q6_K: return QK_K; + case GGML_TYPE_F32: return 1; + default: GGML_ABORT("ggml_blck_size: unsupported type %d", (int) type); + } +} + +void ggml_abort(const char * file, int line, const char * fmt, ...) { + char msg[512]; + va_list args; + va_start(args, fmt); + vsnprintf(msg, sizeof(msg), fmt, args); + va_end(args); + TORCH_CHECK(false, "ggml abort at ", file, ":", line, ": ", msg); +} + +void ggml_cuda_error(const char * stmt, const char * func, const char * file, int line, const char * msg) { + TORCH_CHECK(false, "CUDA error in ", func, " at ", file, ":", line, ": ", stmt, ": ", msg); +} + +int ggml_cuda_get_device() { + int id; + CUDA_CHECK(cudaGetDevice(&id)); + return id; +} + +void ggml_cuda_set_device(int device) { + CUDA_CHECK(cudaSetDevice(device)); +} + +const ggml_cuda_device_info & ggml_cuda_info() { + static ggml_cuda_device_info info = []() { + ggml_cuda_device_info inf = {}; + CUDA_CHECK(cudaGetDeviceCount(&inf.device_count)); + inf.physical_device_count = inf.device_count; + for (int id = 0; id < inf.device_count; ++id) { + cudaDeviceProp prop; + CUDA_CHECK(cudaGetDeviceProperties(&prop, id)); + auto & dev = inf.devices[id]; + dev.cc = 100 * prop.major + 10 * prop.minor; + dev.nsm = prop.multiProcessorCount; + dev.smpb = prop.sharedMemPerBlock; + dev.smpbo = prop.sharedMemPerBlockOptin; + dev.integrated = prop.integrated; + dev.warp_size = prop.warpSize; + dev.total_vram = prop.totalGlobalMem; + dev.physical_device = id; + dev.physical_share_count = 1; + dev.virtual_index = 0; + } + return inf; + }(); + return info; +} + +// Pool backed by torch's caching allocator so transient MMQ scratch (stream-k +// fixup buffers) shares the framework's memory accounting. +struct ggml_cuda_pool_torch : public ggml_cuda_pool { + void * alloc(size_t size, size_t * actual_size) override { + void * ptr = c10::cuda::CUDACachingAllocator::raw_alloc(size); + *actual_size = size; + return ptr; + } + void free(void * ptr, size_t /*size*/) override { + c10::cuda::CUDACachingAllocator::raw_delete(ptr); + } +}; + +std::unique_ptr ggml_backend_cuda_context::new_pool_for_device(int /*device*/, int /*stream_no*/) { + return std::make_unique(); +} + +ggml_backend_cuda_context::~ggml_backend_cuda_context() = default; + +// --------------------------------------------------------------------------- +// MMQ instantiations (Q4_K / Q6_K only) +// --------------------------------------------------------------------------- + +DECL_MMQ_CASE(GGML_TYPE_Q4_K); +DECL_MMQ_CASE(GGML_TYPE_Q6_K); + +// --------------------------------------------------------------------------- +// Torch entry: y = x @ dequant(W).T, W packed ggml rows [nrows, row_bytes] +// --------------------------------------------------------------------------- + +static torch::Tensor ggml_mul_mat_a8_mma(torch::Tensor W, torch::Tensor X, int64_t type, int64_t nrows) { + TORCH_CHECK(W.is_cuda() && X.is_cuda(), "CUDA tensors required"); + TORCH_CHECK(W.dtype() == torch::kUInt8 && W.dim() == 2 && W.is_contiguous()); + TORCH_CHECK(X.dim() == 2 && X.is_contiguous()); + TORCH_CHECK(type == GGML_TYPE_Q4_K || type == GGML_TYPE_Q6_K, "only Q4_K/Q6_K instantiated"); + TORCH_CHECK(W.size(0) == nrows); + + const ggml_type type_x = (ggml_type) type; + const c10::cuda::CUDAGuard guard(W.device()); + cudaStream_t stream = at::cuda::getCurrentCUDAStream(); + + // Upstream's activation quantize kernel reads fp32. + torch::Tensor Xf = X.scalar_type() == torch::kFloat32 ? X : X.to(torch::kFloat32); + + const int64_t ne01 = nrows; // weight rows (output features) + const int64_t ne11 = X.size(0); // tokens + const int64_t ne10 = X.size(1); // input features + const size_t ts = ggml_type_size(type_x); + const int64_t qk = ggml_blck_size(type_x); + TORCH_CHECK((int64_t) W.size(1) * qk == ne10 * (int64_t) ts, "row_bytes mismatch"); + const int64_t s01 = W.size(1) / ts; // row stride in blocks + + const int id = ggml_cuda_get_device(); + const int cc = ggml_cuda_info().devices[id].cc; + const bool fallback = ne01 % 128 != 0; + + const int64_t ne10_padded = GGML_PAD(ne10, MATRIX_ROW_PADDING); + const size_t nbytes_src1_q8_1 = ne11 * ne10_padded * sizeof(block_q8_1_mmq) / QK8_1_MMQ + + ggml_cuda_mmq_get_J_max(type_x, fallback, cc, ne11) * sizeof(block_q8_1_mmq); + + auto opts = torch::TensorOptions().device(W.device()); + torch::Tensor y_q = torch::empty({(int64_t) nbytes_src1_q8_1}, opts.dtype(torch::kUInt8)); + torch::Tensor dst = torch::empty({ne11, ne01}, opts.dtype(torch::kFloat32)); + + quantize_mmq_q8_1_cuda( + Xf.data_ptr(), nullptr, y_q.data_ptr(), type_x, + ne10, /*s01=*/ne10, /*s02=*/ne10 * ne11, /*s03=*/ne10 * ne11, + ne10_padded, ne11, /*ne2=*/1, /*ne3=*/1, stream); + CUDA_CHECK(cudaGetLastError()); + + const int64_t s12 = ne11 * ne10_padded * sizeof(block_q8_1) / (QK8_1 * sizeof(int)); + const mmq_args args = { + (const char *) W.data_ptr(), type_x, (const int *) y_q.data_ptr(), nullptr, nullptr, + dst.data_ptr(), nullptr, + /*ncols_x=*/ne10, /*nrows_x=*/ne01, /*ncols_dst=*/ne11, /*stride_row_x=*/s01, + /*ncols_y=*/ne11, /*nrows_dst=*/ne01, + /*nchannels_x=*/1, /*nchannels_y=*/1, /*stride_channel_x=*/0, /*stride_channel_y=*/s12, /*stride_channel_dst=*/0, + /*nsamples_x=*/1, /*nsamples_y=*/1, /*stride_sample_x=*/0, /*stride_sample_y=*/s12, /*stride_sample_dst=*/0, + /*ncols_max=*/ne11}; + + static ggml_backend_cuda_context ctx(id); + switch (type_x) { + case GGML_TYPE_Q4_K: + mul_mat_q_case(ctx, args, stream); + break; + case GGML_TYPE_Q6_K: + mul_mat_q_case(ctx, args, stream); + break; + default: + GGML_ABORT("unsupported type"); + } + CUDA_CHECK(cudaGetLastError()); + return dst; +} + +// --------------------------------------------------------------------------- +// Torch entry: grouped MoE matmul over flat padded expert slots. +// +// Mirrors upstream's ggml_cuda_mul_mat_q ids branch (mmq.cu): build the +// expert-sorted row maps with mm_ids_helper, quantize each token once and +// scatter to its expert slots (broadcast=true: gate/up, X shared by the +// token's top_k experts) or gather-quantize per routed row (broadcast=false: +// down, X row t*top_k+k belongs to (token t, slot k)), then one mul_mat_q +// launch with expert_bounds. Returns fp32 [tokens*top_k, rows]; row +// t*top_k+k belongs to topk_ids[t][k]. +// --------------------------------------------------------------------------- + +static torch::Tensor ggml_moe_a8_mma( + torch::Tensor X, torch::Tensor W, torch::Tensor topk_ids, int64_t top_k, + int64_t type, int64_t nrows, int64_t tokens, int64_t expert_stride_bytes, + bool broadcast) { + TORCH_CHECK(W.is_cuda() && X.is_cuda() && topk_ids.is_cuda()); + TORCH_CHECK(W.dtype() == torch::kUInt8 && W.dim() == 2 && W.is_contiguous()); + TORCH_CHECK(X.dim() == 2 && X.is_contiguous()); + TORCH_CHECK(topk_ids.dtype() == torch::kInt32 && topk_ids.is_contiguous()); + TORCH_CHECK(topk_ids.dim() == 2 && topk_ids.size(0) == tokens && topk_ids.size(1) == top_k); + TORCH_CHECK(type == GGML_TYPE_Q4_K || type == GGML_TYPE_Q6_K, "only Q4_K/Q6_K instantiated"); + + const ggml_type type_x = (ggml_type) type; + const c10::cuda::CUDAGuard guard(W.device()); + cudaStream_t stream = at::cuda::getCurrentCUDAStream(); + + torch::Tensor Xf = X.scalar_type() == torch::kFloat32 ? X : X.to(torch::kFloat32); + + const int64_t ne02 = W.size(0); // experts (slots) + const int64_t ne01 = nrows; // output features per expert + const int64_t ne10 = X.size(1); // input features + const int64_t ne12 = tokens; + const int64_t n_expert_used = top_k; + TORCH_CHECK(X.size(0) == (broadcast ? tokens : tokens * top_k), "bad activation row count"); + const size_t ts = ggml_type_size(type_x); + const int64_t qk = ggml_blck_size(type_x); + TORCH_CHECK(ne10 % qk == 0); + const int64_t s01 = ne10 / qk; // row stride in blocks + if (expert_stride_bytes == 0) { + expert_stride_bytes = W.size(1); + } + TORCH_CHECK(expert_stride_bytes % (int64_t) ts == 0, + "expert slot stride must be a multiple of the block size"); + TORCH_CHECK(ne01 * s01 * (int64_t) ts <= expert_stride_bytes, "slot smaller than payload"); + const int64_t s02 = expert_stride_bytes / ts; // expert stride in blocks + + const int id = ggml_cuda_get_device(); + const int cc = ggml_cuda_info().devices[id].cc; + const bool fallback = ne01 % 128 != 0; + + auto opts = torch::TensorOptions().device(W.device()); + const int64_t ne_get_rows = ne12 * n_expert_used; + torch::Tensor ids_src1 = torch::empty({ne_get_rows}, opts.dtype(torch::kInt32)); + torch::Tensor ids_dst = torch::empty({ne_get_rows}, opts.dtype(torch::kInt32)); + torch::Tensor expert_bounds = torch::empty({ne02 + 1}, opts.dtype(torch::kInt32)); + + // Broadcast activations (gate/up): each token row is shared by its top_k + // experts -- quantize once and scatter via the inverse map. Per-slot + // activations (down): ids_src1 holds the forward map it*top_k + slot, + // which is exactly the flattened X row -- gather-quantize. + const bool dedup_bcast = broadcast && n_expert_used > 1; + const int nchannels_y = broadcast ? 1 : (int) top_k; + const int sis1 = broadcast ? 1 : (int) top_k; + + ggml_cuda_launch_mm_ids_helper( + (const int32_t *) topk_ids.data_ptr(), (int32_t *) ids_src1.data_ptr(), + (int32_t *) ids_dst.data_ptr(), (int32_t *) expert_bounds.data_ptr(), + ne02, ne12, n_expert_used, nchannels_y, + /*si1=*/(int) top_k, sis1, /*write_inverse=*/dedup_bcast, stream); + CUDA_CHECK(cudaGetLastError()); + + const int64_t ne10_padded = GGML_PAD(ne10, MATRIX_ROW_PADDING); + const size_t nbytes_src1_q8_1 = ne_get_rows * ne10_padded * sizeof(block_q8_1_mmq) / QK8_1_MMQ + + ggml_cuda_mmq_get_J_max(type_x, fallback, cc, /*ne11=*/1) * sizeof(block_q8_1_mmq); + torch::Tensor y_q = torch::empty({(int64_t) nbytes_src1_q8_1}, opts.dtype(torch::kUInt8)); + torch::Tensor dst = torch::empty({ne_get_rows, ne01}, opts.dtype(torch::kFloat32)); + + if (dedup_bcast) { + quantize_scatter_mmq_q8_1_cuda( + Xf.data_ptr(), (const int32_t *) ids_src1.data_ptr(), y_q.data_ptr(), type_x, + ne10, /*stride_token=*/ne10, ne10_padded, ne12, ne_get_rows, n_expert_used, stream); + } else { + // ids_src1[compact] indexes rows of X (stride ne10); ne2 == ne3 == 1. + quantize_mmq_q8_1_cuda( + Xf.data_ptr(), (const int32_t *) ids_src1.data_ptr(), y_q.data_ptr(), type_x, + ne10, /*s01=*/ne10, /*s02=*/0, /*s03=*/0, + ne10_padded, ne_get_rows, /*ne2=*/1, /*ne3=*/1, stream); + } + CUDA_CHECK(cudaGetLastError()); + + // Per-channel strides in the quantized-activation buffer (ne11 == 1). + const int64_t s12 = 1 * ne10_padded * sizeof(block_q8_1) / (QK8_1 * sizeof(int)); + const int64_t s13 = ne12 * s12; + const int64_t s1 = ne01; // dst row stride + const int64_t s2 = n_expert_used * s1; // dst channel (token) stride + const int64_t s3 = ne12 * s2; + + const mmq_args args = { + (const char *) W.data_ptr(), type_x, (const int *) y_q.data_ptr(), + (const int32_t *) ids_dst.data_ptr(), (const int32_t *) expert_bounds.data_ptr(), + dst.data_ptr(), nullptr, + /*ncols_x=*/ne10, /*nrows_x=*/ne01, /*ncols_dst=*/ne_get_rows, /*stride_row_x=*/s01, + /*ncols_y=*/ne_get_rows, /*nrows_dst=*/s1, + /*nchannels_x=*/ne02, /*nchannels_y=*/ne02, /*stride_channel_x=*/s02, /*stride_channel_y=*/s12, /*stride_channel_dst=*/s2, + /*nsamples_x=*/1, /*nsamples_y=*/1, /*stride_sample_x=*/0, /*stride_sample_y=*/s13, /*stride_sample_dst=*/s3, + /*ncols_max=*/ne12}; + + static ggml_backend_cuda_context ctx(id); + switch (type_x) { + case GGML_TYPE_Q4_K: + mul_mat_q_case(ctx, args, stream); + break; + case GGML_TYPE_Q6_K: + mul_mat_q_case(ctx, args, stream); + break; + default: + GGML_ABORT("unsupported type"); + } + CUDA_CHECK(cudaGetLastError()); + return dst; +} + +PYBIND11_MODULE(TORCH_EXTENSION_NAME, m) { + m.def("ggml_mul_mat_a8_mma", &ggml_mul_mat_a8_mma, + "y = x @ dequant(W).T via upstream int8-MMA MMQ (Q4_K/Q6_K)"); + m.def("ggml_moe_a8_mma", &ggml_moe_a8_mma, + "grouped MoE matmul over flat padded expert slots (Q4_K/Q6_K)"); +} diff --git a/python/freetoken/kernel/csrc/gguf_mmq/quantize.cu b/python/freetoken/kernel/csrc/gguf_mmq/quantize.cu new file mode 100644 index 00000000..bcc77239 --- /dev/null +++ b/python/freetoken/kernel/csrc/gguf_mmq/quantize.cu @@ -0,0 +1,697 @@ +#include "quantize.cuh" +#include + +#if defined(BLACKWELL_MMA_AVAILABLE) +// this maps to 256-bit loads in PTX on supported devices, +// and otherwise falls back to 2 128-bit loads +struct __builtin_align__(32) float8 { + float x; float y; float z; float w; + float p; float q; float r; float s; +}; + +#if CUDART_VERSION >= 12080 +static __device__ __forceinline__ float nvfp4_native_scale_error( + const float vals[QK_NVFP4_SUB], const float inv_col_scale, const float inv_scale, const float scale) { + const float scale_dequant = 2.0f * scale; + float err = 0.0f; + +#pragma unroll + for (int k = 0; k < QK_NVFP4_SUB; k += 4) { + const float v0 = vals[k + 0] * inv_col_scale; + const float v1 = vals[k + 1] * inv_col_scale; + const float v2 = vals[k + 2] * inv_col_scale; + const float v3 = vals[k + 3] * inv_col_scale; + + const __nv_fp4x4_e2m1 q(make_float4(v0 * inv_scale, v1 * inv_scale, v2 * inv_scale, v3 * inv_scale)); + const __nv_fp4x4_storage_t q_storage = q.__x; + const __nv_fp4x2_storage_t q_lo = static_cast<__nv_fp4x2_storage_t>(q_storage); + const __nv_fp4x2_storage_t q_hi = static_cast<__nv_fp4x2_storage_t>(q_storage >> 8U); + + const __half2_raw hraw2_lo = __nv_cvt_fp4x2_to_halfraw2(q_lo, __NV_E2M1); + const __half2_raw hraw2_hi = __nv_cvt_fp4x2_to_halfraw2(q_hi, __NV_E2M1); + const __half2 h2_lo = static_cast<__half2>(hraw2_lo); + const __half2 h2_hi = static_cast<__half2>(hraw2_hi); + const float2 dq_lo = __half22float2(h2_lo); + const float2 dq_hi = __half22float2(h2_hi); + + const float err0 = fabsf(v0) - fabsf(dq_lo.x) * scale_dequant; + const float err1 = fabsf(v1) - fabsf(dq_lo.y) * scale_dequant; + const float err2 = fabsf(v2) - fabsf(dq_hi.x) * scale_dequant; + const float err3 = fabsf(v3) - fabsf(dq_hi.y) * scale_dequant; + + err = fmaf(err0, err0, err); + err = fmaf(err1, err1, err); + err = fmaf(err2, err2, err); + err = fmaf(err3, err3, err); + } + + return err; +} +#endif // CUDART_VERSION >= 12080 +#endif // defined(BLACKWELL_MMA_AVAILABLE) + +__launch_bounds__(CUDA_QUANTIZE_BLOCK_SIZE, 1) +static __global__ void quantize_q8_1( + const float * x_ptr, void * vy_ptr, + const int64_t ne00, const int64_t s01, const int64_t s02, const int64_t s03, + const int64_t ne0, const uint32_t ne1, const uint3 ne2) { + ggml_cuda_pdl_lc(); + const float * GGML_CUDA_RESTRICT x = x_ptr; + void * GGML_CUDA_RESTRICT vy = vy_ptr; + const int64_t i0 = (int64_t)blockDim.x*blockIdx.x + threadIdx.x; + + if (i0 >= ne0) { + return; + } + + const int64_t i3 = fastdiv(blockIdx.z, ne2); + const int64_t i2 = blockIdx.z - i3*ne2.z; + const int64_t i1 = blockIdx.y; + + const int64_t & i00 = i0; + const int64_t & i01 = i1; + const int64_t & i02 = i2; + const int64_t & i03 = i3; + + const int64_t i_cont = ((i3*ne2.z + i2) * ne1 + i1) * ne0 + i0; + + block_q8_1 * y = (block_q8_1 *) vy; + + const int64_t ib = i_cont / QK8_1; // block index + const int64_t iqs = i_cont % QK8_1; // quant index + + ggml_cuda_pdl_sync(); + const float xi = i0 < ne00 ? x[i03*s03 + i02*s02 + i01*s01 + i00] : 0.0f; + float amax = fabsf(xi); + float sum = xi; + + amax = warp_reduce_max(amax); + sum = warp_reduce_sum(sum); + + const float d = amax / 127.0f; + const int8_t q = amax == 0.0f ? 0 : roundf(xi / d); + + y[ib].qs[iqs] = q; + + if (iqs > 0) { + return; + } + + y[ib].ds = make_half2(d, sum); +} + +__device__ __forceinline__ uint8_t compute_e8m0_scale(float amax) { + if (!(amax > 0.0f)) { + return 0; + } + + // FP4 E2M1: max exponent (unbiased) is 2. + constexpr int FP4_E2M1_EMAX = 2; + + const float e = log2f(amax); + + // "even" -> round-to-nearest integer, ties-to-even + const int e_int = __float2int_rn(e); + + const int shared_exp = e_int - FP4_E2M1_EMAX; + + int biased = shared_exp + 127; + + biased = max(biased, 0); + biased = min(biased, 254); + + return static_cast(biased); +} + +// scatter: grid over tokens, quantize once, write to all the token's compact rows +template +static __global__ void quantize_mmq_nvfp4( + const float * __restrict__ x, const int32_t * __restrict__ ids, void * __restrict__ vy, float * __restrict__ scale, + const int64_t ne00, const int64_t s01, const int64_t s02, const int64_t s03, + const int64_t ne0, const int64_t ne1, const int64_t ne2, const int n_expert_used) { +#if defined(BLACKWELL_MMA_AVAILABLE) + + const int64_t blocks_per_col = (ne0 + QK_FP4_MMQ - 1) / QK_FP4_MMQ; + + int64_t base_idx; + if constexpr (scatter) { + base_idx = (int64_t) blockIdx.x * s02; // one physical row per token + } else { + const int64_t i2 = blockIdx.y % ne2; + const int64_t i3 = blockIdx.y / ne2; + const int64_t i01 = ids ? ids[blockIdx.x] : blockIdx.x; + base_idx = i3 * s03 + i2 * s02 + i01 * s01; + } + const float * __restrict__ x_row = x + base_idx; + + float amax = 0.0f; + if constexpr (use_aligned_float8) { + for (int64_t i0 = 8 * threadIdx.x; i0 < ne00; i0 += 8 * blockDim.x) { + const float * x_base = x_row + i0; + const float8 v = reinterpret_cast(x_base)[0]; + amax = fmaxf(amax, fabsf(v.x)); + amax = fmaxf(amax, fabsf(v.y)); + amax = fmaxf(amax, fabsf(v.z)); + amax = fmaxf(amax, fabsf(v.w)); + amax = fmaxf(amax, fabsf(v.p)); + amax = fmaxf(amax, fabsf(v.q)); + amax = fmaxf(amax, fabsf(v.r)); + amax = fmaxf(amax, fabsf(v.s)); + } + } else { + for (int64_t i0 = threadIdx.x; i0 < ne00; i0 += blockDim.x) { + amax = fmaxf(amax, fabsf(x_row[i0])); + } + } + + amax = warp_reduce_max(amax); + + __shared__ float warp_amax[CUDA_QUANTIZE_BLOCK_SIZE_MMQ / WARP_SIZE]; + const int lane = threadIdx.x % WARP_SIZE; + const int warp = threadIdx.x / WARP_SIZE; + + if (lane == 0) { + warp_amax[warp] = amax; + } + __syncthreads(); + + if (warp == 0) { + amax = threadIdx.x < int(CUDA_QUANTIZE_BLOCK_SIZE_MMQ / WARP_SIZE) ? warp_amax[lane] : 0.0f; + amax = warp_reduce_max(amax); + if (lane == 0) { + warp_amax[0] = amax / (6.0f * 448.0f); + if constexpr (scatter) { +#pragma unroll + for (int slot = 0; slot < n_expert_used; ++slot) { + const int64_t i = ids[(int64_t) blockIdx.x * n_expert_used + slot]; + scale[i] = warp_amax[0]; + } + } else { + scale[blockIdx.y * ne1 + blockIdx.x] = warp_amax[0]; + } + } + } + __syncthreads(); + + block_fp4_mmq * y = (block_fp4_mmq *) vy; + const int64_t n_subblocks = (ne0 + QK_NVFP4_SUB - 1) / QK_NVFP4_SUB; + + for (int64_t isb = threadIdx.x; isb < n_subblocks; isb += blockDim.x) { + const int64_t i0_base = isb * QK_NVFP4_SUB; + const int64_t k_block = i0_base / QK_FP4_MMQ; + const int sub = (i0_base % QK_FP4_MMQ) / QK_NVFP4_SUB; + + const float row_scale = warp_amax[0]; + const float inv_col_scale = row_scale > 0.0f ? 1.0f / row_scale : 0.0f; + + float vals[QK_NVFP4_SUB]; + if constexpr (use_aligned_float8) { + const float * x_base = x_row + i0_base; + const float8 v0 = i0_base + 7 < ne00 ? reinterpret_cast(x_base)[0] : float8{0.0f, 0.0f, 0.0f, 0.0f, 0.0f, 0.0f, 0.0f, 0.0f}; + const float8 v1 = i0_base + 15 < ne00 ? reinterpret_cast(x_base + 8)[0] : float8{0.0f, 0.0f, 0.0f, 0.0f, 0.0f, 0.0f, 0.0f, 0.0f}; + vals[0] = v0.x; vals[1] = v0.y; vals[2] = v0.z; vals[3] = v0.w; + vals[4] = v0.p; vals[5] = v0.q; vals[6] = v0.r; vals[7] = v0.s; + vals[8] = v1.x; vals[9] = v1.y; vals[10] = v1.z; vals[11] = v1.w; + vals[12] = v1.p; vals[13] = v1.q; vals[14] = v1.r; vals[15] = v1.s; + } else { +#pragma unroll + for (int k = 0; k < QK_NVFP4_SUB; ++k) { + const int64_t i00 = i0_base + k; + vals[k] = i00 < ne00 ? x_row[i00] : 0.0f; + } + } + + uint32_t q0 = 0; + uint32_t q1 = 0; + + float amax_sub = 0.0f; +#pragma unroll + for (int k = 0; k < QK_NVFP4_SUB; ++k) { + amax_sub = fmaxf(amax_sub, fabsf(vals[k] * inv_col_scale)); + } + + static constexpr int test_offsets[5] = { 0, -1, 1, -2, 2 }; + const int first_fp8_code = (int) ggml_cuda_fp32_to_ue4m3(amax_sub / 6.0f); + + uint8_t fp8_code = (uint8_t) first_fp8_code; + float subblock_scale = ggml_cuda_ue4m3_to_fp32(fp8_code); + float inv_scale_err = subblock_scale > 0.0f ? 0.5f / subblock_scale : 0.0f; +#if CUDART_VERSION >= 12080 + float best_err = nvfp4_native_scale_error(vals, inv_col_scale, inv_scale_err, subblock_scale); +#else + float best_err = 0.0f; +#pragma unroll + for (int k = 0; k < QK_NVFP4_SUB; ++k) { + const float v = vals[k] * inv_col_scale; + const uint8_t q = ggml_cuda_float_to_fp4_e2m1(v, inv_scale_err); + const float err_diff = fabsf(v) - fabsf(kvalues_fp4[q & 0x7]) * subblock_scale; + best_err = fmaf(err_diff, err_diff, best_err); + } +#endif // CUDART_VERSION >= 12080 + +#pragma unroll + for (int i = 1; i < 5; ++i) { + const int test_code = first_fp8_code + test_offsets[i]; + if (test_code < 0 || test_code > 0x7e) { + continue; + } + + const float test_scale = ggml_cuda_ue4m3_to_fp32((uint8_t) test_code); + const float test_inv_scale = test_scale > 0.0f ? 0.5f / test_scale : 0.0f; +#if CUDART_VERSION >= 12080 + const float cur_err = nvfp4_native_scale_error(vals, inv_col_scale, test_inv_scale, test_scale); +#else + float cur_err = 0.0f; +#pragma unroll + for (int k = 0; k < QK_NVFP4_SUB; ++k) { + const float v = vals[k] * inv_col_scale; + const uint8_t q = ggml_cuda_float_to_fp4_e2m1(v, test_inv_scale); + const float err_diff = fabsf(v) - fabsf(kvalues_fp4[q & 0x7]) * test_scale; + cur_err = fmaf(err_diff, err_diff, cur_err); + } +#endif // CUDART_VERSION >= 12080 + + if (cur_err < best_err) { + best_err = cur_err; + fp8_code = (uint8_t) test_code; + subblock_scale = test_scale; + } + } +#if CUDART_VERSION >= 12080 + const float inv_scale = subblock_scale > 0.0f ? 0.5f / subblock_scale : 0.0f; + const float s = inv_col_scale * inv_scale; + + __nv_fp4x4_e2m1 q0_lo(make_float4(vals[0] * s, vals[8] * s, vals[1] * s, vals[9] * s)); + __nv_fp4x4_e2m1 q0_hi(make_float4(vals[2] * s, vals[10] * s, vals[3] * s, vals[11] * s)); + __nv_fp4x4_e2m1 q1_lo(make_float4(vals[4] * s, vals[12] * s, vals[5] * s, vals[13] * s)); + __nv_fp4x4_e2m1 q1_hi(make_float4(vals[6] * s, vals[14] * s, vals[7] * s, vals[15] * s)); + + const char2 q0_lo_c = *reinterpret_cast(&q0_lo); + const char2 q0_hi_c = *reinterpret_cast(&q0_hi); + const char2 q1_lo_c = *reinterpret_cast(&q1_lo); + const char2 q1_hi_c = *reinterpret_cast(&q1_hi); + + q0 = uint32_t(uint8_t(q0_lo_c.x)) | (uint32_t(uint8_t(q0_lo_c.y)) << 8) | + (uint32_t(uint8_t(q0_hi_c.x)) << 16) | (uint32_t(uint8_t(q0_hi_c.y)) << 24); + q1 = uint32_t(uint8_t(q1_lo_c.x)) | (uint32_t(uint8_t(q1_lo_c.y)) << 8) | + (uint32_t(uint8_t(q1_hi_c.x)) << 16) | (uint32_t(uint8_t(q1_hi_c.y)) << 24); +#else + const float inv_scale = subblock_scale > 0.0f ? 0.5f / subblock_scale : 0.0f; +#pragma unroll + for (int k = 0; k < QK_NVFP4_SUB / 4; ++k) { + q0 |= uint32_t(ggml_cuda_float_to_fp4_e2m1(vals[k + 0] * inv_col_scale, inv_scale)) << (8 * k); + q0 |= uint32_t(ggml_cuda_float_to_fp4_e2m1(vals[k + 8] * inv_col_scale, inv_scale)) << (8 * k + 4); + q1 |= uint32_t(ggml_cuda_float_to_fp4_e2m1(vals[k + 4] * inv_col_scale, inv_scale)) << (8 * k); + q1 |= uint32_t(ggml_cuda_float_to_fp4_e2m1(vals[k + 12] * inv_col_scale, inv_scale)) << (8 * k + 4); + } +#endif // CUDART_VERSION >= 12080 + + if constexpr (scatter) { +#pragma unroll + for (int slot = 0; slot < n_expert_used; ++slot) { + const int64_t i = ids[(int64_t) blockIdx.x * n_expert_used + slot]; + block_fp4_mmq * yb = y + (k_block * ne1 + i); + uint32_t * yqs = reinterpret_cast(yb->qs); + yqs[2 * sub + 0] = q0; + yqs[2 * sub + 1] = q1; + reinterpret_cast(yb->d4)[sub] = fp8_code; + } + } else { + block_fp4_mmq * yb = y + (blockIdx.y * ((int64_t) blocks_per_col * ne1) + k_block * ne1 + blockIdx.x); + uint32_t * yqs = reinterpret_cast(yb->qs); + yqs[2 * sub + 0] = q0; + yqs[2 * sub + 1] = q1; + reinterpret_cast(yb->d4)[sub] = fp8_code; + } + } +#else + GGML_UNUSED_VARS(x, ids, vy, scale, ne00, s01, s02, s03, ne0, ne1, ne2, n_expert_used); + NO_DEVICE_CODE; // This is for Blackwell NVFP4 activations only. +#endif // defined(BLACKWELL_MMA_AVAILABLE) + +} + +// quantize values in the format mxfp4 is stored which is interleaved nibbles +// i.e. a block a0-a31 is represented as a0a16,a1a17 ...a15a31 +// scatter: grid over tokens, quantize once, write to all the token's compact rows +template +static __global__ void quantize_mmq_mxfp4(const float * __restrict__ x, + const int32_t * __restrict__ ids, + void * __restrict__ vy, + const int64_t ne00, + const int64_t s01, + const int64_t s02, + const int64_t s03, + const int64_t ne0, + const int ne1, + const int ne2, + const int n_expert_used) { + constexpr int vals_per_scale = 32; + constexpr int vals_per_warp = 2 * vals_per_scale; // Each warp processes 2 blocks of 32 = 64 values + + const int warp_id = threadIdx.y; + const int lane_id_32 = threadIdx.x; + + const int nwarps = blockDim.y; + + const int64_t warp_start_offset = (blockIdx.y * nwarps + warp_id) * vals_per_warp; + + if (warp_start_offset >= ne0) { + return; + } + + const int64_t block_fp4_mmq_size = QK_FP4_MMQ; + const int64_t k_block = warp_start_offset / block_fp4_mmq_size; + const int64_t quad_idx_in_block = (warp_start_offset % block_fp4_mmq_size) / vals_per_warp; + + const int group_id = lane_id_32 / 4; + const int lane_in_group = lane_id_32 % 4; + const int base = group_id * 2; + + ggml_cuda_pdl_sync(); + int64_t base_pos; + if constexpr (scatter) { + base_pos = (int64_t) blockIdx.x * s02; // one physical row per token + } else { + const int64_t i2 = blockIdx.z % ne2; + const int64_t i3 = blockIdx.z / ne2; + const int64_t i01 = ids ? ids[blockIdx.x] : blockIdx.x; + base_pos = i3 * s03 + i2 * s02 + i01 * s01; + } + + uint8_t scales[2]; + char2 packed[2]; + +#pragma unroll + for (int b = 0; b < 2; ++b) { + const int64_t i0 = warp_start_offset + b * vals_per_scale + lane_id_32; + const float xi = (i0 < ne00) ? x[base_pos + i0] : 0.0f; + + float amax = fabsf(xi); +#pragma unroll + for (int mask = 16; mask > 0; mask >>= 1) { + amax = fmaxf(amax, __shfl_xor_sync(0xFFFFFFFF, amax, mask, WARP_SIZE)); + } + + const uint8_t e = compute_e8m0_scale(amax); + scales[b] = e; + const float inv_s = (amax == 0.0f) ? 0.0f : __frcp_rn(ggml_cuda_e8m0_to_fp32(e)); + +#if CUDART_VERSION >= 12080 + const float scaled_val = xi * inv_s; + + const float val0 = __shfl_sync(0xFFFFFFFF, scaled_val, base, WARP_SIZE); + const float val1 = __shfl_sync(0xFFFFFFFF, scaled_val, base + 16, WARP_SIZE); + const float val2 = __shfl_sync(0xFFFFFFFF, scaled_val, base + 1, WARP_SIZE); + const float val3 = __shfl_sync(0xFFFFFFFF, scaled_val, base + 17, WARP_SIZE); + + __nv_fp4x4_e2m1 fp4_packed(make_float4(val0, val1, val2, val3)); + packed[b] = *(char2 *) &fp4_packed; +#else + // Fallback: manual FP4 conversion using LUT + const uint8_t q_val = ggml_cuda_float_to_fp4_e2m1(xi, inv_s); + + const uint8_t q_lo_0 = __shfl_sync(0xFFFFFFFF, q_val, base, WARP_SIZE); + const uint8_t q_lo_1 = __shfl_sync(0xFFFFFFFF, q_val, base + 1, WARP_SIZE); + const uint8_t q_hi_0 = __shfl_sync(0xFFFFFFFF, q_val, base + 16, WARP_SIZE); + const uint8_t q_hi_1 = __shfl_sync(0xFFFFFFFF, q_val, base + 17, WARP_SIZE); + + char2 q; + q.x = (q_hi_0 << 4) | q_lo_0; + q.y = (q_hi_1 << 4) | q_lo_1; + packed[b] = q; +#endif // CUDART_VERSION >= 12080 + } + + block_fp4_mmq * y = (block_fp4_mmq *) vy; + if constexpr (scatter) { +#pragma unroll + for (int slot = 0; slot < n_expert_used; ++slot) { + const int64_t i = ids[(int64_t) blockIdx.x * n_expert_used + slot]; + block_fp4_mmq * yb = y + (k_block * ne1 + i); + char2 * yqs2 = (char2 *) yb->qs; + if (lane_in_group == 0) { + yqs2[quad_idx_in_block * 16 + 0 * 8 + group_id] = packed[0]; + yqs2[quad_idx_in_block * 16 + 1 * 8 + group_id] = packed[1]; + } + if (lane_id_32 == 0) { + yb->d4[quad_idx_in_block] = (scales[1] << 8) | scales[0]; + } + } + } else { + const int64_t ib0 = blockIdx.z * ((int64_t) ne1 * (ne0 / block_fp4_mmq_size)); + block_fp4_mmq * yb = y + (ib0 + k_block * ne1 + blockIdx.x); + char2 * yqs2 = (char2 *) yb->qs; + if (lane_in_group == 0) { + yqs2[quad_idx_in_block * 16 + 0 * 8 + group_id] = packed[0]; + yqs2[quad_idx_in_block * 16 + 1 * 8 + group_id] = packed[1]; + } + if (lane_id_32 == 0) { + yb->d4[quad_idx_in_block] = (scales[1] << 8) | scales[0]; + } + } + GGML_UNUSED(n_expert_used); +} + +// scatter: grid over tokens, quantize once, write to all the token's compact rows +template +static __global__ void quantize_mmq_q8_1( + const float * __restrict__ x, const int32_t * __restrict__ ids, void * __restrict__ vy, + const int64_t ne00, const int64_t s01, const int64_t s02, const int64_t s03, + const int64_t ne0, const int ne1, const int ne2, const int n_expert_used) { + + constexpr int vals_per_scale = ds_layout == MMQ_Q8_1_DS_LAYOUT_D2S6 ? 64 : 32; + constexpr int vals_per_sum = ds_layout == MMQ_Q8_1_DS_LAYOUT_D2S6 ? 16 : 32; + + const int64_t i0 = ((int64_t)blockDim.x*blockIdx.y + threadIdx.x)*4; + + if (i0 >= ne0) { + return; + } + + const int64_t i00 = i0; + ggml_cuda_pdl_sync(); + + int64_t base_idx; + if constexpr (scatter) { + base_idx = (int64_t) blockIdx.x * s02; // one physical row per token + } else { + const int64_t i2 = blockIdx.z % ne2; + const int64_t i3 = blockIdx.z / ne2; + const int64_t i01 = ids ? ids[blockIdx.x] : blockIdx.x; + base_idx = i3*s03 + i2*s02 + i01*s01; + } + + const float4 * x4 = (const float4 *) x; + block_q8_1_mmq * y = (block_q8_1_mmq *) vy; + + const int64_t k_block = i0 / QK8_1_MMQ; // column block in the channel + const int64_t iqs = i0 % QK8_1_MMQ; // quant index in block + + // Load 4 floats per thread and calculate max. abs. value between them: + const float4 xi = i0 < ne00 ? x4[(base_idx + i00)/4] : make_float4(0.0f, 0.0f, 0.0f, 0.0f); + float amax = fabsf(xi.x); + amax = fmaxf(amax, fabsf(xi.y)); + amax = fmaxf(amax, fabsf(xi.z)); + amax = fmaxf(amax, fabsf(xi.w)); + + // Exchange max. abs. value between vals_per_scale/4 threads. +#pragma unroll + for (int offset = vals_per_scale/8; offset > 0; offset >>= 1) { + amax = fmaxf(amax, __shfl_xor_sync(0xFFFFFFFF, amax, offset, WARP_SIZE)); + } + + float sum; + if (ds_layout != MMQ_Q8_1_DS_LAYOUT_D4) { + sum = xi.x + xi.y + xi.z + xi.w; + + // Calculate sums across vals_per_sum/4 threads. +#pragma unroll + for (int offset = vals_per_sum/8; offset > 0; offset >>= 1) { + sum += __shfl_xor_sync(0xFFFFFFFF, sum, offset, WARP_SIZE); + } + } + + const float d_inv = 127.0f / amax; + char4 q; + q.x = roundf(xi.x*d_inv); + q.y = roundf(xi.y*d_inv); + q.z = roundf(xi.z*d_inv); + q.w = roundf(xi.w*d_inv); + const float d = 1.0f / d_inv; + + // write the block once (normal) or to each of the token's compact rows (scatter) + const int nwrite = scatter ? n_expert_used : 1; +#pragma unroll + for (int slot = 0; slot < nwrite; ++slot) { + int64_t ib; + if constexpr (scatter) { + const int64_t i = ids[(int64_t) blockIdx.x * n_expert_used + slot]; + ib = k_block*ne1 + i; + } else { + const int64_t ib0 = blockIdx.z*((int64_t)gridDim.x*gridDim.y*blockDim.x/QK8_1); // first block of channel + ib = ib0 + k_block*ne1 + blockIdx.x; + } + + // Write back 4 int8 values as a single 32 bit value for better memory bandwidth: + char4 * yqs4 = (char4 *) y[ib].qs; + yqs4[iqs/4] = q; + + if (ds_layout == MMQ_Q8_1_DS_LAYOUT_D2S6) { + if (iqs % 16 == 0 && iqs < 96) { + y[ib].d2s6[2 + iqs/16] = sum; + if (iqs % 64 == 0) { + y[ib].d2s6[iqs/64] = d; + } + } + } else if (iqs % 32 == 0) { + if (ds_layout == MMQ_Q8_1_DS_LAYOUT_DS4) { + y[ib].ds4[iqs/32] = make_half2(d, sum); + } else { + y[ib].d4[iqs/32] = d; + } + } + } + GGML_UNUSED(n_expert_used); +} + +void quantize_row_q8_1_cuda( + const float * x, const int32_t * ids, void * vy, const ggml_type type_src0, + const int64_t ne00, const int64_t s01, const int64_t s02, const int64_t s03, + const int64_t ne0, const int64_t ne1, const int64_t ne2, const int64_t ne3, cudaStream_t stream) { + GGML_ASSERT(!ids); + GGML_ASSERT(ne0 % QK8_1 == 0); + + const uint3 ne2_fastdiv = init_fastdiv_values(ne2); + + const int64_t block_num_x = (ne0 + CUDA_QUANTIZE_BLOCK_SIZE - 1) / CUDA_QUANTIZE_BLOCK_SIZE; + const dim3 num_blocks(block_num_x, ne1, ne2*ne3); + const dim3 block_size(CUDA_QUANTIZE_BLOCK_SIZE, 1, 1); + const ggml_cuda_kernel_launch_params launch_params = ggml_cuda_kernel_launch_params(num_blocks, block_size, 0, stream); + ggml_cuda_kernel_launch(quantize_q8_1, launch_params, x, vy, ne00, s01, s02, s03, ne0, ne1, ne2_fastdiv); + GGML_UNUSED(type_src0); +} + +void quantize_mmq_q8_1_cuda( + const float * x, const int32_t * ids, void * vy, const ggml_type type_src0, + const int64_t ne00, const int64_t s01, const int64_t s02, const int64_t s03, + const int64_t ne0, const int64_t ne1, const int64_t ne2, const int64_t ne3, cudaStream_t stream) { + GGML_ASSERT(ne00 % 4 == 0); + GGML_ASSERT(ne0 % QK8_1_MMQ == 0); + + // ne1 tends to assume the highest values, therefore use it as the "x" dimension of the CUDA grid: + const int64_t block_num_y = (ne0 + 4*CUDA_QUANTIZE_BLOCK_SIZE_MMQ - 1) / (4*CUDA_QUANTIZE_BLOCK_SIZE_MMQ); + const dim3 num_blocks(ne1, block_num_y, ne2*ne3); + const dim3 block_size(CUDA_QUANTIZE_BLOCK_SIZE_MMQ, 1, 1); + switch (mmq_get_q8_1_ds_layout(type_src0)) { + case MMQ_Q8_1_DS_LAYOUT_D4: + quantize_mmq_q8_1 + <<>>(x, ids, vy, ne00, s01, s02, s03, ne0, ne1, ne2, /*n_expert_used=*/0); + break; + case MMQ_Q8_1_DS_LAYOUT_DS4: + quantize_mmq_q8_1 + <<>>(x, ids, vy, ne00, s01, s02, s03, ne0, ne1, ne2, /*n_expert_used=*/0); + break; + case MMQ_Q8_1_DS_LAYOUT_D2S6: + quantize_mmq_q8_1 + <<>>(x, ids, vy, ne00, s01, s02, s03, ne0, ne1, ne2, /*n_expert_used=*/0); + break; + default: + GGML_ABORT("fatal error"); + break; + } +} + +// scatter=true reuses the quant kernel: grid over tokens, ids = inverse map (token slot -> compact row) +void quantize_scatter_mmq_q8_1_cuda( + const float * x, const int32_t * ids_src1_inv, void * vy, const ggml_type type_src0, + const int64_t ne00, const int64_t stride_token, const int64_t ne0, + const int64_t n_tokens, const int64_t nrows_dst, const int n_expert_used, cudaStream_t stream) { + GGML_ASSERT(ne00 % 4 == 0); + GGML_ASSERT(ne0 % QK8_1_MMQ == 0); + + const int64_t block_num_y = (ne0 + 4*CUDA_QUANTIZE_BLOCK_SIZE_MMQ - 1) / (4*CUDA_QUANTIZE_BLOCK_SIZE_MMQ); + const dim3 num_blocks(n_tokens, block_num_y, 1); + const dim3 block_size(CUDA_QUANTIZE_BLOCK_SIZE_MMQ, 1, 1); + switch (mmq_get_q8_1_ds_layout(type_src0)) { + case MMQ_Q8_1_DS_LAYOUT_D4: + quantize_mmq_q8_1<<>>( + x, ids_src1_inv, vy, ne00, /*s01=*/0, /*s02=*/stride_token, /*s03=*/0, ne0, /*ne1=*/(int) nrows_dst, /*ne2=*/1, n_expert_used); + break; + case MMQ_Q8_1_DS_LAYOUT_DS4: + quantize_mmq_q8_1<<>>( + x, ids_src1_inv, vy, ne00, /*s01=*/0, /*s02=*/stride_token, /*s03=*/0, ne0, /*ne1=*/(int) nrows_dst, /*ne2=*/1, n_expert_used); + break; + case MMQ_Q8_1_DS_LAYOUT_D2S6: + quantize_mmq_q8_1<<>>( + x, ids_src1_inv, vy, ne00, /*s01=*/0, /*s02=*/stride_token, /*s03=*/0, ne0, /*ne1=*/(int) nrows_dst, /*ne2=*/1, n_expert_used); + break; + default: + GGML_ABORT("fatal error"); + break; + } +} + +// scatter=true reuses the quant kernels: grid over tokens, ids = inverse map (token slot -> compact row) +void quantize_scatter_mmq_fp4_cuda( + const float * x, const int32_t * ids_src1_inv, void * vy, float * scale, const ggml_type type_src0, const bool use_aligned_float8, + const int64_t ne00, const int64_t stride_token, const int64_t ne0, + const int64_t n_tokens, const int64_t nrows_dst, const int n_expert_used, cudaStream_t stream) { + GGML_ASSERT(ne0 > 0); + if (type_src0 == GGML_TYPE_NVFP4) { + GGML_ASSERT(scale); + GGML_ASSERT(ne00 % QK_NVFP4 == 0); + const dim3 block_size(CUDA_QUANTIZE_BLOCK_SIZE_MMQ, 1, 1); + const dim3 num_blocks(n_tokens, 1, 1); + if (use_aligned_float8) { + quantize_mmq_nvfp4<<>>( + x, ids_src1_inv, vy, scale, ne00, /*s01=*/0, /*s02=*/stride_token, /*s03=*/0, ne0, /*ne1=*/nrows_dst, /*ne2=*/1, n_expert_used); + } else { + quantize_mmq_nvfp4<<>>( + x, ids_src1_inv, vy, scale, ne00, /*s01=*/0, /*s02=*/stride_token, /*s03=*/0, ne0, /*ne1=*/nrows_dst, /*ne2=*/1, n_expert_used); + } + } else { + GGML_ASSERT(type_src0 == GGML_TYPE_MXFP4); + constexpr int nwarps = 8; + constexpr int vals_per_block = nwarps * 2 * QK_MXFP4; + const int64_t block_num_y = (ne0 + vals_per_block - 1) / vals_per_block; + const dim3 block_size(WARP_SIZE, nwarps, 1); + const dim3 num_blocks(n_tokens, block_num_y, 1); + quantize_mmq_mxfp4<<>>( + x, ids_src1_inv, vy, ne00, /*s01=*/0, /*s02=*/stride_token, /*s03=*/0, ne0, /*ne1=*/(int) nrows_dst, /*ne2=*/1, n_expert_used); + } +} + +void quantize_mmq_fp4_cuda( + const float * x, const int32_t * ids, void * vy, float * scale, const ggml_type type_src0, const bool use_aligned_float8, + const int64_t ne00, const int64_t s01, const int64_t s02, const int64_t s03, + const int64_t ne0, const int64_t ne1, const int64_t ne2, const int64_t ne3, cudaStream_t stream) { + GGML_ASSERT(type_src0 == GGML_TYPE_MXFP4 || type_src0 == GGML_TYPE_NVFP4); + GGML_ASSERT(ne0 > 0); + + if (type_src0 == GGML_TYPE_NVFP4) { + GGML_ASSERT(scale); + GGML_ASSERT(ne00 % QK_NVFP4 == 0); + const dim3 block_size(CUDA_QUANTIZE_BLOCK_SIZE_MMQ, 1, 1); + const dim3 num_blocks(ne1, ne2 * ne3, 1); + if (use_aligned_float8) { + quantize_mmq_nvfp4<<>>( + x, ids, vy, scale, ne00, s01, s02, s03, ne0, ne1, ne2, /*n_expert_used=*/0); + } else { + quantize_mmq_nvfp4<<>>( + x, ids, vy, scale, ne00, s01, s02, s03, ne0, ne1, ne2, /*n_expert_used=*/0); + } + } else { + GGML_ASSERT(ne0 % (2 * QK_MXFP4) == 0); + + constexpr int nwarps = 8; + constexpr int vals_per_warp = 2 * QK_MXFP4; + constexpr int vals_per_block = nwarps * vals_per_warp; + + const int64_t block_num_y = (ne0 + vals_per_block - 1) / vals_per_block; + const dim3 num_blocks(ne1, block_num_y, ne2 * ne3); + const dim3 block_size(WARP_SIZE, nwarps, 1); + + quantize_mmq_mxfp4<<>>(x, ids, vy, ne00, s01, s02, s03, ne0, ne1, ne2, /*n_expert_used=*/0); + } +} diff --git a/python/freetoken/kernel/csrc/gguf_mmq/quantize.cuh b/python/freetoken/kernel/csrc/gguf_mmq/quantize.cuh new file mode 100644 index 00000000..5f08dcbf --- /dev/null +++ b/python/freetoken/kernel/csrc/gguf_mmq/quantize.cuh @@ -0,0 +1,70 @@ +#pragma once + +#include "common.cuh" +#include "mmq.cuh" + +#include + +#define CUDA_QUANTIZE_BLOCK_SIZE 256 +#define CUDA_QUANTIZE_BLOCK_SIZE_MMQ 128 + +static_assert(MATRIX_ROW_PADDING % CUDA_QUANTIZE_BLOCK_SIZE == 0, "Risk of out-of-bounds access."); +static_assert(MATRIX_ROW_PADDING % (4*CUDA_QUANTIZE_BLOCK_SIZE_MMQ) == 0, "Risk of out-of-bounds access."); + +typedef void (*quantize_cuda_t)( + const float * x, const int32_t * ids, void * vy, + ggml_type type_src0, int64_t ne00, int64_t s01, int64_t s02, int64_t s03, + int64_t ne0, int64_t ne1, int64_t ne2, int64_t ne3, cudaStream_t stream); + +void quantize_row_q8_1_cuda( + const float * x, const int32_t * ids, void * vy, + ggml_type type_src0, int64_t ne00, int64_t s01, int64_t s02, int64_t s03, + int64_t ne0, int64_t ne1, int64_t ne2, int64_t ne3, cudaStream_t stream); + +void quantize_mmq_q8_1_cuda( + const float * x, const int32_t * ids, void * vy, + ggml_type type_src0, int64_t ne00, int64_t s01, int64_t s02, int64_t s03, + int64_t ne0, int64_t ne1, int64_t ne2, int64_t ne3, cudaStream_t stream); + +void quantize_mmq_fp4_cuda(const float * x, + const int32_t * ids, + void * vy, + float * scale, + ggml_type type_src0, + bool use_aligned_float8, + int64_t ne00, + int64_t s01, + int64_t s02, + int64_t s03, + int64_t ne0, + int64_t ne1, + int64_t ne2, + int64_t ne3, + cudaStream_t stream); + +// quantize each token once and scatter the block to its compact rows (via the inverse map) +void quantize_scatter_mmq_fp4_cuda(const float * x, + const int32_t * ids_src1_inv, + void * vy, + float * scale, + ggml_type type_src0, + bool use_aligned_float8, + int64_t ne00, + int64_t stride_token, + int64_t ne0, + int64_t n_tokens, + int64_t nrows_dst, + int n_expert_used, + cudaStream_t stream); + +void quantize_scatter_mmq_q8_1_cuda(const float * x, + const int32_t * ids_src1_inv, + void * vy, + ggml_type type_src0, + int64_t ne00, + int64_t stride_token, + int64_t ne0, + int64_t n_tokens, + int64_t nrows_dst, + int n_expert_used, + cudaStream_t stream); diff --git a/python/freetoken/kernel/csrc/gguf_mmq/vecdotq.cuh b/python/freetoken/kernel/csrc/gguf_mmq/vecdotq.cuh new file mode 100644 index 00000000..0f039c73 --- /dev/null +++ b/python/freetoken/kernel/csrc/gguf_mmq/vecdotq.cuh @@ -0,0 +1,1363 @@ +#pragma once + +#include "common.cuh" + +#include + +static __device__ __forceinline__ int get_int_b1(const void * x, const int & i32) { + const uint8_t * x8 = (const uint8_t *) x; + + int x32 = x8[4*i32 + 0] << 0; + x32 |= x8[4*i32 + 1] << 8; + x32 |= x8[4*i32 + 2] << 16; + x32 |= x8[4*i32 + 3] << 24; + + return x32; +} + +static __device__ __forceinline__ int get_int_b2(const void * x, const int & i32) { + const uint16_t * x16 = (const uint16_t *) x; // assume at least 2 byte alignment + + int x32 = x16[2*i32 + 0] << 0; + x32 |= x16[2*i32 + 1] << 16; + + return x32; +} + +static __device__ __forceinline__ int get_int_b4(const void * x, const int & i32) { + return ((const int *) x)[i32]; // assume at least 4 byte alignment +} + +// q4 contains 8 indices with 4 bit each. +// This function selects those bytes from table that are at those indices and returns them as int2. +// The first int contains the bytes with even indices in q4, the second int contains the bytes with odd indices in q4. +static __device__ __forceinline__ int2 get_int_from_table_16(const int & q4, const int8_t * table) { +#if defined(GGML_USE_HIP) + // Load the 16-byte table into four 32-bit unsigned integers. + const uint32_t *values = (const uint32_t *)table; + + const uint32_t q_even = q4; + const uint32_t q_odd = (q4 >> 4); + + // Perform lookups in the lower half of the table (indices 0-7). + uint32_t v_even_low = __builtin_amdgcn_perm(values[1], values[0], q_even & 0x07070707); + uint32_t v_odd_low = __builtin_amdgcn_perm(values[1], values[0], q_odd & 0x07070707); + + // Perform lookups in the upper half of the table (indices 8-15). + uint32_t v_even_high = __builtin_amdgcn_perm(values[3], values[2], q_even & 0x07070707); + uint32_t v_odd_high = __builtin_amdgcn_perm(values[3], values[2], q_odd & 0x07070707); + + // Select between the low and high results based on the MSB of each index nibble. + uint32_t mask_even = 0x03020100 | ((q_even & 0x08080808) >> 1); + uint32_t res_x = __builtin_amdgcn_perm(v_even_high, v_even_low, mask_even); + uint32_t mask_odd = 0x03020100 | ((q_odd & 0x08080808) >> 1); + uint32_t res_y = __builtin_amdgcn_perm(v_odd_high, v_odd_low, mask_odd); + + return make_int2(res_x, res_y); +#elif !defined(GGML_USE_MUSA) + // CUDA does not have an instruction for selecting bytes with 4 bit indices. + // However, __byte_perm is an instruction that selects bytes with 3 bit indices that can be used instead. + const uint32_t * table32 = (const uint32_t *) table; + + // __byte_perm selects bytes based on the lower 16 bits in its third argument. + // Therefore, do 2 iterations over the 32 bits in q4 with 0 and 16 shift. + // To handle the fourth bit, first call _byte_perm both for the low and the high 64 bit of table, using the low 3 bits. + // Then, call __byte_perm again to select from the low and high bytes based on the fourth bit. + uint32_t tmp[2]; + const uint32_t low_high_selection_indices = (0x32103210 | ((q4 & 0x88888888) >> 1)); +#pragma unroll + for (uint32_t i = 0; i < 2; ++i) { + const uint32_t shift = 16 * i; + + const uint32_t low = __byte_perm(table32[0], table32[1], q4 >> shift); + const uint32_t high = __byte_perm(table32[2], table32[3], q4 >> shift); + tmp[i] = __byte_perm(low, high, low_high_selection_indices >> shift); + } + + // tmp contains the bytes from tyble in the same order as the 4 bit indices in q4. + // However, for the result we need ints with all even/odd 4 bit indices in q4. + // Therefore, 2 more calls to __byte_perm to put the bytes in the correct order. + return make_int2(__byte_perm(tmp[0], tmp[1], 0x6420), __byte_perm(tmp[0], tmp[1], 0x7531)); +#else + // Generic implementation. + const int q0_32 = (q4 >> 0) & 0x0F0F0F0F; + const int8_t * q0_8 = (const int8_t *) &q0_32; + const char4 val0_8 = make_char4( + table[q0_8[0]], table[q0_8[1]], table[q0_8[2]], table[q0_8[3]]); + + const int q1_32 = (q4 >> 4) & 0x0F0F0F0F; + const int8_t * q1_8 = (const int8_t *) &q1_32; + const char4 val1_8 = make_char4( + table[q1_8[0]], table[q1_8[1]], table[q1_8[2]], table[q1_8[3]]); + + return make_int2(*((const int *) &val0_8), *((const int *) &val1_8)); +#endif +} + +static __device__ __forceinline__ uint32_t unpack_ksigns(const uint8_t v) { + // v is a 7 bit int, with the 8th sign being encodable as popcnt + // with xor we can "correct" the bit instead of having to mask + const uint32_t p = __popc(v) & 1; + const uint32_t s = v ^ p << 7; + // broadcast over uint to allow for 0x08040201 / 0x80402010 as selectors + return s * 0x01010101; +} + +// VDR = vec dot ratio, how many contiguous integers each thread processes when the vec dot kernel is called +// MMVQ = mul_mat_vec_q, MMQ = mul_mat_q + +#define VDR_Q1_0_Q8_1_MMVQ 1 // Process one 32-element chunk at a time for parallelism +#define VDR_Q1_0_Q8_1_MMQ 4 // Q1_0 has 128 bits (4 ints) per block + +#define VDR_Q2_0_Q8_1_MMVQ 1 // Process one 32-element chunk at a time for parallelism +#define VDR_Q2_0_Q8_1_MMQ 2 // Q2_0 group 64: 128 bits (4 ints) per block, 2 32-element chunks + +#define VDR_Q4_0_Q8_1_MMVQ 2 +#define VDR_Q4_0_Q8_1_MMQ 4 + +template static __device__ __forceinline__ float vec_dot_q4_0_q8_1_impl( + const int * v, const int * u, const float & d4, const half2 & ds8) { + + int sumi = 0; + +#pragma unroll + for (int i = 0; i < vdr; ++i) { + const int vi0 = (v[i] >> 0) & 0x0F0F0F0F; + const int vi1 = (v[i] >> 4) & 0x0F0F0F0F; + + // SIMD dot product of quantized values + sumi = ggml_cuda_dp4a(vi0, u[2*i+0], sumi); + sumi = ggml_cuda_dp4a(vi1, u[2*i+1], sumi); + } + + const float2 ds8f = __half22float2(ds8); + + // second part effectively subtracts 8 from each quant value + return d4 * (sumi * ds8f.x - (8*vdr/QI4_0) * ds8f.y); +} + +#define VDR_Q4_1_Q8_1_MMVQ 2 +#define VDR_Q4_1_Q8_1_MMQ 4 + +template static __device__ __forceinline__ float vec_dot_q4_1_q8_1_impl( + const int * v, const int * u, const half2 & dm4, const half2 & ds8) { + + int sumi = 0; + +#pragma unroll + for (int i = 0; i < vdr; ++i) { + const int vi0 = (v[i] >> 0) & 0x0F0F0F0F; + const int vi1 = (v[i] >> 4) & 0x0F0F0F0F; + + // SIMD dot product of quantized values + sumi = ggml_cuda_dp4a(vi0, u[2*i+0], sumi); + sumi = ggml_cuda_dp4a(vi1, u[2*i+1], sumi); + } + +#ifdef FAST_FP16_AVAILABLE + const float2 tmp = __half22float2(__hmul2(dm4, ds8)); + const float d4d8 = tmp.x; + const float m4s8 = tmp.y; +#else + const float2 dm4f = __half22float2(dm4); + const float2 ds8f = __half22float2(ds8); + const float d4d8 = dm4f.x * ds8f.x; + const float m4s8 = dm4f.y * ds8f.y; +#endif // FAST_FP16_AVAILABLE + + // scale second part of sum by QI8_1/(vdr * QR4_1) to compensate for multiple threads adding it + return sumi * d4d8 + m4s8 / (QI8_1 / (vdr * QR4_1)); +} + +#define VDR_Q5_0_Q8_1_MMVQ 2 +#define VDR_Q5_0_Q8_1_MMQ 4 + +template static __device__ __forceinline__ float vec_dot_q5_0_q8_1_impl( + const int * vl, const int * vh, const int * u, const float & d5, const half2 & ds8) { + + int sumi = 0; + +#pragma unroll + for (int i = 0; i < vdr; ++i) { + int vi0 = (vl[i] >> 0) & 0x0F0F0F0F; // lower 4 qs bits, still need qh as 5th bits + vi0 |= (vh[i] << 4) & 0x00000010; // 0 -> 4 + vi0 |= (vh[i] << 11) & 0x00001000; // 1 -> 12 + vi0 |= (vh[i] << 18) & 0x00100000; // 2 -> 20 + vi0 |= (vh[i] << 25) & 0x10000000; // 3 -> 28 + sumi = ggml_cuda_dp4a(vi0, u[2*i+0], sumi); // SIMD dot product of quantized values + + int vi1 = (vl[i] >> 4) & 0x0F0F0F0F; // upper 4 qs bits, still need qh as 5th bits + vi1 |= (vh[i] >> 12) & 0x00000010; // 16 -> 4 + vi1 |= (vh[i] >> 5) & 0x00001000; // 17 -> 12 + vi1 |= (vh[i] << 2) & 0x00100000; // 18 -> 20 + vi1 |= (vh[i] << 9) & 0x10000000; // 19 -> 28 + sumi = ggml_cuda_dp4a(vi1, u[2*i+1], sumi); // SIMD dot product of quantized values + } + + const float2 ds8f = __half22float2(ds8); + + // second part effectively subtracts 16 from each quant value + return d5 * (sumi * ds8f.x - (16*vdr/QI5_0) * ds8f.y); +} + +#define VDR_Q5_1_Q8_1_MMVQ 2 +#define VDR_Q5_1_Q8_1_MMQ 4 + +template static __device__ __forceinline__ float vec_dot_q5_1_q8_1_impl( + const int * vl, const int * vh, const int * u, const half2 & dm5, const half2 & ds8) { + + int sumi = 0; + +#pragma unroll + for (int i = 0; i < vdr; ++i) { + int vi0 = (vl[i] >> 0) & 0x0F0F0F0F; // lower 4 qs bits, still need qh as 5th bits + vi0 |= (vh[i] << 4) & 0x00000010; // 0 -> 4 + vi0 |= (vh[i] << 11) & 0x00001000; // 1 -> 12 + vi0 |= (vh[i] << 18) & 0x00100000; // 2 -> 20 + vi0 |= (vh[i] << 25) & 0x10000000; // 3 -> 28 + sumi = ggml_cuda_dp4a(vi0, u[2*i+0], sumi); // SIMD dot product of quantized values + + int vi1 = (vl[i] >> 4) & 0x0F0F0F0F; // upper 4 qs bits, still need qh as 5th bits + vi1 |= (vh[i] >> 12) & 0x00000010; // 16 -> 4 + vi1 |= (vh[i] >> 5) & 0x00001000; // 17 -> 12 + vi1 |= (vh[i] << 2) & 0x00100000; // 18 -> 20 + vi1 |= (vh[i] << 9) & 0x10000000; // 19 -> 28 + sumi = ggml_cuda_dp4a(vi1, u[2*i+1], sumi); // SIMD dot product of quantized values + } + +#ifdef FAST_FP16_AVAILABLE + const float2 tmp = __half22float2(__hmul2(dm5, ds8)); + const float d5d8 = tmp.x; + const float m5s8 = tmp.y; +#else + const float2 dm5f = __half22float2(dm5); + const float2 ds8f = __half22float2(ds8); + const float d5d8 = dm5f.x * ds8f.x; + const float m5s8 = dm5f.y * ds8f.y; +#endif // FAST_FP16_AVAILABLE + + // scale second part of sum by QI5_1 / vdr to compensate for multiple threads adding it + return sumi*d5d8 + m5s8 / (QI5_1 / vdr); +} + +#define VDR_Q8_0_Q8_1_MMVQ 2 +#define VDR_Q8_0_Q8_1_MMQ 8 + +template static __device__ __forceinline__ T vec_dot_q8_0_q8_1_impl( + const int * v, const int * u, const T & d8_0, const T & d8_1) { + + int sumi = 0; + +#pragma unroll + for (int i = 0; i < vdr; ++i) { + // SIMD dot product of quantized values + sumi = ggml_cuda_dp4a(v[i], u[i], sumi); + } + + return d8_0*d8_1 * ((T) sumi); +} + +template static __device__ __forceinline__ float vec_dot_q8_1_q8_1_impl( + const int * v, const int * u, const half2 & dm8, const half2 & ds8) { + + int sumi = 0; + +#pragma unroll + for (int i = 0; i < vdr; ++i) { + // SIMD dot product of quantized values + sumi = ggml_cuda_dp4a(v[i], u[i], sumi); + } + +#ifdef FAST_FP16_AVAILABLE + const float2 tmp = __half22float2(__hmul2(dm8, ds8)); + const float d8d8 = tmp.x; + const float m8s8 = tmp.y; +#else + const float2 dm8f = __half22float2(dm8); + const float2 ds8f = __half22float2(ds8); + const float d8d8 = dm8f.x * ds8f.x; + const float m8s8 = dm8f.y * ds8f.y; +#endif // FAST_FP16_AVAILABLE + + // scale second part of sum by QI8_1/ vdr to compensate for multiple threads adding it + return sumi*d8d8 + m8s8 / (QI8_1 / vdr); +} + +template static __device__ __forceinline__ float vec_dot_q8_0_16_q8_1_impl( + const int * v, const int * u, const float * d8_0, const float & d8_1) { + + float sumf = 0.0f; + +#pragma unroll + for (int i0 = 0; i0 < vdr; i0 += QI8_0/2) { + int sumi = 0; + +#pragma unroll + for (int i = i0; i < i0 + QI8_0/2; ++i) { + // SIMD dot product of quantized values + sumi = ggml_cuda_dp4a(v[i], u[i], sumi); + } + + sumf += d8_0[i0/(QI8_0/2)]*sumi; + } + + return d8_1*sumf; +} + +#define VDR_MXFP4_Q8_1_MMVQ 2 +#define VDR_MXFP4_Q8_1_MMQ 4 + +static __device__ __forceinline__ float vec_dot_mxfp4_q8_1( + const void * __restrict__ vbq, const block_q8_1 * __restrict__ bq8_1, const int & kbx, const int & iqs) { + + const block_mxfp4 * bq4 = (const block_mxfp4 *) vbq + kbx; + + const int * q8 = (const int *) bq8_1->qs + iqs; + + int sumi = 0; +#pragma unroll + for (int l = 0; l < VDR_MXFP4_Q8_1_MMVQ; ++l) { + const int aux_q4 = get_int_b1(bq4->qs, iqs + l); + const int2 v = get_int_from_table_16(aux_q4, kvalues_mxfp4); + + sumi = ggml_cuda_dp4a(v.x, q8[l + 0], sumi); + sumi = ggml_cuda_dp4a(v.y, q8[l + 4], sumi); + } + + const float d = ggml_cuda_e8m0_to_fp32(bq4->e) * 0.5f * __low2float(bq8_1->ds); + return d * sumi; +} + +#define VDR_NVFP4_Q8_1_MMVQ 4 +#define VDR_NVFP4_Q8_1_MMQ 8 + +static __device__ __forceinline__ float vec_dot_nvfp4_q8_1( + const void * __restrict__ vbq, + const block_q8_1 * __restrict__ bq8_1, + const int32_t & kbx, + const int32_t & iqs) { + + const block_nvfp4 * bq4 = (const block_nvfp4 *) vbq + kbx; + float sum = 0.0f; +#pragma unroll + for (int i = 0; i < VDR_NVFP4_Q8_1_MMVQ/2; i++) { + const int32_t iqs0 = iqs + 2*i; + const int32_t iqs1 = iqs0 + 1; + const int32_t is = iqs0 >> 1; + const int2 v0 = get_int_from_table_16(get_int_b4(bq4->qs, iqs0), kvalues_mxfp4); + const int2 v1 = get_int_from_table_16(get_int_b4(bq4->qs, iqs1), kvalues_mxfp4); + const block_q8_1 * bq8 = bq8_1 + (is >> 1); + const int32_t i8 = ((is & 1) << 2); + + int sumi = ggml_cuda_dp4a(v0.x, get_int_b4(bq8->qs, i8 + 0), 0); + sumi = ggml_cuda_dp4a(v0.y, get_int_b4(bq8->qs, i8 + 2), sumi); + sumi = ggml_cuda_dp4a(v1.x, get_int_b4(bq8->qs, i8 + 1), sumi); + sumi = ggml_cuda_dp4a(v1.y, get_int_b4(bq8->qs, i8 + 3), sumi); + + const float d = ggml_cuda_ue4m3_to_fp32(bq4->d[is]) * __low2float(bq8->ds); + sum += d * float(sumi); + } + + return sum; +} +#define VDR_Q2_K_Q8_1_MMVQ 1 +#define VDR_Q2_K_Q8_1_MMQ 4 + +// contiguous v/x values +static __device__ __forceinline__ float vec_dot_q2_K_q8_1_impl_mmvq( + const int & v, const int * __restrict__ u, const uint8_t * __restrict__ scales, + const half2 & dm2, const float * __restrict__ d8) { + + float sumf_d = 0.0f; + float sumf_m = 0.0f; + +#pragma unroll + for (int i = 0; i < QR2_K; ++i) { + const int sc = scales[2*i]; + + const int vi = (v >> (2*i)) & 0x03030303; + + sumf_d += d8[i] * (ggml_cuda_dp4a(vi, u[i], 0) * (sc & 0xF)); // SIMD dot product + + // fill int with 4x m + int m = sc >> 4; + m |= m << 8; + m |= m << 16; + sumf_m += d8[i] * ggml_cuda_dp4a(m, u[i], 0); // multiply constant q2_K part with sum of q8_1 values + } + + const float2 dm2f = __half22float2(dm2); + + return dm2f.x*sumf_d - dm2f.y*sumf_m; +} + +// contiguous v/x + u/y values +template +static __device__ __forceinline__ float vec_dot_q2_K_q8_1_impl_mmq( + const int * __restrict__ v, const int * __restrict__ u, const half2 * dm2, const float & d8, const half2 * s8) { + + float sumf = 0.0f; + float sumf_d8 = 0.0f; + +#pragma unroll + for (int i0 = 0; i0 < QR2_K*VDR_Q2_K_Q8_1_MMQ; i0 += QI8_1) { + const float2 dm2f0 = __half22float2(dm2[i0/(QI8_1/2) + 0]); + int sumi_d0 = 0; + + const float2 dm2f1 = __half22float2(dm2[i0/(QI8_1/2) + 1]); + int sumi_d1 = 0; + +#pragma unroll + for (int i = i0; i < i0 + QI8_1/2; ++i) { + sumi_d0 = ggml_cuda_dp4a(v[i], u[i], sumi_d0); + } + sumf_d8 += dm2f0.x * sumi_d0; + +#pragma unroll + for (int i = i0 + QI8_1/2; i < i0 + QI8_1; ++i) { + sumi_d1 = ggml_cuda_dp4a(v[i], u[i], sumi_d1); + } + sumf_d8 += dm2f1.x * sumi_d1; + + if (i0/QI8_1 < ns8) { + const float2 s8f = __half22float2(s8[i0/QI8_1]); + sumf -= dm2f0.y*s8f.x; + sumf -= dm2f1.y*s8f.y; + } else { + int sumi_m0 = 0; +#pragma unroll + for (int i = i0; i < i0 + QI8_1/2; ++i) { + sumi_m0 = ggml_cuda_dp4a(0x01010101, u[i], sumi_m0); + } + sumf_d8 -= dm2f0.y * sumi_m0; + + int sumi_m1 = 0; +#pragma unroll + for (int i = i0 + QI8_1/2; i < i0 + QI8_1; ++i) { + sumi_m1 = ggml_cuda_dp4a(0x01010101, u[i], sumi_m1); + } + sumf_d8 -= dm2f1.y * sumi_m1; + } + } + + return sumf + d8*sumf_d8; +} + +#define VDR_Q3_K_Q8_1_MMVQ 1 +#define VDR_Q3_K_Q8_1_MMQ 2 + +// contiguous v/x values +static __device__ __forceinline__ float vec_dot_q3_K_q8_1_impl_mmvq( + const int & vl, const int & vh, const int * __restrict__ u, const uint8_t * __restrict__ scales, + const int & scale_offset, const float & d3, const float * __restrict__ d8) { + + float sumf = 0.0f; + +#pragma unroll + for (int i = 0; i < QR3_K; ++i) { + const int isc = scale_offset + 2*i; + + const int isc_low = isc % (QK_K/32); + const int sc_shift_low = 4 * (isc / (QK_K/32)); + const int sc_low = (scales[isc_low] >> sc_shift_low) & 0xF; + + const int isc_high = isc % (QK_K/64); + const int sc_shift_high = 2 * (isc / (QK_K/64)); + const int sc_high = ((scales[(QK_K/32) + isc_high] >> sc_shift_high) & 3) << 4; + + const int sc = (sc_low | sc_high) - 32; + + const int vil = (vl >> (2*i)) & 0x03030303; + + const int vih = ((vh >> i) << 2) & 0x04040404; + + const int vi = __vsubss4(vil, vih); + + sumf += d8[i] * (ggml_cuda_dp4a(vi, u[i], 0) * sc); // SIMD dot product + } + + return d3 * sumf; +} + +// contiguous v/x + u/y values +static __device__ __forceinline__ float vec_dot_q3_K_q8_1_impl_mmq( + const int * __restrict__ v, const int * __restrict__ u, const int8_t * __restrict__ scales, + const float & d3, const float & d8) { + + int sumi = 0; + +#pragma unroll + for (int i0 = 0; i0 < QR3_K*VDR_Q3_K_Q8_1_MMQ; i0 += QI8_1/2) { + int sumi_sc = 0; + +#pragma unroll + for (int i = i0; i < i0 + QI8_1/2; ++i) { + sumi_sc = ggml_cuda_dp4a(v[i], u[i], sumi_sc); // SIMD dot product + } + + sumi += sumi_sc * scales[i0 / (QI8_1/2)]; + } + + return d3*d8 * sumi; +} + +#define VDR_Q4_K_Q8_1_MMVQ 2 +#define VDR_Q4_K_Q8_1_MMQ 8 + +// contiguous v/x values +static __device__ __forceinline__ float vec_dot_q4_K_q8_1_impl_vmmq( + const int * __restrict__ v, const int * __restrict__ u, const uint8_t * __restrict__ sc, + const uint8_t * __restrict__ m, const half2 & dm4, const float * __restrict__ d8) { + + float sumf_d = 0.0f; + float sumf_m = 0.0f; + +#pragma unroll + for (int i = 0; i < QR4_K; ++i) { + const int v0i = (v[0] >> (4*i)) & 0x0F0F0F0F; + const int v1i = (v[1] >> (4*i)) & 0x0F0F0F0F; + + const int dot1 = ggml_cuda_dp4a(v1i, u[2*i+1], ggml_cuda_dp4a(v0i, u[2*i+0], 0)); // SIMD dot product + const int dot2 = ggml_cuda_dp4a(0x01010101, u[2*i+1], ggml_cuda_dp4a(0x01010101, u[2*i+0], 0)); // sum of u + + sumf_d += d8[i] * (dot1 * sc[i]); + sumf_m += d8[i] * (dot2 * m[i]); // multiply constant part of q4_K with sum of q8_1 values + } + + const float2 dm4f = __half22float2(dm4); + + return dm4f.x*sumf_d - dm4f.y*sumf_m; +} + +// contiguous v/x + u/y values +static __device__ __forceinline__ float vec_dot_q4_K_q8_1_impl_mmq( + const int * __restrict__ v, const int * __restrict__ u, const uint8_t * __restrict__ sc, + const uint8_t * __restrict__ m, const half2 & dm4, const half2 * __restrict__ ds8) { + + float sumf_d = 0.0f; + float sumf_m = 0.0f; + +#pragma unroll + for (int i = 0; i < QR4_K*VDR_Q4_K_Q8_1_MMQ/QI8_1; ++i) { + int sumi_d = 0; + +#pragma unroll + for (int j = 0; j < QI8_1; ++j) { + sumi_d = ggml_cuda_dp4a((v[j] >> (4*i)) & 0x0F0F0F0F, u[i*QI8_1 + j], sumi_d); // SIMD dot product + } + + const float2 ds8f = __half22float2(ds8[i]); + + sumf_d += ds8f.x * (sc[i] * sumi_d); + sumf_m += ds8f.y * m[i]; // sum of q8_1 block * q4_K min val + } + + const float2 dm4f = __half22float2(dm4); + + return dm4f.x*sumf_d - dm4f.y*sumf_m; +} + +#define VDR_Q5_K_Q8_1_MMVQ 2 +#define VDR_Q5_K_Q8_1_MMQ 8 + +// contiguous v/x values +static __device__ __forceinline__ float vec_dot_q5_K_q8_1_impl_vmmq( + const int * __restrict__ vl, const int * __restrict__ vh, const int * __restrict__ u, const uint8_t * __restrict__ sc, + const uint8_t * __restrict__ m, const half2 & dm5, const float * __restrict__ d8) { + + float sumf_d = 0.0f; + float sumf_m = 0.0f; + +#pragma unroll + for (int i = 0; i < QR5_K; ++i) { + const int vl0i = (vl[0] >> (4*i)) & 0x0F0F0F0F; + const int vl1i = (vl[1] >> (4*i)) & 0x0F0F0F0F; + + const int vh0i = ((vh[0] >> i) << 4) & 0x10101010; + const int vh1i = ((vh[1] >> i) << 4) & 0x10101010; + + const int v0i = vl0i | vh0i; + const int v1i = vl1i | vh1i; + + const int dot1 = ggml_cuda_dp4a(v0i, u[2*i+0], ggml_cuda_dp4a(v1i, u[2*i+1], 0)); // SIMD dot product + const int dot2 = ggml_cuda_dp4a(0x01010101, u[2*i+0], ggml_cuda_dp4a(0x01010101, u[2*i+1], 0)); // sum of u + + sumf_d += d8[i] * (dot1 * sc[i]); + sumf_m += d8[i] * (dot2 * m[i]); + + } + + const float2 dm5f = __half22float2(dm5); + + return dm5f.x*sumf_d - dm5f.y*sumf_m; +} + +// contiguous v/x + u/y values +static __device__ __forceinline__ float vec_dot_q5_K_q8_1_impl_mmq( + const int * __restrict__ v, const int * __restrict__ u, const uint8_t * __restrict__ sc, + const uint8_t * __restrict__ m, const half2 & dm4, const half2 * __restrict__ ds8) { + + float sumf_d = 0.0f; + float sumf_m = 0.0f; + +#pragma unroll + for (int i = 0; i < QR5_K*VDR_Q5_K_Q8_1_MMQ/QI8_1; ++i) { + int sumi_d = 0; + +#pragma unroll + for (int j = 0; j < QI8_1; ++j) { + sumi_d = ggml_cuda_dp4a(v[i*QI8_1 + j], u[i*QI8_1 + j], sumi_d); // SIMD dot product + } + + const float2 ds8f = __half22float2(ds8[i]); + + sumf_d += ds8f.x * (sc[i] * sumi_d); + sumf_m += ds8f.y * m[i]; // sum of q8_1 block * q4_K min val + } + + const float2 dm4f = __half22float2(dm4); + + return dm4f.x*sumf_d - dm4f.y*sumf_m; +} + +#define VDR_Q6_K_Q8_1_MMVQ 1 +#define VDR_Q6_K_Q8_1_MMQ 8 + +// contiguous v/x values +static __device__ __forceinline__ float vec_dot_q6_K_q8_1_impl_mmvq( + const int & vl, const int & vh, const int * __restrict__ u, const int8_t * __restrict__ scales, + const float & d, const float * __restrict__ d8) { + + float sumf = 0.0f; + +#pragma unroll + for (int i = 0; i < QR6_K; ++i) { + const int sc = scales[4*i]; + + const int vil = (vl >> (4*i)) & 0x0F0F0F0F; + + const int vih = ((vh >> (4*i)) << 4) & 0x30303030; + + const int vi = __vsubss4((vil | vih), 0x20202020); // vi = (vil | vih) - 32 + + sumf += d8[i] * (ggml_cuda_dp4a(vi, u[i], 0) * sc); // SIMD dot product + } + + return d*sumf; +} + +// contiguous v/x + u/y values +static __device__ __forceinline__ float vec_dot_q6_K_q8_1_impl_mmq( + const int * __restrict__ v, const int * __restrict__ u, const int8_t * __restrict__ sc, + const float & d6, const float * __restrict__ d8) { + + float sumf_d = 0.0f; + + const int sc_packed = get_int_b4(sc, 0); + const int8_t * sc_reg = (const int8_t *) &sc_packed; + +#pragma unroll + for (int i0 = 0; i0 < VDR_Q6_K_Q8_1_MMQ; i0 += 4) { + int2 sumi_d = {0, 0}; // 2 q6_K scales per q8_1 scale + +#pragma unroll + for (int i = i0; i < i0 + 2; ++i) { + sumi_d.x = ggml_cuda_dp4a(v[2*i+0], u[2*i+0], sumi_d.x); // SIMD dot product + sumi_d.x = ggml_cuda_dp4a(v[2*i+1], u[2*i+1], sumi_d.x); // SIMD dot product + + sumi_d.y = ggml_cuda_dp4a(v[2*i+4], u[2*i+4], sumi_d.y); // SIMD dot product + sumi_d.y = ggml_cuda_dp4a(v[2*i+5], u[2*i+5], sumi_d.y); // SIMD dot product + } + + sumf_d += d8[i0/4] * (sc_reg[i0/2+0]*sumi_d.x + sc_reg[i0/2+1]*sumi_d.y); + } + + return d6 * sumf_d; +} + +static __device__ __forceinline__ float vec_dot_q1_0_q8_1( + const void * __restrict__ vbq, const block_q8_1 * __restrict__ bq8_1, const int & kbx, const int & iqs) { + + const block_q1_0 * bq1_0 = (const block_q1_0 *) vbq + kbx; + + // Q1_0: 128 elements with ONE scale + // Q8_1: 32 elements per block with individual scales + // iqs selects which of the 4 chunks of 32 elements to process (0-3) + + const float d1 = bq1_0->d; + const int16_t * qs = (const int16_t *) bq1_0->qs + iqs * 2; + + // Process only the chunk specified by iqs + const block_q8_1 * bq8_1_chunk = bq8_1 + iqs; + + int sumi = 0; +#pragma unroll + for (int j = 0; j < 2; ++j) { + const int q = qs[j]; + + const int u0 = get_int_b4(bq8_1_chunk->qs, j*4+0); + const int u1 = get_int_b4(bq8_1_chunk->qs, j*4+1); + const int u2 = get_int_b4(bq8_1_chunk->qs, j*4+2); + const int u3 = get_int_b4(bq8_1_chunk->qs, j*4+3); + + // unpack crumbs into nibble indices + const int n0 = __byte_perm(0x11100100, 0x11100100, q >> 0); // [0, 1, 4, 5] [ 8, 9, 12, 13] + const int n1 = __byte_perm(0x11100100, 0x11100100, q >> 2); // [2, 3, 6, 7] [10, 11, 14, 15] + // unpack nibbles into byte values + const int s0 = __byte_perm(0x01FF, 0x01FF, n0 >> 0); + const int s1 = __byte_perm(0x01FF, 0x01FF, n1 >> 0); + const int s2 = __byte_perm(0x01FF, 0x01FF, n0 >> 16); + const int s3 = __byte_perm(0x01FF, 0x01FF, n1 >> 16); + // unshuffle values + const int v0 = __byte_perm(s0, s1, 0x5410); + const int v1 = __byte_perm(s0, s1, 0x7632); + const int v2 = __byte_perm(s2, s3, 0x5410); + const int v3 = __byte_perm(s2, s3, 0x7632); + + sumi = ggml_cuda_dp4a(v0, u0, sumi); + sumi = ggml_cuda_dp4a(v1, u1, sumi); + sumi = ggml_cuda_dp4a(v2, u2, sumi); + sumi = ggml_cuda_dp4a(v3, u3, sumi); + } + + // Apply Q1_0's single scale and this chunk's Q8_1 scale + const float d8 = __low2float(bq8_1_chunk->ds); + return d1 * d8 * sumi; +} + +static __device__ __forceinline__ float vec_dot_q2_0_q8_1( + const void * __restrict__ vbq, const block_q8_1 * __restrict__ bq8_1, const int & kbx, const int & iqs) { + + const block_q2_0 * bq2_0 = (const block_q2_0 *) vbq + kbx; + + // Q2_0 (group 64): 64 elements with ONE scale, 2 bits per element (4 elements per byte) + // Q8_1: 32 elements per block with individual scales + // iqs selects which of the 2 chunks of 32 elements to process (0-1) + + const float d2 = bq2_0->d; + const int16_t * qs = (const int16_t *) bq2_0->qs + iqs * 4; + + // Process only the chunk specified by iqs + const block_q8_1 * bq8_1_chunk = bq8_1 + iqs; + + int sumi = 0; +#pragma unroll + for (int j = 0; j < 4; ++j) { + const int q = qs[j]; + const int u = get_int_b4(bq8_1_chunk->qs, j*2+0); + const int v = get_int_b4(bq8_1_chunk->qs, j*2+1); + + // unpack even and odd crumbs into byte values + const int qe = __byte_perm(0x020100FF, 0x020100FF, q >> 0); + const int qo = __byte_perm(0x020100FF, 0x020100FF, q >> 2); + // unshuffle values + const int qx = __byte_perm(qe, qo, 0x5140); + const int qy = __byte_perm(qe, qo, 0x7362); + + sumi = ggml_cuda_dp4a(u, qx, sumi); + sumi = ggml_cuda_dp4a(v, qy, sumi); + } + + // Apply Q2_0's single scale and this chunk's Q8_1 scale + const float d8 = __low2float(bq8_1_chunk->ds); + return d2 * d8 * sumi; +} + +static __device__ __forceinline__ float vec_dot_q4_0_q8_1( + const void * __restrict__ vbq, const block_q8_1 * __restrict__ bq8_1, const int & kbx, const int & iqs) { + + const block_q4_0 * bq4_0 = (const block_q4_0 *) vbq + kbx; + + int v[VDR_Q4_0_Q8_1_MMVQ]; + int u[2*VDR_Q4_0_Q8_1_MMVQ]; + +#pragma unroll + for (int i = 0; i < VDR_Q4_0_Q8_1_MMVQ; ++i) { + v[i] = get_int_b2(bq4_0->qs, iqs + i); + u[2*i+0] = get_int_b4(bq8_1->qs, iqs + i); + u[2*i+1] = get_int_b4(bq8_1->qs, iqs + i + QI4_0); + } + + return vec_dot_q4_0_q8_1_impl(v, u, bq4_0->d, bq8_1->ds); +} + + +static __device__ __forceinline__ float vec_dot_q4_1_q8_1( + const void * __restrict__ vbq, const block_q8_1 * __restrict__ bq8_1, const int & kbx, const int & iqs) { + + const block_q4_1 * bq4_1 = (const block_q4_1 *) vbq + kbx; + + int v[VDR_Q4_1_Q8_1_MMVQ]; + int u[2*VDR_Q4_1_Q8_1_MMVQ]; + +#pragma unroll + for (int i = 0; i < VDR_Q4_1_Q8_1_MMVQ; ++i) { + v[i] = get_int_b4(bq4_1->qs, iqs + i); + u[2*i+0] = get_int_b4(bq8_1->qs, iqs + i); + u[2*i+1] = get_int_b4(bq8_1->qs, iqs + i + QI4_1); + } + + return vec_dot_q4_1_q8_1_impl(v, u, bq4_1->dm, bq8_1->ds); +} + +static __device__ __forceinline__ float vec_dot_q5_0_q8_1( + const void * __restrict__ vbq, const block_q8_1 * __restrict__ bq8_1, const int & kbx, const int & iqs) { + + const block_q5_0 * bq5_0 = (const block_q5_0 *) vbq + kbx; + + int vl[VDR_Q5_0_Q8_1_MMVQ]; + int vh[VDR_Q5_0_Q8_1_MMVQ]; + int u[2*VDR_Q5_0_Q8_1_MMVQ]; + +#pragma unroll + for (int i = 0; i < VDR_Q5_0_Q8_1_MMVQ; ++i) { + vl[i] = get_int_b2(bq5_0->qs, iqs + i); + vh[i] = get_int_b2(bq5_0->qh, 0) >> (4 * (iqs + i)); + u[2*i+0] = get_int_b4(bq8_1->qs, iqs + i); + u[2*i+1] = get_int_b4(bq8_1->qs, iqs + i + QI5_0); + } + + return vec_dot_q5_0_q8_1_impl(vl, vh, u, bq5_0->d, bq8_1->ds); +} + +static __device__ __forceinline__ float vec_dot_q5_1_q8_1( + const void * __restrict__ vbq, const block_q8_1 * __restrict__ bq8_1, const int & kbx, const int & iqs) { + + const block_q5_1 * bq5_1 = (const block_q5_1 *) vbq + kbx; + + int vl[VDR_Q5_1_Q8_1_MMVQ]; + int vh[VDR_Q5_1_Q8_1_MMVQ]; + int u[2*VDR_Q5_1_Q8_1_MMVQ]; + +#pragma unroll + for (int i = 0; i < VDR_Q5_1_Q8_1_MMVQ; ++i) { + vl[i] = get_int_b4(bq5_1->qs, iqs + i); + vh[i] = get_int_b4(bq5_1->qh, 0) >> (4 * (iqs + i)); + u[2*i+0] = get_int_b4(bq8_1->qs, iqs + i); + u[2*i+1] = get_int_b4(bq8_1->qs, iqs + i + QI5_1); + } + + return vec_dot_q5_1_q8_1_impl(vl, vh, u, bq5_1->dm, bq8_1->ds); +} + +static __device__ __forceinline__ float vec_dot_q8_0_q8_1( + const void * __restrict__ vbq, const block_q8_1 * __restrict__ bq8_1, const int & kbx, const int & iqs) { + + const block_q8_0 * bq8_0 = (const block_q8_0 *) vbq + kbx; + + int v[VDR_Q8_0_Q8_1_MMVQ]; + int u[VDR_Q8_0_Q8_1_MMVQ]; + +#pragma unroll + for (int i = 0; i < VDR_Q8_0_Q8_1_MMVQ; ++i) { + v[i] = get_int_b2(bq8_0->qs, iqs + i); + u[i] = get_int_b4(bq8_1->qs, iqs + i); + } + + return vec_dot_q8_0_q8_1_impl(v, u, bq8_0->d, __low2half(bq8_1->ds)); +} + +static __device__ __forceinline__ float vec_dot_q2_K_q8_1( + const void * __restrict__ vbq, const block_q8_1 * __restrict__ bq8_1, const int & kbx, const int & iqs) { + + const block_q2_K * bq2_K = (const block_q2_K *) vbq + kbx; + + const int bq8_offset = QR2_K * (iqs / QI8_1); + const int scale_offset = iqs - iqs % QI8_1 + (iqs % QI8_1) / (QI8_1/2); + + const uint8_t * scales = bq2_K->scales + scale_offset; + + const int v = get_int_b4(bq2_K->qs, iqs); + int u[QR2_K]; + float d8[QR2_K]; + +#pragma unroll + for (int i = 0; i < QR2_K; ++ i) { + u[i] = get_int_b4(bq8_1[bq8_offset + i].qs, iqs % QI8_1); + d8[i] = __low2float(bq8_1[bq8_offset + i].ds); + } + + return vec_dot_q2_K_q8_1_impl_mmvq(v, u, scales, bq2_K->dm, d8); +} + +static __device__ __forceinline__ float vec_dot_q3_K_q8_1( + const void * __restrict__ vbq, const block_q8_1 * __restrict__ bq8_1, const int & kbx, const int & iqs) { + + const block_q3_K * bq3_K = (const block_q3_K *) vbq + kbx; + + const int bq8_offset = QR3_K * (iqs / (QI3_K/2)); + const int scale_offset = iqs - iqs % QI8_1 + (iqs % QI8_1) / (QI8_1/2); + + const float d = bq3_K->d; + + const int vl = get_int_b2(bq3_K->qs, iqs); + + // invert the mask with ~ so that a 0/1 results in 4/0 being subtracted + const int vh = ~get_int_b2(bq3_K->hmask, iqs % (QI3_K/2)) >> bq8_offset; + + int u[QR3_K]; + float d8[QR3_K]; + +#pragma unroll + for (int i = 0; i < QR3_K; ++i) { + u[i] = get_int_b4(bq8_1[bq8_offset + i].qs, iqs % QI8_1); + d8[i] = __low2float(bq8_1[bq8_offset + i].ds); + } + + return vec_dot_q3_K_q8_1_impl_mmvq(vl, vh, u, bq3_K->scales, scale_offset, d, d8); +} + +static __device__ __forceinline__ float vec_dot_q4_K_q8_1( + const void * __restrict__ vbq, const block_q8_1 * __restrict__ bq8_1, const int & kbx, const int & iqs) { + + const block_q4_K * bq4_K = (const block_q4_K *) vbq + kbx; + + int v[2]; + int u[2*QR4_K]; + float d8[QR4_K]; + + // iqs is in 0,2..30. bq8_offset = iqs/4 -> bq8_offset = 0, 2, 4, 6 + const int bq8_offset = QR4_K * ((iqs/2) / (QI8_1/2)); + + // iqs = 0....3 -> bq8_offset = 0, want q4_offset = 0, 4, 8, 12 + // iqs = 4....7 -> bq8_offset = 2, want q4_offset = 32, 36, 40, 44 + // iqs = 8...11 -> bq8_offset = 4, want q4_offset = 64, 68, 72, 76 + // iqs = 12..15 -> bq8_offset = 6, want q4_offset = 96, 100, 104, 108 + + const int * q4 = (const int *)(bq4_K->qs + 16 * bq8_offset + 4 * ((iqs/2)%4)); + v[0] = q4[0]; + v[1] = q4[4]; + + const uint16_t * scales = (const uint16_t *)bq4_K->scales; + uint16_t aux[2]; + const int j = bq8_offset/2; + if (j < 2) { + aux[0] = scales[j+0] & 0x3f3f; + aux[1] = scales[j+2] & 0x3f3f; + } else { + aux[0] = ((scales[j+2] >> 0) & 0x0f0f) | ((scales[j-2] & 0xc0c0) >> 2); + aux[1] = ((scales[j+2] >> 4) & 0x0f0f) | ((scales[j-0] & 0xc0c0) >> 2); + } + const uint8_t * sc = (const uint8_t *)aux; + const uint8_t * m = sc + 2; + + for (int i = 0; i < QR4_K; ++i) { + const block_q8_1 * bq8i = bq8_1 + bq8_offset + i; + d8[i] = __low2float(bq8i->ds); + + const int * q8 = (const int *)bq8i->qs + ((iqs/2)%4); + u[2*i+0] = q8[0]; + u[2*i+1] = q8[4]; + } + + return vec_dot_q4_K_q8_1_impl_vmmq(v, u, sc, m, bq4_K->dm, d8); +} + +static __device__ __forceinline__ float vec_dot_q5_K_q8_1( + const void * __restrict__ vbq, const block_q8_1 * __restrict__ bq8_1, const int & kbx, const int & iqs) { + + const block_q5_K * bq5_K = (const block_q5_K *) vbq + kbx; + + int vl[2]; + int vh[2]; + int u[2*QR5_K]; + float d8[QR5_K]; + + const int bq8_offset = QR5_K * ((iqs/2) / (QI8_1/2)); + const int * ql = (const int *)(bq5_K->qs + 16 * bq8_offset + 4 * ((iqs/2)%4)); + const int * qh = (const int *)(bq5_K->qh + 4 * ((iqs/2)%4)); + + vl[0] = ql[0]; + vl[1] = ql[4]; + + vh[0] = qh[0] >> bq8_offset; + vh[1] = qh[4] >> bq8_offset; + + const uint16_t * scales = (const uint16_t *)bq5_K->scales; + uint16_t aux[2]; + const int j = bq8_offset/2; + if (j < 2) { + aux[0] = scales[j+0] & 0x3f3f; + aux[1] = scales[j+2] & 0x3f3f; + } else { + aux[0] = ((scales[j+2] >> 0) & 0x0f0f) | ((scales[j-2] & 0xc0c0) >> 2); + aux[1] = ((scales[j+2] >> 4) & 0x0f0f) | ((scales[j-0] & 0xc0c0) >> 2); + } + const uint8_t * sc = (const uint8_t *)aux; + const uint8_t * m = sc + 2; + +#pragma unroll + for (int i = 0; i < QR5_K; ++i) { + const block_q8_1 * bq8i = bq8_1 + bq8_offset + i; + d8[i] = __low2float(bq8i->ds); + + const int * q8 = (const int *)bq8i->qs + ((iqs/2)%4); + u[2*i+0] = q8[0]; + u[2*i+1] = q8[4]; + } + + return vec_dot_q5_K_q8_1_impl_vmmq(vl, vh, u, sc, m, bq5_K->dm, d8); +} + +static __device__ __forceinline__ float vec_dot_q6_K_q8_1( + const void * __restrict__ vbq, const block_q8_1 * __restrict__ bq8_1, const int & kbx, const int & iqs) { + + const block_q6_K * bq6_K = (const block_q6_K *) vbq + kbx; + + const int bq8_offset = 2 * QR6_K * (iqs / (QI6_K/2)) + (iqs % (QI6_K/2)) / (QI6_K/4); + const int scale_offset = (QI6_K/4) * (iqs / (QI6_K/2)) + (iqs % (QI6_K/2)) / (QI6_K/8); + const int vh_shift = 2 * ((iqs % (QI6_K/2)) / (QI6_K/4)); + + const int vl = get_int_b2(bq6_K->ql, iqs); + const int vh = get_int_b2(bq6_K->qh, (QI6_K/4) * (iqs / (QI6_K/2)) + iqs % (QI6_K/4)) >> vh_shift; + + const int8_t * scales = bq6_K->scales + scale_offset; + + int u[QR6_K]; + float d8[QR6_K]; + +#pragma unroll + for (int i = 0; i < QR6_K; ++i) { + u[i] = get_int_b4(bq8_1[bq8_offset + 2*i].qs, iqs % QI8_1); + d8[i] = __low2float(bq8_1[bq8_offset + 2*i].ds); + } + + return vec_dot_q6_K_q8_1_impl_mmvq(vl, vh, u, scales, bq6_K->d, d8); +} + +#define VDR_IQ2_XXS_Q8_1_MMVQ 2 +#define VDR_IQ2_XXS_Q8_1_MMQ 2 + +static __device__ __forceinline__ float vec_dot_iq2_xxs_q8_1( + const void * __restrict__ vbq, const block_q8_1 * __restrict__ bq8_1, const int & kbx, const int & iqs) { + + const block_iq2_xxs * bq2 = (const block_iq2_xxs *) vbq + kbx; + + const int q2 = get_int_b2(bq2->qs, iqs); + const uint8_t * aux8 = (const uint8_t *) &q2; + const uint32_t aux32 = get_int_b2(bq2->qs, iqs + 1); + + int sumi = 0; +#pragma unroll + for (int k0 = 0; k0 < 8; k0 += 2) { + const uint2 grid_pos = ((const uint2*)iq2xxs_grid)[aux8[k0/2]]; + const uint32_t signs = unpack_ksigns(aux32 >> (7 * k0 / 2)); + + const int signs0 = __vcmpne4(signs & 0x08040201, 0); + const int grid0 = __vsub4(grid_pos.x ^ signs0, signs0); + const int u0 = get_int_b4(bq8_1[iqs/2].qs, k0 + 0); + sumi = ggml_cuda_dp4a(grid0, u0, sumi); + + const int signs1 = __vcmpne4(signs & 0x80402010, 0); + const int grid1 = __vsub4(grid_pos.y ^ signs1, signs1); + const int u1 = get_int_b4(bq8_1[iqs/2].qs, k0 + 1); + sumi = ggml_cuda_dp4a(grid1, u1, sumi); + } + + const int ls = aux32 >> 27 | 1; // (scale * 2 + 1) + sumi = sumi * ls / 8; // (sumi * scale + sumi / 2) / 4 + const float d = __half2float(bq2->d) * __low2float(bq8_1[iqs/2].ds); + return d * sumi; +} + +#define VDR_IQ2_XS_Q8_1_MMVQ 2 +#define VDR_IQ2_XS_Q8_1_MMQ 2 + +static __device__ __forceinline__ float vec_dot_iq2_xs_q8_1( + const void * __restrict__ vbq, const block_q8_1 * __restrict__ bq8_1, const int & kbx, const int & iqs) { + + const block_iq2_xs * bq2 = (const block_iq2_xs *) vbq + kbx; + + const int2 q2_packed = make_int2(get_int_b2(bq2->qs, iqs + 0), get_int_b2(bq2->qs, iqs + 1)); + const uint16_t * q2 = (const uint16_t *) &q2_packed; + const int ls0 = bq2->scales[iqs/2] & 0x0F; + const int ls1 = bq2->scales[iqs/2] >> 4; + + int sumi0 = 0; + int sumi1 = 0; +#pragma unroll + for (int l0 = 0; l0 < 8; l0 += 2) { + const uint2 grid_pos = ((const uint2*)iq2xs_grid)[q2[l0/2] & 0x1FF]; + const uint32_t signs = unpack_ksigns(q2[l0/2] >> 9); + + const int signs0 = __vcmpne4(signs & 0x08040201, 0); + const int grid_l = __vsub4(grid_pos.x ^ signs0, signs0); + const int u0 = get_int_b4(bq8_1[iqs/2].qs, l0 + 0); + + const int signs1 = __vcmpne4(signs & 0x80402010, 0); + const int grid_h = __vsub4(grid_pos.y ^ signs1, signs1); + const int u1 = get_int_b4(bq8_1[iqs/2].qs, l0 + 1); + + if (l0 < 4) { + sumi0 = ggml_cuda_dp4a(grid_l, u0, sumi0); + sumi0 = ggml_cuda_dp4a(grid_h, u1, sumi0); + } else { + sumi1 = ggml_cuda_dp4a(grid_l, u0, sumi1); + sumi1 = ggml_cuda_dp4a(grid_h, u1, sumi1); + } + } + const int sumi = (sumi0*ls0 + sumi1*ls1 + (sumi0 + sumi1)/2)/4; + const float d = __half2float(bq2->d) * __low2float(bq8_1[iqs/2].ds); + return d * sumi; +} + +#define VDR_IQ2_S_Q8_1_MMVQ 2 +#define VDR_IQ2_S_Q8_1_MMQ 2 + +static __device__ __forceinline__ float vec_dot_iq2_s_q8_1( + const void * __restrict__ vbq, const block_q8_1 * __restrict__ bq8_1, const int & kbx, const int & iqs) { + + const block_iq2_s * bq2 = (const block_iq2_s *) vbq + kbx; + + const int qs_packed = get_int_b2(bq2->qs, iqs/2); + const uint8_t * qs = (const uint8_t *) &qs_packed; + + const int qh = bq2->qh[iqs/2]; + + const int signs_packed_32 = get_int_b2(bq2->qs, QK_K/32 + iqs/2); + const uint8_t * signs_packed_8 = (const uint8_t *) &signs_packed_32; + + const int ls0 = bq2->scales[iqs/2] & 0x0F; + const int ls1 = bq2->scales[iqs/2] >> 4; + + int sumi0 = 0; + int sumi1 = 0; +#pragma unroll + for (int l0 = 0; l0 < 8; l0 += 2) { + const int * grid_pos = (const int *)(iq2s_grid + (qs[l0/2] | ((qh << (8-l0)) & 0x300))); + + const int signs0 = __vcmpne4(((signs_packed_8[l0/2] & 0x03) << 7) | ((signs_packed_8[l0/2] & 0x0C) << 21), 0x00000000); + const int signs1 = __vcmpne4(((signs_packed_8[l0/2] & 0x30) << 3) | ((signs_packed_8[l0/2] & 0xC0) << 17), 0x00000000); + + const int grid_l = __vsub4(grid_pos[0] ^ signs0, signs0); + const int grid_h = __vsub4(grid_pos[1] ^ signs1, signs1); + + const int u0 = get_int_b4(bq8_1[iqs/2].qs, l0 + 0); + const int u1 = get_int_b4(bq8_1[iqs/2].qs, l0 + 1); + + if (l0 < 4) { + sumi0 = ggml_cuda_dp4a(grid_l, u0, sumi0); + sumi0 = ggml_cuda_dp4a(grid_h, u1, sumi0); + } else { + sumi1 = ggml_cuda_dp4a(grid_l, u0, sumi1); + sumi1 = ggml_cuda_dp4a(grid_h, u1, sumi1); + } + } + const int sumi = (sumi0*ls0 + sumi1*ls1 + (sumi0 + sumi1)/2)/4; + + const float d = __half2float(bq2->d) * __low2float(bq8_1[iqs/2].ds); + return d * sumi; +} + +#define VDR_IQ3_XXS_Q8_1_MMVQ 2 +#define VDR_IQ3_XXS_Q8_1_MMQ 2 + +static __device__ __forceinline__ float vec_dot_iq3_xxs_q8_1( + const void * __restrict__ vbq, const block_q8_1 * __restrict__ bq8_1, const int & kbx, const int & iqs) { + + const block_iq3_xxs * bq3 = (const block_iq3_xxs *) vbq + kbx; + + const int2 q3_packed = make_int2(get_int_b2(bq3->qs, iqs), get_int_b2(bq3->qs, iqs+1)); + const uint8_t * q3 = (const uint8_t *) &q3_packed; + const uint32_t aux32 = get_int_b2(bq3->qs, QK_K/16 + iqs/2); + + int sumi = 0; +#pragma unroll + for (int l0 = 0; l0 < 8; l0 += 2) { + const int2 grid_pos = make_int2(iq3xxs_grid[q3[l0 + 0]], iq3xxs_grid[q3[l0 + 1]]); + const uint32_t signs = unpack_ksigns(aux32 >> (7*l0/2)); + + const int signs0 = __vcmpne4(signs & 0x08040201, 0); + const int grid_l = __vsub4(grid_pos.x ^ signs0, signs0); + + const int u0 = get_int_b4(bq8_1[iqs/2].qs, l0 + 0); + + const int signs1 = __vcmpne4(signs & 0x80402010, 0); + const int grid_h = __vsub4(grid_pos.y ^ signs1, signs1); + + const int u1 = get_int_b4(bq8_1[iqs/2].qs, l0 + 1); + + sumi = ggml_cuda_dp4a(grid_l, u0, sumi); + sumi = ggml_cuda_dp4a(grid_h, u1, sumi); + } + + const int ls = aux32 >> 28; + sumi = (ls*sumi + sumi/2)/2; + const float d = __half2float(bq3->d) * __low2float(bq8_1[iqs/2].ds); + return d * sumi; +} + +#define VDR_IQ3_S_Q8_1_MMVQ 2 +#define VDR_IQ3_S_Q8_1_MMQ 2 + +// TODO: don't use lookup table for signs +static __device__ __forceinline__ float vec_dot_iq3_s_q8_1( + const void * __restrict__ vbq, const block_q8_1 * __restrict__ bq8_1, const int & kbx, const int & iqs) { + + const block_iq3_s * bq3 = (const block_iq3_s *) vbq + kbx; + + const int2 qs_packed = make_int2(get_int_b2(bq3->qs, iqs + 0), get_int_b2(bq3->qs, iqs + 1)); + const uint8_t * qs = (const uint8_t *) &qs_packed; + + const int qh = bq3->qh[iqs/2]; + + const int signs_packed_32 = get_int_b2(bq3->signs, iqs/2); + const uint8_t * signs_packed_8 = (const uint8_t *) &signs_packed_32; + + int sumi = 0; +#pragma unroll + for (int l0 = 0; l0 < 8; l0 += 2) { + const int2 grid_pos = make_int2( + iq3s_grid[qs[l0 + 0] | ((qh << (8 - l0)) & 0x100)], + iq3s_grid[qs[l0 + 1] | ((qh << (7 - l0)) & 0x100)]); + + const int signs0 = __vcmpne4(((signs_packed_8[l0/2] & 0x03) << 7) | ((signs_packed_8[l0/2] & 0x0C) << 21), 0x00000000); + const int signs1 = __vcmpne4(((signs_packed_8[l0/2] & 0x30) << 3) | ((signs_packed_8[l0/2] & 0xC0) << 17), 0x00000000); + + const int grid_l = __vsub4(grid_pos.x ^ signs0, signs0); + const int grid_h = __vsub4(grid_pos.y ^ signs1, signs1); + + const int u0 = get_int_b4(bq8_1[iqs/2].qs, l0 + 0); + const int u1 = get_int_b4(bq8_1[iqs/2].qs, l0 + 1); + + sumi = ggml_cuda_dp4a(grid_l, u0, sumi); + sumi = ggml_cuda_dp4a(grid_h, u1, sumi); + } + + sumi *= 1 + 2*((bq3->scales[iqs/4] >> ((iqs << 1) & 0x04)) & 0x0F); + + const float d = __half2float(bq3->d) * __low2float(bq8_1[iqs/2].ds); + return d * sumi; +} + +#define VDR_IQ1_S_Q8_1_MMVQ 1 +#define VDR_IQ1_S_Q8_1_MMQ 1 + +static __device__ __forceinline__ float vec_dot_iq1_s_q8_1( + const void * __restrict__ vbq, const block_q8_1 * __restrict__ bq8_1, const int & kbx, const int & iqs) { + const block_iq1_s * bq1 = (const block_iq1_s *) vbq + kbx; + + const int qs_packed = get_int_b2(bq1->qs, iqs); + const uint8_t * qs = (const uint8_t *) &qs_packed; + + const int qh = bq1->qh[iqs]; + + int sumi = 0; +#pragma unroll + for (int l0 = 0; l0 < 8; l0 += 2) { + const int grid = iq1s_grid_gpu[qs[l0/2] | (((qh >> 3*(l0/2)) & 0x07) << 8)]; + + const int grid0 = (grid >> 0) & 0x0F0F0F0F; + const int grid1 = (grid >> 4) & 0x0F0F0F0F; + + const int u0 = get_int_b4(bq8_1[iqs].qs, l0 + 0); + const int u1 = get_int_b4(bq8_1[iqs].qs, l0 + 1); + + sumi = ggml_cuda_dp4a(grid0, u0, sumi); + sumi = ggml_cuda_dp4a(grid1, u1, sumi); + } + + const float d1q = __half2float(bq1->d) * (((qh >> 11) & 0x0E) + 1); + const float delta = -1.0f + IQ1S_DELTA - (qh & 0x8000) * (2.0f*IQ1S_DELTA/0x8000); + const float2 ds = __half22float2(bq8_1[iqs].ds); + return d1q * (ds.x*sumi + ds.y*delta); +} + +#define VDR_IQ1_M_Q8_1_MMVQ 1 +#define VDR_IQ1_M_Q8_1_MMQ 1 + +static __device__ __forceinline__ float vec_dot_iq1_m_q8_1( + const void * __restrict__ vbq, const block_q8_1 * __restrict__ bq8_1, const int & kbx, const int & iqs) { + + const block_iq1_m * bq1 = (const block_iq1_m *) vbq + kbx; + + const int qs_packed = get_int_b4(bq1->qs, iqs); + const uint8_t * qs = (const uint8_t *) &qs_packed; + + int sumi[2] = {0}; + float sumf[2] = {0.0f}; +#pragma unroll + for (int l0 = 0; l0 < 8; l0 += 2) { + const int qhl = bq1->qh[2*iqs + l0/4] >> (4 * ((l0/2) % 2)); + + const int grid = iq1s_grid_gpu[qs[l0/2] | ((qhl & 0x07) << 8)]; + + const int grid0 = (grid >> 0) & 0x0F0F0F0F; + const int grid1 = (grid >> 4) & 0x0F0F0F0F; + + const int u0 = get_int_b4(bq8_1[iqs].qs, l0 + 0); + const int u1 = get_int_b4(bq8_1[iqs].qs, l0 + 1); + + sumi[l0/4] = ggml_cuda_dp4a(grid0, u0, sumi[l0/4]); + sumi[l0/4] = ggml_cuda_dp4a(grid1, u1, sumi[l0/4]); + + const float delta = -1.0f + IQ1M_DELTA - (qhl & 0x08) * (2.0f*IQ1M_DELTA/0x08); + int sumy = 0; + sumy = ggml_cuda_dp4a(u0, 0x01010101, sumy); + sumy = ggml_cuda_dp4a(u1, 0x01010101, sumy); + sumf[l0/4] += delta*sumy; + } + + const uint16_t * sc = (const uint16_t *) bq1->scales; + + iq1m_scale_t scale; + scale.u16 = (sc[0] >> 12) | ((sc[1] >> 8) & 0x00F0) | ((sc[2] >> 4) & 0x0F00) | (sc[3] & 0xF000); + const float d = __half2float(scale.f16) * __low2float(bq8_1[iqs].ds); + + const int tmp = sc[iqs/2] >> (6*(iqs%2)); + const int sc0 = 2*((tmp >> 0) & 0x07) + 1; + const int sc1 = 2*((tmp >> 3) & 0x07) + 1; + return d * ((sumi[0] + sumf[0]) * sc0 + (sumi[1] + sumf[1]) * sc1); +} + +#define VDR_IQ4_NL_Q8_1_MMVQ 2 +#define VDR_IQ4_NL_Q8_1_MMQ 4 + +static __device__ __forceinline__ float vec_dot_iq4_nl_q8_1( + const void * __restrict__ vbq, const block_q8_1 * __restrict__ bq8_1, const int & kbx, const int & iqs) { + + const block_iq4_nl * bq4 = (const block_iq4_nl *) vbq + kbx; + + const int * q8 = (const int *) bq8_1->qs + iqs; + + int sumi = 0; +#pragma unroll + for (int l = 0; l < VDR_Q4_0_Q8_1_MMVQ; ++l) { + const int aux_q4 = get_int_b2(bq4->qs, iqs + l); + const int2 v = get_int_from_table_16(aux_q4, kvalues_iq4nl); + + sumi = ggml_cuda_dp4a(v.x, q8[l + 0], sumi); + sumi = ggml_cuda_dp4a(v.y, q8[l + 4], sumi); + } + + const float d = __half2float(bq4->d) * __low2float(bq8_1->ds); + return d * sumi; +} + +#define VDR_IQ4_XS_Q8_1_MMVQ 4 +#define VDR_IQ4_XS_Q8_1_MMQ 4 + +static __device__ __forceinline__ float vec_dot_iq4_xs_q8_1( + const void * __restrict__ vbq, const block_q8_1 * __restrict__ bq8_1, const int & kbx, const int & iqs) { + + const block_iq4_xs * bq4 = (const block_iq4_xs *) vbq + kbx; + + int sumi = 0; +#pragma unroll + for (int j = 0; j < 4; ++j) { + const int aux_q4 = get_int_b4(bq4->qs, iqs + j); + const int2 v = get_int_from_table_16(aux_q4, kvalues_iq4nl); + + const int u0 = get_int_b4(bq8_1[iqs/4].qs, j + 0); + const int u1 = get_int_b4(bq8_1[iqs/4].qs, j + 4); + + sumi = ggml_cuda_dp4a(v.x, u0, sumi); + sumi = ggml_cuda_dp4a(v.y, u1, sumi); + } + + const int ls = ((bq4->scales_l[iqs/8] >> (iqs & 0x04)) & 0x0F) | (((bq4->scales_h >> (iqs/2)) & 0x03) << 4); + sumi *= ls - 32; + + const float d = __half2float(bq4->d) * __low2float(bq8_1[iqs/4].ds); + return d * sumi; +} diff --git a/python/freetoken/kernel/csrc/gguf_mmq/vendors/cuda.h b/python/freetoken/kernel/csrc/gguf_mmq/vendors/cuda.h new file mode 100644 index 00000000..323c9801 --- /dev/null +++ b/python/freetoken/kernel/csrc/gguf_mmq/vendors/cuda.h @@ -0,0 +1,28 @@ +#pragma once + +#include +#include +#include +#include +#include + +#ifdef GGML_USE_NCCL +#include +#endif // GGML_USE_NCCL + +#if CUDART_VERSION >= 11080 +#include +#define FP8_AVAILABLE +#endif // CUDART_VERSION >= 11080 + +#if CUDART_VERSION >= 12080 +#include +#endif // CUDART_VERSION >= 12080 + +#if CUDART_VERSION < 11020 +#define CU_DEVICE_ATTRIBUTE_VIRTUAL_MEMORY_MANAGEMENT_SUPPORTED CU_DEVICE_ATTRIBUTE_VIRTUAL_ADDRESS_MANAGEMENT_SUPPORTED +#define CUBLAS_TF32_TENSOR_OP_MATH CUBLAS_TENSOR_OP_MATH +#define CUBLAS_COMPUTE_16F CUDA_R_16F +#define CUBLAS_COMPUTE_32F CUDA_R_32F +#define cublasComputeType_t cudaDataType_t +#endif // CUDART_VERSION < 11020 diff --git a/python/freetoken/kernel/csrc/jit/fast_index_copy.cuh b/python/freetoken/kernel/csrc/jit/fast_index_copy.cuh index bb83c23e..ce1fe1c4 100644 --- a/python/freetoken/kernel/csrc/jit/fast_index_copy.cuh +++ b/python/freetoken/kernel/csrc/jit/fast_index_copy.cuh @@ -476,6 +476,8 @@ struct MultiIndexCopyParams { const int64_t* __restrict__ dst_ptrs; // [B] device, each base addr of a bank slot cache const int64_t* __restrict__ src_ptrs; // [B] device, each GPU-visible base addr of a bank host source const int64_t* __restrict__ feat_bytes; // [B] device, per-row bytes (multiple of 16) + const int64_t* __restrict__ dst_strides; // [B] device, destination row stride in bytes + const int64_t* __restrict__ src_strides; // [B] device, source row stride in bytes const void* __restrict__ dst_indices; // [L] const void* __restrict__ src_indices; // [L] const int64_t* __restrict__ valid_length; // [1] or null @@ -506,17 +508,19 @@ __global__ __launch_bounds__(kNumThreads) void fast_index_copy_multi( const int64_t col = (u - row * units) << 4; // byte offset within the row const int64_t pd = static_cast(di[row]); const int64_t ps = static_cast(si[row]); - const uint4 v = *reinterpret_cast(src + ps * feat + col); - *reinterpret_cast(dst + pd * feat + col) = v; + const uint4 v = *reinterpret_cast(src + ps * p.src_strides[b] + col); + *reinterpret_cast(dst + pd * p.dst_strides[b] + col) = v; } } template -struct MultiIndexCopyKernel { +struct MultiIndexCopyStridedKernel { static void run( tvm::ffi::TensorView dst_ptrs, tvm::ffi::TensorView src_ptrs, tvm::ffi::TensorView feat_bytes, + tvm::ffi::TensorView dst_strides, + tvm::ffi::TensorView src_strides, tvm::ffi::TensorView dst_indices, tvm::ffi::TensorView src_indices, tvm::ffi::Optional num_indices @@ -530,7 +534,8 @@ struct MultiIndexCopyKernel { auto num_indices_dtype = SymbolicDType{}; TensorMatcher({B}).with_dtype(ptr_dtype).with_device(device) - .verify(dst_ptrs).verify(src_ptrs).verify(feat_bytes); + .verify(dst_ptrs).verify(src_ptrs).verify(feat_bytes) + .verify(dst_strides).verify(src_strides); TensorMatcher({L}).with_dtype(indices_dtype).with_device(device) .verify(dst_indices).verify(src_indices); @@ -546,6 +551,8 @@ struct MultiIndexCopyKernel { static_cast(dst_ptrs.data_ptr()), static_cast(src_ptrs.data_ptr()), static_cast(feat_bytes.data_ptr()), + static_cast(dst_strides.data_ptr()), + static_cast(src_strides.data_ptr()), dst_indices.data_ptr(), src_indices.data_ptr(), valid_length, @@ -560,3 +567,23 @@ struct MultiIndexCopyKernel { device.unwrap())(kernel, params); } }; + +// Backward-compatible uniform-row entry point used by the prebuilt kernel cache. +// Keep its signature stable; variable-size banks compile/load the separately named +// MultiIndexCopyStridedKernel wrapper. +template +struct MultiIndexCopyKernel { + static void run( + tvm::ffi::TensorView dst_ptrs, + tvm::ffi::TensorView src_ptrs, + tvm::ffi::TensorView feat_bytes, + tvm::ffi::TensorView dst_indices, + tvm::ffi::TensorView src_indices, + tvm::ffi::Optional num_indices + ) { + MultiIndexCopyStridedKernel::run( + dst_ptrs, src_ptrs, feat_bytes, feat_bytes, feat_bytes, + dst_indices, src_indices, num_indices + ); + } +}; diff --git a/python/freetoken/kernel/fast_index_copy.py b/python/freetoken/kernel/fast_index_copy.py index 1aaa1303..4f1374be 100644 --- a/python/freetoken/kernel/fast_index_copy.py +++ b/python/freetoken/kernel/fast_index_copy.py @@ -185,6 +185,50 @@ def fast_index_copy_multi_jit( module.launch(dst_ptrs, src_ptrs, feat_bytes, dst_indices, src_indices, num_indices) +@lru_cache(maxsize=None) +def _jit_fast_index_copy_multi_strided_module( + *, num_threads: int, blocks_per_bank: int +) -> Module: + args = make_cpp_args(num_threads, blocks_per_bank) + return load_jit( + "fast_index_copy_multi_strided", + *args, + cuda_files=["fast_index_copy.cuh"], + cuda_wrappers=[("launch", f"&MultiIndexCopyStridedKernel<{args}>::run")], + ) + + +def fast_index_copy_multi_strided_jit( + dst_ptrs: torch.Tensor, + src_ptrs: torch.Tensor, + feat_bytes: torch.Tensor, + dst_strides: torch.Tensor, + src_strides: torch.Tensor, + dst_indices: torch.Tensor, + src_indices: torch.Tensor, + num_indices: torch.Tensor | None = None, + *, + num_threads: int = 1024, + blocks_per_bank: int = 8, +) -> None: + """Fused index copy where compact source rows occupy a larger cache-row prefix.""" + if _skip_fast_index_copy_enabled(): + return + module = _jit_fast_index_copy_multi_strided_module( + num_threads=num_threads, blocks_per_bank=blocks_per_bank + ) + module.launch( + dst_ptrs, + src_ptrs, + feat_bytes, + dst_strides, + src_strides, + dst_indices, + src_indices, + num_indices, + ) + + def update_copy_flag_jit(sync_flag: torch.Tensor, delta: int) -> None: assert sync_flag.is_cuda assert sync_flag.numel() == 1 diff --git a/python/freetoken/kernel/gguf.py b/python/freetoken/kernel/gguf.py index 04a16560..5ba6822c 100644 --- a/python/freetoken/kernel/gguf.py +++ b/python/freetoken/kernel/gguf.py @@ -73,6 +73,59 @@ def _module(): ) +_CSRC_MMQ = pathlib.Path(__file__).parent / "csrc" / "gguf_mmq" + +# Upstream int8-MMA mul_mat_q is only instantiated for these ggml types +# (Q4_K=12, Q6_K=14 -- Ornith's dense projection types). +_MMA_TYPES = frozenset({12, 14}) + + +@functools.cache +def _mma_module(): + """Upstream llama.cpp int8-tensor-core MMQ (csrc/gguf_mmq), sm_75+ hardware. + + Vendored verbatim from llama.cpp master (see mmq_ext.cu header). Separate + extension so the sibling b2899 DP4A module and its build cache are + untouched; compiled lazily the first time the MMA path is selected. + """ + from torch.utils.cpp_extension import load + + extra_cuda_cflags = [ + "-O3", + "--expt-relaxed-constexpr", + "--use_fast_math", + # llama.cpp relies on the implicit half<->float conversions that + # torch's default nvcc flags disable. + "-U__CUDA_NO_HALF_OPERATORS__", + "-U__CUDA_NO_HALF_CONVERSIONS__", + "-U__CUDA_NO_HALF2_OPERATORS__", + "-U__CUDA_NO_BFLOAT16_CONVERSIONS__", + ] + host_cxx = _host_compiler() + if host_cxx is not None: + cxx_path = shutil.which(host_cxx) or host_cxx + extra_cuda_cflags += ["-ccbin", cxx_path] + os.environ["CXX"] = cxx_path + os.environ["CC"] = _c_compiler_for(cxx_path) + + return load( + name="freetoken_gguf_mmq", + sources=[ + str(_CSRC_MMQ / "mmq_ext.cu"), + str(_CSRC_MMQ / "quantize.cu"), + str(_CSRC_MMQ / "mmid.cu"), + ], + extra_include_paths=[str(_CSRC_MMQ)], + extra_cuda_cflags=extra_cuda_cflags, + verbose=True, + ) + + +def mma_mmq_supported(quant_type: int) -> bool: + """Whether the int8-MMA MMQ extension covers ``quant_type``.""" + return quant_type in _MMA_TYPES + + # ---- thin typed wrappers (signatures mirror sgl_kernel.quantization.gguf) ---- @@ -97,6 +150,36 @@ def ggml_mul_mat_a8( return _module().ggml_mul_mat_a8(weight, x, quant_type, row) +def ggml_mul_mat_a8_mma( + weight: torch.Tensor, x: torch.Tensor, quant_type: int, row: int +) -> torch.Tensor: + """Upstream int8-MMA MMQ (Q4_K/Q6_K). Returns fp32 ``[tokens, row]``.""" + return _mma_module().ggml_mul_mat_a8_mma(weight, x, quant_type, row) + + +def ggml_moe_a8_mma( + x: torch.Tensor, + weight: torch.Tensor, + topk_ids: torch.Tensor, + top_k: int, + quant_type: int, + row: int, + tokens: int, + expert_stride_bytes: int = 0, + broadcast: bool = True, +) -> torch.Tensor: + """Upstream int8-MMA grouped expert matmul over flat padded slots. + + ``broadcast=True``: ``x[tokens, in]`` shared by each token's top_k experts + (gate/up). ``broadcast=False``: ``x[tokens*top_k, in]`` with row + ``t*top_k + k`` belonging to ``topk_ids[t][k]`` (down). Returns fp32 + ``[tokens*top_k, row]`` in that same row order. + """ + return _mma_module().ggml_moe_a8_mma( + x, weight, topk_ids, top_k, quant_type, row, tokens, expert_stride_bytes, broadcast + ) + + def ggml_moe_a8( x: torch.Tensor, weight: torch.Tensor, @@ -123,9 +206,17 @@ def ggml_moe_a8_vec( quant_type: int, row: int, tokens: int, + expert_stride_bytes: int = 0, ) -> torch.Tensor: - """MMVQ grouped expert GEMV over stacked experts ``weight[E, row, *]``.""" - return _module().ggml_moe_a8_vec(x, weight, topk_ids, top_k, quant_type, row, tokens) + """MMVQ grouped expert GEMV over stacked experts ``weight[E, row, *]``. + + ``expert_stride_bytes`` == 0 assumes dense contiguous banks; > 0 reads each + expert at that fixed byte offset (padded flat banks for mixed-quant models, + where a layer's real payload occupies the leading bytes of each expert slot). + """ + return _module().ggml_moe_a8_vec( + x, weight, topk_ids, top_k, quant_type, row, tokens, expert_stride_bytes + ) def ggml_moe_get_block_size(quant_type: int) -> int: @@ -136,6 +227,9 @@ def ggml_moe_get_block_size(quant_type: int) -> int: "ggml_dequantize", "ggml_mul_mat_vec_a8", "ggml_mul_mat_a8", + "ggml_mul_mat_a8_mma", + "ggml_moe_a8_mma", + "mma_mmq_supported", "ggml_moe_a8", "ggml_moe_a8_vec", "ggml_moe_get_block_size", diff --git a/python/freetoken/kernel/triton/attention.py b/python/freetoken/kernel/triton/attention.py index c2358d84..9b8e53e8 100644 --- a/python/freetoken/kernel/triton/attention.py +++ b/python/freetoken/kernel/triton/attention.py @@ -11,6 +11,152 @@ _MIN_BLOCK_KV = 32 +def decode_launch_config( + *, + quant_name: str | None, + head_dim: int, + num_q_heads: int, + num_kv_heads: int, + compute_capability: tuple[int, int] | None = None, +) -> tuple[int, int, int]: + """Return ``(kv_splits, block_n, num_warps)`` for grouped decode attention. + + Quantized caches are dequantized before each tensor-core dot and need more + parallel KV partitions than the bf16 path. The Ornith/Qwen3.5 geometry has + measured launch configurations for consumer Ada and Blackwell; other shapes + keep the conservative fallback. + """ + if ( + quant_name == "int4" + and head_dim == 256 + and num_q_heads == 16 + and num_kv_heads == 2 + ): + if compute_capability is not None and compute_capability >= (12, 0): + # RTX 5080 / Triton 3.6: 64 splits and 64-token tiles are 57% faster + # than the sm_89 launch at 262K (0.356 vs 0.822 ms per layer). + return 64, 64, 8 + # BLOCK_N=16 is not safe for the packed-byte loader at this geometry on + # Triton 3.6/sm_89; it silently corrupts attention output. BLOCK_N=32 has + # a numerical regression test and is also the fastest correct 200K launch. + return 32, 32, 4 + if ( + quant_name in {"q8_0", "quant8"} + and head_dim == 256 + and num_q_heads == 16 + and num_kv_heads == 2 + ): + # The backend sees the public name (q8_0) while the low-level kernel + # infers quant8 from the cache tensors. Accept both so CUDA-graph scratch + # is allocated for all 64 splits and the kernel can actually select them. + return 64, 64, 4 + return _MAX_KV_SPLITS, 32, 4 + + +@triton.jit +def _load_kv( + ptr, + scale_ptr, + base, # slot/head byte base (already carries the broadcast), per-row/per-col + elem, # within-head element index (the heads dim), broadcast to match ``base`` + scale_offsets, + mask, + scale_mask, + out_dtype: tl.constexpr, + QUANT: tl.constexpr, + QBLOCK: tl.constexpr, + D_ON_ROWS: tl.constexpr, + EPB: tl.constexpr, +): + """Load a K or V tile, dequantizing it when the pool stores quantized values. + + ``base + elem`` addresses the tile in the KV buffer; ``scale_offsets`` addresses the + matching scales, whose ``head_dim`` extent is ``QBLOCK`` times smaller -- one scale + per block, loaded once and broadcast across the block rather than re-read per + element. + + ``base`` is kept separate from ``elem`` because packed storage halves the byte + extent of ``elem`` only: with ``EPB`` (elements-per-byte) == 2 the within-head + element index maps to ``elem // EPB`` bytes, and the slot/head strides in ``base`` + are *byte* counts that must not be divided. The element's parity (``elem``'s low bit) + selects the low or high nibble. + + ``D_ON_ROWS`` says which way the tile is laid out: the dot-product kernels want K as + ``[D, N]`` and V as ``[N, D]``, and the block axis has to be expanded along whichever + one carries ``head_dim``. + + The scale varies along ``head_dim``, the reduction dimension of ``q @ k``, so it + cannot be folded in after the dot -- the tile is dequantized into ``out_dtype`` (the + query's dtype) and fed to the tensor cores like the bf16 path does. + """ + if QUANT and EPB == 2: + # Nibble-packed GGML Q4_0: uint8 byte per element pair; low nibble = even + # element, high nibble = odd element, and value = (nibble - 8) * signed scale. + # Build a byte-sized tile and interleave its nibbles after the load. The old + # logical-element tile addressed ``elem // 2`` and therefore fetched every byte + # twice. Keeping packed space until unpack halves KV load instructions/traffic. + if D_ON_ROWS: + nb_p: tl.constexpr = scale_offsets.shape[0] + n_p: tl.constexpr = scale_offsets.shape[1] + packed_d: tl.constexpr = nb_p * QBLOCK // EPB + packed_elem = tl.arange(0, packed_d)[:, None] + packed_mask = tl.broadcast_to( + scale_mask[:, None, :], (nb_p, QBLOCK // EPB, n_p) + ).reshape(packed_d, n_p) + else: + n_p: tl.constexpr = scale_offsets.shape[0] + nb_p: tl.constexpr = scale_offsets.shape[1] + packed_d: tl.constexpr = nb_p * QBLOCK // EPB + packed_elem = tl.arange(0, packed_d)[None, :] + packed_mask = tl.broadcast_to( + scale_mask[:, :, None], (n_p, nb_p, QBLOCK // EPB) + ).reshape(n_p, packed_d) + packed = tl.load(ptr + base + packed_elem, mask=packed_mask, other=0) + # Triton 3.6 lowers the integer bit operations directly; unlike modulo and + # division they do not introduce integer arithmetic on every KV element. + lo = (packed & 15).to(tl.float32) + hi = (packed >> 4).to(tl.float32) + if D_ON_ROWS: + vals = tl.interleave(lo.trans(), hi.trans()).trans() + else: + vals = tl.interleave(lo, hi) + # Masked lanes must read as 0 (like the ``other=0.0`` loads in the element path), + # not as the -7 bias, or they would poison the dot when head_dim is not a power + # of two and BLOCK_D probes past D. + if D_ON_ROWS: + logical_mask = tl.broadcast_to( + scale_mask[:, None, :], (nb_p, QBLOCK, n_p) + ).reshape(nb_p * QBLOCK, n_p) + else: + logical_mask = tl.broadcast_to( + scale_mask[:, :, None], (n_p, nb_p, QBLOCK) + ).reshape(n_p, nb_p * QBLOCK) + vals = tl.where(logical_mask, vals - 8.0, 0.0) + scale = tl.load(scale_ptr + scale_offsets, mask=scale_mask, other=0.0) + if D_ON_ROWS: + wide = tl.broadcast_to(scale[:, None, :], (nb_p, QBLOCK, n_p)).reshape(nb_p * QBLOCK, n_p) + else: + wide = tl.broadcast_to(scale[:, :, None], (n_p, nb_p, QBLOCK)).reshape(n_p, nb_p * QBLOCK) + return (vals * wide.to(tl.float32)).to(out_dtype) + + vals = tl.load(ptr + base + elem, mask=mask, other=0.0) + if QUANT: + scale = tl.load(scale_ptr + scale_offsets, mask=scale_mask, other=0.0) + if D_ON_ROWS: + nb: tl.constexpr = scale.shape[0] + n: tl.constexpr = scale.shape[1] + wide = tl.broadcast_to(scale[:, None, :], (nb, QBLOCK, n)).reshape(nb * QBLOCK, n) + else: + n: tl.constexpr = scale.shape[0] + nb: tl.constexpr = scale.shape[1] + wide = tl.broadcast_to(scale[:, :, None], (n, nb, QBLOCK)).reshape(n, nb * QBLOCK) + return (vals.to(tl.float32) * wide.to(tl.float32)).to(out_dtype) + # Both branches must yield the same type for Triton to compile the function, so the + # unquantized path casts too. Callers pass the dtype the tile already has there + # (float32 for the fp32 kernel, the cache's own dtype elsewhere), so it is a no-op. + return vals.to(out_dtype) + + @functools.lru_cache(maxsize=None) def _optin_smem_bytes(device_index: int) -> int: """Per-block opt-in shared-memory budget for a CUDA device (0 if unavailable).""" @@ -47,6 +193,8 @@ def _paged_attention_kernel( q_ptr, k_ptr, v_ptr, + ks_ptr, + vs_ptr, o_ptr, indptr_ptr, indices_ptr, @@ -60,6 +208,10 @@ def _paged_attention_kernel( stride_kh, stride_vs, stride_vh, + stride_kss, + stride_ksh, + stride_vss, + stride_vsh, stride_ot, stride_oh, GROUP: tl.constexpr, @@ -68,6 +220,9 @@ def _paged_attention_kernel( BLOCK_N: tl.constexpr, SLIDING_WINDOW: tl.constexpr, HAS_SINKS: tl.constexpr, + QUANT: tl.constexpr, + QBLOCK: tl.constexpr, + EPB: tl.constexpr, ): q_tok = tl.program_id(0) q_head = tl.program_id(1) @@ -81,6 +236,9 @@ def _paged_attention_kernel( offs_d = tl.arange(0, BLOCK_D) mask_d = offs_d < D + # One scale per QBLOCK elements of head_dim: the tile's block axis. + offs_nb = tl.arange(0, BLOCK_D // QBLOCK) + mask_nb = offs_nb < D // QBLOCK q = tl.load( q_ptr + q_tok * stride_qt + q_head * stride_qh + offs_d, mask=mask_d, @@ -107,13 +265,21 @@ def _paged_attention_kernel( skip_tile = tl.max(mask_n.to(tl.int32), axis=0) == 0 if not skip_tile: slots = tl.load(indices_ptr + kv_start + offs_n, mask=offs_n < kv_len, other=0) - k = tl.load( - k_ptr - + slots[:, None] * stride_ks - + kv_head * stride_kh - + offs_d[None, :], - mask=(offs_n[:, None] < kv_len) & mask_d[None, :], - other=0.0, + kv_mask = (offs_n[:, None] < kv_len) & mask_d[None, :] + kv_scale_mask = (offs_n[:, None] < kv_len) & mask_nb[None, :] + k = _load_kv( + k_ptr, + ks_ptr, + slots[:, None] * stride_ks + kv_head * stride_kh, + offs_d[None, :], + slots[:, None] * stride_kss + kv_head * stride_ksh + offs_nb[None, :], + kv_mask, + kv_scale_mask, + tl.float32, + QUANT, + QBLOCK, + False, + EPB, ).to(tl.float32) scores = tl.sum(q[None, :] * k, axis=1) * sm_scale scores = tl.where(mask_n, scores, -float("inf")) @@ -124,13 +290,19 @@ def _paged_attention_kernel( alpha = tl.exp(m_i - m_new) p = tl.exp(scores - m_new) - v = tl.load( - v_ptr - + slots[:, None] * stride_vs - + kv_head * stride_vh - + offs_d[None, :], - mask=(offs_n[:, None] < kv_len) & mask_d[None, :], - other=0.0, + v = _load_kv( + v_ptr, + vs_ptr, + slots[:, None] * stride_vs + kv_head * stride_vh, + offs_d[None, :], + slots[:, None] * stride_vss + kv_head * stride_vsh + offs_nb[None, :], + kv_mask, + kv_scale_mask, + tl.float32, + QUANT, + QBLOCK, + False, + EPB, ).to(tl.float32) acc = acc * alpha + tl.sum(p[:, None] * v, axis=0) l_i = l_i * alpha + tl.sum(p, axis=0) @@ -149,6 +321,8 @@ def _decode_grouped_stage1_kernel( q_ptr, k_ptr, v_ptr, + ks_ptr, + vs_ptr, sm_scale, indptr_ptr, indices_ptr, @@ -162,6 +336,10 @@ def _decode_grouped_stage1_kernel( stride_kh, stride_vs, stride_vh, + stride_kss, + stride_ksh, + stride_vss, + stride_vsh, stride_mid_ob, stride_mid_oh, stride_mid_os, @@ -179,6 +357,9 @@ def _decode_grouped_stage1_kernel( D: tl.constexpr, DV: tl.constexpr, SLIDING_WINDOW: tl.constexpr, + QUANT: tl.constexpr, + QBLOCK: tl.constexpr, + EPB: tl.constexpr, ): batch_id = tl.program_id(0) head_block_id = tl.program_id(1) @@ -197,6 +378,10 @@ def _decode_grouped_stage1_kernel( offs_dv = tl.arange(0, BLOCK_DV) mask_d = offs_d < D mask_dv = offs_dv < DV + offs_nb = tl.arange(0, BLOCK_D // QBLOCK) + offs_nbv = tl.arange(0, BLOCK_DV // QBLOCK) + mask_nb = offs_nb < D // QBLOCK + mask_nbv = offs_nbv < DV // QBLOCK kv_start = tl.load(indptr_ptr + batch_id) kv_len = tl.load(indptr_ptr + batch_id + 1) - kv_start @@ -219,12 +404,18 @@ def _decode_grouped_stage1_kernel( acc = tl.zeros((BLOCK_H, BLOCK_DV), dtype=tl.float32) q_offsets = batch_id * stride_qt + q_heads[:, None] * stride_qh + offs_d[None, :] - k_base_offsets = kv_head * stride_kh + offs_d[:, None] - v_base_offsets = kv_head * stride_vh + offs_dv[None, :] + k_base_offsets = kv_head * stride_kh + v_base_offsets = kv_head * stride_vh + ks_base_offsets = kv_head * stride_ksh + offs_nb[:, None] + vs_base_offsets = kv_head * stride_vsh + offs_nbv[None, :] if split_end > split_start: q = tl.load(q_ptr + q_offsets, mask=mask_h[:, None] & mask_d[None, :], other=0.0) - q = q.to(k_ptr.dtype.element_ty) + if not QUANT: + # Unquantized: match the cache's dtype as before. Quantized: the cache is + # int8/fp8 and casting q into it would destroy the query -- the dequantized + # K/V tiles are produced in q's dtype instead. + q = q.to(k_ptr.dtype.element_ty) for rel_start in tl.range(split_start, split_end, BLOCK_N): rel_offs = rel_start + tl.arange(0, BLOCK_N) @@ -232,18 +423,36 @@ def _decode_grouped_stage1_kernel( logical_offs = effective_start + rel_offs slots = tl.load(indices_ptr + kv_start + logical_offs, mask=mask_n, other=0) - k = tl.load( - k_ptr + slots[None, :] * stride_ks + k_base_offsets, - mask=mask_n[None, :] & mask_d[:, None], - other=0.0, + k = _load_kv( + k_ptr, + ks_ptr, + slots[None, :] * stride_ks + k_base_offsets, + offs_d[:, None], + slots[None, :] * stride_kss + ks_base_offsets, + mask_n[None, :] & mask_d[:, None], + mask_n[None, :] & mask_nb[:, None], + q.dtype, + QUANT, + QBLOCK, + True, + EPB, ) scores = tl.dot(q, k) * sm_scale scores = tl.where(mask_h[:, None] & mask_n[None, :], scores, -float("inf")) - v = tl.load( - v_ptr + slots[:, None] * stride_vs + v_base_offsets, - mask=mask_n[:, None] & mask_dv[None, :], - other=0.0, + v = _load_kv( + v_ptr, + vs_ptr, + slots[:, None] * stride_vs + v_base_offsets, + offs_dv[None, :], + slots[:, None] * stride_vss + vs_base_offsets, + mask_n[:, None] & mask_dv[None, :], + mask_n[:, None] & mask_nbv[None, :], + q.dtype, + QUANT, + QBLOCK, + False, + EPB, ) m_new = tl.maximum(tl.max(scores, axis=1), m_i) @@ -349,6 +558,42 @@ def _decode_stage2_kernel( ) +def _kv_scale_args(k_cache, v_cache, k_scale, v_scale, epb: int = 1): + """Scale tensors + strides + the QUANT/QBLOCK/EPB constexprs for a kernel launch. + + Unquantized pools pass ``k_scale=None``; the kernels then never touch the scale + pointers, so the KV buffers themselves stand in and ``QUANT=False`` compiles the + dequant away entirely -- the bf16 path emits the same code it did before. + + ``epb`` (elements-per-byte) is 1 for 8-bit element-shaped storage and 2 for packed + int4, whose slab's last dim is D // epb. Callers infer it from + ``head_dim // k_cache.shape[-1]`` when quantized (== 1 unquantized, since there the + slab is element-shaped in the compute dtype). Everything below (``head_dim``, + ``BLOCK_D``, ``offs_d``, scales) stays in LOGICAL element space; only ``_load_kv``'s + byte addressing divides by ``epb``. + """ + if k_scale is None: + assert v_scale is None, "k_scale and v_scale must be given together" + return (k_cache, v_cache, 0, 0, 0, 0, False, 1, epb) + assert v_scale is not None, "k_scale and v_scale must be given together" + assert k_scale.dim() == v_scale.dim() == 3, "scales are [slots, heads, D // block]" + block = k_cache.shape[-1] * epb // k_scale.shape[-1] + assert k_cache.shape[-1] * epb == block * k_scale.shape[-1], ( + f"head_dim {k_cache.shape[-1] * epb} is not a whole number of {k_scale.shape[-1]} blocks" + ) + return ( + k_scale, + v_scale, + k_scale.stride(0), + k_scale.stride(1), + v_scale.stride(0), + v_scale.stride(1), + True, + block, + epb, + ) + + def decode_paged_attention( q: torch.Tensor, k_cache: torch.Tensor, @@ -364,16 +609,22 @@ def decode_paged_attention( sliding_window: int | None = None, sinks: torch.Tensor | None = None, out: torch.Tensor | None = None, + k_scale: torch.Tensor | None = None, + v_scale: torch.Tensor | None = None, ) -> torch.Tensor: """SGLang-style split-k grouped decode attention for one query per request.""" assert q.is_cuda and k_cache.is_cuda and v_cache.is_cuda assert q.dim() == 3 and k_cache.dim() == 3 and v_cache.dim() == 3 batch, num_q_heads, head_dim = q.shape + epb = head_dim // k_cache.shape[-1] + ks, vs, s_kss, s_ksh, s_vss, s_vsh, quant, qblock, epb = _kv_scale_args( + k_cache, v_cache, k_scale, v_scale, epb + ) num_kv_heads = k_cache.shape[1] assert batch == indptr.numel() - 1 assert v_cache.shape[1] == num_kv_heads - assert k_cache.shape[-1] == head_dim and v_cache.shape[-1] == head_dim + assert k_cache.shape[-1] * epb == head_dim and v_cache.shape[-1] * epb == head_dim assert num_q_heads % num_kv_heads == 0 assert attn_logits.shape[0] >= batch assert attn_logits.shape[1] >= num_q_heads @@ -391,6 +642,18 @@ def decode_paged_attention( o = out if out is not None else torch.empty_like(q) sinks_arg = sinks if sinks is not None else q group = num_q_heads // num_kv_heads + quant_name = "int4" if quant and epb == 2 else ("quant8" if quant else None) + preferred_splits, block_n, num_warps = decode_launch_config( + quant_name=quant_name, + head_dim=head_dim, + num_q_heads=num_q_heads, + num_kv_heads=num_kv_heads, + compute_capability=torch.cuda.get_device_capability(q.device), + ) + # Direct kernel callers and older capture buffers may provide less scratch; + # retain correctness and use every split they made available. The backend + # allocates the preferred capacity for new captures. + launch_splits = min(preferred_splits, max_kv_splits) # valid_block_h = heads computed per program (drives the grid + head indexing); block_h = # power-of-two tile size for tl.arange. They differ only for non-power-of-two GQA groups # (e.g. 6), where block_h rounds up and the kernel masks the extra lanes. @@ -400,11 +663,13 @@ def decode_paged_attention( block_dv = triton.next_power_of_2(head_dim) _decode_grouped_stage1_kernel[ - (batch, triton.cdiv(num_q_heads, valid_block_h), max_kv_splits) + (batch, triton.cdiv(num_q_heads, valid_block_h), launch_splits) ]( q, k_cache, v_cache, + ks, + vs, sm_scale, indptr, indices, @@ -418,6 +683,10 @@ def decode_paged_attention( k_cache.stride(1), v_cache.stride(0), v_cache.stride(1), + s_kss, + s_ksh, + s_vss, + s_vsh, attn_logits.stride(0), attn_logits.stride(1), attn_logits.stride(2), @@ -428,14 +697,17 @@ def decode_paged_attention( NUM_Q_HEADS=num_q_heads, BLOCK_D=block_d, BLOCK_DV=block_dv, - BLOCK_N=32, + BLOCK_N=block_n, BLOCK_H=block_h, VALID_BLOCK_H=valid_block_h, MIN_BLOCK_KV=_MIN_BLOCK_KV, D=head_dim, DV=head_dim, SLIDING_WINDOW=sliding_window or 0, - num_warps=4, + QUANT=quant, + QBLOCK=qblock, + EPB=epb, + num_warps=num_warps, num_stages=2, ) _decode_stage2_kernel[(batch, num_q_heads)]( @@ -454,7 +726,7 @@ def decode_paged_attention( attn_lse.stride(2), o.stride(0), o.stride(1), - MAX_KV_SPLITS=max_kv_splits, + MAX_KV_SPLITS=launch_splits, MIN_BLOCK_KV=_MIN_BLOCK_KV, BLOCK_DV=block_dv, DV=head_dim, @@ -471,6 +743,8 @@ def _extend_attention_kernel( q_ptr, k_ptr, v_ptr, + ks_ptr, + vs_ptr, o_ptr, qo_indptr_ptr, kv_indptr_ptr, @@ -484,6 +758,10 @@ def _extend_attention_kernel( stride_kh, stride_vs, stride_vh, + stride_kss, + stride_ksh, + stride_vss, + stride_vsh, stride_ot, stride_oh, GROUP: tl.constexpr, @@ -494,6 +772,9 @@ def _extend_attention_kernel( BLOCK_N: tl.constexpr, SLIDING_WINDOW: tl.constexpr, HAS_SINKS: tl.constexpr, + QUANT: tl.constexpr, + QBLOCK: tl.constexpr, + EPB: tl.constexpr, ): seq_id = tl.program_id(0) q_head = tl.program_id(1) @@ -514,6 +795,10 @@ def _extend_attention_kernel( mask_m = offs_m < q_len mask_d = offs_d < D mask_dv = offs_dv < D + offs_nb = tl.arange(0, BLOCK_D // QBLOCK) + offs_nbv = tl.arange(0, BLOCK_DV // QBLOCK) + mask_nb = offs_nb < D // QBLOCK + mask_nbv = offs_nbv < D // QBLOCK q_abs_pos = prefix_len + offs_m block_q_end = tl.minimum(q_len, (block_m_id + 1) * BLOCK_M) kv_loop_end = tl.minimum(kv_len, prefix_len + block_q_end) @@ -545,13 +830,19 @@ def _extend_attention_kernel( skip_tile = tl.max(tl.max(final_mask.to(tl.int32), axis=1), axis=0) == 0 if not skip_tile: slots = tl.load(kv_indices_ptr + kv_start + kv_offsets, mask=mask_n, other=0) - k = tl.load( - k_ptr - + slots[None, :] * stride_ks - + kv_head * stride_kh - + offs_d[:, None], - mask=mask_n[None, :] & mask_d[:, None], - other=0.0, + k = _load_kv( + k_ptr, + ks_ptr, + slots[None, :] * stride_ks + kv_head * stride_kh, + offs_d[:, None], + slots[None, :] * stride_kss + kv_head * stride_ksh + offs_nb[:, None], + mask_n[None, :] & mask_d[:, None], + mask_n[None, :] & mask_nb[:, None], + q.dtype, + QUANT, + QBLOCK, + True, + EPB, ) scores = tl.dot(q.to(k.dtype), k) * sm_scale scores = tl.where(final_mask, scores, -float("inf")) @@ -562,13 +853,19 @@ def _extend_attention_kernel( alpha = tl.exp(m_i - m_new) p = tl.exp(scores - m_new[:, None]) - v = tl.load( - v_ptr - + slots[:, None] * stride_vs - + kv_head * stride_vh - + offs_dv[None, :], - mask=mask_n[:, None] & mask_dv[None, :], - other=0.0, + v = _load_kv( + v_ptr, + vs_ptr, + slots[:, None] * stride_vs + kv_head * stride_vh, + offs_dv[None, :], + slots[:, None] * stride_vss + kv_head * stride_vsh + offs_nbv[None, :], + mask_n[:, None] & mask_dv[None, :], + mask_n[:, None] & mask_nbv[None, :], + q.dtype, + QUANT, + QBLOCK, + False, + EPB, ) acc = acc * alpha[:, None] + tl.dot(p.to(v.dtype), v) l_i = l_i * alpha + tl.sum(p, axis=1) @@ -592,6 +889,8 @@ def _extend_attention_split_kernel( v_extend_ptr, k_cache_ptr, v_cache_ptr, + ks_ptr, + vs_ptr, o_ptr, qo_indptr_ptr, kv_indptr_ptr, @@ -609,6 +908,10 @@ def _extend_attention_split_kernel( stride_kch, stride_vcs, stride_vch, + stride_kss, + stride_ksh, + stride_vss, + stride_vsh, stride_ot, stride_oh, GROUP: tl.constexpr, @@ -619,6 +922,9 @@ def _extend_attention_split_kernel( BLOCK_N: tl.constexpr, SLIDING_WINDOW: tl.constexpr, HAS_SINKS: tl.constexpr, + QUANT: tl.constexpr, + QBLOCK: tl.constexpr, + EPB: tl.constexpr, ): seq_id = tl.program_id(0) q_head = tl.program_id(1) @@ -637,6 +943,10 @@ def _extend_attention_split_kernel( mask_m = offs_m < q_len mask_d = offs_d < D mask_dv = offs_dv < D + offs_nb = tl.arange(0, BLOCK_D // QBLOCK) + offs_nbv = tl.arange(0, BLOCK_DV // QBLOCK) + mask_nb = offs_nb < D // QBLOCK + mask_nbv = offs_nbv < D // QBLOCK q_abs_pos = prefix_len + offs_m q = tl.load( @@ -672,13 +982,19 @@ def _extend_attention_split_kernel( if not skip_tile: slots = tl.load(kv_indices_ptr + kv_start + kv_offsets, mask=mask_n, other=0) - k = tl.load( - k_cache_ptr - + slots[None, :] * stride_kcs - + kv_head * stride_kch - + offs_d[:, None], - mask=mask_n[None, :] & mask_d[:, None], - other=0.0, + k = _load_kv( + k_cache_ptr, + ks_ptr, + slots[None, :] * stride_kcs + kv_head * stride_kch, + offs_d[:, None], + slots[None, :] * stride_kss + kv_head * stride_ksh + offs_nb[:, None], + mask_n[None, :] & mask_d[:, None], + mask_n[None, :] & mask_nb[:, None], + q.dtype, + QUANT, + QBLOCK, + True, + EPB, ) scores = tl.dot(q.to(k.dtype), k) * sm_scale scores = tl.where(final_mask, scores, -float("inf")) @@ -689,13 +1005,19 @@ def _extend_attention_split_kernel( alpha = tl.exp(m_i - m_new) p = tl.exp(scores - m_new[:, None]) - v = tl.load( - v_cache_ptr - + slots[:, None] * stride_vcs - + kv_head * stride_vch - + offs_dv[None, :], - mask=mask_n[:, None] & mask_dv[None, :], - other=0.0, + v = _load_kv( + v_cache_ptr, + vs_ptr, + slots[:, None] * stride_vcs + kv_head * stride_vch, + offs_dv[None, :], + slots[:, None] * stride_vss + kv_head * stride_vsh + offs_nbv[None, :], + mask_n[:, None] & mask_dv[None, :], + mask_n[:, None] & mask_nbv[None, :], + q.dtype, + QUANT, + QBLOCK, + False, + EPB, ) acc = acc * alpha[:, None] + tl.dot(p.to(v.dtype), v) l_i = l_i * alpha + tl.sum(p, axis=1) @@ -773,17 +1095,23 @@ def extend_paged_attention( out: torch.Tensor | None = None, k_extend: torch.Tensor | None = None, v_extend: torch.Tensor | None = None, + k_scale: torch.Tensor | None = None, + v_scale: torch.Tensor | None = None, ) -> torch.Tensor: """Block-tiled causal prefill/extend attention over paged KV cache.""" assert q.is_cuda and k_cache.is_cuda and v_cache.is_cuda assert q.dim() == 3 and k_cache.dim() == 3 and v_cache.dim() == 3 num_q_tokens, num_q_heads, head_dim = q.shape + epb = head_dim // k_cache.shape[-1] + ks, vs, s_kss, s_ksh, s_vss, s_vsh, quant, qblock, epb = _kv_scale_args( + k_cache, v_cache, k_scale, v_scale, epb + ) num_kv_heads = k_cache.shape[1] assert qo_indptr.numel() == kv_indptr.numel() assert prefix_lens.numel() == qo_indptr.numel() - 1 assert v_cache.shape[1] == num_kv_heads - assert k_cache.shape[-1] == head_dim and v_cache.shape[-1] == head_dim + assert k_cache.shape[-1] * epb == head_dim and v_cache.shape[-1] * epb == head_dim assert num_q_heads % num_kv_heads == 0 if sinks is not None: assert sinks.is_cuda @@ -801,6 +1129,16 @@ def extend_paged_attention( block_m, block_n = _select_extend_tile( head_dim, block_d, _optin_smem_bytes(q.device.index) ) + # On sm_89 the consumer-safe 64x32 tile for D=256 still has room for a + # second software-pipeline stage. It halves cold-chunk time (68 -> 35 ms + # per Ornith full-attention layer at 8K) and remains ~10% faster once an + # 8K quantized prefix is present. Larger-D fallback tiles stay at one stage. + num_stages = 2 if (head_dim, block_m, block_n) == (256, 64, 32) else 1 + # Swept on RTX 5080 (sm_120): 4 warps beat 8 at the consumer (64, 32) D=256 + # tile for both extend kernels (2.01x cold prefill, 1.12x long-Q4-prefix + # extension); sm_89 measured faster with 8. BLOCK_N=16 corrupts the packed + # loader in the extend kernels too and must never be selected here. + num_warps = 4 if torch.cuda.get_device_capability(q.device) >= (12, 0) else 8 grid = (qo_indptr.numel() - 1, num_q_heads, triton.cdiv(max_q_len, block_m)) if k_extend is not None or v_extend is not None: assert k_extend is not None and v_extend is not None @@ -815,6 +1153,8 @@ def extend_paged_attention( v_extend, k_cache, v_cache, + ks, + vs, o, qo_indptr, kv_indptr, @@ -832,6 +1172,10 @@ def extend_paged_attention( k_cache.stride(1), v_cache.stride(0), v_cache.stride(1), + s_kss, + s_ksh, + s_vss, + s_vsh, o.stride(0), o.stride(1), GROUP=num_q_heads // num_kv_heads, @@ -842,8 +1186,11 @@ def extend_paged_attention( BLOCK_N=block_n, SLIDING_WINDOW=sliding_window or 0, HAS_SINKS=sinks is not None, - num_warps=8, - num_stages=1, + QUANT=quant, + QBLOCK=qblock, + EPB=epb, + num_warps=num_warps, + num_stages=num_stages, ) return o @@ -851,6 +1198,8 @@ def extend_paged_attention( q, k_cache, v_cache, + ks, + vs, o, qo_indptr, kv_indptr, @@ -864,6 +1213,10 @@ def extend_paged_attention( k_cache.stride(1), v_cache.stride(0), v_cache.stride(1), + s_kss, + s_ksh, + s_vss, + s_vsh, o.stride(0), o.stride(1), GROUP=num_q_heads // num_kv_heads, @@ -874,8 +1227,11 @@ def extend_paged_attention( BLOCK_N=block_n, SLIDING_WINDOW=sliding_window or 0, HAS_SINKS=sinks is not None, - num_warps=8, - num_stages=1, + QUANT=quant, + QBLOCK=qblock, + EPB=epb, + num_warps=num_warps, + num_stages=num_stages, ) return o @@ -893,6 +1249,8 @@ def paged_attention( sinks: torch.Tensor | None = None, out: torch.Tensor | None = None, block_n: int = 32, + k_scale: torch.Tensor | None = None, + v_scale: torch.Tensor | None = None, ) -> torch.Tensor: """Paged causal attention for one layer. @@ -904,9 +1262,13 @@ def paged_attention( assert q.is_cuda and k_cache.is_cuda and v_cache.is_cuda assert q.dim() == 3 and k_cache.dim() == 3 and v_cache.dim() == 3 num_tokens, num_q_heads, head_dim = q.shape + epb = head_dim // k_cache.shape[-1] + ks, vs, s_kss, s_ksh, s_vss, s_vsh, quant, qblock, epb = _kv_scale_args( + k_cache, v_cache, k_scale, v_scale, epb + ) num_kv_heads = k_cache.shape[1] assert v_cache.shape[1] == num_kv_heads - assert k_cache.shape[-1] == head_dim and v_cache.shape[-1] == head_dim + assert k_cache.shape[-1] * epb == head_dim and v_cache.shape[-1] * epb == head_dim assert num_q_heads % num_kv_heads == 0 if sinks is not None: assert sinks.is_cuda @@ -922,6 +1284,8 @@ def paged_attention( q, k_cache, v_cache, + ks, + vs, o, indptr, indices, @@ -935,6 +1299,10 @@ def paged_attention( k_cache.stride(1), v_cache.stride(0), v_cache.stride(1), + s_kss, + s_ksh, + s_vss, + s_vsh, o.stride(0), o.stride(1), GROUP=num_q_heads // num_kv_heads, @@ -943,6 +1311,9 @@ def paged_attention( BLOCK_N=block_n, SLIDING_WINDOW=sliding_window or 0, HAS_SINKS=sinks is not None, + QUANT=quant, + QBLOCK=qblock, + EPB=epb, num_warps=8 if head_dim >= 256 else 4, num_stages=2, ) diff --git a/python/freetoken/kernel/triton/causal_conv1d_triton.py b/python/freetoken/kernel/triton/causal_conv1d_triton.py index 811132fc..e0fd87dd 100644 --- a/python/freetoken/kernel/triton/causal_conv1d_triton.py +++ b/python/freetoken/kernel/triton/causal_conv1d_triton.py @@ -426,6 +426,7 @@ def causal_conv1d_varlen( cu_seqlens: torch.Tensor, # [batch+1] int32 cache_indices: torch.Tensor, # [batch] int32 has_initial_state: torch.Tensor, # [batch] bool + bias: torch.Tensor | None = None, activation: Optional[str] = "silu", pad_slot_id: int = PAD_SLOT_ID, max_seq_len: Optional[int] = None, @@ -478,7 +479,7 @@ def grid(META): _causal_conv1d_fwd_tiled_kernel[grid]( x, weight, - None, + bias, conv_states, cache_indices, has_initial_state, @@ -499,7 +500,7 @@ def grid(META): stride_o_dim, stride_o_token, pad_slot_id, - HAS_BIAS=False, + HAS_BIAS=bias is not None, KERNEL_WIDTH=width, SILU_ACTIVATION=activation in ["silu", "swish"], HAS_INITIAL_STATES=has_initial_state is not None, @@ -522,6 +523,7 @@ def causal_conv1d_decode( conv_state: torch.Tensor, # [num_slots, conv_dim, state_len>=kernel-1] (in place) weight: torch.Tensor, # [conv_dim, kernel] conv_state_indices: torch.Tensor, # [batch] int32 + bias: torch.Tensor | None = None, activation: Optional[str] = "silu", pad_slot_id: int = PAD_SLOT_ID, ) -> torch.Tensor: @@ -550,7 +552,7 @@ def grid(META): _causal_conv1d_update_kernel[grid]( x, weight, - None, + bias, conv_state, conv_state_indices, out, @@ -572,7 +574,7 @@ def grid(META): stride_o_dim, stride_o_token, pad_slot_id, - HAS_BIAS=False, + HAS_BIAS=bias is not None, KERNEL_WIDTH=width, SILU_ACTIVATION=activation in ["silu", "swish"], IS_CONTINUOUS_BATCHING=conv_state_indices is not None, @@ -607,13 +609,12 @@ def causal_conv1d_fn( ) -> torch.Tensor: """Varlen (prefill) depthwise causal conv with fused silu; conv_states updated in place; returns a fresh output tensor. Adapts the vendored-op signature to the - tuned causal_conv1d_varlen. ``bias`` must be None (the tuned kernel is bias-free). + tuned causal_conv1d_varlen. query_start_loc: (batch+1) int32 cumulative seqlens. seq_lens_cpu: host-side per-request lengths (a Python list) -> batch / max_seq_len without a device->host sync. """ - assert bias is None, "tuned triton causal_conv1d_fn does not support bias" batch = len(seq_lens_cpu) max_seq_len = int(max(seq_lens_cpu)) if batch else 0 return causal_conv1d_varlen( @@ -623,6 +624,7 @@ def causal_conv1d_fn( query_start_loc, cache_indices, has_initial_state, + bias=bias, activation=activation, pad_slot_id=pad_slot_id, max_seq_len=max_seq_len, @@ -648,7 +650,6 @@ def causal_conv1d_update( Accepts x as (batch, dim) or (batch, dim, 1) and returns the matching rank (kernel/causal_conv1d.py passes the 3D form and squeezes the result). """ - assert bias is None, "tuned triton causal_conv1d_update does not support bias" assert cache_seqlens is None, "circular-buffer cache_seqlens not supported" if isinstance(activation, bool): activation = "silu" if activation else None @@ -663,6 +664,7 @@ def causal_conv1d_update( conv_state, weight, conv_state_indices, + bias=bias, activation=activation, pad_slot_id=pad_slot_id, ) # [batch, dim] diff --git a/python/freetoken/kernel/triton/kv_quant.py b/python/freetoken/kernel/triton/kv_quant.py new file mode 100644 index 00000000..00a1ec7f --- /dev/null +++ b/python/freetoken/kernel/triton/kv_quant.py @@ -0,0 +1,191 @@ +"""Quantizing store into a compact KV pool. + +The unquantized path stores K/V with ``kernel/store.py``'s CUDA kernel, which is a +pure byte copy parameterized by element size. Quantized storage has to compute a scale +per block of :data:`~freetoken.kvcache.quant.BLOCK` elements along ``head_dim`` on the +way in, so it gets its own kernel here. The int4 path packs two signed values per byte. + +One program handles one ``(token, kv_head)`` pair: it loads that head's ``head_dim`` +values as a ``[head_dim // BLOCK, BLOCK]`` tile, reduces max-abs along the block, and +writes the quantized values plus one scale per block. K and V are done in the same +program -- they share the token's slot index and the tile geometry, so doing both +halves the launch count and the index math. +""" + +from __future__ import annotations + +import torch +import triton +import triton.language as tl + +from freetoken.kernel.triton.e4m3_compat import round_e4m3 + + +@triton.jit +def _store_kv_quant_kernel( + k_ptr, # [tokens, heads, D] source, compute dtype + v_ptr, + kc_ptr, # [slots, heads, D // EPB] destination, storage dtype + vc_ptr, + ks_ptr, # [slots, heads, D // BLOCK] scales, fp16 + vs_ptr, + indices_ptr, # [tokens] destination slot per token + stride_kt, + stride_kh, + stride_ct, + stride_ch, + stride_st, + stride_sh, + MAX_MAG: tl.constexpr, + IS_INT: tl.constexpr, + EPB: tl.constexpr, + BLOCK: tl.constexpr, + NBLOCK: tl.constexpr, +): + tok = tl.program_id(0) + head = tl.program_id(1) + slot = tl.load(indices_ptr + tok).to(tl.int64) + + if EPB == 2: + # Nibble-packed int4: tile the source as [NBLOCK, BLOCK // 2, 2], where the last + # axis is a (even, odd) element pair that becomes one byte. The scale is still + # per BLOCK (per row), computed over all BLOCK elements of the row. + pair_offs = ( + tl.arange(0, NBLOCK)[:, None, None] * BLOCK + + tl.arange(0, BLOCK // 2)[None, :, None] * 2 + + tl.arange(0, 2)[None, None, :] + ) + byte_offs = tl.arange(0, NBLOCK)[:, None] * (BLOCK // 2) + tl.arange(0, BLOCK // 2)[None, :] + scale_offs = tl.arange(0, NBLOCK) + + for is_v in tl.static_range(2): + src_ptr = v_ptr if is_v else k_ptr + dst_ptr = vc_ptr if is_v else kc_ptr + sc_ptr = vs_ptr if is_v else ks_ptr + + x = tl.load(src_ptr + tok * stride_kt + head * stride_kh + pair_offs).to(tl.float32) + flat_x = x.reshape(NBLOCK, BLOCK) + abs_x = tl.abs(flat_x) + amax = tl.max(abs_x, axis=1) # [NBLOCK] + # Match GGML Q4_0 exactly, including its first-element tie break for + # equally large positive and negative extrema. + extreme_idx = tl.argmax(abs_x, axis=1, tie_break_left=True) + block_idx = tl.arange(0, BLOCK)[None, :] + extreme = tl.sum( + tl.where(block_idx == extreme_idx[:, None], flat_x, 0.0), axis=1 + ) + scale = tl.where(amax > 0, extreme / -8.0, 1.0) + scale = scale.to(sc_ptr.dtype.element_ty).to(tl.float32) + # GGML reference truncates x / d + 8.5 into the unsigned nibble range. + # The value is nonnegative here, so floor is equivalent to truncation. + q = tl.math.div_rn(x, scale[:, None, None]) + q = tl.minimum(tl.maximum(tl.floor(q + 8.5), 0.0), 15.0).to(tl.int8) + # Low nibble = even (index 0), high nibble = odd (index 1). Triton has no + # integer indexing, so split the last axis and pack arithmetically. + even, odd = tl.split(q) + packed = even + odd * 16 + tl.store( + dst_ptr + slot * stride_ct + head * stride_ch + byte_offs, + packed.to(dst_ptr.dtype.element_ty), + ) + tl.store( + sc_ptr + slot * stride_st + head * stride_sh + scale_offs, + scale.to(sc_ptr.dtype.element_ty), + ) + return + + # EPB == 1: 8-bit element-shaped storage, one byte per element. + # [NBLOCK, BLOCK] tile over head_dim: rows are quant blocks, columns the elements + # sharing one scale. + offs = tl.arange(0, NBLOCK)[:, None] * BLOCK + tl.arange(0, BLOCK)[None, :] + scale_offs = tl.arange(0, NBLOCK) + + for is_v in tl.static_range(2): + src_ptr = v_ptr if is_v else k_ptr + dst_ptr = vc_ptr if is_v else kc_ptr + sc_ptr = vs_ptr if is_v else ks_ptr + + x = tl.load(src_ptr + tok * stride_kt + head * stride_kh + offs).to(tl.float32) + amax = tl.max(tl.abs(x), axis=1) + # An all-zero block quantizes to zeros under any positive scale; 1.0 keeps the + # division finite. + scale = tl.where(amax > 0, amax / MAX_MAG, 1.0) + # Round to the stored precision before dividing, so the value written here and + # the value the attention kernels read back are scaled by the identical number. + scale = scale.to(sc_ptr.dtype.element_ty).to(tl.float32) + # div_rn, not `/`: the plain operator is free to lower to a reciprocal multiply, + # which disagrees with the torch reference on values sitting exactly between two + # quantization steps. IEEE round-to-nearest divide makes the two bit-identical. + q = tl.math.div_rn(x, scale[:, None]) + if IS_INT: + # Round half away from zero (what GGUF's Q8_0 does), then clamp -- the + # float->int cast truncates. + q = tl.where(q >= 0, tl.floor(q + 0.5), tl.ceil(q - 0.5)) + q = tl.minimum(tl.maximum(q, -MAX_MAG), MAX_MAG) + else: + # The native fp32 -> float8e4nv downcast does not round to nearest on + # every arch (it lowers as a truncating fp32 -> fp16 -> e4m3 double-round + # on sm_89), so values just above a grid midpoint collapse downward and + # disagree with the RNE torch reference. Round explicitly first. + q = round_e4m3(tl.minimum(tl.maximum(q, -MAX_MAG), MAX_MAG)) + + tl.store( + dst_ptr + slot * stride_ct + head * stride_ch + offs, + q.to(dst_ptr.dtype.element_ty), + ) + tl.store( + sc_ptr + slot * stride_st + head * stride_sh + scale_offs, + scale.to(sc_ptr.dtype.element_ty), + ) + + +def store_kv_quant( + k_cache: torch.Tensor, + k_scale: torch.Tensor, + v_cache: torch.Tensor, + v_scale: torch.Tensor, + indices: torch.Tensor, + k: torch.Tensor, + v: torch.Tensor, + spec, +) -> None: + """Quantize ``k``/``v`` ``[tokens, heads, D]`` into the pool slots ``indices``. + + ``k_cache``/``v_cache`` are ``[slots, heads, D // EPB]`` in the spec's storage dtype + (``D // EPB == D`` for 8-bit, ``D // 2`` for packed int4) and ``k_scale``/``v_scale`` + ``[slots, heads, D // BLOCK]`` in fp16. + """ + from freetoken.kvcache.quant import BLOCK + + num_tokens, num_heads, head_dim = k.shape + if num_tokens == 0: + return + assert head_dim % BLOCK == 0, f"head_dim {head_dim} not a multiple of {BLOCK}" + assert k_cache.shape[1:] == (num_heads, head_dim // spec.elements_per_byte), ( + f"packed cache geometry {tuple(k_cache.shape[1:])} != " + f"{(num_heads, head_dim // spec.elements_per_byte)}" + ) + _store_kv_quant_kernel[(num_tokens, num_heads)]( + k, + v, + k_cache, + v_cache, + k_scale, + v_scale, + indices, + k.stride(0), + k.stride(1), + k_cache.stride(0), + k_cache.stride(1), + k_scale.stride(0), + k_scale.stride(1), + MAX_MAG=spec.max_magnitude, + IS_INT=spec.is_integer, + EPB=spec.elements_per_byte, + BLOCK=BLOCK, + NBLOCK=head_dim // BLOCK, + num_warps=4, + ) + + +__all__ = ["store_kv_quant"] diff --git a/python/freetoken/kvcache/__init__.py b/python/freetoken/kvcache/__init__.py index 1bb352b4..34fcc322 100644 --- a/python/freetoken/kvcache/__init__.py +++ b/python/freetoken/kvcache/__init__.py @@ -106,6 +106,7 @@ def create_kv_pool(config, num_pages: int, device: torch.device, dtype: torch.dt num_swa_tokens=num_swa_tokens, device=device, dtype=dtype, + quant=getattr(config, "kv_quant", None), ) @@ -116,7 +117,11 @@ def create_kvcache_pool( dtype: torch.dtype, device: torch.device, num_swa_tokens: int | None = None, + quant=None, ) -> BaseKVCachePool: + from .quant import NONE + + quant = quant if quant is not None else NONE if model_config.has_swa_attention: from .hybrid_swa_pool import HybridSWAKVCache @@ -128,6 +133,7 @@ def create_kvcache_pool( num_swa_tokens=num_swa_tokens, device=device, dtype=dtype, + quant=quant, ) from .mha_pool import MHAKVCache @@ -207,6 +213,7 @@ def create_kvcache_pool( device=device, dtype=dtype, layer_ids=layer_ids, + quant=quant, ) diff --git a/python/freetoken/kvcache/base.py b/python/freetoken/kvcache/base.py index ae8cf9ec..e63e5909 100644 --- a/python/freetoken/kvcache/base.py +++ b/python/freetoken/kvcache/base.py @@ -21,15 +21,29 @@ def spec_kv_bytes_per_token(spec, config) -> int: x layers, plus the bf16 DSA index-key slab when the spec carries indexer dims. Pure per-spec arithmetic -- pool families compose it over THEIR OWN groups; no family branching here. (2 bytes/elem == the torch.bfloat16 dsa_pool.DSAKVCache._alloc - hardcodes; keep the two in lockstep if the slab dtype ever changes.)""" + hardcodes; keep the two in lockstep if the slab dtype ever changes.) + + A quantized KV pool prices at its scheme's payload bytes plus the amortized fp16 + per-block scale (1.0625 bytes for 8-bit; 0.5625 for packed int4), not the compute + dtype. This number, times every token of every layer, is what frees VRAM for experts. + The index slab stays bf16: it is never quantized. + """ + bytes_per_elem = _kv_bytes_per_element(config) per_token = ( (1 if spec.mla else 2) # MLA latent groups store one slab (V aliases K) * spec.head_dim * div_even(spec.num_kv_heads, config.tp_info.size, allow_replicate=True) - * config.dtype.itemsize * spec.num_layers ) - return per_token + spec.index_head_dim * spec.num_index_layers * 2 + return int(per_token * bytes_per_elem) + spec.index_head_dim * spec.num_index_layers * 2 + + +def _kv_bytes_per_element(config) -> float: + """Storage bytes per K/V element for this engine config's KV pools.""" + quant = getattr(config, "kv_quant", None) + if quant is None or not quant.enabled: + return float(config.dtype.itemsize) + return quant.bytes_per_element(config.dtype) class BaseKVCachePool(ABC): diff --git a/python/freetoken/kvcache/hybrid_swa_pool.py b/python/freetoken/kvcache/hybrid_swa_pool.py index 41c3880e..1d32b40d 100644 --- a/python/freetoken/kvcache/hybrid_swa_pool.py +++ b/python/freetoken/kvcache/hybrid_swa_pool.py @@ -9,6 +9,8 @@ from freetoken.utils import align_ceil, div_even from .base import BaseKVCachePool +from .quant import NONE, KVQuantSpec +from .quant_storage import QuantizedKVStorageMixin @dataclass(frozen=True) @@ -23,9 +25,13 @@ class _KVGroupStorage: k_buffer: torch.Tensor v_buffer: torch.Tensor storage_shape: tuple[int, int, int] + # Per-block scales for an 8-bit group; None when the group stores the compute dtype. + scale_buffer: torch.Tensor | None = None + k_scale: torch.Tensor | None = None + v_scale: torch.Tensor | None = None -class HybridSWAKVCache(BaseKVCachePool): +class HybridSWAKVCache(QuantizedKVStorageMixin, BaseKVCachePool): """SGLang-style wrapper for hybrid full/SWA attention KV storage.""" def __init__( @@ -37,7 +43,9 @@ def __init__( dtype: torch.dtype, device: torch.device, num_swa_tokens: int | None = None, + quant: KVQuantSpec = NONE, ) -> None: + self._quant = quant specs = {group.name: group for group in groups if group.num_layers > 0} if set(specs) != {"full", "swa"}: raise ValueError(f"HybridSWAKVCache requires full and swa groups, got {sorted(specs)}") @@ -80,8 +88,8 @@ def __init__( if self._swa_paged: self._init_swa_paged_state() - @staticmethod def _allocate_group( + self, spec: KVCacheGroupSpec, tp_size: int, outer_size: int, @@ -90,16 +98,20 @@ def _allocate_group( device: torch.device, ) -> _KVGroupStorage: local_kv_heads = div_even(spec.num_kv_heads, tp_size, allow_replicate=True) - buffer = torch.empty( - (2, spec.num_layers, outer_size, inner_size, local_kv_heads, spec.head_dim), - device=device, - dtype=dtype, - ) + logical_head_dim = spec.head_dim + head_dim = logical_head_dim // self._quant.elements_per_byte # packed slabs halve it + phys_shape = (2, spec.num_layers, outer_size, inner_size, local_kv_heads, head_dim) + log_shape = (2, spec.num_layers, outer_size, inner_size, local_kv_heads, logical_head_dim) + buffer = torch.empty(phys_shape, device=device, dtype=self._buffer_dtype(dtype)) + scales = self._alloc_scales(log_shape, device) return _KVGroupStorage( buffer=buffer, k_buffer=buffer[0], v_buffer=buffer[1], - storage_shape=(outer_size * inner_size, local_kv_heads, spec.head_dim), + storage_shape=(outer_size * inner_size, local_kv_heads, head_dim), + scale_buffer=scales, + k_scale=None if scales is None else scales[0], + v_scale=None if scales is None else scales[1], ) @staticmethod @@ -200,6 +212,16 @@ def v_cache(self, index: int) -> torch.Tensor: ref = self.layers_mapping[index] return self._storages[ref.group].v_buffer[ref.index] + def k_scale(self, index: int) -> torch.Tensor | None: + ref = self.layers_mapping[index] + scales = self._storages[ref.group].k_scale + return None if scales is None else scales[ref.index] + + def v_scale(self, index: int) -> torch.Tensor | None: + ref = self.layers_mapping[index] + scales = self._storages[ref.group].v_scale + return None if scales is None else scales[ref.index] + def store_kv( self, k: torch.Tensor, @@ -207,19 +229,20 @@ def store_kv( out_loc: torch.Tensor, layer_id: int, ) -> None: - from freetoken.kernel import store_cache - ref = self.layers_mapping[layer_id] storage = self._storages[ref.group] indices = out_loc if ref.group == "swa": indices = self.translate_loc_from_full_to_swa(out_loc) - store_cache( - k_cache=storage.k_buffer[ref.index].view(storage.storage_shape), - v_cache=storage.v_buffer[ref.index].view(storage.storage_shape), - indices=indices, - k=k, - v=v, + scale_shape = (storage.storage_shape[0], storage.storage_shape[1], -1) + self._store_kv_into( + storage.k_buffer[ref.index].view(storage.storage_shape), + storage.v_buffer[ref.index].view(storage.storage_shape), + None if storage.k_scale is None else storage.k_scale[ref.index].view(scale_shape), + None if storage.v_scale is None else storage.v_scale[ref.index].view(scale_shape), + indices, + k, + v, ) @property @@ -245,24 +268,27 @@ def num_layers(self) -> int: @staticmethod def _group_geometry(group: _KVGroupStorage) -> tuple: # Everything the realloc needs that does NOT pin the old buffer alive: layer count, - # kv heads, head_dim, device, dtype. (Plain ints + device/dtype handles, no tensor.) + # kv heads, head_dim (physical storage dim -- half for packed int4), device, dtype. _, num_layers, _old_outer, _old_inner, local_kv_heads, head_dim = group.buffer.shape return (num_layers, local_kv_heads, head_dim, group.buffer.device, group.buffer.dtype) - @staticmethod - def _alloc_group(geom: tuple, outer_size: int, inner_size: int) -> _KVGroupStorage: + def _alloc_group(self, geom: tuple, outer_size: int, inner_size: int) -> _KVGroupStorage: # Only the outer (page/token) dimension changes; the rest comes from ``geom``. num_layers, local_kv_heads, head_dim, device, dtype = geom - buffer = torch.empty( - (2, num_layers, outer_size, inner_size, local_kv_heads, head_dim), - device=device, - dtype=dtype, - ) + epb = self._quant.elements_per_byte + shape = (2, num_layers, outer_size, inner_size, local_kv_heads, head_dim) + buffer = torch.empty(shape, device=device, dtype=dtype) + # Scales key off the LOGICAL head dim (extent D // BLOCK regardless of packing). + log_shape = (*shape[:-1], head_dim * epb) + scales = self._alloc_scales(log_shape, device) return _KVGroupStorage( buffer=buffer, k_buffer=buffer[0], v_buffer=buffer[1], storage_shape=(outer_size * inner_size, local_kv_heads, head_dim), + scale_buffer=scales, + k_scale=None if scales is None else scales[0], + v_scale=None if scales is None else scales[1], ) def rebuild(self, num_full_pages: int, num_swa_tokens: int | None = None) -> None: @@ -343,12 +369,17 @@ def rebuild_from_config( self.rebuild(num_full_pages=num_pages + 1, num_swa_tokens=num_swa_tokens) def unit_bytes(self) -> tuple[int, int]: + def group_bytes(group: _KVGroupStorage) -> int: + total = group.buffer.numel() * group.buffer.element_size() + if group.scale_buffer is not None: + total += group.scale_buffer.numel() * group.scale_buffer.element_size() + return int(total) + full = self.full_kv_pool.buffer - swa = self.swa_kv_pool.buffer full_tokens = int(full.shape[2]) * int(full.shape[3]) return ( - int(full.numel() * full.element_size()) // full_tokens, - int(swa.numel() * swa.element_size()) // self._swa_num_tokens, + group_bytes(self.full_kv_pool) // full_tokens, + group_bytes(self.swa_kv_pool) // self._swa_num_tokens, ) diff --git a/python/freetoken/kvcache/mha_pool.py b/python/freetoken/kvcache/mha_pool.py index 8ed280b9..c5d61911 100644 --- a/python/freetoken/kvcache/mha_pool.py +++ b/python/freetoken/kvcache/mha_pool.py @@ -7,9 +7,11 @@ from freetoken.utils import div_even from .base import BaseKVCachePool +from .quant import NONE, KVQuantSpec +from .quant_storage import QuantizedKVStorageMixin -class MHAKVCache(BaseKVCachePool): +class MHAKVCache(QuantizedKVStorageMixin, BaseKVCachePool): """ Base class for key-value caches. This class defines the interface for key-value caches used in LLMs. @@ -32,7 +34,9 @@ def __init__( dtype: torch.dtype, device: torch.device, layer_ids: Sequence[int] | None = None, + quant: KVQuantSpec = NONE, ) -> None: + self._quant = quant tp_info = get_tp_info() local_kv_heads = div_even(num_kv_heads, tp_info.size, allow_replicate=True) self._num_layers = num_layers @@ -47,15 +51,21 @@ def __init__( raise ValueError(f"KV layer id {global_id} outside [0, {num_layers})") layer_map[global_id] = dense self._layer_map = layer_map + self._compute_dtype = dtype + storage_head_dim = head_dim // self._quant.elements_per_byte + kv_shape = (2, num_storage_layers, num_pages, page_size, local_kv_heads, storage_head_dim) self._kv_buffer = torch.empty( - (2, num_storage_layers, num_pages, page_size, local_kv_heads, head_dim), - device=device, - dtype=dtype, + kv_shape, device=device, dtype=self._buffer_dtype(dtype) ) self._k_buffer = self._kv_buffer[0] self._v_buffer = self._kv_buffer[1] + # Scales key off the LOGICAL head_dim: extent D // BLOCK regardless of packing. + log_shape = (2, num_storage_layers, num_pages, page_size, local_kv_heads, head_dim) + self._scale_buffer = self._alloc_scales(log_shape, device) + self._k_scale = self._scale_buffer[0] if self._scale_buffer is not None else None + self._v_scale = self._scale_buffer[1] if self._scale_buffer is not None else None self._device = device - self._storage_shape = (num_pages * page_size, local_kv_heads, head_dim) + self._storage_shape = (num_pages * page_size, local_kv_heads, storage_head_dim) def rebuild(self, num_pages: int) -> None: """Reallocate the KV buffer for ``num_pages`` pages IN PLACE. @@ -64,23 +74,29 @@ def rebuild(self, num_pages: int) -> None: existing buffer; only the page count changes. Views and ``_storage_shape`` are refreshed. Object identity is preserved so cached backend references stay valid. """ - _, num_storage_layers, _old_pages, page_size, local_kv_heads, head_dim = self._kv_buffer.shape + _, num_storage_layers, _old_pages, page_size, local_kv_heads, storage_head_dim = self._kv_buffer.shape dtype = self._kv_buffer.dtype device = self._device self._k_buffer = None self._v_buffer = None self._kv_buffer = None + # Drop the scale slab too before reallocating, for the same reason the KV slab is + # dropped: holding the old one alive can OOM a rebuild the target size would fit. + self._k_scale = None + self._v_scale = None + self._scale_buffer = None if device.type == "cuda": torch.cuda.synchronize(device) torch.cuda.empty_cache() - self._kv_buffer = torch.empty( - (2, num_storage_layers, num_pages, page_size, local_kv_heads, head_dim), - device=device, - dtype=dtype, - ) + kv_shape = (2, num_storage_layers, num_pages, page_size, local_kv_heads, storage_head_dim) + self._kv_buffer = torch.empty(kv_shape, device=device, dtype=dtype) self._k_buffer = self._kv_buffer[0] self._v_buffer = self._kv_buffer[1] - self._storage_shape = (num_pages * page_size, local_kv_heads, head_dim) + log_shape = (2, num_storage_layers, num_pages, page_size, local_kv_heads, storage_head_dim * self._quant.elements_per_byte) + self._scale_buffer = self._alloc_scales(log_shape, device) + self._k_scale = self._scale_buffer[0] if self._scale_buffer is not None else None + self._v_scale = self._scale_buffer[1] if self._scale_buffer is not None else None + self._storage_shape = (num_pages * page_size, local_kv_heads, storage_head_dim) @classmethod def kv_cost(cls, config) -> tuple[int, int, int, int]: @@ -101,7 +117,10 @@ def rebuild_from_config( def unit_bytes(self) -> tuple[int, int]: buf = self._kv_buffer tokens = int(buf.shape[2]) * int(buf.shape[3]) - return int(buf.numel() * buf.element_size()) // tokens, 0 + total = buf.numel() * buf.element_size() + if self._scale_buffer is not None: + total += self._scale_buffer.numel() * self._scale_buffer.element_size() + return int(total) // tokens, 0 def _dense(self, layer_id: int) -> int: if self._layer_map is None: @@ -117,6 +136,12 @@ def k_cache(self, index: int) -> torch.Tensor: def v_cache(self, index: int) -> torch.Tensor: return self._v_buffer[self._dense(index)] + def k_scale(self, index: int) -> torch.Tensor | None: + return None if self._k_scale is None else self._k_scale[self._dense(index)] + + def v_scale(self, index: int) -> torch.Tensor | None: + return None if self._v_scale is None else self._v_scale[self._dense(index)] + def store_kv( self, k: torch.Tensor, @@ -124,15 +149,16 @@ def store_kv( out_loc: torch.Tensor, layer_id: int, ) -> None: - from freetoken.kernel import store_cache - dense = self._dense(layer_id) - store_cache( - k_cache=self._k_buffer[dense].view(self._storage_shape), - v_cache=self._v_buffer[dense].view(self._storage_shape), - indices=out_loc, - k=k, - v=v, + scale_shape = (self._storage_shape[0], self._storage_shape[1], -1) + self._store_kv_into( + self._k_buffer[dense].view(self._storage_shape), + self._v_buffer[dense].view(self._storage_shape), + None if self._k_scale is None else self._k_scale[dense].view(scale_shape), + None if self._v_scale is None else self._v_scale[dense].view(scale_shape), + out_loc, + k, + v, ) @property @@ -143,6 +169,10 @@ def device(self) -> torch.device: def dtype(self) -> torch.dtype: return self._kv_buffer.dtype + @property + def compute_dtype(self) -> torch.dtype: + return self._compute_dtype + @property def num_layers(self) -> int: return self._num_layers diff --git a/python/freetoken/kvcache/quant.py b/python/freetoken/kvcache/quant.py new file mode 100644 index 00000000..4ddd627a --- /dev/null +++ b/python/freetoken/kvcache/quant.py @@ -0,0 +1,191 @@ +"""KV-cache quantization schemes with a per-block scale. + +The KV pool normally stores K/V in the model's compute dtype (bf16). Quantized pools +store a compact payload plus a parallel scale tensor holding one fp16 scale per +:data:`BLOCK` elements along ``head_dim`` -- the same block geometry GGUF's Q8_0 uses. +The block is small because KV outliers (mostly in the keys) concentrate in a few +channels, and a block of 32 keeps an outlier from stretching the scale of the whole +head. + +The 8-bit schemes store one element per byte; ``int4`` stores two signed values in each +``uint8`` byte using llama.cpp/GGML Q4_0 scale selection. All share the store kernel and +the dequant path in the attention kernels; the format only changes payload layout and +the divisor mapping a block's extreme onto its representable range. The scale varies +along ``head_dim``, the reduction dimension of ``q @ k``, so attention dequantizes +before the dot. This saves storage bandwidth, not tensor-core compute. +""" + +from __future__ import annotations + +from dataclasses import dataclass + +import torch + +# Elements per scale, along head_dim. Matches GGUF Q8_0's block. +BLOCK = 32 +# One fp16 scale per block. +SCALE_DTYPE = torch.float16 + + +@dataclass(frozen=True) +class KVQuantSpec: + """How a KV pool stores its K/V elements. + + ``name`` is the ``--kv-cache-dtype`` value. ``storage_dtype`` is None for the + unquantized pool, in which case the pool allocates in the compute dtype and no + scale tensor exists. + + ``elements_per_byte`` is 1 for the 8-bit schemes (allocation is element-shaped) and + 2 for int4 (torch has no int4 dtype, so the slab is ``uint8`` with two 4-bit + elements packed per byte and the last dim halves). + """ + + name: str + storage_dtype: torch.dtype | None + # Max-abs of a block maps to this magnitude in the storage dtype. + max_magnitude: float + elements_per_byte: int = 1 + + @property + def enabled(self) -> bool: + return self.storage_dtype is not None + + @property + def is_integer(self) -> bool: + """Integer schemes round; float ones just divide.""" + return self.storage_dtype in (torch.int8, torch.uint8) + + @property + def packed(self) -> bool: + """True when the slab packs several elements per byte (int4).""" + return self.elements_per_byte > 1 + + def bytes_per_element(self, compute_dtype: torch.dtype) -> float: + """Storage bytes per K/V element, scales amortized over the block. + + Unquantized: the compute dtype's itemsize. 8-bit: 1 byte + 2/32 for the fp16 + scale = 1.0625. int4: half a byte + 2/32 = 0.5625. + """ + if not self.enabled: + return float(compute_dtype.itemsize) + return 1.0 / self.elements_per_byte + SCALE_DTYPE.itemsize / BLOCK + + def storage_shape(self, shape: tuple[int, ...]) -> tuple[int, ...]: + """Element-storage slab shape for a logical KV shape (last dim halves when packed).""" + if self.packed: + if shape[-1] % self.elements_per_byte: + raise ValueError( + f"head_dim {shape[-1]} is not a multiple of {self.elements_per_byte}" + ) + return (*shape[:-1], shape[-1] // self.elements_per_byte) + return shape + + def scale_shape(self, shape: tuple[int, ...]) -> tuple[int, ...]: + """Scale-tensor shape for a *logical* KV shape: last dim divided by the block. + + ``shape`` is the element-counted (unpacked) KV geometry, so the scale extent is + the same whether the slab is 8-bit (element-shaped) or int4 (byte-packed). + """ + if shape[-1] % BLOCK: + raise ValueError( + f"head_dim {shape[-1]} is not a multiple of the KV quant block {BLOCK}" + ) + return (*shape[:-1], shape[-1] // BLOCK) + + # ---- reference implementations (correctness oracle for the Triton kernels) ---- + + def quantize(self, x: torch.Tensor) -> tuple[torch.Tensor, torch.Tensor]: + """``x[..., D]`` (float) -> ``(quantized[..., D // epb], scales[..., D // BLOCK])``. + + The quantized tensor has the *storage* shape: element-shaped for 8-bit, + byte-packed (last dim halves) for int4. + """ + assert self.enabled, "quantize() on an unquantized spec" + blocks = x.float().unflatten(-1, (x.shape[-1] // BLOCK, BLOCK)) + if self.packed: + # Match GGML Q4_0. The signed value with the greatest magnitude selects a + # possibly-negative scale; codes 0..15 then represent (code - 8) * scale. + # This uses all 16 nibble values and preserves the block's largest-magnitude + # element exactly, unlike the former symmetric [-7, 7] scheme. + extreme = blocks.gather( + -1, blocks.abs().argmax(dim=-1, keepdim=True) + ).squeeze(-1) + scales = torch.where(extreme != 0, extreme / -8.0, torch.ones_like(extreme)) + scales = scales.to(SCALE_DTYPE) + q = torch.floor(blocks / scales.float().unsqueeze(-1) + 8.5).clamp_(0, 15) + q = q.flatten(-2).to(torch.uint8) + even = q[..., 0::2] + odd = q[..., 1::2] + return even | (odd << 4), scales + + amax = blocks.abs().amax(dim=-1) + scales = torch.where(amax > 0, amax / self.max_magnitude, torch.ones_like(amax)) + # Round the scale to its stored precision BEFORE dividing, so quantize and + # dequantize use the identical value. + scales = scales.to(SCALE_DTYPE) + q = blocks / scales.float().unsqueeze(-1) + if self.is_integer: + # Half away from zero, matching the store kernel. ``Tensor.round`` is + # half-to-even and would disagree on ties. + q = torch.where(q >= 0, (q + 0.5).floor(), (q - 0.5).ceil()) + q = q.clamp_(-self.max_magnitude, self.max_magnitude) + return q.flatten(-2).to(self.storage_dtype), scales + + def dequantize(self, q: torch.Tensor, scales: torch.Tensor) -> torch.Tensor: + """Inverse of :meth:`quantize`, in float32 (logical element shape).""" + assert self.enabled, "dequantize() on an unquantized spec" + if self.packed: + logical_d = q.shape[-1] * self.elements_per_byte + nblock = logical_d // BLOCK + # Each block of BLOCK elements occupies BLOCK // elements_per_byte bytes; + # split each byte's low (even element) and high (odd) nibbles. Operate on + # the integer codes (the caller may pass the packed tensor already floated). + codes = q.to(torch.uint8) + blocks = codes.unflatten(-1, (nblock, BLOCK // self.elements_per_byte)) + values = torch.stack([blocks & 0x0F, blocks >> 4], dim=-1) + values = values.reshape(*blocks.shape[:-1], BLOCK).float() + values = values - 8.0 + else: + values = q.float().unflatten(-1, (q.shape[-1] // BLOCK, BLOCK)) + return (values * scales.float().unsqueeze(-1)).flatten(-2) + + +# int8 symmetric: a block's max-abs maps to 127. +Q8_0 = KVQuantSpec(name="q8_0", storage_dtype=torch.int8, max_magnitude=127.0) +# e4m3: 4-bit exponent, 3-bit mantissa, max finite magnitude 448. +FP8_E4M3 = KVQuantSpec(name="fp8_e4m3", storage_dtype=torch.float8_e4m3fn, max_magnitude=448.0) +# GGML Q4_0, two values per byte: a block's signed extreme selects a scale and all +# 16 codes represent [-8, 7] times that (possibly negative) scale. +INT4 = KVQuantSpec( + name="int4", storage_dtype=torch.uint8, max_magnitude=8.0, elements_per_byte=2 +) +NONE = KVQuantSpec(name="auto", storage_dtype=None, max_magnitude=0.0) + +_BY_NAME = {spec.name: spec for spec in (NONE, Q8_0, FP8_E4M3, INT4)} +_BY_NAME["q4_0"] = INT4 +KV_CACHE_DTYPES = tuple(_BY_NAME) + + +def resolve_kv_quant(name: str | None) -> KVQuantSpec: + """``--kv-cache-dtype`` value -> spec. ``None``/``"auto"`` means unquantized.""" + if name is None: + return NONE + try: + return _BY_NAME[name] + except KeyError: + raise ValueError( + f"unknown --kv-cache-dtype {name!r}; choose from {', '.join(KV_CACHE_DTYPES)}" + ) from None + + +__all__ = [ + "BLOCK", + "SCALE_DTYPE", + "KVQuantSpec", + "KV_CACHE_DTYPES", + "Q8_0", + "FP8_E4M3", + "INT4", + "NONE", + "resolve_kv_quant", +] diff --git a/python/freetoken/kvcache/quant_storage.py b/python/freetoken/kvcache/quant_storage.py new file mode 100644 index 00000000..46638d2c --- /dev/null +++ b/python/freetoken/kvcache/quant_storage.py @@ -0,0 +1,87 @@ +"""Scale-buffer bookkeeping shared by the quantizable KV pools. + +A quantized pool allocates, alongside each K/V slab, a scale slab with the same logical +shape but the last dimension divided by :data:`~freetoken.kvcache.quant.BLOCK`. Packed +formats additionally shrink the K/V slab's physical last dimension. The slabs must be +allocated, rebuilt and freed together, and ``store_kv`` routes to the quantizing kernel +instead of the byte-copy one -- that is all this mixin owns. The pools keep their own +geometry and indexing. +""" + +from __future__ import annotations + +import torch + +from .quant import NONE, SCALE_DTYPE, KVQuantSpec + + +class QuantizedKVStorageMixin: + """Allocation + store routing for pools whose K/V slabs may be compact. + + Subclasses set ``self._quant`` before allocating and call :meth:`_alloc_scales` for + each K/V buffer they create. ``_quant`` defaulting to the unquantized spec keeps + pools that never opt in behaving exactly as before. + """ + + _quant: KVQuantSpec = NONE + + @property + def quant(self) -> KVQuantSpec: + return self._quant + + def _buffer_dtype(self, compute_dtype: torch.dtype) -> torch.dtype: + """Element dtype for a K/V slab under the active scheme.""" + return self._quant.storage_dtype if self._quant.enabled else compute_dtype + + def _buffer_shape(self, kv_shape: tuple[int, ...]) -> tuple[int, ...]: + """Slab shape for a logical KV shape (last dim halves when packed).""" + return self._quant.storage_shape(kv_shape) if self._quant.enabled else kv_shape + + def _alloc_scales(self, kv_shape: tuple[int, ...], device: torch.device) -> torch.Tensor | None: + """Scale slab matching a ``[2, layers, ..., heads, head_dim]`` logical K/V buffer. + + None when unquantized -- callers store that verbatim and the attention path reads + it as "no scales", which is what selects the bf16 kernel branch. ``kv_shape`` is + the unpacked (element-counted) geometry: the scale extent is D // BLOCK regardless + of how the slab packs its elements. + """ + if not self._quant.enabled: + return None + return torch.empty( + self._quant.scale_shape(kv_shape), device=device, dtype=SCALE_DTYPE + ) + + def _store_kv_into( + self, + k_cache: torch.Tensor, + v_cache: torch.Tensor, + k_scale: torch.Tensor | None, + v_scale: torch.Tensor | None, + indices: torch.Tensor, + k: torch.Tensor, + v: torch.Tensor, + ) -> None: + """Write one layer's K/V, quantizing on the way in when the pool is compact.""" + if not self._quant.enabled: + from freetoken.kernel import store_cache + + store_cache(k_cache=k_cache, v_cache=v_cache, indices=indices, k=k, v=v) + return + + from freetoken.kernel.triton.kv_quant import store_kv_quant + + heads, storage_head_dim = k_cache.shape[-2:] + head_dim = storage_head_dim * self._quant.elements_per_byte + store_kv_quant( + k_cache, + k_scale, + v_cache, + v_scale, + indices, + k.view(-1, heads, head_dim), + v.view(-1, heads, head_dim), + self._quant, + ) + + +__all__ = ["QuantizedKVStorageMixin"] diff --git a/python/freetoken/layers/base.py b/python/freetoken/layers/base.py index b1e939d9..f7dab709 100644 --- a/python/freetoken/layers/base.py +++ b/python/freetoken/layers/base.py @@ -42,7 +42,9 @@ def load_state_dict( if isinstance(param, torch.Tensor): item = state_dict.pop(_concat_prefix(prefix, name)) assert isinstance(item, torch.Tensor) - assert param.shape == item.shape and param.dtype == item.dtype + assert param.shape == item.shape and param.dtype == item.dtype, ( + f"{_concat_prefix(prefix, name)}: model {tuple(param.shape)}/{param.dtype} vs ckpt {tuple(item.shape)}/{item.dtype}" + ) setattr(self, name, item) elif isinstance(param, BaseOP): param.load_state_dict( diff --git a/python/freetoken/layers/gguf.py b/python/freetoken/layers/gguf.py index ac49b1a5..e56ba816 100644 --- a/python/freetoken/layers/gguf.py +++ b/python/freetoken/layers/gguf.py @@ -13,6 +13,9 @@ from __future__ import annotations +import functools +import os + import torch from freetoken.models.gguf.dequant import ( @@ -21,7 +24,15 @@ GGML_F16, GGML_F32, GGML_NAME, + GGML_IQ1_S, + GGML_IQ2_S, + GGML_IQ2_XXS, + GGML_IQ3_XXS, + GGML_IQ4_XS, + GGML_Q3_K, GGML_Q4_0, + GGML_Q4_K, + GGML_Q5_K, GGML_Q6_K, GGML_Q8_0, row_bytes, @@ -32,13 +43,90 @@ # ggml type groups for kernel dispatch (subset we build kernels for). _UNQUANTIZED = {GGML_F32, GGML_F16, GGML_BF16} # standard + k-quants: both an MMVQ (small-batch GEMV) and MMQ (large-batch) kernel exist. -_MMVQ = {GGML_Q4_0, GGML_Q8_0, GGML_Q6_K} -_MMQ = {GGML_Q4_0, GGML_Q8_0, GGML_Q6_K} -_DEQUANT = {GGML_Q4_0, GGML_Q8_0, GGML_Q6_K} +_MMVQ = { + GGML_Q4_0, GGML_Q8_0, GGML_Q3_K, GGML_Q4_K, GGML_Q5_K, GGML_Q6_K, + GGML_IQ1_S, GGML_IQ2_S, GGML_IQ2_XXS, GGML_IQ3_XXS, GGML_IQ4_XS, +} +# The vendored CUDA MMQ switch covers the standard + K-quants only (no IQ cases); +# IQ types take the dequant fallback for large batches. +_MMQ = {GGML_Q4_0, GGML_Q8_0, GGML_Q3_K, GGML_Q4_K, GGML_Q5_K, GGML_Q6_K} +_DEQUANT = { + GGML_Q4_0, GGML_Q8_0, GGML_Q3_K, GGML_Q4_K, GGML_Q5_K, GGML_Q6_K, + GGML_IQ1_S, GGML_IQ2_S, GGML_IQ2_XXS, GGML_IQ3_XXS, GGML_IQ4_XS, +} # Below this token count, the MMVQ GEMV kernel wins (matches vLLM's heuristic). _MMVQ_SAFE = 6 +# The donated GGUF MMQ kernel does not use tensor cores. For real prefill +# batches it is much faster to dequantize the comparatively small dense weight +# once and hand the matrix product to cuBLAS. On sm_89 with Ornith's largest +# Q6_K projection the crossover is 32 rows (0.40 ms vs 0.64 ms) and at 8192 +# rows it is 6.1 ms vs 132 ms. This is transient -- packed weights remain the +# persistent representation, so long-context KV/expert capacity is unchanged. +_DEQUANT_GEMM_MIN_ROWS = 32 +# On sm_120 (RTX 5080) the crossover moves down: dequant wins from 24 rows on +# Ornith's Q4_K attention shapes (0.0645 vs 0.0778 ms) while MMQ still wins the +# Q6_K lm_head at 16 (2.08 vs 2.93 ms), so 24 is the safe arch-wide value. +_DEQUANT_GEMM_MIN_ROWS_SM120 = 24 + + +def dequant_gemm_min_rows(compute_capability: tuple[int, int] | None) -> int: + """Row count from which transient dequant+cuBLAS beats the DP4A MMQ kernel.""" + if compute_capability is not None and compute_capability >= (12, 0): + return _DEQUANT_GEMM_MIN_ROWS_SM120 + return _DEQUANT_GEMM_MIN_ROWS + + +@functools.lru_cache(maxsize=8) +def _device_capability(device_index: int) -> tuple[int, int]: + return torch.cuda.get_device_capability(device_index) + + +@functools.lru_cache(maxsize=None) +def _mma_mmq_ok() -> bool: + """JIT-build the upstream int8-MMA MMQ extension once; fall back on failure. + + ``FREETOKEN_GGUF_DISABLE_MMA=1`` forces the DP4A/dequant path -- an escape + hatch if the JIT build misbehaves on a given toolchain, and the A/B control + for benchmarking the port. + """ + import logging + + log = logging.getLogger(__name__) + if os.environ.get("FREETOKEN_GGUF_DISABLE_MMA", "") not in ("", "0"): + log.info("int8-MMA MMQ disabled by FREETOKEN_GGUF_DISABLE_MMA") + return False + + from freetoken.kernel.gguf import _mma_module + + try: + _mma_module() + except Exception as exc: # build/toolchain failure -> DP4A/dequant path + log.warning( + "int8-MMA MMQ extension unavailable, using DP4A/dequant fallback: %s", exc + ) + return False + # One-time, greppable proof of which GEMM path a live server actually runs. + log.info("int8-MMA MMQ ACTIVE for Q4_K/Q6_K (upstream llama.cpp mul_mat_q)") + return True + + +def _use_mma_mmq(quant_type: int, capability: tuple[int, int] | None) -> bool: + """int8-MMA MMQ replaces both the DP4A MMQ band and the dequant+cuBLAS band. + + Measured on sm_120 (RTX 5080, real Ornith tensors): faster than BOTH at + every batch size >= 4 -- 8192-row Q4_K attn_q 1.79 ms vs dequant 2.40 vs + DP4A 22.9; Q6_K lm_head @2048 tokens 17.5 vs 19.2 vs 262. Gated to sm_120 + for now (upstream supports sm_75+, but only Blackwell is measured here); + MMVQ keeps the <= _MMVQ_SAFE decode band. + """ + if capability is None or capability < (12, 0): + return False + from freetoken.kernel.gguf import mma_mmq_supported + + return mma_mmq_supported(quant_type) and _mma_mmq_ok() + def fused_mul_mat_gguf(x: torch.Tensor, qweight: torch.Tensor, qweight_type: int) -> torch.Tensor: """y = x @ dequant(qweight).T, dispatched by batch size and quant type.""" @@ -56,6 +144,18 @@ def fused_mul_mat_gguf(x: torch.Tensor, qweight: torch.Tensor, qweight_type: int if x.shape[0] <= _MMVQ_SAFE and qweight_type in _MMVQ: return ggml_mul_mat_vec_a8(qweight, x, qweight_type, out_features) if qweight_type in _MMQ: + capability = _device_capability(x.device.index) if x.is_cuda else None + if _use_mma_mmq(qweight_type, capability): + from freetoken.kernel.gguf import ggml_mul_mat_a8_mma + + return ggml_mul_mat_a8_mma(qweight, x, qweight_type, out_features).to(x.dtype) + if x.shape[0] >= dequant_gemm_min_rows(capability): + block, type_size = BLOCK_SHAPE[qweight_type] + in_features = qweight.shape[1] // type_size * block + weight = ggml_dequantize( + qweight, qweight_type, out_features, in_features, x.dtype + ) + return x @ weight.T return ggml_mul_mat_a8(qweight, x, qweight_type, out_features) if qweight_type in _DEQUANT: block, type_size = BLOCK_SHAPE[qweight_type] diff --git a/python/freetoken/layers/moe.py b/python/freetoken/layers/moe.py index d68d8ded..3b75985d 100644 --- a/python/freetoken/layers/moe.py +++ b/python/freetoken/layers/moe.py @@ -531,6 +531,20 @@ def _expert_gemm( return fused_experts_gguf_q4_0( hidden_states, gate_up, down, topk_weights, topk_ids, self.activation ) + if fmt == "gguf": + # Mixed-type GGUF experts (per-layer quant types, flat padded slot banks): + # same MMVQ path, geometry from the layer's own type attributes (set via + # make_moe_layer extra_attrs by the model, e.g. laguna). + from freetoken.moe.fused_gguf import fused_experts_gguf + + gate_up, down = views + return fused_experts_gguf( + hidden_states, gate_up, down, topk_weights, topk_ids, self.activation, + gate_up_type=self.gguf_gate_up_type, + down_type=self.gguf_down_type, + gate_up_rows=self.gguf_gate_up_rows, + down_rows=self.gguf_down_rows, + ) if fmt == "mxfp4_triton": # gpt-oss MXFP4 experts (biased, clamped swiglu): transposed split-K GEMV # decode + grouped `_t` prefill. The swiglu scalars live on the layer diff --git a/python/freetoken/models/config.py b/python/freetoken/models/config.py index f6105e1f..9e6c95ac 100644 --- a/python/freetoken/models/config.py +++ b/python/freetoken/models/config.py @@ -1,6 +1,6 @@ from __future__ import annotations import os -from dataclasses import dataclass +from dataclasses import dataclass, field from typing import Any, ClassVar, Dict, List, Literal, Tuple, TypeAlias from freetoken.attention.base import AttnType @@ -199,6 +199,7 @@ def _full_group_attn_type(group: FullAttentionGroupConfig) -> AttnType: class ModelConfig: num_layers: int num_qo_heads: int + num_qo_heads_per_layer: tuple[int, ...] | None = field(default=None, kw_only=True) # hybrid models (laguna) vary per layer; None means uniform num_qo_heads. num_kv_heads: int head_dim: int hidden_size: int @@ -273,6 +274,14 @@ class ModelConfig: has_attn_bias: bool = False has_router_bias: bool = False moe_weight_format: str | None = None + # GGUF checkpoints only: ggml quant type of ``token_embd.weight`` (publisher-dependent + # -- Q6_K in Google's QAT release, Q4_0 in Unsloth's). None for non-GGUF checkpoints. + gguf_embed_quant: int | None = None + # Mixed-type GGUF (laguna): (gate_up_type, down_type) ggml ids per MoE layer, + # read from the file's tensor table. None for uniform-quant checkpoints. + gguf_expert_types: tuple[tuple[int, int], ...] | None = None + # source .gguf path; laguna reads per-tensor quant types from it at conversion + gguf_model_path: str | None = None swiglu_limit: float | None = None hidden_act_alpha: float = 1.702 # Full DeepseekV4Args payload for the DSV4-specific machinery (MLA sparse attention, @@ -291,6 +300,15 @@ class ModelConfig: # Generic execution-path capability flags (set by a model's parse_config) so the engine and # factories stay model-agnostic instead of branching on dsv4_args: single_stream_only: bool = False # model runs one sequence at a time -> force bs=1 + # Explicit sparse-MoE layer set for hybrid mixer-only architectures. None keeps the + # conventional contiguous [first_k_dense_replace, num_layers) layout. + moe_layer_ids: tuple[int, ...] | None = None + # Routed-expert input/output width when it differs from the residual-stream hidden size + # (Nemotron-H projects 4096 -> 1024 around its experts). + expert_hidden_size: int | None = None + expert_gated: bool = True + # Opaque Nemotron-H mixer geometry / per-module quantization metadata. + nemotron_h_args: Any | None = None @property def is_moe(self) -> bool: @@ -303,6 +321,8 @@ def num_moe_layers(self) -> int: Models with leading dense layers (``first_k_dense_replace`` > 0, e.g. GLM-4) only store experts for the trailing layers; everything else has all layers MoE. """ + if self.moe_layer_ids is not None: + return len(self.moe_layer_ids) return self.num_layers - self.first_k_dense_replace @property @@ -372,6 +392,12 @@ def is_linear_layer(self, layer_id: int) -> bool: LinearGatedDeltaGroupConfig, ) + def qo_heads(self, layer_id: int) -> int: + """Query-head count for one layer (per-layer override or the uniform count).""" + if self.num_qo_heads_per_layer is not None: + return self.num_qo_heads_per_layer[layer_id] + return self.num_qo_heads + def attn_type_for_layer(self, layer_id: int) -> AttnType: """Canonical per-layer attention-type lookup (the taxonomy is declared top-down on the attention groups; this is the layer-granular view).""" diff --git a/python/freetoken/models/gemma4/gguf.py b/python/freetoken/models/gemma4/gguf.py index 437822b5..7047817d 100644 --- a/python/freetoken/models/gemma4/gguf.py +++ b/python/freetoken/models/gemma4/gguf.py @@ -48,6 +48,18 @@ def _full_rotary_dim(shim: "GgufConfigShim", full_head_dim: int) -> int: return full_head_dim // 4 +def _embed_quant(shim: "GgufConfigShim") -> int: + """ggml quant type of the token embedding table, read off the file. + + Publishers differ (Google's QAT GGUF stores it Q6_K, Unsloth's Q4_0). A + metadata-only GGUF (an FTW dir's source_metadata.gguf) has no tensor table; fall + back to Q6_K, the type llama.cpp's own gemma4 conversion emits. + """ + from freetoken.models.gguf.reader import gguf_tensor_type + + return gguf_tensor_type(shim.model_path, "token_embd.weight") or GGML_Q6_K + + def parse_gguf_config(shim: "GgufConfigShim") -> ModelConfig: m = shim.metadata @@ -114,6 +126,7 @@ def g(key: str): moe_enabled=True, expert_quant="q4_0", moe_weight_format="q4_0", + gguf_embed_quant=_embed_quant(shim), use_qk_norm=True, attn_sm_scale=1.0, final_logit_softcapping=float(g("final_logit_softcapping")), @@ -230,7 +243,7 @@ def layer_of(name: str) -> int: for t in iter_gguf_tensors(model_path): name = t.name if name == "token_embd.weight": - yield "model.embed_tokens.qweight", t.packed() # Q6_K packed table + yield "model.embed_tokens.qweight", t.packed() # packed table, native quant continue if name == "output_norm.weight": yield "model.norm.weight", _to_bf16(t) @@ -346,6 +359,7 @@ def convert_gemma4_to_gguf(model, config: ModelConfig) -> None: the routed experts (served from the offload cache). """ from freetoken.layers.gguf import GGUFEmbedding, GGUFLinear + embed_quant = config.gguf_embed_quant or GGML_Q6_K def swap_linear(owner, attr, quant_type=GGML_Q4_0): lin = getattr(owner, attr) @@ -360,7 +374,7 @@ def swap_linear(owner, attr, quant_type=GGML_Q4_0): embed = GGUFEmbedding( num_embeddings=config.vocab_size, embedding_dim=config.hidden_size, - quant_type=GGML_Q6_K, + quant_type=embed_quant, embed_scale=config.embedding_scale, ) inner.embed_tokens = embed @@ -372,7 +386,7 @@ def swap_linear(owner, attr, quant_type=GGML_Q4_0): swap_linear(layer.feed_forward.shared_mlp, "down_proj") if config.tie_word_embeddings: - model.lm_head = GGUFTiedLMHead(embed, GGML_Q6_K) + model.lm_head = GGUFTiedLMHead(embed, embed_quant) # -------------------------------------------------------------------------------------- diff --git a/python/freetoken/models/gguf/__init__.py b/python/freetoken/models/gguf/__init__.py index 75e40e4f..ec3f49b4 100644 --- a/python/freetoken/models/gguf/__init__.py +++ b/python/freetoken/models/gguf/__init__.py @@ -5,6 +5,7 @@ gguf_architecture, gguf_config_source, gguf_tensor_names, + gguf_tensor_type, is_gguf_path, iter_gguf_tensors, load_gguf_metadata, @@ -19,6 +20,7 @@ "gguf_architecture", "gguf_config_source", "gguf_tensor_names", + "gguf_tensor_type", "is_gguf_path", "iter_gguf_tensors", "load_gguf_metadata", diff --git a/python/freetoken/models/gguf/config.py b/python/freetoken/models/gguf/config.py index 63b1a18b..abd21a61 100644 --- a/python/freetoken/models/gguf/config.py +++ b/python/freetoken/models/gguf/config.py @@ -18,6 +18,10 @@ # reuses the model classes but a GGUF parse_config / iter_weights). GGUF_ARCH_TO_REGISTRY: dict[str, str] = { "gemma4": "Gemma4GGUFForCausalLM", + "laguna": "LagunaGGUFForCausalLM", + # llama.cpp's name for Qwen3.5-MoE. Ornith-1.5-35B-A3B is a text-only + # fine-tune of this architecture (40 decoder layers plus one MTP layer). + "qwen35moe": "Qwen3_5MoeGGUFForConditionalGeneration", } diff --git a/python/freetoken/models/gguf/dequant.py b/python/freetoken/models/gguf/dequant.py index 77c3ea01..3e009a95 100644 --- a/python/freetoken/models/gguf/dequant.py +++ b/python/freetoken/models/gguf/dequant.py @@ -23,7 +23,15 @@ GGML_F16 = 1 GGML_Q4_0 = 2 GGML_Q8_0 = 8 +GGML_Q3_K = 11 +GGML_Q4_K = 12 +GGML_Q5_K = 13 GGML_Q6_K = 14 +GGML_IQ2_XXS = 16 +GGML_IQ3_XXS = 18 +GGML_IQ1_S = 19 +GGML_IQ2_S = 22 +GGML_IQ4_XS = 23 GGML_BF16 = 30 # (block numel, bytes per block) per ggml type. @@ -33,7 +41,15 @@ GGML_BF16: (1, 2), GGML_Q4_0: (32, 18), GGML_Q8_0: (32, 34), + GGML_Q3_K: (256, 110), + GGML_Q4_K: (256, 144), + GGML_Q5_K: (256, 176), GGML_Q6_K: (256, 210), + GGML_IQ2_XXS: (256, 66), + GGML_IQ3_XXS: (256, 98), + GGML_IQ1_S: (256, 50), + GGML_IQ2_S: (256, 82), + GGML_IQ4_XS: (256, 136), } GGML_NAME = { @@ -42,7 +58,15 @@ GGML_BF16: "BF16", GGML_Q4_0: "Q4_0", GGML_Q8_0: "Q8_0", + GGML_Q3_K: "Q3_K", + GGML_Q4_K: "Q4_K", + GGML_Q5_K: "Q5_K", GGML_Q6_K: "Q6_K", + GGML_IQ2_XXS: "IQ2_XXS", + GGML_IQ3_XXS: "IQ3_XXS", + GGML_IQ1_S: "IQ1_S", + GGML_IQ2_S: "IQ2_S", + GGML_IQ4_XS: "IQ4_XS", } @@ -115,9 +139,65 @@ def dequant_q6_k(raw: torch.Tensor, out_dtype: torch.dtype) -> torch.Tensor: return y.reshape(-1).to(out_dtype) +def _dequant_gguf_py(raw: torch.Tensor, out_dtype: torch.dtype, ggml_type: int) -> torch.Tensor: + """Reference dequant for the K-/IQ-quants, delegated to gguf-py. + + gguf-py carries numpy decoders for every ggml type (it cannot *quantize* the + K/IQ formats, but decoding is all the reference path needs); porting the bit + math here would only duplicate it. + """ + import gguf + import numpy as np + + out = gguf.quants.dequantize( + raw.detach().cpu().contiguous().numpy(), gguf.GGMLQuantizationType(ggml_type) + ) + return torch.from_numpy(np.asarray(out)).to(raw.device, out_dtype).reshape(-1) + + +def dequant_q3_k(raw: torch.Tensor, out_dtype: torch.dtype) -> torch.Tensor: + return _dequant_gguf_py(raw, out_dtype, GGML_Q3_K) + + +def dequant_q4_k(raw: torch.Tensor, out_dtype: torch.dtype) -> torch.Tensor: + return _dequant_gguf_py(raw, out_dtype, GGML_Q4_K) + + +def dequant_q5_k(raw: torch.Tensor, out_dtype: torch.dtype) -> torch.Tensor: + return _dequant_gguf_py(raw, out_dtype, GGML_Q5_K) + + +def dequant_iq2_xxs(raw: torch.Tensor, out_dtype: torch.dtype) -> torch.Tensor: + return _dequant_gguf_py(raw, out_dtype, GGML_IQ2_XXS) + + +def dequant_iq3_xxs(raw: torch.Tensor, out_dtype: torch.dtype) -> torch.Tensor: + return _dequant_gguf_py(raw, out_dtype, GGML_IQ3_XXS) + + +def dequant_iq1_s(raw: torch.Tensor, out_dtype: torch.dtype) -> torch.Tensor: + return _dequant_gguf_py(raw, out_dtype, GGML_IQ1_S) + + +def dequant_iq2_s(raw: torch.Tensor, out_dtype: torch.dtype) -> torch.Tensor: + return _dequant_gguf_py(raw, out_dtype, GGML_IQ2_S) + + +def dequant_iq4_xs(raw: torch.Tensor, out_dtype: torch.dtype) -> torch.Tensor: + return _dequant_gguf_py(raw, out_dtype, GGML_IQ4_XS) + + _DEQUANT = { GGML_Q4_0: dequant_q4_0, GGML_Q6_K: dequant_q6_k, + GGML_Q3_K: dequant_q3_k, + GGML_Q4_K: dequant_q4_k, + GGML_Q5_K: dequant_q5_k, + GGML_IQ2_XXS: dequant_iq2_xxs, + GGML_IQ3_XXS: dequant_iq3_xxs, + GGML_IQ1_S: dequant_iq1_s, + GGML_IQ2_S: dequant_iq2_s, + GGML_IQ4_XS: dequant_iq4_xs, } @@ -141,13 +221,13 @@ def dequantize(raw: torch.Tensor, ggml_type: int, out_dtype: torch.dtype) -> tor "GGML_F32", "GGML_F16", "GGML_BF16", - "GGML_Q4_0", + "GGML_Q4_0", "GGML_Q4_K", "GGML_Q5_K", "GGML_IQ2_XXS", "GGML_IQ3_XXS", "GGML_IQ1_S", "GGML_IQ4_XS", "GGML_Q8_0", "GGML_Q6_K", "GGML_NAME", "BLOCK_SHAPE", "row_bytes", "dequant_q4_0", - "dequant_q6_k", + "dequant_q6_k", "dequant_q4_k", "dequant_q5_k", "dequant_iq2_xxs", "dequant_iq3_xxs", "dequant_iq1_s", "dequant_iq4_xs", "dequantize", ] diff --git a/python/freetoken/models/gguf/reader.py b/python/freetoken/models/gguf/reader.py index b950d929..39b30eec 100644 --- a/python/freetoken/models/gguf/reader.py +++ b/python/freetoken/models/gguf/reader.py @@ -175,6 +175,19 @@ def gguf_tensor_names(model_path: str) -> set[str]: return {t.name for t in _reader(model_path).tensors} +def gguf_tensor_type(model_path: str, name: str) -> int | None: + """The ggml quant type of one tensor, or ``None`` if the file has no such tensor. + + Quant choices are per-tensor in GGUF and differ between publishers (e.g. Google's + QAT release stores ``token_embd.weight`` as Q6_K, Unsloth's as Q4_0), so layer + construction reads the type off the file instead of assuming one. + """ + for t in _reader(model_path).tensors: + if t.name == name: + return int(t.tensor_type) + return None + + __all__ = [ "is_gguf_path", "FTW_METADATA_GGUF", @@ -186,4 +199,5 @@ def gguf_tensor_names(model_path: str) -> set[str]: "gguf_architecture", "iter_gguf_tensors", "gguf_tensor_names", + "gguf_tensor_type", ] diff --git a/python/freetoken/models/gguf/tokenizer.py b/python/freetoken/models/gguf/tokenizer.py index 6d5481c1..e8202174 100644 --- a/python/freetoken/models/gguf/tokenizer.py +++ b/python/freetoken/models/gguf/tokenizer.py @@ -12,8 +12,55 @@ from .reader import gguf_architecture, load_gguf_metadata + +# llama.cpp / GGUF token type values (gguf.TokenType). CONTROL tokens are true +# special tokens, while USER_DEFINED tokens must participate in the added-token +# matcher without being removed by ``skip_special_tokens``. The latter distinction +# matters for Qwen3.5: markers such as ```` and ```` are type 4 and +# the byte-level BPE otherwise splits them into ordinary punctuation/word tokens even +# though their complete strings are present in the base vocabulary. +_GGUF_TOKEN_CONTROL = 3 +_GGUF_TOKEN_USER_DEFINED = 4 + + +def _register_gguf_added_tokens(tokenizer, tokens: list[str], token_types: list[int] | None) -> None: + """Restore GGUF CONTROL/USER_DEFINED matching semantics on a fast tokenizer. + + Transformers' Qwen GGUF converter currently hard-codes only ``<|endoftext|>``, + ``<|im_start|>`` and ``<|im_end|>``. Newer Qwen-family checkpoints carry more + USER_DEFINED tokens in the GGUF table; merely putting those strings in a BPE + vocabulary does not make the tokenizer match them atomically. + """ + if not token_types or len(token_types) != len(tokens): + return + + from tokenizers import AddedToken + + control = [ + AddedToken(token, normalized=False, special=True) + for token, token_type in zip(tokens, token_types) + if int(token_type) == _GGUF_TOKEN_CONTROL + ] + user_defined = [ + AddedToken(token, normalized=False, special=False) + for token, token_type in zip(tokens, token_types) + if int(token_type) == _GGUF_TOKEN_USER_DEFINED + ] + if control: + tokenizer.add_special_tokens({"additional_special_tokens": control}) + if user_defined: + tokenizer.add_tokens(user_defined) + # GGUF architecture -> transformers GGUF tokenizer-converter key. -_TOKENIZER_ARCH = {"gemma4": "gemma4_text"} +# laguna ships a plain gpt2-style BPE (tokenizer.ggml.model = "gpt2"); transformers +# has no "laguna" converter, so route it to the gpt2 one. +_TOKENIZER_ARCH = { + "gemma4": "gemma4_text", + "laguna": "gpt2", + # Qwen3.5-MoE retains Qwen3's tokenizer; only the model architecture label + # changed in llama.cpp. Transformers has no qwen35moe converter key. + "qwen35moe": "qwen3_moe", +} def load_gguf_tokenizer(model_path: str): @@ -28,9 +75,14 @@ def load_gguf_tokenizer(model_path: str): for k, v in meta.items() if k.startswith("tokenizer.ggml.") } - fast, _extra = convert_gguf_tokenizer(conv_arch, tok_dict) - tokens = tok_dict["tokens"] + # Some converters (gpt2) read .bos_token/.eos_token off the skeleton, which the + # GGUF metadata only carries as ids -- materialize the token strings. + for name in ("bos", "eos"): + tid = tok_dict.get(f"{name}_token_id") + if f"{name}_token" not in tok_dict and tid is not None and int(tid) < len(tokens): + tok_dict[f"{name}_token"] = tokens[int(tid)] + fast, _extra = convert_gguf_tokenizer(conv_arch, tok_dict) def tok_for(id_key: str, default: str) -> str: tid = meta.get(f"tokenizer.ggml.{id_key}") @@ -43,9 +95,13 @@ def tok_for(id_key: str, default: str) -> str: tokenizer_object=fast, bos_token=tok_for("bos_token_id", ""), eos_token=turn_end or tok_for("eos_token_id", ""), - unk_token=tok_for("unknown_token_id", ""), + # Qwen has no unknown token. Adding a synthetic would create id + # ``vocab_size`` (one beyond the embedding table) for genuinely unknown + # input, turning a tokenizer fallback into an out-of-bounds lookup. + unk_token=None if arch == "qwen35moe" else tok_for("unknown_token_id", ""), pad_token=tok_for("padding_token_id", ""), ) + _register_gguf_added_tokens(tokenizer, tokens, tok_dict.get("token_type")) chat_template = meta.get("tokenizer.chat_template") if chat_template: tokenizer.chat_template = chat_template @@ -53,7 +109,10 @@ def tok_for(id_key: str, default: str) -> str: def gguf_eos_token_ids(model_path: str, tokenizer) -> set[int]: - """Stop ids for GGUF generation: the formal plus the chat turn end .""" + """Stop ids for GGUF generation: the formal , the chat turn end , the + GGUF-declared eot, and gemma4's tool-response opener <|tool_response> (the model + emits it right after closing a tool call, so it is a stop id upstream too -- + generation_config.json ships eos_token_id [1, 106, 50]).""" meta = load_gguf_metadata(model_path) tokens = meta["tokenizer.ggml.tokens"] ids: set[int] = set() @@ -64,7 +123,10 @@ def gguf_eos_token_ids(model_path: str, tokenizer) -> set[int]: ids.add(int(eid)) # Look the stop tokens up in the vocab directly (convert_tokens_to_ids would map an # absent name to , wrongly adding it as a stop id). - for name in ("", ""): + eot = meta.get("tokenizer.ggml.eot_token_id") + if eot is not None: + ids.add(int(eot)) + for name in ("", "", "<|tool_response>"): try: ids.add(tokens.index(name)) except ValueError: diff --git a/python/freetoken/models/laguna/__init__.py b/python/freetoken/models/laguna/__init__.py new file mode 100644 index 00000000..b9c84d96 --- /dev/null +++ b/python/freetoken/models/laguna/__init__.py @@ -0,0 +1,21 @@ +from .config import parse_config +from .gguf import ( + dummy_gguf_expert_sources, + iter_gguf_weights, + load_gguf_expert_sources, + parse_gguf_config, +) +from .model import LagunaForCausalLM +from .weight import dummy_int4_expert_sources, iter_weights, load_int4_expert_sources + +__all__ = [ + "LagunaForCausalLM", + "dummy_gguf_expert_sources", + "dummy_int4_expert_sources", + "iter_gguf_weights", + "iter_weights", + "load_gguf_expert_sources", + "load_int4_expert_sources", + "parse_config", + "parse_gguf_config", +] diff --git a/python/freetoken/models/laguna/attention.py b/python/freetoken/models/laguna/attention.py new file mode 100644 index 00000000..7afa9945 --- /dev/null +++ b/python/freetoken/models/laguna/attention.py @@ -0,0 +1,109 @@ +from __future__ import annotations + +from typing import TYPE_CHECKING + +import torch +import torch.nn.functional as F + +from freetoken.attention import AttentionSpec +from freetoken.core import get_global_ctx +from freetoken.layers import BaseOP, LinearReplicated, RMSNorm +from freetoken.layers.rotary import get_rope +from freetoken.models.config import FullAttentionGroupConfig, SWAAttentionGroupConfig +from freetoken.utils import nvtx_annotate + +if TYPE_CHECKING: + from freetoken.models.config import ModelConfig + + +class LagunaAttention(BaseOP): + def __init__(self, config: ModelConfig, layer_id: int): + self.layer_id = layer_id + group = config.attention_group_for_layer(layer_id) + if not isinstance(group, (FullAttentionGroupConfig, SWAAttentionGroupConfig)): + raise ValueError(f"LagunaAttention does not support {group.kind!r} layers") + + rotary_config = group.rotary_config + self.head_dim = group.head_dim + self.num_kv_heads = group.num_kv_heads + self.num_qo_heads = config.qo_heads(layer_id) + + self.q_dim = self.num_qo_heads * self.head_dim + self.kv_dim = self.num_kv_heads * self.head_dim + + # Separate q/k/v projections (not LinearQKVMerged): laguna GGUFs quantize + # attn_v at a different ggml type than q/k on some layers (XS Q4_K_M), and + # packed rows of different types cannot be fused into one buffer. + self.q_proj = LinearReplicated(config.hidden_size, self.q_dim, has_bias=False) + self.k_proj = LinearReplicated(config.hidden_size, self.kv_dim, has_bias=False) + self.v_proj = LinearReplicated(config.hidden_size, self.kv_dim, has_bias=False) + self.gate_proj = LinearReplicated(config.hidden_size, self.num_qo_heads, has_bias=False) + self.q_norm = RMSNorm(self.head_dim, eps=config.rms_norm_eps) + self.k_norm = RMSNorm(self.head_dim, eps=config.rms_norm_eps) + self.o_proj = LinearReplicated(self.q_dim, config.hidden_size, has_bias=False) + self.attn_spec = AttentionSpec( + sliding_window=group.sliding_window if isinstance(group, SWAAttentionGroupConfig) else None + ) + + self.rotary = get_rope( + head_dim=self.head_dim, + rotary_dim=rotary_config.rotary_dim, + max_position=rotary_config.max_position, + base=rotary_config.base, + rope_scaling=( + tuple(rotary_config.scaling.items()) + if rotary_config.scaling + else None + ), + ) + + def _apply_rope( + self, + positions: torch.Tensor, + q: torch.Tensor, + k: torch.Tensor, + ) -> tuple[torch.Tensor, torch.Tensor]: + positions = positions.reshape(-1) + if positions.device != q.device or positions.dtype != torch.long: + positions = positions.to(device=q.device, dtype=torch.long) + q_view = q.contiguous().view(q.shape[0], -1) + k_view = k.contiguous().view(k.shape[0], -1) + self.rotary.forward(positions, q_view, k_view) + return q_view.view_as(q), k_view.view_as(k) + + @nvtx_annotate("MHA") + def forward(self, x: torch.Tensor) -> torch.Tensor: + ctx = get_global_ctx() + T = x.shape[0] + + gate = F.softplus(self.gate_proj.forward(x).float()) + q_lin = self.q_proj.forward(x) + k_lin = self.k_proj.forward(x) + v_lin = self.v_proj.forward(x) + + q = q_lin.view(T, self.num_qo_heads, self.head_dim) + k = k_lin.view(T, self.num_kv_heads, self.head_dim) + v = v_lin.view(T, self.num_kv_heads, self.head_dim) + + self.q_norm.forward_inplace(q.view(-1, self.num_qo_heads, self.head_dim)) + self.k_norm.forward_inplace(k.view(-1, self.num_kv_heads, self.head_dim)) + + q, k = self._apply_rope(ctx.batch.positions, q, k) + + k = k.reshape(T, self.kv_dim) + v = v.reshape(T, self.kv_dim) + + o = ctx.attn_backend.forward( + q.contiguous(), + k.contiguous(), + v.contiguous(), + self.layer_id, + ctx.batch, + attn_spec=self.attn_spec, + ) + o = o.view(T, self.num_qo_heads, self.head_dim) + o = o * gate.unsqueeze(-1).to(o.dtype) + return self.o_proj.forward(o.reshape(T, self.q_dim)) + + +__all__ = ["LagunaAttention"] diff --git a/python/freetoken/models/laguna/config.py b/python/freetoken/models/laguna/config.py new file mode 100644 index 00000000..60a29c6c --- /dev/null +++ b/python/freetoken/models/laguna/config.py @@ -0,0 +1,165 @@ +from __future__ import annotations + +import re +from typing import Any + +from freetoken.models.config import ( + FullAttentionGroupConfig, + ModelConfig, + RotaryConfig, + SWAAttentionGroupConfig, +) +from freetoken.models.gguf.dequant import GGML_BF16, GGML_Q4_0 + + +def _field(obj: Any, name: str, default=None): + if isinstance(obj, dict): + return obj.get(name, default) + return getattr(obj, name, default) + + +def _int4_scheme(hf_config: Any) -> dict: + quant = _field(hf_config, "quantization_config") or {} + groups = _field(quant, "config_groups", {}) or {} + for group in groups.values(): + weights = _field(group, "weights", {}) or {} + if ( + int(_field(weights, "num_bits", 0) or 0) == 4 + and str(_field(weights, "type", "")).lower() == "int" + ): + if ( + int(_field(weights, "group_size", 0) or 0) != 32 + or str(_field(weights, "strategy", "")).lower() != "group" + or not bool(_field(weights, "symmetric", False)) + ): + raise ValueError( + "Laguna compressed-tensors INT4 requires symmetric group-wise " + "weights with group_size=32" + ) + return group + raise ValueError( + "Laguna safetensors support requires the compressed-tensors symmetric INT4 scheme" + ) + + +def _matches(patterns: list[str], module_name: str) -> bool: + for pattern in patterns: + if pattern.startswith("re:"): + if re.fullmatch(pattern[3:], module_name): + return True + elif pattern in {"Linear", module_name}: + return True + return False + + +def _expert_types(hf_config: Any) -> tuple[tuple[int, int], ...]: + """Per-MoE-layer storage type, including compressed-tensors ignore rules. + + The published Laguna-S INT4 checkpoint quantizes routed experts in layers 1--39 + and intentionally leaves layers 40--47 in BF16. Carrying that distinction into + the cache avoids silently requantizing the accuracy-sensitive tail. + """ + scheme = _int4_scheme(hf_config) + targets = list(_field(scheme, "targets", []) or []) + quant = _field(hf_config, "quantization_config") or {} + ignores = list(_field(quant, "ignore", []) or []) + dense = len(list(_field(hf_config, "mlp_only_layers", [0]) or [0])) + out = [] + for layer in range(dense, int(_field(hf_config, "num_hidden_layers"))): + module = f"model.layers.{layer}.mlp.experts.0.gate_proj" + is_int4 = _matches(targets, module) and not _matches(ignores, module) + qtype = GGML_Q4_0 if is_int4 else GGML_BF16 + out.append((qtype, qtype)) + return tuple(out) + + +def _rope_config(params: dict, *, head_dim: int, max_position: int) -> RotaryConfig: + partial = float(params.get("partial_rotary_factor", 1.0)) + rope_type = str(params.get("rope_type", "default")) + scaling = None + if rope_type != "default": + scaling = { + key: value + for key, value in params.items() + if key not in {"rope_theta", "partial_rotary_factor"} + } + return RotaryConfig( + head_dim=head_dim, + rotary_dim=int(head_dim * partial), + max_position=max_position, + base=float(params.get("rope_theta", 10000.0)), + scaling=scaling, + ) + + +def parse_config(hf_config: Any) -> ModelConfig: + num_layers = int(hf_config.num_hidden_layers) + head_dim = int(hf_config.head_dim) + max_position = int(hf_config.max_position_embeddings) + head_counts = tuple(int(v) for v in hf_config.num_attention_heads_per_layer) + layer_types = tuple(hf_config.layer_types) + if len(head_counts) != num_layers or len(layer_types) != num_layers: + raise ValueError("Laguna per-layer attention metadata length does not match num_hidden_layers") + + rope = hf_config.rope_parameters + full_rotary = _rope_config(rope["full_attention"], head_dim=head_dim, max_position=max_position) + swa_rotary = _rope_config(rope["sliding_attention"], head_dim=head_dim, max_position=max_position) + full_layers = tuple(i for i, kind in enumerate(layer_types) if kind == "full_attention") + swa_layers = tuple(i for i, kind in enumerate(layer_types) if kind == "sliding_attention") + if len(full_layers) + len(swa_layers) != num_layers: + raise ValueError(f"unsupported Laguna layer_types: {sorted(set(layer_types))}") + + dense_layers = tuple(int(i) for i in getattr(hf_config, "mlp_only_layers", [0])) + if dense_layers != tuple(range(len(dense_layers))): + raise ValueError("FreeToken requires Laguna dense MLP layers to be a leading prefix") + + types = _expert_types(hf_config) + return ModelConfig( + num_layers=num_layers, + num_qo_heads=max(head_counts), + num_qo_heads_per_layer=head_counts, + num_kv_heads=int(hf_config.num_key_value_heads), + head_dim=head_dim, + hidden_size=int(hf_config.hidden_size), + vocab_size=int(hf_config.vocab_size), + intermediate_size=int(hf_config.intermediate_size), + rms_norm_eps=float(hf_config.rms_norm_eps), + rotary_config=full_rotary, + hidden_act=str(getattr(hf_config, "hidden_act", "silu")), + tie_word_embeddings=bool(hf_config.tie_word_embeddings), + num_experts=int(hf_config.num_experts), + num_experts_per_tok=int(hf_config.num_experts_per_tok), + moe_intermediate_size=int(hf_config.moe_intermediate_size), + shared_expert_intermediate_size=int(hf_config.shared_expert_intermediate_size), + n_shared_experts=1, + norm_topk_prob=bool(hf_config.norm_topk_prob), + routed_scaling_factor=float(hf_config.moe_routed_scaling_factor), + first_k_dense_replace=len(dense_layers), + use_qk_norm=True, + model_type="laguna", + architectures=list(hf_config.architectures), + moe_enabled=True, + attention_groups=( + FullAttentionGroupConfig( + name="full", + layer_ids=full_layers, + num_kv_heads=int(hf_config.num_key_value_heads), + head_dim=head_dim, + rotary_config=full_rotary, + ), + SWAAttentionGroupConfig( + name="swa", + layer_ids=swa_layers, + num_kv_heads=int(hf_config.num_key_value_heads), + head_dim=head_dim, + rotary_config=swa_rotary, + sliding_window=int(hf_config.sliding_window), + ), + ), + expert_quant="laguna_int4", + moe_weight_format="laguna_int4", + gguf_expert_types=types, + ) + + +__all__ = ["parse_config"] diff --git a/python/freetoken/models/laguna/gguf.py b/python/freetoken/models/laguna/gguf.py new file mode 100644 index 00000000..749865ec --- /dev/null +++ b/python/freetoken/models/laguna/gguf.py @@ -0,0 +1,545 @@ +"""Laguna GGUF adapter: ModelConfig from GGUF metadata + native-quant conversion. + +Laguna (poolside) is hybrid full/SWA attention (full at ``il % 4 == 0`` with 48 +query heads, SWA elsewhere with 72), sigmoid-routed MoE with a selection-only +score-correction bias, one shared expert, a per-head softplus attention output +gate, and QK-norm. Reference: llama.cpp ``src/models/laguna.cpp``. + +Unsloth's "Dynamic" GGUFs mix quant types per tensor (Q4_K embed/head, Q5_K/Q6_K +attention + dense, IQ1_S/IQ2_XXS/IQ3_XXS/IQ4_XS expert banks), so conversion +swaps dense projections for :class:`DeferredGGUFLinear`, whose packed buffer is +materialized at load time when each tensor's ggml type is known. +""" + +from __future__ import annotations + +from typing import TYPE_CHECKING + +import torch + +from freetoken.layers import BaseOP +from freetoken.models.config import ( + FullAttentionGroupConfig, + ModelConfig, + RotaryConfig, + SWAAttentionGroupConfig, +) + +if TYPE_CHECKING: + from freetoken.models.gguf.config import GgufConfigShim + + +def _embed_quant(shim: "GgufConfigShim") -> int | None: + from freetoken.models.gguf.reader import gguf_tensor_type + + return gguf_tensor_type(shim.model_path, "token_embd.weight") + + +def parse_gguf_config(shim: "GgufConfigShim") -> ModelConfig: + m = shim.metadata + + def g(key: str): + full_key = f"laguna.{key}" + val = m.get(full_key) + if val is None: + raise KeyError(f"missing GGUF metadata key {full_key}") + return val + + num_layers = int(g("block_count")) + hidden = int(g("embedding_length")) + intermediate = int(g("feed_forward_length")) + context = int(g("context_length")) + + head_counts = tuple(int(h) for h in g("attention.head_count")) + assert len(head_counts) == num_layers, "attention.head_count length != block_count" + num_qo_heads = max(head_counts) + + full_count = min(head_counts) + full_layer_ids = tuple(i for i, c in enumerate(head_counts) if c == full_count) + swa_layer_ids = tuple(i for i in range(num_layers) if i not in full_layer_ids) + assert full_layer_ids == tuple(i for i in range(0, num_layers, 4)), ( + "Laguna full attention layers must be exactly i % 4 == 0" + ) + + num_kv_heads = int(g("attention.head_count_kv")) + head_dim = int(g("attention.key_length")) + assert head_dim == int(g("attention.value_length")), "key_length != value_length" + + full_rotary = RotaryConfig( + head_dim=head_dim, + rotary_dim=int(g("rope.dimension_count")), + max_position=context, + base=float(g("rope.freq_base")), + scaling={ + "rope_type": "yarn", + "factor": float(g("rope.scaling.factor")), + # ggml applies yarn_attn_factor verbatim (1.0 here); without it + # freetoken's yarn would default to 1 + 0.1*ln(factor). + "attention_factor": float(g("rope.scaling.yarn_attn_factor")), + "original_max_position_embeddings": int(g("rope.scaling.original_context_length")), + "beta_fast": float(g("rope.scaling.yarn_beta_fast")), + "beta_slow": float(g("rope.scaling.yarn_beta_slow")), + }, + ) + swa_rotary = RotaryConfig( + head_dim=head_dim, + rotary_dim=int(g("rope.dimension_count_swa")), + max_position=context, + base=float(g("rope.freq_base_swa")), + scaling=None, + ) + + return ModelConfig( + num_layers=num_layers, + num_qo_heads=num_qo_heads, + num_qo_heads_per_layer=head_counts, + num_kv_heads=num_kv_heads, + head_dim=head_dim, + hidden_size=hidden, + vocab_size=int(shim.vocab_size), + intermediate_size=intermediate, + rms_norm_eps=float(g("attention.layer_norm_rms_epsilon")), + rotary_config=full_rotary, + hidden_act="silu", + tie_word_embeddings=bool(shim.tie_word_embeddings), + num_experts=int(g("expert_count")), + num_experts_per_tok=int(g("expert_used_count")), + moe_intermediate_size=int(g("expert_feed_forward_length")), + shared_expert_intermediate_size=int(g("expert_shared_feed_forward_length")), + n_shared_experts=1, + norm_topk_prob=bool(g("expert_weights_norm")), + routed_scaling_factor=float(g("expert_weights_scale")), + # The selection-only e_score_correction bias (blk.N.exp_probs_b) is part of + # the laguna arch itself, not flagged in metadata; has_router_bias (a bias + # on the router *linear*) stays False. + first_k_dense_replace=int(g("leading_dense_block_count")), + use_qk_norm=True, + model_type="laguna", + architectures=list(shim.architectures), + moe_enabled=True, + attention_groups=( + FullAttentionGroupConfig( + name="full", + layer_ids=full_layer_ids, + num_kv_heads=num_kv_heads, + head_dim=head_dim, + rotary_config=full_rotary, + ), + SWAAttentionGroupConfig( + name="swa", + layer_ids=swa_layer_ids, + num_kv_heads=num_kv_heads, + head_dim=head_dim, + rotary_config=swa_rotary, + sliding_window=int(g("attention.sliding_window")), + ), + ), + gguf_embed_quant=_embed_quant(shim), + gguf_model_path=shim.model_path, + # Routed experts: mixed per-layer ggml types (Unsloth Dynamic), served by the + # "gguf" offload bank format; types read from the tensor table when present + # (None on a metadata-only FTW source). + expert_quant="gguf", + moe_weight_format="gguf", + gguf_expert_types=_expert_types(shim), + ) + + +def _expert_types(shim: "GgufConfigShim") -> tuple[tuple[int, int], ...] | None: + """(gate_up, down) ggml type per MoE layer, from the file's tensor table. + + gate and up always share a type in the published files (asserted); a + metadata-only GGUF has no tensor table -> None. + """ + from freetoken.models.gguf.reader import gguf_tensor_names + + if not gguf_tensor_names(shim.model_path): + return None + types = gguf_tensor_types(shim.model_path) + num_layers = int(shim.metadata["laguna.block_count"]) + dense = int(shim.metadata["laguna.leading_dense_block_count"]) + out = [] + for i in range(dense, num_layers): + gu = types[f"blk.{i}.ffn_gate_exps.weight"] + up = types[f"blk.{i}.ffn_up_exps.weight"] + dn = types[f"blk.{i}.ffn_down_exps.weight"] + if gu != up: + raise ValueError(f"blk.{i}: gate/up expert banks have different ggml types") + out.append((gu, dn)) + return tuple(out) + + +# -------------------------------------------------------------------------------------- +# Weight loading: GGUF tensor names -> FreeToken laguna module params. +# -------------------------------------------------------------------------------------- + +# Routed expert banks are consumed by the MoE offload cache, not the state dict. +_EXPERT_SUFFIXES = ("ffn_gate_exps.weight", "ffn_up_exps.weight", "ffn_down_exps.weight") + +# Per-layer 1:1 tensors dequantized to a dense dtype (suffix -> (rel name, dtype)). +# The router gate and selection bias are consumed fp32 (top-k boundary fidelity, +# minimax_m3 precedent); norms load bf16. +_DENSE_MAP = { + "attn_norm.weight": ("input_layernorm.weight", "bf16"), + "attn_q_norm.weight": ("self_attn.q_norm.weight", "bf16"), + "attn_k_norm.weight": ("self_attn.k_norm.weight", "bf16"), + "ffn_norm.weight": ("ffn_norm.weight", "bf16"), + "ffn_gate_inp.weight": ("mlp.gate.weight", "f32"), + "exp_probs_b.bias": ("mlp.e_score_correction_bias", "f32"), +} + +# Packed (native-quant) projections that map 1:1 (suffix -> rel name). q/k/v stay +# separate modules: their ggml types differ within a layer in some laguna files +# (XS Q4_K_M quantizes attn_v as Q6_K on half the layers), so packed rows cannot fuse. +_PACKED_MAP = { + "attn_q.weight": "self_attn.q_proj.qweight", + "attn_k.weight": "self_attn.k_proj.qweight", + "attn_v.weight": "self_attn.v_proj.qweight", + "attn_output.weight": "self_attn.o_proj.qweight", + "attn_gate.weight": "self_attn.gate_proj.qweight", + "ffn_down.weight": "mlp.down_proj.qweight", + "ffn_down_shexp.weight": "mlp.shared_experts.down_proj.qweight", +} +_GATE_UP_SLOTS = { + "ffn_gate.weight": ("mlp.gate_up_proj", "gate"), + "ffn_up.weight": ("mlp.gate_up_proj", "up"), + "ffn_gate_shexp.weight": ("mlp.shared_experts.gate_up_proj", "gate"), + "ffn_up_shexp.weight": ("mlp.shared_experts.gate_up_proj", "up"), +} + + +def _require_tp1(what: str) -> None: + from freetoken.distributed import get_tp_info + + if get_tp_info().size > 1: + raise NotImplementedError(f"laguna GGUF {what} currently supports TP=1 only") + + +def gguf_tensor_types(model_path: str) -> dict[str, int]: + """One pass over the tensor table: name -> ggml type (no tensor data touched).""" + from freetoken.models.gguf.reader import iter_gguf_tensors + + return {t.name: t.ggml_type for t in iter_gguf_tensors(model_path)} + + +def iter_gguf_weights( + model_path: str, + device, + *, + include_moe_experts: bool, + include_non_moe: bool, +): + """Yield (param_name, tensor) for every non-expert laguna param. + + Quantized projections stay packed and are yielded as ``.qweight`` (uint8); + q/k/v and gate/up fuse by concatenating packed rows along the output dim + (valid only when the components share one ggml type -- Unsloth Dynamic keeps + fused groups uniform, enforced here). Norms load bf16; the router gate and + exp_probs_b load fp32. Routed experts go to the offload cache (4b), not here. + """ + import torch + + from freetoken.models.gguf.dequant import dequantize + from freetoken.models.gguf.reader import iter_gguf_tensors + + assert not include_moe_experts, ( + "laguna GGUF experts are loaded into the MoE offload cache, not the state dict" + ) + assert include_non_moe + _require_tp1("weight loading") + + def dense(t, kind): + dtype = torch.float32 if kind == "f32" else torch.bfloat16 + return dequantize(t.packed().reshape(-1), t.ggml_type, dtype).reshape(t.shape) + + gate_up_buf: dict[tuple[int, str], dict[str, tuple]] = {} + + for t in iter_gguf_tensors(model_path): + name = t.name + if name == "token_embd.weight": + yield "model.embed_tokens.qweight", t.packed() + continue + if name == "output_norm.weight": + yield "model.norm.weight", dense(t, "bf16") + continue + if name == "output.weight": + yield "lm_head.qweight", t.packed() + continue + if not name.startswith("blk."): + raise ValueError(f"unmapped laguna GGUF tensor: {name}") + if name.endswith(_EXPERT_SUFFIXES): + continue # routed experts -> offload banks + + layer = int(name.split(".")[1]) + suffix = name.split(".", 2)[2] + base = f"model.layers.{layer}" + + if suffix in _DENSE_MAP: + rel, kind = _DENSE_MAP[suffix] + yield f"{base}.{rel}", dense(t, kind) + elif suffix in _PACKED_MAP: + yield f"{base}.{_PACKED_MAP[suffix]}", t.packed() + elif suffix in _GATE_UP_SLOTS: + rel, slot = _GATE_UP_SLOTS[suffix] + gate_up_buf.setdefault((layer, rel), {})[slot] = (t.packed(), t.ggml_type) + else: + raise ValueError(f"unmapped laguna GGUF tensor: {name}") + + for key in [k for k in gate_up_buf if k[0] == layer]: + gu = gate_up_buf[key] + if len(gu) == 2: + if gu["gate"][1] != gu["up"][1]: + raise ValueError(f"blk.{layer}: mixed ggml types across fused gate/up") + yield f"{base}.{key[1]}.qweight", torch.cat( + [gu["gate"][0], gu["up"][0]], dim=0 + ) + del gate_up_buf[key] + + assert not gate_up_buf, f"incomplete gate_up groups: {sorted(gate_up_buf)}" + + +def is_gguf_model(config: ModelConfig) -> bool: + """True when the model was parsed from a GGUF checkpoint (native-quant path).""" + return config.gguf_embed_quant is not None + + +class DeferredGGUFLinear(BaseOP): + """GGUF linear whose quant type is only known at weight-load time. + + Unsloth Dynamic checkpoints choose the ggml type per tensor, so conversion + cannot size the packed buffer up front; the loader calls :meth:`materialize` + with the tensor's recorded type before copying rows in. + """ + + def __init__(self, in_features: int, out_features: int, has_bias: bool = False): + self.in_features = in_features + self.out_features = out_features + self._quant_type: int | None = None + self.qweight: torch.Tensor | None = None + self.bias = torch.empty(out_features) if has_bias else None + + def materialize(self, quant_type: int) -> None: + from freetoken.models.gguf.dequant import row_bytes + + self._quant_type = quant_type + self.qweight = torch.empty( + self.out_features, row_bytes(self.in_features, quant_type), dtype=torch.uint8 + ) + + def forward(self, x: torch.Tensor) -> torch.Tensor: + from freetoken.layers.gguf import fused_mul_mat_gguf + + assert self.qweight is not None and self._quant_type is not None, ( + "DeferredGGUFLinear used before materialize() -- weight was never loaded" + ) + out = fused_mul_mat_gguf(x, self.qweight, self._quant_type) + if self.bias is not None: + out = out + self.bias + return out + + +class LagunaGGUFLMHead(DeferredGGUFLinear): + def forward(self, x: torch.Tensor) -> torch.Tensor: + from freetoken.core import get_global_ctx + from freetoken.layers.gguf import fused_mul_mat_gguf + batch = get_global_ctx().batch + if batch.is_prefill: + x = x[batch.attn_metadata.get_last_indices(batch.size)].contiguous() + assert self.qweight is not None and self._quant_type is not None + return fused_mul_mat_gguf(x, self.qweight, self._quant_type) + + +def _swap(owner, attr: str) -> None: + # _LinearTPImpl exposes local_*_size (== full sizes at TP=1, which the GGUF + # path requires); the weight shape is the ground truth either way. + old = getattr(owner, attr) + out_features, in_features = old.weight.shape + setattr( + owner, + attr, + DeferredGGUFLinear(in_features, out_features, getattr(old, "bias", None) is not None), + ) + + +def convert_laguna_to_gguf(model, config: ModelConfig) -> None: + """In place: replace laguna's dense projections + embedding + head with GGUF ops. + + Swapped: attention qkv/gate/o, the dense layer's MLP, every shared expert, the + embedding, and the (untied) lm_head. Kept dense: the fp32 MoE router gate and + e_score_correction_bias, all RMSNorms (F32 in the GGUF), and the routed expert + banks (they live on the MoE offload cache). + + When the source ``.gguf`` is known (``config.gguf_model_path``), every swapped + module is materialized here from the file's per-tensor ggml types, so the + packed buffers exist before the engine collects ``model.state_dict()``. + """ + from freetoken.layers.gguf import GGUFEmbedding + + types = gguf_tensor_types(config.gguf_model_path) if config.gguf_model_path else None + + def qt(name: str) -> int | None: + return None if types is None else types.get(name) + + def mat(module: DeferredGGUFLinear, tensor_name: str) -> None: + t = qt(tensor_name) + if t is not None: + module.materialize(t) + + model.model.embed_tokens = GGUFEmbedding( + config.vocab_size, config.hidden_size, config.gguf_embed_quant + ) + for i, layer in enumerate(model.model.layers.op_list): + for attr, tname in ( + ("q_proj", "attn_q.weight"), + ("k_proj", "attn_k.weight"), + ("v_proj", "attn_v.weight"), + ("gate_proj", "attn_gate.weight"), + ("o_proj", "attn_output.weight"), + ): + _swap(layer.self_attn, attr) + mat(getattr(layer.self_attn, attr), f"blk.{i}.{tname}") + mlp = layer.mlp + if hasattr(mlp, "gate_up_proj"): # dense leading layer (LagunaMLP) + _swap(mlp, "gate_up_proj") + _swap(mlp, "down_proj") + mat(mlp.gate_up_proj, f"blk.{i}.ffn_gate.weight") + mat(mlp.down_proj, f"blk.{i}.ffn_down.weight") + else: # LagunaSparseMoeBlock: shared expert only (router stays fp32) + _swap(mlp.shared_experts, "gate_up_proj") + _swap(mlp.shared_experts, "down_proj") + mat(mlp.shared_experts.gate_up_proj, f"blk.{i}.ffn_gate_shexp.weight") + mat(mlp.shared_experts.down_proj, f"blk.{i}.ffn_down_shexp.weight") + # Untied output head (output.weight, Q4_K in the target file). + model.lm_head = LagunaGGUFLMHead(config.hidden_size, config.vocab_size) + mat(model.lm_head, "output.weight") + + + +# -------------------------------------------------------------------------------------- +# Routed expert banks (mixed per-layer ggml types) for the MoE offload cache. +# -------------------------------------------------------------------------------------- + + +def _expert_bank_geometry(config: ModelConfig): + """Uniform flat-slot strides across MoE layers: max payload bytes, 64B aligned.""" + from freetoken.models.gguf.dequant import row_bytes + + assert config.gguf_expert_types, "laguna expert banks need gguf_expert_types" + H, I = config.hidden_size, config.moe_intermediate_size + gu_pay = {gu: 2 * I * row_bytes(H, gu) for gu, _ in config.gguf_expert_types} + dn_pay = {dn: H * row_bytes(I, dn) for _, dn in config.gguf_expert_types} + + def align(n: int) -> int: + return (n + 63) // 64 * 64 + + return align(max(gu_pay.values())), align(max(dn_pay.values())) + + +def load_gguf_expert_sources( + model_path: str, config: ModelConfig, *, layer_sink=None +) -> dict[str, list[torch.Tensor]]: + """Per-MoE-layer host banks of the routed experts' native packed bytes. + + Each bank is one flat ``[E, stride]`` uint8 tensor per MoE layer (bank index = + layer_id - first_k_dense_replace): every expert's real payload occupies the + leading bytes of its padded slot, so all layers share one shape and the ggml + MoE kernels read them via ``expert_stride_bytes``. Mirrors the q4_0 loader's + pin pipeline / layer_sink streaming contract. + """ + from freetoken.models.gguf.dequant import row_bytes + from freetoken.models.gguf.reader import iter_gguf_tensors + from freetoken.moe.host_banks import LayerCompletionTracker, PinPipeline, alloc_layer_banks + + _require_tp1("expert banks") + types = config.gguf_expert_types + assert types, "laguna expert banks need gguf_expert_types (tensor table missing?)" + E = config.num_experts + H, I = config.hidden_size, config.moe_intermediate_size + L = len(types) # MoE layers only + gu_stride, dn_stride = _expert_bank_geometry(config) + + specs = { + "gate_up": ((E, gu_stride), torch.uint8), + "down": ((E, dn_stride), torch.uint8), + } + hb = alloc_layer_banks(specs, L) + banks = {name: [b.tensor for b in hb[name]] for name in hb} + seen_gu, seen_dn = set(), set() + + def _load(sink) -> None: + tracker = LayerCompletionTracker(2, hb, sink) if sink is not None else None + gu_parts: dict[int, dict[str, torch.Tensor]] = {} + for t in iter_gguf_tensors(model_path): + name = t.name + if not name.startswith("blk.") or not name.endswith(tuple(_EXPERT_SUFFIXES)): + continue + layer = int(name.split(".")[1]) + bank_id = layer - config.first_k_dense_replace + gu_t, dn_t = types[bank_id] + if name.endswith("ffn_down_exps.weight"): + pay = H * row_bytes(I, dn_t) + banks["down"][bank_id][:, :pay].copy_(t.packed().reshape(E, pay)) + seen_dn.add(bank_id) + if tracker is not None: + tracker.note(bank_id) + else: + # gate and up arrive as separate tensors; each expert's slot holds + # gate rows then up rows (the fused gate_up layout the kernel expects). + half = I * row_bytes(H, gu_t) + part = gu_parts.setdefault(bank_id, {}) + part["gate" if "gate" in name else "up"] = t.packed().reshape(E, half) + if len(part) < 2: + continue + dst = banks["gate_up"][bank_id] + dst[:, :half].copy_(part["gate"]) + dst[:, half : 2 * half].copy_(part["up"]) + del gu_parts[bank_id] + seen_gu.add(bank_id) + if tracker is not None: + tracker.note(bank_id) + assert not gu_parts, f"incomplete expert gate/up layers: {sorted(gu_parts)}" + + if layer_sink is not None: + _load(layer_sink) + elif torch.cuda.is_available(): + with PinPipeline() as pins: + _load(pins) + else: + _load(None) + + want = set(range(L)) + assert seen_gu == want and seen_dn == want, ( + f"missing expert layers: gate_up {sorted(want - seen_gu)}, down {sorted(want - seen_dn)}" + ) + return banks + + +def dummy_gguf_expert_sources(config: ModelConfig) -> dict[str, list[torch.Tensor]]: + """Random banks shaped like ``load_gguf_expert_sources`` output.""" + from freetoken.moe.host_banks import alloc_layer_banks, pin_banks + + E = config.num_experts + L = len(config.gguf_expert_types or ()) + assert L, "laguna dummy expert banks need gguf_expert_types" + gu_stride, dn_stride = _expert_bank_geometry(config) + hb = alloc_layer_banks( + {"gate_up": ((E, gu_stride), torch.uint8), "down": ((E, dn_stride), torch.uint8)}, L + ) + banks = {name: [b.tensor for b in hb[name]] for name in hb} + for t in banks["gate_up"] + banks["down"]: + t.random_(0, 256) + if torch.cuda.is_available(): + pin_banks(hb) + return banks + +__all__ = [ + "parse_gguf_config", + "iter_gguf_weights", + "gguf_tensor_types", + "is_gguf_model", + "DeferredGGUFLinear", + "LagunaGGUFLMHead", + "convert_laguna_to_gguf", + "load_gguf_expert_sources", + "dummy_gguf_expert_sources", +] diff --git a/python/freetoken/models/laguna/model.py b/python/freetoken/models/laguna/model.py new file mode 100644 index 00000000..c0cd77f3 --- /dev/null +++ b/python/freetoken/models/laguna/model.py @@ -0,0 +1,67 @@ +from __future__ import annotations + +import torch +from freetoken.core import get_global_ctx +from freetoken.layers import BaseOP, OPList, ParallelLMHead, RMSNorm, VocabParallelEmbedding +from freetoken.models.blocks import BaseLLMModel +from freetoken.utils import nvtx_annotate + +from .attention import LagunaAttention +from .moe import LagunaMLP, LagunaSparseMoeBlock + + +class LagunaDecoderLayer(BaseOP): + def __init__(self, config, layer_id: int): + self._layer_id = layer_id + self.input_layernorm = RMSNorm(config.hidden_size, eps=config.rms_norm_eps) + self.ffn_norm = RMSNorm(config.hidden_size, eps=config.rms_norm_eps) + self.self_attn = LagunaAttention(config, layer_id) + self.mlp = (LagunaMLP(config.hidden_size, config.intermediate_size) + if layer_id < config.first_k_dense_replace + else LagunaSparseMoeBlock(config, layer_id)) + + @nvtx_annotate("Layer_{}", layer_id_field="_layer_id") + def forward(self, x: torch.Tensor) -> torch.Tensor: + x = x + self.self_attn.forward(self.input_layernorm.forward(x)) + x = x + self.mlp.forward(self.ffn_norm.forward(x)) + return x + + +class LagunaModel(BaseOP): + def __init__(self, config): + self.embed_tokens = VocabParallelEmbedding(config.vocab_size, config.hidden_size) + self.layers = OPList([LagunaDecoderLayer(config, i) for i in range(config.num_layers)]) + self.norm = RMSNorm(config.hidden_size, eps=config.rms_norm_eps) + + def forward(self, input_ids: torch.Tensor) -> torch.Tensor: + x = self.embed_tokens.forward(input_ids) + for layer in self.layers.op_list: + x = layer.forward(x) + return self.norm.forward(x) + + +class LagunaForCausalLM(BaseLLMModel): + def __init__(self, config): + self.model = LagunaModel(config) + self.lm_head = ParallelLMHead(config.vocab_size, config.hidden_size, + tie_word_embeddings=config.tie_word_embeddings, + tied_embedding=None) + super().__init__() + from .gguf import convert_laguna_to_gguf, is_gguf_model + if is_gguf_model(config): + convert_laguna_to_gguf(self, config) + elif config.gguf_model_path: + # GGUF-sourced but no tensor table (a converted FTW dir's metadata-only + # source_metadata.gguf): the per-tensor quant types are unrecoverable, so + # neither module conversion nor expert banks can be built. Refuse loudly + # instead of constructing a dense model that fails weight loading. + raise NotImplementedError( + "laguna FTW conversion is not supported yet -- serve the .gguf file " + "directly (per-tensor quant types live only in its tensor table)" + ) + + def forward(self) -> torch.Tensor: + return self.lm_head.forward(self.model.forward(get_global_ctx().batch.input_ids)) + + +__all__ = ["LagunaDecoderLayer", "LagunaModel", "LagunaForCausalLM"] diff --git a/python/freetoken/models/laguna/moe.py b/python/freetoken/models/laguna/moe.py new file mode 100644 index 00000000..eea90e8e --- /dev/null +++ b/python/freetoken/models/laguna/moe.py @@ -0,0 +1,89 @@ +from __future__ import annotations + +from typing import TYPE_CHECKING, Tuple + +import torch +import torch.nn.functional as F + +from freetoken.layers import BaseOP, LinearReplicated, make_moe_layer, silu_and_mul + +if TYPE_CHECKING: + from freetoken.models.config import ModelConfig + +TopK = Tuple[torch.Tensor, torch.Tensor] + + +class LagunaMLP(BaseOP): + """Plain SwiGLU MLP used by Laguna dense and shared experts.""" + + def __init__(self, hidden_size: int, intermediate_size: int): + self.gate_up_proj = LinearReplicated(hidden_size, 2 * intermediate_size, has_bias=False) + self.down_proj = LinearReplicated(intermediate_size, hidden_size, has_bias=False) + + def forward(self, x: torch.Tensor) -> torch.Tensor: + gate_up = self.gate_up_proj.forward(x) + del x + y = silu_and_mul(gate_up) + del gate_up + return self.down_proj.forward(y) + + +class LagunaSparseMoeBlock(BaseOP): + """Mixture-of-experts block for Laguna.""" + + def __init__(self, config: ModelConfig, layer_id: int): + self.top_k = config.num_experts_per_tok + self.num_experts = config.num_experts + self.norm_topk_prob = config.norm_topk_prob + self.routed_scaling_factor = config.routed_scaling_factor + + # Router weights are fp32; this is required for exact tie-breaking at the top-k boundary. + self.gate = LinearReplicated(config.hidden_size, config.num_experts, has_bias=False) + self.gate.weight = torch.empty(config.num_experts, config.hidden_size, dtype=torch.float32) + self.e_score_correction_bias = torch.empty(config.num_experts, dtype=torch.float32) + + moe_layer_id = layer_id - config.first_k_dense_replace + # Mixed-type GGUF banks ("gguf" offload format): the kernels need this + # layer's ggml types and output-row geometry (see fused_experts_gguf). + extra_attrs = None + if config.gguf_expert_types is not None: + gu_t, dn_t = config.gguf_expert_types[moe_layer_id] + extra_attrs = { + "gguf_gate_up_type": gu_t, + "gguf_down_type": dn_t, + "gguf_gate_up_rows": 2 * config.moe_intermediate_size, + "gguf_down_rows": config.hidden_size, + } + self.experts = make_moe_layer( + config, + layer_id=moe_layer_id, + renormalize=config.norm_topk_prob, + activation="silu", + extra_attrs=extra_attrs, + ) + self.shared_experts = LagunaMLP( + config.hidden_size, + config.shared_expert_intermediate_size * max(1, config.n_shared_experts), + ) + + def _route(self, hidden_states: torch.Tensor) -> TopK: + logits = F.linear(hidden_states.float(), self.gate.weight) + scores = logits.sigmoid() + scores_for_choice = scores + self.e_score_correction_bias + _, topk_ids = torch.topk(scores_for_choice, self.top_k, dim=-1) + topk_weights = scores.gather(-1, topk_ids) + if self.norm_topk_prob: + topk_weights = topk_weights / (topk_weights.sum(dim=-1, keepdim=True) + 1e-20) + topk_weights = topk_weights * self.routed_scaling_factor + return topk_weights.to(torch.float32).contiguous(), topk_ids.to(torch.int32).contiguous() + + def forward(self, hidden_states: torch.Tensor) -> torch.Tensor: + num_tokens, hidden_dim = hidden_states.shape + hidden_states = hidden_states.view(-1, hidden_dim) + topk_weights, topk_ids = self._route(hidden_states) + out = self.experts.routed_forward(hidden_states, topk_weights, topk_ids) + out = out + self.shared_experts.forward(hidden_states) + return out.view(num_tokens, hidden_dim) + + +__all__ = ["LagunaMLP", "LagunaSparseMoeBlock"] diff --git a/python/freetoken/models/laguna/weight.py b/python/freetoken/models/laguna/weight.py new file mode 100644 index 00000000..b7bb4f62 --- /dev/null +++ b/python/freetoken/models/laguna/weight.py @@ -0,0 +1,208 @@ +from __future__ import annotations + +import re +from collections.abc import Iterator + +import torch + +from freetoken.distributed import get_tp_info +from freetoken.models.gguf.dequant import GGML_BF16, GGML_Q4_0, row_bytes +from freetoken.models.loader import ShardReader, drop_page_cache, iter_weight_files + +_EXPERT_RE = re.compile(r"^model\.layers\.\d+\.mlp\.experts\.\d+\.(?:gate|up|down)_proj\.") +_MERGE_PARTS = { + ".mlp.gate_proj.weight": (".mlp.gate_up_proj.weight", "gate"), + ".mlp.up_proj.weight": (".mlp.gate_up_proj.weight", "up"), + ".mlp.shared_expert.gate_proj.weight": (".mlp.shared_experts.gate_up_proj.weight", "gate"), + ".mlp.shared_expert.up_proj.weight": (".mlp.shared_experts.gate_up_proj.weight", "up"), +} + + +def _rename(name: str) -> str: + name = name.replace(".self_attn.g_proj.", ".self_attn.gate_proj.") + name = name.replace(".post_attention_layernorm.", ".ffn_norm.") + name = name.replace(".mlp.shared_expert.", ".mlp.shared_experts.") + name = name.replace(".mlp.experts.e_score_correction_bias", ".mlp.e_score_correction_bias") + return name + + +def iter_weights( + model_path: str, + device: torch.device, + *, + include_moe_experts: bool, + include_non_moe: bool, +) -> Iterator[tuple[str, torch.Tensor]]: + if get_tp_info().size > 1: + raise NotImplementedError("Laguna safetensors currently supports TP=1 only") + if include_moe_experts: + raise NotImplementedError( + "Laguna compressed INT4 experts require --moe-backend offload; " + "resident/fused expert loading is not supported" + ) + if not include_non_moe: + return + + import safetensors + + merge: dict[str, dict[str, torch.Tensor]] = {} + for file in iter_weight_files(model_path): + with safetensors.safe_open(file, framework="pt", device=str(device)) as f: + for raw_name in f.keys(): + if _EXPERT_RE.match(raw_name): + continue + # Scale/shape tensors occur only under routed experts in this artifact, + # but keep this guard explicit for a useful failure on future variants. + if raw_name.endswith((".weight_packed", ".weight_scale", ".weight_shape")): + raise ValueError(f"unsupported quantized non-expert Laguna tensor: {raw_name}") + + merged = None + for suffix, (target, slot) in _MERGE_PARTS.items(): + if raw_name.endswith(suffix): + merged = (raw_name[: -len(suffix)] + target, slot) + break + tensor = f.get_tensor(raw_name) + if merged is None: + name = _rename(raw_name) + # Laguna routes in fp32: the model intentionally allocates this + # parameter as fp32 even though the checkpoint stores it in bf16. + if name.endswith(".mlp.gate.weight"): + tensor = tensor.float() + yield name, tensor + continue + key, slot = merged + slots = merge.setdefault(_rename(key), {}) + slots[slot] = tensor + if len(slots) == 2: + yield _rename(key), torch.cat([slots["gate"], slots["up"]], dim=0) + del merge[_rename(key)] + drop_page_cache(file) + assert not merge, f"incomplete Laguna gate/up groups: {sorted(merge)}" + + +def _ct_int4_to_q4_0(packed: torch.Tensor, scale: torch.Tensor) -> torch.Tensor: + """Repack compressed-tensors symmetric INT4 into GGML Q4_0 row blocks. + + Both encode signed values as an offset nibble (q + 8). Only nibble ordering + differs. Q4_0 stores the scale as fp16, so the checkpoint's bf16 group scale is + rounded once while the 4-bit values remain bit-exact. + """ + if packed.dtype is not torch.int32 or scale.dtype is not torch.bfloat16: + raise ValueError(f"unexpected Laguna INT4 dtypes: {packed.dtype}/{scale.dtype}") + rows, words = packed.shape + groups = scale.shape[1] + if words != groups * 4 or scale.shape[0] != rows: + raise ValueError(f"unexpected Laguna INT4 geometry: {packed.shape}/{scale.shape}") + src = packed.contiguous().view(torch.uint8).reshape(rows, groups, 16) + lo_half, hi_half = src[..., :8], src[..., 8:] + q0 = torch.stack((lo_half & 0x0F, lo_half >> 4), dim=-1).flatten(-2) + q1 = torch.stack((hi_half & 0x0F, hi_half >> 4), dim=-1).flatten(-2) + out = torch.empty((rows, groups, 18), dtype=torch.uint8) + out[..., :2].copy_(scale.to(torch.float16).contiguous().view(torch.uint8).reshape(rows, groups, 2)) + out[..., 2:].copy_(q0 | (q1 << 4)) + return out.reshape(rows, groups * 18) + + +def _bank_payloads(config, qtype: int) -> tuple[int, int]: + H, I = config.hidden_size, config.moe_intermediate_size + if qtype == GGML_Q4_0: + return 2 * I * row_bytes(H, qtype), H * row_bytes(I, qtype) + if qtype == GGML_BF16: + return 2 * I * H * 2, H * I * 2 + raise ValueError(f"unsupported Laguna safetensors expert type {qtype}") + + +def load_int4_expert_sources( + model_path: str, config, *, layer_sink=None +) -> dict[str, list[torch.Tensor]]: + """Load exact mixed INT4/BF16 expert layers into variable-size flat banks.""" + if get_tp_info().size > 1: + raise NotImplementedError("Laguna safetensors expert banks currently support TP=1 only") + from freetoken.moe.host_banks import HostBank, PinPipeline + + types = config.gguf_expert_types + assert types and len(types) == config.num_moe_layers + E, H, I = config.num_experts, config.hidden_size, config.moe_intermediate_size + host = {"gate_up": [], "down": []} + for gu_t, dn_t in types: + gu_pay, _ = _bank_payloads(config, gu_t) + _, dn_pay = _bank_payloads(config, dn_t) + host["gate_up"].append(HostBank((E, gu_pay), torch.uint8)) + host["down"].append(HostBank((E, dn_pay), torch.uint8)) + banks = {name: [bank.tensor for bank in per_layer] for name, per_layer in host.items()} + + reader = ShardReader(model_path, torch.device("cpu")) + pins = PinPipeline() if layer_sink is None and torch.cuda.is_available() else None + try: + for bank_id, (gu_t, dn_t) in enumerate(types): + layer = bank_id + config.first_k_dense_replace + prefix = f"model.layers.{layer}.mlp.experts" + gu_dst, dn_dst = banks["gate_up"][bank_id], banks["down"][bank_id] + for expert in range(E): + ep = f"{prefix}.{expert}" + if gu_t == GGML_Q4_0: + gate = _ct_int4_to_q4_0( + reader.get_tensor(ep + ".gate_proj.weight_packed"), + reader.get_tensor(ep + ".gate_proj.weight_scale"), + ) + up = _ct_int4_to_q4_0( + reader.get_tensor(ep + ".up_proj.weight_packed"), + reader.get_tensor(ep + ".up_proj.weight_scale"), + ) + down = _ct_int4_to_q4_0( + reader.get_tensor(ep + ".down_proj.weight_packed"), + reader.get_tensor(ep + ".down_proj.weight_scale"), + ) + half = I * row_bytes(H, GGML_Q4_0) + gu_dst[expert, :half].copy_(gate.reshape(-1)) + gu_dst[expert, half:].copy_(up.reshape(-1)) + dn_dst[expert].copy_(down.reshape(-1)) + elif gu_t == dn_t == GGML_BF16: + gate = reader.get_tensor(ep + ".gate_proj.weight") + up = reader.get_tensor(ep + ".up_proj.weight") + down = reader.get_tensor(ep + ".down_proj.weight") + if not (gate.dtype is up.dtype is down.dtype is torch.bfloat16): + raise ValueError(f"unexpected BF16 Laguna expert dtype at {ep}") + gu_dst[expert].copy_( + torch.cat((gate, up), dim=0).contiguous().view(torch.uint8).reshape(-1) + ) + dn_dst[expert].copy_(down.contiguous().view(torch.uint8).reshape(-1)) + else: + raise ValueError(f"gate/up and down storage differ at Laguna layer {layer}") + layer_banks = {name: host[name][bank_id] for name in host} + if layer_sink is not None: + layer_sink(bank_id, layer_banks) + elif pins is not None: + pins(bank_id, layer_banks) + finally: + reader.close() + if pins is not None: + pins.wait() + for file in iter_weight_files(model_path): + drop_page_cache(file) + return banks + + +def dummy_int4_expert_sources(config) -> dict[str, list[torch.Tensor]]: + from freetoken.moe.host_banks import HostBank, pin_banks + + host = {"gate_up": [], "down": []} + for gu_t, dn_t in config.gguf_expert_types or (): + gu_pay, _ = _bank_payloads(config, gu_t) + _, dn_pay = _bank_payloads(config, dn_t) + host["gate_up"].append(HostBank((config.num_experts, gu_pay), torch.uint8)) + host["down"].append(HostBank((config.num_experts, dn_pay), torch.uint8)) + banks = {name: [bank.tensor for bank in per_layer] for name, per_layer in host.items()} + for tensor in banks["gate_up"] + banks["down"]: + tensor.random_(0, 256) + if torch.cuda.is_available(): + pin_banks(host) + return banks + + +__all__ = [ + "_ct_int4_to_q4_0", + "dummy_int4_expert_sources", + "iter_weights", + "load_int4_expert_sources", +] diff --git a/python/freetoken/models/nemotron_h/__init__.py b/python/freetoken/models/nemotron_h/__init__.py new file mode 100644 index 00000000..468d19b8 --- /dev/null +++ b/python/freetoken/models/nemotron_h/__init__.py @@ -0,0 +1,18 @@ +from .config import NemotronHArgs, parse_config +from .model import NemotronHForCausalLM +from .weight import ( + dummy_nvfp4_expert_sources, + iter_weights, + load_nvfp4_expert_sources, + load_nvfp4_expert_sources_parallel, +) + +__all__ = [ + "NemotronHArgs", + "NemotronHForCausalLM", + "iter_weights", + "dummy_nvfp4_expert_sources", + "load_nvfp4_expert_sources", + "load_nvfp4_expert_sources_parallel", + "parse_config", +] diff --git a/python/freetoken/models/nemotron_h/config.py b/python/freetoken/models/nemotron_h/config.py new file mode 100644 index 00000000..54711414 --- /dev/null +++ b/python/freetoken/models/nemotron_h/config.py @@ -0,0 +1,144 @@ +from __future__ import annotations + +from dataclasses import dataclass +from typing import Any + +from freetoken.models.config import ( + FullAttentionGroupConfig, + LinearGatedDeltaGroupConfig, + ModelConfig, + RotaryConfig, +) + + +@dataclass(frozen=True) +class NemotronHArgs: + layer_types: tuple[str, ...] + mamba_num_heads: int + mamba_head_dim: int + ssm_state_size: int + n_groups: int + conv_kernel: int + chunk_size: int + mamba_intermediate_size: int + moe_latent_size: int + shared_intermediate_size: int + fp8_modules: frozenset[str] + nvfp4_dense_modules: frozenset[str] + + def module_quant(self, name: str) -> str: + if name in self.fp8_modules: + return "fp8_pertensor" + if name in self.nvfp4_dense_modules: + # These rare dense FP4 matrices are dequantized by the loader. The large + # routed-expert matrices stay native NVFP4 in the offload banks. + return "dequant_bf16" + return "none" + + +def _quantized_modules(hf_config: Any) -> tuple[frozenset[str], frozenset[str]]: + quant = getattr(hf_config, "quantization_config", None) or {} + get = quant.get if isinstance(quant, dict) else lambda k, d=None: getattr(quant, k, d) + layers = get("quantized_layers", {}) or {} + fp8, nvfp4 = set(), set() + for name, spec in layers.items(): + if ".experts." in name: + continue + algo = str((spec or {}).get("quant_algo", "")).lower() + if algo == "fp8": + fp8.add(name) + elif "fp4" in algo: + nvfp4.add(name) + return frozenset(fp8), frozenset(nvfp4) + + +def parse_config(hf_config: Any) -> ModelConfig: + layer_types = tuple( + "mamba" if kind in ("mamba", "linear_attention") else kind + for kind in hf_config.layers_block_type + ) + mamba_ids = tuple(i for i, kind in enumerate(layer_types) if kind == "mamba") + attention_ids = tuple(i for i, kind in enumerate(layer_types) if kind == "attention") + moe_ids = tuple(i for i, kind in enumerate(layer_types) if kind == "moe") + fp8_modules, nvfp4_dense_modules = _quantized_modules(hf_config) + + head_dim = int(getattr(hf_config, "head_dim", 128)) + rotary = RotaryConfig( + head_dim=head_dim, + rotary_dim=head_dim, + max_position=int(hf_config.max_position_embeddings), + base=float(getattr(hf_config, "rope_theta", 10000.0)), + scaling=None, + ) + groups = ( + LinearGatedDeltaGroupConfig( + name="mamba", + layer_ids=mamba_ids, + # LinearStatePool is [value_heads, key_dim, value_dim]. Map those axes to + # Mamba's [heads, state_dim, head_dim]. + num_key_heads=int(hf_config.n_groups), + num_value_heads=int(hf_config.mamba_num_heads), + key_head_dim=int(hf_config.ssm_state_size), + value_head_dim=int(hf_config.mamba_head_dim), + conv_kernel_dim=int(hf_config.conv_kernel), + output_gate=True, + ), + FullAttentionGroupConfig( + name="full", + layer_ids=attention_ids, + num_kv_heads=int(hf_config.num_key_value_heads), + head_dim=head_dim, + rotary_config=rotary, + ), + ) + args = NemotronHArgs( + layer_types=layer_types, + mamba_num_heads=int(hf_config.mamba_num_heads), + mamba_head_dim=int(hf_config.mamba_head_dim), + ssm_state_size=int(hf_config.ssm_state_size), + n_groups=int(hf_config.n_groups), + conv_kernel=int(hf_config.conv_kernel), + chunk_size=int(hf_config.chunk_size), + mamba_intermediate_size=int(hf_config.mamba_num_heads * hf_config.mamba_head_dim), + moe_latent_size=int(hf_config.moe_latent_size), + shared_intermediate_size=int(hf_config.moe_shared_expert_intermediate_size), + fp8_modules=fp8_modules, + nvfp4_dense_modules=nvfp4_dense_modules, + ) + return ModelConfig( + num_layers=int(hf_config.num_hidden_layers), + num_qo_heads=int(hf_config.num_attention_heads), + num_kv_heads=int(hf_config.num_key_value_heads), + head_dim=head_dim, + hidden_size=int(hf_config.hidden_size), + vocab_size=int(hf_config.vocab_size), + intermediate_size=int(hf_config.intermediate_size), + rms_norm_eps=float(hf_config.layer_norm_epsilon), + rotary_config=rotary, + hidden_act="relu2", + tie_word_embeddings=bool(hf_config.tie_word_embeddings), + num_experts=int(hf_config.n_routed_experts), + num_experts_per_tok=int(hf_config.num_experts_per_tok), + moe_intermediate_size=int(hf_config.moe_intermediate_size), + norm_topk_prob=bool(hf_config.norm_topk_prob), + model_type=str(hf_config.model_type), + architectures=list(hf_config.architectures), + moe_enabled=True, + expert_quant="nvfp4", + attn_quant="fp8_pertensor" if fp8_modules else "none", + shared_expert_intermediate_size=int(hf_config.moe_shared_expert_intermediate_size), + routed_scaling_factor=float(hf_config.routed_scaling_factor), + n_group=int(hf_config.n_group), + topk_group=int(hf_config.topk_group), + attention_groups=groups, + moe_layer_ids=moe_ids, + expert_hidden_size=int(hf_config.moe_latent_size), + expert_gated=False, + nemotron_h_args=args, + # One live Mamba state is ~160 MiB at fp32. Keep the initial implementation + # bounded to one session; concurrency can be enabled once state paging lands. + single_stream_only=True, + ) + + +__all__ = ["NemotronHArgs", "parse_config"] diff --git a/python/freetoken/models/nemotron_h/model.py b/python/freetoken/models/nemotron_h/model.py new file mode 100644 index 00000000..1ddb41f3 --- /dev/null +++ b/python/freetoken/models/nemotron_h/model.py @@ -0,0 +1,348 @@ +from __future__ import annotations + +from typing import TYPE_CHECKING + +import torch +import torch.nn.functional as F + +from freetoken.attention.linear import build_fla_metadata +from freetoken.core import get_global_ctx +from freetoken.kernel.causal_conv1d import causal_conv1d_decode, causal_conv1d_varlen +from freetoken.layers import ( + BaseOP, + LinearColParallelMerged, + LinearReplicated, + OPList, + ParallelLMHead, + RMSNorm, + VocabParallelEmbedding, + make_moe_layer, +) +from freetoken.models.blocks import BaseLLMModel +from freetoken.utils import nvtx_annotate + +if TYPE_CHECKING: + from freetoken.models.config import ModelConfig + from .config import NemotronHArgs + + +def _linear(args: "NemotronHArgs", name: str, in_f: int, out_f: int): + if args.module_quant(name) == "fp8_pertensor": + from freetoken.kernel.triton.fp8_pertensor_linear import Fp8PerTensorLinear + + return Fp8PerTensorLinear(in_f, out_f, has_bias=False) + return LinearReplicated(in_f, out_f, has_bias=False) + + +class _DepthwiseConv1d(BaseOP): + def __init__(self, dim: int, kernel: int): + self.weight = torch.empty(dim, 1, kernel) + self.bias = torch.empty(dim) + + +class _MambaGatedRMSNorm(BaseOP): + def __init__(self, size: int, groups: int, eps: float): + self.weight = torch.empty(size) + self.groups = groups + self.group_size = size // groups + self.eps = eps + + def forward(self, x: torch.Tensor, gate: torch.Tensor) -> torch.Tensor: + dtype = x.dtype + x = x.float() * F.silu(gate.float()) + shape = x.shape + grouped = x.view(*shape[:-1], self.groups, self.group_size) + grouped = grouped * torch.rsqrt(grouped.square().mean(-1, keepdim=True) + self.eps) + return grouped.view(shape).to(dtype) * self.weight + + +class NemotronHMamba2Mixer(BaseOP): + def __init__(self, config: "ModelConfig", layer_id: int): + args = config.nemotron_h_args + assert args is not None + self.layer_id = layer_id + self.num_heads = args.mamba_num_heads + self.head_dim = args.mamba_head_dim + self.state_size = args.ssm_state_size + self.n_groups = args.n_groups + self.intermediate_size = args.mamba_intermediate_size + self.conv_dim = self.intermediate_size + 2 * self.n_groups * self.state_size + self.chunk_size = args.chunk_size + prefix = f"backbone.layers.{layer_id}.mixer" + self.in_proj = _linear( + args, f"{prefix}.in_proj", config.hidden_size, + self.intermediate_size + self.conv_dim + self.num_heads, + ) + self.conv1d = _DepthwiseConv1d(self.conv_dim, args.conv_kernel) + self.dt_bias = torch.empty(self.num_heads, dtype=torch.float32) + self.A_log = torch.empty(self.num_heads, dtype=torch.float32) + self.norm = _MambaGatedRMSNorm( + self.intermediate_size, self.n_groups, config.rms_norm_eps + ) + self.D = torch.empty(self.num_heads, dtype=torch.float32) + self.out_proj = _linear( + args, f"{prefix}.out_proj", self.intermediate_size, config.hidden_size + ) + + def _conv(self, conv_in: torch.Tensor, fla, pool, decode: bool) -> torch.Tensor: + li = pool.local_index(self.layer_id) + weight = self.conv1d.weight.squeeze(1) + if decode: + return causal_conv1d_decode( + conv_in, pool.conv_states[li], weight, fla.cache_indices, + bias=self.conv1d.bias, + ) + return causal_conv1d_varlen( + conv_in.transpose(0, 1).contiguous(), weight, pool.conv_states[li], + fla.cu_seqlens, fla.cache_indices, fla.has_initial_state, + bias=self.conv1d.bias, + ).transpose(0, 1) + + def _scan(self, x, dt, B, C, initial): + # Transformers' implementation has a pure-Torch chunk fallback when mamba_ssm is + # absent. It is the reference recurrence and avoids an O(sequence) Python loop. + from transformers.models.nemotron_h.modeling_nemotron_h import mamba2_chunk_scan + + return mamba2_chunk_scan( + x.unsqueeze(0), dt.unsqueeze(0), -torch.exp(self.A_log.float()), + B.unsqueeze(0), C.unsqueeze(0), chunk_size=self.chunk_size, + D=self.D, dt_bias=self.dt_bias, initial_states=initial.unsqueeze(0), + dt_softplus=True, return_final_states=True, + ) + + def _prefill_scan(self, x, dt, B, C, fla, pool) -> torch.Tensor: + li = pool.local_index(self.layer_id) + if fla.fresh_state_indices is not None: + pool.recurrent_states[li].index_fill_(0, fla.fresh_state_indices, 0.0) + outputs = [] + offset = 0 + for req in get_global_ctx().batch.padded_reqs: + length = req.extend_len + slot = req.linear_slot_idx if req.linear_slot_idx is not None else req.table_idx + initial = pool.recurrent_states[li, slot].transpose(-1, -2).contiguous() + sx, sdt, sB, sC = (v[offset:offset + length] for v in (x, dt, B, C)) + + # Hybrid-radix asks for at most one mid-chunk snapshot per request. Split the + # reference scan at that boundary so the donated state is exact. + boundary = None + if req.mamba_last_track_seqlen is not None: + candidate = req.mamba_last_track_seqlen - req.cached_len + if 0 < candidate < length: + boundary = candidate + if boundary is not None: + out1, state1 = self._scan( + sx[:boundary], sdt[:boundary], sB[:boundary], sC[:boundary], initial + ) + assert req.mamba_ping_pong is not None + dst = req.mamba_ping_pong[1 - req.mamba_next_track_idx] + pool.recurrent_states[li, dst].copy_(state1[0].transpose(-1, -2)) + out2, final = self._scan( + sx[boundary:], sdt[boundary:], sB[boundary:], sC[boundary:], state1[0] + ) + out = torch.cat((out1[0], out2[0]), dim=0) + else: + scanned, final = self._scan(sx, sdt, sB, sC, initial) + out = scanned[0] + pool.recurrent_states[li, slot].copy_(final[0].transpose(-1, -2)) + outputs.append(out) + offset += length + return torch.cat(outputs, dim=0) + + def _decode_scan(self, x, dt, B, C, fla, pool) -> torch.Tensor: + from transformers.models.nemotron_h.modeling_nemotron_h import ( + mamba2_selective_state_update, + ) + + li = pool.local_index(self.layer_id) + indices = fla.cache_indices.long() + state = pool.recurrent_states[li].index_select(0, indices).transpose(-1, -2).contiguous() + A = -torch.exp(self.A_log.float())[:, None, None].expand( + -1, self.head_dim, self.state_size + ) + out = mamba2_selective_state_update( + state, x, dt[:, :, None].expand(-1, -1, self.head_dim), A, B, C, + self.D[:, None].expand(-1, self.head_dim), + dt_bias=self.dt_bias[:, None].expand(-1, self.head_dim), dt_softplus=True, + ) + pool.recurrent_states[li].index_copy_(0, indices, state.transpose(-1, -2)) + return out + + def forward(self, hidden_states: torch.Tensor) -> torch.Tensor: + ctx = get_global_ctx() + batch, pool = ctx.batch, ctx.linear_state_pool + assert pool is not None + fla = batch.fla_metadata + if fla is None: + fla = build_fla_metadata(batch, hidden_states.device) + batch.fla_metadata = fla + proj = self.in_proj.forward(hidden_states) + gate, conv_in, dt = torch.split( + proj, [self.intermediate_size, self.conv_dim, self.num_heads], dim=-1 + ) + mixed = self._conv(conv_in, fla, pool, batch.is_decode) + if not batch.is_decode and fla.track_dst is not None: + li = pool.local_index(self.layer_id) + # The causal-conv pool itself is updated to the end of each extend. Preserve + # the raw conv-input window at the earlier radix snapshot boundary separately. + conv_window = conv_in[fla.track_conv_src].transpose(-1, -2).contiguous() + pool.conv_states[li].index_copy_( + 0, fla.track_dst, conv_window.to(pool.conv_states.dtype) + ) + x, B, C = torch.split( + mixed, + [self.intermediate_size, self.n_groups * self.state_size, + self.n_groups * self.state_size], + dim=-1, + ) + x = x.view(-1, self.num_heads, self.head_dim) + B = B.view(-1, self.n_groups, self.state_size) + C = C.view(-1, self.n_groups, self.state_size) + if batch.is_decode: + scanned = self._decode_scan(x, dt, B, C, fla, pool) + else: + scanned = self._prefill_scan(x, dt, B, C, fla, pool) + # The pure-Torch chunk scan accumulates/returns fp32; the reference casts back + # to the input dtype before the gated norm and output projection. + out = self.norm.forward( + scanned.reshape(-1, self.intermediate_size).to(gate.dtype), gate + ) + return self.out_proj.forward(out) + + +class NemotronHAttention(BaseOP): + def __init__(self, config: "ModelConfig", layer_id: int): + args = config.nemotron_h_args + self.layer_id = layer_id + self.num_q = config.num_qo_heads + self.num_kv = config.num_kv_heads + self.head_dim = config.head_dim + self.q_dim = self.num_q * self.head_dim + self.kv_dim = self.num_kv * self.head_dim + # Nemotron-H attention is deliberately position-free: unlike most transformer + # blocks, it applies no RoPE. q/k/v are BF16 in this release; two o projections + # are FP8 and follow the checkpoint's per-module quant map. + self.qkv_proj = LinearColParallelMerged( + config.hidden_size, [self.q_dim, self.kv_dim, self.kv_dim], has_bias=False + ) + self.o_proj = _linear( + args, f"backbone.layers.{layer_id}.mixer.o_proj", + self.q_dim, config.hidden_size, + ) + + @nvtx_annotate("MHA") + def forward(self, x: torch.Tensor) -> torch.Tensor: + ctx = get_global_ctx() + q, k, v = torch.split( + self.qkv_proj.forward(x), [self.q_dim, self.kv_dim, self.kv_dim], dim=-1 + ) + q = q.contiguous().view(-1, self.num_q, self.head_dim) + out = ctx.attn_backend.forward(q, k.contiguous(), v.contiguous(), self.layer_id, ctx.batch) + return self.o_proj.forward(out.reshape(-1, self.q_dim)) + + +class NemotronHMLP(BaseOP): + def __init__(self, config: "ModelConfig", layer_id: int, name: str, width: int): + args = config.nemotron_h_args + prefix = f"backbone.layers.{layer_id}.mixer.{name}" + self.up_proj = _linear(args, f"{prefix}.up_proj", config.hidden_size, width) + self.down_proj = _linear(args, f"{prefix}.down_proj", width, config.hidden_size) + + def forward(self, x: torch.Tensor) -> torch.Tensor: + return self.down_proj.forward(F.relu(self.up_proj.forward(x)).square()) + + +class _NemotronRouter(BaseOP): + def __init__(self, hidden_size: int, num_experts: int): + self.weight = torch.empty(num_experts, hidden_size) + self.e_score_correction_bias = torch.empty(num_experts, dtype=torch.float32) + + def forward(self, x: torch.Tensor) -> tuple[torch.Tensor, torch.Tensor]: + scores = torch.sigmoid(F.linear(x.float(), self.weight.float())) + return scores, scores + self.e_score_correction_bias + + +class NemotronHMoE(BaseOP): + def __init__(self, config: "ModelConfig", layer_id: int, bank_id: int): + args = config.nemotron_h_args + assert args is not None + self.gate = _NemotronRouter(config.hidden_size, config.num_experts) + self.fc1_latent_proj = _linear( + args, f"backbone.layers.{layer_id}.mixer.fc1_latent_proj", + config.hidden_size, args.moe_latent_size, + ) + self.experts = make_moe_layer( + config, layer_id=bank_id, activation="relu2", renormalize=False, + hidden_size=args.moe_latent_size, intermediate_size=config.moe_intermediate_size, + ) + self.fc2_latent_proj = _linear( + args, f"backbone.layers.{layer_id}.mixer.fc2_latent_proj", + args.moe_latent_size, config.hidden_size, + ) + self.shared_experts = NemotronHMLP( + config, layer_id, "shared_experts", args.shared_intermediate_size + ) + self.top_k = config.num_experts_per_tok + self.scale = config.routed_scaling_factor + + def forward(self, x: torch.Tensor) -> torch.Tensor: + scores, choice = self.gate.forward(x) + ids = torch.topk(choice, self.top_k, dim=-1, sorted=False).indices + weights = scores.gather(1, ids) + weights = weights / (weights.sum(dim=-1, keepdim=True) + 1e-20) + weights = (weights * self.scale).float() + latent = self.fc1_latent_proj.forward(x) + routed = self.experts.routed_forward(latent, weights, ids.to(torch.int32)).to(x.dtype) + return self.fc2_latent_proj.forward(routed) + self.shared_experts.forward(x) + + +class NemotronHBlock(BaseOP): + def __init__(self, config: "ModelConfig", layer_id: int, moe_banks: dict[int, int]): + self._layer_id = layer_id + self.norm = RMSNorm(config.hidden_size, eps=config.rms_norm_eps) + kind = config.nemotron_h_args.layer_types[layer_id] + if kind == "mamba": + self.mixer = NemotronHMamba2Mixer(config, layer_id) + elif kind == "attention": + self.mixer = NemotronHAttention(config, layer_id) + elif kind == "moe": + self.mixer = NemotronHMoE(config, layer_id, moe_banks[layer_id]) + else: + raise ValueError(f"unsupported Nemotron-H block type {kind!r}") + + @nvtx_annotate("Layer_{}", layer_id_field="_layer_id") + def forward(self, x: torch.Tensor) -> torch.Tensor: + return x + self.mixer.forward(self.norm.forward(x)) + + +class NemotronHBackbone(BaseOP): + def __init__(self, config: "ModelConfig"): + self.embeddings = VocabParallelEmbedding(config.vocab_size, config.hidden_size) + moe_banks = {layer: bank for bank, layer in enumerate(config.moe_layer_ids or ())} + self.layers = OPList([ + NemotronHBlock(config, layer, moe_banks) for layer in range(config.num_layers) + ]) + self.norm_f = RMSNorm(config.hidden_size, eps=config.rms_norm_eps) + + def forward(self, input_ids: torch.Tensor) -> torch.Tensor: + x = self.embeddings.forward(input_ids) + for layer in self.layers.op_list: + x = layer.forward(x) + return self.norm_f.forward(x) + + +class NemotronHForCausalLM(BaseLLMModel): + def __init__(self, config: "ModelConfig"): + self.backbone = NemotronHBackbone(config) + self.lm_head = ParallelLMHead( + config.vocab_size, config.hidden_size, + tie_word_embeddings=config.tie_word_embeddings, + tied_embedding=self.backbone.embeddings if config.tie_word_embeddings else None, + ) + super().__init__() + + def forward(self) -> torch.Tensor: + hidden = self.backbone.forward(get_global_ctx().batch.input_ids) + return self.lm_head.forward(hidden) + + +__all__ = ["NemotronHForCausalLM"] diff --git a/python/freetoken/models/nemotron_h/weight.py b/python/freetoken/models/nemotron_h/weight.py new file mode 100644 index 00000000..dc5d7ed6 --- /dev/null +++ b/python/freetoken/models/nemotron_h/weight.py @@ -0,0 +1,174 @@ +from __future__ import annotations + +import re +from typing import Iterator + +import safetensors +import torch + +from freetoken.distributed import get_tp_info +from freetoken.models.loader import drop_page_cache, iter_weight_files +from freetoken.models.nvfp4_banks import ( + Nvfp4ExpertSourceSpec, + load_nvfp4_expert_source_banks, + load_nvfp4_expert_source_banks_parallel, +) +from freetoken.utils import cached_load_hf_config +from tqdm import tqdm + +from .config import parse_config + + +_EXPERT_RE = re.compile(r"^backbone\.layers\.\d+\.mixer\.experts\.\d+\.") +_EXPERT_KEY_RE = re.compile( + r"^backbone\.layers\.(?P\d+)\.mixer\.experts\.(?P\d+)\." + r"(?Pup_proj|down_proj)\.(?Pweight|weight_scale|weight_scale_2)$" +) + + +def _layer_to_bank(layer: int, config) -> int | None: + try: + return (config.moe_layer_ids or ()).index(layer) + except ValueError: + return None + + +_SOURCE_SPEC = Nvfp4ExpertSourceSpec( + key_pattern=_EXPERT_KEY_RE, + proj_to_role={"up_proj": "up", "down_proj": "down"}, + layer_to_bank=_layer_to_bank, + desc="Nemotron-H NVFP4 experts", + gated=False, + hidden_size_attr="expert_hidden_size", +) + + +def _dequant_nvfp4(weight, scale, global_scale): + from freetoken.models.qwen3_5_moe.weight import _dequant_nvfp4_weight + + return _dequant_nvfp4_weight(weight, scale, global_scale) + + +def _skip(name: str) -> bool: + return ( + name.startswith("mtp.") + or name.endswith((".k_scale", ".v_scale", ".q_scale", ".prob_scale")) + ) + + +def iter_weights( + model_path: str, + device: torch.device, + *, + include_moe_experts: bool, + include_non_moe: bool, +) -> Iterator[tuple[str, torch.Tensor]]: + del include_moe_experts # routed experts always live in native NVFP4 offload banks + if not include_non_moe: + return + if get_tp_info().size > 1: + raise NotImplementedError("Nemotron-H currently supports TP=1 only") + + config = parse_config(cached_load_hf_config(model_path)) + qkv: dict[str, dict[str, torch.Tensor]] = {} + + for file in tqdm( + iter_weight_files(model_path), desc="Loading Nemotron-H weights", + disable=not get_tp_info().is_primary(), + ): + with safetensors.safe_open(file, framework="pt", device=str(device)) as f: + keys = set(f.keys()) + for name in f.keys(): + if _skip(name) or _EXPERT_RE.match(name): + continue + if name.endswith((".weight_scale", ".weight_scale_2", ".input_scale")): + continue + tensor = f.get_tensor(name) + if name.endswith(".weight"): + base = name[:-7] + scale_name = base + ".weight_scale" + global_name = base + ".weight_scale_2" + if global_name in keys: + tensor = _dequant_nvfp4( + tensor, f.get_tensor(scale_name), f.get_tensor(global_name) + ) + elif scale_name in keys: + # Keep checkpoint-native E4M3 and turn its scalar into the per-row + # vector consumed by FreeToken's W8A16/W8A8 linear. + yield name, tensor + scale = f.get_tensor(scale_name).reshape(1).float() + yield base + ".weight_scale", scale.expand(tensor.shape[0]).contiguous() + input_name = base + ".input_scale" + if input_name in keys: + yield input_name, f.get_tensor(input_name).reshape(()).float() + continue + + match = re.match( + r"^(backbone\.layers\.\d+\.mixer)\.(q_proj|k_proj|v_proj)\.weight$", + name, + ) + if match: + prefix, role = match.groups() + slots = qkv.setdefault(prefix, {}) + slots[role] = tensor + if len(slots) == 3: + yield prefix + ".qkv_proj.weight", torch.cat( + [slots[r] for r in ("q_proj", "k_proj", "v_proj")], dim=0 + ) + del qkv[prefix] + continue + yield name, tensor + drop_page_cache(file) + assert not qkv, f"incomplete Nemotron-H qkv fusions: {list(qkv)}" + + +def load_nvfp4_expert_sources(model_path: str, config, *, layer_sink=None): + return load_nvfp4_expert_source_banks( + model_path, config, _SOURCE_SPEC, drop_page_cache=drop_page_cache, + primary=get_tp_info().is_primary(), layer_sink=layer_sink, + ) + + +def load_nvfp4_expert_sources_parallel( + model_path: str, config, *, workers: int = 8, chunk: int = 8 << 20, layer_sink=None +): + return load_nvfp4_expert_source_banks_parallel( + model_path, config, _SOURCE_SPEC, drop_page_cache=drop_page_cache, + primary=get_tp_info().is_primary(), workers=workers, chunk=chunk, + layer_sink=layer_sink, + ) + + +def dummy_nvfp4_expert_sources(config): + from freetoken.kernel.pinned import alloc_pinned_tensor + + L, E = config.num_moe_layers, config.num_experts + H, I = config.expert_hidden_size, config.moe_intermediate_size + fp8 = torch.float8_e4m3fn + + def bank(*shape, dtype): + return [alloc_pinned_tensor(*shape, dtype=dtype) for _ in range(L)] + + sources = { + "gate_up_packed": bank(E, I, H // 2, dtype=torch.uint8), + "gate_up_scale": bank(E, I, H // 16, dtype=fp8), + "gate_up_global": bank(E, I, dtype=torch.float16), + "down_packed": bank(E, H, I // 2, dtype=torch.uint8), + "down_scale": bank(E, H, I // 16, dtype=fp8), + "down_global": bank(E, H, dtype=torch.float16), + } + for tensor in sources["gate_up_packed"] + sources["down_packed"]: + tensor.random_(0, 256) + for tensor in sources["gate_up_scale"] + sources["down_scale"]: + tensor.fill_(1.0) + for tensor in sources["gate_up_global"] + sources["down_global"]: + tensor.fill_(0.01) + return sources + + +__all__ = [ + "iter_weights", + "load_nvfp4_expert_sources", + "load_nvfp4_expert_sources_parallel", + "dummy_nvfp4_expert_sources", +] diff --git a/python/freetoken/models/nvfp4_banks.py b/python/freetoken/models/nvfp4_banks.py index 0e3ab6a5..66a28a56 100644 --- a/python/freetoken/models/nvfp4_banks.py +++ b/python/freetoken/models/nvfp4_banks.py @@ -22,6 +22,12 @@ class Nvfp4ExpertSourceSpec: proj_to_role: dict[str, str] layer_to_bank: LayerToBank desc: str + # Conventional MoEs concatenate gate|up (2I rows). Nemotron-H has one ungated + # up projection (I rows) followed by ReLU^2. + gated: bool = True + # Optional ModelConfig attribute holding the expert input/output width. The + # residual hidden_size remains unchanged for the rest of the model. + hidden_size_attr: str | None = None def _num_moe_layers(config) -> int: @@ -44,7 +50,7 @@ def _bank_layer(spec: Nvfp4ExpertSourceSpec, layer: int, config) -> int | None: return bank_layer -def _alloc_nvfp4_host_banks(num_layers: int, E: int, H: int, I: int): +def _alloc_nvfp4_host_banks(num_layers: int, E: int, H: int, I: int, *, gated: bool = True): """6 NVFP4 source banks, one ``[E, ...]`` tensor per layer (independent allocations), unpinned (pin-after-fill): register only after fill to skip cudaHostAlloc's slow commit. Caller fills each layer's ``.tensor`` then pins it (per-layer, via @@ -53,9 +59,9 @@ def _alloc_nvfp4_host_banks(num_layers: int, E: int, H: int, I: int): fp8 = torch.float8_e4m3fn return alloc_layer_banks({ - "gate_up_packed": ((E, 2 * I, H // 2), torch.uint8), - "gate_up_scale": ((E, 2 * I, H // 16), fp8), - "gate_up_global": ((E, 2 * I), torch.float16), + "gate_up_packed": ((E, (2 if gated else 1) * I, H // 2), torch.uint8), + "gate_up_scale": ((E, (2 if gated else 1) * I, H // 16), fp8), + "gate_up_global": ((E, (2 if gated else 1) * I), torch.float16), "down_packed": ((E, H, I // 2), torch.uint8), "down_scale": ((E, H, I // 16), fp8), "down_global": ((E, H), torch.float16), @@ -92,7 +98,7 @@ def load_nvfp4_expert_source_banks( weight_map = json.load(f)["weight_map"] E = config.num_experts - H = config.hidden_size + H = getattr(config, spec.hidden_size_attr) if spec.hidden_size_attr else config.hidden_size I = config.moe_intermediate_size num_layers = _num_moe_layers(config) @@ -133,7 +139,7 @@ def load_nvfp4_expert_source_banks( globals_map[key] = f.get_tensor(name).to(torch.float16) drop_page_cache(path) - _hb = _alloc_nvfp4_host_banks(num_layers, E, H, I) # unpinned; pinned after fill + _hb = _alloc_nvfp4_host_banks(num_layers, E, H, I, gated=spec.gated) # unpinned; pinned after fill gate_up_packed = [b.tensor for b in _hb["gate_up_packed"]] gate_up_scale = [b.tensor for b in _hb["gate_up_scale"]] gate_up_global = [b.tensor for b in _hb["gate_up_global"]] @@ -144,7 +150,8 @@ def load_nvfp4_expert_source_banks( from freetoken.moe.host_banks import LayerCompletionTracker, PinPipeline def _load(sink) -> int: - tracker = LayerCompletionTracker(E * 6, _hb, sink) + tensors_per_expert = 6 if spec.gated else 4 + tracker = LayerCompletionTracker(E * tensors_per_expert, _hb, sink) placed = 0 for shard in tqdm(sorted(weight_shards), desc=f"Loading {spec.desc}", disable=not primary): path = os.path.join(folder, shard) @@ -160,7 +167,10 @@ def _load(sink) -> int: if role == "gate": gate_up_packed[bank_layer_id][expert, :I] = tensor elif role == "up": - gate_up_packed[bank_layer_id][expert, I:] = tensor + if spec.gated: + gate_up_packed[bank_layer_id][expert, I:] = tensor + else: + gate_up_packed[bank_layer_id][expert] = tensor elif role == "down": down_packed[bank_layer_id][expert] = tensor else: @@ -171,8 +181,12 @@ def _load(sink) -> int: gate_up_scale[bank_layer_id][expert, :I] = tensor gate_up_global[bank_layer_id][expert, :I] = global_scale elif role == "up": - gate_up_scale[bank_layer_id][expert, I:] = tensor - gate_up_global[bank_layer_id][expert, I:] = global_scale + if spec.gated: + gate_up_scale[bank_layer_id][expert, I:] = tensor + gate_up_global[bank_layer_id][expert, I:] = global_scale + else: + gate_up_scale[bank_layer_id][expert] = tensor + gate_up_global[bank_layer_id][expert] = global_scale elif role == "down": down_scale[bank_layer_id][expert] = tensor down_global[bank_layer_id][expert] = global_scale @@ -189,7 +203,7 @@ def _load(sink) -> int: with PinPipeline() as pins: placed = _load(pins) - expected = num_layers * E * 6 + expected = num_layers * E * (6 if spec.gated else 4) assert placed == expected, f"{spec.desc}: loaded {placed} expert tensors, expected {expected}" return { "gate_up_packed": gate_up_packed, @@ -223,7 +237,7 @@ def load_nvfp4_expert_source_banks_parallel( weight_map = json.load(f)["weight_map"] E = config.num_experts - H = config.hidden_size + H = getattr(config, spec.hidden_size_attr) if spec.hidden_size_attr else config.hidden_size I = config.moe_intermediate_size num_layers = _num_moe_layers(config) @@ -257,7 +271,7 @@ def load_nvfp4_expert_source_banks_parallel( ) drop_page_cache(path) - _hb = _alloc_nvfp4_host_banks(num_layers, E, H, I) # unpinned; pinned after fill + _hb = _alloc_nvfp4_host_banks(num_layers, E, H, I, gated=spec.gated) # unpinned; pinned after fill gate_up_packed = [b.tensor for b in _hb["gate_up_packed"]] gate_up_scale = [b.tensor for b in _hb["gate_up_scale"]] gate_up_global = [b.tensor for b in _hb["gate_up_global"]] @@ -269,7 +283,8 @@ def load_nvfp4_expert_source_banks_parallel( # Pass 2: bulk weight/weight_scale via the common parallel reader; place by name. def _load(sink) -> int: - tracker = LayerCompletionTracker(E * 6, _hb, sink) + tensors_per_expert = 6 if spec.gated else 4 + tracker = LayerCompletionTracker(E * tensors_per_expert, _hb, sink) placed = 0 for name, tensor in iter_expert_tensors_parallel( folder, lambda n: n in weight_info, workers=workers, chunk=chunk @@ -284,7 +299,10 @@ def _load(sink) -> int: if role == "gate": gate_up_packed[bank_layer_id][expert, :I] = tensor elif role == "up": - gate_up_packed[bank_layer_id][expert, I:] = tensor + if spec.gated: + gate_up_packed[bank_layer_id][expert, I:] = tensor + else: + gate_up_packed[bank_layer_id][expert] = tensor else: down_packed[bank_layer_id][expert] = tensor else: @@ -293,8 +311,12 @@ def _load(sink) -> int: gate_up_scale[bank_layer_id][expert, :I] = tensor gate_up_global[bank_layer_id][expert, :I] = g elif role == "up": - gate_up_scale[bank_layer_id][expert, I:] = tensor - gate_up_global[bank_layer_id][expert, I:] = g + if spec.gated: + gate_up_scale[bank_layer_id][expert, I:] = tensor + gate_up_global[bank_layer_id][expert, I:] = g + else: + gate_up_scale[bank_layer_id][expert] = tensor + gate_up_global[bank_layer_id][expert] = g else: down_scale[bank_layer_id][expert] = tensor down_global[bank_layer_id][expert] = g @@ -308,7 +330,7 @@ def _load(sink) -> int: with PinPipeline() as pins: placed = _load(pins) - expected = num_layers * E * 6 + expected = num_layers * E * (6 if spec.gated else 4) assert placed == expected, f"{spec.desc}: loaded {placed} expert tensors, expected {expected}" return { "gate_up_packed": gate_up_packed, diff --git a/python/freetoken/models/qwen3_5_moe/__init__.py b/python/freetoken/models/qwen3_5_moe/__init__.py index 98936e9f..c2ccf237 100644 --- a/python/freetoken/models/qwen3_5_moe/__init__.py +++ b/python/freetoken/models/qwen3_5_moe/__init__.py @@ -1,4 +1,10 @@ from .config import parse_config +from .gguf import ( + dummy_gguf_expert_sources, + iter_gguf_weights, + load_gguf_expert_sources, + parse_gguf_config, +) from .model import Qwen3_5MoEForCausalLM from .weight import ( iter_weights, @@ -11,6 +17,10 @@ __all__ = [ "Qwen3_5MoEForCausalLM", "parse_config", + "parse_gguf_config", + "iter_gguf_weights", + "load_gguf_expert_sources", + "dummy_gguf_expert_sources", "iter_weights", "iter_weights_parallel", "load_nvfp4_expert_sources", diff --git a/python/freetoken/models/qwen3_5_moe/gdn.py b/python/freetoken/models/qwen3_5_moe/gdn.py index 2e732005..8e4eee21 100644 --- a/python/freetoken/models/qwen3_5_moe/gdn.py +++ b/python/freetoken/models/qwen3_5_moe/gdn.py @@ -218,7 +218,8 @@ def forward(self, hidden_states: torch.Tensor) -> torch.Tensor: core_out = core_out.reshape(-1, self.head_v_dim) z = z.reshape(-1, self.head_v_dim) out = self.norm.forward(core_out, z).reshape(total, -1) - return self.out_proj.forward(out) + out = self.out_proj.forward(out) + return out __all__ = ["Qwen3_5GatedDeltaNet"] diff --git a/python/freetoken/models/qwen3_5_moe/gguf.py b/python/freetoken/models/qwen3_5_moe/gguf.py new file mode 100644 index 00000000..94fb4ba3 --- /dev/null +++ b/python/freetoken/models/qwen3_5_moe/gguf.py @@ -0,0 +1,520 @@ +"""Native-GGUF adapter for Qwen3.5-MoE (GGUF architecture ``qwen35moe``). + +Ornith-1.5-35B-A3B uses the standard Qwen3.5 hybrid decoder: three Gated +DeltaNet layers followed by one full-attention layer, 256 top-8 routed experts, +and one gated shared expert. Dense matrices remain in their per-tensor GGUF +Q4_K/Q6_K representation; routed experts are streamed through the mixed-GGUF +offload cache. The final GGUF block is the optional MTP predictor and is not +part of autoregressive serving. +""" + +from __future__ import annotations + +from typing import TYPE_CHECKING + +import torch + +from freetoken.layers import BaseOP +from freetoken.models.config import ( + FullAttentionGroupConfig, + LinearGatedDeltaGroupConfig, + ModelConfig, + RotaryConfig, +) + +if TYPE_CHECKING: + from freetoken.models.gguf.config import GgufConfigShim + + +def _g(shim: "GgufConfigShim", key: str): + full = f"qwen35moe.{key}" + value = shim.metadata.get(full) + if value is None: + raise KeyError(f"missing GGUF metadata key {full}") + return value + + +def _main_layer_count(shim: "GgufConfigShim") -> int: + # llama.cpp includes MTP predictor blocks in block_count and records how many + # there are separately. Transformers' num_hidden_layers excludes them. + total = int(_g(shim, "block_count")) + mtp = int(shim.metadata.get("qwen35moe.nextn_predict_layers", 0)) + if mtp < 0 or mtp >= total: + raise ValueError(f"invalid qwen35moe nextn_predict_layers={mtp} for {total} blocks") + return total - mtp + + +def _expert_types(shim: "GgufConfigShim") -> tuple[tuple[int, int], ...] | None: + from freetoken.models.gguf.reader import gguf_tensor_names + + if not gguf_tensor_names(shim.model_path): + return None + types = gguf_tensor_types(shim.model_path) + out = [] + for layer in range(_main_layer_count(shim)): + gate = types[f"blk.{layer}.ffn_gate_exps.weight"] + up = types[f"blk.{layer}.ffn_up_exps.weight"] + down = types[f"blk.{layer}.ffn_down_exps.weight"] + if gate != up: + raise ValueError(f"blk.{layer}: gate/up expert banks have different GGUF types") + out.append((gate, down)) + return tuple(out) + + +def parse_gguf_config(shim: "GgufConfigShim") -> ModelConfig: + layers = _main_layer_count(shim) + hidden = int(_g(shim, "embedding_length")) + context = int(_g(shim, "context_length")) + num_q = int(_g(shim, "attention.head_count")) + num_kv = int(_g(shim, "attention.head_count_kv")) + head_dim = int(_g(shim, "attention.key_length")) + interval = int(_g(shim, "full_attention_interval")) + full_ids = tuple(i for i in range(layers) if (i + 1) % interval == 0) + linear_ids = tuple(i for i in range(layers) if i not in full_ids) + + rotary = RotaryConfig( + head_dim=head_dim, + rotary_dim=int(_g(shim, "rope.dimension_count")), + max_position=context, + base=float(_g(shim, "rope.freq_base")), + scaling=None, + ) + state_dim = int(_g(shim, "ssm.state_size")) + inner = int(_g(shim, "ssm.inner_size")) + value_heads = inner // state_dim + if inner % state_dim: + raise ValueError(f"qwen35moe ssm.inner_size {inner} is not divisible by {state_dim}") + + from freetoken.models.gguf.reader import gguf_tensor_type + + return ModelConfig( + num_layers=layers, + num_qo_heads=num_q, + num_kv_heads=num_kv, + head_dim=head_dim, + hidden_size=hidden, + vocab_size=int(shim.vocab_size), + intermediate_size=0, + hidden_act="silu", + rms_norm_eps=float(_g(shim, "attention.layer_norm_rms_epsilon")), + tie_word_embeddings=bool(shim.tie_word_embeddings), + rotary_config=rotary, + num_experts=int(_g(shim, "expert_count")), + num_experts_per_tok=int(_g(shim, "expert_used_count")), + moe_intermediate_size=int(_g(shim, "expert_feed_forward_length")), + shared_expert_intermediate_size=int(_g(shim, "expert_shared_feed_forward_length")), + norm_topk_prob=True, + moe_enabled=True, + use_qk_norm=True, + model_type="qwen3_5_moe", + architectures=list(shim.architectures), + attention_groups=( + LinearGatedDeltaGroupConfig( + name="linear", + layer_ids=linear_ids, + num_key_heads=int(_g(shim, "ssm.group_count")), + num_value_heads=value_heads, + key_head_dim=state_dim, + value_head_dim=state_dim, + conv_kernel_dim=int(_g(shim, "ssm.conv_kernel")), + output_gate=True, + ), + FullAttentionGroupConfig( + name="full", + layer_ids=full_ids, + num_kv_heads=num_kv, + head_dim=head_dim, + rotary_config=rotary, + ), + ), + expert_quant="gguf", + moe_weight_format="gguf", + gguf_embed_quant=gguf_tensor_type(shim.model_path, "token_embd.weight"), + gguf_expert_types=_expert_types(shim), + gguf_model_path=shim.model_path, + ) + + +def gguf_tensor_types(model_path: str) -> dict[str, int]: + from freetoken.models.gguf.reader import iter_gguf_tensors + + return {t.name: t.ggml_type for t in iter_gguf_tensors(model_path)} + + +def _dense(t, dtype=torch.bfloat16) -> torch.Tensor: + from freetoken.models.gguf.dequant import dequantize + + return dequantize(t.packed().reshape(-1), t.ggml_type, dtype).reshape(t.shape) + + +def _inverse_v_permutation(config: ModelConfig, *, head_dim: int) -> torch.Tensor: + """Undo llama.cpp's grouped->tiled V-head reorder. + + Qwen's HF/FreeToken GDN layout groups the two V heads belonging to each K + head. llama.cpp rewrites them to tiled order before GGUF quantization so its + broadcast can use ``ggml_repeat``. FreeToken's FLA kernels consume the + original grouped order, therefore every affected GGUF tensor must be mapped + back (see llama.cpp ``_LinearAttentionVReorderBase``). + """ + group = config.linear_attention_group() + assert group is not None + num_k, num_v = group.num_key_heads, group.num_value_heads + per_k = num_v // num_k + perm = torch.arange(num_v * head_dim).reshape(num_k, per_k, head_dim) + perm = perm.permute(1, 0, 2).reshape(-1) + return torch.argsort(perm) + + +def _undo_v_rows(tensor: torch.Tensor, config: ModelConfig, head_dim: int) -> torch.Tensor: + return tensor.index_select(0, _inverse_v_permutation(config, head_dim=head_dim)) + + +_EXPERT_SUFFIXES = ( + "ffn_gate_exps.weight", + "ffn_up_exps.weight", + "ffn_down_exps.weight", +) + + +def iter_gguf_weights( + model_path: str, + device, + *, + include_moe_experts: bool, + include_non_moe: bool, +): + """Yield the 40 serving layers while retaining every matrix in native GGUF form.""" + from freetoken.distributed import get_tp_info + from freetoken.models.gguf.reader import iter_gguf_tensors + from freetoken.utils import cached_load_hf_config + + if get_tp_info().size > 1: + raise NotImplementedError("qwen35moe GGUF currently supports TP=1 only") + assert not include_moe_experts, "qwen35moe GGUF experts require the offload backend" + assert include_non_moe + config = parse_gguf_config(cached_load_hf_config(model_path)) + shared_buf: dict[int, dict[str, tuple[torch.Tensor, int]]] = {} + + packed = { + "attn_q.weight": "self_attn.qkv_proj.q.qweight", + "attn_k.weight": "self_attn.qkv_proj.k.qweight", + "attn_v.weight": "self_attn.qkv_proj.v.qweight", + "attn_output.weight": "self_attn.o_proj.qweight", + "attn_qkv.weight": "linear_attn.in_proj.qkv.qweight", + "attn_gate.weight": "linear_attn.in_proj.z.qweight", + "ssm_beta.weight": "linear_attn.in_proj.b.qweight", + "ssm_alpha.weight": "linear_attn.in_proj.a.qweight", + "ffn_down_shexp.weight": "mlp.shared_expert.down_proj.qweight", + } + dense = { + "attn_norm.weight": "input_layernorm.weight", + "post_attention_norm.weight": "post_attention_layernorm.weight", + "attn_q_norm.weight": "self_attn.q_norm.weight", + "attn_k_norm.weight": "self_attn.k_norm.weight", + "ssm_norm.weight": "linear_attn.norm.weight", + "ffn_gate_inp.weight": "mlp.gate.weight", + } + + for t in iter_gguf_tensors(model_path): + name = t.name + if name == "token_embd.weight": + yield "model.embed_tokens.qweight", t.packed() + continue + if name == "output_norm.weight": + yield "model.norm.weight", _dense(t) + continue + if name == "output.weight": + yield "lm_head.qweight", t.packed() + continue + if not name.startswith("blk."): + raise ValueError(f"unmapped qwen35moe GGUF tensor: {name}") + layer = int(name.split(".")[1]) + if layer >= config.num_layers: + continue # MTP predictor block + suffix = name.split(".", 2)[2] + if suffix in _EXPERT_SUFFIXES: + continue + base = f"model.layers.{layer}" + if suffix in packed: + weight = t.packed() + group = config.linear_attention_group() + assert group is not None + if suffix == "attn_qkv.weight": + qk_rows = 2 * group.num_key_heads * group.key_head_dim + qk, value = weight[:qk_rows], weight[qk_rows:] + weight = torch.cat( + [qk, _undo_v_rows(value, config, group.value_head_dim)], dim=0 + ) + elif suffix == "attn_gate.weight": + weight = _undo_v_rows(weight, config, group.value_head_dim) + elif suffix in ("ssm_beta.weight", "ssm_alpha.weight"): + weight = _undo_v_rows(weight, config, 1) + yield f"{base}.{packed[suffix]}", weight + elif suffix in dense: + yield f"{base}.{dense[suffix]}", _dense(t) + elif suffix == "ffn_gate_inp_shexp.weight": + yield f"{base}.mlp.shared_expert_gate.weight", _dense(t).reshape(1, -1) + elif suffix == "ssm_conv1d.weight": + conv = _dense(t) + group = config.linear_attention_group() + assert group is not None + qk_rows = 2 * group.num_key_heads * group.key_head_dim + conv = torch.cat( + [conv[:qk_rows], _undo_v_rows(conv[qk_rows:], config, group.value_head_dim)], + dim=0, + ) + yield f"{base}.linear_attn.conv1d.weight", conv.unsqueeze(1) + elif suffix == "ssm_dt.bias": + value = _undo_v_rows(_dense(t, torch.float32).reshape(-1, 1), config, 1) + yield f"{base}.linear_attn.dt_bias", value.reshape(-1) + elif suffix == "ssm_a": + # llama.cpp stores the already transformed continuous-time A=-exp(A_log). + a = _dense(t, torch.float32) + if not bool((a < 0).all()): + raise ValueError(f"{name}: expected negative transformed SSM A values") + a = _undo_v_rows(a.reshape(-1, 1), config, 1).reshape(-1) + yield f"{base}.linear_attn.A_log", torch.log(-a) + elif suffix == "ssm_out.weight": + # llama.cpp permutes this matrix's input columns before quantizing. + # K-quants group adjacent input columns, so a lossless packed-byte + # permutation is impossible after quantization; dequantize this one + # projection and restore the original columns in bf16. + out = _dense(t) + group = config.linear_attention_group() + assert group is not None + inv = _inverse_v_permutation(config, head_dim=group.value_head_dim) + yield f"{base}.linear_attn.out_proj.weight", out.index_select(1, inv) + elif suffix in ("ffn_gate_shexp.weight", "ffn_up_shexp.weight"): + slot = "gate" if "gate" in suffix else "up" + shared_buf.setdefault(layer, {})[slot] = (t.packed(), t.ggml_type) + parts = shared_buf[layer] + if len(parts) == 2: + if parts["gate"][1] != parts["up"][1]: + raise ValueError(f"blk.{layer}: shared-expert gate/up GGUF types differ") + yield f"{base}.mlp.shared_expert.gate_up_proj.qweight", torch.cat( + [parts["gate"][0], parts["up"][0]], dim=0 + ) + del shared_buf[layer] + else: + raise ValueError(f"unmapped qwen35moe GGUF tensor: {name}") + assert not shared_buf, f"incomplete shared-expert gate/up groups: {sorted(shared_buf)}" + + +class DeferredGGUFLinear(BaseOP): + def __init__(self, in_features: int, out_features: int): + self.in_features = in_features + self.out_features = out_features + self._quant_type: int | None = None + self.qweight: torch.Tensor | None = None + + def materialize(self, quant_type: int) -> None: + from freetoken.models.gguf.dequant import row_bytes + + self._quant_type = quant_type + self.qweight = torch.empty( + self.out_features, row_bytes(self.in_features, quant_type), dtype=torch.uint8 + ) + + def forward(self, x: torch.Tensor) -> torch.Tensor: + from freetoken.layers.gguf import fused_mul_mat_gguf + + assert self.qweight is not None and self._quant_type is not None + return fused_mul_mat_gguf(x, self.qweight, self._quant_type) + + +class GGUFSplitLinear(BaseOP): + """One logical fused projection backed by independently quantized GGUF parts.""" + + def __init__(self, in_features: int, parts: tuple[tuple[str, int], ...]): + self._part_names = tuple(name for name, _ in parts) + for name, out_features in parts: + setattr(self, name, DeferredGGUFLinear(in_features, out_features)) + + def forward(self, x: torch.Tensor) -> torch.Tensor: + return torch.cat([getattr(self, name).forward(x) for name in self._part_names], dim=-1) + + +class GGUFOutputHead(DeferredGGUFLinear): + def forward(self, x: torch.Tensor) -> torch.Tensor: + from freetoken.core import get_global_ctx + + batch = get_global_ctx().batch + if batch.is_prefill: + x = x[batch.attn_metadata.get_last_indices(batch.size)].contiguous() + return super().forward(x) + + +def is_gguf_model(config: ModelConfig) -> bool: + return config.gguf_model_path is not None and config.expert_quant == "gguf" + + +def convert_qwen3_5_to_gguf(model, config: ModelConfig) -> None: + from freetoken.layers.gguf import GGUFEmbedding + + if not config.gguf_model_path or config.gguf_embed_quant is None: + raise NotImplementedError("qwen35moe FTW conversion is not yet supported") + types = gguf_tensor_types(config.gguf_model_path) + + def materialize(module: DeferredGGUFLinear, name: str) -> None: + module.materialize(types[name]) + + model.model.embed_tokens = GGUFEmbedding( + config.vocab_size, config.hidden_size, config.gguf_embed_quant + ) + for i, layer in enumerate(model.model.layers.op_list): + if layer._is_linear: + g = config.linear_attention_group() + assert g is not None + qkv = 2 * g.num_key_heads * g.key_head_dim + g.num_value_heads * g.value_head_dim + z = g.num_value_heads * g.value_head_dim + layer.linear_attn.in_proj = GGUFSplitLinear( + config.hidden_size, + (("qkv", qkv), ("z", z), ("b", g.num_value_heads), ("a", g.num_value_heads)), + ) + for part, suffix in ( + ("qkv", "attn_qkv.weight"), + ("z", "attn_gate.weight"), + ("b", "ssm_beta.weight"), + ("a", "ssm_alpha.weight"), + ): + materialize(getattr(layer.linear_attn.in_proj, part), f"blk.{i}.{suffix}") + # ssm_out was column-permuted before GGUF quantization. The loader + # dequantizes and restores it to the ordinary bf16 module. + else: + q_out = 2 * config.num_qo_heads * config.head_dim + kv_out = config.num_kv_heads * config.head_dim + layer.self_attn.qkv_proj = GGUFSplitLinear( + config.hidden_size, (("q", q_out), ("k", kv_out), ("v", kv_out)) + ) + for part, suffix in ( + ("q", "attn_q.weight"), + ("k", "attn_k.weight"), + ("v", "attn_v.weight"), + ): + materialize(getattr(layer.self_attn.qkv_proj, part), f"blk.{i}.{suffix}") + layer.self_attn.o_proj = DeferredGGUFLinear( + config.num_qo_heads * config.head_dim, config.hidden_size + ) + materialize(layer.self_attn.o_proj, f"blk.{i}.attn_output.weight") + + shared = layer.mlp.shared_expert + shared.gate_up_proj = DeferredGGUFLinear( + config.hidden_size, 2 * config.shared_expert_intermediate_size + ) + shared.down_proj = DeferredGGUFLinear( + config.shared_expert_intermediate_size, config.hidden_size + ) + materialize(shared.gate_up_proj, f"blk.{i}.ffn_gate_shexp.weight") + materialize(shared.down_proj, f"blk.{i}.ffn_down_shexp.weight") + + model.lm_head = GGUFOutputHead(config.hidden_size, config.vocab_size) + materialize(model.lm_head, "output.weight") + + +def _expert_bank_geometry(config: ModelConfig) -> tuple[int, int]: + from freetoken.models.gguf.dequant import row_bytes + + assert config.gguf_expert_types + h, inter = config.hidden_size, config.moe_intermediate_size + gu = max(2 * inter * row_bytes(h, t) for t, _ in config.gguf_expert_types) + down = max(h * row_bytes(inter, t) for _, t in config.gguf_expert_types) + align = lambda n: (n + 63) // 64 * 64 + return align(gu), align(down) + + +def load_gguf_expert_sources( + model_path: str, config: ModelConfig, *, layer_sink=None +) -> dict[str, list[torch.Tensor]]: + from freetoken.distributed import get_tp_info + from freetoken.models.gguf.dequant import row_bytes + from freetoken.models.gguf.reader import iter_gguf_tensors + from freetoken.moe.host_banks import LayerCompletionTracker, PinPipeline, alloc_layer_banks + + if get_tp_info().size > 1: + raise NotImplementedError("qwen35moe GGUF expert banks currently support TP=1 only") + types = config.gguf_expert_types + assert types and len(types) == config.num_layers + experts, hidden, inter = config.num_experts, config.hidden_size, config.moe_intermediate_size + gu_stride, down_stride = _expert_bank_geometry(config) + host = alloc_layer_banks( + { + "gate_up": ((experts, gu_stride), torch.uint8), + "down": ((experts, down_stride), torch.uint8), + }, + config.num_layers, + ) + banks = {name: [b.tensor for b in per_layer] for name, per_layer in host.items()} + seen_gu, seen_down = set(), set() + + def load(sink) -> None: + tracker = LayerCompletionTracker(2, host, sink) if sink is not None else None + gate_parts: dict[int, dict[str, torch.Tensor]] = {} + for t in iter_gguf_tensors(model_path): + if not t.name.startswith("blk.") or not t.name.endswith(_EXPERT_SUFFIXES): + continue + layer = int(t.name.split(".")[1]) + if layer >= config.num_layers: + continue + gu_type, down_type = types[layer] + if t.name.endswith("ffn_down_exps.weight"): + payload = hidden * row_bytes(inter, down_type) + banks["down"][layer][:, :payload].copy_(t.packed().reshape(experts, payload)) + seen_down.add(layer) + if tracker is not None: + tracker.note(layer) + continue + half = inter * row_bytes(hidden, gu_type) + slot = "gate" if "gate" in t.name else "up" + parts = gate_parts.setdefault(layer, {}) + parts[slot] = t.packed().reshape(experts, half) + if len(parts) == 2: + banks["gate_up"][layer][:, :half].copy_(parts["gate"]) + banks["gate_up"][layer][:, half : 2 * half].copy_(parts["up"]) + del gate_parts[layer] + seen_gu.add(layer) + if tracker is not None: + tracker.note(layer) + assert not gate_parts, f"incomplete qwen35moe expert gate/up layers: {sorted(gate_parts)}" + + if layer_sink is not None: + load(layer_sink) + elif torch.cuda.is_available(): + with PinPipeline() as pins: + load(pins) + else: + load(None) + wanted = set(range(config.num_layers)) + assert seen_gu == wanted and seen_down == wanted, ( + f"missing qwen35moe expert layers: gate_up {sorted(wanted - seen_gu)}, " + f"down {sorted(wanted - seen_down)}" + ) + return banks + + +def dummy_gguf_expert_sources(config: ModelConfig) -> dict[str, list[torch.Tensor]]: + from freetoken.moe.host_banks import alloc_layer_banks, pin_banks + + gu_stride, down_stride = _expert_bank_geometry(config) + host = alloc_layer_banks( + { + "gate_up": ((config.num_experts, gu_stride), torch.uint8), + "down": ((config.num_experts, down_stride), torch.uint8), + }, + config.num_layers, + ) + banks = {name: [b.tensor for b in per_layer] for name, per_layer in host.items()} + for tensor in banks["gate_up"] + banks["down"]: + tensor.random_(0, 256) + if torch.cuda.is_available(): + pin_banks(host) + return banks + + +__all__ = [ + "parse_gguf_config", + "iter_gguf_weights", + "convert_qwen3_5_to_gguf", + "is_gguf_model", + "load_gguf_expert_sources", + "dummy_gguf_expert_sources", +] diff --git a/python/freetoken/models/qwen3_5_moe/model.py b/python/freetoken/models/qwen3_5_moe/model.py index eba7fd24..29bd4928 100644 --- a/python/freetoken/models/qwen3_5_moe/model.py +++ b/python/freetoken/models/qwen3_5_moe/model.py @@ -108,6 +108,10 @@ def __init__(self, config: ModelConfig): tied_embedding=self.model.embed_tokens if config.tie_word_embeddings else None, ) super().__init__() + from .gguf import convert_qwen3_5_to_gguf, is_gguf_model + + if is_gguf_model(config): + convert_qwen3_5_to_gguf(self, config) def forward(self) -> torch.Tensor: output = self.model.forward(get_global_ctx().batch.input_ids) diff --git a/python/freetoken/models/qwen3_5_moe/moe.py b/python/freetoken/models/qwen3_5_moe/moe.py index fc0bb7c2..ca5b43f8 100644 --- a/python/freetoken/models/qwen3_5_moe/moe.py +++ b/python/freetoken/models/qwen3_5_moe/moe.py @@ -68,8 +68,22 @@ def __init__(self, config: ModelConfig, layer_id: int | None = None): weight_format = ( "fp8_block" if getattr(config, "expert_quant", "none") == "fp8_block" else "bf16" ) + extra_attrs = None + if config.gguf_expert_types is not None: + assert layer_id is not None + gu_t, dn_t = config.gguf_expert_types[layer_id] + extra_attrs = { + "gguf_gate_up_type": gu_t, + "gguf_down_type": dn_t, + "gguf_gate_up_rows": 2 * config.moe_intermediate_size, + "gguf_down_rows": config.hidden_size, + } self.experts = make_moe_layer( - config, layer_id=layer_id, renormalize=True, weight_format=weight_format + config, + layer_id=layer_id, + renormalize=True, + weight_format=weight_format, + extra_attrs=extra_attrs, ) self.gate = LinearReplicated(config.hidden_size, config.num_experts, has_bias=False) self.shared_expert = _SharedExpert( diff --git a/python/freetoken/models/register.py b/python/freetoken/models/register.py index 0c033ca0..8516ef88 100644 --- a/python/freetoken/models/register.py +++ b/python/freetoken/models/register.py @@ -58,6 +58,12 @@ class ModelSpec: "freetoken.models.qwen3_5_moe", "Qwen3_5MoEForCausalLM", ), + "Qwen3_5MoeGGUFForConditionalGeneration": ModelSpec( + "freetoken.models.qwen3_5_moe", + "Qwen3_5MoEForCausalLM", + parse_config="parse_gguf_config", + iter_weights="iter_gguf_weights", + ), # Dense Qwen3.x (no "Moe" in the arch name, num_experts==0, e.g. Qwen3.6-27B). Shares the # qwen3_5_moe package: the decoder routes its MLP through the dense Qwen3_5DenseMLP and the # loader handles the compressed-tensors NVFP4 layout. @@ -107,6 +113,16 @@ class ModelSpec: parse_config="parse_gguf_config", iter_weights="iter_gguf_weights", ), + "LagunaGGUFForCausalLM": ModelSpec( + "freetoken.models.laguna", + "LagunaForCausalLM", + parse_config="parse_gguf_config", + iter_weights="iter_gguf_weights", + ), + "LagunaForCausalLM": ModelSpec( + "freetoken.models.laguna", + "LagunaForCausalLM", + ), "GptOssForCausalLM": ModelSpec( "freetoken.models.gpt_oss", "GptOssForCausalLM", @@ -122,6 +138,10 @@ class ModelSpec: "freetoken.models.glm_moe_dsa", "GlmMoeDsaForCausalLM", ), + "NemotronHForCausalLM": ModelSpec( + "freetoken.models.nemotron_h", + "NemotronHForCausalLM", + ), } diff --git a/python/freetoken/models/weight.py b/python/freetoken/models/weight.py index 6a34f3b9..97eb65d9 100644 --- a/python/freetoken/models/weight.py +++ b/python/freetoken/models/weight.py @@ -342,6 +342,26 @@ def load_q4_0_moe_expert_sources( return loader(model_path, model_config, layer_sink=layer_sink) +def load_gguf_moe_expert_sources( + model_path: str, + model_config, + *, + dummy: bool = False, + layer_sink=None, +) -> dict: + """Load (or fabricate) mixed-type GGUF expert banks (flat padded uint8 slots). + + Per-model: dispatches to the model module's ``load_gguf_expert_sources`` / + ``dummy_gguf_expert_sources`` (laguna is the first user).""" + _config, spec = _spec_for_model_path(model_path) + if dummy: + builder = _model_override(spec, "dummy_gguf_expert_sources") + assert builder is not None, "model defines no dummy_gguf_expert_sources" + return builder(model_config) + loader = _load_attr(spec.module, "load_gguf_expert_sources") + return loader(model_path, model_config, layer_sink=layer_sink) + + def _num_moe_layers(config) -> int: value = getattr(config, "num_moe_layers", None) if value is not None: @@ -406,6 +426,7 @@ def bank(*shape: int, dtype: torch.dtype) -> list[torch.Tensor]: __all__ = [ "load_weight", "load_moe_expert_sources", + "load_gguf_moe_expert_sources", "load_nvfp4_moe_expert_sources", "dummy_moe_expert_sources", "dummy_nvfp4_expert_sources", diff --git a/python/freetoken/moe/cpu_executor.py b/python/freetoken/moe/cpu_executor.py index b96205aa..3fe3bed3 100644 --- a/python/freetoken/moe/cpu_executor.py +++ b/python/freetoken/moe/cpu_executor.py @@ -21,6 +21,7 @@ import threading import time import weakref +from types import SimpleNamespace import torch @@ -65,6 +66,7 @@ "gelu_pytorch_tanh": 2, "gpt_oss_swiglu": 3, "swigluoai": 3, + "relu2": 4, } # Weight-format ids must match WFmt in csrc/cpu_moe/cpu_moe_ext.cpp. @@ -156,6 +158,7 @@ def __init__( device: torch.device, swiglu_alpha: float = 1.702, swiglu_limit: float | None = None, + flag_sync: bool | None = None, ) -> None: from freetoken.kernel import _cpu_moe @@ -168,6 +171,8 @@ def __init__( ) if activation not in _ACT_IDS: raise NotImplementedError(f"CPU MoE backend: unsupported activation {activation!r}") + if activation == "relu2" and fmt != "nvfp4": + raise NotImplementedError("CPU ReLU^2 experts currently require native NVFP4 banks") # ABI probe: a stale prebuilt _cpu_moe.so accepts newer act ids without # error and silently computes the wrong activation in the generic # epilogue -- fail loudly with the rebuild instruction instead. (mxfp4 @@ -192,13 +197,16 @@ def __init__( # The per-layer tensors and their pointer tables must outlive the executor # (C++ holds raw addresses into both). self._banks: list[torch.Tensor] = [] - ptrs, (self.H, self.I) = self._resolve_banks(cache.bank_sources, fmt) + ptrs, (self.H, self.I) = self._resolve_banks( + cache.bank_sources, fmt, activation=activation + ) # Decide the flag handshake up front (env + device + a functional stream-memop # probe): its coordinator needs a core of its own, which the auto thread sizing # below reserves (a coordinator time-slicing against the GEMV workers measurably # destabilizes throughput on fully-subscribed boxes). - self._flag_sync = _FLAG_SYNC and device.type == "cuda" + use_flag_sync = _FLAG_SYNC if flag_sync is None else flag_sync + self._flag_sync = use_flag_sync and device.type == "cuda" self._cpu_moe = _cpu_moe # module ref for the decode-path memop calls if self._flag_sync: probe_scratch = alloc_pinned_tensor(1, dtype=torch.int64) @@ -332,7 +340,9 @@ def _make_table(self, layers: list[torch.Tensor]) -> torch.Tensor: self._banks.extend(layers) return table - def _resolve_banks(self, banks: dict, fmt: str) -> tuple[dict, tuple[int, int]]: + def _resolve_banks( + self, banks: dict, fmt: str, *, activation: str + ) -> tuple[dict, tuple[int, int]]: """Return (pointer kwargs for the C++ ctor, (H, I)) for the given format. ``banks[name]`` is a list of ``num_layers`` ``[num_experts, ...]`` tensors @@ -379,14 +389,15 @@ def _resolve_banks(self, banks: dict, fmt: str) -> tuple[dict, tuple[int, int]]: assert gup[0].dtype == torch.uint8 and dnp[0].dtype == torch.uint8, (gup[0].dtype, dnp[0].dtype) assert gus[0].element_size() == 1 and dns[0].element_size() == 1, "block scales must be 1 byte" assert gug[0].dtype == torch.float16 and dng[0].dtype == torch.float16, (gug[0].dtype, dng[0].dtype) - I = int(gup[0].shape[1] // 2) H = int(gup[0].shape[2] * 2) - assert gup[0].shape[1] == 2 * I + I = int(dnp[0].shape[2] * 2) + expected_up_rows = I if activation == "relu2" else 2 * I + assert gup[0].shape[1] == expected_up_rows assert H % 16 == 0 and I % 16 == 0, (H, I) assert tuple(dnp[0].shape[1:]) == (H, I // 2), (dnp[0].shape, H, I) - assert tuple(gus[0].shape[1:]) == (2 * I, H // 16), (gus[0].shape, I, H) + assert tuple(gus[0].shape[1:]) == (expected_up_rows, H // 16), (gus[0].shape, I, H) assert tuple(dns[0].shape[1:]) == (H, I // 16), (dns[0].shape, H, I) - assert tuple(gug[0].shape[1:]) == (2 * I,) and tuple(dng[0].shape[1:]) == (H,) + assert tuple(gug[0].shape[1:]) == (expected_up_rows,) and tuple(dng[0].shape[1:]) == (H,) ptrs = dict( gate_up_ptr=self._make_table(gup).data_ptr(), down_ptr=self._make_table(dnp).data_ptr(), @@ -655,6 +666,136 @@ def raise_if_unhealthy(self) -> None: ) +class MixedGgufCpuMoeExecutor: + """CPU dispatcher for Laguna's per-layer Q4_0/BF16 expert banks. + + The native extension has one weight format per executor. Build one executor + for each format over zero-copy views of the same host banks, then route each + layer to the matching pool. Placeholder pointer-table entries are never run. + """ + + def __init__( + self, + cache, + *, + top_k: int, + activation: str, + apply_router_weight_on_input: bool, + num_threads: int, + max_tokens: int, + device: torch.device, + swiglu_alpha: float = 1.702, + swiglu_limit: float | None = None, + ) -> None: + from freetoken.models.gguf.dequant import GGML_BF16, GGML_Q4_0, row_bytes + + types = getattr(cache, "gguf_expert_types", None) + if not types or len(types) != cache.num_layers: + raise ValueError("mixed GGUF CPU decode requires per-layer expert types") + if any(gu != dn or gu not in (GGML_Q4_0, GGML_BF16) for gu, dn in types): + raise NotImplementedError( + "mixed GGUF CPU decode currently supports only Laguna Q4_0/BF16 layers" + ) + + num_experts = cache.num_experts + hidden = int(cache.expert_hidden_size) + intermediate = int(cache.expert_intermediate_size) + raw_gu = cache.bank_sources["gate_up"] + raw_dn = cache.bank_sources["down"] + + q4_layers = [i for i, (gu, _) in enumerate(types) if gu == GGML_Q4_0] + bf16_layers = [i for i, (gu, _) in enumerate(types) if gu == GGML_BF16] + if not q4_layers or not bf16_layers: + raise ValueError("mixed GGUF executor requires both Q4_0 and BF16 layers") + + q4_gu = { + i: raw_gu[i].view( + num_experts, + 2 * intermediate, + row_bytes(hidden, GGML_Q4_0), + ) + for i in q4_layers + } + q4_dn = { + i: raw_dn[i].view( + num_experts, + hidden, + row_bytes(intermediate, GGML_Q4_0), + ) + for i in q4_layers + } + bf16_gu = { + i: raw_gu[i].view(torch.bfloat16).view( + num_experts, 2 * intermediate, hidden + ) + for i in bf16_layers + } + bf16_dn = { + i: raw_dn[i].view(torch.bfloat16).view( + num_experts, hidden, intermediate + ) + for i in bf16_layers + } + + def proxy(fmt: str, gate_up: dict, down: dict, reference: int): + return SimpleNamespace( + quant_format=fmt, + bank_sources={ + "gate_up": [ + gate_up.get(i, gate_up[reference]) + for i in range(cache.num_layers) + ], + "down": [ + down.get(i, down[reference]) + for i in range(cache.num_layers) + ], + }, + num_layers=cache.num_layers, + num_experts=num_experts, + ) + + common = dict( + top_k=top_k, + activation=activation, + apply_router_weight_on_input=apply_router_weight_on_input, + num_threads=num_threads, + max_tokens=max_tokens, + device=device, + swiglu_alpha=swiglu_alpha, + swiglu_limit=swiglu_limit, + # Two busy-poll coordinators would contend for one reserved core. The + # host-function path is graph-capturable and only used on CPU layers. + flag_sync=False, + ) + self._by_type = { + GGML_Q4_0: CpuMoeExecutor( + proxy("q4_0", q4_gu, q4_dn, q4_layers[0]), **common + ), + GGML_BF16: CpuMoeExecutor( + proxy("bf16", bf16_gu, bf16_dn, bf16_layers[0]), **common + ), + } + self._layer_types = tuple(gu for gu, _ in types) + + def _executor(self, layer_id: int) -> CpuMoeExecutor: + return self._by_type[self._layer_types[layer_id]] + + def decode(self, layer_id: int, *args): + return self._executor(layer_id).decode(layer_id, *args) + + def decode_submit(self, layer_id: int, *args) -> tuple: + executor = self._executor(layer_id) + return executor, executor.decode_submit(layer_id, *args) + + def decode_sync(self, pending: tuple) -> torch.Tensor: + executor, inner = pending + return executor.decode_sync(inner) + + def raise_if_unhealthy(self) -> None: + for executor in self._by_type.values(): + executor.raise_if_unhealthy() + + def _watchdog_main(executor_ref) -> None: """Watchdog daemon body: weakref-deref per tick so the thread never keeps a dead executor alive (see the start site in ``CpuMoeExecutor.__init__``).""" diff --git a/python/freetoken/moe/expert_banks.py b/python/freetoken/moe/expert_banks.py index 8b6116ba..ecb025fc 100644 --- a/python/freetoken/moe/expert_banks.py +++ b/python/freetoken/moe/expert_banks.py @@ -252,6 +252,70 @@ def _q4_0_banks(model_path, model_config, device, dtype, dummy, parallel=False, ) +def _gguf_banks(model_path, model_config, device, dtype, dummy, parallel=False, workers=8, chunk=_PARALLEL_CHUNK, decode_target="gpu", layer_sink=None) -> ExpertBanks: + if parallel: + raise NotImplementedError( + "parallel reader not implemented for gguf: single packed file (see q4_0)" + ) + if decode_target != "gpu": + raise NotImplementedError( + "mixed-type GGUF experts support the GPU offload backend only " + "(no CPU/hybrid executor format id)" + ) + from freetoken.models.weight import load_gguf_moe_expert_sources + + # Mixed-type GGUF routed experts (laguna): per-layer quant types, flat padded + # [E, stride] uint8 slots so every layer shares one bank shape (kernels read the + # leading payload via expert_stride_bytes). + sink = None if dummy else layer_sink + sources = load_gguf_moe_expert_sources(model_path, model_config, dummy=dummy, layer_sink=sink) + return ExpertBanks( + "gguf", {name: sources[name] for name in _BANK_SCHEMAS["gguf"]}, streamed=sink is not None + ) + + +def _laguna_int4_banks( + model_path, + model_config, + device, + dtype, + dummy, + parallel=False, + workers=8, + chunk=_PARALLEL_CHUNK, + decode_target="gpu", + layer_sink=None, +) -> ExpertBanks: + """Poolside compressed-tensors INT4 plus its intentionally BF16 tail layers. + + INT4 tensors are losslessly nibble-reordered into Q4_0 blocks (the bf16 group + scale is rounded to fp16); ignored BF16 expert layers remain BF16. The shared + ``gguf`` execution format understands both per-layer types and variable payloads. + """ + if parallel: + raise NotImplementedError("parallel expert reader is not implemented for Laguna INT4") + if decode_target not in ("gpu", "cpu"): + raise NotImplementedError( + "Laguna compressed INT4 experts support GPU offload and split CPU decode only" + ) + from freetoken.models.laguna.weight import ( + dummy_int4_expert_sources, + load_int4_expert_sources, + ) + + sink = None if dummy else layer_sink + sources = ( + dummy_int4_expert_sources(model_config) + if dummy + else load_int4_expert_sources(model_path, model_config, layer_sink=sink) + ) + return ExpertBanks( + "gguf", + {name: sources[name] for name in _BANK_SCHEMAS["gguf"]}, + streamed=sink is not None, + ) + + def _dsfp4_banks(model_path, model_config, device, dtype, dummy, parallel=False, workers=8, chunk=_PARALLEL_CHUNK, decode_target="gpu", layer_sink=None) -> ExpertBanks: args = model_config.dsv4_args assert args is not None, "ds_fp4 expert banks require dsv4_args on the model config" @@ -301,6 +365,8 @@ def _model_setup_override(model_config): "nvfp4": _nvfp4_banks, "ds_fp4": _dsfp4_banks, "q4_0": _q4_0_banks, + "gguf": _gguf_banks, + "laguna_int4": _laguna_int4_banks, } @@ -398,10 +464,35 @@ def bank_bytes_estimate(model_config) -> int | None: per_expert = _BANK_BYTES_PER_EXPERT.get(fmt) layers = getattr(model_config, "num_moe_layers", None) experts = getattr(model_config, "num_experts", None) - hidden = getattr(model_config, "hidden_size", None) + hidden = getattr(model_config, "expert_hidden_size", None) or getattr( + model_config, "hidden_size", None + ) inter = getattr(model_config, "moe_intermediate_size", None) + if fmt == "laguna_int4" and all((experts, hidden, inter)): + from freetoken.models.gguf.dequant import GGML_BF16, GGML_Q4_0, row_bytes + + types = getattr(model_config, "gguf_expert_types", None) + if types: + + def projection_bytes(qtype: int, rows: int, cols: int) -> int: + if qtype == GGML_Q4_0: + return rows * row_bytes(cols, qtype) + if qtype == GGML_BF16: + return rows * cols * 2 + raise ValueError(f"unsupported Laguna expert type {qtype}") + + return experts * sum( + projection_bytes(gu, 2 * inter, hidden) + + projection_bytes(dn, hidden, inter) + for gu, dn in types + ) if per_expert is None or not all((layers, experts, hidden, inter)): return None + if fmt == "nvfp4" and not getattr(model_config, "expert_gated", True): + # One up matrix (I x H), not gate|up (2I x H). + one_up = inter * (hidden // 2 + hidden // 16 + 2) + down = hidden * (inter // 2 + inter // 16 + 2) + return layers * experts * (one_up + down) return layers * experts * per_expert(hidden, inter) diff --git a/python/freetoken/moe/fused.py b/python/freetoken/moe/fused.py index fe7e417d..476ee35f 100644 --- a/python/freetoken/moe/fused.py +++ b/python/freetoken/moe/fused.py @@ -46,9 +46,24 @@ def fused_topk( from freetoken.kernel.backend import is_triton_kernels_installed - # triton_kernels ships no Windows wheel, and unlike flashinfer/sgl_kernel it is not one - # of the six ops the in-repo triton kernels cover -- so this router needs its own fallback. + # Renormalized routing has an in-tree one-program-per-token Triton kernel (originally + # added for GPT-OSS, but mathematically generic): softmax over the selected logits is + # exactly softmax-over-all -> top-k -> renormalize. Use it for Qwen/Ornith too instead + # of materializing the full probability matrix through several Torch launches. if not is_triton_kernels_installed(): + if ( + renormalize + and gating_output.is_cuda + and gating_output.shape[-1] <= 1024 + ): + from freetoken.kernel.moe_impl import gpt_oss_fused_routing + + topk_weights, topk_ids = gpt_oss_fused_routing(gating_output, topk) + if num_token_non_padded is not None: + indices = torch.arange(0, topk_ids.shape[0], device=topk_ids.device) + topk_ids[indices >= num_token_non_padded, :] = -1 + return topk_weights, topk_ids + global _warned_torch_topk if not _warned_torch_topk: _warned_torch_topk = True diff --git a/python/freetoken/moe/fused_gguf.py b/python/freetoken/moe/fused_gguf.py new file mode 100644 index 00000000..fa08762f --- /dev/null +++ b/python/freetoken/moe/fused_gguf.py @@ -0,0 +1,233 @@ +"""Grouped expert GEMM over mixed-type GGUF banks (borrowed ggml MoE kernels). + +The generalization of :mod:`freetoken.moe.fused_q4_0` for checkpoints whose +routed-expert quant type varies per layer (Unsloth Dynamic laguna: gate/up +IQ1_S or IQ2_XXS, down IQ3_XXS or IQ4_XS). Because per-expert byte sizes then +differ across layers, the banks are FLAT padded slots -- ``[num_slots, +stride_bytes]`` uint8 with each expert's real payload in the leading bytes -- +and the kernels read them via ``expert_stride_bytes``. Geometry (quant type, +output rows) rides in per-call arguments. Decode uses the low-latency MMVQ +kernel; sufficiently large prefills use the grouped MMQ kernel. +""" + +from __future__ import annotations + +import torch + +from freetoken.layers.activation import gelu_and_mul, gelu_tanh_and_mul, silu_and_mul +from freetoken.models.gguf.dequant import GGML_BF16 + +_ACT = {"silu": silu_and_mul, "gelu": gelu_and_mul, "gelu_tanh": gelu_tanh_and_mul} + +# moe_vec's CUDA grid puts (tokens * top_k) rows in grid.z, which CUDA caps at +# 65535. Large prefill chunks (e.g. 16384 tokens * top_8 = 131072) exceed that, +# so calls are split into row-count-bounded pieces. The down projection already +# runs at "top_k=1, tokens=num_tokens*top_k" (one row per selected expert), so +# both calls share one chunking helper keyed off total (rows, top_k) pairs. +_MAX_GRID_Z = 65535 + +# Transient memory bound: each call materializes [rows_in_flight, out_rows] plus a +# q8_1 copy of its activations. On a VRAM-tight offload setup (expert cache eats +# everything the KV pool leaves) a 16k-token prefill chunk at top_8 would allocate +# ~1 GiB in one shot and fault asynchronously, so cap rows well below the grid limit. +_MAX_ROWS_IN_FLIGHT = 16384 + +# MMQ has alignment/setup overhead and its donated kernel is not safe for every +# tiny routed batch. On the RTX 2000 Ada used for the Qwen3.5 GGUF bring-up it +# is already faster at 32 input tokens (Q4_K top-8: 0.61 ms vs 0.71 ms) and the +# advantage grows to 3.1x at 2K. Keep decode and short tails on MMVQ. +_MMQ_MIN_TOKENS = 32 +# On sm_120 (RTX 5080, Ornith blk.0 banks E=256 top-8) grouped MMQ already wins +# at 16 tokens (0.314 vs 0.324 ms) and pulls away from there (0.382 vs 0.475 at +# 24), so prefill tail chunks switch over earlier. +_MMQ_MIN_TOKENS_SM120 = 16 + + +def mmq_min_tokens(compute_capability: tuple[int, int] | None) -> int: + """Token count from which grouped MMQ beats chunked MMVQ for routed experts.""" + if compute_capability is not None and compute_capability >= (12, 0): + return _MMQ_MIN_TOKENS_SM120 + return _MMQ_MIN_TOKENS + + +def _moe_vec_chunked(x, weight, topk_ids, top_k, quant_type, rows, tokens, stride): + from freetoken.kernel.gguf import ggml_moe_a8_vec + + limit = min(_MAX_GRID_Z, _MAX_ROWS_IN_FLIGHT) + if tokens * top_k <= limit: + return ggml_moe_a8_vec(x, weight, topk_ids, top_k, quant_type, rows, tokens, stride) + + chunk = max(1, limit // top_k) + outs = [] + for start in range(0, tokens, chunk): + end = min(start + chunk, tokens) + outs.append( + ggml_moe_a8_vec( + x[start:end], weight, topk_ids[start:end], top_k, quant_type, rows, end - start, stride + ) + ) + return torch.cat(outs, dim=0) + + +# mm_ids_helper in the int8-MMA extension keeps one 4-byte record per token in +# shared memory; stay safely under the ~99KB sm_120 opt-in limit. +_MMA_MAX_TOKENS = 16384 + +# Below this the DP4A grouped MMQ wins: with E=256 top-8 the per-expert row +# count is tiny and mul_mat_q's per-expert MMA tiles waste work (sm_120, +# Ornith geometry: 256 tokens mma 1.64 vs dp4a 1.48 ms; 320 tokens 1.55 vs +# 1.76; 8192 tokens 9.1 vs 38.5 -- the full-prefill-chunk regime is the win). +_MMA_MOE_MIN_TOKENS = 320 + + +def _use_mma_moe(quant_type, stride, x, capability) -> bool: + """Upstream int8-MMA grouped MMQ: sm_120-gated, Q4_K/Q6_K slots only. + + The slot byte stride must be a multiple of the quant block size so the + kernel can address experts in whole blocks (true for Ornith's banks, where + payloads are already 64-byte aligned). + """ + if capability is None or capability < (12, 0): + return False + from freetoken.kernel.gguf import mma_mmq_supported + from freetoken.models.gguf.dequant import BLOCK_SHAPE + + if not mma_mmq_supported(int(quant_type)): + return False + if stride % BLOCK_SHAPE[int(quant_type)][1] != 0: + return False + from freetoken.layers.gguf import _mma_mmq_ok + + return _mma_mmq_ok() + + +def _moe_matmul(x, weight, topk_ids, top_k, quant_type, rows, tokens, stride, *, broadcast=True): + """Choose grouped MMQ for prefill and MMVQ for decode/small tails. + + ``broadcast=True``: ``x[tokens, in]`` shared by each token's top_k experts + (gate/up). ``broadcast=False``: ``x[tokens*top_k, in]`` with row + ``t*top_k + k`` belonging to ``topk_ids[t][k]`` (down). + + ``ggml_moe_a8`` reads experts using ``weight.stride(0)``. A mixed-GGUF + bank is uint8 ``[experts, padded_slot_bytes]``, so that stride is exactly + the byte stride expected by the donated kernel even though the real packed + payload occupies only the beginning of each slot. + """ + from freetoken.layers.gguf import _device_capability + + capability = _device_capability(x.device.index) if x.is_cuda else None + if ( + _MMA_MOE_MIN_TOKENS <= tokens <= _MMA_MAX_TOKENS + and _use_mma_moe(quant_type, stride, x, capability) + ): + from freetoken.kernel.gguf import ggml_moe_a8_mma + + out = ggml_moe_a8_mma( + x, weight, topk_ids.contiguous(), top_k, int(quant_type), + rows, tokens, stride, broadcast, + ) + return out.to(x.dtype) + if not broadcast: + # The DP4A/vec kernels take per-slot rows as a flat top_k=1 call. + topk_ids = topk_ids.reshape(-1, 1) + tokens = tokens * top_k + top_k = 1 + if tokens >= mmq_min_tokens(capability): + from freetoken.kernel.gguf import ggml_moe_a8, ggml_moe_get_block_size + from freetoken.moe.fused import moe_align_block_size + + block_size = ggml_moe_get_block_size(int(quant_type)) + if block_size: + sorted_ids, expert_ids, num_tokens_post_padded = moe_align_block_size( + topk_ids, block_size, weight.shape[0] + ) + return ggml_moe_a8( + x, + weight, + sorted_ids, + expert_ids, + num_tokens_post_padded, + int(quant_type), + rows, + top_k, + tokens, + ) + return _moe_vec_chunked( + x, weight, topk_ids, top_k, quant_type, rows, tokens, stride + ) + + +def fused_experts_gguf( + hidden_states: torch.Tensor, + gate_up_q: torch.Tensor, # [num_slots, gu_stride] uint8 (flat padded slots) + down_q: torch.Tensor, # [num_slots, dn_stride] uint8 + topk_weights: torch.Tensor, + topk_ids: torch.Tensor, + activation: str, + *, + gate_up_type: int, + down_type: int, + gate_up_rows: int, # 2 * intermediate + down_rows: int, # hidden +) -> torch.Tensor: + act_fn = _ACT.get(activation) + if act_fn is None: + raise ValueError(f"unsupported MoE activation {activation!r}") + + num_tokens = hidden_states.shape[0] + top_k = topk_ids.shape[1] + assert gate_up_q.dim() == 2 and down_q.dim() == 2, "gguf banks are flat padded slots" + + # Safetensors Laguna-S keeps its last expert layers in BF16. Variable-size + # cache rows place the real payload at the start of each padded slot, so expose + # that prefix as ordinary dense expert tensors and reuse the native BF16 MoE. + if gate_up_type == down_type == GGML_BF16: + from freetoken.moe.fused import fused_experts_impl + + hidden = hidden_states.shape[-1] + intermediate = gate_up_rows // 2 + gu_elems = gate_up_rows * hidden + dn_elems = down_rows * intermediate + # The leading payload is contiguous within each slot, while the slot-to-slot + # stride includes padding for the largest layer. ``view`` preserves that outer + # stride, giving the dense kernel an exact zero-copy 3-D view. + gate_up = gate_up_q[:, : gu_elems * 2].view(torch.bfloat16).view( + gate_up_q.shape[0], gate_up_rows, hidden + ) + down = down_q[:, : dn_elems * 2].view(torch.bfloat16).view( + down_q.shape[0], down_rows, intermediate + ) + return fused_experts_impl( + # fused_experts_impl writes its input in place. Laguna evaluates the + # shared expert afterwards from the original hidden states, so preserve + # that input just as the quantized path does. + hidden_states.clone(), + gate_up, + down, + topk_weights, + topk_ids, + activation, + False, + ) + if gate_up_type == GGML_BF16 or down_type == GGML_BF16: + raise ValueError("mixed BF16/quantized projections within one expert layer are unsupported") + + gate_up = _moe_matmul( + hidden_states, gate_up_q, topk_ids, top_k, int(gate_up_type), + gate_up_rows, num_tokens, gate_up_q.shape[1], + ) + inter = act_fn(gate_up) + # Down pass: one selected-expert row per (token, k). _moe_matmul flattens to + # a top_k=1 call for the DP4A/vec kernels (row-major [num_tokens, top_k] -> + # contiguous [num_tokens*top_k, 1]); the MMA path keeps the 2-D ids. + out = _moe_matmul( + inter, down_q, topk_ids, top_k, int(down_type), + down_rows, num_tokens, down_q.shape[1], broadcast=False, + ) + out = out.reshape(num_tokens, top_k, down_rows) * topk_weights.reshape( + num_tokens, top_k, 1 + ).to(out.dtype) + return out.sum(dim=1) + + +__all__ = ["fused_experts_gguf"] diff --git a/python/freetoken/moe/fused_nvfp4.py b/python/freetoken/moe/fused_nvfp4.py index 5ed8d766..f7cafc01 100644 --- a/python/freetoken/moe/fused_nvfp4.py +++ b/python/freetoken/moe/fused_nvfp4.py @@ -43,6 +43,9 @@ def _run_act( gpt-oss swiglu over the banks' uninterleaved [gate; up] halves) carries the per-model ``act_alpha``/``act_limit`` scalars; the plain *_and_mul kinds ignore them.""" + if activation == "relu2": + torch.square(torch.relu(gate_up), out=out) + return if activation == "swigluoai": swigluoai_and_mul(gate_up, out, alpha=act_alpha, limit=act_limit) return @@ -173,7 +176,7 @@ def _fused_experts_decode_nvfp4( M, H = hidden_states.shape top_k = topk_ids.shape[1] two_i = gate_up_packed.shape[1] - inter = two_i // 2 + inter = two_i if activation == "relu2" else two_i // 2 dev, dt = hidden_states.device, hidden_states.dtype ic1 = torch.empty((M, top_k, two_i), device=dev, dtype=dt) @@ -317,7 +320,7 @@ def fused_experts_nvfp4( M, H = hidden_states.shape top_k = topk_ids.shape[1] two_i = gate_up_packed.shape[1] - inter = two_i // 2 + inter = two_i if activation == "relu2" else two_i // 2 dev, dt = hidden_states.device, hidden_states.dtype cfg = _prefill_config(M) diff --git a/python/freetoken/moe/offload_cache.py b/python/freetoken/moe/offload_cache.py index 6ee76406..7ea24bc3 100644 --- a/python/freetoken/moe/offload_cache.py +++ b/python/freetoken/moe/offload_cache.py @@ -45,6 +45,9 @@ # native GGUF Q4_0 experts: packed block bytes per output row, dequantized inside # the borrowed ggml MoE kernels. gate_up [L*E, 2I, H//32*18], down [L*E, H, I//32*18]. "q4_0": ("gate_up", "down"), + # Mixed-type GGUF (laguna): flat padded uint8 slots [E, stride_bytes]; the + # per-layer quant geometry lives on the MoE layer, not the bank shape. + "gguf": ("gate_up", "down"), # native ModelOpt rows for the Triton inline-dequant kernels: packed e2m1 codes + # fp8-e4m3 per-16 block scales + per-output-row fp16 globals (w1/w3 carry distinct # globals, and folding them into the e4m3 block scales would underflow) @@ -86,6 +89,9 @@ "nvfp4": lambda H, I: 2 * I * (H // 2 + H // 16 + 2) + H * (I // 2 + I // 16 + 2), "mxfp4": lambda H, I: 2 * I * (H // 2 + H // 32 + 2) + H * (I // 2 + I // 32 + 2), "ds_fp4": lambda H, I: 2 * I * (H // 2 + H // 32) + H * (I // 2 + I // 32), + # Upper bound for the mixed Poolside artifact. Its early layers are groupwise + # INT4, while ignored tail layers are BF16 and therefore determine slot-cache size. + "laguna_int4": lambda H, I: 3 * I * H * 2, } # vLLM's marlin grouped-GEMM hands the full [cache_size] slot cache as its expert @@ -147,6 +153,10 @@ def __post_init__(self) -> None: # offload/PCIe path. Set by the engine after construction (empty = all-GPU, # all layers = the plain --moe-backend cpu case). self.cpu_layer_ids: frozenset = frozenset() + # Allow non-pinned layers to keep GPU expert compute by gathering each + # decode step's misses through a bounded pinned host/device staging pair. + # The engine enables this before set_bank_sources and disables CUDA graphs. + self.pageable_gpu = False # num_experts floor + nvfp4_marlin slot cap, shared with the runtime-rebuild path. self.validate_rebuild(self.cache_size) assert not self.prefill_overlap or self.cache_size >= 2 * self.num_experts, ( @@ -246,12 +256,29 @@ def __post_init__(self) -> None: self._copy_dst_ptrs: torch.Tensor | None = None self._copy_src_ptrs: list[torch.Tensor] | None = None self._copy_feat_bytes: torch.Tensor | None = None + self._copy_feat_bytes_by_layer: list[torch.Tensor] | None = None + self._copy_dst_strides: torch.Tensor | None = None + self._copy_src_strides: list[torch.Tensor] | None = None + self._variable_bank_rows: set[str] = set() + self._bank_cache_shapes: dict[str, tuple[int, ...]] = {} # The layer whose misses ensure_experts/materialize_layer staged last; consumed # by copy_missing to pick the per-layer source (part of the same pending-copy # state as evict_slots/src_indices/num_indices). # _pending_whole_layer records WHICH staged it: the pageable branch is only sound after materialize_layer self._pending_src_layer: int | None = None self._pending_whole_layer = False + # Lazily allocated by _copy_missing_pageable. Capacity follows the largest + # miss set observed (decode is normally batch=1/top-k sized), not E or the + # 59+ GiB source banks. Metadata buffers are pinned so one D2H fence obtains + # the device-produced LRU plan before the CPU gather. + self._pageable_stage_capacity = 0 + self._pageable_host_staging: list[torch.Tensor] = [] + self._pageable_device_staging: list[torch.Tensor] = [] + self._pageable_stage_src_indices: torch.Tensor | None = None + self._pageable_stage_src_ptrs: torch.Tensor | None = None + self._pageable_num_host: torch.Tensor | None = None + self._pageable_src_host: torch.Tensor | None = None + self._pageable_dst_host: torch.Tensor | None = None # Per-bank [2, num_experts, ...] double-buffer views over the slot cache's # first 2 * num_experts slots (set up when prefill_overlap is enabled). self.prefill_bank_buffers: list[torch.Tensor] = [] @@ -291,7 +318,10 @@ def set_bank_sources( -- the cache machinery is layout-agnostic and just moves rows. ``layer_residency`` labels each layer with a ``HostResidency`` value (default: all pinned). - Non-pinned (LOCKED/PAGEABLE) layers have no device address: they must already be routed to the CPU executor (``cpu_layer_ids``, set BEFORE this call), the copy plan skips their rows, and their only movement is ``copy_missing``'s whole-layer pageable prefill branch -- which is why prefill overlap is incompatible with them. + Non-pinned (LOCKED/PAGEABLE) layers have no device address. Normally they + must be routed to ``cpu_layer_ids``; with ``pageable_gpu`` they instead use + bounded decode staging and still compute on GPU. Prefill overlap remains + incompatible because it requires direct registered source addresses. """ from freetoken.moe.host_banks import HostResidency @@ -305,7 +335,7 @@ def set_bank_sources( i for i, r in enumerate(residency) if r != HostResidency.PINNED.value ) if unpinned: - if not unpinned <= self.cpu_layer_ids: + if not self.pageable_gpu and not unpinned <= self.cpu_layer_ids: raise ValueError( f"non-pinned layers {sorted(unpinned - self.cpu_layer_ids)} are not in " f"cpu_layer_ids: a layer without a device address can only decode on " @@ -318,20 +348,35 @@ def set_bank_sources( ) self._unpinned_layers = unpinned self.layer_residency = list(residency) + self._variable_bank_rows.clear() + self._bank_cache_shapes.clear() + self.bank_sources.clear() + self.bank_caches.clear() for name in self.bank_schema: per_layer = sources[name] assert len(per_layer) == self.num_layers, (name, len(per_layer)) head = per_layer[0] + dtype = head.dtype + row_numels = [] for layer_id, source in enumerate(per_layer): assert source.is_contiguous(), f"bank {name!r} layer {layer_id} must be contiguous" assert source.size(0) == self.num_experts, (name, layer_id, source.shape) - assert source.shape == head.shape and source.dtype == head.dtype, ( - name, layer_id, source.shape, source.dtype, - ) + assert source.dtype == dtype, (name, layer_id, source.dtype, dtype) + row_numels.append(math.prod(source.shape[1:])) self.bank_sources[name] = list(per_layer) + if len(set(row_numels)) == 1: + cache_tail = tuple(head.shape[1:]) + else: + # Mixed-precision layers have different compact payload sizes. The GPU + # slot uses the largest flat row; each source occupies only its prefix. + self._variable_bank_rows.add(name) + cache_tail = (max(row_numels),) + if dtype is not torch.uint8: + raise ValueError("variable-size expert rows must use flat uint8 storage") + self._bank_cache_shapes[name] = cache_tail self.bank_caches[name] = torch.empty( - (self.cache_size, *head.shape[1:]), - dtype=head.dtype, + (self.cache_size, *cache_tail), + dtype=dtype, device=self.device, ) self.banks = [(self.bank_sources[n], self.bank_caches[n]) for n in self.bank_schema] @@ -351,50 +396,73 @@ def _build_copy_plan(self) -> None: self._copy_dst_ptrs = None self._copy_src_ptrs = None self._copy_feat_bytes = None + self._copy_feat_bytes_by_layer = None + self._copy_dst_strides = None + self._copy_src_strides = None self._copy_dst_ptrs_host: list[int] = [] self._copy_src_ptrs_host: list[list[int]] = [] self._copy_feat_bytes_host: list[int] = [] + self._copy_feat_bytes_by_layer_host: list[list[int]] = [] + self._copy_dst_strides_host: list[int] = [] + self._copy_src_strides_host: list[list[int]] = [] self._gather_bank_ids: list[int] = [] self._gather_dst_ptrs: torch.Tensor | None = None self._gather_feat_bytes: torch.Tensor | None = None - if not _FUSED_COPY or self.device.type != "cuda" or not self.banks: + if (not _FUSED_COPY and not self._variable_bank_rows) or self.device.type != "cuda" or not self.banks: return from freetoken.kernel.pinned import device_ptr - dst_ptrs, feats = [], [] + dst_ptrs, dst_strides = [], [] layer_src_ptrs = [[] for _ in range(self.num_layers)] + layer_src_strides = [[] for _ in range(self.num_layers)] + layer_feats = [[] for _ in range(self.num_layers)] for per_layer, cache in self.banks: - feat = math.prod(per_layer[0].shape[1:]) * per_layer[0].element_size() - if feat % 16 != 0 or cache.data_ptr() % 16 != 0: + dst_stride = math.prod(cache.shape[1:]) * cache.element_size() + if dst_stride % 16 != 0 or cache.data_ptr() % 16 != 0: return # leave fused disabled; copy_missing uses the per-bank path for layer_id, source in enumerate(per_layer): + src_stride = math.prod(source.shape[1:]) * source.element_size() + if src_stride % 16 != 0: + return if layer_id in self._unpinned_layers: # unregistered layer: no device alias exists, and the row is never consumed (CPU decode; pageable prefill) # a 0 placeholder keeps the descriptor shape layer_src_ptrs[layer_id].append(0) - continue - # The kernel dereferences these on the GPU, so store each host bank's - # device alias (== data_ptr() under UVA identity; differs on - # Windows/WDDM). - src_dev = device_ptr(source) - if src_dev % 16 != 0: - return - layer_src_ptrs[layer_id].append(src_dev) + else: + # The kernel dereferences these on the GPU, so store each host bank's + # device alias (== data_ptr() under UVA identity; differs on WDDM). + src_dev = device_ptr(source) + if src_dev % 16 != 0: + return + layer_src_ptrs[layer_id].append(src_dev) + layer_src_strides[layer_id].append(src_stride) + layer_feats[layer_id].append(src_stride) dst_ptrs.append(cache.data_ptr()) - feats.append(feat) + dst_strides.append(dst_stride) self._copy_dst_ptrs = torch.tensor(dst_ptrs, dtype=torch.int64, device=self.device) self._copy_src_ptrs = [ torch.tensor(ptrs, dtype=torch.int64, device=self.device) for ptrs in layer_src_ptrs ] - self._copy_feat_bytes = torch.tensor(feats, dtype=torch.int64, device=self.device) + self._copy_src_strides = [ + torch.tensor(v, dtype=torch.int64, device=self.device) for v in layer_src_strides + ] + self._copy_feat_bytes_by_layer = [ + torch.tensor(v, dtype=torch.int64, device=self.device) for v in layer_feats + ] + self._copy_dst_strides = torch.tensor(dst_strides, dtype=torch.int64, device=self.device) + # Full cache-row sizes are used only for cache-to-cache hit gathers. + self._copy_feat_bytes = self._copy_dst_strides.clone() self._copy_dst_ptrs_host = dst_ptrs self._copy_src_ptrs_host = layer_src_ptrs - self._copy_feat_bytes_host = feats + self._copy_feat_bytes_host = dst_strides + self._copy_feat_bytes_by_layer_host = layer_feats + self._copy_dst_strides_host = dst_strides + self._copy_src_strides_host = layer_src_strides # hit-D2D gather serves only the big banks; small banks are whole-layer # H2D entries (see _SMALL_BANK_FEAT_BYTES), so their rows never need D2D. - self._gather_bank_ids = [i for i, f in enumerate(feats) if f >= _SMALL_BANK_FEAT_BYTES] - if len(self._gather_bank_ids) == len(feats): + self._gather_bank_ids = [i for i, f in enumerate(dst_strides) if f >= _SMALL_BANK_FEAT_BYTES] + if len(self._gather_bank_ids) == len(dst_strides): self._gather_dst_ptrs = self._copy_dst_ptrs self._gather_feat_bytes = self._copy_feat_bytes elif self._gather_bank_ids: @@ -450,7 +518,7 @@ def rebuild(self, cache_size: int) -> None: for name in self.bank_schema: head = self.bank_sources[name][0] self.bank_caches[name] = torch.empty( - (cache_size, *head.shape[1:]), dtype=head.dtype, device=self.device + (cache_size, *self._bank_cache_shapes[name]), dtype=head.dtype, device=self.device ) self.banks = [(self.bank_sources[n], self.bank_caches[n]) for n in self.bank_schema] self._build_copy_plan() # slot caches were reallocated -> refresh fused-copy addrs @@ -640,8 +708,15 @@ def prefetch_prefill_layer(self, layer_id: int) -> None: def copy() -> None: self._invalidate_prefill_buffer(buffer_id) - for (per_layer, _), buffer in zip(self.banks, self.prefill_bank_buffers): - buffer[buffer_id].copy_(per_layer[layer_id], non_blocking=True) + for name, (per_layer, _), buffer in zip( + self.bank_schema, self.banks, self.prefill_bank_buffers + ): + src = per_layer[layer_id] + dst = buffer[buffer_id] + if name in self._variable_bank_rows: + dst[:, : src.shape[1]].copy_(src, non_blocking=True) + else: + dst.copy_(src, non_blocking=True) if self._prefill_hit_d2d_active: self._prefetch_split(layer_id, buffer_id) @@ -751,18 +826,34 @@ def _prefetch_split(self, layer_id: int, buffer_id: int) -> None: starts = miss[run_starts] lengths = np.diff(np.concatenate((run_starts, [miss.size]))) dst, src, nbytes = [], [], [] - for b, feat in enumerate(self._copy_feat_bytes_host): - if feat < _SMALL_BANK_FEAT_BYTES: + layer_feats = self._copy_feat_bytes_by_layer_host[layer_id] + src_strides = self._copy_src_strides_host[layer_id] + for b, feat in enumerate(layer_feats): + dst_stride = self._copy_dst_strides_host[b] + src_stride = src_strides[b] + if feat < _SMALL_BANK_FEAT_BYTES and feat == dst_stride == src_stride: # Whole layer as one entry, EVEN with zero misses: it keeps every # batch entry above the driver's async floor and covers the hit # rows the gather skips for these banks. - dst.append(self._copy_dst_ptrs_host[b] + buffer_id * E * feat) + dst.append(self._copy_dst_ptrs_host[b] + buffer_id * E * dst_stride) src.append(self._copy_src_ptrs_host[layer_id][b]) nbytes.append(E * feat) elif miss.size: - dst.extend(self._copy_dst_ptrs_host[b] + (buffer_id * E + starts) * feat) - src.extend(self._copy_src_ptrs_host[layer_id][b] + starts * feat) - nbytes.extend(lengths * feat) + if feat == dst_stride == src_stride: + dst.extend(self._copy_dst_ptrs_host[b] + (buffer_id * E + starts) * dst_stride) + src.extend(self._copy_src_ptrs_host[layer_id][b] + starts * src_stride) + nbytes.extend(lengths * feat) + else: + for expert in miss: + dst.append( + self._copy_dst_ptrs_host[b] + + (buffer_id * E + int(expert)) * dst_stride + ) + src.append( + self._copy_src_ptrs_host[layer_id][b] + + int(expert) * src_stride + ) + nbytes.append(feat) if dst: self._batch_memcpy( torch.tensor(dst, dtype=torch.int64), @@ -965,12 +1056,133 @@ def decode_routing_stats(self) -> dict: "norm_entropy": norm_ent, } + def _ensure_pageable_stage(self, capacity: int, layer_id: int) -> None: + """Allocate bounded pinned+device rows for pageable GPU miss staging.""" + if capacity <= self._pageable_stage_capacity: + return + # Geometric growth avoids reallocating when consecutive decode steps have + # slightly different miss counts. Never size this from num_experts: Nemotron's + # six-bank row is ~3 MiB, while bs=1 needs at most top-k rows. + capacity = 1 << max(0, (capacity - 1).bit_length()) + host, device = [], [] + for per_layer, _cache in self.banks: + source = per_layer[layer_id] + host.append( + torch.empty( + (capacity, *source.shape[1:]), + dtype=source.dtype, + device="cpu", + pin_memory=True, + ) + ) + device.append( + torch.empty( + (capacity, *source.shape[1:]), + dtype=source.dtype, + device=self.device, + ) + ) + self._pageable_host_staging = host + self._pageable_device_staging = device + # The fused copy binding requires src/dst index tensors to have identical + # static lengths even though num_indices gates the active prefix. + # Keep the index vector the same static length as evict_slots. ``capacity`` + # is rounded up geometrically and may exceed that length near a non-power-of-two + # cache ceiling; num_indices still limits reads to the live staged rows. + self._pageable_stage_src_indices = torch.arange( + self.evict_slots.numel(), dtype=torch.int32, device=self.device + ) + self._pageable_stage_src_ptrs = torch.tensor( + [x.data_ptr() for x in device], dtype=torch.int64, device=self.device + ) + self._pageable_stage_capacity = capacity + total = sum(x.numel() * x.element_size() for x in host) + logger.info( + f"pageable GPU expert staging ready: capacity={capacity} rows, " + f"pinned_host={total / 2**20:.1f} MiB, device={total / 2**20:.1f} MiB" + ) + + def _copy_missing_pageable(self, layer_id: int) -> None: + """Gather pageable expert rows into a small pinned buffer, then GPU slots. + + ``ensure_experts`` produces the LRU copy plan on-device. Copying that tiny + plan to pinned host memory introduces one stream fence for an overflow layer; + afterwards CPU ``index_select`` performs the RAM gather, H2D copies enqueue + on the current stream, and one fused D2D scatter places rows in their LRU slots. + """ + if self.device.type != "cuda": + raise RuntimeError("pageable GPU expert staging requires CUDA") + + # Allocate metadata on the first call, then fetch the full small plan arrays. + # Their maximum is cache_size (a few KiB), avoiding a second sync after n is known. + if self._pageable_num_host is None: + self._pageable_num_host = torch.empty(1, dtype=torch.int64, pin_memory=True) + self._pageable_src_host = torch.empty( + self.src_indices.numel(), dtype=torch.int32, pin_memory=True + ) + self._pageable_dst_host = torch.empty( + self.evict_slots.numel(), dtype=torch.int32, pin_memory=True + ) + stream = torch.cuda.current_stream(self.device) + self._pageable_num_host.copy_(self.num_indices, non_blocking=True) + self._pageable_src_host.copy_(self.src_indices, non_blocking=True) + self._pageable_dst_host.copy_(self.evict_slots, non_blocking=True) + stream.synchronize() + n = int(self._pageable_num_host[0]) + if n == 0: + return + if n > self.src_indices.numel(): + raise RuntimeError(f"invalid pageable expert miss count {n}") + self._ensure_pageable_stage(n, layer_id) + + src_ids = self._pageable_src_host[:n].long() + for (per_layer, _cache), host_stage, device_stage in zip( + self.banks, self._pageable_host_staging, self._pageable_device_staging + ): + source = per_layer[layer_id] + torch.index_select(source, 0, src_ids, out=host_stage[:n]) + device_stage[:n].copy_(host_stage[:n], non_blocking=True) + + # All current in-tree fixed-row formats use the fused plan. Variable-row + # GGUF needs per-layer strides and is deliberately rejected until it has a + # corresponding staging descriptor. + if self._variable_bank_rows or not self._copy_fused_ok: + from freetoken.kernel import fast_index_copy_jit + + if self._variable_bank_rows: + raise NotImplementedError( + "--moe-pageable-gpu does not yet support variable-size expert rows" + ) + for (_per_layer, cache), stage in zip(self.banks, self._pageable_device_staging): + fast_index_copy_jit( + cache, + self.evict_slots, + stage, + self._pageable_stage_src_indices, + self.num_indices, + ) + return + + from freetoken.kernel.fast_index_copy import fast_index_copy_multi_jit + + fast_index_copy_multi_jit( + self._copy_dst_ptrs, + self._pageable_stage_src_ptrs, + self._copy_feat_bytes, + self.evict_slots, + self._pageable_stage_src_indices, + self.num_indices, + ) + def copy_missing(self) -> None: assert self.banks, "set_bank_sources must register the banks first" layer_id = self._pending_src_layer assert layer_id is not None, "no staged misses (ensure_experts/materialize_layer first)" if layer_id in self._unpinned_layers: if not self._pending_whole_layer: + if self.pageable_gpu: + self._copy_missing_pageable(layer_id) + return raise RuntimeError( f"layer {layer_id} is unpinned: its only copy is the whole-layer " f"pageable materialize (position == expert id); ensure_experts's " @@ -979,28 +1191,51 @@ def copy_missing(self) -> None: # the only copy a non-pinned layer ever needs is the non-overlap prefill materialize, which schedules the whole layer into slots [0, num_experts) with position == expert id -- a plain synchronous pageable H2D copy # never CUDA-graph captured: prefill is not captured, and decode never reaches this branch (it routes to the CPU executor) for per_layer, cache in self.banks: - cache[: self.num_experts].copy_(per_layer[layer_id]) + source = per_layer[layer_id] + if source.shape[1:] == cache.shape[1:]: + cache[: self.num_experts].copy_(source) + else: + cache[: self.num_experts, : source.shape[1]].copy_(source) return if self._copy_fused_ok: - from freetoken.kernel.fast_index_copy import fast_index_copy_multi_jit + from freetoken.kernel.fast_index_copy import ( + fast_index_copy_multi_jit, + fast_index_copy_multi_strided_jit, + ) # One launch copies the missing rows for every bank (instead of one launch per # bank). evict_slots/src_indices/num_indices are shared across banks; # src_indices holds layer-local expert rows, resolved against this layer's # source pointers (layer_id is a static int per captured graph node). - fast_index_copy_multi_jit( - self._copy_dst_ptrs, - self._copy_src_ptrs[layer_id], - self._copy_feat_bytes, - self.evict_slots, - self.src_indices, - self.num_indices, - ) + if self._variable_bank_rows: + fast_index_copy_multi_strided_jit( + self._copy_dst_ptrs, + self._copy_src_ptrs[layer_id], + self._copy_feat_bytes_by_layer[layer_id], + self._copy_dst_strides, + self._copy_src_strides[layer_id], + self.evict_slots, + self.src_indices, + self.num_indices, + ) + else: + fast_index_copy_multi_jit( + self._copy_dst_ptrs, + self._copy_src_ptrs[layer_id], + self._copy_feat_bytes, + self.evict_slots, + self.src_indices, + self.num_indices, + ) return from freetoken.kernel import fast_index_copy_jit - for per_layer, cache in self.banks: + for name, (per_layer, cache) in zip(self.bank_schema, self.banks): + if name in self._variable_bank_rows: + raise RuntimeError( + "variable-size expert banks require the fused strided copy kernel" + ) fast_index_copy_jit( cache, self.evict_slots, diff --git a/python/freetoken/server/args.py b/python/freetoken/server/args.py index 4954c5f5..7d14a74a 100644 --- a/python/freetoken/server/args.py +++ b/python/freetoken/server/args.py @@ -89,6 +89,7 @@ def parse_args( """ from freetoken.attention import validate_attn_backend from freetoken.kvcache import SUPPORTED_CACHE_MANAGER + from freetoken.kvcache.quant import KV_CACHE_DTYPES from freetoken.moe import SUPPORTED_MOE_BACKENDS def _parse_moe_cache_rate(value: str) -> float: @@ -109,6 +110,15 @@ def _positive_int(value: str) -> int: raise argparse.ArgumentTypeError("must be >= 1") return n + def _positive_ratio(value: str) -> float: + try: + ratio = float(value) + except ValueError as exc: + raise argparse.ArgumentTypeError("must be a number in (0, 1]") from exc + if not 0 < ratio <= 1: + raise argparse.ArgumentTypeError("must be in (0, 1]") + return ratio + def _infer_tool_call_parser(model_path: str) -> str: try: from freetoken.utils import cached_load_hf_config @@ -138,6 +148,10 @@ def _infer_tool_call_parser(model_path: str) -> str: return "muse_glimmer" if "gemma4" in marker: return "gemma4" + # Nemotron-3 Super ships the Qwen3-Coder XML tool grammar + # () in its chat template. + if any(tag in marker for tag in ("nemotron-3", "nemotron_3", "nemotronh")): + return "qwen3_coder" if ( "qwen3_5" in marker or "qwen3.5" in marker @@ -175,6 +189,9 @@ def _infer_reasoning_parser(model_path: str) -> str | None: marker = " ".join(candidates).lower() if "gpt_oss" in marker or "gpt-oss" in marker or "gptoss" in marker: return "gpt_oss" + # Nemotron-3 Super's generation prompt opens an implicit block. + if any(tag in marker for tag in ("nemotron-3", "nemotron_3", "nemotronh")): + return "qwen3" if "deepseek" in marker and any( tag in marker for tag in ("v4", "deepseek_v4", "v3.2", "v32") ): @@ -345,6 +362,21 @@ def _infer_reasoning_parser(model_path: str) -> str | None: help="Set the page size for system management.", ) + parser.add_argument( + "--kv-cache-dtype", + type=str, + choices=list(KV_CACHE_DTYPES), + default=ServerArgs.kv_cache_dtype, + help=( + "KV-cache element storage. 'auto' keeps the compute dtype (bf16); 'q8_0' and " + "'fp8_e4m3' use 1.0625 bytes/element, while 'q4_0' (also accepted as 'int4') " + "uses 0.5625 bytes/element with llama.cpp-compatible GGML Q4_0 quantization. " + "Each includes an fp16 scale per 32 head-dim elements. q8_0 is the most accurate " + "compact format; q4_0 maximizes capacity. Needs the triton " + "attention backend and head_dim divisible by 32." + ), + ) + parser.add_argument( "--attention-backend", "--attn", @@ -371,6 +403,18 @@ def _infer_reasoning_parser(model_path: str) -> str | None: "as a GDN-aware radix (cross-request GDN-state prefix reuse); pass 'naive' to opt out.", ) + parser.add_argument( + "--swa-full-tokens-ratio", + type=_positive_ratio, + default=ServerArgs.swa_full_tokens_ratio, + help=( + "Sliding-window KV-pool size as a fraction of the full-attention KV token " + "capacity. The runtime concurrency/window floor still applies. Lower values " + "leave more VRAM for long-context full-attention KV; effective only for " + "sliding-window models with radix caching." + ), + ) + parser.add_argument( "--enable-cache-report", action="store_true", @@ -539,6 +583,18 @@ def _infer_reasoning_parser(model_path: str) -> str | None: ), ) + parser.add_argument( + "--moe-pageable-gpu", + action="store_true", + default=ServerArgs.moe_pageable_gpu, + help=( + "On hosts with a CUDA pinning quota (notably WSL), keep expert-bank " + "overflow layers pageable and stage only routed cache misses through a " + "small pinned buffer. All expert math remains on GPU, at the cost of " + "eager decode and an extra RAM copy for overflow layers." + ), + ) + parser.add_argument( "--moe-hybrid-max-fetch", type=int, diff --git a/tasks/laguna-handover.md b/tasks/laguna-handover.md new file mode 100644 index 00000000..7ee5e8a9 --- /dev/null +++ b/tasks/laguna-handover.md @@ -0,0 +1,165 @@ +# Laguna GGUF port — handover + +State as of 2026-08-23. Written so a fresh session on a **bigger host** can finish +validating the port against **Laguna-S-2.1** (the XS model was only a stand-in for +bring-up on a 23 GB / 16 GB-VRAM box, which cannot hold S). + +The plan and verified model facts live in `tasks/laguna-todo.md` — read that first, +it is the spec. This file says what is done, what is not, and what will bite you. + +## One-line status + +Phases 1–4 are implemented, reviewed and unit-tested (81 tests green: `tests/models` ++ `tests/kernels/test_gguf_quant_types.py`). Phase 5 (end-to-end validation on real +weights) is **not done**: no token has ever been generated by this port. + +## What works, with evidence + +| Area | Evidence | +|---|---| +| GGML types Q4_K/Q5_K/IQ1_S/IQ2_XXS/IQ3_XXS/IQ4_XS (dequant, mmvq, mmq where it exists, moe_vec) | `tests/kernels/test_gguf_quant_types.py` (27 tests, CUDA vs gguf-py on identical bytes) | +| Config/registry/tokenizer from GGUF metadata | `tests/models/test_laguna_config.py`, committed fixture `tests/fixtures/laguna-s-2.1-metadata.gguf` | +| Attention (per-layer heads, QK-norm, dual rope, softplus gate), MoE router, decoder residuals | `tests/models/test_laguna_modules.py` (stubbed forward vs independent reference) | +| Weight loading, deferred materialization, expert banks | `tests/models/test_laguna_weights.py` (writes a real tiny laguna GGUF with gguf-py) | +| Name-map coverage on the real S file | all 814 tensors mapped or in the expert skip-set; `iter_gguf_weights` yielded 529 params with no duplicates | +| Expert-bank byte path on the real S file | layer 1 / expert 5 gate_up matmul vs gguf-py dequant reference: **0.5 % rel err** | +| S bring-up reached | metadata → weights → expert banks → MoE cache sizing → KV allocation. Died only on host RAM (S needs ~37 GiB pinned; box had 23 GB) | +| XS bring-up reached | same, plus KV alloc of **262144 tokens fp8 = 8.79 GiB** with `--kv-reserve-tokens 262144`; process was killed manually before the ready flip | + +## Validation done on the APEX-Mini model (XS-size) + +Validated end-to-end on `Laguna-XS-2.1-APEX-I-Mini.gguf` (Q3_K/Q4_K/Q5_K/Q6_K/IQ2_S, +12.8 GB) on the 16 GB / 23 GB box: + +- **NIAH 3/3 at 250,054 tokens** -- needle recovered at 10%/50%/90% depth, exact + passcode each time (~433 tok/s prefill). This exercises YaRN over the full window, + the full/SWA split, QK-norm, the softplus gate, and routing, so a llama.cpp token + diff is considered redundant. +- **Decode**: 157-162 tok/s at 64k ctx (8984 expert slots), 21-23 tok/s at ~250k ctx + (2441 slots -- PCIe-bound). 262k KV costs 8.79 GiB fp8, so on 16 GB the 262k and + 64k configs cannot both be "fast". + +## What is NOT done + +1. **S-model validation.** Everything above is XS. S is geometry-identical but 48 + layers / 3072 hidden / 1024 expert, needs >=48 GB RAM (see sizing below) -- run + the same NIAH + decode recipe there. This is the whole point of the handover. +2. **SWA-boundary + eos stop-behaviour** not explicitly tested (short prompts in the + NIAH reasoning block exercised window crossing implicitly, but no dedicated test). +3. **Pre/post-fill perf on IQ types** (IQ2_S, IQ1_S, IQ3_XXS have no MMQ kernel, so + the dense-quant prefill path dequantizes; MoE prefill uses moe_vec). Tune only if + S prefill is slow. +4. **hybrid/cpu MoE backends refuse `gguf`.** No `WFmt` case exists in the compiled + C++ CPU executor, so `--moe-backend hybrid` errors. S host (CPU-bw > PCIe) is where + hybrid matters -- see the full scope in "Things that will bite you" #8 (a C++ SIMD + kernel port, not a Python change). +5. **FTW conversion refused** on purpose (metadata-only GGUF drops per-tensor types). + Serve the `.gguf` directly. +6. **TP=1 only**, text-only. + +## How to run it on the big host + +```bash +ft serve --model /path/Laguna-S-2.1-UD-IQ1_S.gguf \ + --kv-cache-dtype q8_0 --num-tokens 65536 --kv-reserve-tokens 65536 +``` + +- `--kv-reserve-tokens` matters: without it the MoE cache auto-sizer eats the VRAM the + KV pool then needs, and allocation fails late (seen on this box). +- `--kv-cache-dtype q8_0|fp8_e4m3` comes from the *other* (uncommitted) KV-quant + workstream in this tree. With plain `auto` (bf16) the KV cost doubles. + +### Host sizing for S (measured/derived, not guessed) + +- Expert banks, real bytes: **28.8 GiB**. As allocated by this port (uniform padded + slots, see below): **36.9 GiB pinned host RAM**. Budget ≥ 48 GB RAM. +- Dense weights (VRAM): ~3–4 GiB. KV @64k q8_0: ~1.4 GiB (only the 12 full-attention + layers hold full context; the 36 SWA layers are capped at a 512 window). +- VRAM left over becomes the expert LRU cache; more is better, nothing breaks if small. + +## Things that will bite you + +1. **Padding waste (worth fixing).** `OffloadMoeCache.set_bank_sources` asserts every + layer's bank has the same shape, but Laguna's per-layer expert quant types differ, so + `load_gguf_expert_sources` pads every expert slot to the *max* type's size. On S that + is +8.1 GiB of pure waste (28.8 → 36.9). Options, cheapest first: + - group MoE layers by `(gate_up_type, down_type)` and give each group its own cache + (3 groups on S), or + - teach the cache per-layer strides (the kernels already take `expert_stride_bytes`, + so only the host-side bank/copy-plan bookkeeping is in the way). +2. **Mixed types inside one layer.** XS quantizes `attn_v` differently from `attn_q/k` on + half its layers, which is why q/k/v are **separate projections** (not `LinearQKVMerged`). + Do not "optimize" them back into a fused buffer — packed rows of different ggml types + cannot be concatenated. Gate/up fusion *is* still done and is type-checked at load. +3. **Deferred materialization ordering.** The engine collects `model.state_dict()` before + iterating weights, so `convert_laguna_to_gguf` must materialize every + `DeferredGGUFLinear` up front from the file's tensor table (it does — via + `config.gguf_model_path`). If you move that call, loading breaks with missing keys. +4. **`num_qo_heads` is the max (72 on S)**; per-layer counts come from + `config.qo_heads(layer_id)`. Triton allocates decode scratch at the max and gets the + real count from `q.shape[1]` at call time. FlashInfer plans one global head count and + is therefore **not** usable for Laguna; SWA forces the triton backend anyway. +5. **`expert_stride_bytes`** was added to all 19 vendored `moe_vec_*` launchers in + `python/freetoken/kernel/csrc/gguf/`. Value 0 = old dense behaviour, so existing + formats (q4_0/gemma4) are unaffected. If you re-vendor those files from upstream you + will drop this patch. +6. **moe_vec grid.z cap (the crash).** The `moe_vec_q` kernel indexes experts via + `blockIdx.z`, and CUDA caps grid-z at 65535 rows = `tokens*top_k`. `fused_gguf.py` + chunks calls to `min(65535, 16384)` rows for BOTH the grid limit and transient VRAM + use (a 16k-token x top-8 prefill chunk materializes ~1 GiB). The 16384 cap was the + fix for a "CUDA driver error: device not ready" that surfaced asynchronously. +7. **Offload-cache port hygiene.** Orphaned spawn workers hold rendezvous port 1920 with + a bare `multiprocessing` cmdline, so `pkill -f "ft serve"` does NOT kill them; the + next `ft serve` dies EADDRINUSE. Kill by port (`ss -tlnp | grep ':1920 '`) before + restarting, or wait ~60 s for TIME_WAIT to drain. `serve_supervised.sh` in + `.claude/scratch/` does this sweep + drain. +8. **hybrid/cpu fix (for the S host).** To enable `--moe-backend hybrid` for `gguf`: + - Python side: add a `"gguf"` entry to `_WFMT_IDS` in + `python/freetoken/moe/cpu_executor.py` and a `_resolve_gguf_banks` that reads the + flat `[E, stride]` uint8 banks with per-layer `(gate_up_type, down_type)` from + `config.gguf_expert_types`. + - **The hard part (not "~1 file"): the CPU executor's hot path is a compiled C++ + extension** (`python/freetoken/kernel/csrc/cpu_moe/cpu_moe_ext.cpp`) whose `WFmt` + enum has no `gguf` case. It needs vec-dot kernels for every type laguna uses + (Q3_K, Q4_K, Q5_K, Q6_K, IQ1_S, IQ2_XXS, IQ3_XXS, IQ4_XS — ported from llama.cpp's + ggml-cpu AVX-512 paths) plus per-layer type plumbing from `_resolve_gguf_banks`. + That is a C++ kernel port, several hundred lines of SIMD per family plus the + dispatch -- not a Python fallback (the Python `dequantize` is gguf-py/numpy, + far too slow for the decode GEMV to beat PCIe). + - Scope note: only worth it where CPU-bw > PCIe (i.e. the S host's calibration, NOT + this 16 GB box — its own `ft bench bw` profile recommends `offload` for every + format, so hybrid would lose here regardless). + +## Map of the change + +New: +- `python/freetoken/models/laguna/` — `gguf.py` (config parse, name map, weight iter, + deferred linears, expert-bank loader), `attention.py`, `moe.py`, `model.py` +- `python/freetoken/moe/fused_gguf.py` — mixed-type expert GEMV +- `tests/models/test_laguna_{config,modules,weights}.py`, + `tests/kernels/test_gguf_quant_types.py`, `tests/fixtures/laguna-s-2.1-metadata.gguf` + +Modified: +- `kernel/csrc/gguf/{moe_vec.cuh,gguf_kernel.cu}`, `kernel/gguf.py` — `expert_stride_bytes` +- `models/gguf/dequant.py`, `layers/gguf.py` — the six new ggml types +- `layers/moe.py`, `moe/{expert_banks,offload_cache}.py`, `models/weight.py` — the + `"gguf"` expert-bank format end to end +- `models/config.py` — `num_qo_heads_per_layer` + `qo_heads()`, `gguf_model_path`, + `gguf_expert_types` +- `models/gguf/{config,tokenizer,reader,__init__}.py`, `models/register.py` — laguna + registry entry, gpt2-converter routing, `gguf_tensor_type` +- `models/gemma4/gguf.py`, `layers/base.py` — small shared-infra pickups + +## The other workstream in this repo + +The KV-cache quantization effort (`tasks/todo.md`, `kvcache/quant*.py`, +`kernel/triton/kv_quant.py`, `attention/triton.py`, `engine/*`, `server/args.py`, its +tests) is **separate** but shipped in the commit right after this one, because the two +meet at `--kv-cache-dtype`: the `q8_0` / `fp8_e4m3` flag used in every serve command +above comes from there, and without it laguna's KV falls back to bf16 (2x the bytes). + +Its own status: 53 tests green plus the 33 pre-existing triton-attention tests, but +step 9 of `tasks/todo.md` is open — no needle-in-246k, no perplexity vs bf16, no +measured expert-slot / tok-s gain. So `q8_0` vs `fp8_e4m3` as the default is still +undecided, and on the big host it is worth settling that on the same run that +validates laguna: both questions need one loaded model and a long context. diff --git a/tasks/laguna-todo.md b/tasks/laguna-todo.md new file mode 100644 index 00000000..b074e041 --- /dev/null +++ b/tasks/laguna-todo.md @@ -0,0 +1,169 @@ +# Laguna-S-2.1 GGUF support (unsloth Laguna-S-2.1-UD-IQ1_S.gguf) + +Goal: `ft serve --model Laguna-S-2.1-UD-IQ1_S.gguf --kv-cache-dtype q8_0 --num-tokens 65536` +loads and generates correctly on the RTX 5080 (16 GB), experts on the MoE offload cache. + +## Ground truth (verified from the file header + llama.cpp origin/master) + +Single unsplit GGUF, 33,766,781,984 bytes. **No split-reader work needed for this file.** + +Metadata (`laguna.*`): +- block_count 48, embedding_length 3072, vocab 100352, context_length 262144 +- head_count per layer: `[48,72,72,72] * 12` (48 ⇒ full attention at `il%4==0`, 72 ⇒ SWA) +- head_count_kv 8 (uniform), key/value_length 128, sliding_window 512 +- rope full layers: dim 64 (partial), θ=500000, YaRN factor 32, orig_ctx 8192, + attn_factor 1.0, beta_fast 32, beta_slow 1 +- rope SWA layers: dim 128, θ=10000, **plain rope, no YaRN** (freq_scale 1.0) +- rms_eps 1e-6; leading_dense_block_count 1; expert_count 256, used 10, + expert_ff 1024, shared_expert_ff 1024, weights_norm true, weights_scale 2.5, + gating_func 2 (sigmoid); dense ffn 12288 +- tokenizer: gpt2 BPE, pre="laguna", bos 2, eos 2, eot 24, add_bos true, + chat template embedded +- Tensor quant types by role (per-tensor, "Unsloth Dynamic" = mixed): + - token_embd / output: Q4_K (untied head) + - attn q/k/v/o, attn_gate, dense ffn, shexp gate/up: Q5_K (layer 47: Q6_K) + - ffn_down (dense), down_shexp: Q6_K (one Q8_0) + - expert banks: gate/up IQ1_S (33 layers) or IQ2_XXS (14), down IQ3_XXS (45) or IQ4_XS (2) + - norms, router (ffn_gate_inp), exp_probs_b: F32 +- Layer schedule: layer 0 dense SwiGLU; layers 1..47 MoE (expert bank index = layer_id-1) + +Reference semantics (llama.cpp `src/models/laguna.cpp`, copy in +`/tmp/claude-1000/-home-lucas-ai-FreeToken/61016eb3-71fd-4d7b-97c6-816d5fa0839a/scratchpad/llamacpp-laguna.cpp`): +- QK RMSNorm at head_dim level (weight shape [128]) before rope, Qwen3-style +- attn_gate: `g = softplus(g_proj(pre-norm hidden))`, per-head ([3072, n_head_il]); + multiply attention output per head (broadcast over head_dim) **before** o_proj +- router: fp32 logits → sigmoid → +exp_probs_b bias for top-10 selection only → + gather **unbiased** sigmoid scores → renormalize → ×2.5 → weighted expert sum; + shared expert always added +- pre-attn and pre-ffn RMSNorm only (no post norms); final norm + untied lm_head + +CUDA kernel coverage (verified in `python/freetoken/kernel/csrc/gguf/`): mmvq, mmq +(Q4_K/Q5_K), dequantize and moe_vec for **all** the types above already exist and are +dispatched by ggml type id in `gguf_kernel.cu`. Only the Python-side tables gate them. + +Blockers found in Python side: +- `models/gguf/dequant.py`: only F32/F16/BF16/Q4_0/Q8_0/Q6_K in tables +- `layers/gguf.py`: `_MMVQ/_MMQ/_DEQUANT = {Q4_0, Q8_0, Q6_K}` +- `models/weight.py::load_q4_0_moe_expert_sources`: Q4_0-only expert banks +- `ModelConfig`: scalar `num_qo_heads` (Laguna needs per-layer 48/72) +- GGUF registry: only gemma4 + +Model download running in background task `bp45xxl7b` → +`~/.cache/huggingface/hub/models--unsloth--Laguna-S-2.1-GGUF/...`. +Header fixture: scratchpad `laguna_sparse.gguf` (metadata + tensor table, sparse data). + +## Orchestration + +Implementer: `pc-gpt-5-3-codex-spark` (weak — tasks kept small, exact files/symbols given). +Escalation (user 2026-08-23): on the next Spark failure (context death, usage limit, +botched task), switch the implementer seat to `pc-gpt-5-6-luna` permanently. +Reviewer per phase: `pc-gpt-5-6-terra` `[[effort: medium]]` — findings **with fixes**. +If a phase needs >2 review rounds, the lead (Fable) reviews and fixes directly. + +## Phase 1 — GGML quant-type plumbing (no model code) + +- [x] 1a. `models/gguf/dequant.py`: add ids Q4_K=12, Q5_K=13, IQ2_XXS=16, IQ3_XXS=18, + IQ1_S=19, IQ4_XS=23; BLOCK_SHAPE (256,144)/(256,176)/(256,66)/(256,98)/(256,50)/(256,136); + GGML_NAME entries. Reference dequant: delegate to gguf-py `gguf.quants.dequantize` + (verify installed gguf-py handles these types; else port only what tests need). +- [x] 1b. `layers/gguf.py`: extend `_MMVQ` with all six; `_MMQ` with Q4_K, Q5_K only; + `_DEQUANT` per actual `ggml_dequantize` switch coverage in `gguf_kernel.cu` + (verify each case id before adding). IQ types have no MMQ → prefill path must + fall back to dequant+bf16 matmul; confirm `fused_mul_mat_gguf` already routes that. +- [x] 1c. Tests `tests/kernels/test_gguf_quant_types.py`: per type — CUDA + `ggml_dequantize` vs gguf-py reference on random packed blocks; `mmvq` matvec vs + `F.linear` on dequantized weight; `moe_vec` for IQ1_S/IQ2_XXS/IQ3_XXS/IQ4_XS. +- [x] R1. Terra review round(s) → fixes → tests green. + +Phase 1 notes: gguf-py cannot quantize K/IQ formats, so tests use random +safe-scaled packed bytes with gguf-py dequant as reference; matmul refs model +q8_1 activation quantization; moe_vec asserted bit-exact vs mmvq. MMQ exists +for Q4_K/Q5_K only among the new types (IQ prefill falls back to dequant). +Lead (Fable) fixed tests after 2 implementer rounds. + +## Phase 2 — config, registry, tokenizer + +- [x] 2a. `models/config.py`: add optional per-layer qo-head counts + (`num_qo_heads_per_layer: tuple[int, ...] | None = None`) with accessor defaulting + to scalar `num_qo_heads`; audit consumers that assume the scalar for Q width + (KV geometry is uniform: 8 KV heads × 128 — KV pools unaffected). +- [x] 2b. New `models/laguna/` package: `config.py::parse_gguf_config` building + ModelConfig from the metadata above (full/SWA schedule from head-count array; + per-layer rope params; MoE fields; eos {2,24}); registry entries + (`gguf/config.py::GGUF_ARCH_TO_REGISTRY["laguna"]`, `register.py` ModelSpec + `LagunaGGUFForCausalLM`). +- [x] 2c. `models/gguf/tokenizer.py`: make it arch-generic where gemma-specific + (eos ids from `tokenizer.ggml.eos_token_id` + `eot_token_id`; verify transformers + converts gpt2/"laguna" pre BPE from GGUF; else load via tokenizer.ggml.* fields). +- [x] 2d. Tests: config parse from a metadata dict fixture; registry dispatch; + tokenizer eos/bos/chat-template presence (uses scratchpad header fixture). +- [x] R2. Terra review → fixes. + +Phase 2 notes: yarn scaling needs explicit attention_factor=1.0 (metadata +yarn_attn_factor) or freetoken defaults to ggml-incompatible mscale. Tokenizer +routes laguna->gpt2 converter with bos/eos strings materialized from ids. +Committed metadata-only fixture tests/fixtures/laguna-s-2.1-metadata.gguf (3.5MB). +Registry entry live; model stub raises a clear in-progress error until Phase 3/4. +Review: 1 round (blocker attention_factor + hardening), fixes by Spark. + +## Phase 3 — model modules (torch, quant-agnostic bf16 reference first) + +Phase 3 wiring notes (from Terra's R2 review): FlashInfer plans a single global +qo_head count and LinearQKVMerged/LinearOProj take one Q/O width — attention +modules must be built with `config.qo_heads(layer_id)` and pass it to QKV/O +projections, reshapes, gate logic, and the GQA/tensor-core decision; Triton +decode scratch may keep max-72 allocation but gets the real per-layer count at +invocation. attn backend for SWA is triton-only. + +- [x] 3a. `models/laguna/attention.py`: per-layer head count, QK head-dim RMSNorm, + partial-rope (dim 64 yarn) / full-rope (dim 128 plain) per layer type, per-head + softplus gate (fp32) applied before o_proj. Reuse `layers/rotary.py::get_rope` + (verify partial+yarn support), existing attention backend plumbing (SWA hybrid + pool as in gemma4). +- [x] 3b. `models/laguna/moe.py` + `mlp`: router semantics above; reuse existing MoE + offload machinery (find sigmoid+bias router precedent, e.g. glm/minimax/afmoe + style in repo); shared expert; dense layer 0. Expert bank index = layer_id-1. +- [x] 3c. `models/laguna/model.py`: decoder wiring, final norm, untied head; + `convert_laguna_to_gguf` pass swapping Linear/Embedding → GGUFLinear/GGUFEmbedding + **keeping each tensor's own ggml type** (unlike gemma4's uniform assumption). +- [x] 3d. Unit tests vs handwritten torch reference: one full layer, one SWA layer, + gate math, router top-10 selection/bias/renorm/scale, dense vs MoE layer. +- [x] R3. Terra review — semantics clean; loader-contract findings folded into Phase 4; + test-quality findings fixed in round 1. Module tests: 6, all green. + +## Phase 4 — weight loading + +- [x] 4a. `models/laguna/gguf.py::iter_gguf_weights`: name map (`blk.N.attn_q` etc. → + module params, table in ground truth above); norms/router/bias to f32/bf16; + quantized tensors kept packed with per-tensor type. + R3 contract (blocker): Engine collects model.state_dict() BEFORE iterating + weights, so DeferredGGUFLinear must be materialized earlier — add + `gguf_model_path` to ModelConfig (set from shim.model_path), have + convert_laguna_to_gguf read the tensor table (name→ggml type) and + materialize every swapped module up front using the same name map 4a uses + (share one mapping helper). Also (major): the untied lm_head must select + last-token rows on prefill like ParallelLMHead/GGUFTiedLMHead + (batch.attn_metadata.get_last_indices(batch.size)) — wrap DeferredGGUFLinear + in a LagunaGGUFLMHead that does the gather before the fused matmul. +- [x] 4b. Generalize `models/weight.py::load_q4_0_moe_expert_sources` → type-aware + expert-bank loader (per-layer, per-projection ggml type — gate/up/down differ); + wire type ids through the offload cache to `moe_vec` dispatch. +- [x] 4c. Test: synthetic tiny laguna GGUF (write with gguf-py) loads end-to-end with + dummy weights; every tensor consumed exactly once; unknown tensor → clear error. +- [x] R4. Sol (medium) review — clean except FTW-conversion blocker; laguna FTW + now rejects loudly (documented limitation). Kernel scope independently + cleared by a Terra sub-lens before the reviewer switch. + +## Phase 5 — e2e validation (lead-driven) + +- [ ] 5a. Load real file; `ft serve --kv-cache-dtype q8_0 --num-tokens 65536`; + one-token decode, short prefill; VRAM/expert-cache occupancy sane. +- [ ] 5b. Greedy next-token comparison vs llama.cpp on fixed prompts. +- [ ] 5c. Short needle + SWA-boundary (>512 tok) prompts; eos 2/24 stop behavior. +- [ ] Review section here. + +## Initial limitations + +TP=1 only; FTW conversion unsupported (serve the .gguf directly); text-only; no speculative decoding; split-GGUF variants (BF16 etc.) +out of scope; prefill for IQ expert types goes through moe_vec/dequant fallback +(optimize later if slow). diff --git a/tasks/lessons.md b/tasks/lessons.md new file mode 100644 index 00000000..38f2f4f4 --- /dev/null +++ b/tasks/lessons.md @@ -0,0 +1,19 @@ + +## 2026-08-26 (Ornith phase 2 live A/B) +- Never grade model output by grepping a raw SSE stream: tokens split codes + across `data:` events. Concatenate the JSON `text` fields first. (Caused 3 + false NEEDLE_MISS on a run that was actually 3/3.) +- A "server startup timeout" on this stack is usually one of: (a) stale + torch_extensions `lock` from a SIGKILLed build -- the loader sleep-polls it + forever; (b) swap-cold expert-bank build. Diagnose via /proc//wchan + (hrtimer_nanosleep) + lock mtime BEFORE extending timeouts or killing. +- `pkill -f "ft serve"` does NOT kill the workers: they are + `multiprocessing.spawn` stubs holding ~20 GiB shmem banks. Kill by venv path + and verify with `free -g`, or the next run OOMs the box (took the terminal + down 3x). +- `--num-tokens` is GPU KV capacity, not host RAM. Don't guess flag semantics + in explanations to the user -- read the argparse help first. +- Structure live perf claims as env-gated A/B on the same box minutes apart + (FREETOKEN_GGUF_DISABLE_MMA=1), with a path-proof mechanism decided BEFORE + the run; in-server INFO logging is swallowed by the log handler, so proof + must be external (JIT cache touch, wall-time signature). diff --git a/tasks/todo.md b/tasks/todo.md new file mode 100644 index 00000000..8eb777ef --- /dev/null +++ b/tasks/todo.md @@ -0,0 +1,121 @@ +# Ornith RTX 5080 full-context optimization + +- [x] Add reproducible Ornith Q4_0 attention and serving benchmark controls. +- [x] Record unchanged-main RTX 5080 synthetic baselines. +- [x] Sweep and implement numerically safe sm_120 attention launch geometry. +- [x] Sweep Ornith Q4_K/Q6_K dense and routed-expert GGUF dispatch on sm_120. +- [x] Implement only repeatable sm_120 wins with safe non-sm_120 fallbacks. +- [x] Verify focused tests, full non-slow tests, and lint. +- [x] Validate a live near-262K request and long-context retrieval. +- [x] Document the final RTX 5080 launch command and measured before/after results. + +## Review (2026-08-25, RTX 5080 sm_120, Torch 2.11/CUDA 13/Triton 3.6, WSL) + +### Implemented +- `decode_launch_config` is architecture-aware (`compute_capability` param). Ornith + Q4_0 decode on sm_120 uses (kv_splits=64, block_n=64, warps=8): 0.356 ms vs + 0.822 ms per 262K full-attention layer with the sm_89 tuple (2.31x). sm_89 and + the conservative fallback are unchanged; scratch/CUDA-graph capacity follows. +- `extend_paged_attention` uses num_warps=4 on sm_120 (8 elsewhere): 1.12x on the + production long-Q4-prefix split kernel, 2.01x on the fused cold-prefill kernel. + BLOCK_N=16 was reconfirmed to silently corrupt the packed loader on sm_120 in + BOTH extend kernels and remains excluded. +- `benchmarks/bench_ornith_attention.py` (oracle-gated decode/prefill/extend + sweeps at the exact Ornith geometry) + full-context flags on + `bench_decode_moe.py` (`--max-context`, `--kv-cache-dtype`, `--prefill-chunk`, + `--prefill-hit-d2d`); GPU-free unit tests under `tests/benchmarks/`. +- `docs/models.md`: sm_120 full-262K launch command (explicit + `--attention-backend triton`; sm_120 auto-resolves to FlashInfer, which cannot + read the quantized KV pool). + +### Measured but intentionally not changed +- The vendored GGUF DP4A kernels predate llama.cpp's int8-tensor-core MMQ + rewrite (as do vLLM/SGLang's copies); porting that is phase 2b (below). + +## Phase 2a (2026-08-25): arch-aware GGUF dispatch thresholds + +- [x] `layers/gguf.py`: `dequant_gemm_min_rows(cc)` — 24 on sm_120, 32 + elsewhere (Q4_K attn shapes cross at 24: dequant 0.0645 vs MMQ 0.0778 ms; + 16 would regress the Q6_K lm_head where MMQ still wins at 16). +- [x] `moe/fused_gguf.py`: `mmq_min_tokens(cc)` — 16 on sm_120, 32 elsewhere + (grouped MMQ 0.314 vs vec 0.324 ms @16; 0.382 vs 0.475 @24). +- [x] `_MMVQ_SAFE` left at 6 (per-shape ambiguous on sm_120). +- [x] Tests: `tests/kernels/test_gguf_dispatch.py` — pure threshold tests for + both archs + CUDA dispatch-branch tests with faked capability (18 passed). +- [x] Live re-verify on real Ornith blk.3.attn_q (Q4_K): 24 rows now 0.0690 ms + via dequant vs 0.0806 ms with the old threshold; 16/32 rows unchanged. +- [x] Kernel+benchmark suites 123 passed; full non-slow suite failures A/B + identical to clean main (7 failures; laguna errors are suite-order artifacts + present in the clean-main baseline too); ruff clean. + +## Phase 2b (2026-08-25): upstream int8-MMA MMQ port (Q4_K/Q6_K) + +- [x] Vendored llama.cpp master `eab8ee41` CUDA MMQ verbatim into + `python/freetoken/kernel/csrc/gguf_mmq/` (mmq/mma/load-tiles/vec-dot/configs/ + quantize/mmid + the ggml headers). `mmq_ext.cu` is the only hand-written + file: backend shims (device info, torch-allocator pool, abort/error) + torch + bindings; only Q4_K/Q6_K `mul_mat_q` cases are instantiated. +- [x] Dense: `ggml_mul_mat_a8_mma` wired into `fused_mul_mat_gguf` on sm_120 + for rows > `_MMVQ_SAFE` (replaces BOTH the DP4A-MMQ band and dequant+cuBLAS). + Measured (real Ornith tensors, embedder stopped): Q4_K attn_q 8192 rows + 1.79 ms vs dequant 2.40 vs DP4A 22.9; Q6_K lm_head @2048 tokens 17.5 vs 19.2 + vs 262; wins at every rows >= 4. Build failure falls back to the old path. +- [x] MoE: `ggml_moe_a8_mma` (upstream ids path: mm_ids_helper + + scatter/gather q8_1_mmq quantize + expert_bounds mul_mat_q) wired into + `_moe_matmul` for 320 <= tokens <= 16384 on sm_120; broadcast (gate/up) and + per-slot (down) forms; padded flat slots addressed in whole blocks (stride + must divide by block size -- true for Ornith banks). Ornith geometry + (E=256 top-8): 8192 tokens gate_up 4.16 ms vs DP4A 23.2, down 4.90 vs 15.3; + DP4A keeps 16..319 (crossover ~288), MMVQ keeps decode. +- [x] Numerics: MMA matches dequant reference within activation-quant noise on + real Ornith tensors (rel <= 0.013, on par with DP4A) and on random-safe + packed bytes vs gguf-py (`tests/kernels/test_gguf_mma.py`); MoE broadcast + + gather verified vs per-expert dense reference and vs the vec kernel + (end-to-end `fused_experts_gguf` rel ~1e-3). +- [x] Tests: dispatch-branch tests extended (MMA seams, `_use_mma_moe` gate); + `test_dense_gguf_prefill_uses_dequantized_cublas_result` pinned to the + dequant branch it validates. Kernels+benchmarks: 336 passed, 1 skipped. + Full non-slow suite: same 7 pre-existing failures as clean main. +- [x] Live A/B at the production 262K config (2026-08-26, hostile prompt: + 28K words seeded non-repetitive text, ~50K tokens / 6+ chunks, three distinct + needles at 10/50/90% depth, greedy, 320-token decode): + - MMA: 13.88 s wall, 3/3 needles exact, 0 tracebacks (prefill ~4,700 tok/s). + - Fallback (FREETOKEN_GGUF_DISABLE_MMA=1, same box/prompt minutes apart): + 21.94 s wall, 3/3 needles exact, 0 tracebacks (prefill ~2,700 tok/s). + - Net: ~1.75x prefill, 1.58x end-to-end TTFT+decode; identical answers. + - Path evidence: the MMA leg's worker warmup demonstrably blocked polling the + freetoken_gguf_mmq JIT lock (only `_mma_module()` touches it) and ran at + the faster wall time; the fallback leg never touched that cache. The + in-log `int8-MMA MMQ ACTIVE` INFO line is swallowed by the server's log + handler -- known instrumentation gap, not a dispatch gap. +- OPS HAZARD found: `torch.utils.cpp_extension.load` leaves a stale `lock` in + ~/.cache/torch_extensions/ when a building/loading process is SIGKILLed; the + next server then hangs in warmup FOREVER, sleep-polling it (looks like a + startup hang). Fix on sight: `rm ~/.cache/torch_extensions/py312_cu130/*/lock`. + Also: 3 host OOMs during testing were the ~20 GiB shmem expert banks + agent + processes on the 29 GiB WSL box; mitigated with a 12 GiB swapfile + (/swapfile-claude, left enabled) + oom_score_adj (serve 800, terminal -600). + A killed `ft serve` leaves `multiprocessing.spawn` workers holding the banks: + `pkill -9 -f "FreeToken/.venv/bin/python3"` and check `free -g`. +- Flaky pre-existing `test_reference_roundtrip_error_is_within_the_scheme_envelope[int4]` + (~5% failure, unseeded randn) now seeded. + +### Live 262K validation (Ornith-1.5-35B-Q4_K_M, one RTX 5080 16 GB) +Command: `ft serve --model ~/ai/models/Ornith-1.5-35B-Q4_K_M.gguf +--attention-backend triton --kv-cache-dtype q4_0 --num-tokens 262144 +--kv-reserve-tokens 262144 --max-seq-len-override 262144 +--max-running-requests 1 --moe-backend offload --moe-cache-auto +--max-prefill-length 8192` +- Auto-sizing: 4,835 expert slots + 262,263 KV pages, prefill overlap on. +- Cold ~259,400-token prefill: 210–220 s (~1,230 tok/s sustained). +- Decode at ~259K context: 99–104 tok/s (Ada baseline: 33.67 tok/s at 170K). +- NIAH 3/3 exact: passcode `7391-ALPHA` recovered at 10%/50%/90% depth + (greedy, through the model's `` block). +- Radix-cached TTFT for a repeated full-context prompt: 4.9 s. +- No crash, OOM, or worker restart across the whole run. + +### Test/lint status +- `tests/kernels/test_triton_attention.py` + `test_kv_quant.py`: 75 passed. +- `tests/benchmarks`: 8 passed; ruff clean on all changed files. +- Full non-slow suite: 9 failures + 6 errors pre-exist on clean main + (`moe_pageable_gpu` config-test drift), byte-identical A/B vs baseline. diff --git a/tests/benchmarks/test_bench_ornith_attention.py b/tests/benchmarks/test_bench_ornith_attention.py new file mode 100644 index 00000000..df835f16 --- /dev/null +++ b/tests/benchmarks/test_bench_ornith_attention.py @@ -0,0 +1,84 @@ +"""Lightweight, GPU-free tests for the Ornith attention bench's case-building. + +The bench itself needs CUDA (it calls the production Triton kernels), so these tests +only cover the pure argparse/case-expansion logic -- kept as plain functions with no +torch import specifically so this file can catch sweep-construction bugs (wrong case +count, wrong op routing, geometry drift) on any machine, CI included. +""" + +from __future__ import annotations + +import sys +from pathlib import Path + +import pytest + +sys.path.insert(0, str(Path(__file__).resolve().parents[2] / "benchmarks")) +import bench_ornith_attention as bench # noqa: E402 + + +def test_default_geometry_matches_the_tuned_decode_launch(): + # This is the exact shape kernel/triton/attention.py::decode_launch_config + # special-cases for packed int4 (BLOCK_N=32/32 splits/4 warps). If either drifts, + # the bench silently stops exercising the tuned path -- pin both here. + assert (bench.Q_HEADS, bench.KV_HEADS, bench.HEAD_DIM) == (16, 2, 256) + + +def test_quant_alias_covers_every_cli_choice(): + for name in bench.QUANT_CHOICES: + assert name in bench._QUANT_ALIAS + assert bench._QUANT_ALIAS["int4"] == bench._QUANT_ALIAS["q4_0"] == "int4" + assert bench._QUANT_ALIAS["bf16"] == "auto" # the unquantized pool, no oracle check + + +def test_default_args_build_the_documented_case_counts(): + args = bench.parse_args([]) + cases = bench.build_cases(args) + + n_decode = len(args.kv_quant) * len(args.decode_lengths) * len(args.batch_sizes) * 1 + n_prefill = len(args.kv_quant) * len(args.prefill_chunk_sizes) + n_extend = len(args.kv_quant) * len(args.decode_lengths) * len(args.extend_chunk_sizes) + assert len(cases) == n_decode + n_prefill + n_extend + + assert sum(isinstance(c, bench.DecodeCase) for c in cases) == n_decode + assert sum(isinstance(c, bench.PrefillCase) for c in cases) == n_prefill + assert sum(isinstance(c, bench.ExtendCase) for c in cases) == n_extend + assert {c.op for c in cases} == set(bench.OPS) + + +def test_ops_filter_restricts_to_the_requested_families(): + args = bench.parse_args(["--ops", "decode"]) + cases = bench.build_cases(args) + assert cases and all(c.op == "decode" for c in cases) + + +def test_max_kv_splits_sweep_multiplies_only_decode_cases(): + base = bench.build_cases(bench.parse_args(["--ops", "decode", "--decode-lengths", "1024"])) + swept = bench.build_cases( + bench.parse_args(["--ops", "decode", "--decode-lengths", "1024", "--max-kv-splits", "8", "16", "32"]) + ) + assert len(swept) == len(base) * 3 + assert {c.max_kv_splits for c in swept} == {8, 16, 32} + assert base[0].max_kv_splits is None # default: let the kernel pick its own tuned preference + + +def test_decode_case_field_layout(): + case = bench.DecodeCase(quant="int4", ctx_len=131072, batch=4, max_kv_splits=32) + assert case.op == "decode" + assert (case.quant, case.ctx_len, case.batch, case.max_kv_splits) == ("int4", 131072, 4, 32) + + +def test_build_cases_is_deterministic(): + args = bench.parse_args(["--kv-quant", "q8_0", "int4"]) + first = bench.build_cases(args) + second = bench.build_cases(args) + assert first == second + + +def test_help_exits_cleanly(): + """--help / case-building must stay usable on a machine with no GPU: torch/triton are + imported lazily inside the run_* functions, never at module scope or in parse_args.""" + with pytest.raises(SystemExit) as exc: + bench.parse_args(["--help"]) + assert exc.value.code == 0 + assert "torch" not in vars(bench) # confirms parse_args never pulled torch into scope diff --git a/tests/engine/test_cache_budget.py b/tests/engine/test_cache_budget.py index a164f0b4..d80b2b8e 100644 --- a/tests/engine/test_cache_budget.py +++ b/tests/engine/test_cache_budget.py @@ -292,6 +292,7 @@ class StubConfig: memory_ratio = 0.9 moe_prefill_overlap = True kv_reserve_tokens = 0 + num_page_override = None swa_full_tokens_ratio = 0.2 swa_num_pages_override = None model_config = StubModelConfig() @@ -331,6 +332,53 @@ class StubBanks: assert (size, pages, overlap) == expected +def test_engine_auto_moe_reserves_explicit_kv_geometry(monkeypatch): + import freetoken.engine.cache_budget as budget + from freetoken.engine.engine import Engine + + captured = {} + + def fake_resolve(**kwargs): + captured.update(kwargs) + return 8, 25, True + + monkeypatch.setattr(budget, "resolve_moe_cache_auto", fake_resolve) + + class StubModelConfig: + num_experts = 4 + num_moe_layers = 2 + + @staticmethod + def linear_attention_group(): + return None + + class StubConfig: + memory_ratio = 0.9 + moe_prefill_overlap = True + kv_reserve_tokens = 32 + num_page_override = 25 + model_config = StubModelConfig() + + class StubPool: + @staticmethod + def kv_cost(_config): + return 10, 0, 16, 0 + + class StubBanks: + quant_format = "bf16" + sources = { + "gate_up": [torch.zeros(4, 2, dtype=torch.float16)], + "down": [torch.zeros(4, 2, dtype=torch.float16)], + } + + engine = Engine.__new__(Engine) + engine._baseline_free = 10_000 + engine._weights_bytes = 1_000 + engine._pool_cls = StubPool + assert engine._resolve_auto_moe_cache_size(StubConfig(), StubBanks()) == (8, 25, True) + assert captured["kv_reserve_tokens"] == 25 * 16 + + # --------------------------------------------------------------------------- # offload-cache sizing guard + auto-resolution (_require_offload_cache_size / _adjust_config), # the floor rule compute_cache_floors documents above. diff --git a/tests/engine/test_kv_cache_dtype_gating.py b/tests/engine/test_kv_cache_dtype_gating.py new file mode 100644 index 00000000..900fb716 --- /dev/null +++ b/tests/engine/test_kv_cache_dtype_gating.py @@ -0,0 +1,72 @@ +"""--kv-cache-dtype gating: the combinations the quantized path does not implement +must be refused at config time, not at the first kernel launch.""" + +from __future__ import annotations + +import pytest + +from freetoken.engine.engine import _validate_kv_cache_dtype +from freetoken.kvcache.quant import FP8_E4M3, INT4, NONE, Q8_0 +from freetoken.models.config import KVCacheGroupSpec + + +class _Cfg: + def __init__(self, quant=Q8_0, backend="triton"): + self.kv_quant = quant + self.attention_backend = backend + + +class _Model: + def __init__(self, *specs): + self._specs = specs + + def kv_cache_group_specs(self): + return self._specs + + +def _spec(name="full", head_dim=256, mla=False, index_head_dim=0): + return KVCacheGroupSpec( + name=name, + layer_ids=(0, 1), + num_kv_heads=2, + head_dim=head_dim, + sliding_window=None, + mla=mla, + index_head_dim=index_head_dim, + ) + + +@pytest.mark.parametrize("quant", [Q8_0, FP8_E4M3, INT4], ids=lambda spec: spec.name) +def test_triton_backend_with_aligned_head_dim_is_accepted(quant): + _validate_kv_cache_dtype(_Cfg(quant=quant), _Model(_spec(head_dim=256), _spec("swa", 512))) + + +def test_auto_dtype_skips_every_check(): + # Unquantized configs must pass even on backends/pools the quantized path rejects. + _validate_kv_cache_dtype(_Cfg(quant=NONE, backend="fi"), _Model(_spec(mla=True))) + + +@pytest.mark.parametrize("backend", ["fi", "fa", "trtllm", "fa,fi", "triton,fi"]) +def test_non_triton_backends_are_rejected(backend): + with pytest.raises(ValueError, match="needs the triton attention backend"): + _validate_kv_cache_dtype(_Cfg(backend=backend), _Model(_spec())) + + +def test_mla_pool_is_rejected(): + with pytest.raises(ValueError, match="MLA/DSA"): + _validate_kv_cache_dtype(_Cfg(), _Model(_spec(mla=True))) + + +def test_dsa_index_tier_is_rejected(): + with pytest.raises(ValueError, match="MLA/DSA"): + _validate_kv_cache_dtype(_Cfg(), _Model(_spec(index_head_dim=128))) + + +def test_head_dim_not_a_multiple_of_the_block_is_rejected(): + with pytest.raises(ValueError, match="multiple of 32"): + _validate_kv_cache_dtype(_Cfg(), _Model(_spec(head_dim=80))) + + +def test_the_error_names_the_offending_group(): + with pytest.raises(ValueError, match=r"swa \(head_dim 100\)"): + _validate_kv_cache_dtype(_Cfg(), _Model(_spec(head_dim=256), _spec("swa", 100))) diff --git a/tests/fixtures/laguna-s-2.1-metadata.gguf b/tests/fixtures/laguna-s-2.1-metadata.gguf new file mode 100644 index 00000000..0369262e Binary files /dev/null and b/tests/fixtures/laguna-s-2.1-metadata.gguf differ diff --git a/tests/kernels/test_gguf_dispatch.py b/tests/kernels/test_gguf_dispatch.py new file mode 100644 index 00000000..7400c5bc --- /dev/null +++ b/tests/kernels/test_gguf_dispatch.py @@ -0,0 +1,191 @@ +"""Arch-aware GGUF kernel dispatch thresholds (sm_120 vs Ada/default). + +The threshold *functions* are pure and tested on both archs without a GPU; the +dispatch-site tests stub the CUDA kernels and fake the device capability, so +they only need any CUDA device (branch selection, not numerics). +""" +from __future__ import annotations + +import pytest +import torch + +import freetoken.layers.gguf as layers_gguf +import freetoken.moe.fused_gguf as moe_gguf +from freetoken.layers.gguf import dequant_gemm_min_rows +from freetoken.models.gguf.dequant import BLOCK_SHAPE, GGML_Q4_K +from freetoken.moe.fused_gguf import mmq_min_tokens + + +@pytest.mark.parametrize( + "capability,expected", + [(None, 32), ((8, 9), 32), ((9, 0), 32), ((12, 0), 24), ((12, 1), 24)], +) +def test_dequant_gemm_min_rows(capability, expected): + assert dequant_gemm_min_rows(capability) == expected + + +@pytest.mark.parametrize( + "capability,expected", + [(None, 32), ((8, 9), 32), ((9, 0), 32), ((12, 0), 16), ((12, 1), 16)], +) +def test_mmq_min_tokens(capability, expected): + assert mmq_min_tokens(capability) == expected + + +@pytest.fixture +def cuda(): + if not torch.cuda.is_available(): + pytest.skip("CUDA required") + + +@pytest.mark.parametrize( + "capability,rows,expected_branch", + [ + ((12, 0), 24, "dequant"), + ((12, 0), 23, "mmq"), + ((8, 9), 24, "mmq"), + ((8, 9), 32, "dequant"), + ], +) +def test_dense_dispatch_branch(cuda, monkeypatch, capability, rows, expected_branch): + import freetoken.kernel.gguf as kernel_gguf + + monkeypatch.setattr(layers_gguf, "_device_capability", lambda i: capability) + # The MMA path (tested separately) sits above the dequant/DP4A crossover. + monkeypatch.setattr(layers_gguf, "_use_mma_mmq", lambda qt, cc: False) + block, type_size = BLOCK_SHAPE[GGML_Q4_K] + in_features, out_features = 256, 8 + qweight = torch.zeros( + (out_features, in_features // block * type_size), dtype=torch.uint8, device="cuda" + ) + x = torch.zeros((rows, in_features), dtype=torch.float16, device="cuda") + called = [] + monkeypatch.setattr( + kernel_gguf, + "ggml_dequantize", + lambda *a, **k: called.append("dequant") + or torch.zeros((out_features, in_features), dtype=x.dtype, device="cuda"), + ) + monkeypatch.setattr( + kernel_gguf, + "ggml_mul_mat_a8", + lambda *a, **k: called.append("mmq") + or torch.zeros((rows, out_features), dtype=x.dtype, device="cuda"), + ) + out = layers_gguf.fused_mul_mat_gguf(x, qweight, GGML_Q4_K) + assert called == [expected_branch] + assert out.shape == (rows, out_features) + + +@pytest.mark.parametrize( + "capability,expect_mma", + [((12, 0), True), ((12, 1), True), ((8, 9), False), ((9, 0), False)], +) +def test_dense_dispatch_mma_branch(cuda, monkeypatch, capability, expect_mma): + """On sm_120, Q4_K rows above the MMVQ band route to int8-MMA MMQ.""" + import freetoken.kernel.gguf as kernel_gguf + + monkeypatch.setattr(layers_gguf, "_device_capability", lambda i: capability) + monkeypatch.setattr(layers_gguf, "_mma_mmq_ok", lambda: True) + block, type_size = BLOCK_SHAPE[GGML_Q4_K] + in_features, out_features, rows = 256, 8, 64 + qweight = torch.zeros( + (out_features, in_features // block * type_size), dtype=torch.uint8, device="cuda" + ) + x = torch.zeros((rows, in_features), dtype=torch.float16, device="cuda") + called = [] + monkeypatch.setattr( + kernel_gguf, + "ggml_mul_mat_a8_mma", + lambda *a, **k: called.append("mma") + or torch.zeros((rows, out_features), dtype=torch.float32, device="cuda"), + ) + monkeypatch.setattr( + kernel_gguf, + "ggml_dequantize", + lambda *a, **k: called.append("dequant") + or torch.zeros((out_features, in_features), dtype=x.dtype, device="cuda"), + ) + out = layers_gguf.fused_mul_mat_gguf(x, qweight, GGML_Q4_K) + assert called == (["mma"] if expect_mma else ["dequant"]) + assert out.dtype == x.dtype + + +@pytest.mark.parametrize( + "capability,tokens,expect_mmq", + [ + ((12, 0), 16, True), + ((12, 0), 15, False), + ((8, 9), 16, False), + ((8, 9), 32, True), + ], +) +def test_moe_dispatch_branch(cuda, monkeypatch, capability, tokens, expect_mmq): + import freetoken.kernel.gguf as kernel_gguf + + monkeypatch.setattr(layers_gguf, "_device_capability", lambda i: capability) + monkeypatch.setattr(moe_gguf, "_use_mma_moe", lambda *a: False) + considered = [] + # Returning 0 makes the MMQ branch fall through to the (stubbed) vec path, + # so the sentinel records threshold crossing without running any kernel. + monkeypatch.setattr( + kernel_gguf, + "ggml_moe_get_block_size", + lambda qt: considered.append("mmq") or 0, + ) + sentinel = torch.zeros((tokens, 4)) + monkeypatch.setattr(moe_gguf, "_moe_vec_chunked", lambda *a, **k: sentinel) + x = torch.zeros((tokens, 8), dtype=torch.float16, device="cuda") + topk_ids = torch.zeros((tokens, 2), dtype=torch.int32, device="cuda") + weight = torch.zeros((4, 16), dtype=torch.uint8, device="cuda") + out = moe_gguf._moe_matmul( + x, weight, topk_ids, 2, GGML_Q4_K, rows=4, tokens=tokens, stride=16 + ) + assert (considered == ["mmq"]) is expect_mmq + assert out is sentinel + + +def test_moe_use_mma_gate(monkeypatch): + """_use_mma_moe: sm_120 + supported type + block-aligned stride only.""" + monkeypatch.setattr(layers_gguf, "_mma_mmq_ok", lambda: True) + block, type_size = BLOCK_SHAPE[GGML_Q4_K] + x = torch.zeros(4, 8) + assert moe_gguf._use_mma_moe(GGML_Q4_K, 4 * type_size, x, (12, 0)) + assert not moe_gguf._use_mma_moe(GGML_Q4_K, 4 * type_size, x, (8, 9)) + assert not moe_gguf._use_mma_moe(GGML_Q4_K, 4 * type_size + 1, x, (12, 0)) + assert not moe_gguf._use_mma_moe(2, 64, x, (12, 0)) # Q4_0: not instantiated + + +@pytest.mark.parametrize( + "tokens,broadcast,expect_mma", + [(320, True, True), (320, False, True), (319, True, False), (16385, True, False)], +) +def test_moe_dispatch_mma_branch(cuda, monkeypatch, tokens, broadcast, expect_mma): + import freetoken.kernel.gguf as kernel_gguf + + monkeypatch.setattr(layers_gguf, "_device_capability", lambda i: (12, 0)) + monkeypatch.setattr(moe_gguf, "_use_mma_moe", lambda *a: True) + top_k = 2 + called = [] + mma_out = torch.zeros(tokens * top_k, 4) + monkeypatch.setattr( + kernel_gguf, "ggml_moe_a8_mma", lambda *a, **k: called.append("mma") or mma_out + ) + vec_out = torch.zeros(tokens * top_k, 4) + monkeypatch.setattr(moe_gguf, "_moe_vec_chunked", lambda *a, **k: vec_out) + import freetoken.kernel.gguf as kg + + monkeypatch.setattr(kg, "ggml_moe_get_block_size", lambda qt: 0) + rows_x = tokens if broadcast else tokens * top_k + x = torch.zeros((rows_x, 8), dtype=torch.float16, device="cuda") + topk_ids = torch.zeros((tokens, top_k), dtype=torch.int32, device="cuda") + weight = torch.zeros((4, 16), dtype=torch.uint8, device="cuda") + out = moe_gguf._moe_matmul( + x, weight, topk_ids, top_k, GGML_Q4_K, rows=4, tokens=tokens, stride=16, + broadcast=broadcast, + ) + assert (called == ["mma"]) is expect_mma + if expect_mma: + assert out.dtype == x.dtype # fp32 kernel output cast back + else: + assert out is vec_out diff --git a/tests/kernels/test_gguf_mma.py b/tests/kernels/test_gguf_mma.py new file mode 100644 index 00000000..db17967f --- /dev/null +++ b/tests/kernels/test_gguf_mma.py @@ -0,0 +1,117 @@ +"""Numeric checks for the upstream int8-MMA MMQ extension (Q4_K/Q6_K). + +Same strategy as test_gguf_quant_types: random-but-safe packed bytes (fp16 +scale fields masked small), gguf-py's decode of the same bytes as reference. +Runs only where the dispatch would actually select the MMA path (sm_120+). +""" +from __future__ import annotations + +import numpy as np +import pytest +import torch + +if not torch.cuda.is_available(): + pytest.skip("CUDA required", allow_module_level=True) +if torch.cuda.get_device_capability() < (12, 0): + pytest.skip("int8-MMA MMQ path is sm_120-gated", allow_module_level=True) + +import gguf + +from freetoken.models.gguf.dequant import BLOCK_SHAPE, GGML_Q4_K, GGML_Q6_K + + +def _packed_rows(qtype: int, rows: int, seed: int) -> np.ndarray: + rng = np.random.default_rng(seed) + raw = rng.integers(0, 256, (rows, BLOCK_SHAPE[qtype][1]), dtype=np.uint8) + u16 = raw.view(np.uint16) + u16 &= np.uint16(0x3BFF) # clears sign, caps exponent -> |value| < 1 + return raw + + +@pytest.mark.parametrize("qtype", [GGML_Q4_K, GGML_Q6_K]) +@pytest.mark.parametrize("tokens", [7, 16, 129]) +def test_mma_matches_reference(qtype, tokens): + from freetoken.kernel.gguf import ggml_mul_mat_a8_mma + + out_features = 320 # not a multiple of 128 -> exercises the fallback kernel + block, type_size = BLOCK_SHAPE[qtype] + in_features = 2 * block + raw = _packed_rows(qtype, out_features * in_features // block, seed=qtype * 100 + tokens) + weight = torch.from_numpy(raw.reshape(out_features, -1).copy()).cuda() + ref_w = torch.from_numpy( + gguf.quants.dequantize(raw, gguf.GGMLQuantizationType(qtype)) + ).float().reshape(out_features, in_features).cuda() + + torch.manual_seed(0) + x = torch.randn(tokens, in_features, dtype=torch.float32, device="cuda") + got = ggml_mul_mat_a8_mma(weight, x, qtype, out_features) + ref = x @ ref_w.T + rel = ((got - ref).norm() / (ref.norm() + 1e-12)).item() + assert got.shape == (tokens, out_features) + assert rel < 0.02, rel + + +@pytest.mark.parametrize("qtype", [GGML_Q4_K, GGML_Q6_K]) +@pytest.mark.parametrize("broadcast", [True, False]) +def test_moe_mma_matches_reference(qtype, broadcast): + from freetoken.kernel.gguf import ggml_moe_a8_mma + + experts, out_f, tokens, top_k = 16, 320, 9, 4 + block, type_size = BLOCK_SHAPE[qtype] + in_f = 2 * block + pad = 2 * type_size # padded slots, still block-aligned + payload = out_f * (in_f // block) * type_size + stride = payload + pad + + rng = np.random.default_rng(3) + bank = torch.zeros(experts, stride, dtype=torch.uint8) + ws = [] + for e in range(experts): + raw = _packed_rows(qtype, out_f * in_f // block, seed=e) + bank[e, :payload] = torch.from_numpy(raw.reshape(-1)) + ws.append( + torch.from_numpy(gguf.quants.dequantize(raw, gguf.GGMLQuantizationType(qtype))) + .float().reshape(out_f, in_f) + ) + bank = bank.cuda() + w = torch.stack(ws).cuda() + topk_ids = torch.from_numpy( + np.stack([rng.permutation(experts)[:top_k] for _ in range(tokens)]) + ).int().cuda() + + torch.manual_seed(2) + rows_x = tokens if broadcast else tokens * top_k + x = torch.randn(rows_x, in_f, dtype=torch.float32, device="cuda") + if broadcast: + ref = torch.stack( + [x[t] @ w[topk_ids[t, k]].T for t in range(tokens) for k in range(top_k)] + ) + else: + ref = torch.stack( + [x[t * top_k + k] @ w[topk_ids[t, k]].T for t in range(tokens) for k in range(top_k)] + ) + got = ggml_moe_a8_mma(x, bank, topk_ids, top_k, qtype, out_f, tokens, stride, broadcast) + rel = ((got - ref).norm() / (ref.norm() + 1e-12)).item() + assert got.shape == (tokens * top_k, out_f) + assert rel < 0.02, rel + + +def test_mma_multiple_of_128_rows(): + from freetoken.kernel.gguf import ggml_mul_mat_a8_mma + + qtype = GGML_Q4_K + out_features = 256 # multiple of 128 -> non-fallback kernel + block, type_size = BLOCK_SHAPE[qtype] + in_features = 2 * block + raw = _packed_rows(qtype, out_features * in_features // block, seed=7) + weight = torch.from_numpy(raw.reshape(out_features, -1).copy()).cuda() + ref_w = torch.from_numpy( + gguf.quants.dequantize(raw, gguf.GGMLQuantizationType(qtype)) + ).float().reshape(out_features, in_features).cuda() + + torch.manual_seed(1) + x = torch.randn(33, in_features, dtype=torch.float32, device="cuda") + got = ggml_mul_mat_a8_mma(weight, x, qtype, out_features) + ref = x @ ref_w.T + rel = ((got - ref).norm() / (ref.norm() + 1e-12)).item() + assert rel < 0.02, rel diff --git a/tests/kernels/test_gguf_quant_types.py b/tests/kernels/test_gguf_quant_types.py new file mode 100644 index 00000000..f7612b79 --- /dev/null +++ b/tests/kernels/test_gguf_quant_types.py @@ -0,0 +1,233 @@ +"""Coverage tests for the GGUF quant types added for Laguna (Q4_K/Q5_K/IQ*). + +Strategy: gguf-py cannot *quantize* K/IQ formats, but it can *dequantize* any +packed bytes. So every test builds a weight from random-but-safe packed bytes +(fp16 scale fields masked small so the kernels' fp16 intermediates cannot +overflow -- real weights are O(1), random fp16 scales are not) and compares the +CUDA kernels against gguf-py's decode of the SAME bytes. +""" +from __future__ import annotations + +import numpy as np +import pytest +import torch +import torch.nn.functional as F + +if not torch.cuda.is_available(): + pytest.skip("CUDA required", allow_module_level=True) + +import gguf + +from freetoken.models.gguf.dequant import ( + BLOCK_SHAPE, + GGML_IQ1_S, + GGML_IQ2_S, + GGML_IQ2_XXS, + GGML_IQ3_XXS, + GGML_IQ4_XS, + GGML_Q3_K, + GGML_Q4_K, + GGML_Q5_K, + dequantize, +) + +TYPES = [ + GGML_Q3_K, GGML_Q4_K, GGML_Q5_K, + GGML_IQ1_S, GGML_IQ2_S, GGML_IQ2_XXS, GGML_IQ3_XXS, GGML_IQ4_XS, +] + + +def _packed_rows(qtype: int, rows: int, seed: int) -> np.ndarray: + """Random packed rows with every 16-bit field masked to a small positive + fp16 (exponent forced below 1.0) so any scale interpretation stays tiny and + fp16 kernel intermediates cannot overflow. Payload bits keep plenty of + entropy for the sub-block codes.""" + rng = np.random.default_rng(seed) + raw = rng.integers(0, 256, (rows, BLOCK_SHAPE[qtype][1]), dtype=np.uint8) + u16 = raw.view(np.uint16) if raw.shape[1] % 2 == 0 else None + if u16 is None: # odd row_bytes (IQ1_S is 50 -> even; guard anyway) + raw[:, 1::2] &= 0x3B + return raw + u16 &= np.uint16(0x3BFF) # clears sign, caps exponent -> |value| < 1 + return raw + + +def _reference(raw: np.ndarray, qtype: int) -> torch.Tensor: + return torch.from_numpy( + gguf.quants.dequantize(raw, gguf.GGMLQuantizationType(qtype)) + ).float() + + +def _q8_1_activations(x: torch.Tensor) -> torch.Tensor: + """Model the kernels' q8_1 activation quantization (int8 per 32-block with an + fp16 absmax/127 scale) so matmul references carry the same rounding.""" + blocks = x.float().reshape(x.shape[0], -1, 32) + scale = (blocks.abs().amax(dim=-1, keepdim=True) / 127.0).half().float() + q = torch.where(scale > 0, (blocks / scale).round().clamp(-127, 127), blocks) + return (q * scale).reshape(x.shape) + + +def _randn(shape, seed: int) -> torch.Tensor: + g = torch.Generator(device="cuda").manual_seed(seed) + return torch.randn(*shape, generator=g, device="cuda", dtype=torch.bfloat16) + + +@pytest.mark.parametrize("qtype", TYPES) +def test_python_reference_matches_gguf(qtype): + """The freetoken reference dequant (used by non-CUDA callers) agrees with + gguf-py on identical bytes.""" + raw = _packed_rows(qtype, rows=2, seed=qtype) + ours = dequantize(torch.from_numpy(raw), qtype, torch.float32).reshape(2, -1) + ref = _reference(raw, qtype).reshape(2, -1) + torch.testing.assert_close(ours, ref, rtol=1e-5, atol=1e-6) + + +@pytest.mark.parametrize("qtype", TYPES) +def test_bytes_cuda_matches_gguf_reference(qtype): + from freetoken.kernel.gguf import ggml_dequantize + + raw = _packed_rows(qtype, rows=4, seed=qtype) + packed = torch.from_numpy(raw).cuda() + got = ggml_dequantize(packed, qtype, 4, BLOCK_SHAPE[qtype][0], torch.float32).cpu() + ref = _reference(raw, qtype).reshape(4, -1) + torch.testing.assert_close(got, ref, rtol=2e-2, atol=2e-3) + + +@pytest.mark.parametrize("qtype", TYPES) +def test_mmvq_matches_linear(qtype): + from freetoken.kernel.gguf import ggml_mul_mat_vec_a8 + + block = BLOCK_SHAPE[qtype][0] + rows, cols = 8, 2 * block + raw = _packed_rows(qtype, rows=rows * (cols // block), seed=qtype + 1) + raw = np.ascontiguousarray(raw.reshape(rows, -1)) + packed = torch.from_numpy(raw).cuda() + w = _reference(raw, qtype).reshape(rows, cols).cuda() + x = _randn((1, cols), seed=qtype + 10) + got = ggml_mul_mat_vec_a8(packed, x, qtype, rows).float() + ref = F.linear(_q8_1_activations(x), w) + tol = 5e-3 * ref.abs().max().clamp(min=1.0) + assert (got.reshape(-1) - ref.reshape(-1)).abs().max() <= tol + + +@pytest.mark.parametrize("qtype", [GGML_Q3_K, GGML_Q4_K, GGML_Q5_K]) +def test_mmq_matches_linear(qtype): + from freetoken.kernel.gguf import ggml_mul_mat_a8 + + block = BLOCK_SHAPE[qtype][0] + rows, cols, batch = 8, 2 * block, 8 + raw = _packed_rows(qtype, rows=rows * (cols // block), seed=qtype + 2) + raw = np.ascontiguousarray(raw.reshape(rows, -1)) + packed = torch.from_numpy(raw).cuda() + w = _reference(raw, qtype).reshape(rows, cols).cuda() + x = _randn((batch, cols), seed=qtype + 20) + got = ggml_mul_mat_a8(packed, x, qtype, rows).float() + ref = F.linear(_q8_1_activations(x), w) + tol = 5e-3 * ref.abs().max().clamp(min=1.0) + assert (got - ref).abs().max() <= tol + + +@pytest.mark.parametrize("qtype", [GGML_Q4_K, GGML_Q5_K]) +def test_dense_gguf_prefill_uses_dequantized_cublas_result(qtype, monkeypatch): + import freetoken.layers.gguf as layers_gguf + from freetoken.kernel.gguf import ggml_dequantize + from freetoken.layers.gguf import fused_mul_mat_gguf + + # On sm_120 Q4_K/Q6_K prefill dispatches to int8-MMA MMQ instead (different + # rounding; covered by test_gguf_mma). This test validates the dequant path. + monkeypatch.setattr(layers_gguf, "_use_mma_mmq", lambda qt, cc: False) + + block = BLOCK_SHAPE[qtype][0] + rows, cols, batch = 32, 2 * block, 32 + raw = _packed_rows(qtype, rows=rows * (cols // block), seed=qtype + 70) + raw = np.ascontiguousarray(raw.reshape(rows, -1)) + packed = torch.from_numpy(raw).cuda() + x = _randn((batch, cols), seed=qtype + 71) + + got = fused_mul_mat_gguf(x, packed, qtype) + dense = ggml_dequantize(packed, qtype, rows, cols, x.dtype) + expected = x @ dense.T + torch.testing.assert_close(got, expected, rtol=0, atol=0) + + +@pytest.mark.parametrize( + "qtype", [GGML_Q3_K, GGML_Q4_K, GGML_IQ2_S, GGML_IQ2_XXS, GGML_IQ3_XXS, GGML_IQ1_S, GGML_IQ4_XS] +) +def test_moe_vec_matches_mmvq(qtype): + """moe_vec shares mmvq's vec_dot; per selected expert it must reproduce the + (reference-validated) mmvq result on that expert's rows bit-exactly.""" + from freetoken.kernel.gguf import ggml_moe_a8_vec, ggml_mul_mat_vec_a8 + + block = BLOCK_SHAPE[qtype][0] + experts, rows, cols, top_k = 4, 4, block, 2 + raw = _packed_rows(qtype, rows=experts * rows, seed=qtype + 3) + bank = np.ascontiguousarray(raw.reshape(experts, rows, -1)) + packed = torch.from_numpy(bank).cuda() + x = _randn((1, cols), seed=qtype + 30) + topk_ids = torch.tensor([[1, 3]], device="cuda", dtype=torch.int32) + out = ggml_moe_a8_vec(x, packed, topk_ids, top_k, qtype, rows, 1) + out = out.reshape(top_k, rows) + for j, e in enumerate([1, 3]): + one = torch.from_numpy(np.ascontiguousarray(bank[e])).cuda() + ref = ggml_mul_mat_vec_a8(one, x, qtype, rows).reshape(rows) + torch.testing.assert_close(out[j], ref, rtol=0.0, atol=0.0) + + +@pytest.mark.parametrize("qtype", [GGML_IQ1_S, GGML_IQ2_S, GGML_IQ3_XXS]) +def test_moe_vec_expert_stride_padded_bank(qtype): + """Mixed-quant banks store each expert's payload in the leading bytes of a + padded flat slot; expert_stride_bytes must reproduce the dense-bank result.""" + from freetoken.kernel.gguf import ggml_moe_a8_vec + + block = BLOCK_SHAPE[qtype][0] + experts, rows, cols, top_k = 4, 4, block, 2 + raw = _packed_rows(qtype, rows=experts * rows, seed=qtype + 40) + dense = torch.from_numpy(np.ascontiguousarray(raw.reshape(experts, rows, -1))).cuda() + payload = rows * raw.shape[1] // 1 # bytes per expert (row_bytes * rows) + payload = rows * dense.shape[2] + stride = payload + 64 # pad each expert slot + flat = torch.zeros(experts, stride, dtype=torch.uint8, device="cuda") + flat[:, :payload] = dense.reshape(experts, payload) + x = _randn((1, cols), seed=qtype + 41) + topk_ids = torch.tensor([[1, 3]], device="cuda", dtype=torch.int32) + ref = ggml_moe_a8_vec(x, dense, topk_ids, top_k, qtype, rows, 1) + got = ggml_moe_a8_vec(x, flat, topk_ids, top_k, qtype, rows, 1, stride) + torch.testing.assert_close(got, ref, rtol=0.0, atol=0.0) + + +def test_moe_mmq_skips_capacity_block_after_live_prefix(): + """The align buffers have spare capacity beyond ``num_tokens_post_padded``. + + A launch block can start exactly at the live-prefix boundary. It must return + before reading the intentionally invalid spare entries; otherwise the old + strict-``>`` guard uses -1 as an activation/output offset and faults. + """ + from freetoken.kernel.gguf import ggml_moe_a8 + + qtype = GGML_Q4_K + cols, rows = BLOCK_SHAPE[qtype][0], 32 + raw = _packed_rows(qtype, rows=rows, seed=91) + packed = torch.from_numpy(np.ascontiguousarray(raw.reshape(1, rows, -1))).cuda() + x = _randn((1, cols), seed=92) + # MMQ_X_Q4_K == 4. The first block is the entire live aligned prefix; + # the second block exists only because the buffers have spare capacity. + sorted_ids = torch.tensor( + [0, 1, 1, 1, -1, -1, -1, -1], dtype=torch.int32, device="cuda" + ) + expert_ids = torch.zeros((2,), dtype=torch.int32, device="cuda") + num_tokens_post_padded = torch.tensor([4], dtype=torch.int32, device="cuda") + + got = ggml_moe_a8( + x, + packed, + sorted_ids, + expert_ids, + num_tokens_post_padded, + qtype, + rows, + 1, + 1, + ) + torch.cuda.synchronize() + assert got.shape == (1, rows) + assert torch.isfinite(got).all() diff --git a/tests/kernels/test_kv_quant.py b/tests/kernels/test_kv_quant.py new file mode 100644 index 00000000..6c199d88 --- /dev/null +++ b/tests/kernels/test_kv_quant.py @@ -0,0 +1,343 @@ +from __future__ import annotations + +import pytest +import torch + +from freetoken.kvcache.quant import BLOCK, FP8_E4M3, INT4, NONE, Q8_0, resolve_kv_quant + +SPECS = [Q8_0, FP8_E4M3, INT4] +IDS = [spec.name for spec in SPECS] + +cuda_only = pytest.mark.skipif(not torch.cuda.is_available(), reason="needs CUDA") + + +def _kv(tokens: int, heads: int, dim: int, device="cuda", seed=0) -> torch.Tensor: + g = torch.Generator(device=device).manual_seed(seed) + return torch.randn(tokens, heads, dim, generator=g, device=device, dtype=torch.bfloat16) + + +def test_bytes_per_element_amortizes_the_scale(): + # 8 bits of payload + one fp16 scale per 32 elements. + assert Q8_0.bytes_per_element(torch.bfloat16) == 1.0 + 2 / 32 + assert FP8_E4M3.bytes_per_element(torch.bfloat16) == 1.0 + 2 / 32 + # Packed int4 stores two values per byte, with the same scale slab. + assert INT4.bytes_per_element(torch.bfloat16) == 0.5 + 2 / 32 + assert INT4.storage_shape((7, 4, 256)) == (7, 4, 128) + # Unquantized pools price at the compute dtype. + assert NONE.bytes_per_element(torch.bfloat16) == 2.0 + assert NONE.bytes_per_element(torch.float32) == 4.0 + + +def test_resolve_and_scale_shape(): + assert resolve_kv_quant(None) is NONE + assert resolve_kv_quant("auto") is NONE + assert resolve_kv_quant("q8_0") is Q8_0 + assert resolve_kv_quant("int4") is INT4 + assert resolve_kv_quant("q4_0") is INT4 + assert Q8_0.scale_shape((7, 4, 256)) == (7, 4, 8) + with pytest.raises(ValueError, match="not a multiple"): + Q8_0.scale_shape((7, 4, 100)) + + +@pytest.mark.parametrize("spec", SPECS, ids=IDS) +def test_reference_roundtrip_error_is_within_the_scheme_envelope(spec): + torch.manual_seed(0) + """Each scheme's round-trip error must sit inside the bound its format implies. + + int8 rounds onto a uniform grid of step ``amax/127``, so the error is at most half a + step -- except for a value at the block's extreme, which the clamp can push to a + full step once the fp16 scale rounds down. One step is the honest envelope. e4m3 + carries 3 mantissa bits, so its error is relative, up to ~2^-4 of each value, which + against the block's amax is bounded the same way. + """ + x = torch.randn(64, 4, 256, dtype=torch.float32) + q, scales = spec.quantize(x) + back = spec.dequantize(q, scales) + + blocks = x.unflatten(-1, (x.shape[-1] // BLOCK, BLOCK)) + amax = blocks.abs().amax(dim=-1, keepdim=True) + err = (back - x).unflatten(-1, (x.shape[-1] // BLOCK, BLOCK)).abs() + bound = 1.0 / spec.max_magnitude if spec.is_integer else 1.0 / 2**4 + assert (err <= amax * bound + 1e-6).all() + # And the typical error should sit well under the worst case, not at it. + assert err.mean() <= amax.mean() * bound * 0.5 + + +def test_int4_matches_ggml_q4_0_codes_in_element_order(): + x = torch.tensor([-8.0, -7.0, -1.0, 0.0, 1.0, 6.0, 7.0, 0.0] * 4).view(1, 1, BLOCK) + packed, scales = INT4.quantize(x) + + # The negative extreme selects scale=1; code = value + 8. Low nibble is the even + # logical element and high nibble is the odd logical element. + assert scales.item() == 1.0 + assert packed[0, 0, :4].tolist() == [0x10, 0x87, 0xE9, 0x8F] + torch.testing.assert_close(INT4.dequantize(packed, scales), x) + + +def test_int8_beats_fp8_on_a_flat_block_and_loses_on_a_spiky_one(): + """The tradeoff the scheme choice turns on, pinned as a test. + + Within a block, int8 spends its codes uniformly and fp8 spends them + logarithmically. So a block of similar magnitudes favours int8, and a block where + one outlier dwarfs the rest favours fp8 -- which is exactly why the block is 32 + elements and not a whole head. + """ + flat = torch.full((1, 1, BLOCK), 1.0) + flat[..., ::2] = 0.9 + spiky = torch.full((1, 1, BLOCK), 0.01) + spiky[..., 0] = 100.0 + + def rel_err(spec, x): + back = spec.dequantize(*spec.quantize(x)) + return ((back - x).abs() / x.abs()).mean().item() + + assert rel_err(Q8_0, flat) < rel_err(FP8_E4M3, flat) + assert rel_err(FP8_E4M3, spiky) < rel_err(Q8_0, spiky) + + +@cuda_only +@pytest.mark.parametrize("spec", SPECS, ids=IDS) +@pytest.mark.parametrize("head_dim", [256, 512]) +def test_store_kernel_matches_the_reference_quantizer(spec, head_dim): + from freetoken.kernel.triton.kv_quant import store_kv_quant + + tokens, heads, slots = 37, 3, 64 + k = _kv(tokens, heads, head_dim, seed=1) + v = _kv(tokens, heads, head_dim, seed=2) + # Scatter to non-contiguous slots: the kernel must honour the index indirection. + indices = torch.randperm(slots, device="cuda")[:tokens].to(torch.int32) + + kc = torch.zeros(slots, heads, head_dim // spec.elements_per_byte, device="cuda", dtype=spec.storage_dtype) + vc = torch.zeros_like(kc) + ks = torch.zeros(slots, heads, head_dim // BLOCK, device="cuda", dtype=torch.float16) + vs = torch.zeros_like(ks) + + store_kv_quant(kc, ks, vc, vs, indices, k, v, spec) + + idx = indices.to(torch.long) + for src, cache, scale in ((k, kc, ks), (v, vc, vs)): + want_q, want_s = spec.quantize(src.float()) + torch.testing.assert_close(scale[idx].float(), want_s.float(), rtol=0, atol=0) + # Compare dequantized values: int8 is exact, fp8 codes compare through float. + got = spec.dequantize(cache[idx].float(), scale[idx]) + torch.testing.assert_close(got, spec.dequantize(want_q.float(), want_s), rtol=0, atol=0) + + +@cuda_only +@pytest.mark.parametrize("spec", SPECS, ids=IDS) +def test_store_kernel_leaves_untouched_slots_alone(spec): + from freetoken.kernel.triton.kv_quant import store_kv_quant + + heads, head_dim, slots = 2, 256, 16 + k = _kv(4, heads, head_dim, seed=3) + v = _kv(4, heads, head_dim, seed=4) + indices = torch.tensor([1, 3, 5, 7], device="cuda", dtype=torch.int32) + + kc = torch.zeros(slots, heads, head_dim // spec.elements_per_byte, device="cuda", dtype=spec.storage_dtype) + vc = torch.zeros_like(kc) + ks = torch.zeros(slots, heads, head_dim // BLOCK, device="cuda", dtype=torch.float16) + vs = torch.zeros_like(ks) + store_kv_quant(kc, ks, vc, vs, indices, k, v, spec) + + untouched = [s for s in range(slots) if s not in {1, 3, 5, 7}] + assert (kc[untouched].float() == 0).all() + assert (ks[untouched] == 0).all() + + +@cuda_only +@pytest.mark.parametrize("spec", SPECS, ids=IDS) +def test_store_kernel_handles_an_all_zero_head(spec): + """A zero block has no max to scale by; it must store zeros, not NaNs.""" + from freetoken.kernel.triton.kv_quant import store_kv_quant + + heads, head_dim = 2, 256 + k = torch.zeros(1, heads, head_dim, device="cuda", dtype=torch.bfloat16) + v = torch.zeros_like(k) + indices = torch.zeros(1, device="cuda", dtype=torch.int32) + + kc = torch.empty(4, heads, head_dim // spec.elements_per_byte, device="cuda", dtype=spec.storage_dtype) + vc = torch.empty_like(kc) + ks = torch.empty(4, heads, head_dim // BLOCK, device="cuda", dtype=torch.float16) + vs = torch.empty_like(ks) + store_kv_quant(kc, ks, vc, vs, indices, k, v, spec) + + # Zero must dequantize back to zero. int4 encodes 0 as the offset nibble (0x88), + # not 0x00, so compare the DEQUANTIZED values, not the raw packed bytes. + assert (spec.dequantize(kc[0].float(), ks[0]).abs() == 0).all() + assert torch.isfinite(ks[0]).all() and (ks[0] > 0).all() + + +# -------------------------------------------------------------------------------------- +# Attention over a quantized pool. +# +# The gate is equivalence against the bf16 kernel fed the SAME dequantized values: that +# isolates "did the dequant path compute attention correctly" from "how much does 8-bit +# storage cost", which is a separate, looser assertion below. +# -------------------------------------------------------------------------------------- + + +def _quantized_pool(spec, k_bf16, v_bf16): + """Store bf16 K/V into a quantized pool; return (kq, ks, vq, vs, k_deq, v_deq).""" + from freetoken.kernel.triton.kv_quant import store_kv_quant + + slots, heads, dim = k_bf16.shape + epb = spec.elements_per_byte + kq = torch.zeros(slots, heads, dim // epb, device="cuda", dtype=spec.storage_dtype) + vq = torch.zeros_like(kq) + ks = torch.zeros(slots, heads, dim // BLOCK, device="cuda", dtype=torch.float16) + vs = torch.zeros_like(ks) + indices = torch.arange(slots, device="cuda", dtype=torch.int32) + store_kv_quant(kq, ks, vq, vs, indices, k_bf16, v_bf16, spec) + # What the attention kernel will effectively see, in the dtype it dequantizes into. + k_deq = spec.dequantize(kq.float(), ks).to(torch.bfloat16).reshape(slots, heads, dim) + v_deq = spec.dequantize(vq.float(), vs).to(torch.bfloat16).reshape(slots, heads, dim) + return kq, ks, vq, vs, k_deq, v_deq + + +@cuda_only +@pytest.mark.parametrize("spec", SPECS, ids=IDS) +@pytest.mark.parametrize("head_dim", [256, 512]) +def test_paged_attention_over_quantized_pool(spec, head_dim): + from freetoken.kernel.triton.attention import paged_attention + + slots, q_heads, kv_heads = 96, 8, 2 + q = _kv(6, q_heads, head_dim, seed=5) + k = _kv(slots, kv_heads, head_dim, seed=6) + v = _kv(slots, kv_heads, head_dim, seed=7) + kq, ks, vq, vs, k_deq, v_deq = _quantized_pool(spec, k, v) + + indptr = torch.tensor([0, 40, 96], device="cuda", dtype=torch.int32) + indices = torch.arange(slots, device="cuda", dtype=torch.int32) + q_to_req = torch.tensor([0, 0, 0, 1, 1, 1], device="cuda", dtype=torch.int32) + q_pos = torch.tensor([10, 25, 39, 5, 30, 55], device="cuda", dtype=torch.int32) + kw = dict(indptr=indptr, indices=indices, q_to_req=q_to_req, q_positions=q_pos, + sm_scale=head_dim**-0.5) + + got = paged_attention(q=q, k_cache=kq, v_cache=vq, k_scale=ks, v_scale=vs, **kw) + want = paged_attention(q=q, k_cache=k_deq, v_cache=v_deq, **kw) + torch.testing.assert_close(got, want, rtol=2e-2, atol=2e-2) + + +@cuda_only +@pytest.mark.parametrize("spec", SPECS, ids=IDS) +@pytest.mark.parametrize("head_dim", [256, 512]) +def test_decode_attention_over_quantized_pool(spec, head_dim): + from freetoken.kernel.triton.attention import decode_paged_attention + + slots, q_heads, kv_heads, batch = 128, 8, 2, 3 + q = _kv(batch, q_heads, head_dim, seed=8) + k = _kv(slots, kv_heads, head_dim, seed=9) + v = _kv(slots, kv_heads, head_dim, seed=10) + kq, ks, vq, vs, k_deq, v_deq = _quantized_pool(spec, k, v) + + indptr = torch.tensor([0, 40, 90, 128], device="cuda", dtype=torch.int32) + indices = torch.arange(slots, device="cuda", dtype=torch.int32) + q_pos = torch.tensor([39, 49, 37], device="cuda", dtype=torch.int32) + splits = 4 + logits = torch.zeros(batch, q_heads, splits, head_dim, device="cuda", dtype=torch.float32) + lse = torch.zeros(batch, q_heads, splits, device="cuda", dtype=torch.float32) + nsplits = torch.full((batch,), splits, device="cuda", dtype=torch.int32) + kw = dict(indptr=indptr, indices=indices, q_positions=q_pos, attn_logits=logits, + attn_lse=lse, num_kv_splits=nsplits, max_kv_splits=splits, + sm_scale=head_dim**-0.5) + + got = decode_paged_attention(q=q, k_cache=kq, v_cache=vq, k_scale=ks, v_scale=vs, **kw) + want = decode_paged_attention(q=q, k_cache=k_deq, v_cache=v_deq, **kw) + torch.testing.assert_close(got, want, rtol=2e-2, atol=2e-2) + + +@cuda_only +def test_ornith_q4_tuned_decode_matches_dequantized_oracle(): + """Exercise the exact launch selected in production for Ornith. + + The former BLOCK_N=16 tuning passed generic 8-head tests but silently corrupted + packed Q4 attention at Ornith's 16-query-head/2-KV-head geometry. + """ + from freetoken.kernel.triton.attention import decode_paged_attention + + slots, q_heads, kv_heads, head_dim = 67, 16, 2, 256 + q = _kv(1, q_heads, head_dim, seed=81) + k = _kv(slots, kv_heads, head_dim, seed=82) + v = _kv(slots, kv_heads, head_dim, seed=83) + kq, ks, vq, vs, k_deq, v_deq = _quantized_pool(INT4, k, v) + indptr = torch.tensor([0, slots], device="cuda", dtype=torch.int32) + indices = torch.arange(slots, device="cuda", dtype=torch.int32) + q_pos = torch.tensor([slots - 1], device="cuda", dtype=torch.int32) + + def run(k_cache, v_cache, splits, k_scale=None, v_scale=None): + logits = torch.empty(1, q_heads, splits, head_dim, device="cuda", dtype=torch.float32) + lse = torch.empty(1, q_heads, splits, device="cuda", dtype=torch.float32) + nsplits = torch.full((1,), splits, device="cuda", dtype=torch.int32) + return decode_paged_attention( + q, k_cache, v_cache, indptr, indices, q_pos, logits, lse, nsplits, + splits, head_dim**-0.5, k_scale=k_scale, v_scale=v_scale, + ) + + # Exercise both architecture-specific production choices regardless of which + # GPU runs the suite; the launch geometry, not the device name, owns correctness. + tuned_splits = 64 + got = run(kq, vq, tuned_splits, ks, vs) + want = run(k_deq, v_deq, 8) + torch.testing.assert_close(got, want, rtol=2e-2, atol=2e-2) + + +@cuda_only +@pytest.mark.parametrize("spec", SPECS, ids=IDS) +@pytest.mark.parametrize("split", [False, True], ids=["fused", "split"]) +def test_extend_attention_over_quantized_pool(spec, split): + """Both extend paths. The split kernel reads the freshly-computed K/V in bf16 and + only the prefix from the quantized pool, so it exercises the mixed case.""" + from freetoken.kernel.triton.attention import extend_paged_attention + + head_dim, slots, q_heads, kv_heads = 256, 64, 8, 2 + q_len, prefix = 8, 24 + q = _kv(q_len, q_heads, head_dim, seed=11) + k = _kv(slots, kv_heads, head_dim, seed=12) + v = _kv(slots, kv_heads, head_dim, seed=13) + kq, ks, vq, vs, k_deq, v_deq = _quantized_pool(spec, k, v) + + qo_indptr = torch.tensor([0, q_len], device="cuda", dtype=torch.int32) + kv_indptr = torch.tensor([0, prefix + q_len], device="cuda", dtype=torch.int32) + kv_indices = torch.arange(prefix + q_len, device="cuda", dtype=torch.int32) + prefix_lens = torch.tensor([prefix], device="cuda", dtype=torch.int32) + extend = {} + if split: + extend = dict(k_extend=_kv(q_len, kv_heads, head_dim, seed=14), + v_extend=_kv(q_len, kv_heads, head_dim, seed=15)) + kw = dict(qo_indptr=qo_indptr, kv_indptr=kv_indptr, kv_indices=kv_indices, + prefix_lens=prefix_lens, max_q_len=q_len, sm_scale=head_dim**-0.5, **extend) + + got = extend_paged_attention(q=q, k_cache=kq, v_cache=vq, k_scale=ks, v_scale=vs, **kw) + want = extend_paged_attention(q=q, k_cache=k_deq, v_cache=v_deq, **kw) + torch.testing.assert_close(got, want, rtol=2e-2, atol=2e-2) + + +@cuda_only +@pytest.mark.parametrize("spec", SPECS, ids=IDS) +def test_quantized_attention_tracks_the_bf16_pool(spec): + """The end-to-end cost of 8-bit storage: attention over a quantized pool against + attention over the original bf16 one. This is the number that matters for quality, + and it is looser than the kernel-equivalence gate above by construction.""" + from freetoken.kernel.triton.attention import paged_attention + + head_dim, slots, q_heads, kv_heads = 256, 128, 8, 2 + q = _kv(4, q_heads, head_dim, seed=16) + k = _kv(slots, kv_heads, head_dim, seed=17) + v = _kv(slots, kv_heads, head_dim, seed=18) + kq, ks, vq, vs, _, _ = _quantized_pool(spec, k, v) + + indptr = torch.tensor([0, slots], device="cuda", dtype=torch.int32) + indices = torch.arange(slots, device="cuda", dtype=torch.int32) + q_to_req = torch.zeros(4, device="cuda", dtype=torch.int32) + q_pos = torch.tensor([20, 60, 100, 127], device="cuda", dtype=torch.int32) + kw = dict(indptr=indptr, indices=indices, q_to_req=q_to_req, q_positions=q_pos, + sm_scale=head_dim**-0.5) + + got = paged_attention(q=q, k_cache=kq, v_cache=vq, k_scale=ks, v_scale=vs, **kw) + ref = paged_attention(q=q, k_cache=k, v_cache=v, **kw) + rel = ((got.float() - ref.float()).norm() / ref.float().norm()).item() + # 8-bit ~1% here; int4's 4-bit mantissa is inherently ~7x coarser, so it gets a + # looser (still meaningful) bound. The strict per-kernel equivalence is pinned by + # the tests above; this only whats the TOTAL storage cost. + bound = 0.16 if spec.packed else 0.05 + assert rel < bound, f"{spec.name}: relative error {rel:.4f} vs bf16 pool (bound {bound})" diff --git a/tests/kernels/test_triton_attention.py b/tests/kernels/test_triton_attention.py index 6f4afca9..8d8e4e3e 100644 --- a/tests/kernels/test_triton_attention.py +++ b/tests/kernels/test_triton_attention.py @@ -314,6 +314,31 @@ def test_decode_triton_attention_matches_reference( torch.testing.assert_close(actual.float(), expected.float(), atol=2e-2, rtol=2e-2) +def test_decode_launch_config_selects_ornith_quantized_tuning_only(): + from freetoken.kernel.triton.attention import decode_launch_config + + assert decode_launch_config( + quant_name="int4", head_dim=256, num_q_heads=16, num_kv_heads=2, + compute_capability=(8, 9), + ) == (32, 32, 4) + assert decode_launch_config( + quant_name="int4", head_dim=256, num_q_heads=16, num_kv_heads=2, + compute_capability=(12, 0), + ) == (64, 64, 8) + assert decode_launch_config( + quant_name="q8_0", head_dim=256, num_q_heads=16, num_kv_heads=2 + ) == (64, 64, 4) + assert decode_launch_config( + quant_name="quant8", head_dim=256, num_q_heads=16, num_kv_heads=2 + ) == (64, 64, 4) + assert decode_launch_config( + quant_name=None, head_dim=256, num_q_heads=16, num_kv_heads=2 + ) == (8, 32, 4) + assert decode_launch_config( + quant_name="int4", head_dim=256, num_q_heads=24, num_kv_heads=4 + ) == (8, 32, 4) + + @pytest.mark.skipif(not torch.cuda.is_available(), reason="Triton attention needs CUDA") @pytest.mark.parametrize(("num_q_heads", "num_kv_heads"), [(24, 4), (20, 4), (28, 4)]) def test_decode_triton_attention_non_pow2_group(num_q_heads: int, num_kv_heads: int): diff --git a/tests/kvcache/test_kv_quant_pool.py b/tests/kvcache/test_kv_quant_pool.py new file mode 100644 index 00000000..95a8ffd4 --- /dev/null +++ b/tests/kvcache/test_kv_quant_pool.py @@ -0,0 +1,232 @@ +"""KV pools backed by compact storage: allocation, store/read-back, cost, rebuild.""" + +from __future__ import annotations + +import pytest +import torch + +from freetoken.kvcache.quant import BLOCK, FP8_E4M3, INT4, NONE, Q8_0 + +from .test_hybrid_swa_kv_cache import _kv_group_specs, _patch_tp + +SPECS = [Q8_0, FP8_E4M3, INT4] +IDS = [spec.name for spec in SPECS] + +# Measured relative L2 error of a round trip through each scheme on Gaussian KV with a +# 32-element block: q8_0 ~0.005, fp8_e4m3 ~0.024, int4 ~0.09. q8_0 remains the quality +# default; int4 trades accuracy for roughly 89% more capacity than the 8-bit formats. +MAX_REL_ERR = {Q8_0.name: 0.01, FP8_E4M3.name: 0.03, INT4.name: 0.12} + +cuda_only = pytest.mark.skipif(not torch.cuda.is_available(), reason="needs CUDA") + + +def _swa_pool(quant, device="cuda", num_full_pages=64, num_swa_tokens=32): + from freetoken.kvcache.hybrid_swa_pool import HybridSWAKVCache + + return HybridSWAKVCache( + groups=_kv_group_specs(), + num_layers=6, + num_full_pages=num_full_pages, + page_size=1, + num_swa_tokens=num_swa_tokens, + device=torch.device(device), + dtype=torch.bfloat16, + quant=quant, + ) + + +def _mha_pool(quant, device="cuda", num_pages=64): + from freetoken.kvcache.mha_pool import MHAKVCache + + return MHAKVCache( + num_kv_heads=2, + num_layers=4, + head_dim=256, + num_pages=num_pages, + page_size=1, + dtype=torch.bfloat16, + device=torch.device(device), + quant=quant, + ) + + +@cuda_only +@pytest.mark.parametrize("spec", SPECS, ids=IDS) +def test_swa_pool_allocates_compact_slabs_and_matching_scales(monkeypatch, spec): + _patch_tp(monkeypatch) + pool = _swa_pool(spec) + + # Layer 0 is SWA (head_dim 256, 8 kv heads), layer 2 full (head_dim 512, 2 heads). + for layer, head_dim, heads in ((0, 256, 8), (2, 512, 2)): + k = pool.k_cache(layer) + s = pool.k_scale(layer) + assert k.dtype == spec.storage_dtype + assert s.dtype == torch.float16 + assert k.shape[-2:] == (heads, head_dim // spec.elements_per_byte) + assert s.shape[-2:] == (heads, head_dim // BLOCK) + assert s.shape[:-1] == k.shape[:-1] + + +@cuda_only +def test_unquantized_pool_keeps_bf16_and_has_no_scales(monkeypatch): + _patch_tp(monkeypatch) + pool = _swa_pool(NONE) + assert pool.k_cache(0).dtype == torch.bfloat16 + assert pool.k_scale(0) is None + assert pool.v_scale(0) is None + assert not pool.quant.enabled + + +@cuda_only +@pytest.mark.parametrize("spec", SPECS, ids=IDS) +@pytest.mark.parametrize("layer", [0, 2], ids=["swa", "full"]) +def test_store_kv_round_trips_through_the_quantized_pool(monkeypatch, spec, layer): + _patch_tp(monkeypatch) + pool = _swa_pool(spec) + heads, storage_head_dim = pool.k_cache(layer).shape[-2:] + head_dim = storage_head_dim * spec.elements_per_byte + + tokens = 8 + g = torch.Generator(device="cuda").manual_seed(0) + k = torch.randn(tokens, heads, head_dim, generator=g, device="cuda", dtype=torch.bfloat16) + v = torch.randn(tokens, heads, head_dim, generator=g, device="cuda", dtype=torch.bfloat16) + out_loc = torch.arange(1, tokens + 1, device="cuda", dtype=torch.int32) + if pool.is_swa_layer(layer): + pool.alloc_swa(out_loc) + + pool.store_kv(k, v, out_loc, layer) + + slots = ( + pool.translate_loc_from_full_to_swa(out_loc) if pool.is_swa_layer(layer) else out_loc + ).to(torch.long) + got = spec.dequantize( + pool.k_cache(layer).view(-1, heads, storage_head_dim)[slots].float(), + pool.k_scale(layer).view(-1, heads, head_dim // BLOCK)[slots], + ) + # Storing is lossy by construction; what must hold is that it round-trips to within + # the scheme's envelope, not that it is exact. + rel = ((got - k.float()).norm() / k.float().norm()).item() + assert rel < MAX_REL_ERR[spec.name], ( + f"{spec.name} layer {layer}: relative round-trip error {rel:.4f}" + ) + + +@cuda_only +@pytest.mark.parametrize("spec", SPECS, ids=IDS) +def test_unit_bytes_counts_the_scale_slab(monkeypatch, spec): + """Budgeting must include the compact payload and per-block scales.""" + _patch_tp(monkeypatch) + quantized = _swa_pool(spec) + plain = _swa_pool(NONE) + + q_full, q_swa = quantized.unit_bytes() + p_full, p_swa = plain.unit_bytes() + for q, p in ((q_full, p_full), (q_swa, p_swa)): + assert q == pytest.approx(p * (spec.bytes_per_element(torch.bfloat16) / 2.0), rel=1e-3) + + +@cuda_only +@pytest.mark.parametrize("spec", SPECS, ids=IDS) +def test_rebuild_reallocates_scales_and_keeps_identity(monkeypatch, spec): + _patch_tp(monkeypatch) + pool = _swa_pool(spec, num_full_pages=64, num_swa_tokens=32) + before = id(pool) + + pool.rebuild(num_full_pages=128, num_swa_tokens=64) + + assert id(pool) == before, "rebuild must preserve object identity" + for layer in (0, 2): + k, s = pool.k_cache(layer), pool.k_scale(layer) + assert k.dtype == spec.storage_dtype + assert s is not None and s.dtype == torch.float16 + assert s.shape[:-1] == k.shape[:-1] + assert s.shape[-1] == k.shape[-1] * spec.elements_per_byte // BLOCK + assert pool.k_cache(2).shape[0] == 128 + assert pool.k_cache(0).shape[0] == 64 + + +@cuda_only +@pytest.mark.parametrize("spec", SPECS, ids=IDS) +def test_mha_pool_quantizes_and_round_trips(monkeypatch, spec): + from freetoken.distributed.info import DistributedInfo + + monkeypatch.setattr( + "freetoken.kvcache.mha_pool.get_tp_info", lambda: DistributedInfo(rank=0, size=1) + ) + pool = _mha_pool(spec) + assert pool.k_cache(0).dtype == spec.storage_dtype + assert pool.k_cache(0).shape[-1] == 256 // spec.elements_per_byte + assert pool.k_scale(0).shape[-1] == 256 // BLOCK + + g = torch.Generator(device="cuda").manual_seed(1) + k = torch.randn(6, 2, 256, generator=g, device="cuda", dtype=torch.bfloat16) + v = torch.randn(6, 2, 256, generator=g, device="cuda", dtype=torch.bfloat16) + out_loc = torch.arange(1, 7, device="cuda", dtype=torch.int32) + pool.store_kv(k, v, out_loc, 0) + + idx = out_loc.to(torch.long) + got = spec.dequantize( + pool.k_cache(0).view(-1, 2, 256 // spec.elements_per_byte)[idx].float(), + pool.k_scale(0).view(-1, 2, 256 // BLOCK)[idx], + ) + rel = ((got - k.float()).norm() / k.float().norm()).item() + assert rel < MAX_REL_ERR[spec.name] + + pool.rebuild(32) + assert pool.k_cache(0).shape[0] == 32 + assert pool.k_scale(0).shape[0] == 32 + + +def test_cost_model_prices_the_quantized_pool_below_bf16(monkeypatch): + """The compact formats reduce the same logical geometry's memory footprint.""" + from freetoken.kvcache.base import spec_kv_bytes_per_token + from freetoken.distributed.info import DistributedInfo + from types import SimpleNamespace + + spec = _kv_group_specs()[0] # full group: 2 kv heads, head_dim 512, 2 layers + tp = DistributedInfo(rank=0, size=1) + + def cfg(quant): + return SimpleNamespace(tp_info=tp, dtype=torch.bfloat16, kv_quant=quant) + + plain = spec_kv_bytes_per_token(spec, cfg(NONE)) + quantized = spec_kv_bytes_per_token(spec, cfg(Q8_0)) + packed = spec_kv_bytes_per_token(spec, cfg(INT4)) + assert quantized == pytest.approx(plain * (Q8_0.bytes_per_element(torch.bfloat16) / 2.0), rel=1e-3) + assert packed == pytest.approx(plain * (INT4.bytes_per_element(torch.bfloat16) / 2.0), rel=1e-3) + assert packed < quantized + # A config with no kv_quant attribute at all must price as bf16 (back-compat with + # every caller that predates the flag). + legacy = spec_kv_bytes_per_token(spec, SimpleNamespace(tp_info=tp, dtype=torch.bfloat16)) + assert legacy == plain + + +@cuda_only +def test_q8_0_stores_kv_more_accurately_than_fp8(monkeypatch): + """Why q8_0 is the default, measured rather than argued. + + Both schemes cost the same bytes and the same kernel work here, so the choice is + purely numerical. On a 32-element block int8's uniform grid beats e4m3's 3-bit + mantissa by several times -- fp8's advantage only shows up with a whole-head scale, + where one outlier would crush everything else, or on kernels with native fp8 support + (which the SWA path cannot use anyway). + """ + _patch_tp(monkeypatch) + g = torch.Generator(device="cuda").manual_seed(7) + k = torch.randn(16, 8, 256, generator=g, device="cuda", dtype=torch.bfloat16) + v = torch.randn(16, 8, 256, generator=g, device="cuda", dtype=torch.bfloat16) + out_loc = torch.arange(1, 17, device="cuda", dtype=torch.int32) + + errs = {} + for spec in SPECS: + pool = _swa_pool(spec, num_swa_tokens=64) + pool.alloc_swa(out_loc) + pool.store_kv(k, v, out_loc, 0) + slots = pool.translate_loc_from_full_to_swa(out_loc).to(torch.long) + got = spec.dequantize( + pool.k_cache(0).view(-1, 8, 256 // spec.elements_per_byte)[slots].float(), + pool.k_scale(0).view(-1, 8, 256 // BLOCK)[slots], + ) + errs[spec.name] = ((got - k.float()).norm() / k.float().norm()).item() + + assert errs[Q8_0.name] < errs[FP8_E4M3.name] / 2, errs diff --git a/tests/models/test_gguf_tokenizer.py b/tests/models/test_gguf_tokenizer.py new file mode 100644 index 00000000..f1f26799 --- /dev/null +++ b/tests/models/test_gguf_tokenizer.py @@ -0,0 +1,47 @@ +from __future__ import annotations + + +def _tiny_fast_tokenizer(): + from tokenizers import Tokenizer + from tokenizers.models import BPE + from transformers import PreTrainedTokenizerFast + + # With no merge rule the base BPE spells as three tokens even though + # the complete string has an assigned vocabulary id. GGUF USER_DEFINED + # registration must make the complete string win atomically. + backend = Tokenizer( + BPE( + vocab={"<": 0, "think": 1, ">": 2, "": 3, "": 4}, + merges=[], + unk_token=None, + ) + ) + return PreTrainedTokenizerFast(tokenizer_object=backend) + + +def test_registers_user_defined_tokens_atomically_without_hiding_them(): + from freetoken.models.gguf.tokenizer import _register_gguf_added_tokens + + tokenizer = _tiny_fast_tokenizer() + assert tokenizer.encode("", add_special_tokens=False) != [3] + + _register_gguf_added_tokens( + tokenizer, + ["<", "think", ">", "", ""], + [1, 1, 1, 4, 3], + ) + + assert tokenizer.encode("", add_special_tokens=False) == [3] + assert tokenizer.decode([3], skip_special_tokens=True) == "" + assert tokenizer.encode("", add_special_tokens=False) == [4] + assert tokenizer.decode([4], skip_special_tokens=True) == "" + + +def test_ignores_missing_or_malformed_token_type_table(): + from freetoken.models.gguf.tokenizer import _register_gguf_added_tokens + + tokenizer = _tiny_fast_tokenizer() + before = tokenizer.encode("", add_special_tokens=False) + _register_gguf_added_tokens(tokenizer, [""], None) + _register_gguf_added_tokens(tokenizer, [""], [4, 4]) + assert tokenizer.encode("", add_special_tokens=False) == before diff --git a/tests/models/test_laguna_config.py b/tests/models/test_laguna_config.py new file mode 100644 index 00000000..adf102d0 --- /dev/null +++ b/tests/models/test_laguna_config.py @@ -0,0 +1,169 @@ +from __future__ import annotations + +from pathlib import Path +from types import SimpleNamespace + +import pytest + +FIXTURE = Path(__file__).resolve().parent.parent / "fixtures" / "laguna-s-2.1-metadata.gguf" + + +def _safetensors_config(): + layers = 48 + return SimpleNamespace( + architectures=["LagunaForCausalLM"], + num_hidden_layers=layers, + head_dim=128, + max_position_embeddings=1_048_576, + num_attention_heads_per_layer=[48 if i % 4 == 0 else 72 for i in range(layers)], + layer_types=[ + "full_attention" if i % 4 == 0 else "sliding_attention" + for i in range(layers) + ], + rope_parameters={ + "full_attention": { + "rope_theta": 500_000.0, + "rope_type": "yarn", + "factor": 128.0, + "original_max_position_embeddings": 8192, + "beta_slow": 1.0, + "beta_fast": 32.0, + "attention_factor": 1.4852030263919618, + "partial_rotary_factor": 0.5, + }, + "sliding_attention": { + "rope_type": "default", + "rope_theta": 10_000.0, + "partial_rotary_factor": 1.0, + }, + }, + mlp_only_layers=[0], + num_key_value_heads=8, + hidden_size=3072, + vocab_size=100352, + intermediate_size=12288, + rms_norm_eps=1e-6, + tie_word_embeddings=False, + num_experts=256, + num_experts_per_tok=10, + moe_intermediate_size=1024, + shared_expert_intermediate_size=1024, + norm_topk_prob=True, + moe_routed_scaling_factor=2.5, + sliding_window=512, + quantization_config={ + "quant_method": "compressed-tensors", + "config_groups": { + "group_0": { + "targets": [ + r"re:.*layers\.\d+\..*(gate_proj|up_proj|down_proj)$" + ], + "weights": { + "num_bits": 4, + "type": "int", + "group_size": 32, + "strategy": "group", + "symmetric": True, + }, + } + }, + "ignore": [ + r"re:^model\.layers\.(?:40|41|42|43|44|45|46|47)\.mlp\.experts\.[0-9]+\.(?:gate_proj|up_proj|down_proj)$" + ], + }, + ) + + +def test_laguna_config_parse_and_attention_groups(): + from freetoken.models.gguf.config import build_gguf_shim + from freetoken.models.laguna.gguf import parse_gguf_config + + cfg = parse_gguf_config(build_gguf_shim(str(FIXTURE))) + + assert cfg.num_layers == 48 + assert cfg.num_qo_heads == 72 + assert all(v == 48 for v in cfg.num_qo_heads_per_layer[0::4]) + assert all(v == 72 for i, v in enumerate(cfg.num_qo_heads_per_layer) if i % 4 != 0) + assert cfg.qo_heads(0) == 48 + assert cfg.qo_heads(1) == 72 + assert cfg.num_kv_heads == 8 + assert cfg.head_dim == 128 + assert cfg.hidden_size == 3072 + assert cfg.vocab_size == 100352 + assert cfg.first_k_dense_replace == 1 + assert cfg.num_experts == 256 + assert cfg.num_experts_per_tok == 10 + assert cfg.moe_intermediate_size == 1024 + assert cfg.shared_expert_intermediate_size == 1024 + assert cfg.n_shared_experts == 1 + assert cfg.routed_scaling_factor == 2.5 + assert cfg.norm_topk_prob is True + assert cfg.tie_word_embeddings is False + assert cfg.model_type == "laguna" + assert cfg.moe_enabled is True + assert cfg.use_qk_norm is True + assert cfg.rms_norm_eps == pytest.approx(1e-6) + assert cfg.rotary_config.max_position == 262144 + + assert len(cfg.attention_groups) == 2 + full = cfg.attention_groups[0] + swa = cfg.attention_groups[1] + + assert tuple(full.layer_ids) == tuple(range(0, 48, 4)) + assert tuple(swa.layer_ids) == tuple(i for i in range(0, 48) if i not in set(full.layer_ids)) + assert swa.sliding_window == 512 + + assert full.rotary_config.rotary_dim == 64 + assert full.rotary_config.base == 500000.0 + assert full.rotary_config.scaling is not None + assert full.rotary_config.scaling["rope_type"] == "yarn" + assert full.rotary_config.scaling["factor"] == 32.0 + assert full.rotary_config.scaling["attention_factor"] == 1.0 + assert full.rotary_config.scaling["beta_fast"] == 32.0 + assert full.rotary_config.scaling["beta_slow"] == 1.0 + assert full.rotary_config.scaling["original_max_position_embeddings"] == 8192 + + assert swa.rotary_config.rotary_dim == 128 + assert swa.rotary_config.base == 10000.0 + assert swa.rotary_config.scaling is None + + assert cfg.gguf_embed_quant is None + + +def test_laguna_gguf_arch_registry_map(): + from freetoken.models import register + from freetoken.models.gguf.config import GGUF_ARCH_TO_REGISTRY + + assert GGUF_ARCH_TO_REGISTRY["laguna"] == "LagunaGGUFForCausalLM" + assert "LagunaGGUFForCausalLM" in register._MODEL_REGISTRY + + +def test_laguna_safetensors_config_and_mixed_expert_types(): + from freetoken.models import register + from freetoken.models.gguf.dequant import GGML_BF16, GGML_Q4_0 + from freetoken.models.laguna.config import parse_config + + cfg = parse_config(_safetensors_config()) + assert cfg.architectures == ["LagunaForCausalLM"] + assert cfg.num_layers == 48 and cfg.num_moe_layers == 47 + assert cfg.rotary_config.max_position == 1_048_576 + assert cfg.rotary_config.rotary_dim == 64 + assert cfg.attention_groups[1].sliding_window == 512 + assert cfg.expert_quant == cfg.moe_weight_format == "laguna_int4" + assert cfg.gguf_expert_types == ( + ((GGML_Q4_0, GGML_Q4_0),) * 39 + + ((GGML_BF16, GGML_BF16),) * 8 + ) + assert "LagunaForCausalLM" in register._MODEL_REGISTRY + + +def test_laguna_tokenizer_and_eos_tokens(): + from freetoken.models.gguf.tokenizer import gguf_eos_token_ids, load_gguf_tokenizer + + tok = load_gguf_tokenizer(str(FIXTURE)) + assert tok.eos_token_id == 2 + assert gguf_eos_token_ids(str(FIXTURE), tok) == {2, 24} + assert tok.chat_template + + ids = tok("def foo(): return 1").input_ids + assert tok.decode(ids).endswith("def foo(): return 1") diff --git a/tests/models/test_laguna_modules.py b/tests/models/test_laguna_modules.py new file mode 100644 index 00000000..c6ac2144 --- /dev/null +++ b/tests/models/test_laguna_modules.py @@ -0,0 +1,124 @@ +from __future__ import annotations + +from dataclasses import replace +from pathlib import Path +from types import SimpleNamespace + +import numpy as np +import pytest +import torch +import torch.nn.functional as F + +FIXTURE = Path(__file__).resolve().parent.parent / "fixtures" / "laguna-s-2.1-metadata.gguf" + + +@pytest.fixture(scope="module", autouse=True) +def _tp_one(): + from freetoken.distributed import set_tp_info + + set_tp_info(rank=0, size=1) + + +def _tiny_config(): + from freetoken.models.gguf.config import build_gguf_shim + from freetoken.models.laguna.gguf import parse_gguf_config + from freetoken.models.config import FullAttentionGroupConfig, RotaryConfig, SWAAttentionGroupConfig + + cfg = parse_gguf_config(build_gguf_shim(str(FIXTURE))) + full_rope = replace(cfg.attention_groups[0].rotary_config, head_dim=32, rotary_dim=32) + swa_rope = replace(cfg.attention_groups[1].rotary_config, head_dim=32, rotary_dim=32) + groups = ( + FullAttentionGroupConfig("full", (0, 4), 2, 32, full_rope), + SWAAttentionGroupConfig("swa", (1, 2, 3, 5, 6, 7), 2, 32, swa_rope, 512), + ) + return replace(cfg, num_layers=8, num_qo_heads=6, + num_qo_heads_per_layer=(4, 6, 6, 6, 4, 6, 6, 6), num_kv_heads=2, + head_dim=32, hidden_size=64, intermediate_size=96, moe_intermediate_size=16, + shared_expert_intermediate_size=16, vocab_size=128, num_experts=8, + num_experts_per_tok=3, gguf_embed_quant=None, + gguf_model_path=None, rotary_config=replace(cfg.rotary_config, head_dim=32, rotary_dim=32), + attention_groups=groups) + + +def test_attention_geometry(monkeypatch): + import freetoken.models.laguna.attention as attention_mod + from freetoken.models.laguna.attention import LagunaAttention + monkeypatch.setattr(attention_mod, "get_rope", lambda **kw: SimpleNamespace(rotary_dim=kw["rotary_dim"])) + cfg = _tiny_config(); full = LagunaAttention(cfg, 0); swa = LagunaAttention(cfg, 1) + assert full.num_qo_heads == 4 and full.q_proj.weight.shape == (128, 64) + assert full.k_proj.weight.shape == (64, 64) and full.v_proj.weight.shape == (64, 64) + assert full.rotary.rotary_dim == 32 and full.attn_spec.sliding_window is None + assert swa.num_qo_heads == 6 and swa.attn_spec.sliding_window == 512 + assert swa.gate_proj.weight.shape[0] == 6 + + +def test_attention_forward_applies_gate_independently(monkeypatch): + import freetoken.models.laguna.attention as attention_mod + from freetoken.models.laguna.attention import LagunaAttention + monkeypatch.setattr(attention_mod, "get_rope", lambda **kw: SimpleNamespace(rotary_dim=kw["rotary_dim"], forward=lambda p, q, k: (q, k))) + cfg = _tiny_config(); x = torch.randn(3, 64) + for layer_id, heads in ((0, 4), (1, 6)): + attn = LagunaAttention(cfg, layer_id); seen = {} + qv = torch.randn(3, heads * 32); kv = torch.randn(3, 2 * 32); vv = torch.randn(3, 2 * 32) + gate = torch.randn(3, heads); backend = torch.randn(3, heads, 32, dtype=torch.float16) + def proj(key, out): + class P: + def forward(self, z): seen[key] = z; return out + return P() + class O: + def forward(self, z): seen["o"] = z; return z + class N: + def forward_inplace(self, z): return z + attn.q_proj, attn.k_proj, attn.v_proj = proj("q", qv), proj("k", kv), proj("v", vv) + attn.gate_proj, attn.o_proj = proj("gate", gate), O(); attn.q_norm = attn.k_norm = N() + ctx = SimpleNamespace(batch=SimpleNamespace(positions=torch.arange(3)), attn_backend=SimpleNamespace(forward=lambda *a, **k: backend)) + monkeypatch.setattr(attention_mod, "get_global_ctx", lambda: ctx) + got = attn.forward(x) + assert seen["q"] is x and seen["k"] is x and seen["v"] is x and seen["gate"] is x + expected = backend.view(3, heads, 32) * F.softplus(gate.float()).unsqueeze(-1).to(backend.dtype) + torch.testing.assert_close(seen["o"], expected.reshape(3, heads * 32)); torch.testing.assert_close(got, seen["o"]) + + +def test_router_matches_reference_and_full_forward(): + from freetoken.models.laguna.moe import LagunaSparseMoeBlock + torch.manual_seed(0); blk = LagunaSparseMoeBlock.__new__(LagunaSparseMoeBlock) + blk.top_k, blk.num_experts, blk.norm_topk_prob, blk.routed_scaling_factor = 3, 8, True, 2.5 + blk.gate = SimpleNamespace(weight=torch.randn(8, 64)); blk.e_score_correction_bias = torch.randn(8) + x = torch.randn(5, 64); scores = (x @ blk.gate.weight.T).sigmoid(); ids = torch.topk(scores + blk.e_score_correction_bias, 3, dim=-1).indices + weights = scores.gather(-1, ids); weights = weights / (weights.sum(-1, keepdim=True) + 1e-20) * 2.5 + got_w, got_ids = blk._route(x); torch.testing.assert_close(got_ids, ids.to(torch.int32)); torch.testing.assert_close(got_w, weights) + blk.experts = SimpleNamespace(routed_forward=lambda h, w, i: h * w.sum(-1, keepdim=True)); blk.shared_experts = SimpleNamespace(forward=lambda h: h + 7) + torch.testing.assert_close(blk.forward(x), x * got_w.sum(-1, keepdim=True) + x + 7) + + +def test_decoder_residual_semantics(monkeypatch): + import freetoken.models.laguna.attention as attention_mod + from freetoken.models.laguna.model import LagunaDecoderLayer + monkeypatch.setattr(attention_mod, "get_rope", lambda **kw: SimpleNamespace(rotary_dim=kw["rotary_dim"])) + layer = LagunaDecoderLayer(_tiny_config(), 0); x = torch.randn(3, 64) + layer.input_layernorm = layer.ffn_norm = SimpleNamespace(forward=lambda z: z * 2) + layer.self_attn = SimpleNamespace(forward=lambda z: z + 1); layer.mlp = SimpleNamespace(forward=lambda z: z * 3) + h = x + (x * 2 + 1); torch.testing.assert_close(layer.forward(x), h + h * 2 * 3) + + +def test_laguna_mlp_fused_layout(monkeypatch): + import freetoken.models.laguna.moe as moe_mod + from freetoken.models.laguna.moe import LagunaMLP + monkeypatch.setattr(moe_mod, "silu_and_mul", lambda z: F.silu(z[..., :4]) * z[..., 4:]) + m = LagunaMLP(8, 4); torch.manual_seed(2); m.gate_up_proj.weight.copy_(torch.randn_like(m.gate_up_proj.weight)); m.down_proj.weight.copy_(torch.randn_like(m.down_proj.weight)) + x = torch.randn(3, 8); wg, wu = m.gate_up_proj.weight[:4], m.gate_up_proj.weight[4:] + m.gate_up_proj.forward = lambda z: F.linear(z, m.gate_up_proj.weight); m.down_proj.forward = lambda z: F.linear(z, m.down_proj.weight) + expected = F.linear(F.silu(F.linear(x, wg)) * F.linear(x, wu), m.down_proj.weight) + torch.testing.assert_close(m.forward(x), expected) + + +def test_deferred_gguf_linear_q8(): + from freetoken.models.gguf.dequant import GGML_Q8_0, row_bytes + from freetoken.models.laguna.gguf import DeferredGGUFLinear + layer = DeferredGGUFLinear(64, 32) + with pytest.raises(AssertionError): layer.forward(torch.randn(2, 64)) + layer.materialize(GGML_Q8_0); assert layer.qweight.shape == (32, row_bytes(64, GGML_Q8_0)) + if not torch.cuda.is_available(): pytest.skip("CUDA required for fused GGUF forward") + import gguf + rng = np.random.default_rng(1); weight = rng.standard_normal((32, 64), dtype=np.float32); packed = gguf.quants.quantize(weight, gguf.GGMLQuantizationType.Q8_0) + layer.qweight = torch.from_numpy(np.ascontiguousarray(packed)).cuda(); x = torch.randn(2, 64, device="cuda", dtype=torch.bfloat16); got = layer.forward(x).float(); blocks = x.float().reshape(2, -1, 32); scale = (blocks.abs().amax(dim=-1, keepdim=True) / 127).half().float(); aq = torch.where(scale > 0, (blocks / scale).round().clamp(-127, 127), blocks).mul(scale).reshape_as(x); ref = F.linear(aq, torch.from_numpy(gguf.quants.dequantize(packed, gguf.GGMLQuantizationType.Q8_0)).float().cuda()); assert (got - ref).abs().max() <= 5e-3 * ref.abs().max().clamp(min=1.0) diff --git a/tests/models/test_laguna_weights.py b/tests/models/test_laguna_weights.py new file mode 100644 index 00000000..51b64487 --- /dev/null +++ b/tests/models/test_laguna_weights.py @@ -0,0 +1,264 @@ +"""End-to-end Laguna GGUF weight loading over a synthetic tiny checkpoint. + +Writes a real (tiny) laguna GGUF with gguf-py -- Q8_0 quantized projections and +expert banks, F32 norms/router -- then exercises config parsing, the name map, +``iter_gguf_weights`` (every tensor consumed exactly once, fused buffers built), +deferred materialization, and the mixed-type expert-bank loader. +""" +from __future__ import annotations + +import numpy as np +import pytest +import torch + +import gguf + +import freetoken.distributed.info as di +from freetoken.models.gguf.dequant import GGML_Q4_0, GGML_Q8_0, dequantize, row_bytes + +# Tiny geometry: 4 layers (full at 0), heads 4 full / 6 swa, kv 2, head_dim 32. +L, H, FF = 4, 64, 96 +HEADS = [4, 6, 6, 6] +KV, HD = 2, 32 +E, TOPK, I, SHI = 8, 3, 32, 32 +VOCAB = 128 + + +@pytest.fixture(scope="module") +def tiny_gguf(tmp_path_factory): + path = str(tmp_path_factory.mktemp("laguna") / "tiny-laguna.gguf") + w = gguf.GGUFWriter(path, "laguna") + w.add_block_count(L) + w.add_context_length(4096) + w.add_embedding_length(H) + w.add_feed_forward_length(FF) + w.add_head_count(HEADS) + w.add_head_count_kv(KV) + w.add_key_length(HD) + w.add_value_length(HD) + w.add_layer_norm_rms_eps(1e-6) + w.add_sliding_window(512) + w.add_rope_freq_base(500000.0) + w.add_rope_dimension_count(16) + # SWA rope mirrors + yarn keys (raw kv names, mirroring the real file). + w.add_float32("laguna.rope.freq_base_swa", 10000.0) + w.add_uint32("laguna.rope.dimension_count_swa", HD) + w.add_string("laguna.rope.scaling.type", "yarn") + w.add_float32("laguna.rope.scaling.factor", 32.0) + w.add_uint32("laguna.rope.scaling.original_context_length", 8192) + w.add_float32("laguna.rope.scaling.yarn_attn_factor", 1.0) + w.add_float32("laguna.rope.scaling.yarn_beta_fast", 32.0) + w.add_float32("laguna.rope.scaling.yarn_beta_slow", 1.0) + w.add_expert_count(E) + w.add_expert_used_count(TOPK) + w.add_expert_feed_forward_length(I) + w.add_expert_shared_feed_forward_length(SHI) + w.add_bool("laguna.expert_weights_norm", True) + w.add_float32("laguna.expert_weights_scale", 2.5) + w.add_uint32("laguna.expert_gating_func", 2) + w.add_uint32("laguna.leading_dense_block_count", 1) + w.add_uint32("laguna.vocab_size", VOCAB) + # Minimal gpt2 tokenizer metadata so the shim can size the vocab. + w.add_tokenizer_model("gpt2") + w.add_token_list([f"" for i in range(VOCAB)]) + w.add_token_types([1] * VOCAB) + w.add_token_merges([]) + w.add_bos_token_id(2) + w.add_eos_token_id(2) + w.add_uint32("tokenizer.ggml.eot_token_id", 24) + + rng = np.random.default_rng(0) + q8 = gguf.GGMLQuantizationType.Q8_0 + + def quant(name, rows, cols): + data = rng.standard_normal((rows, cols)).astype(np.float32) + w.add_tensor(name, gguf.quants.quantize(data, q8), raw_dtype=q8) + + def f32(name, *shape): + w.add_tensor(name, rng.standard_normal(shape).astype(np.float32)) + + quant("token_embd.weight", VOCAB, H) + quant("output.weight", VOCAB, H) + f32("output_norm.weight", H) + for i in range(L): + p = f"blk.{i}." + nh = HEADS[i] + f32(p + "attn_norm.weight", H) + f32(p + "attn_q_norm.weight", HD) + f32(p + "attn_k_norm.weight", HD) + f32(p + "ffn_norm.weight", H) + quant(p + "attn_q.weight", nh * HD, H) + quant(p + "attn_k.weight", KV * HD, H) + quant(p + "attn_v.weight", KV * HD, H) + quant(p + "attn_output.weight", H, nh * HD) + quant(p + "attn_gate.weight", nh, H) + if i == 0: + quant(p + "ffn_gate.weight", FF, H) + quant(p + "ffn_up.weight", FF, H) + quant(p + "ffn_down.weight", H, FF) + else: + f32(p + "ffn_gate_inp.weight", E, H) + f32(p + "exp_probs_b.bias", E) + quant(p + "ffn_gate_shexp.weight", SHI, H) + quant(p + "ffn_up_shexp.weight", SHI, H) + quant(p + "ffn_down_shexp.weight", H, SHI) + for role, rows, cols in ( + ("ffn_gate_exps", I, H), + ("ffn_up_exps", I, H), + ("ffn_down_exps", H, I), + ): + data = rng.standard_normal((E, rows, cols)).astype(np.float32) + w.add_tensor( + p + role + ".weight", + gguf.quants.quantize(data.reshape(E * rows, cols), q8).reshape(E, rows, -1), + raw_dtype=q8, + ) + w.write_header_to_file() + w.write_kv_data_to_file() + w.write_tensors_to_file() + w.close() + return path + + +@pytest.fixture(autouse=True) +def _tp1(): + try: + di.get_tp_info() + except RuntimeError: + di.set_tp_info(0, 1) + + +def _config(path): + from freetoken.models.gguf.config import build_gguf_shim + from freetoken.models.laguna.gguf import parse_gguf_config + + return parse_gguf_config(build_gguf_shim(path)) + + +def test_config_and_expert_types(tiny_gguf): + cfg = _config(tiny_gguf) + assert cfg.num_layers == L and cfg.num_qo_heads == 6 + assert cfg.gguf_embed_quant == GGML_Q8_0 + assert cfg.expert_quant == "gguf" and cfg.moe_weight_format == "gguf" + assert cfg.gguf_expert_types == ((GGML_Q8_0, GGML_Q8_0),) * (L - 1) + + +def test_iter_weights_complete_and_fused(tiny_gguf): + from freetoken.models.laguna.gguf import iter_gguf_weights + + got = dict(iter_gguf_weights(tiny_gguf, "cpu", include_moe_experts=False, include_non_moe=True)) + # split q/k/v per layer (mixed-type files forbid fusing) + for i, nh in enumerate(HEADS): + assert got[f"model.layers.{i}.self_attn.q_proj.qweight"].shape == (nh * HD, row_bytes(H, GGML_Q8_0)) + assert got[f"model.layers.{i}.self_attn.k_proj.qweight"].shape == (KV * HD, row_bytes(H, GGML_Q8_0)) + assert got[f"model.layers.{i}.self_attn.v_proj.qweight"].shape == (KV * HD, row_bytes(H, GGML_Q8_0)) + t = got["model.layers.0.mlp.gate_up_proj.qweight"] + assert t.shape == (2 * FF, row_bytes(H, GGML_Q8_0)) + t = got["model.layers.1.mlp.shared_experts.gate_up_proj.qweight"] + assert t.shape == (2 * SHI, row_bytes(H, GGML_Q8_0)) + assert got["model.layers.1.mlp.gate.weight"].dtype == torch.float32 + assert got["model.layers.1.mlp.e_score_correction_bias"].dtype == torch.float32 + assert got["model.layers.0.ffn_norm.weight"].dtype == torch.bfloat16 + assert got["lm_head.qweight"].shape == (VOCAB, row_bytes(H, GGML_Q8_0)) + + +def test_unknown_tensor_rejected(tiny_gguf, tmp_path): + from freetoken.models.laguna.gguf import iter_gguf_weights + + w = gguf.GGUFWriter(str(tmp_path / "bad.gguf"), "laguna") + w.add_block_count(1) + w.add_tensor("blk.0.mystery.weight", np.zeros((4, 4), dtype=np.float32)) + w.write_header_to_file(); w.write_kv_data_to_file(); w.write_tensors_to_file(); w.close() + with pytest.raises(ValueError, match="mystery"): + list(iter_gguf_weights(str(tmp_path / "bad.gguf"), "cpu", + include_moe_experts=False, include_non_moe=True)) + + +def test_expert_bank_loader(tiny_gguf): + from freetoken.models.laguna.gguf import _expert_bank_geometry, load_gguf_expert_sources + + cfg = _config(tiny_gguf) + banks = load_gguf_expert_sources(tiny_gguf, cfg) + gu_s, dn_s = _expert_bank_geometry(cfg) + assert len(banks["gate_up"]) == L - 1 and len(banks["down"]) == L - 1 + for t in banks["gate_up"]: + assert t.shape == (E, gu_s) and t.dtype == torch.uint8 + # payload bytes decode to the source values via gguf-py + half = I * row_bytes(H, GGML_Q8_0) + blob = banks["gate_up"][0][:, : 2 * half] + dec = gguf.quants.dequantize( + np.ascontiguousarray(blob.reshape(E * 2 * I, -1).numpy()), + gguf.GGMLQuantizationType.Q8_0, + ) + assert np.isfinite(dec).all() and dec.std() > 0.5 # real data, not padding + + +def test_deferred_materialization(tiny_gguf): + cfg = _config(tiny_gguf) + from freetoken.models.laguna.gguf import DeferredGGUFLinear + + # conversion materializes from the file's tensor table; emulate on one module + mod = DeferredGGUFLinear(H, 6 * HD) + mod.materialize(GGML_Q8_0) + assert mod.qweight.shape == (6 * HD, row_bytes(H, GGML_Q8_0)) + assert cfg.gguf_model_path == tiny_gguf + + +def test_compressed_tensors_int4_repack_is_bit_faithful(): + from freetoken.models.laguna.weight import _ct_int4_to_q4_0 + + torch.manual_seed(4) + rows, groups = 5, 3 + q = torch.randint(-8, 8, (rows, groups, 32), dtype=torch.int32) + unsigned = q + 8 + words = torch.zeros((rows, groups, 4), dtype=torch.int32) + for word in range(4): + for nibble in range(8): + words[..., word] |= unsigned[..., word * 8 + nibble] << (4 * nibble) + packed = words.reshape(rows, groups * 4) + scales = torch.randn(rows, groups).abs().add_(0.01).to(torch.bfloat16) + + got = _ct_int4_to_q4_0(packed, scales) + assert got.shape == (rows, groups * 18) + decoded = dequantize(got.reshape(-1), GGML_Q4_0, torch.float32).reshape( + rows, groups, 32 + ) + expected = q.float() * scales.to(torch.float16).float().unsqueeze(-1) + torch.testing.assert_close(decoded, expected, rtol=0, atol=0) + + +def test_safetensors_dense_weight_mapping(tmp_path): + safetensors = pytest.importorskip("safetensors.torch") + from freetoken.models.laguna.weight import iter_weights + + tensors = { + "model.embed_tokens.weight": torch.randn(8, 4, dtype=torch.bfloat16), + "model.layers.0.post_attention_layernorm.weight": torch.randn(4), + "model.layers.0.self_attn.g_proj.weight": torch.randn(2, 4), + "model.layers.0.mlp.gate_proj.weight": torch.randn(3, 4), + "model.layers.0.mlp.up_proj.weight": torch.randn(3, 4), + "model.layers.1.mlp.shared_expert.gate_proj.weight": torch.randn(2, 4), + "model.layers.1.mlp.shared_expert.up_proj.weight": torch.randn(2, 4), + "model.layers.1.mlp.gate.weight": torch.randn(2, 4, dtype=torch.bfloat16), + "model.layers.1.mlp.experts.e_score_correction_bias": torch.randn(2), + "model.layers.1.mlp.experts.0.gate_proj.weight_packed": torch.zeros( + 2, 2, dtype=torch.int32 + ), + } + safetensors.save_file(tensors, tmp_path / "model.safetensors") + got = dict( + iter_weights( + str(tmp_path), + torch.device("cpu"), + include_moe_experts=False, + include_non_moe=True, + ) + ) + + assert "model.layers.0.self_attn.gate_proj.weight" in got + assert "model.layers.0.ffn_norm.weight" in got + assert got["model.layers.0.mlp.gate_up_proj.weight"].shape == (6, 4) + assert got["model.layers.1.mlp.shared_experts.gate_up_proj.weight"].shape == (4, 4) + assert got["model.layers.1.mlp.gate.weight"].dtype == torch.float32 + assert "model.layers.1.mlp.e_score_correction_bias" in got + assert not any("weight_packed" in name for name in got) diff --git a/tests/models/test_nemotron_h.py b/tests/models/test_nemotron_h.py new file mode 100644 index 00000000..1f601719 --- /dev/null +++ b/tests/models/test_nemotron_h.py @@ -0,0 +1,102 @@ +from types import SimpleNamespace + +import torch + +from freetoken.distributed import get_tp_info, set_tp_info +from freetoken.models.nemotron_h.config import parse_config +from freetoken.models.nemotron_h.model import NemotronHForCausalLM +from freetoken.models.register import get_model_spec +from freetoken.moe.expert_banks import bank_bytes_estimate +from freetoken.moe.fused_nvfp4 import _run_act +from freetoken.utils import torch_dtype + + +def _hf_config(): + layers = ["mamba", "moe", "mamba", "attention", "moe"] + quantized = { + "backbone.layers.0.mixer.in_proj": {"quant_algo": "FP8"}, + "backbone.layers.0.mixer.out_proj": {"quant_algo": "FP8"}, + "backbone.layers.1.mixer.fc1_latent_proj": {"quant_algo": "FP8"}, + "backbone.layers.1.mixer.experts.0.up_proj": {"quant_algo": "NVFP4"}, + "backbone.layers.1.mixer.experts.0.down_proj": {"quant_algo": "NVFP4"}, + } + return SimpleNamespace( + layers_block_type=layers, + quantization_config={"quant_algo": "MIXED_PRECISION", "quantized_layers": quantized}, + head_dim=128, + max_position_embeddings=262144, + rope_theta=10000.0, + n_groups=8, + mamba_num_heads=128, + mamba_head_dim=64, + ssm_state_size=128, + conv_kernel=4, + chunk_size=128, + moe_latent_size=1024, + moe_shared_expert_intermediate_size=5376, + num_key_value_heads=2, + num_hidden_layers=len(layers), + num_attention_heads=32, + hidden_size=4096, + vocab_size=131072, + intermediate_size=2688, + layer_norm_epsilon=1e-5, + tie_word_embeddings=False, + n_routed_experts=512, + num_experts_per_tok=22, + moe_intermediate_size=2688, + norm_topk_prob=True, + routed_scaling_factor=5.0, + n_group=1, + topk_group=1, + model_type="nemotron_h", + architectures=["NemotronHForCausalLM"], + ) + + +def test_config_maps_mixer_and_expert_geometry(): + config = parse_config(_hf_config()) + assert config.num_moe_layers == 2 + assert config.moe_layer_ids == (1, 4) + assert config.expert_hidden_size == 1024 + assert not config.expert_gated + assert config.single_stream_only + assert config.attention_groups[0].layer_ids == (0, 2) + assert config.attention_groups[1].layer_ids == (3,) + assert config.nemotron_h_args.module_quant( + "backbone.layers.0.mixer.in_proj" + ) == "fp8_pertensor" + + +def test_offload_model_has_no_resident_expert_tensors(): + try: + get_tp_info() + except RuntimeError: + set_tp_info(0, 1) + config = parse_config(_hf_config()) + object.__setattr__(config, "moe_backend", "offload") + with torch.device("meta"), torch_dtype(torch.bfloat16): + model = NemotronHForCausalLM(config) + state = model.state_dict() + assert not any(".experts." in key for key in state) + assert state["backbone.layers.0.mixer.in_proj.weight"].dtype == torch.float8_e4m3fn + assert state["backbone.layers.1.mixer.gate.weight"].shape == (512, 4096) + + +def test_ungated_nvfp4_bank_estimate(): + config = parse_config(_hf_config()) + H, I = 1024, 2688 + per_expert = I * (H // 2 + H // 16 + 2) + H * (I // 2 + I // 16 + 2) + assert bank_bytes_estimate(config) == 2 * 512 * per_expert + + +def test_relu2_expert_activation_is_ungated(): + x = torch.tensor([[-2.0, 0.5, 3.0]]) + out = torch.empty_like(x) + _run_act("relu2", x, out, 1.702, 7.0) + torch.testing.assert_close(out, torch.tensor([[0.0, 0.25, 9.0]])) + + +def test_registry_entry(): + spec = get_model_spec("NemotronHForCausalLM") + assert spec.module == "freetoken.models.nemotron_h" diff --git a/tests/models/test_qwen3_5_gguf.py b/tests/models/test_qwen3_5_gguf.py new file mode 100644 index 00000000..a5235b25 --- /dev/null +++ b/tests/models/test_qwen3_5_gguf.py @@ -0,0 +1,91 @@ +from __future__ import annotations + +from types import SimpleNamespace + +import torch + + +def _shim(): + metadata = { + "qwen35moe.block_count": 41, + "qwen35moe.nextn_predict_layers": 1, + "qwen35moe.embedding_length": 2048, + "qwen35moe.context_length": 262144, + "qwen35moe.attention.head_count": 16, + "qwen35moe.attention.head_count_kv": 2, + "qwen35moe.attention.key_length": 256, + "qwen35moe.attention.layer_norm_rms_epsilon": 1e-6, + "qwen35moe.full_attention_interval": 4, + "qwen35moe.rope.dimension_count": 64, + "qwen35moe.rope.freq_base": 10_000_000.0, + "qwen35moe.ssm.state_size": 128, + "qwen35moe.ssm.inner_size": 4096, + "qwen35moe.ssm.group_count": 16, + "qwen35moe.ssm.conv_kernel": 4, + "qwen35moe.expert_count": 256, + "qwen35moe.expert_used_count": 8, + "qwen35moe.expert_feed_forward_length": 512, + "qwen35moe.expert_shared_feed_forward_length": 512, + } + return SimpleNamespace( + metadata=metadata, + model_path="ornith.gguf", + vocab_size=248320, + tie_word_embeddings=False, + architectures=["Qwen3_5MoeGGUFForConditionalGeneration"], + ) + + +def test_qwen35moe_gguf_config_excludes_mtp_and_builds_hybrid_groups(monkeypatch): + from freetoken.models.gguf import reader + from freetoken.models.qwen3_5_moe import gguf + + monkeypatch.setattr(reader, "gguf_tensor_type", lambda path, name: 12) + monkeypatch.setattr(gguf, "_expert_types", lambda shim: ((12, 14),) * 40) + config = gguf.parse_gguf_config(_shim()) + + assert config.num_layers == 40 + assert config.rotary_config.max_position == 262144 + assert config.rotary_config.rotary_dim == 64 + assert config.num_experts == 256 + assert config.num_experts_per_tok == 8 + assert config.moe_intermediate_size == 512 + assert config.shared_expert_intermediate_size == 512 + assert config.expert_quant == "gguf" + assert config.gguf_embed_quant == 12 + assert len(config.gguf_expert_types) == 40 + + linear = config.linear_attention_group() + assert linear is not None + assert len(linear.layer_ids) == 30 + assert linear.num_key_heads == 16 + assert linear.num_value_heads == 32 + assert linear.key_head_dim == linear.value_head_dim == 128 + full = next(group for group in config.attention_groups if group.name == "full") + assert tuple(full.layer_ids) == tuple(range(3, 40, 4)) + + +def test_qwen35moe_gguf_registry_mapping(): + from freetoken.models import register + from freetoken.models.gguf.config import GGUF_ARCH_TO_REGISTRY + + key = "Qwen3_5MoeGGUFForConditionalGeneration" + assert GGUF_ARCH_TO_REGISTRY["qwen35moe"] == key + assert key in register._MODEL_REGISTRY + + from freetoken.models.gguf.tokenizer import _TOKENIZER_ARCH + + assert _TOKENIZER_ARCH["qwen35moe"] == "qwen3_moe" + + +def test_inverse_v_permutation_restores_grouped_head_order(monkeypatch): + from freetoken.models.gguf import reader + from freetoken.models.qwen3_5_moe import gguf + + monkeypatch.setattr(reader, "gguf_tensor_type", lambda path, name: 12) + monkeypatch.setattr(gguf, "_expert_types", lambda shim: ((12, 14),) * 40) + config = gguf.parse_gguf_config(_shim()) + grouped = torch.arange(32 * 3).reshape(32 * 3, 1) + # llama.cpp stores [G0v0, G1v0, ..., G0v1, G1v1, ...]. + tiled = grouped.reshape(16, 2, 3).permute(1, 0, 2).reshape(32 * 3, 1) + assert torch.equal(gguf._undo_v_rows(tiled, config, 3), grouped) diff --git a/tests/moe/test_cpu_moe.py b/tests/moe/test_cpu_moe.py index f496de31..6546979f 100644 --- a/tests/moe/test_cpu_moe.py +++ b/tests/moe/test_cpu_moe.py @@ -164,6 +164,52 @@ def test_cpu_decode_nvfp4_matches_dequant_then_gpu(bs): assert rel < 3e-2, f"nvfp4 bs={bs} rel err {rel.item()}" +def test_cpu_decode_nvfp4_ungated_relu2_matches_dequant_reference(): + """Nemotron-H's single up projection (no gate half) and ReLU^2 epilogue.""" + from freetoken.kernel.triton.nvfp4_dequant import dequant_nvfp4 + from freetoken.moe.cpu_executor import CpuMoeExecutor + + torch.manual_seed(2718) + L, E, H, I, top_k, bs = 2, 8, 256, 128, 2, 3 + cache = _make_nvfp4_cache(L, E, H, I, seed=19) + # The helper creates conventional [gate;up] rows; Nemotron stores just one I-row + # projection. Keep the first half and make every per-layer view contiguous. + for name in ("gate_up_packed", "gate_up_scale", "gate_up_global"): + cache.bank_sources[name] = [ + tensor[:, :I].contiguous() for tensor in cache.bank_sources[name] + ] + + dev = torch.device("cuda") + ex = CpuMoeExecutor( + cache, top_k=top_k, activation="relu2", + apply_router_weight_on_input=False, num_threads=4, max_tokens=bs, device=dev, + ) + hidden = torch.randn(bs, H, device=dev, dtype=torch.bfloat16) + ids = torch.randint(0, E, (bs, top_k), device=dev, dtype=torch.int32) + weights = torch.rand(bs, top_k, device=dev, dtype=torch.float32) + actual = ex.decode(1, hidden, weights, ids).float() + torch.cuda.synchronize() + + banks = cache.bank_sources + slots = torch.arange(E, device=dev, dtype=torch.int32) + up = dequant_nvfp4( + banks["gate_up_packed"][1].to(dev), banks["gate_up_scale"][1].to(dev), + banks["gate_up_global"][1].to(dev), slots, dtype=torch.bfloat16, + ).float() + down = dequant_nvfp4( + banks["down_packed"][1].to(dev), banks["down_scale"][1].to(dev), + banks["down_global"][1].to(dev), slots, dtype=torch.bfloat16, + ).float() + reference = torch.zeros(bs, H, device=dev) + for token in range(bs): + for route in range(top_k): + expert = int(ids[token, route]) + act = torch.relu(up[expert] @ hidden[token].float()).square().bfloat16().float() + reference[token] += float(weights[token, route]) * (down[expert] @ act) + rel = (actual - reference).abs().max() / (reference.abs().max() + 1e-6) + assert rel < 3e-2, f"nvfp4 relu2 relative error {rel.item()}" + + @pytest.mark.parametrize("bs", [1, 4]) def test_cpu_decode_nvfp4_swigluoai_matches_dequant_reference(bs): """MiniMax-M3's swigluoai routed experts on the CPU executor (ActKind 3 through diff --git a/tests/moe/test_cpu_moe_mixed_gguf.py b/tests/moe/test_cpu_moe_mixed_gguf.py new file mode 100644 index 00000000..7f825f86 --- /dev/null +++ b/tests/moe/test_cpu_moe_mixed_gguf.py @@ -0,0 +1,185 @@ +from __future__ import annotations + +from types import SimpleNamespace + +import pytest +import torch + + +def test_mixed_gguf_prefill_dispatches_to_grouped_mmq(monkeypatch): + import freetoken.kernel.gguf as kernel + import freetoken.moe.fused as fused + import freetoken.moe.fused_gguf as mod + + calls = [] + sentinel = torch.empty(1) + monkeypatch.setattr(kernel, "ggml_moe_get_block_size", lambda _qtype: 4) + monkeypatch.setattr( + fused, + "moe_align_block_size", + lambda ids, block, experts: ( + calls.append(("align", block, experts)) or torch.empty(1, dtype=torch.int32), + torch.empty(1, dtype=torch.int32), + torch.empty(1, dtype=torch.int32), + ), + ) + monkeypatch.setattr( + kernel, + "ggml_moe_a8", + lambda *args: calls.append(("mmq", args[-2], args[-1])) or sentinel, + ) + monkeypatch.setattr( + mod, + "_moe_vec_chunked", + lambda *args: calls.append(("mmvq", args[-3], args[-2])) or sentinel, + ) + + x = torch.empty(32, 64) + weight = torch.empty(256, 1024, dtype=torch.uint8) + ids = torch.zeros(32, 8, dtype=torch.int32) + got = mod._moe_matmul(x, weight, ids, 8, 12, 128, 32, 1024) + + assert got is sentinel + assert calls == [("align", 4, 256), ("mmq", 8, 32)] + + +def test_mixed_gguf_decode_stays_on_mmvq(monkeypatch): + import freetoken.moe.fused_gguf as mod + + calls = [] + sentinel = torch.empty(1) + monkeypatch.setattr( + mod, + "_moe_vec_chunked", + lambda *args: calls.append((args[3], args[4])) or sentinel, + ) + got = mod._moe_matmul( + torch.empty(1, 64), + torch.empty(256, 1024, dtype=torch.uint8), + torch.zeros(1, 8, dtype=torch.int32), + 8, + 12, + 128, + 1, + 1024, + ) + + assert got is sentinel + assert calls == [(8, 12)] + + +def test_mixed_gguf_cpu_executor_builds_zero_copy_views_and_dispatches(monkeypatch): + import freetoken.moe.cpu_executor as mod + from freetoken.models.gguf.dequant import GGML_BF16, GGML_Q4_0, row_bytes + + created = [] + + class FakeExecutor: + def __init__(self, cache, **kwargs): + self.cache = cache + self.kwargs = kwargs + created.append(self) + + def decode(self, layer_id, *args): + return self.cache.quant_format, layer_id + + def decode_submit(self, layer_id, *args): + return self.cache.quant_format, layer_id + + def decode_sync(self, pending): + return pending + + def raise_if_unhealthy(self): + return None + + monkeypatch.setattr(mod, "CpuMoeExecutor", FakeExecutor) + experts, hidden, intermediate = 2, 64, 32 + q4_gu = torch.empty( + experts * 2 * intermediate * row_bytes(hidden, GGML_Q4_0), + dtype=torch.uint8, + ).view(experts, -1) + q4_dn = torch.empty( + experts * hidden * row_bytes(intermediate, GGML_Q4_0), + dtype=torch.uint8, + ).view(experts, -1) + bf_gu = torch.empty( + experts * 2 * intermediate * hidden * 2, dtype=torch.uint8 + ).view(experts, -1) + bf_dn = torch.empty( + experts * hidden * intermediate * 2, dtype=torch.uint8 + ).view(experts, -1) + cache = SimpleNamespace( + num_layers=2, + num_experts=experts, + gguf_expert_types=( + (GGML_Q4_0, GGML_Q4_0), + (GGML_BF16, GGML_BF16), + ), + expert_hidden_size=hidden, + expert_intermediate_size=intermediate, + bank_sources={"gate_up": [q4_gu, bf_gu], "down": [q4_dn, bf_dn]}, + ) + + executor = mod.MixedGgufCpuMoeExecutor( + cache, + top_k=2, + activation="silu", + apply_router_weight_on_input=False, + num_threads=1, + max_tokens=1, + device=torch.device("cpu"), + ) + assert {item.cache.quant_format for item in created} == {"q4_0", "bf16"} + q4 = next(item.cache for item in created if item.cache.quant_format == "q4_0") + bf16 = next(item.cache for item in created if item.cache.quant_format == "bf16") + assert q4.bank_sources["gate_up"][0].shape == ( + experts, + 2 * intermediate, + row_bytes(hidden, GGML_Q4_0), + ) + assert bf16.bank_sources["down"][1].shape == (experts, hidden, intermediate) + assert q4.bank_sources["gate_up"][0].data_ptr() == q4_gu.data_ptr() + assert bf16.bank_sources["down"][1].data_ptr() == bf_dn.data_ptr() + assert executor.decode(0) == ("q4_0", 0) + assert executor.decode(1) == ("bf16", 1) + assert executor.decode_sync(executor.decode_submit(1)) == ("bf16", 1) + + +@pytest.mark.skipif(not torch.cuda.is_available(), reason="needs CUDA") +def test_mixed_gguf_bf16_layer_matches_native_fused_experts(): + from freetoken.models.gguf.dequant import GGML_BF16 + from freetoken.moe.fused import fused_experts_impl + from freetoken.moe.fused_gguf import fused_experts_gguf + + experts, hidden, intermediate, tokens, top_k = 4, 64, 32, 3, 2 + torch.manual_seed(8) + gate_up = torch.randn( + experts, 2 * intermediate, hidden, device="cuda", dtype=torch.bfloat16 + ) + down = torch.randn( + experts, hidden, intermediate, device="cuda", dtype=torch.bfloat16 + ) + gate_up_bytes = gate_up.contiguous().view(torch.uint8).reshape(experts, -1) + down_bytes = down.contiguous().view(torch.uint8).reshape(experts, -1) + x = torch.randn(tokens, hidden, device="cuda", dtype=torch.bfloat16) + ids = torch.tensor([[0, 1], [2, 3], [1, 2]], device="cuda", dtype=torch.int32) + weights = torch.rand(tokens, top_k, device="cuda", dtype=torch.float32) + + original = x.clone() + got = fused_experts_gguf( + x, + gate_up_bytes, + down_bytes, + weights, + ids, + "silu", + gate_up_type=GGML_BF16, + down_type=GGML_BF16, + gate_up_rows=2 * intermediate, + down_rows=hidden, + ) + expected = fused_experts_impl( + x.clone(), gate_up, down, weights, ids, "silu", False + ) + torch.testing.assert_close(x, original, rtol=0, atol=0) + torch.testing.assert_close(got, expected, rtol=0, atol=0) diff --git a/tests/moe/test_fused_copy.py b/tests/moe/test_fused_copy.py index 7a0c0864..27d645ab 100644 --- a/tests/moe/test_fused_copy.py +++ b/tests/moe/test_fused_copy.py @@ -74,3 +74,50 @@ def test_fused_copy_matches_per_bank(num_indices): for b, (r, (_, c)) in enumerate(zip(ref, cache.banks)): assert torch.equal(r, c), f"bank {b} (feat={FEATS[b]}) fused != per-bank at num_indices={num_indices}" + + +@CUDA +@pytest.mark.slow +@pytest.mark.parametrize("layer_id", [0, 1]) +def test_fused_copy_variable_source_rows_fill_cache_prefix(layer_id): + dev = torch.device("cuda") + cache = OffloadMoeCache( + num_layers=2, + num_experts=4, + cache_size=8, + device=dev, + cache_policy="lru", + prefill_overlap=False, + quant_format="gguf", + ) + sources = { + "gate_up": [ + torch.randint(0, 256, (4, 256), dtype=torch.uint8, device=dev), + torch.randint(0, 256, (4, 512), dtype=torch.uint8, device=dev), + ], + "down": [ + torch.randint(0, 256, (4, 512), dtype=torch.uint8, device=dev), + torch.randint(0, 256, (4, 256), dtype=torch.uint8, device=dev), + ], + } + cache.set_bank_sources(sources) + assert cache._copy_fused_ok + assert cache._variable_bank_rows == {"gate_up", "down"} + + cache._pending_src_layer = layer_id + cache.num_indices.fill_(2) + cache.evict_slots[:2] = torch.tensor([1, 5], dtype=torch.int32, device=dev) + cache.src_indices[:2] = torch.tensor([3, 0], dtype=torch.int32, device=dev) + for _, bank in cache.banks: + bank.fill_(0xA5) + cache.copy_missing() + torch.cuda.synchronize() + + for name, (_, bank) in zip(cache.bank_schema, cache.banks): + source = sources[name][layer_id] + feat = source.shape[1] + torch.testing.assert_close(bank[1, :feat], source[3]) + torch.testing.assert_close(bank[5, :feat], source[0]) + if feat < bank.shape[1]: + assert torch.all(bank[1, feat:] == 0xA5) + assert torch.all(bank[5, feat:] == 0xA5) diff --git a/tests/moe/test_offload.py b/tests/moe/test_offload.py index 422ca867..5b4baff0 100644 --- a/tests/moe/test_offload.py +++ b/tests/moe/test_offload.py @@ -729,6 +729,28 @@ def test_set_bank_sources_locked_layer_requires_cpu_layer_ids(): ) +def test_set_bank_sources_pageable_gpu_accepts_unpinned_gpu_layer(): + from freetoken.moe.host_banks import HostResidency + from freetoken.moe.offload_cache import OffloadMoeCache + + _init_tp() + cache = OffloadMoeCache( + num_layers=2, num_experts=4, cache_size=8, device=torch.device("cpu"), + ) + cache.pageable_gpu = True + sources = { + "gate_up": [torch.randn(4, 32, 8) for _ in range(2)], + "down": [torch.randn(4, 8, 16) for _ in range(2)], + } + cache.set_bank_sources( + sources, + layer_residency=[HostResidency.PINNED.value, HostResidency.PAGEABLE.value], + ) + + assert cache.cpu_layer_ids == frozenset() + assert cache._unpinned_layers == frozenset({1}) + + def test_set_bank_sources_locked_layer_rejects_prefill_overlap(): # prefill overlap DMAs from registered banks; a LOCKED layer cannot feed it from freetoken.moe.host_banks import HostResidency diff --git a/tests/server/test_parser_auto_selection.py b/tests/server/test_parser_auto_selection.py index 78b1663e..15bb290f 100644 --- a/tests/server/test_parser_auto_selection.py +++ b/tests/server/test_parser_auto_selection.py @@ -30,6 +30,8 @@ # Families with no thinking format of their own. Everything else must resolve to a real reasoning # parser, and a new architecture landing here is the bug this file exists to catch. NO_REASONING_FORMAT = { + "LagunaForCausalLM", + "LagunaGGUFForCausalLM", "LlamaForCausalLM", "MistralForCausalLM", "Mistral3ForConditionalGeneration", @@ -38,7 +40,11 @@ # `llama3` is the end of the cascade -- the answer when nothing matched. GENERIC_TOOL_CALL_FALLBACK = "llama3" -NO_DEDICATED_TOOL_FORMAT = {"LlamaForCausalLM"} +NO_DEDICATED_TOOL_FORMAT = { + "LagunaForCausalLM", + "LagunaGGUFForCausalLM", + "LlamaForCausalLM", +} class _Config: @@ -94,3 +100,18 @@ def test_an_explicit_choice_beats_inference(): pinned, _ = parse_args(["--model", ANON_PATH, "--reasoning-parser", "qwen3"]) assert off.reasoning_parser is None assert pinned.reasoning_parser == "qwen3" + + +def test_swa_full_tokens_ratio_cli_surface(): + config = _Config({"architectures": ["LlamaForCausalLM"], "torch_dtype": "bfloat16"}) + with patch("freetoken.utils.cached_load_hf_config", lambda _path: config): + args, _ = parse_args( + ["--model", ANON_PATH, "--swa-full-tokens-ratio", "0.006"] + ) + assert args.swa_full_tokens_ratio == pytest.approx(0.006) + + +@pytest.mark.parametrize("ratio", ["0", "-0.1", "1.1", "not-a-number"]) +def test_swa_full_tokens_ratio_rejects_invalid_values(ratio): + with pytest.raises(SystemExit): + parse_args(["--model", ANON_PATH, "--swa-full-tokens-ratio", ratio])