feat(qwen4): support Qwen3.8 Flash Next NVFP4, vision, and Windows serving

.dockerignore

@@ -0,0 +1,9 @@
+.git
+.github
+.pytest_cache
+.venv
+**/__pycache__
+**/*.pyc
+build
+dist
+tests

benchmarks/bench_qwen4_qsa.py

@@ -0,0 +1,81 @@
+"""Microbenchmark Qwen4-Exp QSA selection and exact sparse attention."""
+
+from __future__ import annotations
+
+import argparse
+import statistics
+
+import torch
+
+from freetoken.attention.qsa import select_qsa_logical_rows
+from freetoken.kernel.triton.qsa import qsa_sparse_gqa
+
+
+def _median_ms(fn, warmup: int, repeats: int) -> float:
+    for _ in range(warmup):
+        fn()
+    samples = []
+    for _ in range(repeats):
+        start = torch.cuda.Event(enable_timing=True)
+        stop = torch.cuda.Event(enable_timing=True)
+        start.record()
+        fn()
+        stop.record()
+        stop.synchronize()
+        samples.append(start.elapsed_time(stop))
+    return statistics.median(samples)
+
+
+def main() -> None:
+    parser = argparse.ArgumentParser()
+    parser.add_argument("--context", type=int, default=262_144)
+    parser.add_argument("--repeats", type=int, default=50)
+    args = parser.parse_args()
+    if not torch.cuda.is_available():
+        raise RuntimeError("CUDA is required")
+    if args.context % 4:
+        raise ValueError("context must divide by QSA's compression ratio of four")
+
+    device = torch.device("cuda")
+    query = torch.randn(1, 4, 128, dtype=torch.bfloat16, device=device)
+    compressed = torch.randn(
+        args.context // 4, 1, 128, dtype=torch.bfloat16, device=device
+    )
+    position = torch.tensor([args.context - 1], dtype=torch.int64, device=device)
+    selection_ms = _median_ms(
+        lambda: select_qsa_logical_rows(
+            query,
+            compressed,
+            position,
+            compress_ratio=4,
+            token_budget=2048,
+        ),
+        5,
+        args.repeats,
+    )
+
+    attention_query = torch.randn(1, 24, 256, dtype=torch.bfloat16, device=device)
+    keys = torch.randn(2048, 2, 256, dtype=torch.bfloat16, device=device)
+    values = torch.randn_like(keys)
+    rows = torch.arange(2048, dtype=torch.int32, device=device).view(1, -1)
+    counts = torch.tensor([2048], dtype=torch.int32, device=device)
+    attention_ms = _median_ms(
+        lambda: qsa_sparse_gqa(
+            attention_query, keys, values, rows, counts, 256**-0.5
+        ),
+        5,
+        args.repeats,
+    )
+    print(
+        {
+            "gpu": torch.cuda.get_device_name(device),
+            "context": args.context,
+            "selection_ms_per_layer": round(selection_ms, 3),
+            "attention_ms_per_layer": round(attention_ms, 3),
+            "qsa_ms_for_12_layers": round(12 * (selection_ms + attention_ms), 3),
+        }
+    )
+
+
+if __name__ == "__main__":
+    main()

benchmarks/qwen38-flash-next-rtx3090-128k.json

@@ -0,0 +1,19 @@
+{
+  "date": "2026-08-26",
+  "platform": "Windows, NVIDIA RTX 3090",
+  "model": "Qwen3.8-Flash-Next NVFP4 FTW",
+  "context_length": 131072,
+  "kv_cache_gib": 3.09,
+  "moe_cache_size": 2048,
+  "gpu_used_by_load_gib": 19.974,
+  "prompt_tokens": 37,
+  "output_tokens": 127,
+  "overall_output_tokens_per_second": 3.574,
+  "steady_decode_tokens_per_second": 5.791,
+  "moe_cache_miss_rate": 0.1651,
+  "notes": [
+    "This result uses a short text prompt and a single request.",
+    "Steady decode excludes model load, prefill, and time to first token.",
+    "The 128K KV allocation permits a 2048-slot expert cache on a 24 GiB GPU."
+  ]
+}

benchmarks/qwen38-flash-next-rtx3090-bw.json

@@ -0,0 +1,50 @@
+{
+  "version": 4,
+  "timestamp": "2026-08-26T18:36:33-04:00",
+  "epoch": 1787783793,
+  "gpu": {
+    "index": 0,
+    "name": "NVIDIA GeForce RTX 3090"
+  },
+  "cpu": {
+    "physical_cores": 32,
+    "threads_used": 32
+  },
+  "threshold": 2.0,
+  "ceilings": {
+    "cpu_stream_read_gbs": 46.31,
+    "pcie_linear_h2d_gbs": 12.31,
+    "pcie_linear_d2h_gbs": 13.16
+  },
+  "dtypes": {},
+  "dtype_kernels": {},
+  "workloads": {
+    "qwen3.8-flash-next": {
+      "model": {
+        "name": "qwen3.8-flash-next",
+        "hidden": 2560,
+        "inter": 640,
+        "experts": 512,
+        "top_k": 10
+      },
+      "kernels": {
+        "nvfp4": {
+          "expert_bytes": 2772480,
+          "synth_experts": 512,
+          "cpu_moe_gbs": 0.19,
+          "cpu_moe_isa": "avx2+vnni(nvfp4-w4a8)",
+          "isa_sweep": null,
+          "pcie_gather_gbs": 9.73,
+          "cpu_moe_overlap_gbs": 0.13,
+          "pcie_gather_overlap_gbs": 9.68,
+          "ratio": 0.02,
+          "recommended": "offload",
+          "note": null
+        }
+      },
+      "recommended_moe_backend": {
+        "nvfp4": "offload"
+      }
+    }
+  }
+}

benchmarks/qwen38-flash-next-server-validation.json

@@ -0,0 +1,32 @@
+{
+  "date": "2026-08-26",
+  "platform": "Windows, NVIDIA RTX 3090",
+  "model": "Qwen3.8-Flash-Next NVFP4 FTW",
+  "served_model_name": "qwen3.8-flash-next-nvfp4",
+  "context_length": 262144,
+  "kv_cache_gib": 6.19,
+  "api": {
+    "health": "passed",
+    "models": "passed",
+    "chat_completions_text": {
+      "expected": "FREETOKEN_SERVER_OK",
+      "actual": "FREETOKEN_SERVER_OK",
+      "prompt_tokens": 21,
+      "completion_tokens": 6
+    },
+    "chat_completions_vision": {
+      "image": "assets/desktop-console.png",
+      "resolution": "2920x1944",
+      "expected": "FreeToken Desktop",
+      "actual": "FreeToken Desktop",
+      "prompt_tokens": 5589,
+      "completion_tokens": 6,
+      "reported_prefill_tokens_per_second": 143.53
+    }
+  },
+  "notes": [
+    "OpenAI image_url data URI input passed through the multiprocess server.",
+    "Pixel transport uses BF16 because the vision patch projection converts pixels to BF16 before its first operation.",
+    "The validation server was stopped after the checks."
+  ]
+}