Fix v0.4.0 Web media helper compatibility

.github/workflows/ci.yml

@@ -15,8 +15,12 @@ env:
 
 jobs:
   build-webgpu-bridge:
-    name: Build WebGPU Bridge (WASM)
+    name: ${{ matrix.upstream == 'pinned' && 'Build WebGPU Bridge (WASM)' || 'Build WebGPU Bridge (WASM, v0.4.0)' }}
     runs-on: ubuntu-latest
+    strategy:
+      fail-fast: false
+      matrix:
+        upstream: [pinned, v0.4.0]
     env:
       LLAMA_WEBGPU_SMOKE_MODEL_URL: https://huggingface.co/aladar/llama-2-tiny-random-GGUF/resolve/main/llama-2-tiny-random.gguf
       LLAMA_WEBGPU_SMOKE_MODEL_SHA256: 81f226c62d28ed4a1a9b9fa080fcd9f0cc40e0f9d5680036583ff98fbcd035cb
@@ -39,6 +43,8 @@ jobs:
       LLAMA_WEBGPU_TTS_MMPROJ_SHA256: 6fd65188839bcd6ecc91b277ad471e22a0edfada4699a0fe82f1165c18cfcce2
     steps:
       - uses: actions/checkout@v4
+        with:
+          persist-credentials: false
 
       - name: Setup Node.js
         uses: actions/setup-node@v4
@@ -77,6 +83,9 @@ jobs:
       - name: Validate text-to-speech API contract
         run: python3 scripts/verify_text_to_speech_api.py
 
+      - name: Validate media helper compatibility
+        run: python3 scripts/mtmd_compat_contract_test.py
+
       - name: Validate wasm64 runtime patch contract
         run: python3 scripts/wasm64_runtime_patch_contract_test.py
 
@@ -118,6 +127,12 @@ jobs:
         run: |
           git clone --depth 1 --branch "$LLAMA_CPP_TAG" https://github.com/ggml-org/llama.cpp.git third_party/llama_cpp
 
+      - name: Select exact v0.4.0 compatibility revision
+        if: matrix.upstream == 'v0.4.0'
+        run: |
+          git -C third_party/llama_cpp fetch --depth 1 origin refs/tags/v0.4.0:refs/tags/v0.4.0
+          git -C third_party/llama_cpp checkout --detach 5266f24da75dc449bd56cbed7addb9c8e4a6a73e
+
       - name: Setup Emscripten SDK
         uses: mymindstorm/setup-emsdk@v14
         with:
@@ -161,22 +176,22 @@ jobs:
         if: failure()
         uses: actions/upload-artifact@v4
         with:
-          name: state-persistence-smoke-artifacts
+          name: state-persistence-smoke-artifacts-${{ matrix.upstream }}
           path: ${{ env.LLAMA_WEBGPU_SMOKE_ARTIFACTS_DIR }}
           if-no-files-found: ignore
 
       - name: Upload multimodal smoke diagnostics
         if: failure()
         uses: actions/upload-artifact@v4
         with:
-          name: multimodal-smoke-artifacts
+          name: multimodal-smoke-artifacts-${{ matrix.upstream }}
           path: ${{ env.LLAMA_WEBGPU_MULTIMODAL_ARTIFACTS_DIR }}
           if-no-files-found: ignore
 
       - name: Upload bridge artifacts
         uses: actions/upload-artifact@v4
         with:
-          name: webgpu-bridge-dist
+          name: ${{ matrix.upstream == 'pinned' && 'webgpu-bridge-dist' || 'webgpu-bridge-dist-v0.4.0' }}
           path: |
             ${{ runner.temp }}/webgpu_bridge_dist/llama_webgpu_bridge.js
             ${{ runner.temp }}/webgpu_bridge_dist/llama_webgpu_bridge_worker.js

AGENTS.md

@@ -68,6 +68,7 @@ npm run check:js
 python3 -m py_compile scripts/verify_state_persistence_api.py scripts/verify_text_to_speech_api.py scripts/verify_ci_reliability.py scripts/state_persistence_browser_smoke.py scripts/multimodal_browser_smoke.py scripts/speech_to_text_browser_smoke.py scripts/text_to_speech_browser_smoke.py
 python3 scripts/verify_state_persistence_api.py
 python3 scripts/verify_text_to_speech_api.py
+python3 scripts/mtmd_compat_contract_test.py
 python3 scripts/verify_ci_reliability.py
 ```
 
@@ -135,6 +136,10 @@ automated qualification run binds the candidate digest it is about to publish.
 
 - CI build gate: `.github/workflows/ci.yml`
   - Resolves the default llama.cpp checkout from `llama_cpp.version`.
+  - Builds wasm32/memory64 against that pin and exact v0.4.0 in separate matrix
+    lanes, with media-helper API contracts plus real state and image smokes.
+    Preserve the original pinned check/artifact identities and isolate each
+    lane's artifact uploads. Neither lane publishes or changes the source pin.
   - Resolves `emsdk.version`, installs that exact compiler, verifies the active
     `emcc` identity, and contract-tests all five required wasm64 WASMFS patches.
   - Never dispatches asset publication. Bridge source changes, including changes
@@ -373,6 +378,7 @@ After publishing assets tag:
 - Every llama.cpp pin update must pass checksum-pinned real multimodal inference
   in both direct and worker runtimes; a successful WASM build alone is not
   sufficient.
-- Speech-capable asset releases must pass the required local qualification in
-  wasm32 and memory64; never run the large Qwen3-ASR/Qwen3-TTS pairs on hosted
-  runners.
+- Speech-capable asset releases must pass the hosted automated qualification
+  workflow's exact required memory/runtime matrix before publication. Keep
+  these heavy gates outside ordinary CI; reproduce locally with the individual
+  smoke scripts above, not the workflow-only combined qualifier.

CMakeLists.txt

@@ -62,6 +62,22 @@ set(GGML_BLAS OFF CACHE BOOL "" FORCE)
 
 add_subdirectory("${LLAMA_CPP_DIR}" "${CMAKE_BINARY_DIR}/llama_cpp")
 
+include(CheckCXXSourceCompiles)
+include(CMakePushCheckState)
+cmake_push_check_state(RESET)
+set(CMAKE_REQUIRED_INCLUDES
+    "${LLAMA_CPP_DIR}/include"
+    "${LLAMA_CPP_DIR}/ggml/include"
+    "${LLAMA_CPP_DIR}/tools/mtmd")
+# Recheck when a build directory is reused with a different upstream checkout.
+unset(LLAMADART_MTMD_HELPER_HAS_OPTIONS CACHE)
+# Inspect declarations without calling upstream symbols or running target code.
+check_cxx_source_compiles("
+#include <mtmd-helper.h>
+int main() { return sizeof(decltype(mtmd_helper_init_opt_default())) == 0; }
+" LLAMADART_MTMD_HELPER_HAS_OPTIONS)
+cmake_pop_check_state()
+
 find_package(Threads REQUIRED)
 
 set(MTMD_AUDIO_SRC "${LLAMA_CPP_DIR}/tools/mtmd/mtmd-audio.cpp")
@@ -113,6 +129,7 @@ add_executable(llama_webgpu_core
 target_compile_features(llama_webgpu_core PRIVATE cxx_std_17)
 
 target_compile_definitions(llama_webgpu_core PRIVATE
+    LLAMADART_MTMD_HELPER_HAS_OPTIONS=$<BOOL:${LLAMADART_MTMD_HELPER_HAS_OPTIONS}>
     LLAMADART_WEBGPU_PTHREAD_POOL_SIZE=${LLAMADART_WEBGPU_PTHREAD_POOL_SIZE}
 )
 

README.md

@@ -181,11 +181,15 @@ This repo includes a wasm build gate in:
 
 - `.github/workflows/ci.yml`
 
-It builds against the pinned `llama.cpp` tag in `llama_cpp.version`, runs the JS
-bridge build/type-check gate, uploads build artifacts, and runs the static CI
-reliability contract:
+It builds wasm32 and memory64 against both the pinned `llama.cpp` tag in
+`llama_cpp.version` and the exact v0.4.0 compatibility revision. Both lanes run
+the JS/compatibility contracts and real state-persistence and multimodal browser
+smokes. The pinned lane retains `webgpu-bridge-dist`; the compatibility lane
+uploads `webgpu-bridge-dist-v0.4.0`. Neither changes a pin or publishes assets.
+To run the media-helper and static CI contracts locally:
 
 ```bash
+python3 scripts/mtmd_compat_contract_test.py
 python3 scripts/verify_ci_reliability.py
 ```
 

scripts/mtmd_compat_contract_test.py

@@ -0,0 +1,78 @@
+#!/usr/bin/env python3
+"""Compile both media-helper API shapes and guard all production call sites."""
+
+import os
+from pathlib import Path
+import shlex
+import subprocess
+import tempfile
+import unittest
+
+
+ROOT = Path(__file__).resolve().parents[1]
+
+
+class MediaCompatibilityTest(unittest.TestCase):
+    def test_both_upstream_api_shapes(self):
+        for options in (False, True):
+            with self.subTest(options=options), tempfile.TemporaryDirectory() as tmp:
+                directory = Path(tmp)
+                extra = ", mtmd_helper_init_opt options" if options else ""
+                check = "assert(options.sentinel == 73);" if options else ""
+                declarations = """
+struct mtmd_helper_init_opt { int sentinel; };
+inline mtmd_helper_init_opt mtmd_helper_init_opt_default() { return {73}; }
+""" if options else ""
+                (directory / "mtmd-helper.h").write_text(
+                    """#pragma once
+#include <cassert>
+#include <cstddef>
+struct mtmd_context {};
+struct mtmd_helper_bitmap_wrapper { int result; };
+inline mtmd_context expected_context;
+inline const unsigned char expected_bytes[] = {1, 2, 3};
+inline const char expected_path[] = "reference.wav";
+""" + declarations + f"""
+inline mtmd_helper_bitmap_wrapper mtmd_helper_bitmap_init_from_buf(
+    mtmd_context *ctx, const unsigned char *bytes, size_t size, bool placeholder{extra}) {{
+  assert(ctx == &expected_context && bytes == expected_bytes && size == 3);
+  assert(!placeholder); {check}
+  return {{17}};
+}}
+inline mtmd_helper_bitmap_wrapper mtmd_helper_bitmap_init_from_file(
+    mtmd_context *ctx, const char *path, bool placeholder{extra}) {{
+  assert(ctx == &expected_context && path == expected_path);
+  assert(!placeholder); {check}
+  return {{29}};
+}}
+""", encoding="utf-8")
+                source = directory / "test.cpp"
+                source.write_text("""
+#include "llama_webgpu_mtmd_compat.h"
+int main() {
+  assert(llama_webgpu_bitmap_from_buffer(&expected_context, expected_bytes, 3).result == 17);
+  assert(llama_webgpu_bitmap_from_file(&expected_context, expected_path).result == 29);
+}
+""", encoding="utf-8")
+                executable = directory / "test"
+                subprocess.run([
+                    *shlex.split(os.environ.get("CXX", "c++")), "-std=c++17",
+                    "-Wall", "-Wextra", "-Werror",
+                    f"-DLLAMADART_MTMD_HELPER_HAS_OPTIONS={int(options)}",
+                    f"-I{directory}", f"-I{ROOT / 'src'}", str(source),
+                    "-o", str(executable),
+                ], check=True, capture_output=True, text=True)
+                subprocess.run([str(executable)], check=True)
+
+    def test_production_routes_through_compatibility_helpers(self):
+        core = (ROOT / "src/llama_webgpu_core.cpp").read_text()
+        tts = (ROOT / "src/llama_webgpu_tts.cpp").read_text()
+        self.assertEqual(core.count("llama_webgpu_bitmap_from_file("), 1)
+        self.assertEqual(core.count("llama_webgpu_bitmap_from_buffer("), 1)
+        self.assertEqual(tts.count("llama_webgpu_bitmap_from_buffer("), 1)
+        for source in (core, tts):
+            self.assertNotIn("mtmd_helper_bitmap_init_from_", source)
+
+
+if __name__ == "__main__":
+    unittest.main()

scripts/multimodal_browser_smoke.py

@@ -118,6 +118,49 @@ def write_harness(web_root: Path) -> None:
         await bridge.loadMultimodalProjector('/multimodal-mmproj.gguf');
         assert(bridge.supportsVision(), `${mode} did not report vision support`);
 
+        // Normal image ingestion resizes through RGB. Also exercise the native
+        // file/encoded entry points so helper API compatibility cannot hide
+        // behind that independent path. The direct runtime owns this FS.
+        if (disableWorker) {
+          const core = bridge._runtime?._core;
+          assert(core?.FS && typeof core.ccall === 'function',
+            'direct runtime core is unavailable for media compatibility checks');
+          const path = '/mtmd-compat.png';
+          core.FS.writeFile(path, imageBytes);
+          try {
+            assert(
+              core.ccall('llamadart_webgpu_media_add_file', 'number',
+                ['string'], [path]) === 0,
+              'native file helper failed to decode the real PNG',
+            );
+            core.ccall('llamadart_webgpu_media_clear_pending', null, [], []);
+            assert(
+              core.ccall('llamadart_webgpu_media_add_encoded', 'number',
+                ['array', 'number'], [imageBytes, imageBytes.length]) === 0,
+              'native buffer helper failed to decode the real PNG',
+            );
+            core.ccall('llamadart_webgpu_media_clear_pending', null, [], []);
+            assert(
+              core.ccall('llamadart_webgpu_media_add_encoded', 'number',
+                ['array', 'number'], [imageBytes, 0]) === -3,
+              'native buffer helper must reject empty input',
+            );
+            assert(
+              core.ccall('llamadart_webgpu_media_add_encoded', 'number',
+                ['array', 'number'], [imageBytes.subarray(0, 8), 8]) === -4,
+              'native buffer helper must reject a truncated PNG',
+            );
+            assert(
+              core.ccall('llamadart_webgpu_media_add_file', 'number',
+                ['string'], ['/mtmd-compat-missing.png']) === -4,
+              'native file helper must reject a missing file',
+            );
+          } finally {
+            core.ccall('llamadart_webgpu_media_clear_pending', null, [], []);
+            core.FS.unlink(path);
+          }
+        }
+
         const output = await bridge.createCompletion('what do you see?', {
           nPredict: 64,
           temp: 0,

scripts/verify_ci_reliability.py

@@ -1223,6 +1223,28 @@ def main() -> int:
         "ci.yml must resolve the llama.cpp tag from llama_cpp.version, support explicit dispatch, and avoid hard-coded stale defaults",
         errors,
     )
+    require(
+        "upstream: [pinned, v0.4.0]" in ci
+        and "5266f24da75dc449bd56cbed7addb9c8e4a6a73e" in ci
+        and "'Build WebGPU Bridge (WASM)'" in ci
+        and "'webgpu-bridge-dist-v0.4.0'" in ci
+        and "state-persistence-smoke-artifacts-${{ matrix.upstream }}" in ci
+        and "multimodal-smoke-artifacts-${{ matrix.upstream }}" in ci
+        and "python3 scripts/mtmd_compat_contract_test.py" in ci,
+        "CI must retain the pinned check and artifact identities, test exact v0.4.0 media compatibility, and isolate matrix artifacts",
+        errors,
+    )
+    require(
+        "bridge._runtime?._core" in multimodal_smoke
+        and "llamadart_webgpu_media_add_file" in multimodal_smoke
+        and "llamadart_webgpu_media_add_encoded" in multimodal_smoke
+        and "[imageBytes, 0]) === -3" in multimodal_smoke
+        and "[imageBytes.subarray(0, 8), 8]) === -4" in multimodal_smoke
+        and "['/mtmd-compat-missing.png']) === -4" in multimodal_smoke
+        and "core.FS.unlink(path)" in multimodal_smoke,
+        "multimodal smoke must exercise direct file/encoded PNG helpers, reject empty input, and clean its runtime-owned fixture",
+        errors,
+    )
     require(
         "python3 -m playwright install chromium" in ci
         and "playwright install --with-deps chromium" not in ci,

src/llama_webgpu_core.cpp

@@ -24,6 +24,7 @@
 #include "mtmd.h"
 
 #include "llama_webgpu_embedding_json.h"
+#include "llama_webgpu_mtmd_compat.h"
 #include "llama_webgpu_tts.h"
 
 namespace {
@@ -1461,7 +1462,7 @@ EMSCRIPTEN_KEEPALIVE int32_t llamadart_webgpu_media_add_file(
   }
 
   mtmd_helper_bitmap_wrapper media =
-      mtmd_helper_bitmap_init_from_file(g_state.mm_ctx, media_path, false);
+      llama_webgpu_bitmap_from_file(g_state.mm_ctx, media_path);
   if (media.bitmap == nullptr) {
     set_error("Failed to decode media file content");
     return -4;
@@ -1490,11 +1491,10 @@ EMSCRIPTEN_KEEPALIVE int32_t llamadart_webgpu_media_add_encoded(
     return -3;
   }
 
-  mtmd_helper_bitmap_wrapper media = mtmd_helper_bitmap_init_from_buf(
+  mtmd_helper_bitmap_wrapper media = llama_webgpu_bitmap_from_buffer(
       g_state.mm_ctx,
       bytes,
-      static_cast<size_t>(length),
-      false);
+      static_cast<size_t>(length));
   if (media.bitmap == nullptr) {
     set_error("Failed to decode encoded media bytes");
     return -4;

src/llama_webgpu_mtmd_compat.h

@@ -0,0 +1,25 @@
+#pragma once
+
+#include "mtmd-helper.h"
+
+// v0.4.0 added helper options. Preserve older upstream builds and use the
+// upstream defaults without changing the bridge's non-placeholder behavior.
+static inline mtmd_helper_bitmap_wrapper llama_webgpu_bitmap_from_buffer(
+    mtmd_context * ctx, const unsigned char * bytes, size_t length) {
+#if LLAMADART_MTMD_HELPER_HAS_OPTIONS
+  return mtmd_helper_bitmap_init_from_buf(
+      ctx, bytes, length, false, mtmd_helper_init_opt_default());
+#else
+  return mtmd_helper_bitmap_init_from_buf(ctx, bytes, length, false);
+#endif
+}
+
+static inline mtmd_helper_bitmap_wrapper llama_webgpu_bitmap_from_file(
+    mtmd_context * ctx, const char * path) {
+#if LLAMADART_MTMD_HELPER_HAS_OPTIONS
+  return mtmd_helper_bitmap_init_from_file(
+      ctx, path, false, mtmd_helper_init_opt_default());
+#else
+  return mtmd_helper_bitmap_init_from_file(ctx, path, false);
+#endif
+}

src/llama_webgpu_tts.cpp

@@ -1,4 +1,5 @@
 #include "llama_webgpu_tts.h"
+#include "llama_webgpu_mtmd_compat.h"
 
 #include <atomic>
 #include <cmath>
@@ -290,11 +291,10 @@ llama_webgpu_tts_status llama_webgpu_tts_start(
   tts->owns_embedding_mode = true;
 
   if (request.speaker_audio_length > 0) {
-    mtmd_helper_bitmap_wrapper wrapper = mtmd_helper_bitmap_init_from_buf(
+    mtmd_helper_bitmap_wrapper wrapper = llama_webgpu_bitmap_from_buffer(
         tts->mtmd,
         request.speaker_audio,
-        request.speaker_audio_length,
-        false);
+        request.speaker_audio_length);
     if (wrapper.bitmap == nullptr || !mtmd_bitmap_is_audio(wrapper.bitmap)) {
       if (wrapper.bitmap != nullptr) {
         mtmd_bitmap_free(wrapper.bitmap);