From 02872f264c79b70542f456072a4c5d6fc0671825 Mon Sep 17 00:00:00 2001 From: Jhin Lee Date: Sat, 5 Sep 2026 08:39:58 -0400 Subject: [PATCH 1/2] fix: support v0.4.0 Web media helper options --- .github/workflows/ci.yml | 23 ++++++-- CMakeLists.txt | 17 ++++++ README.md | 10 ++-- scripts/mtmd_compat_contract_test.py | 78 ++++++++++++++++++++++++++++ scripts/multimodal_browser_smoke.py | 43 +++++++++++++++ scripts/verify_ci_reliability.py | 22 ++++++++ src/llama_webgpu_core.cpp | 8 +-- src/llama_webgpu_mtmd_compat.h | 25 +++++++++ src/llama_webgpu_tts.cpp | 6 +-- 9 files changed, 218 insertions(+), 14 deletions(-) create mode 100644 scripts/mtmd_compat_contract_test.py create mode 100644 src/llama_webgpu_mtmd_compat.h diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 2233d54..627b110 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -15,8 +15,12 @@ env: jobs: build-webgpu-bridge: - name: Build WebGPU Bridge (WASM) + name: ${{ matrix.upstream == 'pinned' && 'Build WebGPU Bridge (WASM)' || 'Build WebGPU Bridge (WASM, v0.4.0)' }} runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + upstream: [pinned, v0.4.0] env: LLAMA_WEBGPU_SMOKE_MODEL_URL: https://huggingface.co/aladar/llama-2-tiny-random-GGUF/resolve/main/llama-2-tiny-random.gguf LLAMA_WEBGPU_SMOKE_MODEL_SHA256: 81f226c62d28ed4a1a9b9fa080fcd9f0cc40e0f9d5680036583ff98fbcd035cb @@ -39,6 +43,8 @@ jobs: LLAMA_WEBGPU_TTS_MMPROJ_SHA256: 6fd65188839bcd6ecc91b277ad471e22a0edfada4699a0fe82f1165c18cfcce2 steps: - uses: actions/checkout@v4 + with: + persist-credentials: false - name: Setup Node.js uses: actions/setup-node@v4 @@ -77,6 +83,9 @@ jobs: - name: Validate text-to-speech API contract run: python3 scripts/verify_text_to_speech_api.py + - name: Validate media helper compatibility + run: python3 scripts/mtmd_compat_contract_test.py + - name: Validate wasm64 runtime patch contract run: python3 scripts/wasm64_runtime_patch_contract_test.py @@ -118,6 +127,12 @@ jobs: run: | git clone --depth 1 --branch "$LLAMA_CPP_TAG" https://github.com/ggml-org/llama.cpp.git third_party/llama_cpp + - name: Select exact v0.4.0 compatibility revision + if: matrix.upstream == 'v0.4.0' + run: | + git -C third_party/llama_cpp fetch --depth 1 origin refs/tags/v0.4.0:refs/tags/v0.4.0 + git -C third_party/llama_cpp checkout --detach 5266f24da75dc449bd56cbed7addb9c8e4a6a73e + - name: Setup Emscripten SDK uses: mymindstorm/setup-emsdk@v14 with: @@ -161,7 +176,7 @@ jobs: if: failure() uses: actions/upload-artifact@v4 with: - name: state-persistence-smoke-artifacts + name: state-persistence-smoke-artifacts-${{ matrix.upstream }} path: ${{ env.LLAMA_WEBGPU_SMOKE_ARTIFACTS_DIR }} if-no-files-found: ignore @@ -169,14 +184,14 @@ jobs: if: failure() uses: actions/upload-artifact@v4 with: - name: multimodal-smoke-artifacts + name: multimodal-smoke-artifacts-${{ matrix.upstream }} path: ${{ env.LLAMA_WEBGPU_MULTIMODAL_ARTIFACTS_DIR }} if-no-files-found: ignore - name: Upload bridge artifacts uses: actions/upload-artifact@v4 with: - name: webgpu-bridge-dist + name: ${{ matrix.upstream == 'pinned' && 'webgpu-bridge-dist' || 'webgpu-bridge-dist-v0.4.0' }} path: | ${{ runner.temp }}/webgpu_bridge_dist/llama_webgpu_bridge.js ${{ runner.temp }}/webgpu_bridge_dist/llama_webgpu_bridge_worker.js diff --git a/CMakeLists.txt b/CMakeLists.txt index 6f6bd47..759c78e 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -62,6 +62,22 @@ set(GGML_BLAS OFF CACHE BOOL "" FORCE) add_subdirectory("${LLAMA_CPP_DIR}" "${CMAKE_BINARY_DIR}/llama_cpp") +include(CheckCXXSourceCompiles) +include(CMakePushCheckState) +cmake_push_check_state(RESET) +set(CMAKE_REQUIRED_INCLUDES + "${LLAMA_CPP_DIR}/include" + "${LLAMA_CPP_DIR}/ggml/include" + "${LLAMA_CPP_DIR}/tools/mtmd") +# Recheck when a build directory is reused with a different upstream checkout. +unset(LLAMADART_MTMD_HELPER_HAS_OPTIONS CACHE) +# Inspect declarations without calling upstream symbols or running target code. +check_cxx_source_compiles(" +#include +int main() { return sizeof(decltype(mtmd_helper_init_opt_default())) == 0; } +" LLAMADART_MTMD_HELPER_HAS_OPTIONS) +cmake_pop_check_state() + find_package(Threads REQUIRED) set(MTMD_AUDIO_SRC "${LLAMA_CPP_DIR}/tools/mtmd/mtmd-audio.cpp") @@ -113,6 +129,7 @@ add_executable(llama_webgpu_core target_compile_features(llama_webgpu_core PRIVATE cxx_std_17) target_compile_definitions(llama_webgpu_core PRIVATE + LLAMADART_MTMD_HELPER_HAS_OPTIONS=$ LLAMADART_WEBGPU_PTHREAD_POOL_SIZE=${LLAMADART_WEBGPU_PTHREAD_POOL_SIZE} ) diff --git a/README.md b/README.md index b2244cd..a2cf19d 100644 --- a/README.md +++ b/README.md @@ -181,11 +181,15 @@ This repo includes a wasm build gate in: - `.github/workflows/ci.yml` -It builds against the pinned `llama.cpp` tag in `llama_cpp.version`, runs the JS -bridge build/type-check gate, uploads build artifacts, and runs the static CI -reliability contract: +It builds wasm32 and memory64 against both the pinned `llama.cpp` tag in +`llama_cpp.version` and the exact v0.4.0 compatibility revision. Both lanes run +the JS/compatibility contracts and real state-persistence and multimodal browser +smokes. The pinned lane retains `webgpu-bridge-dist`; the compatibility lane +uploads `webgpu-bridge-dist-v0.4.0`. Neither changes a pin or publishes assets. +To run the media-helper and static CI contracts locally: ```bash +python3 scripts/mtmd_compat_contract_test.py python3 scripts/verify_ci_reliability.py ``` diff --git a/scripts/mtmd_compat_contract_test.py b/scripts/mtmd_compat_contract_test.py new file mode 100644 index 0000000..66dd451 --- /dev/null +++ b/scripts/mtmd_compat_contract_test.py @@ -0,0 +1,78 @@ +#!/usr/bin/env python3 +"""Compile both media-helper API shapes and guard all production call sites.""" + +import os +from pathlib import Path +import shlex +import subprocess +import tempfile +import unittest + + +ROOT = Path(__file__).resolve().parents[1] + + +class MediaCompatibilityTest(unittest.TestCase): + def test_both_upstream_api_shapes(self): + for options in (False, True): + with self.subTest(options=options), tempfile.TemporaryDirectory() as tmp: + directory = Path(tmp) + extra = ", mtmd_helper_init_opt options" if options else "" + check = "assert(options.sentinel == 73);" if options else "" + declarations = """ +struct mtmd_helper_init_opt { int sentinel; }; +inline mtmd_helper_init_opt mtmd_helper_init_opt_default() { return {73}; } +""" if options else "" + (directory / "mtmd-helper.h").write_text( + """#pragma once +#include +#include +struct mtmd_context {}; +struct mtmd_helper_bitmap_wrapper { int result; }; +inline mtmd_context expected_context; +inline const unsigned char expected_bytes[] = {1, 2, 3}; +inline const char expected_path[] = "reference.wav"; +""" + declarations + f""" +inline mtmd_helper_bitmap_wrapper mtmd_helper_bitmap_init_from_buf( + mtmd_context *ctx, const unsigned char *bytes, size_t size, bool placeholder{extra}) {{ + assert(ctx == &expected_context && bytes == expected_bytes && size == 3); + assert(!placeholder); {check} + return {{17}}; +}} +inline mtmd_helper_bitmap_wrapper mtmd_helper_bitmap_init_from_file( + mtmd_context *ctx, const char *path, bool placeholder{extra}) {{ + assert(ctx == &expected_context && path == expected_path); + assert(!placeholder); {check} + return {{29}}; +}} +""", encoding="utf-8") + source = directory / "test.cpp" + source.write_text(""" +#include "llama_webgpu_mtmd_compat.h" +int main() { + assert(llama_webgpu_bitmap_from_buffer(&expected_context, expected_bytes, 3).result == 17); + assert(llama_webgpu_bitmap_from_file(&expected_context, expected_path).result == 29); +} +""", encoding="utf-8") + executable = directory / "test" + subprocess.run([ + *shlex.split(os.environ.get("CXX", "c++")), "-std=c++17", + "-Wall", "-Wextra", "-Werror", + f"-DLLAMADART_MTMD_HELPER_HAS_OPTIONS={int(options)}", + f"-I{directory}", f"-I{ROOT / 'src'}", str(source), + "-o", str(executable), + ], check=True, capture_output=True, text=True) + subprocess.run([str(executable)], check=True) + + def test_production_routes_through_compatibility_helpers(self): + core = (ROOT / "src/llama_webgpu_core.cpp").read_text() + tts = (ROOT / "src/llama_webgpu_tts.cpp").read_text() + self.assertEqual(core.count("llama_webgpu_bitmap_from_file("), 1) + self.assertEqual(core.count("llama_webgpu_bitmap_from_buffer("), 1) + self.assertEqual(tts.count("llama_webgpu_bitmap_from_buffer("), 1) + for source in (core, tts): + self.assertNotIn("mtmd_helper_bitmap_init_from_", source) + + +if __name__ == "__main__": + unittest.main() diff --git a/scripts/multimodal_browser_smoke.py b/scripts/multimodal_browser_smoke.py index 4eb9a90..7b47063 100644 --- a/scripts/multimodal_browser_smoke.py +++ b/scripts/multimodal_browser_smoke.py @@ -118,6 +118,49 @@ def write_harness(web_root: Path) -> None: await bridge.loadMultimodalProjector('/multimodal-mmproj.gguf'); assert(bridge.supportsVision(), `${mode} did not report vision support`); + // Normal image ingestion resizes through RGB. Also exercise the native + // file/encoded entry points so helper API compatibility cannot hide + // behind that independent path. The direct runtime owns this FS. + if (disableWorker) { + const core = bridge._runtime?._core; + assert(core?.FS && typeof core.ccall === 'function', + 'direct runtime core is unavailable for media compatibility checks'); + const path = '/mtmd-compat.png'; + core.FS.writeFile(path, imageBytes); + try { + assert( + core.ccall('llamadart_webgpu_media_add_file', 'number', + ['string'], [path]) === 0, + 'native file helper failed to decode the real PNG', + ); + core.ccall('llamadart_webgpu_media_clear_pending', null, [], []); + assert( + core.ccall('llamadart_webgpu_media_add_encoded', 'number', + ['array', 'number'], [imageBytes, imageBytes.length]) === 0, + 'native buffer helper failed to decode the real PNG', + ); + core.ccall('llamadart_webgpu_media_clear_pending', null, [], []); + assert( + core.ccall('llamadart_webgpu_media_add_encoded', 'number', + ['array', 'number'], [imageBytes, 0]) === -3, + 'native buffer helper must reject empty input', + ); + assert( + core.ccall('llamadart_webgpu_media_add_encoded', 'number', + ['array', 'number'], [imageBytes.subarray(0, 8), 8]) === -4, + 'native buffer helper must reject a truncated PNG', + ); + assert( + core.ccall('llamadart_webgpu_media_add_file', 'number', + ['string'], ['/mtmd-compat-missing.png']) === -4, + 'native file helper must reject a missing file', + ); + } finally { + core.ccall('llamadart_webgpu_media_clear_pending', null, [], []); + core.FS.unlink(path); + } + } + const output = await bridge.createCompletion('what do you see?', { nPredict: 64, temp: 0, diff --git a/scripts/verify_ci_reliability.py b/scripts/verify_ci_reliability.py index b633b65..f91f3d6 100644 --- a/scripts/verify_ci_reliability.py +++ b/scripts/verify_ci_reliability.py @@ -1223,6 +1223,28 @@ def main() -> int: "ci.yml must resolve the llama.cpp tag from llama_cpp.version, support explicit dispatch, and avoid hard-coded stale defaults", errors, ) + require( + "upstream: [pinned, v0.4.0]" in ci + and "5266f24da75dc449bd56cbed7addb9c8e4a6a73e" in ci + and "'Build WebGPU Bridge (WASM)'" in ci + and "'webgpu-bridge-dist-v0.4.0'" in ci + and "state-persistence-smoke-artifacts-${{ matrix.upstream }}" in ci + and "multimodal-smoke-artifacts-${{ matrix.upstream }}" in ci + and "python3 scripts/mtmd_compat_contract_test.py" in ci, + "CI must retain the pinned check and artifact identities, test exact v0.4.0 media compatibility, and isolate matrix artifacts", + errors, + ) + require( + "bridge._runtime?._core" in multimodal_smoke + and "llamadart_webgpu_media_add_file" in multimodal_smoke + and "llamadart_webgpu_media_add_encoded" in multimodal_smoke + and "[imageBytes, 0]) === -3" in multimodal_smoke + and "[imageBytes.subarray(0, 8), 8]) === -4" in multimodal_smoke + and "['/mtmd-compat-missing.png']) === -4" in multimodal_smoke + and "core.FS.unlink(path)" in multimodal_smoke, + "multimodal smoke must exercise direct file/encoded PNG helpers, reject empty input, and clean its runtime-owned fixture", + errors, + ) require( "python3 -m playwright install chromium" in ci and "playwright install --with-deps chromium" not in ci, diff --git a/src/llama_webgpu_core.cpp b/src/llama_webgpu_core.cpp index eac2717..83f801e 100644 --- a/src/llama_webgpu_core.cpp +++ b/src/llama_webgpu_core.cpp @@ -24,6 +24,7 @@ #include "mtmd.h" #include "llama_webgpu_embedding_json.h" +#include "llama_webgpu_mtmd_compat.h" #include "llama_webgpu_tts.h" namespace { @@ -1461,7 +1462,7 @@ EMSCRIPTEN_KEEPALIVE int32_t llamadart_webgpu_media_add_file( } mtmd_helper_bitmap_wrapper media = - mtmd_helper_bitmap_init_from_file(g_state.mm_ctx, media_path, false); + llama_webgpu_bitmap_from_file(g_state.mm_ctx, media_path); if (media.bitmap == nullptr) { set_error("Failed to decode media file content"); return -4; @@ -1490,11 +1491,10 @@ EMSCRIPTEN_KEEPALIVE int32_t llamadart_webgpu_media_add_encoded( return -3; } - mtmd_helper_bitmap_wrapper media = mtmd_helper_bitmap_init_from_buf( + mtmd_helper_bitmap_wrapper media = llama_webgpu_bitmap_from_buffer( g_state.mm_ctx, bytes, - static_cast(length), - false); + static_cast(length)); if (media.bitmap == nullptr) { set_error("Failed to decode encoded media bytes"); return -4; diff --git a/src/llama_webgpu_mtmd_compat.h b/src/llama_webgpu_mtmd_compat.h new file mode 100644 index 0000000..6ea9719 --- /dev/null +++ b/src/llama_webgpu_mtmd_compat.h @@ -0,0 +1,25 @@ +#pragma once + +#include "mtmd-helper.h" + +// v0.4.0 added helper options. Preserve older upstream builds and use the +// upstream defaults without changing the bridge's non-placeholder behavior. +static inline mtmd_helper_bitmap_wrapper llama_webgpu_bitmap_from_buffer( + mtmd_context * ctx, const unsigned char * bytes, size_t length) { +#if LLAMADART_MTMD_HELPER_HAS_OPTIONS + return mtmd_helper_bitmap_init_from_buf( + ctx, bytes, length, false, mtmd_helper_init_opt_default()); +#else + return mtmd_helper_bitmap_init_from_buf(ctx, bytes, length, false); +#endif +} + +static inline mtmd_helper_bitmap_wrapper llama_webgpu_bitmap_from_file( + mtmd_context * ctx, const char * path) { +#if LLAMADART_MTMD_HELPER_HAS_OPTIONS + return mtmd_helper_bitmap_init_from_file( + ctx, path, false, mtmd_helper_init_opt_default()); +#else + return mtmd_helper_bitmap_init_from_file(ctx, path, false); +#endif +} diff --git a/src/llama_webgpu_tts.cpp b/src/llama_webgpu_tts.cpp index 874f2f9..1954f1a 100644 --- a/src/llama_webgpu_tts.cpp +++ b/src/llama_webgpu_tts.cpp @@ -1,4 +1,5 @@ #include "llama_webgpu_tts.h" +#include "llama_webgpu_mtmd_compat.h" #include #include @@ -290,11 +291,10 @@ llama_webgpu_tts_status llama_webgpu_tts_start( tts->owns_embedding_mode = true; if (request.speaker_audio_length > 0) { - mtmd_helper_bitmap_wrapper wrapper = mtmd_helper_bitmap_init_from_buf( + mtmd_helper_bitmap_wrapper wrapper = llama_webgpu_bitmap_from_buffer( tts->mtmd, request.speaker_audio, - request.speaker_audio_length, - false); + request.speaker_audio_length); if (wrapper.bitmap == nullptr || !mtmd_bitmap_is_audio(wrapper.bitmap)) { if (wrapper.bitmap != nullptr) { mtmd_bitmap_free(wrapper.bitmap); From 20e4663d97302e7906c7557c31ab4c98dbaabcef Mon Sep 17 00:00:00 2001 From: Jhin Lee Date: Sat, 5 Sep 2026 08:45:25 -0400 Subject: [PATCH 2/2] docs: align Web compatibility and qualification guidance --- AGENTS.md | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/AGENTS.md b/AGENTS.md index 8822253..0d5c1bf 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -68,6 +68,7 @@ npm run check:js python3 -m py_compile scripts/verify_state_persistence_api.py scripts/verify_text_to_speech_api.py scripts/verify_ci_reliability.py scripts/state_persistence_browser_smoke.py scripts/multimodal_browser_smoke.py scripts/speech_to_text_browser_smoke.py scripts/text_to_speech_browser_smoke.py python3 scripts/verify_state_persistence_api.py python3 scripts/verify_text_to_speech_api.py +python3 scripts/mtmd_compat_contract_test.py python3 scripts/verify_ci_reliability.py ``` @@ -135,6 +136,10 @@ automated qualification run binds the candidate digest it is about to publish. - CI build gate: `.github/workflows/ci.yml` - Resolves the default llama.cpp checkout from `llama_cpp.version`. + - Builds wasm32/memory64 against that pin and exact v0.4.0 in separate matrix + lanes, with media-helper API contracts plus real state and image smokes. + Preserve the original pinned check/artifact identities and isolate each + lane's artifact uploads. Neither lane publishes or changes the source pin. - Resolves `emsdk.version`, installs that exact compiler, verifies the active `emcc` identity, and contract-tests all five required wasm64 WASMFS patches. - Never dispatches asset publication. Bridge source changes, including changes @@ -373,6 +378,7 @@ After publishing assets tag: - Every llama.cpp pin update must pass checksum-pinned real multimodal inference in both direct and worker runtimes; a successful WASM build alone is not sufficient. -- Speech-capable asset releases must pass the required local qualification in - wasm32 and memory64; never run the large Qwen3-ASR/Qwen3-TTS pairs on hosted - runners. +- Speech-capable asset releases must pass the hosted automated qualification + workflow's exact required memory/runtime matrix before publication. Keep + these heavy gates outside ordinary CI; reproduce locally with the individual + smoke scripts above, not the workflow-only combined qualifier.