Implement manifest-driven shader pipeline with runtime library, hot-reload, and metadata

CMakeLists.txt

@@ -114,6 +114,8 @@ add_library(render_renderer STATIC
   engine/render/renderer_types.hpp
   engine/render/buffer_types.hpp
   engine/render/shader_types.hpp
+  engine/render/shader_library.hpp
+  engine/render/shader_library.cpp
   engine/render/texture_types.hpp
   engine/render/bgfx/renderer_bgfx.cpp
 )
@@ -186,6 +188,11 @@ if(RENDER_BUILD_TESTS)
   render_apply_project_options(render_renderer_lifecycle_tests)
   render_apply_warnings(render_renderer_lifecycle_tests)
 
+  add_executable(render_shader_pipeline_tests tests/render/shader_pipeline_tests.cpp)
+  target_link_libraries(render_shader_pipeline_tests PRIVATE render::renderer render::filesystem)
+  render_apply_project_options(render_shader_pipeline_tests)
+  render_apply_warnings(render_shader_pipeline_tests)
+
   add_test(NAME unit.platform.types COMMAND render_platform_types_tests)
   set_tests_properties(unit.platform.types PROPERTIES LABELS "unit;platform")
   add_test(NAME unit.core.runtime COMMAND render_core_runtime_tests)
@@ -196,12 +203,14 @@ if(RENDER_BUILD_TESTS)
   set_tests_properties(unit.filesystem.service PROPERTIES LABELS "unit;filesystem")
   add_test(NAME unit.renderer.lifecycle COMMAND render_renderer_lifecycle_tests)
   set_tests_properties(unit.renderer.lifecycle PROPERTIES LABELS "unit;renderer")
+  add_test(NAME unit.renderer.shader_pipeline COMMAND render_shader_pipeline_tests)
+  set_tests_properties(unit.renderer.shader_pipeline PROPERTIES LABELS "unit;renderer;shader")
   add_test(NAME headless.smoke.startup COMMAND render_headless_validation)
   set_tests_properties(headless.smoke.startup PROPERTIES LABELS "headless;smoke")
 
   add_custom_target(render_test_unit
     COMMAND ${CMAKE_CTEST_COMMAND} --output-on-failure --label-regex unit
-    DEPENDS render_platform_types_tests render_core_runtime_tests render_serialization_tests render_filesystem_tests render_renderer_lifecycle_tests
+    DEPENDS render_platform_types_tests render_core_runtime_tests render_serialization_tests render_filesystem_tests render_renderer_lifecycle_tests render_shader_pipeline_tests
     WORKING_DIRECTORY "${CMAKE_BINARY_DIR}"
     COMMENT "Running render unit tests"
   )
@@ -220,6 +229,7 @@ set(RENDER_PACKAGE_SHADER_RELATIVE_DIR "shaders/bin")
 
 install(TARGETS render_shell RUNTIME DESTINATION "${CMAKE_INSTALL_BINDIR}")
 install(DIRECTORY "${CMAKE_BINARY_DIR}/bin/shaders/bin/" DESTINATION "shaders/bin" OPTIONAL)
+install(DIRECTORY "${CMAKE_BINARY_DIR}/bin/shaders/metadata/" DESTINATION "shaders/metadata" OPTIONAL)
 
 add_custom_target(render_package_stage
   COMMAND ${CMAKE_COMMAND} -E rm -rf "${RENDER_PACKAGE_STAGE_DIR}"

cmake/ShaderCompilation.cmake

@@ -1,24 +1,39 @@
 function(render_setup_shader_compilation)
-  set(RENDER_SHADER_OUTPUT_DIR "${CMAKE_BINARY_DIR}/bin/shaders/bin" CACHE PATH "Directory for compiled shader binaries")
+  set(RENDER_SHADER_OUTPUT_ROOT "${CMAKE_BINARY_DIR}/bin/shaders" CACHE PATH "Root directory for compiled shader artifacts")
+  set(RENDER_SHADER_MANIFEST "${CMAKE_SOURCE_DIR}/shaders/shaders.cmake" CACHE FILEPATH "Shader manifest file")
   set(RENDER_BGFX_SHADERC "" CACHE FILEPATH "Path to bgfx shaderc executable")
+  set(RENDER_SHADER_BACKENDS "" CACHE STRING "Semicolon-separated backend list override (dx11;spirv;glsl;metal)")
   option(RENDER_REQUIRE_SHADER_COMPILATION "Fail configure/build checks when shader compilation is unavailable" OFF)
 
   set(shaderc_candidate "${RENDER_BGFX_SHADERC}")
   if(NOT shaderc_candidate AND TARGET shaderc)
     set(shaderc_candidate "$<TARGET_FILE:shaderc>")
   endif()
 
+  if(RENDER_SHADER_BACKENDS)
+    set(backend_list "${RENDER_SHADER_BACKENDS}")
+  elseif(WIN32)
+    set(backend_list "dx11")
+  elseif(APPLE)
+    set(backend_list "metal")
+  else()
+    set(backend_list "spirv;glsl")
+  endif()
+
   add_custom_target(render_shaders
-    COMMAND ${CMAKE_COMMAND} -E make_directory "${RENDER_SHADER_OUTPUT_DIR}"
+    COMMAND ${CMAKE_COMMAND} -E make_directory "${RENDER_SHADER_OUTPUT_ROOT}/bin"
+    COMMAND ${CMAKE_COMMAND} -E make_directory "${RENDER_SHADER_OUTPUT_ROOT}/metadata"
     COMMAND ${CMAKE_COMMAND} -E make_directory "${CMAKE_BINARY_DIR}/logs"
     COMMAND ${CMAKE_COMMAND}
       -DRENDER_SOURCE_DIR=${CMAKE_SOURCE_DIR}
-      -DRENDER_SHADER_OUTPUT_DIR=${RENDER_SHADER_OUTPUT_DIR}
+      -DRENDER_SHADER_OUTPUT_ROOT=${RENDER_SHADER_OUTPUT_ROOT}
+      -DRENDER_SHADER_MANIFEST=${RENDER_SHADER_MANIFEST}
       -DRENDER_BGFX_SHADERC=${shaderc_candidate}
       -DRENDER_REQUIRE_SHADER_COMPILATION=${RENDER_REQUIRE_SHADER_COMPILATION}
+      -DRENDER_SHADER_BACKENDS=${backend_list}
       -DRENDER_SHADER_LOG_FILE=${CMAKE_BINARY_DIR}/logs/shader_compilation.log
       -P "${CMAKE_SOURCE_DIR}/cmake/compile_shaders.cmake"
-    COMMENT "Compiling bgfx shaders"
+    COMMENT "Compiling render shader pipeline assets"
     VERBATIM
   )
 

cmake/compile_shaders.cmake

@@ -1,5 +1,9 @@
-if(NOT EXISTS "${RENDER_SOURCE_DIR}/shaders/src")
-  message(FATAL_ERROR "Shader source directory missing: ${RENDER_SOURCE_DIR}/shaders/src")
+if(NOT EXISTS "${RENDER_SOURCE_DIR}/shaders")
+  message(FATAL_ERROR "Shader source directory missing: ${RENDER_SOURCE_DIR}/shaders")
+endif()
+
+if(NOT EXISTS "${RENDER_SHADER_MANIFEST}")
+  message(FATAL_ERROR "Shader manifest missing: ${RENDER_SHADER_MANIFEST}")
 endif()
 
 if(NOT DEFINED RENDER_SHADER_LOG_FILE)
@@ -20,7 +24,7 @@ if(NOT RENDER_BGFX_SHADERC)
   if(RENDER_REQUIRE_SHADER_COMPILATION)
     message(FATAL_ERROR "RENDER_BGFX_SHADERC not set and RENDER_REQUIRE_SHADER_COMPILATION=ON")
   endif()
-  message(WARNING "RENDER_BGFX_SHADERC not set; skipping shader compilation. Expected precompiled files in shaders/bin/<backend>.")
+  message(WARNING "RENDER_BGFX_SHADERC not set; skipping shader compilation. Expected staged files in ${RENDER_SHADER_OUTPUT_ROOT}.")
   render_shader_log("skipped: shaderc unavailable")
   return()
 endif()
@@ -34,62 +38,182 @@ if(NOT EXISTS "${RENDER_BGFX_SHADERC}")
   return()
 endif()
 
-set(SHADER_SOURCE_DIR "${RENDER_SOURCE_DIR}/shaders/src")
-set(SHADER_OUTPUT_DIR "${RENDER_SHADER_OUTPUT_DIR}")
-file(MAKE_DIRECTORY "${SHADER_OUTPUT_DIR}")
+include("${RENDER_SHADER_MANIFEST}")
+
+set(SHADER_ROOT "${RENDER_SOURCE_DIR}/shaders")
+set(SHADER_SOURCE_ROOT "${SHADER_ROOT}")
+set(SHADER_OUTPUT_BIN_ROOT "${RENDER_SHADER_OUTPUT_ROOT}/bin")
+set(SHADER_OUTPUT_META_ROOT "${RENDER_SHADER_OUTPUT_ROOT}/metadata")
+
+if(NOT RENDER_SHADER_BACKENDS)
+  message(FATAL_ERROR "RENDER_SHADER_BACKENDS is empty")
+endif()
+
+set(RENDER_SHADER_STAGES vs fs cs)
+
+function(render_backend_profile backend stage out_platform_var out_profile_var)
+  if("${backend}" STREQUAL "spirv")
+    set(platform linux PARENT_SCOPE)
+    if("${stage}" STREQUAL "cs")
+      set(profile spirv15-12 PARENT_SCOPE)
+    else()
+      set(profile spirv PARENT_SCOPE)
+    endif()
+  elseif("${backend}" STREQUAL "glsl")
+    set(platform linux PARENT_SCOPE)
+    if("${stage}" STREQUAL "cs")
+      set(profile 430 PARENT_SCOPE)
+    else()
+      set(profile 330 PARENT_SCOPE)
+    endif()
+  elseif("${backend}" STREQUAL "metal")
+    set(platform osx PARENT_SCOPE)
+    set(profile metal PARENT_SCOPE)
+  elseif("${backend}" STREQUAL "dx11")
+    set(platform windows PARENT_SCOPE)
+    if("${stage}" STREQUAL "vs")
+      set(profile vs_5_0 PARENT_SCOPE)
+    elseif("${stage}" STREQUAL "fs")
+      set(profile ps_5_0 PARENT_SCOPE)
+    else()
+      set(profile cs_5_0 PARENT_SCOPE)
+    endif()
+  else()
+    message(FATAL_ERROR "Unsupported shader backend '${backend}'")
+  endif()
+endfunction()
+
+function(render_compile_stage program category stage backend source_rel variant defines)
+  set(source_path "${SHADER_SOURCE_ROOT}/${source_rel}")
+  if(NOT EXISTS "${source_path}")
+    message(FATAL_ERROR "Shader source not found for ${program}/${stage}: ${source_path}")
+  endif()
+
+  render_backend_profile("${backend}" "${stage}" platform profile)
+
+  if("${stage}" STREQUAL "vs")
+    set(shader_type "vertex")
+  elseif("${stage}" STREQUAL "fs")
+    set(shader_type "fragment")
+  elseif("${stage}" STREQUAL "cs")
+    set(shader_type "compute")
+  else()
+    message(FATAL_ERROR "Unsupported shader stage '${stage}'")
+  endif()
+
+  set(output_dir "${SHADER_OUTPUT_BIN_ROOT}/${backend}/${category}/${program}/${variant}")
+  file(MAKE_DIRECTORY "${output_dir}")
+  set(output_file "${output_dir}/${stage}.bin")
+
+  set(stage_args
+    --type ${shader_type}
+    --platform ${platform}
+    --profile ${profile}
+    --varyingdef "${SHADER_ROOT}/includes/varying_color.def.sc"
+    --include-dir "${SHADER_ROOT}"
+    --include-dir "${SHADER_ROOT}/includes"
+    -f "${source_path}"
+    -o "${output_file}"
+  )
+
+  if(defines)
+    list(APPEND stage_args --define "${defines}")
+  endif()
+
+  execute_process(
+    COMMAND "${RENDER_BGFX_SHADERC}" ${stage_args}
+    RESULT_VARIABLE shader_result
+    OUTPUT_VARIABLE shader_stdout
+    ERROR_VARIABLE shader_stderr
+  )
+
+  if(NOT shader_result EQUAL 0)
+    message(FATAL_ERROR
+      "Failed to compile shader\n"
+      "  program=${program}\n"
+      "  category=${category}\n"
+      "  stage=${stage}\n"
+      "  backend=${backend}\n"
+      "  variant=${variant}\n"
+      "  source=${source_rel}\n"
+      "  defines=${defines}\n"
+      "  stdout=${shader_stdout}\n"
+      "  stderr=${shader_stderr}")
+  endif()
+
+  file(SHA256 "${output_file}" artifact_hash)
+
+  set(meta_dir "${SHADER_OUTPUT_META_ROOT}/${backend}/${category}/${program}/${variant}")
+  file(MAKE_DIRECTORY "${meta_dir}")
+  set(meta_file "${meta_dir}/${stage}.json")
+
+  string(REPLACE "\"" "\\\"" safe_defines "${defines}")
+
+  file(WRITE "${meta_file}" "{\n")
+  file(APPEND "${meta_file}" "  \"schema_version\": 1,\n")
+  file(APPEND "${meta_file}" "  \"program\": \"${program}\",\n")
+  file(APPEND "${meta_file}" "  \"category\": \"${category}\",\n")
+  file(APPEND "${meta_file}" "  \"variant\": \"${variant}\",\n")
+  file(APPEND "${meta_file}" "  \"stage\": \"${stage}\",\n")
+  file(APPEND "${meta_file}" "  \"backend\": \"${backend}\",\n")
+  file(APPEND "${meta_file}" "  \"source\": \"${source_rel}\",\n")
+  file(APPEND "${meta_file}" "  \"compiled_binary\": \"${backend}/${category}/${program}/${variant}/${stage}.bin\",\n")
+  file(APPEND "${meta_file}" "  \"defines\": \"${safe_defines}\",\n")
+  file(APPEND "${meta_file}" "  \"binary_sha256\": \"${artifact_hash}\"\n")
+  file(APPEND "${meta_file}" "}\n")
+
+  render_shader_log("[ok] ${backend}/${category}/${program}/${variant}/${stage}.bin")
+endfunction()
+
+function(render_emit_program_metadata program category backend variant defines)
+  set(meta_dir "${SHADER_OUTPUT_META_ROOT}/${backend}/${category}/${program}/${variant}")
+  file(MAKE_DIRECTORY "${meta_dir}")
+  set(program_file "${meta_dir}/program.json")
+  string(REPLACE "\"" "\\\"" safe_defines "${defines}")
+
+  file(WRITE "${program_file}" "{\n")
+  file(APPEND "${program_file}" "  \"schema_version\": 1,\n")
+  file(APPEND "${program_file}" "  \"program\": \"${program}\",\n")
+  file(APPEND "${program_file}" "  \"category\": \"${category}\",\n")
+  file(APPEND "${program_file}" "  \"backend\": \"${backend}\",\n")
+  file(APPEND "${program_file}" "  \"variant\": \"${variant}\",\n")
+  file(APPEND "${program_file}" "  \"stages\": [\"vs\", \"fs\"],\n")
+  file(APPEND "${program_file}" "  \"defines\": \"${safe_defines}\",\n")
+  file(APPEND "${program_file}" "  \"vertex_binary\": \"${backend}/${category}/${program}/${variant}/vs.bin\",\n")
+  file(APPEND "${program_file}" "  \"fragment_binary\": \"${backend}/${category}/${program}/${variant}/fs.bin\",\n")
+  file(APPEND "${program_file}" "  \"reflection_files\": [\n")
+  file(APPEND "${program_file}" "    \"${backend}/${category}/${program}/${variant}/vs.json\",\n")
+  file(APPEND "${program_file}" "    \"${backend}/${category}/${program}/${variant}/fs.json\"\n")
+  file(APPEND "${program_file}" "  ]\n")
+  file(APPEND "${program_file}" "}\n")
+
+  render_shader_log("[meta] ${backend}/${category}/${program}/${variant}/program.json")
+endfunction()
+
+file(MAKE_DIRECTORY "${SHADER_OUTPUT_BIN_ROOT}")
+file(MAKE_DIRECTORY "${SHADER_OUTPUT_META_ROOT}")
 render_shader_log("shaderc: ${RENDER_BGFX_SHADERC}")
-render_shader_log("output: ${SHADER_OUTPUT_DIR}")
-
-set(RENDER_BACKENDS spirv glsl metal dx11)
-set(RENDER_SHADER_STAGES vs fs)
-set(RENDER_SHADER_NAMES statement4)
-
-foreach(backend IN LISTS RENDER_BACKENDS)
-  file(MAKE_DIRECTORY "${SHADER_OUTPUT_DIR}/${backend}")
-
-  foreach(stage IN LISTS RENDER_SHADER_STAGES)
-    foreach(name IN LISTS RENDER_SHADER_NAMES)
-      set(in_file "${SHADER_SOURCE_DIR}/${stage}_${name}.sc")
-      set(out_file "${SHADER_OUTPUT_DIR}/${backend}/${stage}_${name}.bin")
-
-      if(stage STREQUAL "vs")
-        set(shader_type "vertex")
-      else()
-        set(shader_type "fragment")
-      endif()
-
-      if(backend STREQUAL "spirv")
-        set(platform linux)
-        set(profile spirv)
-      elseif(backend STREQUAL "glsl")
-        set(platform linux)
-        set(profile 330)
-      elseif(backend STREQUAL "metal")
-        set(platform osx)
-        set(profile metal)
-      elseif(backend STREQUAL "dx11")
-        set(platform windows)
-        if(stage STREQUAL "vs")
-          set(profile vs_5_0)
-        else()
-          set(profile ps_5_0)
-        endif()
-      endif()
-
-      execute_process(
-        COMMAND "${RENDER_BGFX_SHADERC}" --type ${shader_type} --platform ${platform} --profile ${profile}
-          --include-dir "${SHADER_SOURCE_DIR}" -f "${in_file}" -o "${out_file}"
-        RESULT_VARIABLE shader_result
-        OUTPUT_VARIABLE shader_stdout
-        ERROR_VARIABLE shader_stderr
-      )
-
-      if(NOT shader_result EQUAL 0)
-        message(FATAL_ERROR "Failed to compile ${in_file} for ${backend}:\n${shader_stdout}\n${shader_stderr}")
-      endif()
-      render_shader_log("[ok] ${backend}/${stage}_${name}.bin")
-    endforeach()
+render_shader_log("output root: ${RENDER_SHADER_OUTPUT_ROOT}")
+
+foreach(backend IN LISTS RENDER_SHADER_BACKENDS)
+  foreach(entry IN LISTS RENDER_SHADER_PROGRAMS)
+    string(REPLACE "|" ";" entry_fields "${entry}")
+    list(LENGTH entry_fields entry_count)
+    if(NOT entry_count EQUAL 6)
+      message(FATAL_ERROR "Invalid shader manifest entry '${entry}'. Expected 6 fields")
+    endif()
+
+    list(GET entry_fields 0 program)
+    list(GET entry_fields 1 category)
+    list(GET entry_fields 2 vs_source)
+    list(GET entry_fields 3 fs_source)
+    list(GET entry_fields 4 variant)
+    list(GET entry_fields 5 defines)
+
+    render_compile_stage("${program}" "${category}" vs "${backend}" "${vs_source}" "${variant}" "${defines}")
+    render_compile_stage("${program}" "${category}" fs "${backend}" "${fs_source}" "${variant}" "${defines}")
+    render_emit_program_metadata("${program}" "${category}" "${backend}" "${variant}" "${defines}")
   endforeach()
 endforeach()
 
-message(STATUS "Shader compilation complete: ${SHADER_OUTPUT_DIR}")
+message(STATUS "Shader pipeline compilation complete: ${RENDER_SHADER_OUTPUT_ROOT}")

cmake/validate_package_layout.cmake

@@ -27,6 +27,7 @@ endif()
 set(stage_dir "${RENDER_PACKAGE_STAGE_DIR}")
 set(bin_dir "${stage_dir}/${RENDER_PACKAGE_BIN_SUBDIR}")
 set(shader_dir "${stage_dir}/${RENDER_PACKAGE_SHADER_RELATIVE_DIR}")
+set(shader_metadata_dir "${stage_dir}/shaders/metadata")
 
 if(WIN32)
   set(shell_name "render_shell.exe")
@@ -54,6 +55,20 @@ if(shader_count EQUAL 0)
 endif()
 render_package_log("[ok] compiled shader count: ${shader_count}")
 
+if(NOT EXISTS "${shader_metadata_dir}")
+  message(FATAL_ERROR "Packaging validation failed: missing shader metadata directory ${shader_metadata_dir}")
+endif()
+render_package_log("[ok] shader metadata directory: ${shader_metadata_dir}")
+
+file(GLOB_RECURSE shader_metadata_files
+  "${shader_metadata_dir}/*.json"
+)
+list(LENGTH shader_metadata_files metadata_count)
+if(metadata_count EQUAL 0)
+  message(FATAL_ERROR "Packaging validation failed: no shader metadata files found in ${shader_metadata_dir}")
+endif()
+render_package_log("[ok] shader metadata count: ${metadata_count}")
+
 set(archive_base "${stage_dir}/../render-package")
 if(WIN32)
   set(archive_path "${archive_base}.zip")

docs/build.md

@@ -23,12 +23,17 @@ Runtime dependencies can come from:
 - SDL3 system package for SDL only (`find_package(SDL3 CONFIG)`)
 - FetchContent fallback (`-DRENDER_ALLOW_FETCHCONTENT=ON`)
 
-## Shader compilation options
-
-- `render_shell` expects compiled shaders at runtime under `bin/shaders/bin/<backend>/`.
-- The `render_shader_check` target drives shader compilation and validation.
+## Shader pipeline options
+
+- Shader sources and conventions live under `shaders/` with manifest-driven entries in `shaders/shaders.cmake`.
+- `render_shaders` compiles binaries and reflection metadata under `<build>/bin/shaders/`.
+- Runtime expects:
+  - binaries: `bin/shaders/bin/<backend>/<category>/<program>/<variant>/<stage>.bin`
+  - metadata: `bin/shaders/metadata/<backend>/<category>/<program>/<variant>/*.json`
+- `render_shader_check` drives the same shader pipeline used by shell/tests/packaging.
 - If `RENDER_BGFX_SHADERC` is not set, compilation is skipped unless `RENDER_REQUIRE_SHADER_COMPILATION=ON`.
 - Enable `RENDER_BGFX_BUILD_TOOLS=ON` to build bgfx tooling (including `shaderc`) when available.
+- Use `RENDER_SHADER_BACKENDS` to override host-default backend list.
 
 ## Quick Start (Linux/macOS)