From 8f1f3c63f3fca876b853ab16977ba724a3be4fbe Mon Sep 17 00:00:00 2001 From: Jhin Lee Date: Mon, 17 Aug 2026 13:22:38 -0400 Subject: [PATCH 1/3] feat: select Windows CUDA sidecars --- AGENTS.md | 5 + CHANGELOG.md | 6 + hook/build.dart | 221 ++++++++++++++++- .../backends/llama_cpp/llama_cpp_service.dart | 57 ++++- .../llama_cpp/windows_cuda_selector.dart | 112 +++++++++ lib/src/hook/native_bundle_config.dart | 34 +++ lib/src/hook/windows_cuda_pack.dart | 226 ++++++++++++++++++ .../llama_cpp/llama_cpp_service_test.dart | 22 ++ .../llama_cpp/windows_cuda_selector_test.dart | 82 +++++++ .../hook/build_hook_integration_test.dart | 168 +++++++++++++ test/unit/hook/native_bundle_config_test.dart | 32 +++ test/unit/hook/windows_cuda_pack_test.dart | 144 +++++++++++ website/docs/getting-started/installation.md | 20 ++ website/docs/platforms/native-build-hooks.md | 7 + website/docs/platforms/support-matrix.md | 10 +- 15 files changed, 1137 insertions(+), 9 deletions(-) create mode 100644 lib/src/backends/llama_cpp/windows_cuda_selector.dart create mode 100644 lib/src/hook/windows_cuda_pack.dart create mode 100644 test/unit/backends/llama_cpp/windows_cuda_selector_test.dart create mode 100644 test/unit/hook/windows_cuda_pack_test.dart diff --git a/AGENTS.md b/AGENTS.md index 9e97c528..8e96e515 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -203,6 +203,11 @@ WEBGPU_BRIDGE_ASSETS_TAG= ./scripts/fetch_webgpu_bridge_assets.sh See `website/docs/maintainers/native-and-web-sync.md` for the full maintainer procedure. +Windows x64 CUDA sidecars are selected only when the `cuda` backend is +requested. Keep `llamadart_windows_cuda` values limited to `12`, `13`, or +`both`; `13` is the sidecar default. A `both` build may package both dependency +families but must select and load exactly one CUDA backend per process. + ## Changelog And Releases - Never add unreleased work to an already-published version section in diff --git a/CHANGELOG.md b/CHANGELOG.md index 5da2db85..ad865791 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,11 @@ ## Unreleased +* Added verified Windows x64 CUDA sidecar selection for native llama.cpp + builds. CUDA 13 is the default when CUDA is requested, with CUDA 12 and a + portable `both` option available through `llamadart_windows_cuda`; dual-pack + builds probe the target NVIDIA driver/GPU and load only one matching CUDA + dependency family per process. + * Updated the default llama.cpp native runtime pin to `leehack/llamadart-native@b10453`, picking up llama.cpp fixes for a Granite Speech heap overflow, a DOTS OCR out-of-bounds write, and a Step3VL diff --git a/hook/build.dart b/hook/build.dart index a46d2d3c..63de9c6e 100644 --- a/hook/build.dart +++ b/hook/build.dart @@ -12,6 +12,7 @@ import 'package:logging/logging.dart'; import 'package:path/path.dart' as path; import 'package:llamadart/src/hook/native_bundle_config.dart'; +import 'package:llamadart/src/hook/windows_cuda_pack.dart'; const _llamaCppTag = 'b10453'; const _nativeRepoSlug = 'leehack/llamadart-native'; @@ -156,6 +157,7 @@ const _litertLmBundleSpecs = <_LiteRtLmBundleSpec>[ const _dynamicLibraryExtensions = {'.so', '.dylib', '.dll'}; final _windowsCudartPattern = RegExp(r'^cudart64(?:[_-]?\d+)?\.dll$'); final _windowsCublasPattern = RegExp(r'^cublas64(?:[_-]?\d+)?\.dll$'); +final _windowsCublasLtPattern = RegExp(r'^cublaslt64(?:[_-]?\d+)?\.dll$'); final _linuxVersionedSoPattern = RegExp(r'\.so\.\d+$'); final _nativeTagPattern = RegExp(r'^[A-Za-z0-9][A-Za-z0-9._-]*$'); final _githubRepoSegmentPattern = RegExp(r'^[A-Za-z0-9_.-]+$'); @@ -350,11 +352,49 @@ void main(List args) async { log: log, ); - final libraryPaths = _collectDynamicLibraryPaths(bundleDir); + var libraryPaths = _collectDynamicLibraryPaths(bundleDir); if (libraryPaths.isEmpty) { throw Exception('No dynamic libraries found in ${bundleDir.path}.'); } + final requestedBackends = parseRequestedBackends( + bundle: spec.bundle, + rawUserConfig: input.userDefines[nativeBackendUserDefineKey], + ); + final wantsWindowsCudaSidecar = + spec.bundle == 'windows-x64' && + requestedBackends?.contains('cuda') == true; + if (wantsWindowsCudaSidecar) { + final rawCudaSelection = + input.userDefines[nativeWindowsCudaUserDefineKey]; + final legacyCudaAvailable = libraryPaths.any(_isWindowsCudaLibraryPath); + if (rawCudaSelection == null && legacyCudaAvailable) { + log.info( + 'Using the legacy CUDA module from ${nativeConfig.sourceLabel}. ' + 'A sidecar-capable native release will default CUDA selection to 13.', + ); + } else { + final selection = resolveWindowsCudaBundleSelection(rawCudaSelection); + final coreLibrary = _findWindowsGgmlBaseLibrary(libraryPaths); + final sidecarDirectories = await _acquireWindowsCudaSidecars( + packageRoot: pkgRoot, + nativeConfig: nativeConfig, + selection: selection, + coreLibrary: coreLibrary, + log: log, + ); + libraryPaths = [ + ...libraryPaths.where((entry) => !_isWindowsCudaLibraryPath(entry)), + for (final directory in sidecarDirectories) + ..._collectDynamicLibraryPaths(directory), + ]; + log.info( + 'Using Windows CUDA sidecar selection: ' + '${selection.majors.join(', ')}.', + ); + } + } + final libraries = describeNativeLibraries(libraryPaths); if (!libraries.any((library) => library.isPrimary)) { throw Exception( @@ -1236,6 +1276,185 @@ Uri? _resolveNativePath(HookInputUserDefines userDefines) { return resolvedPath; } +bool _isWindowsCudaLibraryPath(String filePath) { + final fileName = path.basename(filePath).toLowerCase(); + return (fileName.startsWith('ggml-cuda') && fileName.endsWith('.dll')) || + _windowsCudartPattern.hasMatch(fileName) || + _windowsCublasPattern.hasMatch(fileName) || + _windowsCublasLtPattern.hasMatch(fileName); +} + +File _findWindowsGgmlBaseLibrary(List libraryPaths) { + final matches = libraryPaths + .where( + (entry) => describeNativeLibrary(entry).canonicalName == 'ggml-base', + ) + .map(File.new) + .toList(growable: false); + if (matches.length != 1) { + throw Exception( + 'Expected exactly one Windows ggml-base library, found ${matches.length}.', + ); + } + return matches.single; +} + +Future> _acquireWindowsCudaSidecars({ + required String packageRoot, + required _NativeBundleConfig nativeConfig, + required WindowsCudaBundleSelection selection, + required File coreLibrary, + required Logger log, +}) async { + final bundleCache = _bundleCacheDirectory( + packageRoot: packageRoot, + nativeConfig: nativeConfig, + bundle: 'windows-x64', + ); + final sidecarCache = path.join(bundleCache, 'cuda-sidecars'); + await Directory(sidecarCache).create(recursive: true); + final release = await _acquireWindowsCudaReleaseManifest( + nativeConfig: nativeConfig, + cacheDirectory: sidecarCache, + log: log, + ); + if (release.nativeTag != nativeConfig.tag) { + throw FormatException( + 'CUDA sidecar release tag ${release.nativeTag} does not match ' + '${nativeConfig.tag}.', + ); + } + + final directories = []; + for (final major in selection.majors) { + final archiveName = + 'llamadart-native-windows-x64-cuda$major-${nativeConfig.tag}.tar.gz'; + final expectedArchiveSha256 = release.assetDigests[archiveName]; + if (expectedArchiveSha256 == null) { + throw Exception( + 'Native release ${nativeConfig.tag} does not publish $archiveName.', + ); + } + final variantCache = path.join(sidecarCache, 'cuda$major'); + final extractedDirectory = Directory(path.join(variantCache, 'extracted')); + if (extractedDirectory.existsSync()) { + try { + await verifyWindowsCudaPackDirectory( + directory: extractedDirectory, + expectedCudaMajor: major, + release: release, + coreLibrary: coreLibrary, + ); + directories.add(extractedDirectory); + continue; + } on FormatException catch (error) { + log.warning('Cached CUDA $major sidecar is stale; refreshing: $error'); + await extractedDirectory.delete(recursive: true); + } + } + + final cachedArchive = File(path.join(variantCache, archiveName)); + await cachedArchive.parent.create(recursive: true); + final localArchive = _resolveLocalWindowsCudaAsset( + nativeConfig: nativeConfig, + assetName: archiveName, + ); + final archive = localArchive ?? cachedArchive; + if (localArchive == null && cachedArchive.existsSync()) { + if (await sha256File(cachedArchive) != expectedArchiveSha256) { + log.warning( + 'Cached CUDA $major archive digest differs; redownloading.', + ); + await cachedArchive.delete(); + } + } + if (!archive.existsSync()) { + await _downloadReleaseAsset( + repository: nativeConfig.repository, + nativeTag: nativeConfig.tag, + assetName: archiveName, + destinationPath: archive.path, + log: log, + ); + } + if (await sha256File(archive) != expectedArchiveSha256) { + if (localArchive == null && archive.existsSync()) { + await archive.delete(); + } + throw Exception('CUDA $major sidecar archive checksum mismatch.'); + } + + await _extractCachedArchive( + archivePath: archive.path, + extractedDir: extractedDirectory, + cacheDir: variantCache, + log: log, + ); + await verifyWindowsCudaPackDirectory( + directory: extractedDirectory, + expectedCudaMajor: major, + release: release, + coreLibrary: coreLibrary, + ); + directories.add(extractedDirectory); + } + return directories; +} + +Future _acquireWindowsCudaReleaseManifest({ + required _NativeBundleConfig nativeConfig, + required String cacheDirectory, + required Logger log, +}) async { + const assetName = 'assets.json'; + final localManifest = _resolveLocalWindowsCudaAsset( + nativeConfig: nativeConfig, + assetName: assetName, + ); + final manifestFile = + localManifest ?? File(path.join(cacheDirectory, assetName)); + if (!manifestFile.existsSync()) { + await _downloadReleaseAsset( + repository: nativeConfig.repository, + nativeTag: nativeConfig.tag, + assetName: assetName, + destinationPath: manifestFile.path, + log: log, + ); + } + return WindowsCudaReleaseManifest.parse(await manifestFile.readAsString()); +} + +File? _resolveLocalWindowsCudaAsset({ + required _NativeBundleConfig nativeConfig, + required String assetName, +}) { + final localPath = nativeConfig.localPath; + if (localPath == null) { + return null; + } + final localFilePath = localPath.toFilePath(); + final root = File(localFilePath).existsSync() + ? File(localFilePath).parent.path + : localFilePath; + final candidates = [ + path.join(root, assetName), + path.join(root, nativeConfig.tag, assetName), + path.join(root, nativeConfig.tag, 'windows-x64', assetName), + path.join(root, 'windows-x64', assetName), + ]; + for (final candidate in candidates) { + final file = File(candidate); + if (file.existsSync()) { + return file; + } + } + throw Exception( + 'Local native source $root is missing required CUDA sidecar asset ' + '$assetName.', + ); +} + Future _acquireBundleDirectory({ required String packageRoot, required _NativeBundleConfig nativeConfig, diff --git a/lib/src/backends/llama_cpp/llama_cpp_service.dart b/lib/src/backends/llama_cpp/llama_cpp_service.dart index 12cbe973..bb23fc9e 100644 --- a/lib/src/backends/llama_cpp/llama_cpp_service.dart +++ b/lib/src/backends/llama_cpp/llama_cpp_service.dart @@ -24,6 +24,7 @@ import '../../core/template/chat_template_engine.dart'; import 'load_param_helpers.dart'; import 'bindings.dart'; import 'llama_cpp_raw_bindings.dart' as raw_bindings; +import 'windows_cuda_selector.dart'; const _llamadartWrapperAssetId = 'package:llamadart/llamadart_wrapper'; @@ -2387,13 +2388,16 @@ class LlamaCppService { candidates.addAll(fileNameCandidates); } - _preloadWindowsBackendDependencies(backend); - for (final candidate in candidates) { if (path.isAbsolute(candidate) && !File(candidate).existsSync()) { continue; } + _preloadWindowsBackendDependencies( + backend, + cudaMajor: windowsCudaMajorFromFileName(path.basename(candidate)), + ); + final alteredSearchPathHandle = _preloadWindowsBackendModule( candidate, backend, @@ -2537,7 +2541,7 @@ class LlamaCppService { } } - void _preloadWindowsBackendDependencies(String backend) { + void _preloadWindowsBackendDependencies(String backend, {int? cudaMajor}) { if (!Platform.isWindows) { return; } @@ -2548,7 +2552,8 @@ class LlamaCppService { } final cacheKey = - '$backend|${path.normalize(backendModuleDirectory).toLowerCase()}'; + '$backend|${cudaMajor ?? 'legacy'}|' + '${path.normalize(backendModuleDirectory).toLowerCase()}'; if (_preloadedBackendDependencyLibraries.containsKey(cacheKey)) { return; } @@ -2559,6 +2564,7 @@ class LlamaCppService { for (final dependencyPath in windowsBackendDependencyPaths( backendModuleDirectory, backend, + cudaMajor: cudaMajor, )) { try { handles.add(DynamicLibrary.open(dependencyPath)); @@ -2594,6 +2600,7 @@ class LlamaCppService { static List windowsBackendDependencyPaths( String directoryPath, String backend, { + int? cudaMajor, Iterable? fileNames, }) { if (backend != 'cuda') { @@ -2612,6 +2619,9 @@ class LlamaCppService { if (lower.startsWith('cudart64_') || lower.startsWith('cublas64_') || lower.startsWith('cublaslt64_')) { + if (cudaMajor != null && !lower.endsWith('_$cudaMajor.dll')) { + continue; + } selected.add(name); } } @@ -3404,7 +3414,44 @@ class LlamaCppService { dynamicNames.sort(_compareAndroidCpuLibraryCandidates); } candidates.addAll(dynamicNames); - final resolved = candidates.toList(growable: false); + final resolved = candidates.toList(); + if (backend == 'cuda' && Platform.isWindows) { + final versioned = resolved + .where((name) => windowsCudaMajorFromFileName(name) != null) + .toList(growable: false); + final availableMajors = versioned + .map(windowsCudaMajorFromFileName) + .whereType() + .toSet(); + if (availableMajors.length > 1) { + final probe = probeWindowsCudaDriver(); + final selectedMajor = probe == null + ? null + : selectWindowsCudaMajor( + availableMajors: availableMajors, + probe: probe, + ); + if (selectedMajor == null) { + _recordStartupDiagnostic( + 'No bundled Windows CUDA sidecar matches the installed NVIDIA ' + 'driver and visible GPU compute capabilities.', + ); + resolved.removeWhere( + (name) => windowsCudaMajorFromFileName(name) != null, + ); + } else { + _recordStartupDiagnostic( + 'Selected bundled Windows CUDA $selectedMajor sidecar from ' + 'driver API ${probe!.driverApiVersion} and compute capabilities ' + '${probe.computeCapabilities.join(', ')}.', + ); + resolved.removeWhere((name) { + final major = windowsCudaMajorFromFileName(name); + return major != null && major != selectedMajor; + }); + } + } + } if (backend == 'cpu' && Platform.isAndroid) { resolved.sort(_compareAndroidCpuLibraryCandidates); } diff --git a/lib/src/backends/llama_cpp/windows_cuda_selector.dart b/lib/src/backends/llama_cpp/windows_cuda_selector.dart new file mode 100644 index 00000000..d0a120ac --- /dev/null +++ b/lib/src/backends/llama_cpp/windows_cuda_selector.dart @@ -0,0 +1,112 @@ +// ignore_for_file: public_member_api_docs + +import 'dart:ffi'; + +import 'package:ffi/ffi.dart'; + +typedef _CuInitNative = Int32 Function(Uint32 flags); +typedef _CuInitDart = int Function(int flags); +typedef _CuDriverGetVersionNative = Int32 Function(Pointer version); +typedef _CuDriverGetVersionDart = int Function(Pointer version); +typedef _CuDeviceGetCountNative = Int32 Function(Pointer count); +typedef _CuDeviceGetCountDart = int Function(Pointer count); +typedef _CuDeviceComputeCapabilityNative = + Int32 Function(Pointer major, Pointer minor, Int32 device); +typedef _CuDeviceComputeCapabilityDart = + int Function(Pointer major, Pointer minor, int device); + +class WindowsCudaDriverProbe { + final int driverApiVersion; + final List computeCapabilities; + + const WindowsCudaDriverProbe({ + required this.driverApiVersion, + required this.computeCapabilities, + }); +} + +int? windowsCudaMajorFromFileName(String fileName) { + final match = RegExp( + r'^ggml-cuda-(12|13)(?:-[^\\/]+)*\.dll$', + caseSensitive: false, + ).firstMatch(fileName); + return int.tryParse(match?.group(1) ?? ''); +} + +int? selectWindowsCudaMajor({ + required Set availableMajors, + required WindowsCudaDriverProbe probe, +}) { + if (probe.computeCapabilities.isEmpty) { + return null; + } + final minimumCapability = probe.computeCapabilities.reduce( + (value, element) => value < element ? value : element, + ); + if (availableMajors.contains(13) && + probe.driverApiVersion >= 13000 && + minimumCapability >= 75) { + return 13; + } + if (availableMajors.contains(12) && + probe.driverApiVersion >= 12000 && + minimumCapability >= 50) { + return 12; + } + return null; +} + +WindowsCudaDriverProbe? probeWindowsCudaDriver() { + try { + final library = DynamicLibrary.open('nvcuda.dll'); + final cuInit = library.lookupFunction<_CuInitNative, _CuInitDart>('cuInit'); + final cuDriverGetVersion = library + .lookupFunction<_CuDriverGetVersionNative, _CuDriverGetVersionDart>( + 'cuDriverGetVersion', + ); + final cuDeviceGetCount = library + .lookupFunction<_CuDeviceGetCountNative, _CuDeviceGetCountDart>( + 'cuDeviceGetCount', + ); + final cuDeviceComputeCapability = library + .lookupFunction< + _CuDeviceComputeCapabilityNative, + _CuDeviceComputeCapabilityDart + >('cuDeviceComputeCapability'); + if (cuInit(0) != 0) { + return null; + } + + final driverVersion = calloc(); + final deviceCount = calloc(); + final major = calloc(); + final minor = calloc(); + try { + if (cuDriverGetVersion(driverVersion) != 0 || + cuDeviceGetCount(deviceCount) != 0 || + deviceCount.value <= 0) { + return null; + } + final capabilities = []; + for (var device = 0; device < deviceCount.value; device++) { + if (cuDeviceComputeCapability(major, minor, device) == 0) { + capabilities.add(major.value * 10 + minor.value); + } + } + if (capabilities.isEmpty) { + return null; + } + return WindowsCudaDriverProbe( + driverApiVersion: driverVersion.value, + computeCapabilities: List.unmodifiable(capabilities), + ); + } finally { + calloc.free(driverVersion); + calloc.free(deviceCount); + calloc.free(major); + calloc.free(minor); + } + } catch (_) { + return null; + } +} diff --git a/lib/src/hook/native_bundle_config.dart b/lib/src/hook/native_bundle_config.dart index 46c4e47b..c02274b2 100644 --- a/lib/src/hook/native_bundle_config.dart +++ b/lib/src/hook/native_bundle_config.dart @@ -10,6 +10,17 @@ const String nativePathUserDefineKey = 'llamadart_native_path'; const String nativeRuntimesUserDefineKey = 'llamadart_native_runtimes'; const String nativeRuntimeLlamaCpp = 'llama_cpp'; const String nativeRuntimeLiteRtLm = 'litert_lm'; +const String nativeWindowsCudaUserDefineKey = 'llamadart_windows_cuda'; + +enum WindowsCudaBundleSelection { cuda12, cuda13, both } + +extension WindowsCudaBundleSelectionValues on WindowsCudaBundleSelection { + List get majors => switch (this) { + WindowsCudaBundleSelection.cuda12 => const [12], + WindowsCudaBundleSelection.cuda13 => const [13], + WindowsCudaBundleSelection.both => const [12, 13], + }; +} const List allNativeRuntimes = [ nativeRuntimeLlamaCpp, @@ -414,6 +425,29 @@ List defaultNativeRuntimesForBundle(String bundle) { return defaultNativeRuntimes; } +WindowsCudaBundleSelection resolveWindowsCudaBundleSelection( + Object? rawUserConfig, +) { + if (rawUserConfig == null) { + return WindowsCudaBundleSelection.cuda13; + } + + final normalized = switch (rawUserConfig) { + int value => value.toString(), + String value => value.trim().toLowerCase(), + _ => '', + }; + return switch (normalized) { + '12' || 'cuda12' || 'cuda-12' => WindowsCudaBundleSelection.cuda12, + '13' || 'cuda13' || 'cuda-13' => WindowsCudaBundleSelection.cuda13, + 'both' => WindowsCudaBundleSelection.both, + _ => throw FormatException( + 'hooks.user_defines.llamadart.$nativeWindowsCudaUserDefineKey must be ' + '12, 13, or both.', + ), + }; +} + List selectBackendsForBundle({ required NativeBundleSpec spec, required Set availableBackends, diff --git a/lib/src/hook/windows_cuda_pack.dart b/lib/src/hook/windows_cuda_pack.dart new file mode 100644 index 00000000..69a6bb09 --- /dev/null +++ b/lib/src/hook/windows_cuda_pack.dart @@ -0,0 +1,226 @@ +// ignore_for_file: public_member_api_docs + +import 'dart:convert'; +import 'dart:io'; + +import 'package:crypto/crypto.dart'; +import 'package:path/path.dart' as path; + +const int windowsCudaPackContractVersion = 3; + +class WindowsCudaReleaseManifest { + final String nativeTag; + final String llamaCppTag; + final String llamaCppCommit; + final Map assetDigests; + + const WindowsCudaReleaseManifest({ + required this.nativeTag, + required this.llamaCppTag, + required this.llamaCppCommit, + required this.assetDigests, + }); + + factory WindowsCudaReleaseManifest.parse(String source) { + final decoded = jsonDecode(source); + if (decoded is! Map) { + throw const FormatException('Native release manifest must be an object.'); + } + final artifacts = decoded['artifacts']; + if (artifacts is! List) { + throw const FormatException( + 'Native release manifest artifacts must be an array.', + ); + } + final digests = {}; + for (final entry in artifacts) { + if (entry is! Map) { + throw const FormatException('Native release artifact is invalid.'); + } + final file = entry['file']; + final sha256 = entry['sha256']; + if (file is! String || + sha256 is! String || + !RegExp(r'^[0-9a-f]{64}$').hasMatch(sha256)) { + throw const FormatException( + 'Native release artifact name or SHA-256 is invalid.', + ); + } + if (digests.containsKey(file)) { + throw FormatException('Duplicate native release artifact: $file.'); + } + digests[file] = sha256; + } + + final nativeTag = decoded['tag']; + final llamaCppTag = decoded['llama_cpp_tag']; + final llamaCppCommit = decoded['llama_cpp_commit']; + if (nativeTag is! String || + llamaCppTag is! String || + llamaCppCommit is! String || + !RegExp(r'^[0-9a-f]{40}$').hasMatch(llamaCppCommit)) { + throw const FormatException( + 'Native release provenance metadata is invalid.', + ); + } + return WindowsCudaReleaseManifest( + nativeTag: nativeTag, + llamaCppTag: llamaCppTag, + llamaCppCommit: llamaCppCommit, + assetDigests: Map.unmodifiable(digests), + ); + } +} + +class WindowsCudaPackManifest { + final int cudaMajor; + final String backendLibrary; + final Map files; + + const WindowsCudaPackManifest({ + required this.cudaMajor, + required this.backendLibrary, + required this.files, + }); +} + +Future sha256File(File file) async { + return sha256.bind(file.openRead()).first.then((digest) => digest.toString()); +} + +Future verifyWindowsCudaPackDirectory({ + required Directory directory, + required int expectedCudaMajor, + required WindowsCudaReleaseManifest release, + required File coreLibrary, +}) async { + final manifestFile = File(path.join(directory.path, 'cuda-pack.json')); + if (!manifestFile.existsSync()) { + throw const FormatException('CUDA sidecar is missing cuda-pack.json.'); + } + final decoded = jsonDecode(await manifestFile.readAsString()); + if (decoded is! Map) { + throw const FormatException('CUDA sidecar manifest must be an object.'); + } + if (decoded['contract_version'] != windowsCudaPackContractVersion) { + throw FormatException( + 'Unsupported CUDA sidecar contract version: ' + '${decoded['contract_version']}.', + ); + } + if (decoded['native_release_tag'] != release.nativeTag || + decoded['llama_cpp_tag'] != release.llamaCppTag || + decoded['llama_cpp_commit'] != release.llamaCppCommit) { + throw const FormatException( + 'CUDA sidecar provenance does not match the native release.', + ); + } + if (decoded['platform'] != 'windows' || decoded['arch'] != 'x64') { + throw const FormatException('CUDA sidecar must target Windows x64.'); + } + if (decoded['backend'] != 'cuda' || + decoded['cuda_major'] != expectedCudaMajor) { + throw FormatException( + 'CUDA sidecar variant does not match CUDA $expectedCudaMajor.', + ); + } + + final expectedBackend = 'ggml-cuda-$expectedCudaMajor.dll'; + if (decoded['backend_library'] != expectedBackend) { + throw FormatException('CUDA sidecar backend must be $expectedBackend.'); + } + final expectedCompatibility = expectedCudaMajor == 12 + ? const { + 'minimum_compute_capability': 50, + 'minimum_driver_family': 525, + 'minimum_driver_api': 12000, + } + : const { + 'minimum_compute_capability': 75, + 'minimum_driver_family': 580, + 'minimum_driver_api': 13000, + }; + final compatibility = decoded['compatibility']; + if (compatibility is! Map || + compatibility.length != expectedCompatibility.length || + !expectedCompatibility.entries.every( + (entry) => compatibility[entry.key] == entry.value, + )) { + throw FormatException( + 'CUDA $expectedCudaMajor compatibility metadata differs from the ' + 'supported contract.', + ); + } + final coreCompatibility = decoded['core_compatibility']; + if (coreCompatibility is! Map || + coreCompatibility['library'] != 'ggml-base.dll' || + coreCompatibility['sha256'] != await sha256File(coreLibrary)) { + throw const FormatException( + 'CUDA sidecar does not match the selected native core.', + ); + } + + final expectedNames = { + expectedBackend, + 'cudart64_$expectedCudaMajor.dll', + 'cublas64_$expectedCudaMajor.dll', + 'cublasLt64_$expectedCudaMajor.dll', + }; + final fileEntries = decoded['files']; + if (fileEntries is! List) { + throw const FormatException('CUDA sidecar file manifest is invalid.'); + } + final files = {}; + for (final entry in fileEntries) { + if (entry is! Map || + entry['name'] is! String || + entry['sha256'] is! String || + entry['size'] is! int) { + throw const FormatException('CUDA sidecar file entry is invalid.'); + } + final name = entry['name']! as String; + final digest = entry['sha256']! as String; + final size = entry['size']! as int; + if (!expectedNames.contains(name) || + !RegExp(r'^[0-9a-f]{64}$').hasMatch(digest) || + size < 0 || + files.containsKey(name)) { + throw FormatException('CUDA sidecar file entry is invalid: $name.'); + } + files[name] = (sha256: digest, size: size); + } + if (files.keys.toSet().difference(expectedNames).isNotEmpty || + expectedNames.difference(files.keys.toSet()).isNotEmpty) { + throw const FormatException( + 'CUDA sidecar payload differs from the required dependency family.', + ); + } + + final actualEntries = directory + .listSync() + .whereType() + .map((file) => path.basename(file.path)) + .toSet(); + if (actualEntries.difference({ + ...expectedNames, + 'cuda-pack.json', + }).isNotEmpty || + expectedNames.difference(actualEntries).isNotEmpty) { + throw const FormatException( + 'CUDA sidecar contains missing or unexpected files.', + ); + } + for (final MapEntry(key: name, value: expected) in files.entries) { + final file = File(path.join(directory.path, name)); + if (file.lengthSync() != expected.size || + await sha256File(file) != expected.sha256) { + throw FormatException('CUDA sidecar payload digest differs: $name.'); + } + } + + return WindowsCudaPackManifest( + cudaMajor: expectedCudaMajor, + backendLibrary: expectedBackend, + files: Map.unmodifiable(files), + ); +} diff --git a/test/unit/backends/llama_cpp/llama_cpp_service_test.dart b/test/unit/backends/llama_cpp/llama_cpp_service_test.dart index e0cc4e29..3bcc589f 100644 --- a/test/unit/backends/llama_cpp/llama_cpp_service_test.dart +++ b/test/unit/backends/llama_cpp/llama_cpp_service_test.dart @@ -1325,6 +1325,28 @@ void main() { ); }); + test('isolates the selected Windows CUDA dependency family', () { + final dependencyPaths = LlamaCppService.windowsBackendDependencyPaths( + tempRoot.path, + 'cuda', + cudaMajor: 13, + fileNames: const [ + 'cudart64_12.dll', + 'cublas64_12.dll', + 'cublasLt64_12.dll', + 'cudart64_13.dll', + 'cublas64_13.dll', + 'cublasLt64_13.dll', + ], + ); + + expect(dependencyPaths, [ + path.join(tempRoot.path, 'cudart64_13.dll'), + path.join(tempRoot.path, 'cublas64_13.dll'), + path.join(tempRoot.path, 'cublasLt64_13.dll'), + ]); + }); + test('falls back to hook cache extracted bundle directory', () { final extractedDir = Directory( path.join( diff --git a/test/unit/backends/llama_cpp/windows_cuda_selector_test.dart b/test/unit/backends/llama_cpp/windows_cuda_selector_test.dart new file mode 100644 index 00000000..7bd32260 --- /dev/null +++ b/test/unit/backends/llama_cpp/windows_cuda_selector_test.dart @@ -0,0 +1,82 @@ +@TestOn('vm') +library; + +import 'package:test/test.dart'; + +import 'package:llamadart/src/backends/llama_cpp/windows_cuda_selector.dart'; + +void main() { + test('recognizes only versioned CUDA sidecar backend names', () { + expect(windowsCudaMajorFromFileName('ggml-cuda-12.dll'), 12); + expect(windowsCudaMajorFromFileName('ggml-cuda-13-windows-x64.dll'), 13); + expect(windowsCudaMajorFromFileName('ggml-cuda.dll'), isNull); + expect(windowsCudaMajorFromFileName('ggml-vulkan-13.dll'), isNull); + }); + + test('prefers CUDA 13 when the driver and GPU satisfy its contract', () { + expect( + selectWindowsCudaMajor( + availableMajors: const {12, 13}, + probe: const WindowsCudaDriverProbe( + driverApiVersion: 13000, + computeCapabilities: [89], + ), + ), + 13, + ); + }); + + test('falls back to CUDA 12 at driver and GPU boundaries', () { + for (final probe in const [ + WindowsCudaDriverProbe( + driverApiVersion: 12000, + computeCapabilities: [89], + ), + WindowsCudaDriverProbe( + driverApiVersion: 13000, + computeCapabilities: [70], + ), + ]) { + expect( + selectWindowsCudaMajor(availableMajors: const {12, 13}, probe: probe), + 12, + ); + } + }); + + test('uses the oldest visible GPU for a mixed-GPU system', () { + expect( + selectWindowsCudaMajor( + availableMajors: const {12, 13}, + probe: const WindowsCudaDriverProbe( + driverApiVersion: 13000, + computeCapabilities: [89, 70], + ), + ), + 12, + ); + }); + + test('rejects unsupported driver and compute capability combinations', () { + expect( + selectWindowsCudaMajor( + availableMajors: const {12, 13}, + probe: const WindowsCudaDriverProbe( + driverApiVersion: 11080, + computeCapabilities: [89], + ), + ), + isNull, + ); + expect( + selectWindowsCudaMajor( + availableMajors: const {12, 13}, + probe: const WindowsCudaDriverProbe( + driverApiVersion: 13000, + computeCapabilities: [49], + ), + ), + isNull, + ); + }); +} diff --git a/test/unit/hook/build_hook_integration_test.dart b/test/unit/hook/build_hook_integration_test.dart index 297c80c7..38c199a9 100644 --- a/test/unit/hook/build_hook_integration_test.dart +++ b/test/unit/hook/build_hook_integration_test.dart @@ -30,6 +30,7 @@ void main() { final archivePath = '$cacheRelativeDir/llamadart-native-windows-x64-$nativeTag.tar.gz'; final archiveFile = File(archivePath); + final cudaSidecarDir = Directory('$cacheRelativeDir/cuda-sidecars'); setUpAll(() async { if (backupDir.existsSync()) { @@ -68,12 +69,18 @@ void main() { if (archiveFile.existsSync()) { await archiveFile.delete(); } + if (cudaSidecarDir.existsSync()) { + await cudaSidecarDir.delete(recursive: true); + } }); tearDownAll(() async { if (archiveFile.existsSync()) { await archiveFile.delete(); } + if (cudaSidecarDir.existsSync()) { + await cudaSidecarDir.delete(recursive: true); + } if (bundleDir.existsSync()) { await bundleDir.delete(recursive: true); } @@ -186,6 +193,93 @@ void main() { ); }); + test( + 'build hook replaces legacy CUDA with the explicit CUDA 13 sidecar', + () async { + await _writeCudaSidecarFixtures( + cacheDirectory: cudaSidecarDir, + coreLibrary: File( + path.join(bundleDir.path, 'ggml-base-windows-x64.dll'), + ), + nativeTag: nativeTag, + majors: const [13], + ); + final userDefines = PackageUserDefines( + workspacePubspec: PackageUserDefinesSource( + defines: { + 'llamadart_windows_cuda': '13', + 'llamadart_native_runtimes': ['llama_cpp'], + 'llamadart_native_backends': { + 'platforms': { + 'windows-x64': ['cuda'], + }, + }, + }, + basePath: Directory.current.uri, + ), + ); + + await testCodeBuildHook( + mainMethod: build_hook.main, + targetOS: OS.windows, + targetArchitecture: Architecture.x64, + userDefines: userDefines, + check: (input, output) { + final emittedNames = _emittedFileNames(output); + expect(emittedNames, contains('ggml-cuda-13.dll')); + expect(emittedNames, contains('cudart64_13.dll')); + expect(emittedNames, isNot(contains('ggml-cuda-windows-x64.dll'))); + expect(emittedNames, isNot(contains('cudart64_12.dll'))); + }, + ); + }, + ); + + test( + 'build hook can bundle both isolated CUDA dependency families', + () async { + await _writeCudaSidecarFixtures( + cacheDirectory: cudaSidecarDir, + coreLibrary: File( + path.join(bundleDir.path, 'ggml-base-windows-x64.dll'), + ), + nativeTag: nativeTag, + majors: const [12, 13], + ); + final userDefines = PackageUserDefines( + workspacePubspec: PackageUserDefinesSource( + defines: { + 'llamadart_windows_cuda': 'both', + 'llamadart_native_runtimes': ['llama_cpp'], + 'llamadart_native_backends': { + 'platforms': { + 'windows-x64': ['cuda'], + }, + }, + }, + basePath: Directory.current.uri, + ), + ); + + await testCodeBuildHook( + mainMethod: build_hook.main, + targetOS: OS.windows, + targetArchitecture: Architecture.x64, + userDefines: userDefines, + check: (input, output) { + final emittedNames = _emittedFileNames(output); + for (final major in const [12, 13]) { + expect(emittedNames, contains('ggml-cuda-$major.dll')); + expect(emittedNames, contains('cudart64_$major.dll')); + expect(emittedNames, contains('cublas64_$major.dll')); + expect(emittedNames, contains('cublasLt64_$major.dll')); + } + expect(emittedNames, isNot(contains('ggml-cuda-windows-x64.dll'))); + }, + ); + }, + ); + test('build hook can emit llama.cpp runtime without LiteRT-LM', () async { final userDefines = PackageUserDefines( workspacePubspec: PackageUserDefinesSource( @@ -737,6 +831,80 @@ Future _writeBundleArchive({ await archiveFile.writeAsBytes(gzBytes); } +Future _writeCudaSidecarFixtures({ + required Directory cacheDirectory, + required File coreLibrary, + required String nativeTag, + required List majors, +}) async { + await cacheDirectory.create(recursive: true); + final artifacts = >[]; + final coreDigest = sha256.convert(await coreLibrary.readAsBytes()).toString(); + for (final major in majors) { + final archiveName = + 'llamadart-native-windows-x64-cuda$major-$nativeTag.tar.gz'; + artifacts.add({'file': archiveName, 'sha256': 'a' * 64}); + final extracted = Directory( + path.join(cacheDirectory.path, 'cuda$major', 'extracted'), + ); + await extracted.create(recursive: true); + final payload = { + 'ggml-cuda-$major.dll': 'backend-$major', + 'cudart64_$major.dll': 'cudart-$major', + 'cublas64_$major.dll': 'cublas-$major', + 'cublasLt64_$major.dll': 'cublas-lt-$major', + }; + final files = >[]; + for (final MapEntry(key: name, value: contents) in payload.entries) { + final file = File(path.join(extracted.path, name)); + await file.writeAsString(contents); + files.add({ + 'name': name, + 'sha256': sha256.convert(await file.readAsBytes()).toString(), + 'size': await file.length(), + }); + } + await File(path.join(extracted.path, 'cuda-pack.json')).writeAsString( + jsonEncode({ + 'contract_version': 3, + 'native_release_tag': nativeTag, + 'llama_cpp_tag': 'b-test', + 'llama_cpp_commit': '1' * 40, + 'platform': 'windows', + 'arch': 'x64', + 'backend': 'cuda', + 'cuda_version': major == 12 ? '12.4' : '13.3', + 'cuda_major': major, + 'backend_library': 'ggml-cuda-$major.dll', + 'compatibility': major == 12 + ? const { + 'minimum_compute_capability': 50, + 'minimum_driver_family': 525, + 'minimum_driver_api': 12000, + } + : const { + 'minimum_compute_capability': 75, + 'minimum_driver_family': 580, + 'minimum_driver_api': 13000, + }, + 'core_compatibility': { + 'library': 'ggml-base.dll', + 'sha256': coreDigest, + }, + 'files': files, + }), + ); + } + await File(path.join(cacheDirectory.path, 'assets.json')).writeAsString( + jsonEncode({ + 'tag': nativeTag, + 'llama_cpp_tag': 'b-test', + 'llama_cpp_commit': '1' * 40, + 'artifacts': artifacts, + }), + ); +} + String _localPathCacheRoot({ required String localRootPath, required String nativeTag, diff --git a/test/unit/hook/native_bundle_config_test.dart b/test/unit/hook/native_bundle_config_test.dart index e6a9c166..9e04deba 100644 --- a/test/unit/hook/native_bundle_config_test.dart +++ b/test/unit/hook/native_bundle_config_test.dart @@ -7,6 +7,38 @@ import 'package:test/test.dart'; import 'package:llamadart/src/hook/native_bundle_config.dart'; void main() { + group('resolveWindowsCudaBundleSelection', () { + test('defaults to CUDA 13', () { + expect( + resolveWindowsCudaBundleSelection(null), + WindowsCudaBundleSelection.cuda13, + ); + }); + + test('accepts CUDA 12, CUDA 13, and both', () { + expect( + resolveWindowsCudaBundleSelection('12'), + WindowsCudaBundleSelection.cuda12, + ); + expect( + resolveWindowsCudaBundleSelection(13), + WindowsCudaBundleSelection.cuda13, + ); + expect( + resolveWindowsCudaBundleSelection('both'), + WindowsCudaBundleSelection.both, + ); + expect(WindowsCudaBundleSelection.both.majors, const [12, 13]); + }); + + test('rejects unknown values', () { + expect( + () => resolveWindowsCudaBundleSelection('auto'), + throwsA(isA()), + ); + }); + }); + group('resolveNativeBundleSpec', () { test('resolves android arm64 with cpu+vulkan defaults', () { final spec = resolveNativeBundleSpec( diff --git a/test/unit/hook/windows_cuda_pack_test.dart b/test/unit/hook/windows_cuda_pack_test.dart new file mode 100644 index 00000000..1e272322 --- /dev/null +++ b/test/unit/hook/windows_cuda_pack_test.dart @@ -0,0 +1,144 @@ +@TestOn('vm') +library; + +import 'dart:convert'; +import 'dart:io'; + +import 'package:path/path.dart' as path; +import 'package:test/test.dart'; + +import 'package:llamadart/src/hook/windows_cuda_pack.dart'; + +void main() { + const nativeTag = 'native-test'; + const llamaTag = 'b-test'; + const llamaCommit = '1111111111111111111111111111111111111111'; + + test('parses exact release provenance and asset digests', () { + final release = WindowsCudaReleaseManifest.parse( + jsonEncode({ + 'tag': nativeTag, + 'llama_cpp_tag': llamaTag, + 'llama_cpp_commit': llamaCommit, + 'artifacts': [ + {'file': 'cuda.tar.gz', 'sha256': '2' * 64}, + ], + }), + ); + expect(release.nativeTag, nativeTag); + expect(release.llamaCppTag, llamaTag); + expect(release.assetDigests['cuda.tar.gz'], '2' * 64); + }); + + test( + 'verifies pack provenance, core, dependency family, and payloads', + () async { + await _withPack((directory, core, release) async { + final manifest = await verifyWindowsCudaPackDirectory( + directory: directory, + expectedCudaMajor: 13, + release: release, + coreLibrary: core, + ); + expect(manifest.cudaMajor, 13); + expect(manifest.backendLibrary, 'ggml-cuda-13.dll'); + }); + }, + ); + + test('rejects wrong core and corrupt payloads', () async { + await _withPack((directory, core, release) async { + final wrongCore = File(path.join(directory.parent.path, 'wrong-core.dll')) + ..writeAsBytesSync(const [9]); + await expectLater( + verifyWindowsCudaPackDirectory( + directory: directory, + expectedCudaMajor: 13, + release: release, + coreLibrary: wrongCore, + ), + throwsA(isA()), + ); + + File( + path.join(directory.path, 'ggml-cuda-13.dll'), + ).writeAsBytesSync(const [8]); + await expectLater( + verifyWindowsCudaPackDirectory( + directory: directory, + expectedCudaMajor: 13, + release: release, + coreLibrary: core, + ), + throwsA(isA()), + ); + }); + }); +} + +Future _withPack( + Future Function( + Directory directory, + File core, + WindowsCudaReleaseManifest release, + ) + body, +) async { + final root = await Directory.systemTemp.createTemp('cuda-pack-test-'); + try { + final directory = Directory(path.join(root.path, 'pack'))..createSync(); + final core = File(path.join(root.path, 'ggml-base.dll')) + ..writeAsBytesSync(const [1, 2, 3]); + final payload = >{ + 'ggml-cuda-13.dll': const [4], + 'cudart64_13.dll': const [5], + 'cublas64_13.dll': const [6], + 'cublasLt64_13.dll': const [7], + }; + final files = >[]; + for (final MapEntry(key: name, value: bytes) in payload.entries) { + final file = File(path.join(directory.path, name)) + ..writeAsBytesSync(bytes); + files.add({ + 'name': name, + 'sha256': await sha256File(file), + 'size': bytes.length, + }); + } + File(path.join(directory.path, 'cuda-pack.json')).writeAsStringSync( + jsonEncode({ + 'contract_version': windowsCudaPackContractVersion, + 'native_release_tag': 'native-test', + 'llama_cpp_tag': 'b-test', + 'llama_cpp_commit': '1' * 40, + 'platform': 'windows', + 'arch': 'x64', + 'backend': 'cuda', + 'cuda_version': '13.3', + 'cuda_major': 13, + 'backend_library': 'ggml-cuda-13.dll', + 'compatibility': const { + 'minimum_compute_capability': 75, + 'minimum_driver_family': 580, + 'minimum_driver_api': 13000, + }, + 'core_compatibility': { + 'library': 'ggml-base.dll', + 'sha256': await sha256File(core), + }, + 'files': files, + }), + ); + final release = WindowsCudaReleaseManifest.parse( + jsonEncode({ + 'tag': 'native-test', + 'llama_cpp_tag': 'b-test', + 'llama_cpp_commit': '1' * 40, + 'artifacts': const [], + }), + ); + await body(directory, core, release); + } finally { + await root.delete(recursive: true); + } +} diff --git a/website/docs/getting-started/installation.md b/website/docs/getting-started/installation.md index d32ec7c6..3ceff504 100644 --- a/website/docs/getting-started/installation.md +++ b/website/docs/getting-started/installation.md @@ -95,6 +95,11 @@ hooks: cpu_profile: full # default: full; use compact for baseline-only CPU linux-x64: [vulkan, cuda] windows-x64: [vulkan, cuda] + + # Windows x64 only, and only when the cuda backend is selected. + # Defaults to "13". Use "12" for older NVIDIA drivers/GPUs, or + # "both" for one portable package with runtime compatibility selection. + llamadart_windows_cuda: "13" ``` Module availability is platform/arch specific and tied to the selected native @@ -119,6 +124,21 @@ gh release list --repo leehack/llamadart-native --limit 20 Before overriding, confirm the release includes the asset for your target. The hook downloads files named `llamadart-native--.tar.gz`, for example `llamadart-native-windows-x64-b10453.tar.gz`. + +Sidecar-capable Windows x64 releases publish CUDA separately from the core +bundle. `llamadart_windows_cuda` accepts `"13"` (the default), `"12"`, or +`"both"`. The `both` option bundles both dependency families but loads exactly +one: CUDA 13 when the driver API and every visible GPU satisfy compute +capability 7.5+, otherwise CUDA 12 when every visible GPU satisfies compute +capability 5.0+ +GPU. This is runtime selection, so the build machine's GPU is not used as a +proxy for the target computer. Expect `both` to add roughly 1.1 GB of compressed +native assets for the currently audited packs. + +The hook verifies the sidecar archive digest, native/llama.cpp release +provenance, contract version, target architecture, matching `ggml-base` digest, +dependency family, and every extracted file before reporting native assets. +CUDA 12 and CUDA 13 are never loaded together in one process. For local testing, `llamadart_native_path` may point directly at a bundle archive, at an extracted bundle directory, or at a directory containing `//`, `/`, or the expected archive file. diff --git a/website/docs/platforms/native-build-hooks.md b/website/docs/platforms/native-build-hooks.md index e15ccca9..ab8423db 100644 --- a/website/docs/platforms/native-build-hooks.md +++ b/website/docs/platforms/native-build-hooks.md @@ -136,6 +136,13 @@ runtime layout for the selected architecture. to defaults. - On `windows-x64`, the hook additionally validates CUDA/BLAS runtime dependencies before accepting a bundle. +- Windows x64 CUDA sidecars default to CUDA 13 when `cuda` is selected. + `llamadart_windows_cuda` can select CUDA 12 or `both`; the latter packages + both versions but probes the target NVIDIA driver/GPU and loads only one + matching backend and dependency family per process. +- CUDA sidecars are accepted only when their release provenance, archive and + payload digests, contract version, architecture, compatibility metadata, and + `ggml-base` digest match the selected native core. - LiteRT-LM archives are validated after extraction by checking the required runtime libraries; corrupt or incomplete cached archives are refreshed before the build continues. diff --git a/website/docs/platforms/support-matrix.md b/website/docs/platforms/support-matrix.md index 0fd6f510..0eab1e58 100644 --- a/website/docs/platforms/support-matrix.md +++ b/website/docs/platforms/support-matrix.md @@ -376,6 +376,10 @@ no valid entries remain, selection falls back to `cpu_profile` (or default the default cache search is not suitable. - `windows-x64` performs extra runtime dependency validation: - `cuda` requires `cudart` and `cublas` DLLs. + - Sidecar-capable releases default selected CUDA builds to CUDA 13; + `llamadart_windows_cuda: "12"` supports older compatible NVIDIA systems, + while `"both"` ships both and selects one from the target driver API and + GPU compute capability before backend loading. - `blas` requires OpenBLAS DLL. - If `llamadart_native_tag` points at a release without a matching bundle asset, the native-assets hook fails while downloading that asset. @@ -389,9 +393,9 @@ no valid entries remain, selection falls back to `cpu_profile` (or default - Native source overrides do not regenerate Dart FFI bindings or symbol lookups, so they are only safe with compatible native binaries. - If you change `llamadart_native_tag`, `llamadart_native_repository`, - `llamadart_native_path`, `llamadart_native_runtimes`, or - `llamadart_native_backends`, run `flutter clean` once to clear stale - native-asset outputs. + `llamadart_native_path`, `llamadart_native_runtimes`, + `llamadart_native_backends`, or `llamadart_windows_cuda`, run `flutter clean` + once to clear stale native-asset outputs. - If a native release tag is republished with refreshed assets, also run `flutter clean` before rebuilding so an older same-tag extracted bundle does not stay in use. From d6c94dd363e82bbba696be646379eba9058d66ec Mon Sep 17 00:00:00 2001 From: Jhin Lee Date: Mon, 17 Aug 2026 13:36:14 -0400 Subject: [PATCH 2/3] fix: harden CUDA sidecar verification --- lib/src/hook/windows_cuda_pack.dart | 16 ++++++++++++---- test/unit/hook/windows_cuda_pack_test.dart | 16 ++++++++++++++++ website/docs/getting-started/installation.md | 4 ++-- website/docs/platforms/support-matrix.md | 2 +- 4 files changed, 31 insertions(+), 7 deletions(-) diff --git a/lib/src/hook/windows_cuda_pack.dart b/lib/src/hook/windows_cuda_pack.dart index 69a6bb09..ddddfb9f 100644 --- a/lib/src/hook/windows_cuda_pack.dart +++ b/lib/src/hook/windows_cuda_pack.dart @@ -196,10 +196,18 @@ Future verifyWindowsCudaPackDirectory({ ); } - final actualEntries = directory - .listSync() - .whereType() - .map((file) => path.basename(file.path)) + final extractedEntities = directory.listSync(followLinks: false); + if (extractedEntities.any( + (entity) => + FileSystemEntity.typeSync(entity.path, followLinks: false) != + FileSystemEntityType.file, + )) { + throw const FormatException( + 'CUDA sidecar contains an unexpected directory or link.', + ); + } + final actualEntries = extractedEntities + .map((entity) => path.basename(entity.path)) .toSet(); if (actualEntries.difference({ ...expectedNames, diff --git a/test/unit/hook/windows_cuda_pack_test.dart b/test/unit/hook/windows_cuda_pack_test.dart index 1e272322..34aaa989 100644 --- a/test/unit/hook/windows_cuda_pack_test.dart +++ b/test/unit/hook/windows_cuda_pack_test.dart @@ -74,6 +74,22 @@ void main() { ); }); }); + + test('rejects an unexpected payload directory', () async { + await _withPack((directory, core, release) async { + Directory(path.join(directory.path, 'nested')).createSync(); + + await expectLater( + verifyWindowsCudaPackDirectory( + directory: directory, + expectedCudaMajor: 13, + release: release, + coreLibrary: core, + ), + throwsA(isA()), + ); + }); + }); } Future _withPack( diff --git a/website/docs/getting-started/installation.md b/website/docs/getting-started/installation.md index 3ceff504..dfb73262 100644 --- a/website/docs/getting-started/installation.md +++ b/website/docs/getting-started/installation.md @@ -130,8 +130,8 @@ bundle. `llamadart_windows_cuda` accepts `"13"` (the default), `"12"`, or `"both"`. The `both` option bundles both dependency families but loads exactly one: CUDA 13 when the driver API and every visible GPU satisfy compute capability 7.5+, otherwise CUDA 12 when every visible GPU satisfies compute -capability 5.0+ -GPU. This is runtime selection, so the build machine's GPU is not used as a +capability 5.0+. This is runtime selection, so the build machine's GPU is not +used as a proxy for the target computer. Expect `both` to add roughly 1.1 GB of compressed native assets for the currently audited packs. diff --git a/website/docs/platforms/support-matrix.md b/website/docs/platforms/support-matrix.md index 0eab1e58..de081d75 100644 --- a/website/docs/platforms/support-matrix.md +++ b/website/docs/platforms/support-matrix.md @@ -376,7 +376,7 @@ no valid entries remain, selection falls back to `cpu_profile` (or default the default cache search is not suitable. - `windows-x64` performs extra runtime dependency validation: - `cuda` requires `cudart` and `cublas` DLLs. - - Sidecar-capable releases default selected CUDA builds to CUDA 13; + - Sidecar-capable releases use CUDA 13 by default when CUDA is selected; `llamadart_windows_cuda: "12"` supports older compatible NVIDIA systems, while `"both"` ships both and selects one from the target driver API and GPU compute capability before backend loading. From 3fe4b881d2a54cc23e0b3c8287dbc75d0de4eb75 Mon Sep 17 00:00:00 2001 From: Jhin Lee Date: Mon, 17 Aug 2026 15:11:52 -0400 Subject: [PATCH 3/3] Normalize CUDA sidecar candidate paths --- lib/src/backends/llama_cpp/windows_cuda_selector.dart | 3 ++- test/unit/backends/llama_cpp/windows_cuda_selector_test.dart | 2 ++ 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/lib/src/backends/llama_cpp/windows_cuda_selector.dart b/lib/src/backends/llama_cpp/windows_cuda_selector.dart index d0a120ac..95b0024f 100644 --- a/lib/src/backends/llama_cpp/windows_cuda_selector.dart +++ b/lib/src/backends/llama_cpp/windows_cuda_selector.dart @@ -26,10 +26,11 @@ class WindowsCudaDriverProbe { } int? windowsCudaMajorFromFileName(String fileName) { + final baseName = fileName.split(RegExp(r'[/\\]')).last; final match = RegExp( r'^ggml-cuda-(12|13)(?:-[^\\/]+)*\.dll$', caseSensitive: false, - ).firstMatch(fileName); + ).firstMatch(baseName); return int.tryParse(match?.group(1) ?? ''); } diff --git a/test/unit/backends/llama_cpp/windows_cuda_selector_test.dart b/test/unit/backends/llama_cpp/windows_cuda_selector_test.dart index 7bd32260..8338ec1e 100644 --- a/test/unit/backends/llama_cpp/windows_cuda_selector_test.dart +++ b/test/unit/backends/llama_cpp/windows_cuda_selector_test.dart @@ -9,6 +9,8 @@ void main() { test('recognizes only versioned CUDA sidecar backend names', () { expect(windowsCudaMajorFromFileName('ggml-cuda-12.dll'), 12); expect(windowsCudaMajorFromFileName('ggml-cuda-13-windows-x64.dll'), 13); + expect(windowsCudaMajorFromFileName(r'C:\runtime\ggml-cuda-12.dll'), 12); + expect(windowsCudaMajorFromFileName('/runtime/ggml-cuda-13.dll'), 13); expect(windowsCudaMajorFromFileName('ggml-cuda.dll'), isNull); expect(windowsCudaMajorFromFileName('ggml-vulkan-13.dll'), isNull); });