Add selectable Windows CUDA release sidecars

AGENTS.md

@@ -203,6 +203,11 @@ WEBGPU_BRIDGE_ASSETS_TAG=<tag> ./scripts/fetch_webgpu_bridge_assets.sh
 See `website/docs/maintainers/native-and-web-sync.md` for the full maintainer
 procedure.
 
+Windows x64 CUDA sidecars are selected only when the `cuda` backend is
+requested. Keep `llamadart_windows_cuda` values limited to `12`, `13`, or
+`both`; `13` is the sidecar default. A `both` build may package both dependency
+families but must select and load exactly one CUDA backend per process.
+
 ## Changelog And Releases
 
 - Never add unreleased work to an already-published version section in

CHANGELOG.md

@@ -1,5 +1,11 @@
 ## Unreleased
 
+* Added verified Windows x64 CUDA sidecar selection for native llama.cpp
+  builds. CUDA 13 is the default when CUDA is requested, with CUDA 12 and a
+  portable `both` option available through `llamadart_windows_cuda`; dual-pack
+  builds probe the target NVIDIA driver/GPU and load only one matching CUDA
+  dependency family per process.
+
 * Updated the default llama.cpp native runtime pin to
   `leehack/llamadart-native@b10453`, picking up llama.cpp fixes for a Granite
   Speech heap overflow, a DOTS OCR out-of-bounds write, and a Step3VL

hook/build.dart

@@ -12,6 +12,7 @@ import 'package:logging/logging.dart';
 import 'package:path/path.dart' as path;
 
 import 'package:llamadart/src/hook/native_bundle_config.dart';
+import 'package:llamadart/src/hook/windows_cuda_pack.dart';
 
 const _llamaCppTag = 'b10453';
 const _nativeRepoSlug = 'leehack/llamadart-native';
@@ -156,6 +157,7 @@ const _litertLmBundleSpecs = <_LiteRtLmBundleSpec>[
 const _dynamicLibraryExtensions = {'.so', '.dylib', '.dll'};
 final _windowsCudartPattern = RegExp(r'^cudart64(?:[_-]?\d+)?\.dll$');
 final _windowsCublasPattern = RegExp(r'^cublas64(?:[_-]?\d+)?\.dll$');
+final _windowsCublasLtPattern = RegExp(r'^cublaslt64(?:[_-]?\d+)?\.dll$');
 final _linuxVersionedSoPattern = RegExp(r'\.so\.\d+$');
 final _nativeTagPattern = RegExp(r'^[A-Za-z0-9][A-Za-z0-9._-]*$');
 final _githubRepoSegmentPattern = RegExp(r'^[A-Za-z0-9_.-]+$');
@@ -350,11 +352,49 @@ void main(List<String> args) async {
         log: log,
       );
 
-      final libraryPaths = _collectDynamicLibraryPaths(bundleDir);
+      var libraryPaths = _collectDynamicLibraryPaths(bundleDir);
       if (libraryPaths.isEmpty) {
         throw Exception('No dynamic libraries found in ${bundleDir.path}.');
       }
 
+      final requestedBackends = parseRequestedBackends(
+        bundle: spec.bundle,
+        rawUserConfig: input.userDefines[nativeBackendUserDefineKey],
+      );
+      final wantsWindowsCudaSidecar =
+          spec.bundle == 'windows-x64' &&
+          requestedBackends?.contains('cuda') == true;
+      if (wantsWindowsCudaSidecar) {
+        final rawCudaSelection =
+            input.userDefines[nativeWindowsCudaUserDefineKey];
+        final legacyCudaAvailable = libraryPaths.any(_isWindowsCudaLibraryPath);
+        if (rawCudaSelection == null && legacyCudaAvailable) {
+          log.info(
+            'Using the legacy CUDA module from ${nativeConfig.sourceLabel}. '
+            'A sidecar-capable native release will default CUDA selection to 13.',
+          );
+        } else {
+          final selection = resolveWindowsCudaBundleSelection(rawCudaSelection);
+          final coreLibrary = _findWindowsGgmlBaseLibrary(libraryPaths);
+          final sidecarDirectories = await _acquireWindowsCudaSidecars(
+            packageRoot: pkgRoot,
+            nativeConfig: nativeConfig,
+            selection: selection,
+            coreLibrary: coreLibrary,
+            log: log,
+          );
+          libraryPaths = [
+            ...libraryPaths.where((entry) => !_isWindowsCudaLibraryPath(entry)),
+            for (final directory in sidecarDirectories)
+              ..._collectDynamicLibraryPaths(directory),
+          ];
+          log.info(
+            'Using Windows CUDA sidecar selection: '
+            '${selection.majors.join(', ')}.',
+          );
+        }
+      }
+
       final libraries = describeNativeLibraries(libraryPaths);
       if (!libraries.any((library) => library.isPrimary)) {
         throw Exception(
@@ -1236,6 +1276,185 @@ Uri? _resolveNativePath(HookInputUserDefines userDefines) {
   return resolvedPath;
 }
 
+bool _isWindowsCudaLibraryPath(String filePath) {
+  final fileName = path.basename(filePath).toLowerCase();
+  return (fileName.startsWith('ggml-cuda') && fileName.endsWith('.dll')) ||
+      _windowsCudartPattern.hasMatch(fileName) ||
+      _windowsCublasPattern.hasMatch(fileName) ||
+      _windowsCublasLtPattern.hasMatch(fileName);
+}
+
+File _findWindowsGgmlBaseLibrary(List<String> libraryPaths) {
+  final matches = libraryPaths
+      .where(
+        (entry) => describeNativeLibrary(entry).canonicalName == 'ggml-base',
+      )
+      .map(File.new)
+      .toList(growable: false);
+  if (matches.length != 1) {
+    throw Exception(
+      'Expected exactly one Windows ggml-base library, found ${matches.length}.',
+    );
+  }
+  return matches.single;
+}
+
+Future<List<Directory>> _acquireWindowsCudaSidecars({
+  required String packageRoot,
+  required _NativeBundleConfig nativeConfig,
+  required WindowsCudaBundleSelection selection,
+  required File coreLibrary,
+  required Logger log,
+}) async {
+  final bundleCache = _bundleCacheDirectory(
+    packageRoot: packageRoot,
+    nativeConfig: nativeConfig,
+    bundle: 'windows-x64',
+  );
+  final sidecarCache = path.join(bundleCache, 'cuda-sidecars');
+  await Directory(sidecarCache).create(recursive: true);
+  final release = await _acquireWindowsCudaReleaseManifest(
+    nativeConfig: nativeConfig,
+    cacheDirectory: sidecarCache,
+    log: log,
+  );
+  if (release.nativeTag != nativeConfig.tag) {
+    throw FormatException(
+      'CUDA sidecar release tag ${release.nativeTag} does not match '
+      '${nativeConfig.tag}.',
+    );
+  }
+
+  final directories = <Directory>[];
+  for (final major in selection.majors) {
+    final archiveName =
+        'llamadart-native-windows-x64-cuda$major-${nativeConfig.tag}.tar.gz';
+    final expectedArchiveSha256 = release.assetDigests[archiveName];
+    if (expectedArchiveSha256 == null) {
+      throw Exception(
+        'Native release ${nativeConfig.tag} does not publish $archiveName.',
+      );
+    }
+    final variantCache = path.join(sidecarCache, 'cuda$major');
+    final extractedDirectory = Directory(path.join(variantCache, 'extracted'));
+    if (extractedDirectory.existsSync()) {
+      try {
+        await verifyWindowsCudaPackDirectory(
+          directory: extractedDirectory,
+          expectedCudaMajor: major,
+          release: release,
+          coreLibrary: coreLibrary,
+        );
+        directories.add(extractedDirectory);
+        continue;
+      } on FormatException catch (error) {
+        log.warning('Cached CUDA $major sidecar is stale; refreshing: $error');
+        await extractedDirectory.delete(recursive: true);
+      }
+    }
+
+    final cachedArchive = File(path.join(variantCache, archiveName));
+    await cachedArchive.parent.create(recursive: true);
+    final localArchive = _resolveLocalWindowsCudaAsset(
+      nativeConfig: nativeConfig,
+      assetName: archiveName,
+    );
+    final archive = localArchive ?? cachedArchive;
+    if (localArchive == null && cachedArchive.existsSync()) {
+      if (await sha256File(cachedArchive) != expectedArchiveSha256) {
+        log.warning(
+          'Cached CUDA $major archive digest differs; redownloading.',
+        );
+        await cachedArchive.delete();
+      }
+    }
+    if (!archive.existsSync()) {
+      await _downloadReleaseAsset(
+        repository: nativeConfig.repository,
+        nativeTag: nativeConfig.tag,
+        assetName: archiveName,
+        destinationPath: archive.path,
+        log: log,
+      );
+    }
+    if (await sha256File(archive) != expectedArchiveSha256) {
+      if (localArchive == null && archive.existsSync()) {
+        await archive.delete();
+      }
+      throw Exception('CUDA $major sidecar archive checksum mismatch.');
+    }
+
+    await _extractCachedArchive(
+      archivePath: archive.path,
+      extractedDir: extractedDirectory,
+      cacheDir: variantCache,
+      log: log,
+    );
+    await verifyWindowsCudaPackDirectory(
+      directory: extractedDirectory,
+      expectedCudaMajor: major,
+      release: release,
+      coreLibrary: coreLibrary,
+    );
+    directories.add(extractedDirectory);
+  }
+  return directories;
+}
+
+Future<WindowsCudaReleaseManifest> _acquireWindowsCudaReleaseManifest({
+  required _NativeBundleConfig nativeConfig,
+  required String cacheDirectory,
+  required Logger log,
+}) async {
+  const assetName = 'assets.json';
+  final localManifest = _resolveLocalWindowsCudaAsset(
+    nativeConfig: nativeConfig,
+    assetName: assetName,
+  );
+  final manifestFile =
+      localManifest ?? File(path.join(cacheDirectory, assetName));
+  if (!manifestFile.existsSync()) {
+    await _downloadReleaseAsset(
+      repository: nativeConfig.repository,
+      nativeTag: nativeConfig.tag,
+      assetName: assetName,
+      destinationPath: manifestFile.path,
+      log: log,
+    );
+  }
+  return WindowsCudaReleaseManifest.parse(await manifestFile.readAsString());
+}
+
+File? _resolveLocalWindowsCudaAsset({
+  required _NativeBundleConfig nativeConfig,
+  required String assetName,
+}) {
+  final localPath = nativeConfig.localPath;
+  if (localPath == null) {
+    return null;
+  }
+  final localFilePath = localPath.toFilePath();
+  final root = File(localFilePath).existsSync()
+      ? File(localFilePath).parent.path
+      : localFilePath;
+  final candidates = <String>[
+    path.join(root, assetName),
+    path.join(root, nativeConfig.tag, assetName),
+    path.join(root, nativeConfig.tag, 'windows-x64', assetName),
+    path.join(root, 'windows-x64', assetName),
+  ];
+  for (final candidate in candidates) {
+    final file = File(candidate);
+    if (file.existsSync()) {
+      return file;
+    }
+  }
+  throw Exception(
+    'Local native source $root is missing required CUDA sidecar asset '
+    '$assetName.',
+  );
+}
+
 Future<Directory> _acquireBundleDirectory({
   required String packageRoot,
   required _NativeBundleConfig nativeConfig,

lib/src/backends/llama_cpp/llama_cpp_service.dart

@@ -24,6 +24,7 @@ import '../../core/template/chat_template_engine.dart';
 import 'load_param_helpers.dart';
 import 'bindings.dart';
 import 'llama_cpp_raw_bindings.dart' as raw_bindings;
+import 'windows_cuda_selector.dart';
 
 const _llamadartWrapperAssetId = 'package:llamadart/llamadart_wrapper';
 
@@ -2387,13 +2388,16 @@ class LlamaCppService {
       candidates.addAll(fileNameCandidates);
     }
 
-    _preloadWindowsBackendDependencies(backend);
-
     for (final candidate in candidates) {
       if (path.isAbsolute(candidate) && !File(candidate).existsSync()) {
         continue;
       }
 
+      _preloadWindowsBackendDependencies(
+        backend,
+        cudaMajor: windowsCudaMajorFromFileName(path.basename(candidate)),
+      );
+
       final alteredSearchPathHandle = _preloadWindowsBackendModule(
         candidate,
         backend,
@@ -2537,7 +2541,7 @@ class LlamaCppService {
     }
   }
 
-  void _preloadWindowsBackendDependencies(String backend) {
+  void _preloadWindowsBackendDependencies(String backend, {int? cudaMajor}) {
     if (!Platform.isWindows) {
       return;
     }
@@ -2548,7 +2552,8 @@ class LlamaCppService {
     }
 
     final cacheKey =
-        '$backend|${path.normalize(backendModuleDirectory).toLowerCase()}';
+        '$backend|${cudaMajor ?? 'legacy'}|'
+        '${path.normalize(backendModuleDirectory).toLowerCase()}';
     if (_preloadedBackendDependencyLibraries.containsKey(cacheKey)) {
       return;
     }
@@ -2559,6 +2564,7 @@ class LlamaCppService {
     for (final dependencyPath in windowsBackendDependencyPaths(
       backendModuleDirectory,
       backend,
+      cudaMajor: cudaMajor,
     )) {
       try {
         handles.add(DynamicLibrary.open(dependencyPath));
@@ -2594,6 +2600,7 @@ class LlamaCppService {
   static List<String> windowsBackendDependencyPaths(
     String directoryPath,
     String backend, {
+    int? cudaMajor,
     Iterable<String>? fileNames,
   }) {
     if (backend != 'cuda') {
@@ -2612,6 +2619,9 @@ class LlamaCppService {
       if (lower.startsWith('cudart64_') ||
           lower.startsWith('cublas64_') ||
           lower.startsWith('cublaslt64_')) {
+        if (cudaMajor != null && !lower.endsWith('_$cudaMajor.dll')) {
+          continue;
+        }
         selected.add(name);
       }
     }
@@ -3404,7 +3414,44 @@ class LlamaCppService {
       dynamicNames.sort(_compareAndroidCpuLibraryCandidates);
     }
     candidates.addAll(dynamicNames);
-    final resolved = candidates.toList(growable: false);
+    final resolved = candidates.toList();
+    if (backend == 'cuda' && Platform.isWindows) {
+      final versioned = resolved
+          .where((name) => windowsCudaMajorFromFileName(name) != null)
+          .toList(growable: false);
+      final availableMajors = versioned
+          .map(windowsCudaMajorFromFileName)
+          .whereType<int>()
+          .toSet();
+      if (availableMajors.length > 1) {
+        final probe = probeWindowsCudaDriver();
+        final selectedMajor = probe == null
+            ? null
+            : selectWindowsCudaMajor(
+                availableMajors: availableMajors,
+                probe: probe,
+              );
+        if (selectedMajor == null) {
+          _recordStartupDiagnostic(
+            'No bundled Windows CUDA sidecar matches the installed NVIDIA '
+            'driver and visible GPU compute capabilities.',
+          );
+          resolved.removeWhere(
+            (name) => windowsCudaMajorFromFileName(name) != null,
+          );
+        } else {
+          _recordStartupDiagnostic(
+            'Selected bundled Windows CUDA $selectedMajor sidecar from '
+            'driver API ${probe!.driverApiVersion} and compute capabilities '
+            '${probe.computeCapabilities.join(', ')}.',
+          );
+          resolved.removeWhere((name) {
+            final major = windowsCudaMajorFromFileName(name);
+            return major != null && major != selectedMajor;
+          });
+        }
+      }
+    }
     if (backend == 'cpu' && Platform.isAndroid) {
       resolved.sort(_compareAndroidCpuLibraryCandidates);
     }