diff --git a/.settings.yaml b/.settings.yaml index b08ee364d..f97c59f9b 100644 --- a/.settings.yaml +++ b/.settings.yaml @@ -202,7 +202,7 @@ testing: # GPU CI runtime pins consumed through .github/actions/load-versions. # renovate: datasource=helm depName=gpu-operator depType=testing - gpu_operator_chart_version: 'v26.3.3' + gpu_operator_chart_version: 'v26.7.0' # Base image for the CI smoke-test snapshot-agent image. The agent binary is # static Go and detects GPUs driver-free via NFD/PCI, so it no longer needs a # CUDA base / nvidia-smi. Matches the NVIDIA static distroless base used for diff --git a/api/aicr/v1/server.yaml b/api/aicr/v1/server.yaml index c02974cff..67a0282de 100644 --- a/api/aicr/v1/server.yaml +++ b/api/aicr/v1/server.yaml @@ -650,7 +650,7 @@ paths: value: version: "580.105.08" enabled: true - useOpenKernelModules: true + kernelModuleType: auto "400": description: > Invalid request. Common causes: (1) no criteria provided — all diff --git a/demos/dynamic.md b/demos/dynamic.md index 8c5caf331..f2c1be1b1 100644 --- a/demos/dynamic.md +++ b/demos/dynamic.md @@ -98,7 +98,7 @@ cat ./bundle/010-gpu-operator/cluster-values.yaml # # Generated by AICR # --- # driver: -# version: 580.173.02 +# version: 595.91.07 ``` And `install.sh` applies it after the baked values, so it wins: @@ -116,12 +116,12 @@ Three quick checks, the same ones the chainsaw suite enforces: ```shell # a) gone from the baked values.yaml (no match) -grep -c 'version: 580' ./bundle/010-gpu-operator/values.yaml +grep -c 'version: 595' ./bundle/010-gpu-operator/values.yaml # 0 # b) present in cluster-values.yaml grep version ./bundle/010-gpu-operator/cluster-values.yaml -# version: 580.173.02 +# version: 595.91.07 # c) install.sh references cluster-values.yaml grep -q cluster-values.yaml ./bundle/010-gpu-operator/install.sh && echo wired @@ -193,7 +193,7 @@ rm -rf ./b-oci oras pull "$REF" -o ./b-oci (cd ./b-oci && aicr verify .) cat ./b-oci/010-gpu-operator/cluster-values.yaml -# version: 580.173.02 +# version: 595.91.07 ``` > ttl.sh is public; anything pushed is world-readable until it expires. Fine for a throwaway demo bundle, not for anything sensitive — push to your own registry by pointing `spec.bundle.output.target` (or `--output`) at it, e.g. `oci://registry.example.com/aicr-bundle:tag`. Transport is separate: `spec.bundle.registry.plainHTTP: true` only switches HTTP instead of HTTPS (enable it only when the registry is intentionally served over plain HTTP, such as a local test registry); leave it off for any HTTPS registry, and use `spec.bundle.registry.insecureTLS: true` to skip TLS verification against a self-signed HTTPS registry. diff --git a/demos/query.md b/demos/query.md index 17cb7bddd..c73cc23d5 100644 --- a/demos/query.md +++ b/demos/query.md @@ -38,7 +38,7 @@ aicr query \ --selector components.gpu-operator.values.driver.version ``` -> `580.173.02` +> `595.91.07` Subtree — full driver block: @@ -53,8 +53,8 @@ enabled: true maxParallelUpgrades: 5 rdma: enabled: false -useOpenKernelModules: true -version: 580.173.02 +kernelModuleType: auto +version: 595.91.07 ``` ## Differentiation: Same Selector, Different Criteria diff --git a/docs/integrator/aks-gpu-setup.md b/docs/integrator/aks-gpu-setup.md index 695172902..6d11a5e16 100644 --- a/docs/integrator/aks-gpu-setup.md +++ b/docs/integrator/aks-gpu-setup.md @@ -376,7 +376,7 @@ required so the device plugin's volume-mounts allocation strategy still works for legitimately allocated pods, but it leaves the **volume-mounts device-request path open** (a pod that declares a `/dev/null`-backed mount whose destination is under `/var/run/nvidia-container-devices` can still select -devices — the pinned toolkit v1.19.1 accepts the volume-mount device request +devices — the pinned toolkit v1.20.0 accepts the volume-mount device request only when the mount *source* is `/dev/null`). This is the same posture as GPU-Operator-managed mode — not a regression — but it means full multi-tenant isolation additionally requires an admission policy restricting a diff --git a/docs/integrator/automation.md b/docs/integrator/automation.md index 9c46cd411..94333e5b3 100644 --- a/docs/integrator/automation.md +++ b/docs/integrator/automation.md @@ -231,7 +231,7 @@ spec: # Helm chart from upstream - repoURL: https://helm.ngc.nvidia.com/nvidia chart: gpu-operator - targetRevision: v26.3.3 + targetRevision: v26.7.0 helm: valueFiles: # Values live under the numbered bundle dir (NNN-/) diff --git a/docs/integrator/data-flow.md b/docs/integrator/data-flow.md index 833eb0453..b0e7007fc 100644 --- a/docs/integrator/data-flow.md +++ b/docs/integrator/data-flow.md @@ -634,7 +634,7 @@ spec: sources: # Helm chart from upstream - repoURL: https://helm.ngc.nvidia.com/nvidia - targetRevision: v26.3.3 + targetRevision: v26.7.0 chart: gpu-operator helm: valueFiles: diff --git a/docs/integrator/recipe-development.md b/docs/integrator/recipe-development.md index 4985364b6..a469decc3 100644 --- a/docs/integrator/recipe-development.md +++ b/docs/integrator/recipe-development.md @@ -888,7 +888,7 @@ spec: # recipes/registry.yaml - name: gpu-operator helm: - defaultVersion: v26.3.3 # Changed from v26.3.2 + defaultVersion: v26.7.0 # Changed from v26.3.3 ``` **Adding components:** diff --git a/docs/user/air-gap-mirror.md b/docs/user/air-gap-mirror.md index e495b804d..43f2544d3 100644 --- a/docs/user/air-gap-mirror.md +++ b/docs/user/air-gap-mirror.md @@ -112,7 +112,7 @@ metadata: name: aicr-images spec: images: - - name: nvcr.io/nvidia/gpu-operator:v26.3.3 + - name: nvcr.io/nvidia/gpu-operator:v26.7.0 - name: registry.k8s.io/nfd/node-feature-discovery:v0.19.0 # ... --- @@ -124,7 +124,7 @@ spec: charts: - name: gpu-operator repoURL: oci://ghcr.io/nvidia - version: v26.3.3 + version: v26.7.0 # ... ``` @@ -187,13 +187,13 @@ components: - name: aicr-images required: true images: - - nvcr.io/nvidia/gpu-operator:v26.3.3 + - nvcr.io/nvidia/gpu-operator:v26.7.0 - registry.k8s.io/nfd/node-feature-discovery:v0.19.0 # ... charts: - name: gpu-operator url: oci://ghcr.io/nvidia/gpu-operator - version: v26.3.3 + version: v26.7.0 namespace: gpu-operator # ... ``` diff --git a/docs/user/api-reference.md b/docs/user/api-reference.md index 22b57e01c..e869e3e75 100644 --- a/docs/user/api-reference.md +++ b/docs/user/api-reference.md @@ -737,7 +737,7 @@ curl -X POST "http://localhost:8080/v1/bundle" \ "apiVersion": "aicr.run/v1alpha2", "kind": "RecipeResult", "componentRefs": [ - {"name": "gpu-operator", "type": "Helm", "chart": "gpu-operator", "source": "https://helm.ngc.nvidia.com/nvidia", "version": "v26.3.3", "namespace": "gpu-operator", "valuesFile": "components/gpu-operator/values.yaml"}, + {"name": "gpu-operator", "type": "Helm", "chart": "gpu-operator", "source": "https://helm.ngc.nvidia.com/nvidia", "version": "v26.7.0", "namespace": "gpu-operator", "valuesFile": "components/gpu-operator/values.yaml"}, {"name": "network-operator", "type": "Helm", "chart": "network-operator", "source": "https://helm.ngc.nvidia.com/nvidia", "version": "26.1.1", "namespace": "nvidia-network-operator", "valuesFile": "components/network-operator/values.yaml"} ], "deploymentOrder": ["gpu-operator", "network-operator"] diff --git a/docs/user/cli-reference.md b/docs/user/cli-reference.md index 73e6c5c1e..45270b5c7 100644 --- a/docs/user/cli-reference.md +++ b/docs/user/cli-reference.md @@ -1988,7 +1988,7 @@ The `--vendor-charts` flag pulls upstream Helm chart bytes into the bundle at bu my-bundle/ 001-gpu-operator/ Chart.yaml # wrapper, declares the vendored subchart - charts/gpu-operator-v26.3.3.tgz # vendored upstream tarball + charts/gpu-operator-v26.7.0.tgz # vendored upstream tarball values.yaml # values nested under the subchart name cluster-values.yaml # dynamic values, also nested install.sh # helm upgrade --install ./ ... @@ -2017,10 +2017,10 @@ kind: BundleProvenance vendoredCharts: - name: gpu-operator chart: gpu-operator - version: v26.3.3 + version: v26.7.0 repository: https://helm.ngc.nvidia.com/nvidia sha256: abc123... - tarballName: gpu-operator-v26.3.3.tgz + tarballName: gpu-operator-v26.7.0.tgz pullerVersion: helm-cli v3.20.2 ``` diff --git a/docs/user/container-images.md b/docs/user/container-images.md index 112201971..4098434d0 100644 --- a/docs/user/container-images.md +++ b/docs/user/container-images.md @@ -42,7 +42,7 @@ _Rendering fidelity:_ `catalog-parity: charts are rendered with the shared recip | gatekeeper | helm | gatekeeper/gatekeeper | 3.22.2 | 3 | | gcp-driver-installer | manifest | — | — | 3 | | gke-nccl-tcpxo | manifest | — | — | 4 | -| gpu-operator | helm | nvidia/gpu-operator | v26.3.3 | 15 | +| gpu-operator | helm | nvidia/gpu-operator | v26.7.0 | 15 | | gpu-operator-ocp | manifest | — | — | 0 | | gpu-operator-ocp-olm | manifest | — | — | 0 | | grove | helm | grove-charts | v0.1.0-alpha.8 | 1 | @@ -64,8 +64,8 @@ _Rendering fidelity:_ `catalog-parity: charts are rendered with the shared recip | nfd-ocp-olm | manifest | — | — | 0 | | nodewright-customizations | manifest | — | — | 5 | | nodewright-operator | helm | nodewright | v0.17.1 | 3 | -| nvidia-dra-driver-gpu | helm | dra-driver-nvidia-gpu | 0.4.1 | 1 | -| nvidia-dra-driver-gpu-ocp | helm | dra-driver-nvidia-gpu | 0.4.1 | 1 | +| nvidia-dra-driver-gpu | helm | dra-driver-nvidia-gpu | 0.5.0 | 1 | +| nvidia-dra-driver-gpu-ocp | helm | dra-driver-nvidia-gpu | 0.5.0 | 1 | | nvsentinel | helm | nvsentinel | v1.20.0 | 6 | | prometheus-adapter | helm | prometheus-community/prometheus-adapter | 5.3.0 | 1 | | prometheus-adapter-ocp | helm | prometheus-community/prometheus-adapter | 5.3.0 | 1 | @@ -151,20 +151,20 @@ _No images extracted._ ### gpu-operator - `docker.io/library/busybox:1.38.0@sha256:dc2d74b28e4cf8984fa52af1f39bc7c3d9c73760b41a74d629f5d11b1ab28616` -- `nvcr.io/nvidia/cloud-native/dcgm:4.5.2-1-ubuntu22.04` -- `nvcr.io/nvidia/cloud-native/gdrdrv:v2.5.2` -- `nvcr.io/nvidia/cloud-native/k8s-cc-manager:v0.4.0` -- `nvcr.io/nvidia/cloud-native/k8s-driver-manager:v0.11.0` -- `nvcr.io/nvidia/cloud-native/k8s-mig-manager:v0.14.2` -- `nvcr.io/nvidia/cloud-native/nvidia-fs:2.27.3` -- `nvcr.io/nvidia/cloud-native/nvidia-sandbox-device-plugin:v0.0.3` -- `nvcr.io/nvidia/cloud-native/vgpu-device-manager:v0.4.2` +- `nvcr.io/nvidia/cloud-native/dcgm:4.6.0-1-ubuntu24.04` +- `nvcr.io/nvidia/cloud-native/gdrdrv:v2.6` +- `nvcr.io/nvidia/cloud-native/k8s-cc-manager:v0.4.3` +- `nvcr.io/nvidia/cloud-native/k8s-driver-manager:v0.12.0` +- `nvcr.io/nvidia/cloud-native/k8s-mig-manager:v0.15.0` +- `nvcr.io/nvidia/cloud-native/nvidia-fs:2.29.4` +- `nvcr.io/nvidia/cloud-native/nvidia-sandbox-device-plugin:v0.0.5` +- `nvcr.io/nvidia/cloud-native/vgpu-device-manager:v0.5.0` - `nvcr.io/nvidia/driver:580.173.02` -- `nvcr.io/nvidia/gpu-operator:v26.3.3` -- `nvcr.io/nvidia/k8s-device-plugin:v0.19.3` -- `nvcr.io/nvidia/k8s/container-toolkit:v1.19.1` -- `nvcr.io/nvidia/k8s/dcgm-exporter:4.5.3-4.8.2-distroless` -- `nvcr.io/nvidia/kubevirt-gpu-device-plugin:v1.5.0` +- `nvcr.io/nvidia/gpu-operator:v26.7.0` +- `nvcr.io/nvidia/k8s-device-plugin:v0.20.0` +- `nvcr.io/nvidia/k8s/container-toolkit:v1.20.0` +- `nvcr.io/nvidia/k8s/dcgm-exporter:4.6.0-4.8.3-distroless` +- `nvcr.io/nvidia/kubevirt-gpu-device-plugin:v1.6.0` ### gpu-operator-ocp @@ -281,11 +281,11 @@ _No images extracted._ ### nvidia-dra-driver-gpu -- `registry.k8s.io/dra-driver-nvidia/dra-driver-nvidia-gpu:v0.4.1` +- `registry.k8s.io/dra-driver-nvidia/dra-driver-nvidia-gpu:v0.5.0` ### nvidia-dra-driver-gpu-ocp -- `registry.k8s.io/dra-driver-nvidia/dra-driver-nvidia-gpu:v0.4.1` +- `registry.k8s.io/dra-driver-nvidia/dra-driver-nvidia-gpu:v0.5.0` ### nvsentinel @@ -450,7 +450,7 @@ Y = artifact attached, - = artifact absent, ? = could not probe. Image Sig SBOM Prov Notes -------------------------------------------------------------- --- ---- ---- ----- - registry.k8s.io/dra-driver-nvidia/dra-driver-nvidia-gpu:v0.4.1 Y - - + registry.k8s.io/dra-driver-nvidia/dra-driver-nvidia-gpu:v0.5.0 Y - - Summary: 1/1 signed · 0/1 SBOM · 0/1 provenance ``` diff --git a/examples/recipes/aks-training.yaml b/examples/recipes/aks-training.yaml index 418788d30..cdd8d4dcc 100644 --- a/examples/recipes/aks-training.yaml +++ b/examples/recipes/aks-training.yaml @@ -52,7 +52,7 @@ componentRefs: chart: gpu-operator type: Helm source: https://helm.ngc.nvidia.com/nvidia - version: v26.3.3 + version: v26.7.0 valuesFile: components/gpu-operator/values-aks-training.yaml dependencyRefs: - nfd @@ -151,7 +151,7 @@ componentRefs: chart: dra-driver-nvidia-gpu type: Helm source: oci://registry.k8s.io/dra-driver-nvidia/charts - version: 0.4.1 + version: 0.5.0 valuesFile: components/nvidia-dra-driver-gpu/values.yaml overrides: controller: diff --git a/examples/recipes/eks-gb200-ubuntu-training-with-validation.yaml b/examples/recipes/eks-gb200-ubuntu-training-with-validation.yaml index c9b91a3f8..49bbe8429 100644 --- a/examples/recipes/eks-gb200-ubuntu-training-with-validation.yaml +++ b/examples/recipes/eks-gb200-ubuntu-training-with-validation.yaml @@ -61,9 +61,9 @@ validation: deployment: constraints: - name: gpu-operator.version - value: "== v26.3.3" + value: "== v26.7.0" severity: warning - remediation: "Update GPU Operator to v26.3.3" + remediation: "Update GPU Operator to v26.7.0" checks: - expected-resources @@ -100,7 +100,7 @@ componentRefs: chart: gpu-operator type: Helm source: https://helm.ngc.nvidia.com/nvidia - version: v26.3.3 + version: v26.7.0 valuesFile: components/gpu-operator/values-eks-training.yaml expectedResources: - kind: Deployment @@ -129,7 +129,7 @@ componentRefs: chart: dra-driver-nvidia-gpu type: Helm source: oci://registry.k8s.io/dra-driver-nvidia/charts - version: 0.4.1 + version: 0.5.0 valuesFile: components/nvidia-dra-driver-gpu/values.yaml dependencyRefs: - gpu-operator diff --git a/examples/recipes/eks-training.yaml b/examples/recipes/eks-training.yaml index e4dc850a9..8de030f01 100644 --- a/examples/recipes/eks-training.yaml +++ b/examples/recipes/eks-training.yaml @@ -41,7 +41,7 @@ componentRefs: chart: gpu-operator type: Helm source: https://helm.ngc.nvidia.com/nvidia - version: v26.3.3 + version: v26.7.0 valuesFile: components/gpu-operator/values-eks-training.yaml dependencyRefs: - cert-manager diff --git a/pkg/bundler/testdata/stock_render_golden.yaml b/pkg/bundler/testdata/stock_render_golden.yaml index d43d9f13f..02c850371 100644 --- a/pkg/bundler/testdata/stock_render_golden.yaml +++ b/pkg/bundler/testdata/stock_render_golden.yaml @@ -3,51 +3,51 @@ # # One entry per leaf overlay: a digest over its fully rendered helm-deployer # bundle tree (sorted relative paths paired with per-file content hashes). -a100-aks-ubuntu-training-kubeflow: 265940a7ef6f82bfe9752e5e488b416d5d54aa0245b29f0262796978547af68d -a100-any: 457dd24ba4f8b9d44af7f68ddc1f4e9d46d0e6eac30b0eceb2e979e1e9c2d2db -a100-eks-ubuntu-training-kubeflow: 9a77e69e7ea639be6100c4c2c0cb49bf87059348020c98f5c561158a22d452e7 -a100-gke-cos-training-kubeflow: 0fb904af67e4ad85cb1b71efed94c8cd3966fc36f14cc80d9b50d5ffe7babde5 -a100-oke-ubuntu-training-kubeflow: 92c1a420bccca84c7ca7e74d8960af6c3662bce4c708c45ccaa386f209dc1c9e -b200-any: b8e028ff78b7f142c3e157143a258510c4f44579e31f13c183796767de53d862 -b200-gke-cos-inference-dynamo: cd3c475c9bb709b8164571be18ec625b38263a762d91d16f89049e75fc934eb4 -b200-gke-cos-training-kubeflow: f35da200bec023578aaaeebe337775d9b62718cfd2ac0c568c4a56f69575fbb3 -bcm-inference: 2a7d525459b585c3b1639eb556de8feb682b27f6bb8be3a7015de7773ba7c235 -gb200-any: ff2c8128bc91fce4a48d650d160659b08272a346a044f7955d3b23109fde6e8d -gb200-eks-ubuntu-inference-dynamo: 637a4fd32933c220453d3732363c9a56bd75dd91cdd886a9b0571b666670744d -gb200-eks-ubuntu-training-kubeflow: e4d2b1cc86cf33b7741e881a984f6943353bf2e1b85d918b8d3e82cf50c8fa60 -gb200-eks-ubuntu-training-slurm: d356fec2562b9d3f8adcccb2344402f41270ca301f353affd6529aaa86e430d4 -gb200-oke-ubuntu-inference-dynamo: 0304fffcd27d04827ae561b32ced55b9ff87e0f6049bda30a075afd49f24ad11 -gb200-oke-ubuntu-training-kubeflow: 3ed8582775ca602b48eb88ff05d228db3f4189f4230f31f72cf355be8d5241b0 -gb300-any: a67ce96e16a0a22d85f22c2803f029e49b1befb4226a60cdf9820fb6b6e77ea2 -gb300-eks-ubuntu-inference-dynamo: c725636167448e64ef1d567613aded4bcd7ede0137a3d1e4b9c70be1ec20c61a -gb300-eks-ubuntu-training-kubeflow: e84019cb236bae67eef79a8166214295b066d1514ebb869619898925be48522d -h100-aks-ubuntu-inference-dynamo: 54061313326c300b83a0627f415002bd080c7c0e338e19bcce7996446a791065 -h100-aks-ubuntu-training-kubeflow: ea0180c91126bdda49e3641538cfef03961358ebe4d883e893d53a82b26d4bd7 -h100-aks-ubuntu-training-slurm: 7f9c80295d770456adca24b66bc7681e42fd9db54fcb97d2541c657c939645c9 -h100-any: 7e98352317f4d2a753255ec776e7b7bc605ad288f3856b4a467a15f8844aa903 -h100-bcm-ubuntu-training: c73ab59f00b5425a0be7c8aaac6107469c8fb59e54eecce9cf4937704821611d -h100-eks-ubuntu-inference-dynamo: 6a3e47afa2018591b22f2fd96105d75ec23b631fafb4c0f30f42ba77af6f537a -h100-eks-ubuntu-inference-nim: 3b9b4fe53c967fa95e56e2cf6e631a55f044d50f76423e11ac176d839e005d2d -h100-eks-ubuntu-training-kubeflow: 18be43e357c98753a069d5db99f2dbe79f6bbd7c98c91286715968f183639cb0 -h100-eks-ubuntu-training-slurm: fdf26391dd6d2e10b9e853f3bd480c8968f2749948cb58a4a1cf5a1b34b85325 -h100-gke-cos-inference-dynamo: a8527e38ea90f346eee045c6b1b448263c83258c3d99864f73b4e9f86d28d520 -h100-gke-cos-training-kubeflow: 304639b08cf1899fc5cb8bb7a5c1a4f9ff652896ae18a92123c802b2f4cfc341 -h100-gke-cos-training-slurm: 0efee10e13589930111b9f5064430d2dcac463d2d2cf6032a6a7687be2085958 -h100-kind-inference-dynamo: 8cea3d208001c5d7bb542d83a75e2ab641722981b279db9391a300b8f9d1f20c -h100-kind-training-kubeflow: 1203d2e475a51a3e07d3a62a87ad2dca91c24eda3aea800aca86bf2dfee5a80a -h100-kind-training-slurm: 8a27b1ae4e91cda80c4a32d456552952f2ec0a28af10e9e0317219b551ddd133 -h200-any: ceee05a6ed9dd0f218919a90187de851e16e93cf89293c9746d5f0e46bbda222 -h200-eks-inference: 2900b7d96bc48a27eb5fedda75b99b010b9165bc6058372d2de3b991f64c4982 -h200-eks-training: 4b3074954d10a81d057ac746c7bddf3664637dfd002cd0955348ec3dee11eb20 -l40s-any: dabbb23da635f2b6db7803be55841a7305f884ce7798dbf79d952686004f09d8 -l40s-oke-inference: 8a8884dcf64b9feaa708b152cc4eaf1b9c09d790de295cf2b0e5a5ca464154f1 -l40s-oke-training: f9b741b390f9f29451c7ad16da56602bbf6497bec067be6b19242cffdf1e1de7 -monitoring-hpa: 32917a470d982044c3d49960dbd5e62ab56cd870375bfac23c35d7a4b3f4298a -ocp-inference-nim: 016fc59c13e901de5556f09f3cb3d5a90f91b9826d34f126fde1e066b311605f -ocp-training: 652f7ccdf52009adafe993da111857aa67390678f382b4c0caf0aecd0d613635 -rtx-pro-6000-any: 9115c5d61d9941130e448f29b9fbcd834f6d15a46fa4edf3239afc44fcdd36f0 -rtx-pro-6000-eks-ubuntu-inference-dynamo: 8c35c21289dc4a56f3e035da9d35427c9a1f2402c643febda2c6b397692b2a49 -rtx-pro-6000-eks-ubuntu-inference-nim: feb5f1d72ebd95459c9df0891420d20fac42c6b8c16f0e8fa551da61a19aef1d -rtx-pro-6000-eks-ubuntu-training-kubeflow: 1dfe905ac8d891fba4ce797de6532320024fe2616653e4703be5fc8e29716d77 -rtx-pro-6000-lke-ubuntu-inference: c5d53e0adbe9c500ecd5eb45d2482e632f1c414657a08ebef053a8aab90fc3cb -rtx-pro-6000-lke-ubuntu-training: 39e9cc1636a11d2568d9d50ecf3744f5773c4870772727661c40be63bac33280 +a100-aks-ubuntu-training-kubeflow: 179e890bbea38115389a4e16ca19854c0a8e1a8dacd7dd62cae17a57e810aed1 +a100-any: 44c73314feb82b7dcc5c22cfd16d78d928e2dc47d6bdca334968dd9b39d78d0b +a100-eks-ubuntu-training-kubeflow: cd2be5a192926d9f91e439167477a3fe3db6d449c1ea5bf9be65a939f343a347 +a100-gke-cos-training-kubeflow: c8a208cd7df159bc6613d6737b787db3769a6b1255a73c969a98e0f28bff7387 +a100-oke-ubuntu-training-kubeflow: 6705797efa62ae99242fe45309aae39f2c02b44dfc17f536f2f7597b2fd11a7c +b200-any: 83c8acdda32fad2afd441d66fbe7ba3d47565527bd4b630c8e9b669918bd8020 +b200-gke-cos-inference-dynamo: 03b5805d397f75d387b8e8b93fb073638d15e92e2e785f82b6a914d9a402cda2 +b200-gke-cos-training-kubeflow: 5bf0d4e35ccb60d9291db3ad91eae4a6ed1da3c6e3b04e5c8530be5947a25a2f +bcm-inference: 1c0a7bb58b9fe346a58e15ee59c49046c9a2003e1ba27dada7642b56f7419818 +gb200-any: fc0974eabd9041f7e29697303fca0358d9eece203d462e4a63fa21f4843d1d86 +gb200-eks-ubuntu-inference-dynamo: 5a0e6a53d4044c69369169e07c1907c09239d114b7399f0c851f8fcc5275f71a +gb200-eks-ubuntu-training-kubeflow: 8895cdb1981c7c008bc0f631a5c01c1a02f02730c836661f2d4a605ee71f597c +gb200-eks-ubuntu-training-slurm: 12923102f3c7e46670f17214ac5d0d03dc64ebd8b938e7952633ab07c3255c7b +gb200-oke-ubuntu-inference-dynamo: fa5a57601733e82bb763de3234166e3f72fc89543fc6a9721f8e762d75e9a714 +gb200-oke-ubuntu-training-kubeflow: 8499a0e45d1d446bf987a64b4612fba8705d7255dcee08d9e440408a8952fea4 +gb300-any: 6299dd58a7c29d668071ac97adcc6221005816368ecfeca676c74787e3126625 +gb300-eks-ubuntu-inference-dynamo: e0a01e5542dfc641b7ce2b7e4570b7a73efa951597d1af95b8e0b7a54f581b6b +gb300-eks-ubuntu-training-kubeflow: 2300a31efb361f4c46302b1680202c8d63adb083b2c981f6ea5950996b0f6b81 +h100-aks-ubuntu-inference-dynamo: 90e70e9240cbc4f82b387b71804c3a34dcbdad4ad6cec4a765bb8c6a52a9f90d +h100-aks-ubuntu-training-kubeflow: e0b99b6e04b877ffce34635aa75cb52e1adc265962c2173ff52793690f072d94 +h100-aks-ubuntu-training-slurm: 432ddd1c8084e6515c428e6727496ced19f3136b41ba254702d3938c7baf266c +h100-any: b6d3c92581c3b0e86731c36cbcbd420cbd5cc34ac2ba87aeefd765b95a938c9d +h100-bcm-ubuntu-training: 1d143be546c4b2c5fb12eadec4beb8d7297f8706ee9642f83c74b3d600775352 +h100-eks-ubuntu-inference-dynamo: 917a36151dbbdb20145addda2abc9600aeaed3e475c9c99121d9c24437ac016b +h100-eks-ubuntu-inference-nim: eab44cbf067e48785f190e710cf9e4c21ec9b516b10d6a4ab808a7e626b59a1e +h100-eks-ubuntu-training-kubeflow: d473a308bd3399ada67498431ecba245c7a7c8ee1ad9c3a7229bdaac300137c2 +h100-eks-ubuntu-training-slurm: 3de4aa7e2d7589d945e0d5ebb9ea07dde2afb53422354bc5562a375ed3f6b8b9 +h100-gke-cos-inference-dynamo: 8b01f66e9dc3877c8349be6853914666b5ac2a0c901ca8d7f9f389ca7f918df3 +h100-gke-cos-training-kubeflow: 1812fe1059ac056cba9db7248d7ab18a37336acf138ea424a65e88c16693ce7b +h100-gke-cos-training-slurm: f62fbb6425fdaa606ec786f94889deaa15406e525033046bfdcf2e8c699e02c4 +h100-kind-inference-dynamo: 943ef4d74e8efeaf136c8b9f3771f19493c6bd595f4fc5ae1c9806b7d30c912e +h100-kind-training-kubeflow: 5420fbf7f27fa89df22cac52b37e41793b246932cc92ec5306abf91e63abf0fe +h100-kind-training-slurm: 9c1bb61f8fc2afcc772ff1a427611bb0b59732af751357ec7b0149d8d4a70d05 +h200-any: 378bd79e8f5eb801e52ef2d56e1f349c484367e60d8302ce64ace7654e8ef7c2 +h200-eks-inference: 7da11486decd1747a9fda05996be2e5584b6217f2acdf46637dfd9957d9944cd +h200-eks-training: d3c01bff1eace5f71df68470e60be0dac0ccd97b864c8f9ec3e4570117b9dfd4 +l40s-any: 34c6144a769834d1a491050ea4f21052ed00e28183e576eb3150c22888f90e78 +l40s-oke-inference: eb70336d93a39ea95c30275708047a90f8ab29117002735cb36df3740eab55ec +l40s-oke-training: 987818b884ea09e48335427178f30a28de5862ff1407334af56e99e77a31167b +monitoring-hpa: d563d7261ab44da64f6af4a7f9b33a6281797815232990d43014fc7aa360e8b2 +ocp-inference-nim: a728bed625f568760bf291d54e066a443e2c141b239dc5fb7f4bc96dcac87c39 +ocp-training: 32a047795bcb4c9ea6363e18de6d4f213e6b8c9003106cb4a54ccffd3a6b0ef1 +rtx-pro-6000-any: dafcf7588df850699cd260e9ddeaff374bef6018e4a713fed1a72791d62ab931 +rtx-pro-6000-eks-ubuntu-inference-dynamo: 754541a260c92f495b17fe4a127829c76c2554a5bbdf58b45c253d444f864e3b +rtx-pro-6000-eks-ubuntu-inference-nim: eb6e7ae0b710db7570ddc12f79f0495bba16c36483a4b305fe5bc9061f314ec7 +rtx-pro-6000-eks-ubuntu-training-kubeflow: 32123c9c89208aa6b916bd0099326c890c8b41697a9339bf2170455a2c87a801 +rtx-pro-6000-lke-ubuntu-inference: 7e3ce1dead651a89ba35bfbf40a43e69a5f37f467df9601727e51ca181beedba +rtx-pro-6000-lke-ubuntu-training: a3e7f3a094d0da80f80496b0da88d81c0678746c272b46ec97f12e3ead4d845b diff --git a/pkg/bundler/validations/checks.go b/pkg/bundler/validations/checks.go index 22765a67a..2a2afe6fd 100644 --- a/pkg/bundler/validations/checks.go +++ b/pkg/bundler/validations/checks.go @@ -464,7 +464,7 @@ func driverAbsentRemedy(service recipe.CriteriaServiceType, os recipe.CriteriaOS "DaemonSet alongside it; see " + "docs/integrator/gke-gpu-setup.md." case recipe.CriteriaOSUbuntu: - // The pinned GPU Operator (v26.3.3) supports driver management + // The pinned GPU Operator supports driver management // on GKE only on Ubuntu node images with containerd. return "On GKE Ubuntu node images the GPU Operator can manage " + "the driver: bundle in GPU-Operator-managed mode: " + @@ -643,11 +643,11 @@ func effectiveComponentValues(ctx context.Context, recipeResult *recipe.RecipeRe // path.Clean'd (trailing-slash spellings compare equal, mirroring // pkg/recipe/driver_root_lockstep_test.go), declared empty string → // the default (the operator's own transformForDriverInstallDir treats -// "" identically to the default, gpu-operator v26.3.3). An explicitly +// "" identically to the default, gpu-operator v26.7.0). An explicitly // null or non-map hostPaths section is rejected with a blocking // message: Helm null-coalescing deletes a null key together with its // chart defaults, so the chart's unconditional .Values.hostPaths.rootFS -// access (clusterpolicy.yaml, v26.3.3) fails at install. A declared +// access (clusterpolicy.yaml, v26.7.0) fails at install. A declared // value that cleans to a relative path is rejected too — host-path // mounts require absolute paths. func resolveInstallDir(values map[string]any, componentName string) (string, bool, []string) { @@ -680,7 +680,7 @@ func resolveInstallDir(values map[string]any, componentName string) (string, boo // rejected rather than silently defaulted: the emitted values // would carry it verbatim, and the pinned ClusterPolicy CRD // types hostPaths.driverInstallDir as a string (gpu-operator - // v26.3.3 nvidia.com_clusterpolicies.yaml), so the install + // v26.7.0 nvidia.com_clusterpolicies.yaml), so the install // fails while a defaulted check would have validated against // /run/nvidia/driver instead. return installDir, false, []string{fmt.Sprintf( @@ -693,7 +693,7 @@ func resolveInstallDir(values map[string]any, componentName string) (string, boo if dir == "" { // Intentionally default-equivalent: the operator's own // transformForDriverInstallDir early-returns on "" exactly like - // the default (gpu-operator v26.3.3, controllers/object_controls.go). + // the default (gpu-operator v26.7.0, controllers/object_controls.go). return installDir, false, nil } cleaned := path.Clean(dir) @@ -716,7 +716,7 @@ func resolveInstallDir(values map[string]any, componentName string) (string, boo // resolveDRARoot resolves the effective nvidia-dra-driver-gpu // nvidiaDriverRoot for Rule 2. Only a genuinely ABSENT key falls back to -// the chart-default assumption ("/", DRA chart v0.4.1 values.yaml). A +// the chart-default assumption ("/", DRA chart v0.5.0 values.yaml). A // present null, empty-string, or non-string value is rejected: unlike the // gpu-operator's driverInstallDir (where "" is default-equivalent, see // resolveInstallDir), the DRA chart pipes the raw value through @@ -866,7 +866,7 @@ func nvsentinelDynamicGuardViolations(bundlerConfig *config.Config, componentNam // null-coalescing deletes the key together with its chart defaults, // so .Values.
is nil at render time and the gpu-operator // templates fail on unconditional field access (e.g. -// .Values.driver.manager.repository in _helpers.tpl, v26.3.3) — +// .Values.driver.manager.repository in _helpers.tpl, v26.7.0) — // ownership cannot be verified and the install would fail anyway. // A non-boolean toggle is rejected because the chart renders the // value unquoted, so YAML re-typing at install time can flip it to a @@ -1705,7 +1705,7 @@ func CheckNVSentinelDriverLabelDetectable(ctx context.Context, componentName str } // defaultRuntimeClassName is the shared chart default: the gpu-operator -// chart ships operator.runtimeClass: nvidia (v26.3.3, verified against +// chart ships operator.runtimeClass: nvidia (v26.7.0, verified against // the pinned chart values), and nvsentinel's metadata-collector subchart // ships runtimeClassName: "nvidia" (v1.9.0, charts/metadata-collector/ // values.yaml:31). Either side left unset therefore resolves to this diff --git a/pkg/bundler/validations/checks_test.go b/pkg/bundler/validations/checks_test.go index c72760c32..833f1718f 100644 --- a/pkg/bundler/validations/checks_test.go +++ b/pkg/bundler/validations/checks_test.go @@ -1330,7 +1330,7 @@ func TestCheckDriverOwnershipCoherence(t *testing.T) { // the key together with its chart defaults, so .Values.driver // is nil at render time and the chart's unconditional field // accesses (_helpers.tpl .Values.driver.manager.repository, - // v26.3.3) fail at install. Reject rather than default. The + // v26.7.0) fail at install. Reject rather than default. The // reachable vector is a top-level --set-json null: the typed // merge assigns it verbatim (mergeTypedValueByPath), while the // recipe-side overlay merge drops nil-valued keys before the @@ -1357,7 +1357,7 @@ func TestCheckDriverOwnershipCoherence(t *testing.T) { // hostPaths: null is the same hazard as a null driver/toolkit // section: Helm null-coalescing deletes the chart defaults and // clusterpolicy.yaml's unconditional .Values.hostPaths.rootFS - // access fails at install (v26.3.3). + // access fails at install (v26.7.0). name: "--set-json hostPaths=null → rejected", recipeResult: result("", aks, gpuOpRef(driverOn())), bundlerConfig: config.NewConfig(config.WithValueOverridesTypedPaths([]config.TypedComponentPath{ diff --git a/pkg/client/v1/gpu_driver_state.go b/pkg/client/v1/gpu_driver_state.go index dd4d2da7e..59801164d 100644 --- a/pkg/client/v1/gpu_driver_state.go +++ b/pkg/client/v1/gpu_driver_state.go @@ -109,7 +109,7 @@ func driverAbsentRemedy(service recipe.CriteriaServiceType, os recipe.CriteriaOS "DaemonSet and pins the driver version — nothing to deploy " + "by hand; see docs/integrator/gke-gpu-setup.md." case recipe.CriteriaOSUbuntu: - // The pinned GPU Operator (v26.3.3) supports driver management + // The pinned GPU Operator supports driver management // on GKE only on Ubuntu node images with containerd. return "On GKE Ubuntu node images the GPU Operator can manage " + "the driver: bundle in GPU-Operator-managed mode: " + diff --git a/pkg/recipe/computedomain_numnodes_test.go b/pkg/recipe/computedomain_numnodes_test.go new file mode 100644 index 000000000..799bd92cc --- /dev/null +++ b/pkg/recipe/computedomain_numnodes_test.go @@ -0,0 +1,248 @@ +// Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package recipe + +import ( + "io/fs" + "regexp" + "strings" + "testing" +) + +// specKeyRE matches the document's top-level `spec:` mapping key. +var specKeyRE = regexp.MustCompile(`^(\s*)spec\s*:\s*$`) + +// numNodesChildRE matches `numNodes:` at a given exact indentation. +func numNodesChildRE(indent string) *regexp.Regexp { + return regexp.MustCompile(`^` + regexp.QuoteMeta(indent) + `numNodes\s*:`) +} + +// specHasNumNodes reports whether a single YAML document declares numNodes as a +// DIRECT CHILD of spec. +// +// Path-aware on purpose. An earlier version matched `numNodes:` anywhere in the +// document, which accepted `metadata.numNodes` — a key Kubernetes ignores, while +// the required `spec.numNodes` stays absent and admission still fails. Matching +// the key without its parent is not a weaker check, it is the wrong check. +// +// Comment lines are stripped first: these manifests legitimately discuss +// "spec.numNodes: Required value" in prose, and a scan that does not strip them +// matches that instead of the real key, passing even when the key is deleted. +// +// A full YAML parse is unavailable — the manifests are Helm templates containing +// {{ }} expressions that no YAML parser accepts — so this walks indentation. +func specHasNumNodes(doc string) bool { + var lines []string + for _, line := range strings.Split(doc, "\n") { + if strings.HasPrefix(strings.TrimSpace(line), "#") || strings.TrimSpace(line) == "" { + continue + } + lines = append(lines, line) + } + for i, line := range lines { + m := specKeyRE.FindStringSubmatch(line) + if m == nil { + continue + } + specIndent := m[1] + var childRE *regexp.Regexp + for _, sub := range lines[i+1:] { + subIndent := sub[:len(sub)-len(strings.TrimLeft(sub, " \t"))] + // Dedent to spec's level or shallower ends the spec mapping. + if len(subIndent) <= len(specIndent) { + break + } + if childRE == nil { + childRE = numNodesChildRE(subIndent) + } + if childRE.MatchString(sub) { + return true + } + } + } + return false +} + +// computeDomainDocsMissingNumNodes returns the 0-based indexes of YAML +// documents that declare kind: ComputeDomain without spec.numNodes. +// +// Scoped per document: a multi-document manifest where one ComputeDomain sets +// the key and a second omits it would satisfy a whole-file scan while still +// failing admission. No such manifest exists in the catalog today; the guard is +// document-scoped so adding one cannot silently bypass it. +func computeDomainDocsMissingNumNodes(content string) []int { + var missing []int + for i, doc := range strings.Split(content, "\n---") { + if !strings.Contains(doc, "kind: ComputeDomain") { + continue + } + if !specHasNumNodes(doc) { + missing = append(missing, i) + } + } + return missing +} + +// TestComputeDomainManifestsSetNumNodes guards the fresh-install CRD-overlap +// hazard introduced by GPU Operator v26.7.0. +// +// Two charts in the catalog ship a CRD named computedomains.resource.nvidia.com: +// the standalone nvidia-dra-driver-gpu chart, and — new in gpu-operator +// v26.7.0 — the GPU Operator chart. The two copies are NOT identical. The +// operator's is a stale snapshot that lists numNodes as required and supplies +// no `default: 0`; the DRA driver's makes it optional with a default. +// +// Helm installs crds/ only when the CRD is absent and never upgrades it, and +// gpu-operator is ordered before nvidia-dra-driver-gpu. So on a FRESH cluster +// the operator's stricter copy is the one that lands, and any ComputeDomain CR +// omitting spec.numNodes is rejected by the API server with +// "spec.numNodes: Required value". Structural defaulting cannot rescue it +// because that copy carries no default. Neither chart installs a webhook that +// could supply the field. +// +// An UPGRADED cluster masks this: it already has the permissive copy installed +// by DRA 0.4.1, so the CR still admits. That asymmetry is why this is a unit +// guard rather than something an upgrade-path e2e would catch. +// +// The invariant: every ComputeDomain CR shipped in the catalog must set +// spec.numNodes explicitly, so it is valid under BOTH CRD copies regardless of +// which chart installed the CRD first. +// +// See PR #2439 and issue #1087 for the driver-root analog of this +// cross-component coupling problem. +func TestComputeDomainManifestsSetNumNodes(t *testing.T) { + t.Parallel() + + efs := GetEmbeddedFS() + + var checked int + err := fs.WalkDir(efs, ".", func(path string, d fs.DirEntry, err error) error { + if err != nil { + return err + } + if d.IsDir() || !strings.HasSuffix(path, ".yaml") { + return nil + } + raw, readErr := efs.ReadFile(path) + if readErr != nil { + return readErr + } + content := string(raw) + if !strings.Contains(content, "kind: ComputeDomain") { + return nil + } + checked++ + + for _, idx := range computeDomainDocsMissingNumNodes(content) { + t.Errorf("%s: YAML document %d declares kind: ComputeDomain but does not set spec.numNodes.\n"+ + " GPU Operator v26.7.0 ships a ComputeDomain CRD copy that marks numNodes\n"+ + " REQUIRED with no default, and it is installed before the DRA driver's\n"+ + " permissive copy. On a fresh cluster this CR is rejected at admission with\n"+ + " \"spec.numNodes: Required value\".\n"+ + " Set numNodes explicitly (0 is correct under IMEXDaemonsWithDNSNames=true,\n"+ + " the DRA driver default, where each IMEX daemon starts without waiting for\n"+ + " a quorum). See PR #2439.", path, idx) + } + return nil + }) + if err != nil { + t.Fatalf("walking embedded recipes: %v", err) + } + + // Fail closed on a vacuous pass: if the walk matched nothing, the guard is + // silently inert and a regression would go unnoticed. + if checked == 0 { + t.Fatal("no ComputeDomain manifests found in the embedded recipes — " + + "this guard is vacuous. Either the manifests moved, or the embed " + + "pattern no longer covers them.") + } + t.Logf("verified %d ComputeDomain manifest(s) set spec.numNodes", checked) +} + +// TestComputeDomainScannerCases pins the scanner's behavior directly, so the +// catalog guard above cannot quietly stop discriminating if the catalog changes. +// Each case is a shape that has either fooled a previous version of this +// scanner or must keep working. +func TestComputeDomainScannerCases(t *testing.T) { + t.Parallel() + + const header = "apiVersion: resource.nvidia.com/v1beta1\nkind: ComputeDomain\n" + + tests := []struct { + name string + doc string + wantMissing bool + }{ + { + name: "spec.numNodes present", + doc: header + "metadata:\n name: cd\nspec:\n numNodes: 0\n channel:\n allocationMode: All\n", + }, + { + name: "spec.numNodes absent", + doc: header + "metadata:\n name: cd\nspec:\n channel:\n allocationMode: All\n", + wantMissing: true, + }, + { + // Regression: an earlier scanner matched numNodes anywhere in the + // document, so this passed while admission would still fail. + name: "numNodes under metadata, not spec", + doc: header + "metadata:\n name: cd\n numNodes: 0\nspec:\n channel:\n allocationMode: All\n", + wantMissing: true, + }, + { + // Regression: an earlier scanner did not strip comments, so the + // prose in the real manifest satisfied it even with the key gone. + name: "numNodes only mentioned in a comment", + doc: header + "metadata:\n name: cd\nspec:\n # numNodes: Required value\n channel:\n allocationMode: All\n", + wantMissing: true, + }, + { + name: "nested numNodes does not satisfy the direct-child rule", + doc: header + "metadata:\n name: cd\nspec:\n channel:\n numNodes: 0\n", + // numNodes exists but under spec.channel, not spec. + wantMissing: true, + }, + { + name: "templated value is acceptable", + doc: header + "metadata:\n name: cd\nspec:\n numNodes: {{ .Values.numNodes }}\n", + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + got := !specHasNumNodes(tt.doc) + if got != tt.wantMissing { + t.Errorf("specHasNumNodes reported missing=%v, want %v\ndoc:\n%s", + got, tt.wantMissing, tt.doc) + } + }) + } +} + +// TestComputeDomainMultiDocument covers the per-document scoping: a file where +// one ComputeDomain is valid and a second is not must report only the second. +func TestComputeDomainMultiDocument(t *testing.T) { + t.Parallel() + + content := "apiVersion: v1\nkind: ConfigMap\nmetadata:\n name: unrelated\n" + + "\n---\napiVersion: resource.nvidia.com/v1beta1\nkind: ComputeDomain\nmetadata:\n name: ok\nspec:\n numNodes: 0\n" + + "\n---\napiVersion: resource.nvidia.com/v1beta1\nkind: ComputeDomain\nmetadata:\n name: bad\nspec:\n channel:\n allocationMode: All\n" + + missing := computeDomainDocsMissingNumNodes(content) + if len(missing) != 1 || missing[0] != 2 { + t.Errorf("missing documents = %v, want [2] (only the third document lacks spec.numNodes)", missing) + } +} diff --git a/pkg/recipe/driver_root_lockstep_test.go b/pkg/recipe/driver_root_lockstep_test.go index 1372e6141..e1c83fdc7 100644 --- a/pkg/recipe/driver_root_lockstep_test.go +++ b/pkg/recipe/driver_root_lockstep_test.go @@ -74,8 +74,8 @@ import ( // **Why "explicitly set" matters for the lockstep case.** An empty value // falls through to the upstream chart's bundled default, which the test // cannot read — and per-component defaults differ (GPU Operator chart -// 26.3.3 defaults driverInstallDir to /run/nvidia/driver, but DRA chart -// 25.12.0 defaults nvidiaDriverRoot to /). Relying on chart defaults is +// 26.7.0 defaults driverInstallDir to /run/nvidia/driver, but DRA chart +// 0.5.0 defaults nvidiaDriverRoot to /). Relying on chart defaults is // itself drift waiting to happen on the next chart bump, so when the // lockstep applies the test treats "not explicitly set on both" as a // failure. @@ -211,7 +211,7 @@ func TestDriverRootLockstep(t *testing.T) { t.Errorf( "overlay %q: both nvidia-dra-driver-gpu.nvidiaDriverRoot and gpu-operator.hostPaths.driverInstallDir are unset.\n"+ " Both must be set explicitly to the same path. Chart defaults differ across components\n"+ - " (gpu-operator chart 26.3.3: /run/nvidia/driver; dra chart 25.12.0: /), so an unset value\n"+ + " (gpu-operator chart 26.7.0: /run/nvidia/driver; dra chart 0.5.0: /), so an unset value\n"+ " is drift waiting to happen on the next chart bump.\n"+ " See issue #1087.", name) diff --git a/pkg/recipe/testdata/catalog_parity_golden.yaml b/pkg/recipe/testdata/catalog_parity_golden.yaml index 95fe07389..06cffd0e0 100644 --- a/pkg/recipe/testdata/catalog_parity_golden.yaml +++ b/pkg/recipe/testdata/catalog_parity_golden.yaml @@ -3,51 +3,51 @@ # # One entry per leaf overlay: sha256 of its deterministically-marshalled # resolved recipe. A moved digest means that recipe's resolved bytes changed. -a100-aks-ubuntu-training-kubeflow: 3d8d77ac7ac29bf13253410e0b0486bc322d69d66d8778884df3375619566d58 -a100-any: 62d1581b21fecd3465b69659bac3dc9e5438eb2612c98c1acdd7b19cc9237084 -a100-eks-ubuntu-training-kubeflow: 0196adfa0c77f230d95ed3ebee2bdade8ac720f631cd3bbe2b8cec7d55094a10 -a100-gke-cos-training-kubeflow: 514f996360b251a6ef657ba1a715b5705be03d6b6dccb53887ee96554a83ffb6 -a100-oke-ubuntu-training-kubeflow: 310a6ee2abdd7f8508882429d657c01bd23458ba635236c410934825e4f617dc -b200-any: 63dbc4fe27b84321e8b1991cd98c13f85395a3919d13e0c6891599eaa1f1e729 -b200-gke-cos-inference-dynamo: 8174db8270ea447d9b307f21be9945b919ed3bcdacc702d2961d2ff75fb60c35 -b200-gke-cos-training-kubeflow: 96723738cb0baaaec300c95d5cfbc08a18d999fd0adb52aef3c88436561da1e7 -bcm-inference: 92fa64d3af61026891b37cbf692d2663be7b2484078ab65e2653d18fc2bfe6f8 -gb200-any: 9ee6ab187b4c1b93cd859d86ada35e922d0873ffc6fe892cb084ad98baf09727 -gb200-eks-ubuntu-inference-dynamo: 8babae6413d104da16060f18d919c80fd97ff983bd32f9876bd5f72da7fd6be6 -gb200-eks-ubuntu-training-kubeflow: 63b2fbd3a195bac682bb45263e5d5d0ec479a6571dd38ef0505eeb4f287b29e9 -gb200-eks-ubuntu-training-slurm: c35e33618c0a8bb548bbe3c5d6765faf90cebe1427478ef97089f7a30bb37a6c -gb200-oke-ubuntu-inference-dynamo: 6f01848824dcbefb0f58ec56ee9e911fff1470c9f67c0d0a9ba0de1dd1e648d2 -gb200-oke-ubuntu-training-kubeflow: 3959f47474edfac9b53a65939117d4364551fde0c9324246f195f144fe3beb12 -gb300-any: 7f68607dffbbfc912f19b00ffa833caa81ca675d25bc22cf612dbd0b76f39475 -gb300-eks-ubuntu-inference-dynamo: b14a3d99503949b8bfcdc81455da4af389a339deb24ea784637a7901297d70ad -gb300-eks-ubuntu-training-kubeflow: 5303e5ebf8dc2ec13c2cff60c7caba2ef53d1a65bfd69d06f4f78cbb93835c49 -h100-aks-ubuntu-inference-dynamo: 9fe4d82412d250d4004efc56fa4ddd6b8b1a9166387455e96ac2c2284d323107 -h100-aks-ubuntu-training-kubeflow: 72a8ca777c333e0db82717f53a706ae9155163ff0c73fd4e5ed8d71badda5811 -h100-aks-ubuntu-training-slurm: ef48d77171817bb5b24815d128457c246e0e5dc0fe850aa6881276952e590055 -h100-any: f5a55f03948075adde9b1b24fd650d98fa4d35b85f394d071ad3365ba05588a5 -h100-bcm-ubuntu-training: 76746f45add1fb6b0d7b6441b4d9581acceb3b5d3d4800b943c1cf460c92fbc5 -h100-eks-ubuntu-inference-dynamo: 0a31f03010849f76b385ce81c21e499af884f41bee89de47b9bacf03f8b6e311 -h100-eks-ubuntu-inference-nim: 9c6a4839bf5b59f620093427de0e138a9fafaf1f55f3fa7f555b1ee33b914a3a -h100-eks-ubuntu-training-kubeflow: bfcbd5e14fe43209f2401e429d39b5f7a52d049832db07e5b9595d130ab6e2fc -h100-eks-ubuntu-training-slurm: 8d29c2be5c3c0dc8396b0df53111a485f6d57771394df8ef135e4b052d681064 -h100-gke-cos-inference-dynamo: 45190af6c5b5d76f69a4f36d9c0cb800c450bdef54ca5b522ad49f605d1bad2a -h100-gke-cos-training-kubeflow: 43de2291cc0adb9b8d89494179d86eeef35994b2ad4fd4062a87265d6753ec6b -h100-gke-cos-training-slurm: 414f95e1a231b1c86bcbe70729b820a95533367dec0c9e3f7e36514431fad1d9 -h100-kind-inference-dynamo: 0e2d552b62b55b91f960c9721ed128b1d07f68be8517799473fbaa3baa1cc6e5 -h100-kind-training-kubeflow: 4bb03659faaa6a9a66d58dfc95572bdb54b75010c219c4bbefbd0eafeb8ed867 -h100-kind-training-slurm: 0245c786638ade1023ba4dc49e0f2efd56d2f789bdf447e946bbc3a3e652e734 -h200-any: acf1986d76037dddd025125eaa34eb7ecf3d31fe62cb80e7dc89dd042ebb42f5 -h200-eks-inference: d1ff148001722e9d5e23606bd7770f8120506f08c10f0c132e0c379f535609c5 -h200-eks-training: 98d04b3c81a059a0321451a39772af8d0d3310ad1fb861837f87d82dd997c4a1 -l40s-any: 89210ba5815f93ce3d7b1a2a78f7ba75d083556e940603a1bc812eac9aeccd0a -l40s-oke-inference: 6c75ef93e86fb21dbdf73ac97089b894ca8da832164dc807e94592493ab7f7cc -l40s-oke-training: 2a35011bd59cf3c727cea158d504c51bd59cba069620d3d6a4988d68972dee74 -monitoring-hpa: 15e93304e7e68997e7b15a0be4af360a29f10c869502e6dccb6a6cefb6519fd5 -ocp-inference-nim: 49245ecfc91f67fcc2ed0b305b00046a4164628cafc75ab98542e6dafa626821 -ocp-training: aaa8cd7f45ef42235b075936461d715e2e017ea2802df9e8017de38304a17915 -rtx-pro-6000-any: 9639d67639256a44be8566f3f8eda49ceb4025b6b140a3bc2a4b6d4768d07ffa -rtx-pro-6000-eks-ubuntu-inference-dynamo: cbe11051d979ae20e83471142c4582e1504fd24f55898a00afcededbe143bba6 -rtx-pro-6000-eks-ubuntu-inference-nim: 08b73d58d0d60681be55ee1402bdec5bf1367a0cb137a273ca27bec0788cc020 -rtx-pro-6000-eks-ubuntu-training-kubeflow: c2595e2994d56a735714103eedd30c6afd45921f458bf026c3432017b5eba60e -rtx-pro-6000-lke-ubuntu-inference: 28cdab0531f8ed6018391aba7cab1fbfad8ba551392de1b9138f3b88042f5340 -rtx-pro-6000-lke-ubuntu-training: 312c4cf49d0e8cec0377cd9858338d7106226a2252d81828a75c1d426a194a2f +a100-aks-ubuntu-training-kubeflow: 52a19136ff27701be3ebd3291d0838b405dd4200b9ea2ad1f15f5e9231afc164 +a100-any: eedbde8040b6909da0e42d2af6345881ca5fcaacaa3ac83b222ba998c5c07c5e +a100-eks-ubuntu-training-kubeflow: 9918500dfc098857fa643a8427d378ad205fc7914566095c8678c2b837b93e7a +a100-gke-cos-training-kubeflow: e2dedc04e012cf16d146c953bc1f6a58237afd305e1978b58976d67d5695cb37 +a100-oke-ubuntu-training-kubeflow: c2f71c8a4bb59e2cca85fc007740eede1ded4a6514ec46071d7172f707c0fa13 +b200-any: 2d0d453bcdcd89d5db6b0aab435aad942458b108a7b0060a48e73c88a96e11ca +b200-gke-cos-inference-dynamo: 37147bb9daf7782bd58349c97ef01fa72b70fb46f1e01746d8725869a83ecfc6 +b200-gke-cos-training-kubeflow: e219956eaff20825678d485dd656e6071a6a91afc180af9abe54122d93191164 +bcm-inference: c608118b2f1c74e2d1d2d9defa90918f9da2cf6122afbf133d605f8a0154ea87 +gb200-any: 548063fe33335835470c1bbf9fb9ace134061d06a8910409e68d90423377a41d +gb200-eks-ubuntu-inference-dynamo: b15f4dd4cf5224ca76f7524f15611ed22910c4ee90334fe160594cb1592e9ecc +gb200-eks-ubuntu-training-kubeflow: fc6b234dd59ec815ba4bc67c695ff04057dbf3a233ba68662c54b96450ba035b +gb200-eks-ubuntu-training-slurm: 4b2007f13eb5c953620a3e18271e3f5b8d8a407258a36c1c47882012490d0470 +gb200-oke-ubuntu-inference-dynamo: 4ce2e85e13d125948e91614f0d07ebb70968ea0d93beec9feba03863f420f282 +gb200-oke-ubuntu-training-kubeflow: 3f0394a1b85b3d16147a31ecf55088d899c3b103ead7cb2f8a25183cb3a1575d +gb300-any: 10eb8b5cc815ca6a190412e8e344d261c0153cc65e2bf8266d89f6d821171da3 +gb300-eks-ubuntu-inference-dynamo: 9a4d41b7e6c57ec6cb44df5f6af64ca3f4d15a4568d6dfaaa16dfc0e5b791877 +gb300-eks-ubuntu-training-kubeflow: 460cb834803767f98f70f6bad7e006d22ebd79f83aa4c62f3feace0f896bd579 +h100-aks-ubuntu-inference-dynamo: e16910a1555ed867d5c85b63d6bea4861e585f1c4cbffeb81b9315fe04feb551 +h100-aks-ubuntu-training-kubeflow: 0ef4f87a06b668338e8416eb7b85e5ac63f472ebbd478c69db1a6e36d9cdac6a +h100-aks-ubuntu-training-slurm: f43f166e8d0af0447fa581e227fff3088d8a0d5efb9bac8cda4aff2f6cf84d04 +h100-any: 879ed75bd46ef020cc9f31eb03bceb49debe3719c2dcf567d072ac5b13ba3d2e +h100-bcm-ubuntu-training: 2d25d1fa54d969088215f4f95d2eabd2aead572a4223d2421f6d9ffaaf15be61 +h100-eks-ubuntu-inference-dynamo: 0b20a1afb8336d0fb7abe60dd5308cb2b76b96df2a4932fe18196468a08b1012 +h100-eks-ubuntu-inference-nim: 00460f237a30b03854875ce52a9e7a58217878192cfeb3f52745b26d1ee66c18 +h100-eks-ubuntu-training-kubeflow: 0940f0e8420302f91cd11f3d0451406de9259408245ef9410423ad09bb369766 +h100-eks-ubuntu-training-slurm: 939f6422a3b79e68b5a43225a838ecba9d2878d57cd9b4a98207d6d2860b1193 +h100-gke-cos-inference-dynamo: 904aeab7e68bce844fc9a72a9a79f27d28f77d044d510fdcbaecce596cefe983 +h100-gke-cos-training-kubeflow: 2ec3f85e22a551c48bca2a67af7211965c405f7a6fe7e4fa8be3a161311943a9 +h100-gke-cos-training-slurm: b74e224e7362f803da02ed60e090954d3c70d964cdc6db62df4bac4b5a060f70 +h100-kind-inference-dynamo: b17a7431bf9b5cd97211d343274324bdf7871b44b504f77bd575f54092c993db +h100-kind-training-kubeflow: 49f22bb0ba0b62c8c728cb801e99af9d4515ecbff4151100a0788bd3f4c1dcf2 +h100-kind-training-slurm: 891a37a0ff30b83bae3252eaae8305fb1fc4f29ddd3cafcb751f561308b738a7 +h200-any: 959608a2eb8060c6df2ef7caf453a1500f1adc8790b8c228851c603d15b6e908 +h200-eks-inference: dd2814a49106bfae9623bd310587ae4bcf378fba20039726796272fe23a571cc +h200-eks-training: cd7439667773a77b523a7337e672f4b8a9828bf88af618436094dc556746acb1 +l40s-any: 83b2b92d8b43e8f23754ff9608f02a6d64d7de28f8ccb3b7d6ae67679cffac93 +l40s-oke-inference: 9761d9172ba2c7561e0406392e2c0828450f6e3d5537c61bdaf5732d80fca487 +l40s-oke-training: 4b40b25e4a5620d3cea304df67209b397e3813190ee2ff4c7dd5798690ecd10d +monitoring-hpa: 7156deeffed71f32316b4a2501c771033d67dc2de3d3181311e985cc3625e557 +ocp-inference-nim: 08f9f5d03f030160eb6951bd238c24a400c41d62dcf721589abb3d265df8e3fc +ocp-training: 3998c76724c4189cc463e5cf473a3cd97d8c0eaf3299336be7f1ffac9cb11e6f +rtx-pro-6000-any: 62760c1f1e8287ae8c37b20d306546469bed84a742ea2b143aad52f942575f7e +rtx-pro-6000-eks-ubuntu-inference-dynamo: 2e14248538447738e7ea1699b0c0730dfd1e18b8aa8588ab0540903874e3bc2e +rtx-pro-6000-eks-ubuntu-inference-nim: b1c918b05e42118edcb51a164a9e251c944243d3e8d0127855ec361f8740ee52 +rtx-pro-6000-eks-ubuntu-training-kubeflow: cb833e1bcc917f105a106f4e36186ee55c665c641c22bd0a5da4206284b04daf +rtx-pro-6000-lke-ubuntu-inference: 1de10952036333b9f1d475b5ae212d54e4ed226ff3a44c17b13415bb708e5368 +rtx-pro-6000-lke-ubuntu-training: 296c4559de7977f238303e787407e57489a0dae8b24249f9f416a0425d4d48c7 diff --git a/recipes/components/gpu-operator-ocp/values.yaml b/recipes/components/gpu-operator-ocp/values.yaml index 1f090147d..8b931edbc 100644 --- a/recipes/components/gpu-operator-ocp/values.yaml +++ b/recipes/components/gpu-operator-ocp/values.yaml @@ -160,7 +160,7 @@ daemonsets: # operator.upgradeCRD: Helm chart setting, not a ClusterPolicy field. # operator.resources: Helm chart setting for the operator Deployment, not CR. # driver.version: OCP operator manages driver version via the certified driver container. -# driver.useOpenKernelModules: OCP uses pre-built driver containers from the certified catalog. +# driver.kernelModuleType: OCP uses pre-built driver containers from the certified catalog. # driver.maxParallelUpgrades: Use daemonsets.rollingUpdate.maxUnavailable instead. # devicePlugin.env: OCP sets device plugin env via the operator's own defaults. # validator: OCP operator manages validation internally. diff --git a/recipes/components/gpu-operator/manifests/nvidia-toolkit-hardening-aks.yaml b/recipes/components/gpu-operator/manifests/nvidia-toolkit-hardening-aks.yaml index f8a440c85..885dbc00a 100644 --- a/recipes/components/gpu-operator/manifests/nvidia-toolkit-hardening-aks.yaml +++ b/recipes/components/gpu-operator/manifests/nvidia-toolkit-hardening-aks.yaml @@ -55,7 +55,7 @@ # device-request path open — the same posture as GPU-Operator-managed mode, # not a regression — so full multi-tenant isolation additionally requires # an admission policy restricting a /dev/null-backed mount whose destination -# is under /var/run/nvidia-container-devices (the pinned toolkit v1.19.1 +# is under /var/run/nvidia-container-devices (the pinned toolkit v1.20.0 # accepts the volume-mount device request only when the mount SOURCE is # /dev/null). secure-accelerator-access does not # exercise the volume-mounts path, so a green result is not proof of diff --git a/recipes/components/gpu-operator/values.yaml b/recipes/components/gpu-operator/values.yaml index a107a5943..569d102d2 100644 --- a/recipes/components/gpu-operator/values.yaml +++ b/recipes/components/gpu-operator/values.yaml @@ -147,16 +147,60 @@ gfd: enabled: true driver: - # NVIDIA's recommended driver for the v26.3.3 chart; matches the - # GB200+EFA floor so a single global pin covers H100/B200/GB200 EKS. + # HELD at 580.173.02 — deliberately BEHIND the v26.7.0 chart default of + # 595.91.07. Do not advance without EFA validation on P6e hardware. + # + # NVIDIA documents R595 as backward incompatible with P6e-GB200 EFA. R595 + # drops NVreg_GrdmaPciTopoCheckOverride (the R580-era escape hatch) and + # substitutes a topology check: EFA must sit in an IOMMU group, or GPU and + # EFA must share a root port. p6e-gb200/gb300 satisfies neither — measured + # on p6e-gb300r.36xlarge, where EFA (0000:97:00.0) has no iommu_group and + # sits under pci0000:84 while the GPU (0000:9d:00.0) is in group 0 under + # pci0000:98. + # + # The failure is proven, not inferred. Running the NCCL NET workload under + # 595 with the preflight bypassed, both nodes selected aws-ofi-nccl/EFA/RDMA + # and then aborted, with the kernel logging: + # NVRM: dma-buf attach failed: topology not supported for mapping type FORCE_PCIE + # Under 580.173.02 on the same two nodes, NCCL NET passed at 43.16 GB/s. + # + # Scope: the AWS P6e Grace-Blackwell + EFA GPUDirect path specifically. NVLS + # passed under 595 (840 GB/s), as did deployment and conformance; NVLink + # never touches EFA. + # + # Holding here is a supported combination, not an untested one: GPU Operator + # v26.7.0 lists 580.173.02 as supported (595.91.07 is merely its default), + # and DRA driver 0.5.0 requires >= 580. Pinned rather than inherited so a + # chart bump is an explicit driver decision, and so BOM/attestation inputs + # stay deterministic. + # + # Do not substitute a newer R580 patch without re-validating EFA — + # 580.173.02 is the exact measured baseline. Per-family overrides, so + # non-P6e families can advance independently, are tracked separately. version: 580.173.02 enabled: true - useOpenKernelModules: true + # Replaces useOpenKernelModules, which the chart deprecated and made a + # no-op (identical NOTE in v26.3.3 and v26.7.0 values.yaml), so the old + # key had already stopped having any effect before this pin moved. + # + # "auto" pins today's effective behaviour, not a change: it is the chart + # default both before and after this bump, and it is what the OCP + # ClusterPolicy template already falls back to + # (components/gpu-operator-ocp/manifests/clusterpolicy.yaml). Per the + # ClusterPolicy CRD, auto selects the recommended module type from the + # host's GPU devices and driver branch — which yields open modules on + # Blackwell/Grace, where they are required. + # + # Setting "open" instead would pin the intent the dead key expressed, but + # that IS a behaviour change on any hardware where auto currently selects + # proprietary, so it needs its own validation rather than riding a version + # bump. + kernelModuleType: auto maxParallelUpgrades: 5 rdma: # Default off: nvidia_peermem only loads against Mellanox MOFED # symbols. AWS EFA (EKS p4d/p5/p5e) and Linode have no MOFED, so - # peermem fails to load and v26.3.3's stricter driver-validation + # peermem fails to load and the chart's stricter driver-validation # init container blocks the rest of the GPU stack. Overlays that # ship MOFED (AKS via network-operator) explicitly re-enable this. enabled: false @@ -189,8 +233,9 @@ validator: nfd: enabled: false -# Confidential Compute Manager defaults to enabled in chart v26.3.x; keep -# it off until AICR has explicit CC-capable hardware support. +# Confidential Compute Manager defaults to enabled in the upstream chart +# (verified still true in v26.7.0); keep it off until AICR has explicit +# CC-capable hardware support. ccManager: enabled: false diff --git a/recipes/components/nvidia-dra-driver-gpu-ocp/manifests/scc-rolebinding.yaml b/recipes/components/nvidia-dra-driver-gpu-ocp/manifests/scc-rolebinding.yaml index a04352c57..d0ecd406e 100644 --- a/recipes/components/nvidia-dra-driver-gpu-ocp/manifests/scc-rolebinding.yaml +++ b/recipes/components/nvidia-dra-driver-gpu-ocp/manifests/scc-rolebinding.yaml @@ -26,7 +26,7 @@ # SCC (see CodeRabbit CWE-732 finding on PR #2057). Name confirmed via: # helm template nvidia-dra-driver-gpu-ocp \ # oci://registry.k8s.io/dra-driver-nvidia/charts/dra-driver-nvidia-gpu \ -# --version 0.4.1 -f values.yaml --namespace nvidia-dra-driver +# --version 0.5.0 -f values.yaml --namespace nvidia-dra-driver apiVersion: rbac.authorization.k8s.io/v1 kind: RoleBinding metadata: diff --git a/recipes/components/slinky-slurm/manifests/compute-domain.yaml b/recipes/components/slinky-slurm/manifests/compute-domain.yaml index 3bf024787..3af09f839 100644 --- a/recipes/components/slinky-slurm/manifests/compute-domain.yaml +++ b/recipes/components/slinky-slurm/manifests/compute-domain.yaml @@ -29,6 +29,21 @@ metadata: name: slinky-slurm-imex namespace: {{ .Release.Namespace }} spec: + # Pinned explicitly rather than relying on the CRD default. GPU Operator + # v26.7.0 ships its own copy of resource.nvidia.com_computedomains.yaml in + # crds/, and that copy is a stale snapshot: it lists numNodes as REQUIRED + # and carries no `default: 0`, where the DRA driver chart's copy makes it + # optional with a default. Helm installs crds/ only when the CRD is absent + # and never upgrades it, and gpu-operator is ordered before + # nvidia-dra-driver-gpu, so on a fresh cluster the operator's stricter copy + # wins and a CR omitting numNodes is rejected with "spec.numNodes: Required + # value". Setting it here is valid under both copies. + # + # 0 is the correct value, not merely a placeholder: with + # IMEXDaemonsWithDNSNames=true (the DRA driver default) each IMEX daemon + # starts immediately instead of waiting for a quorum. Matches + # buildComputeDomain() in validators/performance/nccl_all_reduce_bw_constraint.go. + numNodes: 0 channel: allocationMode: All resourceClaimTemplate: diff --git a/recipes/overlays/ocp.yaml b/recipes/overlays/ocp.yaml index babdda2d6..2867b475f 100644 --- a/recipes/overlays/ocp.yaml +++ b/recipes/overlays/ocp.yaml @@ -22,7 +22,7 @@ spec: service: ocp constraints: - # nvidia-dra-driver-gpu-ocp (chart 0.4.1) declares kubeVersion: + # nvidia-dra-driver-gpu-ocp (chart 0.5.0) declares kubeVersion: # '>=1.32.0-0'. Raised from >= 1.29 so DRA's floor doesn't silently # pass recipe-time constraint checks and then fail at `helm install`. # This effectively requires OCP 4.19+ (first OCP minor shipping k8s 1.32). diff --git a/recipes/registry.yaml b/recipes/registry.yaml index abbf297a2..5abd739d7 100644 --- a/recipes/registry.yaml +++ b/recipes/registry.yaml @@ -100,7 +100,7 @@ components: helm: defaultRepository: https://helm.ngc.nvidia.com/nvidia defaultChart: nvidia/gpu-operator - defaultVersion: v26.3.3 + defaultVersion: v26.7.0 defaultNamespace: gpu-operator nodeScheduling: system: @@ -396,7 +396,7 @@ components: helm: defaultRepository: oci://registry.k8s.io/dra-driver-nvidia/charts defaultChart: dra-driver-nvidia-gpu - defaultVersion: "0.4.1" + defaultVersion: "0.5.0" defaultNamespace: nvidia-dra-driver nodeScheduling: system: @@ -1101,7 +1101,7 @@ components: helm: defaultRepository: oci://registry.k8s.io/dra-driver-nvidia/charts defaultChart: dra-driver-nvidia-gpu - defaultVersion: "0.4.1" + defaultVersion: "0.5.0" defaultNamespace: nvidia-dra-driver nodeScheduling: system: diff --git a/tests/chainsaw/cli/cuj1-training/assert-bundle-scheduling.yaml b/tests/chainsaw/cli/cuj1-training/assert-bundle-scheduling.yaml index 5b59ffdac..c38d7c7d3 100644 --- a/tests/chainsaw/cli/cuj1-training/assert-bundle-scheduling.yaml +++ b/tests/chainsaw/cli/cuj1-training/assert-bundle-scheduling.yaml @@ -39,14 +39,14 @@ daemonsets: # ── Driver: nvidia_peermem off on EKS (AWS EFA path uses aws-ofi-nccl) # nvidia_peermem only loads against Mellanox MOFED symbols; on AWS EFA -# (p4d/p5/p5e) it fails to load and v26.3.3's strict driver-validation +# (p4d/p5/p5e) it fails to load and the chart's strict driver-validation # init container blocks the rest of the GPU stack. NCCL multi-node on # EFA uses libfabric via aws-ofi-nccl, not nvidia_peermem. driver: enabled: true rdma: enabled: false - useOpenKernelModules: true + kernelModuleType: auto # ── GDRCopy: GPU-direct memory for high-performance training ───────── gdrcopy: