From 3afd931bc136b24aac2649ee6124515e5161c2b5 Mon Sep 17 00:00:00 2001 From: Yuan Chen Date: Fri, 28 Aug 2026 09:28:15 -0700 Subject: [PATCH 01/12] chore(recipes): bump gpu-operator to v26.7.0, driver to 595.91.07, DRA to 0.5.0 Signed-off-by: Yuan Chen --- recipes/components/gpu-operator/values.yaml | 12 ++++++++---- .../manifests/scc-rolebinding.yaml | 2 +- recipes/overlays/ocp.yaml | 2 +- recipes/registry.yaml | 6 +++--- 4 files changed, 13 insertions(+), 9 deletions(-) diff --git a/recipes/components/gpu-operator/values.yaml b/recipes/components/gpu-operator/values.yaml index a107a5943..de4866fd3 100644 --- a/recipes/components/gpu-operator/values.yaml +++ b/recipes/components/gpu-operator/values.yaml @@ -147,16 +147,20 @@ gfd: enabled: true driver: - # NVIDIA's recommended driver for the v26.3.3 chart; matches the - # GB200+EFA floor so a single global pin covers H100/B200/GB200 EKS. - version: 580.173.02 + # NVIDIA's recommended driver for the v26.7.0 chart; a single global + # pin covers H100/B200/GB200 EKS. Pinned rather than inherited from the + # chart default so a future chart bump is an explicit driver decision + # instead of a silent fleet-wide driver change, and so BOM/attestation + # inputs stay deterministic. Moves with the chart: holding an older + # driver against a newer chart is a combination upstream does not test. + version: 595.91.07 enabled: true useOpenKernelModules: true maxParallelUpgrades: 5 rdma: # Default off: nvidia_peermem only loads against Mellanox MOFED # symbols. AWS EFA (EKS p4d/p5/p5e) and Linode have no MOFED, so - # peermem fails to load and v26.3.3's stricter driver-validation + # peermem fails to load and the chart's stricter driver-validation # init container blocks the rest of the GPU stack. Overlays that # ship MOFED (AKS via network-operator) explicitly re-enable this. enabled: false diff --git a/recipes/components/nvidia-dra-driver-gpu-ocp/manifests/scc-rolebinding.yaml b/recipes/components/nvidia-dra-driver-gpu-ocp/manifests/scc-rolebinding.yaml index a04352c57..d0ecd406e 100644 --- a/recipes/components/nvidia-dra-driver-gpu-ocp/manifests/scc-rolebinding.yaml +++ b/recipes/components/nvidia-dra-driver-gpu-ocp/manifests/scc-rolebinding.yaml @@ -26,7 +26,7 @@ # SCC (see CodeRabbit CWE-732 finding on PR #2057). Name confirmed via: # helm template nvidia-dra-driver-gpu-ocp \ # oci://registry.k8s.io/dra-driver-nvidia/charts/dra-driver-nvidia-gpu \ -# --version 0.4.1 -f values.yaml --namespace nvidia-dra-driver +# --version 0.5.0 -f values.yaml --namespace nvidia-dra-driver apiVersion: rbac.authorization.k8s.io/v1 kind: RoleBinding metadata: diff --git a/recipes/overlays/ocp.yaml b/recipes/overlays/ocp.yaml index babdda2d6..2867b475f 100644 --- a/recipes/overlays/ocp.yaml +++ b/recipes/overlays/ocp.yaml @@ -22,7 +22,7 @@ spec: service: ocp constraints: - # nvidia-dra-driver-gpu-ocp (chart 0.4.1) declares kubeVersion: + # nvidia-dra-driver-gpu-ocp (chart 0.5.0) declares kubeVersion: # '>=1.32.0-0'. Raised from >= 1.29 so DRA's floor doesn't silently # pass recipe-time constraint checks and then fail at `helm install`. # This effectively requires OCP 4.19+ (first OCP minor shipping k8s 1.32). diff --git a/recipes/registry.yaml b/recipes/registry.yaml index abbf297a2..5abd739d7 100644 --- a/recipes/registry.yaml +++ b/recipes/registry.yaml @@ -100,7 +100,7 @@ components: helm: defaultRepository: https://helm.ngc.nvidia.com/nvidia defaultChart: nvidia/gpu-operator - defaultVersion: v26.3.3 + defaultVersion: v26.7.0 defaultNamespace: gpu-operator nodeScheduling: system: @@ -396,7 +396,7 @@ components: helm: defaultRepository: oci://registry.k8s.io/dra-driver-nvidia/charts defaultChart: dra-driver-nvidia-gpu - defaultVersion: "0.4.1" + defaultVersion: "0.5.0" defaultNamespace: nvidia-dra-driver nodeScheduling: system: @@ -1101,7 +1101,7 @@ components: helm: defaultRepository: oci://registry.k8s.io/dra-driver-nvidia/charts defaultChart: dra-driver-nvidia-gpu - defaultVersion: "0.4.1" + defaultVersion: "0.5.0" defaultNamespace: nvidia-dra-driver nodeScheduling: system: From 5933de880dd58af51cf0e695cb578be5d8197591 Mon Sep 17 00:00:00 2001 From: Yuan Chen Date: Fri, 28 Aug 2026 09:30:11 -0700 Subject: [PATCH 02/12] docs: regenerate container-images BOM for the gpu-operator and DRA bump Signed-off-by: Yuan Chen --- docs/user/container-images.md | 38 +++++++++++++++++------------------ 1 file changed, 19 insertions(+), 19 deletions(-) diff --git a/docs/user/container-images.md b/docs/user/container-images.md index 112201971..858e9c90f 100644 --- a/docs/user/container-images.md +++ b/docs/user/container-images.md @@ -42,7 +42,7 @@ _Rendering fidelity:_ `catalog-parity: charts are rendered with the shared recip | gatekeeper | helm | gatekeeper/gatekeeper | 3.22.2 | 3 | | gcp-driver-installer | manifest | — | — | 3 | | gke-nccl-tcpxo | manifest | — | — | 4 | -| gpu-operator | helm | nvidia/gpu-operator | v26.3.3 | 15 | +| gpu-operator | helm | nvidia/gpu-operator | v26.7.0 | 15 | | gpu-operator-ocp | manifest | — | — | 0 | | gpu-operator-ocp-olm | manifest | — | — | 0 | | grove | helm | grove-charts | v0.1.0-alpha.8 | 1 | @@ -64,8 +64,8 @@ _Rendering fidelity:_ `catalog-parity: charts are rendered with the shared recip | nfd-ocp-olm | manifest | — | — | 0 | | nodewright-customizations | manifest | — | — | 5 | | nodewright-operator | helm | nodewright | v0.17.1 | 3 | -| nvidia-dra-driver-gpu | helm | dra-driver-nvidia-gpu | 0.4.1 | 1 | -| nvidia-dra-driver-gpu-ocp | helm | dra-driver-nvidia-gpu | 0.4.1 | 1 | +| nvidia-dra-driver-gpu | helm | dra-driver-nvidia-gpu | 0.5.0 | 1 | +| nvidia-dra-driver-gpu-ocp | helm | dra-driver-nvidia-gpu | 0.5.0 | 1 | | nvsentinel | helm | nvsentinel | v1.20.0 | 6 | | prometheus-adapter | helm | prometheus-community/prometheus-adapter | 5.3.0 | 1 | | prometheus-adapter-ocp | helm | prometheus-community/prometheus-adapter | 5.3.0 | 1 | @@ -151,20 +151,20 @@ _No images extracted._ ### gpu-operator - `docker.io/library/busybox:1.38.0@sha256:dc2d74b28e4cf8984fa52af1f39bc7c3d9c73760b41a74d629f5d11b1ab28616` -- `nvcr.io/nvidia/cloud-native/dcgm:4.5.2-1-ubuntu22.04` -- `nvcr.io/nvidia/cloud-native/gdrdrv:v2.5.2` -- `nvcr.io/nvidia/cloud-native/k8s-cc-manager:v0.4.0` -- `nvcr.io/nvidia/cloud-native/k8s-driver-manager:v0.11.0` -- `nvcr.io/nvidia/cloud-native/k8s-mig-manager:v0.14.2` -- `nvcr.io/nvidia/cloud-native/nvidia-fs:2.27.3` -- `nvcr.io/nvidia/cloud-native/nvidia-sandbox-device-plugin:v0.0.3` -- `nvcr.io/nvidia/cloud-native/vgpu-device-manager:v0.4.2` -- `nvcr.io/nvidia/driver:580.173.02` -- `nvcr.io/nvidia/gpu-operator:v26.3.3` -- `nvcr.io/nvidia/k8s-device-plugin:v0.19.3` -- `nvcr.io/nvidia/k8s/container-toolkit:v1.19.1` -- `nvcr.io/nvidia/k8s/dcgm-exporter:4.5.3-4.8.2-distroless` -- `nvcr.io/nvidia/kubevirt-gpu-device-plugin:v1.5.0` +- `nvcr.io/nvidia/cloud-native/dcgm:4.6.0-1-ubuntu24.04` +- `nvcr.io/nvidia/cloud-native/gdrdrv:v2.6` +- `nvcr.io/nvidia/cloud-native/k8s-cc-manager:v0.4.3` +- `nvcr.io/nvidia/cloud-native/k8s-driver-manager:v0.12.0` +- `nvcr.io/nvidia/cloud-native/k8s-mig-manager:v0.15.0` +- `nvcr.io/nvidia/cloud-native/nvidia-fs:2.29.4` +- `nvcr.io/nvidia/cloud-native/nvidia-sandbox-device-plugin:v0.0.5` +- `nvcr.io/nvidia/cloud-native/vgpu-device-manager:v0.5.0` +- `nvcr.io/nvidia/driver:595.91.07` +- `nvcr.io/nvidia/gpu-operator:v26.7.0` +- `nvcr.io/nvidia/k8s-device-plugin:v0.20.0` +- `nvcr.io/nvidia/k8s/container-toolkit:v1.20.0` +- `nvcr.io/nvidia/k8s/dcgm-exporter:4.6.0-4.8.3-distroless` +- `nvcr.io/nvidia/kubevirt-gpu-device-plugin:v1.6.0` ### gpu-operator-ocp @@ -281,11 +281,11 @@ _No images extracted._ ### nvidia-dra-driver-gpu -- `registry.k8s.io/dra-driver-nvidia/dra-driver-nvidia-gpu:v0.4.1` +- `registry.k8s.io/dra-driver-nvidia/dra-driver-nvidia-gpu:v0.5.0` ### nvidia-dra-driver-gpu-ocp -- `registry.k8s.io/dra-driver-nvidia/dra-driver-nvidia-gpu:v0.4.1` +- `registry.k8s.io/dra-driver-nvidia/dra-driver-nvidia-gpu:v0.5.0` ### nvsentinel From 072a666a0383cbdfef22dc467310c35d1331cd3d Mon Sep 17 00:00:00 2001 From: Yuan Chen Date: Fri, 28 Aug 2026 09:38:03 -0700 Subject: [PATCH 03/12] test: regenerate catalog and stock-render golden digests for the version bump Signed-off-by: Yuan Chen --- pkg/bundler/testdata/stock_render_golden.yaml | 96 +++++++++---------- .../testdata/catalog_parity_golden.yaml | 96 +++++++++---------- 2 files changed, 96 insertions(+), 96 deletions(-) diff --git a/pkg/bundler/testdata/stock_render_golden.yaml b/pkg/bundler/testdata/stock_render_golden.yaml index d43d9f13f..5e9ddbad9 100644 --- a/pkg/bundler/testdata/stock_render_golden.yaml +++ b/pkg/bundler/testdata/stock_render_golden.yaml @@ -3,51 +3,51 @@ # # One entry per leaf overlay: a digest over its fully rendered helm-deployer # bundle tree (sorted relative paths paired with per-file content hashes). -a100-aks-ubuntu-training-kubeflow: 265940a7ef6f82bfe9752e5e488b416d5d54aa0245b29f0262796978547af68d -a100-any: 457dd24ba4f8b9d44af7f68ddc1f4e9d46d0e6eac30b0eceb2e979e1e9c2d2db -a100-eks-ubuntu-training-kubeflow: 9a77e69e7ea639be6100c4c2c0cb49bf87059348020c98f5c561158a22d452e7 -a100-gke-cos-training-kubeflow: 0fb904af67e4ad85cb1b71efed94c8cd3966fc36f14cc80d9b50d5ffe7babde5 -a100-oke-ubuntu-training-kubeflow: 92c1a420bccca84c7ca7e74d8960af6c3662bce4c708c45ccaa386f209dc1c9e -b200-any: b8e028ff78b7f142c3e157143a258510c4f44579e31f13c183796767de53d862 -b200-gke-cos-inference-dynamo: cd3c475c9bb709b8164571be18ec625b38263a762d91d16f89049e75fc934eb4 -b200-gke-cos-training-kubeflow: f35da200bec023578aaaeebe337775d9b62718cfd2ac0c568c4a56f69575fbb3 -bcm-inference: 2a7d525459b585c3b1639eb556de8feb682b27f6bb8be3a7015de7773ba7c235 -gb200-any: ff2c8128bc91fce4a48d650d160659b08272a346a044f7955d3b23109fde6e8d -gb200-eks-ubuntu-inference-dynamo: 637a4fd32933c220453d3732363c9a56bd75dd91cdd886a9b0571b666670744d -gb200-eks-ubuntu-training-kubeflow: e4d2b1cc86cf33b7741e881a984f6943353bf2e1b85d918b8d3e82cf50c8fa60 -gb200-eks-ubuntu-training-slurm: d356fec2562b9d3f8adcccb2344402f41270ca301f353affd6529aaa86e430d4 -gb200-oke-ubuntu-inference-dynamo: 0304fffcd27d04827ae561b32ced55b9ff87e0f6049bda30a075afd49f24ad11 -gb200-oke-ubuntu-training-kubeflow: 3ed8582775ca602b48eb88ff05d228db3f4189f4230f31f72cf355be8d5241b0 -gb300-any: a67ce96e16a0a22d85f22c2803f029e49b1befb4226a60cdf9820fb6b6e77ea2 -gb300-eks-ubuntu-inference-dynamo: c725636167448e64ef1d567613aded4bcd7ede0137a3d1e4b9c70be1ec20c61a -gb300-eks-ubuntu-training-kubeflow: e84019cb236bae67eef79a8166214295b066d1514ebb869619898925be48522d -h100-aks-ubuntu-inference-dynamo: 54061313326c300b83a0627f415002bd080c7c0e338e19bcce7996446a791065 -h100-aks-ubuntu-training-kubeflow: ea0180c91126bdda49e3641538cfef03961358ebe4d883e893d53a82b26d4bd7 -h100-aks-ubuntu-training-slurm: 7f9c80295d770456adca24b66bc7681e42fd9db54fcb97d2541c657c939645c9 -h100-any: 7e98352317f4d2a753255ec776e7b7bc605ad288f3856b4a467a15f8844aa903 -h100-bcm-ubuntu-training: c73ab59f00b5425a0be7c8aaac6107469c8fb59e54eecce9cf4937704821611d -h100-eks-ubuntu-inference-dynamo: 6a3e47afa2018591b22f2fd96105d75ec23b631fafb4c0f30f42ba77af6f537a -h100-eks-ubuntu-inference-nim: 3b9b4fe53c967fa95e56e2cf6e631a55f044d50f76423e11ac176d839e005d2d -h100-eks-ubuntu-training-kubeflow: 18be43e357c98753a069d5db99f2dbe79f6bbd7c98c91286715968f183639cb0 -h100-eks-ubuntu-training-slurm: fdf26391dd6d2e10b9e853f3bd480c8968f2749948cb58a4a1cf5a1b34b85325 -h100-gke-cos-inference-dynamo: a8527e38ea90f346eee045c6b1b448263c83258c3d99864f73b4e9f86d28d520 -h100-gke-cos-training-kubeflow: 304639b08cf1899fc5cb8bb7a5c1a4f9ff652896ae18a92123c802b2f4cfc341 -h100-gke-cos-training-slurm: 0efee10e13589930111b9f5064430d2dcac463d2d2cf6032a6a7687be2085958 -h100-kind-inference-dynamo: 8cea3d208001c5d7bb542d83a75e2ab641722981b279db9391a300b8f9d1f20c -h100-kind-training-kubeflow: 1203d2e475a51a3e07d3a62a87ad2dca91c24eda3aea800aca86bf2dfee5a80a -h100-kind-training-slurm: 8a27b1ae4e91cda80c4a32d456552952f2ec0a28af10e9e0317219b551ddd133 -h200-any: ceee05a6ed9dd0f218919a90187de851e16e93cf89293c9746d5f0e46bbda222 -h200-eks-inference: 2900b7d96bc48a27eb5fedda75b99b010b9165bc6058372d2de3b991f64c4982 -h200-eks-training: 4b3074954d10a81d057ac746c7bddf3664637dfd002cd0955348ec3dee11eb20 -l40s-any: dabbb23da635f2b6db7803be55841a7305f884ce7798dbf79d952686004f09d8 -l40s-oke-inference: 8a8884dcf64b9feaa708b152cc4eaf1b9c09d790de295cf2b0e5a5ca464154f1 -l40s-oke-training: f9b741b390f9f29451c7ad16da56602bbf6497bec067be6b19242cffdf1e1de7 -monitoring-hpa: 32917a470d982044c3d49960dbd5e62ab56cd870375bfac23c35d7a4b3f4298a -ocp-inference-nim: 016fc59c13e901de5556f09f3cb3d5a90f91b9826d34f126fde1e066b311605f -ocp-training: 652f7ccdf52009adafe993da111857aa67390678f382b4c0caf0aecd0d613635 -rtx-pro-6000-any: 9115c5d61d9941130e448f29b9fbcd834f6d15a46fa4edf3239afc44fcdd36f0 -rtx-pro-6000-eks-ubuntu-inference-dynamo: 8c35c21289dc4a56f3e035da9d35427c9a1f2402c643febda2c6b397692b2a49 -rtx-pro-6000-eks-ubuntu-inference-nim: feb5f1d72ebd95459c9df0891420d20fac42c6b8c16f0e8fa551da61a19aef1d -rtx-pro-6000-eks-ubuntu-training-kubeflow: 1dfe905ac8d891fba4ce797de6532320024fe2616653e4703be5fc8e29716d77 -rtx-pro-6000-lke-ubuntu-inference: c5d53e0adbe9c500ecd5eb45d2482e632f1c414657a08ebef053a8aab90fc3cb -rtx-pro-6000-lke-ubuntu-training: 39e9cc1636a11d2568d9d50ecf3744f5773c4870772727661c40be63bac33280 +a100-aks-ubuntu-training-kubeflow: 6ebbccf8d2e7b42ffced91482b0204461cf427d214d1bd0c72047ab0a817075e +a100-any: 1574bec32707462c1b4b5f61111b9db8a17ed2d6e085c03d197bdc252b53f66b +a100-eks-ubuntu-training-kubeflow: c7188e09cac23357027d5f88e1b6400a81ca0730595049b6d2c18b79389b5ea8 +a100-gke-cos-training-kubeflow: f82dfc6cff16f94e4b6d9a149cafd492b3de2638805ec8710016a513b3239f01 +a100-oke-ubuntu-training-kubeflow: d41b6533c1a0c7c146ef1d461234898bc643ca1cb92e93fb64ab7325ff1b080b +b200-any: c45ab3caedd55af9e76d4b24db0cbc8393dc8f6be8fa8ac915453a28c2281833 +b200-gke-cos-inference-dynamo: d6f9103bce0828001485ea93e5e045318a499e81f16c6e3ab4a26d298e20b50e +b200-gke-cos-training-kubeflow: c31aded6fe3354cba4702de9855d9edbef6eee30da284b97e342e57101ef70a4 +bcm-inference: bc7e8b71e3d73de8d849c64c82e2177c2e8d8a5771a0bc5df66e14fa84bc34a6 +gb200-any: 2223bb69c04629a557137818972dfa8b9d952e6477d2cd1b5e9a3a3a7ece65ea +gb200-eks-ubuntu-inference-dynamo: 307d88b7d991d1eeaec847367eeed334b9871e5c250ffd91c30659ba9f5d3d06 +gb200-eks-ubuntu-training-kubeflow: 8782d40e1b816c9d568baa0ec9e96ffb37b8b69aa99b10dc36b9ba0f6b676e7c +gb200-eks-ubuntu-training-slurm: 5937ef8e29869dc1e5c306777da79471d181d6136898d9371cebec994c898073 +gb200-oke-ubuntu-inference-dynamo: 3b0326748d71af2c0e92835894026c00e7cdf505bb075f71ad2ff8663c0b168a +gb200-oke-ubuntu-training-kubeflow: 7a00164f66b9e19a3a6b9bdafbb4c87d6c039b8b40bef810da34bab7fbb1ad45 +gb300-any: 0c0b8b78181e33588c154219ff44ecc4914518905917bd21f986fa5016732201 +gb300-eks-ubuntu-inference-dynamo: d0afefee4a8e1cc7e1a7fb22fda6e5f59f7b6cc57139a51dd5189a3288c4fcfe +gb300-eks-ubuntu-training-kubeflow: 0332c753b04f8cf32ca4e237614e925cf4f922f11cf9f6e8a64834869256869e +h100-aks-ubuntu-inference-dynamo: f6dbcdd49599a1c5c8158ac6666bef521e3cd63309c9487de0e5f93d338926d9 +h100-aks-ubuntu-training-kubeflow: d4a64648e8f02dbd9614a97f9ca217ced9c8a48598246f86fec868f770d44749 +h100-aks-ubuntu-training-slurm: 00928d1150c128288712beab835e4dcfa7200bf65926ee05f27cde48a3459690 +h100-any: 5d6032050aeac10384c0808b3b32d811f7ec87fd5e4d816d03c391e13c94a93f +h100-bcm-ubuntu-training: 8085e79850c12c513e5d857b3d9ddb3d49eaab28f9237a1ee596198d96b34710 +h100-eks-ubuntu-inference-dynamo: 1cf513f5383e847def40f5f382e00bc48e35e0f33963abc59514cf50814f2a71 +h100-eks-ubuntu-inference-nim: c3525c9ddf07c3d51a6de71f998fe8aaaae66c5f7783b41acc57843b8b9ca11c +h100-eks-ubuntu-training-kubeflow: 9a2e562cc16ed98561230812e5dda5257a5a8cab0665579c6e6918ffcc26b7b1 +h100-eks-ubuntu-training-slurm: 0a1fc59b1144ec4f7e16b544ae9ebcf189f78447bb8dae42b073141f332d90d8 +h100-gke-cos-inference-dynamo: f001d10c88bd0e2c609c74fc1ae60a9752bbb67b8195ead55245d75f3aa89534 +h100-gke-cos-training-kubeflow: 2cfc19faee0144f43895e504380b8491ace0eb1121c6005fb9cbf19a81310d97 +h100-gke-cos-training-slurm: 001ca331ba2dd7f512959923e530f15ebf0b1f03035c6cc3fdafabb48fc0663a +h100-kind-inference-dynamo: 959f28abbdc9d05c56476c993c23fa58404edc634c30f3c60f1bf70173791127 +h100-kind-training-kubeflow: 77e8dd0ad1b78af14be19765d0c855415aa93bb3a7b3e1dd8135d464b28798b3 +h100-kind-training-slurm: eef18f418b69b879964e69cbd3f7a3c3a28c2451a2fd03c3cf60f811bbb9bc19 +h200-any: a8c74e455e6fb774df5724cdefc6d5e0823e7d64a48504ecc97c4099a1292795 +h200-eks-inference: f2a2c69a5098c74859d530518504af6dbcd80db25331a625e7e4ddfada396984 +h200-eks-training: dfd4e12aa5c5d5a69e28f1fd38b4a13bc48fc8a0d1046fd2cafa6113c4d7a397 +l40s-any: 03d4042acd620aa3a697df010c5e2e5b42388c820be7cafeaf6c6636bb443fe9 +l40s-oke-inference: cd3986c301fd7355a883ba41cbee9dd02f782bfd79fd334f40cf1fc7c725366a +l40s-oke-training: 79094f30620f87280b7f271080642cad6915b3b35ea09880919b2c269541025b +monitoring-hpa: 3fd6a80124d0e346ff3c42f1692d36c74626414a7dad33da56f6237233a3cb63 +ocp-inference-nim: a728bed625f568760bf291d54e066a443e2c141b239dc5fb7f4bc96dcac87c39 +ocp-training: 32a047795bcb4c9ea6363e18de6d4f213e6b8c9003106cb4a54ccffd3a6b0ef1 +rtx-pro-6000-any: 4930e5dfc65a125e9ffebfe9a997adef7a123432fd05d584dc4984a2e72ab893 +rtx-pro-6000-eks-ubuntu-inference-dynamo: 2be567e056aceea081f5e8e676724617e6986febb64e5f21992362bc80d14bc8 +rtx-pro-6000-eks-ubuntu-inference-nim: 1672ed446085dc84d395288edfbf1be31789151e9ab5800379d7fa2cdbb6fcb6 +rtx-pro-6000-eks-ubuntu-training-kubeflow: 976ad5d9d70517825faa9491987f1f6a2059337ed69cc2a66a05809aee4f23c0 +rtx-pro-6000-lke-ubuntu-inference: d98dba61633284f0bbba26df5f3525839984d26c424353d57133a4e9441c29aa +rtx-pro-6000-lke-ubuntu-training: 23f03d02c253c4be476e760dab8e8ba1d808d443af13913b558e8a7681041fa0 diff --git a/pkg/recipe/testdata/catalog_parity_golden.yaml b/pkg/recipe/testdata/catalog_parity_golden.yaml index 95fe07389..06cffd0e0 100644 --- a/pkg/recipe/testdata/catalog_parity_golden.yaml +++ b/pkg/recipe/testdata/catalog_parity_golden.yaml @@ -3,51 +3,51 @@ # # One entry per leaf overlay: sha256 of its deterministically-marshalled # resolved recipe. A moved digest means that recipe's resolved bytes changed. -a100-aks-ubuntu-training-kubeflow: 3d8d77ac7ac29bf13253410e0b0486bc322d69d66d8778884df3375619566d58 -a100-any: 62d1581b21fecd3465b69659bac3dc9e5438eb2612c98c1acdd7b19cc9237084 -a100-eks-ubuntu-training-kubeflow: 0196adfa0c77f230d95ed3ebee2bdade8ac720f631cd3bbe2b8cec7d55094a10 -a100-gke-cos-training-kubeflow: 514f996360b251a6ef657ba1a715b5705be03d6b6dccb53887ee96554a83ffb6 -a100-oke-ubuntu-training-kubeflow: 310a6ee2abdd7f8508882429d657c01bd23458ba635236c410934825e4f617dc -b200-any: 63dbc4fe27b84321e8b1991cd98c13f85395a3919d13e0c6891599eaa1f1e729 -b200-gke-cos-inference-dynamo: 8174db8270ea447d9b307f21be9945b919ed3bcdacc702d2961d2ff75fb60c35 -b200-gke-cos-training-kubeflow: 96723738cb0baaaec300c95d5cfbc08a18d999fd0adb52aef3c88436561da1e7 -bcm-inference: 92fa64d3af61026891b37cbf692d2663be7b2484078ab65e2653d18fc2bfe6f8 -gb200-any: 9ee6ab187b4c1b93cd859d86ada35e922d0873ffc6fe892cb084ad98baf09727 -gb200-eks-ubuntu-inference-dynamo: 8babae6413d104da16060f18d919c80fd97ff983bd32f9876bd5f72da7fd6be6 -gb200-eks-ubuntu-training-kubeflow: 63b2fbd3a195bac682bb45263e5d5d0ec479a6571dd38ef0505eeb4f287b29e9 -gb200-eks-ubuntu-training-slurm: c35e33618c0a8bb548bbe3c5d6765faf90cebe1427478ef97089f7a30bb37a6c -gb200-oke-ubuntu-inference-dynamo: 6f01848824dcbefb0f58ec56ee9e911fff1470c9f67c0d0a9ba0de1dd1e648d2 -gb200-oke-ubuntu-training-kubeflow: 3959f47474edfac9b53a65939117d4364551fde0c9324246f195f144fe3beb12 -gb300-any: 7f68607dffbbfc912f19b00ffa833caa81ca675d25bc22cf612dbd0b76f39475 -gb300-eks-ubuntu-inference-dynamo: b14a3d99503949b8bfcdc81455da4af389a339deb24ea784637a7901297d70ad -gb300-eks-ubuntu-training-kubeflow: 5303e5ebf8dc2ec13c2cff60c7caba2ef53d1a65bfd69d06f4f78cbb93835c49 -h100-aks-ubuntu-inference-dynamo: 9fe4d82412d250d4004efc56fa4ddd6b8b1a9166387455e96ac2c2284d323107 -h100-aks-ubuntu-training-kubeflow: 72a8ca777c333e0db82717f53a706ae9155163ff0c73fd4e5ed8d71badda5811 -h100-aks-ubuntu-training-slurm: ef48d77171817bb5b24815d128457c246e0e5dc0fe850aa6881276952e590055 -h100-any: f5a55f03948075adde9b1b24fd650d98fa4d35b85f394d071ad3365ba05588a5 -h100-bcm-ubuntu-training: 76746f45add1fb6b0d7b6441b4d9581acceb3b5d3d4800b943c1cf460c92fbc5 -h100-eks-ubuntu-inference-dynamo: 0a31f03010849f76b385ce81c21e499af884f41bee89de47b9bacf03f8b6e311 -h100-eks-ubuntu-inference-nim: 9c6a4839bf5b59f620093427de0e138a9fafaf1f55f3fa7f555b1ee33b914a3a -h100-eks-ubuntu-training-kubeflow: bfcbd5e14fe43209f2401e429d39b5f7a52d049832db07e5b9595d130ab6e2fc -h100-eks-ubuntu-training-slurm: 8d29c2be5c3c0dc8396b0df53111a485f6d57771394df8ef135e4b052d681064 -h100-gke-cos-inference-dynamo: 45190af6c5b5d76f69a4f36d9c0cb800c450bdef54ca5b522ad49f605d1bad2a -h100-gke-cos-training-kubeflow: 43de2291cc0adb9b8d89494179d86eeef35994b2ad4fd4062a87265d6753ec6b -h100-gke-cos-training-slurm: 414f95e1a231b1c86bcbe70729b820a95533367dec0c9e3f7e36514431fad1d9 -h100-kind-inference-dynamo: 0e2d552b62b55b91f960c9721ed128b1d07f68be8517799473fbaa3baa1cc6e5 -h100-kind-training-kubeflow: 4bb03659faaa6a9a66d58dfc95572bdb54b75010c219c4bbefbd0eafeb8ed867 -h100-kind-training-slurm: 0245c786638ade1023ba4dc49e0f2efd56d2f789bdf447e946bbc3a3e652e734 -h200-any: acf1986d76037dddd025125eaa34eb7ecf3d31fe62cb80e7dc89dd042ebb42f5 -h200-eks-inference: d1ff148001722e9d5e23606bd7770f8120506f08c10f0c132e0c379f535609c5 -h200-eks-training: 98d04b3c81a059a0321451a39772af8d0d3310ad1fb861837f87d82dd997c4a1 -l40s-any: 89210ba5815f93ce3d7b1a2a78f7ba75d083556e940603a1bc812eac9aeccd0a -l40s-oke-inference: 6c75ef93e86fb21dbdf73ac97089b894ca8da832164dc807e94592493ab7f7cc -l40s-oke-training: 2a35011bd59cf3c727cea158d504c51bd59cba069620d3d6a4988d68972dee74 -monitoring-hpa: 15e93304e7e68997e7b15a0be4af360a29f10c869502e6dccb6a6cefb6519fd5 -ocp-inference-nim: 49245ecfc91f67fcc2ed0b305b00046a4164628cafc75ab98542e6dafa626821 -ocp-training: aaa8cd7f45ef42235b075936461d715e2e017ea2802df9e8017de38304a17915 -rtx-pro-6000-any: 9639d67639256a44be8566f3f8eda49ceb4025b6b140a3bc2a4b6d4768d07ffa -rtx-pro-6000-eks-ubuntu-inference-dynamo: cbe11051d979ae20e83471142c4582e1504fd24f55898a00afcededbe143bba6 -rtx-pro-6000-eks-ubuntu-inference-nim: 08b73d58d0d60681be55ee1402bdec5bf1367a0cb137a273ca27bec0788cc020 -rtx-pro-6000-eks-ubuntu-training-kubeflow: c2595e2994d56a735714103eedd30c6afd45921f458bf026c3432017b5eba60e -rtx-pro-6000-lke-ubuntu-inference: 28cdab0531f8ed6018391aba7cab1fbfad8ba551392de1b9138f3b88042f5340 -rtx-pro-6000-lke-ubuntu-training: 312c4cf49d0e8cec0377cd9858338d7106226a2252d81828a75c1d426a194a2f +a100-aks-ubuntu-training-kubeflow: 52a19136ff27701be3ebd3291d0838b405dd4200b9ea2ad1f15f5e9231afc164 +a100-any: eedbde8040b6909da0e42d2af6345881ca5fcaacaa3ac83b222ba998c5c07c5e +a100-eks-ubuntu-training-kubeflow: 9918500dfc098857fa643a8427d378ad205fc7914566095c8678c2b837b93e7a +a100-gke-cos-training-kubeflow: e2dedc04e012cf16d146c953bc1f6a58237afd305e1978b58976d67d5695cb37 +a100-oke-ubuntu-training-kubeflow: c2f71c8a4bb59e2cca85fc007740eede1ded4a6514ec46071d7172f707c0fa13 +b200-any: 2d0d453bcdcd89d5db6b0aab435aad942458b108a7b0060a48e73c88a96e11ca +b200-gke-cos-inference-dynamo: 37147bb9daf7782bd58349c97ef01fa72b70fb46f1e01746d8725869a83ecfc6 +b200-gke-cos-training-kubeflow: e219956eaff20825678d485dd656e6071a6a91afc180af9abe54122d93191164 +bcm-inference: c608118b2f1c74e2d1d2d9defa90918f9da2cf6122afbf133d605f8a0154ea87 +gb200-any: 548063fe33335835470c1bbf9fb9ace134061d06a8910409e68d90423377a41d +gb200-eks-ubuntu-inference-dynamo: b15f4dd4cf5224ca76f7524f15611ed22910c4ee90334fe160594cb1592e9ecc +gb200-eks-ubuntu-training-kubeflow: fc6b234dd59ec815ba4bc67c695ff04057dbf3a233ba68662c54b96450ba035b +gb200-eks-ubuntu-training-slurm: 4b2007f13eb5c953620a3e18271e3f5b8d8a407258a36c1c47882012490d0470 +gb200-oke-ubuntu-inference-dynamo: 4ce2e85e13d125948e91614f0d07ebb70968ea0d93beec9feba03863f420f282 +gb200-oke-ubuntu-training-kubeflow: 3f0394a1b85b3d16147a31ecf55088d899c3b103ead7cb2f8a25183cb3a1575d +gb300-any: 10eb8b5cc815ca6a190412e8e344d261c0153cc65e2bf8266d89f6d821171da3 +gb300-eks-ubuntu-inference-dynamo: 9a4d41b7e6c57ec6cb44df5f6af64ca3f4d15a4568d6dfaaa16dfc0e5b791877 +gb300-eks-ubuntu-training-kubeflow: 460cb834803767f98f70f6bad7e006d22ebd79f83aa4c62f3feace0f896bd579 +h100-aks-ubuntu-inference-dynamo: e16910a1555ed867d5c85b63d6bea4861e585f1c4cbffeb81b9315fe04feb551 +h100-aks-ubuntu-training-kubeflow: 0ef4f87a06b668338e8416eb7b85e5ac63f472ebbd478c69db1a6e36d9cdac6a +h100-aks-ubuntu-training-slurm: f43f166e8d0af0447fa581e227fff3088d8a0d5efb9bac8cda4aff2f6cf84d04 +h100-any: 879ed75bd46ef020cc9f31eb03bceb49debe3719c2dcf567d072ac5b13ba3d2e +h100-bcm-ubuntu-training: 2d25d1fa54d969088215f4f95d2eabd2aead572a4223d2421f6d9ffaaf15be61 +h100-eks-ubuntu-inference-dynamo: 0b20a1afb8336d0fb7abe60dd5308cb2b76b96df2a4932fe18196468a08b1012 +h100-eks-ubuntu-inference-nim: 00460f237a30b03854875ce52a9e7a58217878192cfeb3f52745b26d1ee66c18 +h100-eks-ubuntu-training-kubeflow: 0940f0e8420302f91cd11f3d0451406de9259408245ef9410423ad09bb369766 +h100-eks-ubuntu-training-slurm: 939f6422a3b79e68b5a43225a838ecba9d2878d57cd9b4a98207d6d2860b1193 +h100-gke-cos-inference-dynamo: 904aeab7e68bce844fc9a72a9a79f27d28f77d044d510fdcbaecce596cefe983 +h100-gke-cos-training-kubeflow: 2ec3f85e22a551c48bca2a67af7211965c405f7a6fe7e4fa8be3a161311943a9 +h100-gke-cos-training-slurm: b74e224e7362f803da02ed60e090954d3c70d964cdc6db62df4bac4b5a060f70 +h100-kind-inference-dynamo: b17a7431bf9b5cd97211d343274324bdf7871b44b504f77bd575f54092c993db +h100-kind-training-kubeflow: 49f22bb0ba0b62c8c728cb801e99af9d4515ecbff4151100a0788bd3f4c1dcf2 +h100-kind-training-slurm: 891a37a0ff30b83bae3252eaae8305fb1fc4f29ddd3cafcb751f561308b738a7 +h200-any: 959608a2eb8060c6df2ef7caf453a1500f1adc8790b8c228851c603d15b6e908 +h200-eks-inference: dd2814a49106bfae9623bd310587ae4bcf378fba20039726796272fe23a571cc +h200-eks-training: cd7439667773a77b523a7337e672f4b8a9828bf88af618436094dc556746acb1 +l40s-any: 83b2b92d8b43e8f23754ff9608f02a6d64d7de28f8ccb3b7d6ae67679cffac93 +l40s-oke-inference: 9761d9172ba2c7561e0406392e2c0828450f6e3d5537c61bdaf5732d80fca487 +l40s-oke-training: 4b40b25e4a5620d3cea304df67209b397e3813190ee2ff4c7dd5798690ecd10d +monitoring-hpa: 7156deeffed71f32316b4a2501c771033d67dc2de3d3181311e985cc3625e557 +ocp-inference-nim: 08f9f5d03f030160eb6951bd238c24a400c41d62dcf721589abb3d265df8e3fc +ocp-training: 3998c76724c4189cc463e5cf473a3cd97d8c0eaf3299336be7f1ffac9cb11e6f +rtx-pro-6000-any: 62760c1f1e8287ae8c37b20d306546469bed84a742ea2b143aad52f942575f7e +rtx-pro-6000-eks-ubuntu-inference-dynamo: 2e14248538447738e7ea1699b0c0730dfd1e18b8aa8588ab0540903874e3bc2e +rtx-pro-6000-eks-ubuntu-inference-nim: b1c918b05e42118edcb51a164a9e251c944243d3e8d0127855ec361f8740ee52 +rtx-pro-6000-eks-ubuntu-training-kubeflow: cb833e1bcc917f105a106f4e36186ee55c665c641c22bd0a5da4206284b04daf +rtx-pro-6000-lke-ubuntu-inference: 1de10952036333b9f1d475b5ae212d54e4ed226ff3a44c17b13415bb708e5368 +rtx-pro-6000-lke-ubuntu-training: 296c4559de7977f238303e787407e57489a0dae8b24249f9f416a0425d4d48c7 From ad2a3e7b9ad7ea7be0c2d4f7be85a59568c4fa79 Mon Sep 17 00:00:00 2001 From: Yuan Chen Date: Fri, 28 Aug 2026 09:54:15 -0700 Subject: [PATCH 04/12] docs(recipes): record the GB200+EFA floor history on the driver pin Signed-off-by: Yuan Chen --- recipes/components/gpu-operator/values.yaml | 17 +++++++++++------ 1 file changed, 11 insertions(+), 6 deletions(-) diff --git a/recipes/components/gpu-operator/values.yaml b/recipes/components/gpu-operator/values.yaml index de4866fd3..a5de768be 100644 --- a/recipes/components/gpu-operator/values.yaml +++ b/recipes/components/gpu-operator/values.yaml @@ -147,12 +147,17 @@ gfd: enabled: true driver: - # NVIDIA's recommended driver for the v26.7.0 chart; a single global - # pin covers H100/B200/GB200 EKS. Pinned rather than inherited from the - # chart default so a future chart bump is an explicit driver decision - # instead of a silent fleet-wide driver change, and so BOM/attestation - # inputs stay deterministic. Moves with the chart: holding an older - # driver against a newer chart is a combination upstream does not test. + # Single global pin covering H100/B200/GB200 EKS. Pinned rather than + # inherited from the chart default so a future chart bump is an explicit + # driver decision instead of a silent fleet-wide driver change, and so + # BOM/attestation inputs stay deterministic. + # + # History: the previous pin (580.173.02) was deliberately AHEAD of the + # v26.3.3 chart default (580.126.20) to clear a GB200+EFA floor. This + # value is the v26.7.0 chart default and is far above that floor, so the + # pin and the chart recommendation coincide again — but they are not the + # same thing. If a future chart default regresses below a GB200+EFA + # requirement, hold the pin ahead rather than following the chart. version: 595.91.07 enabled: true useOpenKernelModules: true From 454a6cb894a07a64355bd39dfa62f4e277d99b96 Mon Sep 17 00:00:00 2001 From: Yuan Chen Date: Fri, 28 Aug 2026 10:16:09 -0700 Subject: [PATCH 05/12] fix(recipes): pin ComputeDomain numNodes and refresh stale version references gpu-operator v26.7.0 ships a stale copy of the ComputeDomain CRD in crds/ that marks numNodes required with no default. Helm installs crds/ only when absent and gpu-operator is ordered before nvidia-dra-driver-gpu, so on a fresh cluster the operator's copy wins and the slinky-slurm ComputeDomain CR is rejected with "spec.numNodes: Required value". Pin numNodes: 0 explicitly, which is valid under both CRD copies and is the correct value under IMEXDaemonsWithDNSNames. Also refresh version references the bump made stale: the GPU CI chart pin, a chainsaw comment mirroring the driver-validation rationale, the ccManager and toolkit-hardening comments, and a hand-written sample in the BOM page. Signed-off-by: Yuan Chen --- .settings.yaml | 2 +- docs/user/container-images.md | 2 +- pkg/bundler/testdata/stock_render_golden.yaml | 10 +++++----- .../manifests/nvidia-toolkit-hardening-aks.yaml | 2 +- recipes/components/gpu-operator/values.yaml | 5 +++-- .../slinky-slurm/manifests/compute-domain.yaml | 15 +++++++++++++++ .../cuj1-training/assert-bundle-scheduling.yaml | 2 +- 7 files changed, 27 insertions(+), 11 deletions(-) diff --git a/.settings.yaml b/.settings.yaml index b08ee364d..f97c59f9b 100644 --- a/.settings.yaml +++ b/.settings.yaml @@ -202,7 +202,7 @@ testing: # GPU CI runtime pins consumed through .github/actions/load-versions. # renovate: datasource=helm depName=gpu-operator depType=testing - gpu_operator_chart_version: 'v26.3.3' + gpu_operator_chart_version: 'v26.7.0' # Base image for the CI smoke-test snapshot-agent image. The agent binary is # static Go and detects GPUs driver-free via NFD/PCI, so it no longer needs a # CUDA base / nvidia-smi. Matches the NVIDIA static distroless base used for diff --git a/docs/user/container-images.md b/docs/user/container-images.md index 858e9c90f..351730f98 100644 --- a/docs/user/container-images.md +++ b/docs/user/container-images.md @@ -450,7 +450,7 @@ Y = artifact attached, - = artifact absent, ? = could not probe. Image Sig SBOM Prov Notes -------------------------------------------------------------- --- ---- ---- ----- - registry.k8s.io/dra-driver-nvidia/dra-driver-nvidia-gpu:v0.4.1 Y - - + registry.k8s.io/dra-driver-nvidia/dra-driver-nvidia-gpu:v0.5.0 Y - - Summary: 1/1 signed · 0/1 SBOM · 0/1 provenance ``` diff --git a/pkg/bundler/testdata/stock_render_golden.yaml b/pkg/bundler/testdata/stock_render_golden.yaml index 5e9ddbad9..bc31acfbf 100644 --- a/pkg/bundler/testdata/stock_render_golden.yaml +++ b/pkg/bundler/testdata/stock_render_golden.yaml @@ -3,7 +3,7 @@ # # One entry per leaf overlay: a digest over its fully rendered helm-deployer # bundle tree (sorted relative paths paired with per-file content hashes). -a100-aks-ubuntu-training-kubeflow: 6ebbccf8d2e7b42ffced91482b0204461cf427d214d1bd0c72047ab0a817075e +a100-aks-ubuntu-training-kubeflow: 03953f4ca87794693143872440318e5c03a71eeb2b26633074c15a07bc7ea2a0 a100-any: 1574bec32707462c1b4b5f61111b9db8a17ed2d6e085c03d197bdc252b53f66b a100-eks-ubuntu-training-kubeflow: c7188e09cac23357027d5f88e1b6400a81ca0730595049b6d2c18b79389b5ea8 a100-gke-cos-training-kubeflow: f82dfc6cff16f94e4b6d9a149cafd492b3de2638805ec8710016a513b3239f01 @@ -15,15 +15,15 @@ bcm-inference: bc7e8b71e3d73de8d849c64c82e2177c2e8d8a5771a0bc5df66e14fa84bc34a6 gb200-any: 2223bb69c04629a557137818972dfa8b9d952e6477d2cd1b5e9a3a3a7ece65ea gb200-eks-ubuntu-inference-dynamo: 307d88b7d991d1eeaec847367eeed334b9871e5c250ffd91c30659ba9f5d3d06 gb200-eks-ubuntu-training-kubeflow: 8782d40e1b816c9d568baa0ec9e96ffb37b8b69aa99b10dc36b9ba0f6b676e7c -gb200-eks-ubuntu-training-slurm: 5937ef8e29869dc1e5c306777da79471d181d6136898d9371cebec994c898073 +gb200-eks-ubuntu-training-slurm: 2701547d3ea6619bb99cc43c0109b88d23ccb763336b2d25f2fe0877aa3ffb6c gb200-oke-ubuntu-inference-dynamo: 3b0326748d71af2c0e92835894026c00e7cdf505bb075f71ad2ff8663c0b168a gb200-oke-ubuntu-training-kubeflow: 7a00164f66b9e19a3a6b9bdafbb4c87d6c039b8b40bef810da34bab7fbb1ad45 gb300-any: 0c0b8b78181e33588c154219ff44ecc4914518905917bd21f986fa5016732201 gb300-eks-ubuntu-inference-dynamo: d0afefee4a8e1cc7e1a7fb22fda6e5f59f7b6cc57139a51dd5189a3288c4fcfe gb300-eks-ubuntu-training-kubeflow: 0332c753b04f8cf32ca4e237614e925cf4f922f11cf9f6e8a64834869256869e -h100-aks-ubuntu-inference-dynamo: f6dbcdd49599a1c5c8158ac6666bef521e3cd63309c9487de0e5f93d338926d9 -h100-aks-ubuntu-training-kubeflow: d4a64648e8f02dbd9614a97f9ca217ced9c8a48598246f86fec868f770d44749 -h100-aks-ubuntu-training-slurm: 00928d1150c128288712beab835e4dcfa7200bf65926ee05f27cde48a3459690 +h100-aks-ubuntu-inference-dynamo: fd7227664378fa578b36d12bac10e082a4d36e2ba016ed275bdc93ce14336b7a +h100-aks-ubuntu-training-kubeflow: 7b36f0149ca75ca0911705b63b37f126d5c3dd5c251e3532615fcd1213052af7 +h100-aks-ubuntu-training-slurm: 2aa10941a7d9077a4a29bbd088e564f36bc830a2a5c8d7ae04ea2a517a6d08cd h100-any: 5d6032050aeac10384c0808b3b32d811f7ec87fd5e4d816d03c391e13c94a93f h100-bcm-ubuntu-training: 8085e79850c12c513e5d857b3d9ddb3d49eaab28f9237a1ee596198d96b34710 h100-eks-ubuntu-inference-dynamo: 1cf513f5383e847def40f5f382e00bc48e35e0f33963abc59514cf50814f2a71 diff --git a/recipes/components/gpu-operator/manifests/nvidia-toolkit-hardening-aks.yaml b/recipes/components/gpu-operator/manifests/nvidia-toolkit-hardening-aks.yaml index f8a440c85..885dbc00a 100644 --- a/recipes/components/gpu-operator/manifests/nvidia-toolkit-hardening-aks.yaml +++ b/recipes/components/gpu-operator/manifests/nvidia-toolkit-hardening-aks.yaml @@ -55,7 +55,7 @@ # device-request path open — the same posture as GPU-Operator-managed mode, # not a regression — so full multi-tenant isolation additionally requires # an admission policy restricting a /dev/null-backed mount whose destination -# is under /var/run/nvidia-container-devices (the pinned toolkit v1.19.1 +# is under /var/run/nvidia-container-devices (the pinned toolkit v1.20.0 # accepts the volume-mount device request only when the mount SOURCE is # /dev/null). secure-accelerator-access does not # exercise the volume-mounts path, so a green result is not proof of diff --git a/recipes/components/gpu-operator/values.yaml b/recipes/components/gpu-operator/values.yaml index a5de768be..d7cfd4c35 100644 --- a/recipes/components/gpu-operator/values.yaml +++ b/recipes/components/gpu-operator/values.yaml @@ -198,8 +198,9 @@ validator: nfd: enabled: false -# Confidential Compute Manager defaults to enabled in chart v26.3.x; keep -# it off until AICR has explicit CC-capable hardware support. +# Confidential Compute Manager defaults to enabled in the upstream chart +# (verified still true in v26.7.0); keep it off until AICR has explicit +# CC-capable hardware support. ccManager: enabled: false diff --git a/recipes/components/slinky-slurm/manifests/compute-domain.yaml b/recipes/components/slinky-slurm/manifests/compute-domain.yaml index 3bf024787..3af09f839 100644 --- a/recipes/components/slinky-slurm/manifests/compute-domain.yaml +++ b/recipes/components/slinky-slurm/manifests/compute-domain.yaml @@ -29,6 +29,21 @@ metadata: name: slinky-slurm-imex namespace: {{ .Release.Namespace }} spec: + # Pinned explicitly rather than relying on the CRD default. GPU Operator + # v26.7.0 ships its own copy of resource.nvidia.com_computedomains.yaml in + # crds/, and that copy is a stale snapshot: it lists numNodes as REQUIRED + # and carries no `default: 0`, where the DRA driver chart's copy makes it + # optional with a default. Helm installs crds/ only when the CRD is absent + # and never upgrades it, and gpu-operator is ordered before + # nvidia-dra-driver-gpu, so on a fresh cluster the operator's stricter copy + # wins and a CR omitting numNodes is rejected with "spec.numNodes: Required + # value". Setting it here is valid under both copies. + # + # 0 is the correct value, not merely a placeholder: with + # IMEXDaemonsWithDNSNames=true (the DRA driver default) each IMEX daemon + # starts immediately instead of waiting for a quorum. Matches + # buildComputeDomain() in validators/performance/nccl_all_reduce_bw_constraint.go. + numNodes: 0 channel: allocationMode: All resourceClaimTemplate: diff --git a/tests/chainsaw/cli/cuj1-training/assert-bundle-scheduling.yaml b/tests/chainsaw/cli/cuj1-training/assert-bundle-scheduling.yaml index 5b59ffdac..858d25528 100644 --- a/tests/chainsaw/cli/cuj1-training/assert-bundle-scheduling.yaml +++ b/tests/chainsaw/cli/cuj1-training/assert-bundle-scheduling.yaml @@ -39,7 +39,7 @@ daemonsets: # ── Driver: nvidia_peermem off on EKS (AWS EFA path uses aws-ofi-nccl) # nvidia_peermem only loads against Mellanox MOFED symbols; on AWS EFA -# (p4d/p5/p5e) it fails to load and v26.3.3's strict driver-validation +# (p4d/p5/p5e) it fails to load and the chart's strict driver-validation # init container blocks the rest of the GPU stack. NCCL multi-node on # EFA uses libfabric via aws-ofi-nccl, not nvidia_peermem. driver: From 881a108235bd3f1ee7d83c9dbde460513c39076d Mon Sep 17 00:00:00 2001 From: Yuan Chen Date: Fri, 28 Aug 2026 10:20:38 -0700 Subject: [PATCH 06/12] docs: align docs, demos and examples with the bumped component versions Verified each version-specific behavioral claim in Go comments against v26.7.0 / DRA 0.5.0 before updating the cited version: transformForDriverInstallDir still early-returns on "", clusterpolicy.yaml still accesses .Values.hostPaths.rootFS unconditionally, the ClusterPolicy CRD still types driverInstallDir as a string, _helpers.tpl still reads .Values.driver.manager.repository, operator.runtimeClass still defaults to nvidia, driverInstallDir still defaults to /run/nvidia/driver, and the DRA chart still defaults nvidiaDriverRoot to "/". The GKE Ubuntu driver-management note is de-versioned rather than re-pinned, since it is a support-matrix claim rather than a chart fact. Docs, demos and example recipes carried the old versions in copyable samples and in one exact-match constraint (== v26.3.3), which would have failed validation against a v26.7.0 deployment. Signed-off-by: Yuan Chen --- demos/dynamic.md | 6 +++--- demos/query.md | 4 ++-- docs/integrator/automation.md | 2 +- docs/integrator/data-flow.md | 2 +- docs/integrator/recipe-development.md | 2 +- docs/user/air-gap-mirror.md | 8 ++++---- docs/user/api-reference.md | 2 +- docs/user/cli-reference.md | 6 +++--- examples/recipes/aks-training.yaml | 4 ++-- ...ks-gb200-ubuntu-training-with-validation.yaml | 8 ++++---- examples/recipes/eks-training.yaml | 2 +- pkg/bundler/validations/checks.go | 16 ++++++++-------- pkg/client/v1/gpu_driver_state.go | 2 +- pkg/recipe/driver_root_lockstep_test.go | 4 ++-- 14 files changed, 34 insertions(+), 34 deletions(-) diff --git a/demos/dynamic.md b/demos/dynamic.md index 8c5caf331..a178fb171 100644 --- a/demos/dynamic.md +++ b/demos/dynamic.md @@ -98,7 +98,7 @@ cat ./bundle/010-gpu-operator/cluster-values.yaml # # Generated by AICR # --- # driver: -# version: 580.173.02 +# version: 595.91.07 ``` And `install.sh` applies it after the baked values, so it wins: @@ -121,7 +121,7 @@ grep -c 'version: 580' ./bundle/010-gpu-operator/values.yaml # b) present in cluster-values.yaml grep version ./bundle/010-gpu-operator/cluster-values.yaml -# version: 580.173.02 +# version: 595.91.07 # c) install.sh references cluster-values.yaml grep -q cluster-values.yaml ./bundle/010-gpu-operator/install.sh && echo wired @@ -193,7 +193,7 @@ rm -rf ./b-oci oras pull "$REF" -o ./b-oci (cd ./b-oci && aicr verify .) cat ./b-oci/010-gpu-operator/cluster-values.yaml -# version: 580.173.02 +# version: 595.91.07 ``` > ttl.sh is public; anything pushed is world-readable until it expires. Fine for a throwaway demo bundle, not for anything sensitive — push to your own registry by pointing `spec.bundle.output.target` (or `--output`) at it, e.g. `oci://registry.example.com/aicr-bundle:tag`. Transport is separate: `spec.bundle.registry.plainHTTP: true` only switches HTTP instead of HTTPS (enable it only when the registry is intentionally served over plain HTTP, such as a local test registry); leave it off for any HTTPS registry, and use `spec.bundle.registry.insecureTLS: true` to skip TLS verification against a self-signed HTTPS registry. diff --git a/demos/query.md b/demos/query.md index 17cb7bddd..b5a457b8e 100644 --- a/demos/query.md +++ b/demos/query.md @@ -38,7 +38,7 @@ aicr query \ --selector components.gpu-operator.values.driver.version ``` -> `580.173.02` +> `595.91.07` Subtree — full driver block: @@ -54,7 +54,7 @@ maxParallelUpgrades: 5 rdma: enabled: false useOpenKernelModules: true -version: 580.173.02 +version: 595.91.07 ``` ## Differentiation: Same Selector, Different Criteria diff --git a/docs/integrator/automation.md b/docs/integrator/automation.md index 9c46cd411..94333e5b3 100644 --- a/docs/integrator/automation.md +++ b/docs/integrator/automation.md @@ -231,7 +231,7 @@ spec: # Helm chart from upstream - repoURL: https://helm.ngc.nvidia.com/nvidia chart: gpu-operator - targetRevision: v26.3.3 + targetRevision: v26.7.0 helm: valueFiles: # Values live under the numbered bundle dir (NNN-/) diff --git a/docs/integrator/data-flow.md b/docs/integrator/data-flow.md index 833eb0453..b0e7007fc 100644 --- a/docs/integrator/data-flow.md +++ b/docs/integrator/data-flow.md @@ -634,7 +634,7 @@ spec: sources: # Helm chart from upstream - repoURL: https://helm.ngc.nvidia.com/nvidia - targetRevision: v26.3.3 + targetRevision: v26.7.0 chart: gpu-operator helm: valueFiles: diff --git a/docs/integrator/recipe-development.md b/docs/integrator/recipe-development.md index 4985364b6..a469decc3 100644 --- a/docs/integrator/recipe-development.md +++ b/docs/integrator/recipe-development.md @@ -888,7 +888,7 @@ spec: # recipes/registry.yaml - name: gpu-operator helm: - defaultVersion: v26.3.3 # Changed from v26.3.2 + defaultVersion: v26.7.0 # Changed from v26.3.3 ``` **Adding components:** diff --git a/docs/user/air-gap-mirror.md b/docs/user/air-gap-mirror.md index e495b804d..43f2544d3 100644 --- a/docs/user/air-gap-mirror.md +++ b/docs/user/air-gap-mirror.md @@ -112,7 +112,7 @@ metadata: name: aicr-images spec: images: - - name: nvcr.io/nvidia/gpu-operator:v26.3.3 + - name: nvcr.io/nvidia/gpu-operator:v26.7.0 - name: registry.k8s.io/nfd/node-feature-discovery:v0.19.0 # ... --- @@ -124,7 +124,7 @@ spec: charts: - name: gpu-operator repoURL: oci://ghcr.io/nvidia - version: v26.3.3 + version: v26.7.0 # ... ``` @@ -187,13 +187,13 @@ components: - name: aicr-images required: true images: - - nvcr.io/nvidia/gpu-operator:v26.3.3 + - nvcr.io/nvidia/gpu-operator:v26.7.0 - registry.k8s.io/nfd/node-feature-discovery:v0.19.0 # ... charts: - name: gpu-operator url: oci://ghcr.io/nvidia/gpu-operator - version: v26.3.3 + version: v26.7.0 namespace: gpu-operator # ... ``` diff --git a/docs/user/api-reference.md b/docs/user/api-reference.md index 22b57e01c..e869e3e75 100644 --- a/docs/user/api-reference.md +++ b/docs/user/api-reference.md @@ -737,7 +737,7 @@ curl -X POST "http://localhost:8080/v1/bundle" \ "apiVersion": "aicr.run/v1alpha2", "kind": "RecipeResult", "componentRefs": [ - {"name": "gpu-operator", "type": "Helm", "chart": "gpu-operator", "source": "https://helm.ngc.nvidia.com/nvidia", "version": "v26.3.3", "namespace": "gpu-operator", "valuesFile": "components/gpu-operator/values.yaml"}, + {"name": "gpu-operator", "type": "Helm", "chart": "gpu-operator", "source": "https://helm.ngc.nvidia.com/nvidia", "version": "v26.7.0", "namespace": "gpu-operator", "valuesFile": "components/gpu-operator/values.yaml"}, {"name": "network-operator", "type": "Helm", "chart": "network-operator", "source": "https://helm.ngc.nvidia.com/nvidia", "version": "26.1.1", "namespace": "nvidia-network-operator", "valuesFile": "components/network-operator/values.yaml"} ], "deploymentOrder": ["gpu-operator", "network-operator"] diff --git a/docs/user/cli-reference.md b/docs/user/cli-reference.md index 73e6c5c1e..45270b5c7 100644 --- a/docs/user/cli-reference.md +++ b/docs/user/cli-reference.md @@ -1988,7 +1988,7 @@ The `--vendor-charts` flag pulls upstream Helm chart bytes into the bundle at bu my-bundle/ 001-gpu-operator/ Chart.yaml # wrapper, declares the vendored subchart - charts/gpu-operator-v26.3.3.tgz # vendored upstream tarball + charts/gpu-operator-v26.7.0.tgz # vendored upstream tarball values.yaml # values nested under the subchart name cluster-values.yaml # dynamic values, also nested install.sh # helm upgrade --install ./ ... @@ -2017,10 +2017,10 @@ kind: BundleProvenance vendoredCharts: - name: gpu-operator chart: gpu-operator - version: v26.3.3 + version: v26.7.0 repository: https://helm.ngc.nvidia.com/nvidia sha256: abc123... - tarballName: gpu-operator-v26.3.3.tgz + tarballName: gpu-operator-v26.7.0.tgz pullerVersion: helm-cli v3.20.2 ``` diff --git a/examples/recipes/aks-training.yaml b/examples/recipes/aks-training.yaml index 418788d30..cdd8d4dcc 100644 --- a/examples/recipes/aks-training.yaml +++ b/examples/recipes/aks-training.yaml @@ -52,7 +52,7 @@ componentRefs: chart: gpu-operator type: Helm source: https://helm.ngc.nvidia.com/nvidia - version: v26.3.3 + version: v26.7.0 valuesFile: components/gpu-operator/values-aks-training.yaml dependencyRefs: - nfd @@ -151,7 +151,7 @@ componentRefs: chart: dra-driver-nvidia-gpu type: Helm source: oci://registry.k8s.io/dra-driver-nvidia/charts - version: 0.4.1 + version: 0.5.0 valuesFile: components/nvidia-dra-driver-gpu/values.yaml overrides: controller: diff --git a/examples/recipes/eks-gb200-ubuntu-training-with-validation.yaml b/examples/recipes/eks-gb200-ubuntu-training-with-validation.yaml index c9b91a3f8..49bbe8429 100644 --- a/examples/recipes/eks-gb200-ubuntu-training-with-validation.yaml +++ b/examples/recipes/eks-gb200-ubuntu-training-with-validation.yaml @@ -61,9 +61,9 @@ validation: deployment: constraints: - name: gpu-operator.version - value: "== v26.3.3" + value: "== v26.7.0" severity: warning - remediation: "Update GPU Operator to v26.3.3" + remediation: "Update GPU Operator to v26.7.0" checks: - expected-resources @@ -100,7 +100,7 @@ componentRefs: chart: gpu-operator type: Helm source: https://helm.ngc.nvidia.com/nvidia - version: v26.3.3 + version: v26.7.0 valuesFile: components/gpu-operator/values-eks-training.yaml expectedResources: - kind: Deployment @@ -129,7 +129,7 @@ componentRefs: chart: dra-driver-nvidia-gpu type: Helm source: oci://registry.k8s.io/dra-driver-nvidia/charts - version: 0.4.1 + version: 0.5.0 valuesFile: components/nvidia-dra-driver-gpu/values.yaml dependencyRefs: - gpu-operator diff --git a/examples/recipes/eks-training.yaml b/examples/recipes/eks-training.yaml index e4dc850a9..8de030f01 100644 --- a/examples/recipes/eks-training.yaml +++ b/examples/recipes/eks-training.yaml @@ -41,7 +41,7 @@ componentRefs: chart: gpu-operator type: Helm source: https://helm.ngc.nvidia.com/nvidia - version: v26.3.3 + version: v26.7.0 valuesFile: components/gpu-operator/values-eks-training.yaml dependencyRefs: - cert-manager diff --git a/pkg/bundler/validations/checks.go b/pkg/bundler/validations/checks.go index 22765a67a..2a2afe6fd 100644 --- a/pkg/bundler/validations/checks.go +++ b/pkg/bundler/validations/checks.go @@ -464,7 +464,7 @@ func driverAbsentRemedy(service recipe.CriteriaServiceType, os recipe.CriteriaOS "DaemonSet alongside it; see " + "docs/integrator/gke-gpu-setup.md." case recipe.CriteriaOSUbuntu: - // The pinned GPU Operator (v26.3.3) supports driver management + // The pinned GPU Operator supports driver management // on GKE only on Ubuntu node images with containerd. return "On GKE Ubuntu node images the GPU Operator can manage " + "the driver: bundle in GPU-Operator-managed mode: " + @@ -643,11 +643,11 @@ func effectiveComponentValues(ctx context.Context, recipeResult *recipe.RecipeRe // path.Clean'd (trailing-slash spellings compare equal, mirroring // pkg/recipe/driver_root_lockstep_test.go), declared empty string → // the default (the operator's own transformForDriverInstallDir treats -// "" identically to the default, gpu-operator v26.3.3). An explicitly +// "" identically to the default, gpu-operator v26.7.0). An explicitly // null or non-map hostPaths section is rejected with a blocking // message: Helm null-coalescing deletes a null key together with its // chart defaults, so the chart's unconditional .Values.hostPaths.rootFS -// access (clusterpolicy.yaml, v26.3.3) fails at install. A declared +// access (clusterpolicy.yaml, v26.7.0) fails at install. A declared // value that cleans to a relative path is rejected too — host-path // mounts require absolute paths. func resolveInstallDir(values map[string]any, componentName string) (string, bool, []string) { @@ -680,7 +680,7 @@ func resolveInstallDir(values map[string]any, componentName string) (string, boo // rejected rather than silently defaulted: the emitted values // would carry it verbatim, and the pinned ClusterPolicy CRD // types hostPaths.driverInstallDir as a string (gpu-operator - // v26.3.3 nvidia.com_clusterpolicies.yaml), so the install + // v26.7.0 nvidia.com_clusterpolicies.yaml), so the install // fails while a defaulted check would have validated against // /run/nvidia/driver instead. return installDir, false, []string{fmt.Sprintf( @@ -693,7 +693,7 @@ func resolveInstallDir(values map[string]any, componentName string) (string, boo if dir == "" { // Intentionally default-equivalent: the operator's own // transformForDriverInstallDir early-returns on "" exactly like - // the default (gpu-operator v26.3.3, controllers/object_controls.go). + // the default (gpu-operator v26.7.0, controllers/object_controls.go). return installDir, false, nil } cleaned := path.Clean(dir) @@ -716,7 +716,7 @@ func resolveInstallDir(values map[string]any, componentName string) (string, boo // resolveDRARoot resolves the effective nvidia-dra-driver-gpu // nvidiaDriverRoot for Rule 2. Only a genuinely ABSENT key falls back to -// the chart-default assumption ("/", DRA chart v0.4.1 values.yaml). A +// the chart-default assumption ("/", DRA chart v0.5.0 values.yaml). A // present null, empty-string, or non-string value is rejected: unlike the // gpu-operator's driverInstallDir (where "" is default-equivalent, see // resolveInstallDir), the DRA chart pipes the raw value through @@ -866,7 +866,7 @@ func nvsentinelDynamicGuardViolations(bundlerConfig *config.Config, componentNam // null-coalescing deletes the key together with its chart defaults, // so .Values.
is nil at render time and the gpu-operator // templates fail on unconditional field access (e.g. -// .Values.driver.manager.repository in _helpers.tpl, v26.3.3) — +// .Values.driver.manager.repository in _helpers.tpl, v26.7.0) — // ownership cannot be verified and the install would fail anyway. // A non-boolean toggle is rejected because the chart renders the // value unquoted, so YAML re-typing at install time can flip it to a @@ -1705,7 +1705,7 @@ func CheckNVSentinelDriverLabelDetectable(ctx context.Context, componentName str } // defaultRuntimeClassName is the shared chart default: the gpu-operator -// chart ships operator.runtimeClass: nvidia (v26.3.3, verified against +// chart ships operator.runtimeClass: nvidia (v26.7.0, verified against // the pinned chart values), and nvsentinel's metadata-collector subchart // ships runtimeClassName: "nvidia" (v1.9.0, charts/metadata-collector/ // values.yaml:31). Either side left unset therefore resolves to this diff --git a/pkg/client/v1/gpu_driver_state.go b/pkg/client/v1/gpu_driver_state.go index dd4d2da7e..59801164d 100644 --- a/pkg/client/v1/gpu_driver_state.go +++ b/pkg/client/v1/gpu_driver_state.go @@ -109,7 +109,7 @@ func driverAbsentRemedy(service recipe.CriteriaServiceType, os recipe.CriteriaOS "DaemonSet and pins the driver version — nothing to deploy " + "by hand; see docs/integrator/gke-gpu-setup.md." case recipe.CriteriaOSUbuntu: - // The pinned GPU Operator (v26.3.3) supports driver management + // The pinned GPU Operator supports driver management // on GKE only on Ubuntu node images with containerd. return "On GKE Ubuntu node images the GPU Operator can manage " + "the driver: bundle in GPU-Operator-managed mode: " + diff --git a/pkg/recipe/driver_root_lockstep_test.go b/pkg/recipe/driver_root_lockstep_test.go index 1372e6141..bd38abf53 100644 --- a/pkg/recipe/driver_root_lockstep_test.go +++ b/pkg/recipe/driver_root_lockstep_test.go @@ -74,7 +74,7 @@ import ( // **Why "explicitly set" matters for the lockstep case.** An empty value // falls through to the upstream chart's bundled default, which the test // cannot read — and per-component defaults differ (GPU Operator chart -// 26.3.3 defaults driverInstallDir to /run/nvidia/driver, but DRA chart +// 26.7.0 defaults driverInstallDir to /run/nvidia/driver, but DRA chart // 25.12.0 defaults nvidiaDriverRoot to /). Relying on chart defaults is // itself drift waiting to happen on the next chart bump, so when the // lockstep applies the test treats "not explicitly set on both" as a @@ -211,7 +211,7 @@ func TestDriverRootLockstep(t *testing.T) { t.Errorf( "overlay %q: both nvidia-dra-driver-gpu.nvidiaDriverRoot and gpu-operator.hostPaths.driverInstallDir are unset.\n"+ " Both must be set explicitly to the same path. Chart defaults differ across components\n"+ - " (gpu-operator chart 26.3.3: /run/nvidia/driver; dra chart 25.12.0: /), so an unset value\n"+ + " (gpu-operator chart 26.7.0: /run/nvidia/driver; dra chart 0.5.0: /), so an unset value\n"+ " is drift waiting to happen on the next chart bump.\n"+ " See issue #1087.", name) From 6be2d859b17819e8628ee669a52c0276e52247aa Mon Sep 17 00:00:00 2001 From: Yuan Chen Date: Fri, 28 Aug 2026 10:39:41 -0700 Subject: [PATCH 07/12] test(recipe): guard ComputeDomain manifests against the CRD-overlap hazard MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit gpu-operator v26.7.0 and the standalone DRA driver both ship a computedomains.resource.nvidia.com CRD, and the copies differ: the operator's marks numNodes required with no default. Helm installs crds/ only when absent and gpu-operator is ordered first, so a fresh cluster gets the stricter copy and rejects any ComputeDomain CR omitting numNodes. An upgraded cluster masks this because DRA 0.4.1 already installed the permissive copy, so no upgrade-path e2e would catch a regression here. The guard ignores comment lines: these manifests legitimately discuss "spec.numNodes: Required value" in prose, and a naive substring scan matched that instead of the key — passing even with the key deleted. Verified by deleting numNodes and confirming the test fails, then restoring it. Also fails closed when the walk matches no manifests, so it cannot go silently inert. Signed-off-by: Yuan Chen --- pkg/recipe/computedomain_numnodes_test.go | 124 ++++++++++++++++++++++ 1 file changed, 124 insertions(+) create mode 100644 pkg/recipe/computedomain_numnodes_test.go diff --git a/pkg/recipe/computedomain_numnodes_test.go b/pkg/recipe/computedomain_numnodes_test.go new file mode 100644 index 000000000..b2cbe4bda --- /dev/null +++ b/pkg/recipe/computedomain_numnodes_test.go @@ -0,0 +1,124 @@ +// Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package recipe + +import ( + "io/fs" + "regexp" + "strings" + "testing" +) + +// numNodesKeyRE matches an indented mapping key. Anchored per line so a key +// inside a comment or a quoted error string cannot satisfy it. +var numNodesKeyRE = regexp.MustCompile(`(?m)^[ \t]+numNodes[ \t]*:`) + +// hasYAMLKey reports whether content declares key as a real mapping key, +// ignoring comment lines. +func hasYAMLKey(content, key string) bool { + var b strings.Builder + for _, line := range strings.Split(content, "\n") { + if strings.HasPrefix(strings.TrimSpace(line), "#") { + continue + } + b.WriteString(line) + b.WriteString("\n") + } + if key != "numNodes" { + panic("hasYAMLKey: only numNodes is supported") + } + return numNodesKeyRE.MatchString(b.String()) +} + +// TestComputeDomainManifestsSetNumNodes guards the fresh-install CRD-overlap +// hazard introduced by GPU Operator v26.7.0. +// +// Two charts in the catalog ship a CRD named computedomains.resource.nvidia.com: +// the standalone nvidia-dra-driver-gpu chart, and — new in gpu-operator +// v26.7.0 — the GPU Operator chart. The two copies are NOT identical. The +// operator's is a stale snapshot that lists numNodes as required and supplies +// no `default: 0`; the DRA driver's makes it optional with a default. +// +// Helm installs crds/ only when the CRD is absent and never upgrades it, and +// gpu-operator is ordered before nvidia-dra-driver-gpu. So on a FRESH cluster +// the operator's stricter copy is the one that lands, and any ComputeDomain CR +// omitting spec.numNodes is rejected by the API server with +// "spec.numNodes: Required value". Structural defaulting cannot rescue it +// because that copy carries no default. Neither chart installs a webhook that +// could supply the field. +// +// An UPGRADED cluster masks this: it already has the permissive copy installed +// by DRA 0.4.1, so the CR still admits. That asymmetry is why this is a unit +// guard rather than something an upgrade-path e2e would catch. +// +// The invariant: every ComputeDomain CR shipped in the catalog must set +// spec.numNodes explicitly, so it is valid under BOTH CRD copies regardless of +// which chart installed the CRD first. +// +// See PR #2439 and issue #1087 for the driver-root analogue of this +// cross-component coupling problem. +func TestComputeDomainManifestsSetNumNodes(t *testing.T) { + t.Parallel() + + efs := GetEmbeddedFS() + + var checked int + err := fs.WalkDir(efs, ".", func(path string, d fs.DirEntry, err error) error { + if err != nil { + return err + } + if d.IsDir() || !strings.HasSuffix(path, ".yaml") { + return nil + } + raw, readErr := efs.ReadFile(path) + if readErr != nil { + return readErr + } + content := string(raw) + if !strings.Contains(content, "kind: ComputeDomain") { + return nil + } + checked++ + + // The manifests are Helm templates, so a full YAML parse is not + // available. Strip comment lines FIRST — the surrounding prose in + // these files legitimately discusses "spec.numNodes: Required value", + // and a naive substring scan matches that instead of the real key, + // producing a guard that passes even when the key is deleted. + if !hasYAMLKey(content, "numNodes") { + t.Errorf("%s declares kind: ComputeDomain but does not set spec.numNodes.\n"+ + " GPU Operator v26.7.0 ships a ComputeDomain CRD copy that marks numNodes\n"+ + " REQUIRED with no default, and it is installed before the DRA driver's\n"+ + " permissive copy. On a fresh cluster this CR is rejected at admission with\n"+ + " \"spec.numNodes: Required value\".\n"+ + " Set numNodes explicitly (0 is correct under IMEXDaemonsWithDNSNames=true,\n"+ + " the DRA driver default, where each IMEX daemon starts without waiting for\n"+ + " a quorum). See PR #2439.", path) + } + return nil + }) + if err != nil { + t.Fatalf("walking embedded recipes: %v", err) + } + + // Fail closed on a vacuous pass: if the walk matched nothing, the guard is + // silently inert and a regression would go unnoticed. + if checked == 0 { + t.Fatal("no ComputeDomain manifests found in the embedded recipes — " + + "this guard is vacuous. Either the manifests moved, or the embed " + + "pattern no longer covers them.") + } + t.Logf("verified %d ComputeDomain manifest(s) set spec.numNodes", checked) +} From 9cb342bb9b18f98c2028d98e97a5cbcd9ebdf57c Mon Sep 17 00:00:00 2001 From: Yuan Chen Date: Fri, 28 Aug 2026 10:59:55 -0700 Subject: [PATCH 08/12] fix: correct misspell lint failure and finish the version-reference refresh misspell rejects "analogue"; the US spelling is required by the lint gate. Three stale references the earlier refresh missed: - demos/dynamic.md grepped for 'version: 580' to prove the dynamic value had been removed from baked values. With the pin at 595.91.07 that grep returns 0 whether or not the substitution worked, so the check no longer proved anything. The chainsaw suite it mirrors uses a 999.99.99 sentinel and is unaffected. - the driver-root lockstep comment still named DRA chart 25.12.0 while its own error text had been updated to 0.5.0. - the AKS setup guide still named toolkit v1.19.1 while the manifest it documents now names v1.20.0. Signed-off-by: Yuan Chen --- demos/dynamic.md | 2 +- docs/integrator/aks-gpu-setup.md | 2 +- pkg/recipe/computedomain_numnodes_test.go | 2 +- pkg/recipe/driver_root_lockstep_test.go | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/demos/dynamic.md b/demos/dynamic.md index a178fb171..f2c1be1b1 100644 --- a/demos/dynamic.md +++ b/demos/dynamic.md @@ -116,7 +116,7 @@ Three quick checks, the same ones the chainsaw suite enforces: ```shell # a) gone from the baked values.yaml (no match) -grep -c 'version: 580' ./bundle/010-gpu-operator/values.yaml +grep -c 'version: 595' ./bundle/010-gpu-operator/values.yaml # 0 # b) present in cluster-values.yaml diff --git a/docs/integrator/aks-gpu-setup.md b/docs/integrator/aks-gpu-setup.md index 695172902..6d11a5e16 100644 --- a/docs/integrator/aks-gpu-setup.md +++ b/docs/integrator/aks-gpu-setup.md @@ -376,7 +376,7 @@ required so the device plugin's volume-mounts allocation strategy still works for legitimately allocated pods, but it leaves the **volume-mounts device-request path open** (a pod that declares a `/dev/null`-backed mount whose destination is under `/var/run/nvidia-container-devices` can still select -devices — the pinned toolkit v1.19.1 accepts the volume-mount device request +devices — the pinned toolkit v1.20.0 accepts the volume-mount device request only when the mount *source* is `/dev/null`). This is the same posture as GPU-Operator-managed mode — not a regression — but it means full multi-tenant isolation additionally requires an admission policy restricting a diff --git a/pkg/recipe/computedomain_numnodes_test.go b/pkg/recipe/computedomain_numnodes_test.go index b2cbe4bda..657051da6 100644 --- a/pkg/recipe/computedomain_numnodes_test.go +++ b/pkg/recipe/computedomain_numnodes_test.go @@ -67,7 +67,7 @@ func hasYAMLKey(content, key string) bool { // spec.numNodes explicitly, so it is valid under BOTH CRD copies regardless of // which chart installed the CRD first. // -// See PR #2439 and issue #1087 for the driver-root analogue of this +// See PR #2439 and issue #1087 for the driver-root analog of this // cross-component coupling problem. func TestComputeDomainManifestsSetNumNodes(t *testing.T) { t.Parallel() diff --git a/pkg/recipe/driver_root_lockstep_test.go b/pkg/recipe/driver_root_lockstep_test.go index bd38abf53..e1c83fdc7 100644 --- a/pkg/recipe/driver_root_lockstep_test.go +++ b/pkg/recipe/driver_root_lockstep_test.go @@ -75,7 +75,7 @@ import ( // falls through to the upstream chart's bundled default, which the test // cannot read — and per-component defaults differ (GPU Operator chart // 26.7.0 defaults driverInstallDir to /run/nvidia/driver, but DRA chart -// 25.12.0 defaults nvidiaDriverRoot to /). Relying on chart defaults is +// 0.5.0 defaults nvidiaDriverRoot to /). Relying on chart defaults is // itself drift waiting to happen on the next chart bump, so when the // lockstep applies the test treats "not explicitly set on both" as a // failure. From e557a83ae5c5d979b2da69958abc759143eb093a Mon Sep 17 00:00:00 2001 From: Yuan Chen Date: Fri, 28 Aug 2026 11:36:39 -0700 Subject: [PATCH 09/12] test: scope the ComputeDomain guard per YAML document MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The guard matched numNodes anywhere in the file. A multi-document manifest where one ComputeDomain sets the key and a second omits it would pass while still failing admission, as would an unrelated resource carrying a numNodes key. No such manifest exists today; scoping per document means adding one cannot silently bypass the guard. Two independent reviews raised this. Dropping the whole-file helper also removes its fake generic signature — it took a key parameter, ignored it in the regex, and panicked on anything but "numNodes". Controls verified both ways: deleting numNodes fails, and appending a second ComputeDomain document without it fails naming document 2. Restoring passes. Also updates two comments in checks_test.go that mirror comments in checks.go which this branch already moved to v26.7.0; the claims were verified against the v26.7.0 chart and still hold. Signed-off-by: Yuan Chen --- pkg/bundler/validations/checks_test.go | 4 +- pkg/recipe/computedomain_numnodes_test.go | 55 ++++++++++++++--------- 2 files changed, 37 insertions(+), 22 deletions(-) diff --git a/pkg/bundler/validations/checks_test.go b/pkg/bundler/validations/checks_test.go index c72760c32..833f1718f 100644 --- a/pkg/bundler/validations/checks_test.go +++ b/pkg/bundler/validations/checks_test.go @@ -1330,7 +1330,7 @@ func TestCheckDriverOwnershipCoherence(t *testing.T) { // the key together with its chart defaults, so .Values.driver // is nil at render time and the chart's unconditional field // accesses (_helpers.tpl .Values.driver.manager.repository, - // v26.3.3) fail at install. Reject rather than default. The + // v26.7.0) fail at install. Reject rather than default. The // reachable vector is a top-level --set-json null: the typed // merge assigns it verbatim (mergeTypedValueByPath), while the // recipe-side overlay merge drops nil-valued keys before the @@ -1357,7 +1357,7 @@ func TestCheckDriverOwnershipCoherence(t *testing.T) { // hostPaths: null is the same hazard as a null driver/toolkit // section: Helm null-coalescing deletes the chart defaults and // clusterpolicy.yaml's unconditional .Values.hostPaths.rootFS - // access fails at install (v26.3.3). + // access fails at install (v26.7.0). name: "--set-json hostPaths=null → rejected", recipeResult: result("", aks, gpuOpRef(driverOn())), bundlerConfig: config.NewConfig(config.WithValueOverridesTypedPaths([]config.TypedComponentPath{ diff --git a/pkg/recipe/computedomain_numnodes_test.go b/pkg/recipe/computedomain_numnodes_test.go index 657051da6..b190b3a61 100644 --- a/pkg/recipe/computedomain_numnodes_test.go +++ b/pkg/recipe/computedomain_numnodes_test.go @@ -25,21 +25,41 @@ import ( // inside a comment or a quoted error string cannot satisfy it. var numNodesKeyRE = regexp.MustCompile(`(?m)^[ \t]+numNodes[ \t]*:`) -// hasYAMLKey reports whether content declares key as a real mapping key, -// ignoring comment lines. -func hasYAMLKey(content, key string) bool { - var b strings.Builder - for _, line := range strings.Split(content, "\n") { - if strings.HasPrefix(strings.TrimSpace(line), "#") { +// computeDomainDocsMissingNumNodes returns the 0-based indexes of YAML +// documents that declare kind: ComputeDomain without a numNodes key. +// +// Scoped per document rather than per file. A multi-document manifest where one +// ComputeDomain sets numNodes and a second omits it would satisfy a whole-file +// scan while still failing admission, and so would an unrelated resource that +// happens to carry a numNodes key. No such manifest exists in the catalog +// today; the guard is document-scoped so that adding one cannot silently +// bypass it. +// +// Comment lines are stripped first: these manifests legitimately discuss +// "spec.numNodes: Required value" in prose, and a naive substring scan matches +// that instead of the real key, passing even when the key is deleted. +// +// A full YAML parse is unavailable — the manifests are Helm templates and +// contain {{ }} expressions that no YAML parser accepts. +func computeDomainDocsMissingNumNodes(content string) []int { + var missing []int + for i, doc := range strings.Split(content, "\n---") { + if !strings.Contains(doc, "kind: ComputeDomain") { continue } - b.WriteString(line) - b.WriteString("\n") - } - if key != "numNodes" { - panic("hasYAMLKey: only numNodes is supported") + var b strings.Builder + for _, line := range strings.Split(doc, "\n") { + if strings.HasPrefix(strings.TrimSpace(line), "#") { + continue + } + b.WriteString(line) + b.WriteString("\n") + } + if !numNodesKeyRE.MatchString(b.String()) { + missing = append(missing, i) + } } - return numNodesKeyRE.MatchString(b.String()) + return missing } // TestComputeDomainManifestsSetNumNodes guards the fresh-install CRD-overlap @@ -92,20 +112,15 @@ func TestComputeDomainManifestsSetNumNodes(t *testing.T) { } checked++ - // The manifests are Helm templates, so a full YAML parse is not - // available. Strip comment lines FIRST — the surrounding prose in - // these files legitimately discusses "spec.numNodes: Required value", - // and a naive substring scan matches that instead of the real key, - // producing a guard that passes even when the key is deleted. - if !hasYAMLKey(content, "numNodes") { - t.Errorf("%s declares kind: ComputeDomain but does not set spec.numNodes.\n"+ + for _, idx := range computeDomainDocsMissingNumNodes(content) { + t.Errorf("%s: YAML document %d declares kind: ComputeDomain but does not set spec.numNodes.\n"+ " GPU Operator v26.7.0 ships a ComputeDomain CRD copy that marks numNodes\n"+ " REQUIRED with no default, and it is installed before the DRA driver's\n"+ " permissive copy. On a fresh cluster this CR is rejected at admission with\n"+ " \"spec.numNodes: Required value\".\n"+ " Set numNodes explicitly (0 is correct under IMEXDaemonsWithDNSNames=true,\n"+ " the DRA driver default, where each IMEX daemon starts without waiting for\n"+ - " a quorum). See PR #2439.", path) + " a quorum). See PR #2439.", path, idx) } return nil }) From 9bb2c8efd075958350b16c849b297e4b80d850e2 Mon Sep 17 00:00:00 2001 From: Yuan Chen Date: Fri, 28 Aug 2026 11:43:05 -0700 Subject: [PATCH 10/12] refactor(recipes): replace no-op useOpenKernelModules with kernelModuleType The chart deprecated useOpenKernelModules and made it a no-op; the identical NOTE appears in v26.3.3 and v26.7.0 values.yaml, so the key had already stopped having any effect before this pin moved. Our clusters have been running the chart default kernelModuleType: auto regardless of what that line said. Pinning "auto" records today's effective behaviour rather than changing it: it is the chart default on both versions and the fallback the OCP ClusterPolicy template already renders. Setting "open" would pin the intent the dead key expressed, but that is a real behaviour change wherever auto currently selects proprietary, so it needs its own validation rather than riding a version bump. Render goldens move for 46 of 48 leaves. The two that do not, ocp-inference-nim and ocp-training, consume gpu-operator-ocp and its separate values file. Catalog goldens are unchanged because values content affects rendering but not recipe resolution. Signed-off-by: Yuan Chen --- pkg/bundler/testdata/stock_render_golden.yaml | 92 +++++++++---------- recipes/components/gpu-operator/values.yaml | 18 +++- 2 files changed, 63 insertions(+), 47 deletions(-) diff --git a/pkg/bundler/testdata/stock_render_golden.yaml b/pkg/bundler/testdata/stock_render_golden.yaml index bc31acfbf..9ce4a1e59 100644 --- a/pkg/bundler/testdata/stock_render_golden.yaml +++ b/pkg/bundler/testdata/stock_render_golden.yaml @@ -3,51 +3,51 @@ # # One entry per leaf overlay: a digest over its fully rendered helm-deployer # bundle tree (sorted relative paths paired with per-file content hashes). -a100-aks-ubuntu-training-kubeflow: 03953f4ca87794693143872440318e5c03a71eeb2b26633074c15a07bc7ea2a0 -a100-any: 1574bec32707462c1b4b5f61111b9db8a17ed2d6e085c03d197bdc252b53f66b -a100-eks-ubuntu-training-kubeflow: c7188e09cac23357027d5f88e1b6400a81ca0730595049b6d2c18b79389b5ea8 -a100-gke-cos-training-kubeflow: f82dfc6cff16f94e4b6d9a149cafd492b3de2638805ec8710016a513b3239f01 -a100-oke-ubuntu-training-kubeflow: d41b6533c1a0c7c146ef1d461234898bc643ca1cb92e93fb64ab7325ff1b080b -b200-any: c45ab3caedd55af9e76d4b24db0cbc8393dc8f6be8fa8ac915453a28c2281833 -b200-gke-cos-inference-dynamo: d6f9103bce0828001485ea93e5e045318a499e81f16c6e3ab4a26d298e20b50e -b200-gke-cos-training-kubeflow: c31aded6fe3354cba4702de9855d9edbef6eee30da284b97e342e57101ef70a4 -bcm-inference: bc7e8b71e3d73de8d849c64c82e2177c2e8d8a5771a0bc5df66e14fa84bc34a6 -gb200-any: 2223bb69c04629a557137818972dfa8b9d952e6477d2cd1b5e9a3a3a7ece65ea -gb200-eks-ubuntu-inference-dynamo: 307d88b7d991d1eeaec847367eeed334b9871e5c250ffd91c30659ba9f5d3d06 -gb200-eks-ubuntu-training-kubeflow: 8782d40e1b816c9d568baa0ec9e96ffb37b8b69aa99b10dc36b9ba0f6b676e7c -gb200-eks-ubuntu-training-slurm: 2701547d3ea6619bb99cc43c0109b88d23ccb763336b2d25f2fe0877aa3ffb6c -gb200-oke-ubuntu-inference-dynamo: 3b0326748d71af2c0e92835894026c00e7cdf505bb075f71ad2ff8663c0b168a -gb200-oke-ubuntu-training-kubeflow: 7a00164f66b9e19a3a6b9bdafbb4c87d6c039b8b40bef810da34bab7fbb1ad45 -gb300-any: 0c0b8b78181e33588c154219ff44ecc4914518905917bd21f986fa5016732201 -gb300-eks-ubuntu-inference-dynamo: d0afefee4a8e1cc7e1a7fb22fda6e5f59f7b6cc57139a51dd5189a3288c4fcfe -gb300-eks-ubuntu-training-kubeflow: 0332c753b04f8cf32ca4e237614e925cf4f922f11cf9f6e8a64834869256869e -h100-aks-ubuntu-inference-dynamo: fd7227664378fa578b36d12bac10e082a4d36e2ba016ed275bdc93ce14336b7a -h100-aks-ubuntu-training-kubeflow: 7b36f0149ca75ca0911705b63b37f126d5c3dd5c251e3532615fcd1213052af7 -h100-aks-ubuntu-training-slurm: 2aa10941a7d9077a4a29bbd088e564f36bc830a2a5c8d7ae04ea2a517a6d08cd -h100-any: 5d6032050aeac10384c0808b3b32d811f7ec87fd5e4d816d03c391e13c94a93f -h100-bcm-ubuntu-training: 8085e79850c12c513e5d857b3d9ddb3d49eaab28f9237a1ee596198d96b34710 -h100-eks-ubuntu-inference-dynamo: 1cf513f5383e847def40f5f382e00bc48e35e0f33963abc59514cf50814f2a71 -h100-eks-ubuntu-inference-nim: c3525c9ddf07c3d51a6de71f998fe8aaaae66c5f7783b41acc57843b8b9ca11c -h100-eks-ubuntu-training-kubeflow: 9a2e562cc16ed98561230812e5dda5257a5a8cab0665579c6e6918ffcc26b7b1 -h100-eks-ubuntu-training-slurm: 0a1fc59b1144ec4f7e16b544ae9ebcf189f78447bb8dae42b073141f332d90d8 -h100-gke-cos-inference-dynamo: f001d10c88bd0e2c609c74fc1ae60a9752bbb67b8195ead55245d75f3aa89534 -h100-gke-cos-training-kubeflow: 2cfc19faee0144f43895e504380b8491ace0eb1121c6005fb9cbf19a81310d97 -h100-gke-cos-training-slurm: 001ca331ba2dd7f512959923e530f15ebf0b1f03035c6cc3fdafabb48fc0663a -h100-kind-inference-dynamo: 959f28abbdc9d05c56476c993c23fa58404edc634c30f3c60f1bf70173791127 -h100-kind-training-kubeflow: 77e8dd0ad1b78af14be19765d0c855415aa93bb3a7b3e1dd8135d464b28798b3 -h100-kind-training-slurm: eef18f418b69b879964e69cbd3f7a3c3a28c2451a2fd03c3cf60f811bbb9bc19 -h200-any: a8c74e455e6fb774df5724cdefc6d5e0823e7d64a48504ecc97c4099a1292795 -h200-eks-inference: f2a2c69a5098c74859d530518504af6dbcd80db25331a625e7e4ddfada396984 -h200-eks-training: dfd4e12aa5c5d5a69e28f1fd38b4a13bc48fc8a0d1046fd2cafa6113c4d7a397 -l40s-any: 03d4042acd620aa3a697df010c5e2e5b42388c820be7cafeaf6c6636bb443fe9 -l40s-oke-inference: cd3986c301fd7355a883ba41cbee9dd02f782bfd79fd334f40cf1fc7c725366a -l40s-oke-training: 79094f30620f87280b7f271080642cad6915b3b35ea09880919b2c269541025b -monitoring-hpa: 3fd6a80124d0e346ff3c42f1692d36c74626414a7dad33da56f6237233a3cb63 +a100-aks-ubuntu-training-kubeflow: ba4809f205a8ddf94afa75a8a3623101d69874e477edcf3140890b35015fee46 +a100-any: 67228bd14328a01b06335b6b805ba8f66696cd18c3e6b80a24d9b0b4ea12b551 +a100-eks-ubuntu-training-kubeflow: 77ba19db6c1a621dc703b7efcb74e678e807c11d25964c119e61637780847c00 +a100-gke-cos-training-kubeflow: b00d861db41fb6b3d67c97706ab6ddf7b2ddc7f31d718f868b4e88586d918229 +a100-oke-ubuntu-training-kubeflow: 9e20629fee09306cd88265ad42c838ea8b3df9ad5b4a76e93adaa09552a49809 +b200-any: a0f3f6e9dc5ffcb4d0c96c32b735b885767e99ddd3e0a2447c8ade25a8d8d930 +b200-gke-cos-inference-dynamo: 2402e86bcd3453bc9604693879b7056c938d2b05911104fcca1aeec3413f03be +b200-gke-cos-training-kubeflow: cc5df420bee36fa2af8ab3dc5a690f38ca11ab8d0d3f3bba32d3889c6867a98c +bcm-inference: f06817c3866ae75d3c46c73fae1d121831f7a22ae4cb1edc2e18f9854e1f53cf +gb200-any: 580846f807cf370252c28428bc46ef13293f2303be04795e956d401ae145e108 +gb200-eks-ubuntu-inference-dynamo: 966c6929259615769e2bc887e0952f40ab5e6e50174f97d7409b6d589fe8c81f +gb200-eks-ubuntu-training-kubeflow: 98a44bea88ea04692e3e64e8ac4682095fb2e8f7141473a5bc1b21b2e5d55e50 +gb200-eks-ubuntu-training-slurm: d5f7ec4abdf22825cddf5beaa9410b090add7195b21b6f3eaa3e47f98fba0954 +gb200-oke-ubuntu-inference-dynamo: 1741a806e5b94997adfaebe2224128d78e1741bc94b5e0887f7ef49f0b0a0a4b +gb200-oke-ubuntu-training-kubeflow: ea86c42a1a9a4b51353b0f69b9abe80eaa881f0e0f640e384e9487c199127a8a +gb300-any: e703f2cd97e2aaec3b5f920a795e331e0d8a268e17da2450d688f853280db7a8 +gb300-eks-ubuntu-inference-dynamo: fa2fe0191625a7a5e9498406933154bbbc2f5de8ddf3233a9b64f60b1403070f +gb300-eks-ubuntu-training-kubeflow: ac3dd6280831114cdd367ae37780e12ee16c10b2c575ab67afb334badfe62e0b +h100-aks-ubuntu-inference-dynamo: ccac7522740d262e3ab12af286e7c3d7f58231673a7ded30018cecc5154cfa10 +h100-aks-ubuntu-training-kubeflow: fcd0b8598fade1c7086dd85c7ba301cad67b187294704b456294feac1ad59804 +h100-aks-ubuntu-training-slurm: 983f86e00d4b1423c308604034e3a05f068c3e6dc7cb74426ddeef5f644d1df3 +h100-any: 2c0e0386ec460ca8eeb3593bb22b68db7b95abe6fba51aea50df7f277fdb5503 +h100-bcm-ubuntu-training: eb3eb1572c56f89e7d8108915879f5debf6eb3892563bd12a366d05f5e52e120 +h100-eks-ubuntu-inference-dynamo: 82c84d76b24b45cf3845933cd567f3bbd84d92e5e7105ce3c46396d65000db8f +h100-eks-ubuntu-inference-nim: 415292845ee10056c61890ed5c5eb469c072e0786709c59173d9d7ed7555ecfb +h100-eks-ubuntu-training-kubeflow: 478fc650545d1b0caa9761373fd6ab7765f1c5fe09f6ddecf6ebb71d976f3e05 +h100-eks-ubuntu-training-slurm: 9a444f808d45439f6a8903a03e8610a94444c9fa4783634f0c20f72077e7db5c +h100-gke-cos-inference-dynamo: cb6f151bcd7f5ad7b81ceea35649320ee8c8842236d749d9f6837a833bc1dd60 +h100-gke-cos-training-kubeflow: feae5f125622fa0c472d2cf64cf8c6088cec1160cf95501b98c8fecd404fd734 +h100-gke-cos-training-slurm: 698bbed35f699b9f28abfa13863f38d6dc36fd62180027db0b87cad89cfe13ae +h100-kind-inference-dynamo: b7b33b07faaed415c9a120fb43bf78d58cf606783103b26d3c5edd4f0ed51a18 +h100-kind-training-kubeflow: 4412529fcd051ad8bc389993ca52d9526e1180805ede6e42b29fcbc85a64fd99 +h100-kind-training-slurm: a4f8479c9c210735ac66953e77e1fe4a1c5bd5ae8262178e300ce78e496a44e7 +h200-any: 737f444bf832d184f91f768cf3b58839c1ba36abf5578da79cf4c0938e2f1f07 +h200-eks-inference: d1fac6d45f2eae71ab0a0e51f2e942176b5f6e28a22132de8b9171b74e8e078a +h200-eks-training: cf5add53acd55c0275f0de66589b13bf5943b2c9d48d13f8f389936ddcaad52d +l40s-any: a4385e2dc809867437b44748b26a57ed2bc463d15254f73a532c8fb9dcdcb872 +l40s-oke-inference: 8e7fa89c8a94ddf517f1171cb2b6775ce37d0ab3a86fbc56fdb5324b99a2b493 +l40s-oke-training: 1f3ce9538778d67b32f1ef068000f1c336daaa812c61eb49ba03139c207500f6 +monitoring-hpa: 19bc810c681b40bc40d5f90821feba53cb1a60805f9f47ddcf79e2388e582c01 ocp-inference-nim: a728bed625f568760bf291d54e066a443e2c141b239dc5fb7f4bc96dcac87c39 ocp-training: 32a047795bcb4c9ea6363e18de6d4f213e6b8c9003106cb4a54ccffd3a6b0ef1 -rtx-pro-6000-any: 4930e5dfc65a125e9ffebfe9a997adef7a123432fd05d584dc4984a2e72ab893 -rtx-pro-6000-eks-ubuntu-inference-dynamo: 2be567e056aceea081f5e8e676724617e6986febb64e5f21992362bc80d14bc8 -rtx-pro-6000-eks-ubuntu-inference-nim: 1672ed446085dc84d395288edfbf1be31789151e9ab5800379d7fa2cdbb6fcb6 -rtx-pro-6000-eks-ubuntu-training-kubeflow: 976ad5d9d70517825faa9491987f1f6a2059337ed69cc2a66a05809aee4f23c0 -rtx-pro-6000-lke-ubuntu-inference: d98dba61633284f0bbba26df5f3525839984d26c424353d57133a4e9441c29aa -rtx-pro-6000-lke-ubuntu-training: 23f03d02c253c4be476e760dab8e8ba1d808d443af13913b558e8a7681041fa0 +rtx-pro-6000-any: 39eabfa120d603d57963e4ebbf2ba500dae687dbb06c483d65d541b884f582bb +rtx-pro-6000-eks-ubuntu-inference-dynamo: f44ae470c5229ad94285f12076b4c9d9d3bab9021245bc8f5f85635bd773d857 +rtx-pro-6000-eks-ubuntu-inference-nim: 3fd68742176ba25430952efa41751fe092e16b9904566fa5f21ba4e1a0619605 +rtx-pro-6000-eks-ubuntu-training-kubeflow: 825f138e88d319e325226cfc1686e542f42a8809644e4d82817e9e5b2612a3e7 +rtx-pro-6000-lke-ubuntu-inference: 883e38b844d236e4bf233962a5a9f09fce8388a3966a77cc37d40f11248ee7d3 +rtx-pro-6000-lke-ubuntu-training: c61067119e33798f0e1b91a0f097daf1542b80846b34a99c30225654cefae1ca diff --git a/recipes/components/gpu-operator/values.yaml b/recipes/components/gpu-operator/values.yaml index d7cfd4c35..e71d8a7e5 100644 --- a/recipes/components/gpu-operator/values.yaml +++ b/recipes/components/gpu-operator/values.yaml @@ -160,7 +160,23 @@ driver: # requirement, hold the pin ahead rather than following the chart. version: 595.91.07 enabled: true - useOpenKernelModules: true + # Replaces useOpenKernelModules, which the chart deprecated and made a + # no-op (identical NOTE in v26.3.3 and v26.7.0 values.yaml), so the old + # key had already stopped having any effect before this pin moved. + # + # "auto" pins today's effective behaviour, not a change: it is the chart + # default both before and after this bump, and it is what the OCP + # ClusterPolicy template already falls back to + # (components/gpu-operator-ocp/manifests/clusterpolicy.yaml). Per the + # ClusterPolicy CRD, auto selects the recommended module type from the + # host's GPU devices and driver branch — which yields open modules on + # Blackwell/Grace, where they are required. + # + # Setting "open" instead would pin the intent the dead key expressed, but + # that IS a behaviour change on any hardware where auto currently selects + # proprietary, so it needs its own validation rather than riding a version + # bump. + kernelModuleType: auto maxParallelUpgrades: 5 rdma: # Default off: nvidia_peermem only loads against Mellanox MOFED From e41d1d354efb3358ef3c0e2ce1e34c07bdc4f961 Mon Sep 17 00:00:00 2001 From: Yuan Chen Date: Fri, 28 Aug 2026 12:07:41 -0700 Subject: [PATCH 11/12] fix: make the ComputeDomain guard path-aware and update kernelModuleType consumers MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The guard matched numNodes anywhere in the ComputeDomain document, so moving the key to metadata.numNodes still passed while Kubernetes would reject the absent spec.numNodes. Matching a key without its parent is not a weaker check, it is the wrong check. The scanner now requires numNodes as a DIRECT CHILD of spec, walking indentation because the manifests are Helm templates no YAML parser accepts. Adds table cases covering every shape that has fooled a previous version of this scanner or must keep working: present, absent, under metadata, comment-only, nested under spec.channel, and templated. Plus a multi-document case asserting only the offending document is reported. Replacing useOpenKernelModules also broke four consumers that were not updated with it — a chainsaw bundle assertion (which fails CLI E2E), the OpenAPI response example, the query demo output, and the OCP values comment. Changing a values key requires sweeping its consumers, the same discipline applied to the version strings in this branch. Signed-off-by: Yuan Chen --- api/aicr/v1/server.yaml | 2 +- demos/query.md | 2 +- pkg/recipe/computedomain_numnodes_test.go | 157 +++++++++++++++--- .../components/gpu-operator-ocp/values.yaml | 2 +- .../assert-bundle-scheduling.yaml | 2 +- 5 files changed, 137 insertions(+), 28 deletions(-) diff --git a/api/aicr/v1/server.yaml b/api/aicr/v1/server.yaml index c02974cff..67a0282de 100644 --- a/api/aicr/v1/server.yaml +++ b/api/aicr/v1/server.yaml @@ -650,7 +650,7 @@ paths: value: version: "580.105.08" enabled: true - useOpenKernelModules: true + kernelModuleType: auto "400": description: > Invalid request. Common causes: (1) no criteria provided — all diff --git a/demos/query.md b/demos/query.md index b5a457b8e..c73cc23d5 100644 --- a/demos/query.md +++ b/demos/query.md @@ -53,7 +53,7 @@ enabled: true maxParallelUpgrades: 5 rdma: enabled: false -useOpenKernelModules: true +kernelModuleType: auto version: 595.91.07 ``` diff --git a/pkg/recipe/computedomain_numnodes_test.go b/pkg/recipe/computedomain_numnodes_test.go index b190b3a61..799bd92cc 100644 --- a/pkg/recipe/computedomain_numnodes_test.go +++ b/pkg/recipe/computedomain_numnodes_test.go @@ -21,41 +21,74 @@ import ( "testing" ) -// numNodesKeyRE matches an indented mapping key. Anchored per line so a key -// inside a comment or a quoted error string cannot satisfy it. -var numNodesKeyRE = regexp.MustCompile(`(?m)^[ \t]+numNodes[ \t]*:`) +// specKeyRE matches the document's top-level `spec:` mapping key. +var specKeyRE = regexp.MustCompile(`^(\s*)spec\s*:\s*$`) -// computeDomainDocsMissingNumNodes returns the 0-based indexes of YAML -// documents that declare kind: ComputeDomain without a numNodes key. +// numNodesChildRE matches `numNodes:` at a given exact indentation. +func numNodesChildRE(indent string) *regexp.Regexp { + return regexp.MustCompile(`^` + regexp.QuoteMeta(indent) + `numNodes\s*:`) +} + +// specHasNumNodes reports whether a single YAML document declares numNodes as a +// DIRECT CHILD of spec. // -// Scoped per document rather than per file. A multi-document manifest where one -// ComputeDomain sets numNodes and a second omits it would satisfy a whole-file -// scan while still failing admission, and so would an unrelated resource that -// happens to carry a numNodes key. No such manifest exists in the catalog -// today; the guard is document-scoped so that adding one cannot silently -// bypass it. +// Path-aware on purpose. An earlier version matched `numNodes:` anywhere in the +// document, which accepted `metadata.numNodes` — a key Kubernetes ignores, while +// the required `spec.numNodes` stays absent and admission still fails. Matching +// the key without its parent is not a weaker check, it is the wrong check. // // Comment lines are stripped first: these manifests legitimately discuss -// "spec.numNodes: Required value" in prose, and a naive substring scan matches -// that instead of the real key, passing even when the key is deleted. +// "spec.numNodes: Required value" in prose, and a scan that does not strip them +// matches that instead of the real key, passing even when the key is deleted. +// +// A full YAML parse is unavailable — the manifests are Helm templates containing +// {{ }} expressions that no YAML parser accepts — so this walks indentation. +func specHasNumNodes(doc string) bool { + var lines []string + for _, line := range strings.Split(doc, "\n") { + if strings.HasPrefix(strings.TrimSpace(line), "#") || strings.TrimSpace(line) == "" { + continue + } + lines = append(lines, line) + } + for i, line := range lines { + m := specKeyRE.FindStringSubmatch(line) + if m == nil { + continue + } + specIndent := m[1] + var childRE *regexp.Regexp + for _, sub := range lines[i+1:] { + subIndent := sub[:len(sub)-len(strings.TrimLeft(sub, " \t"))] + // Dedent to spec's level or shallower ends the spec mapping. + if len(subIndent) <= len(specIndent) { + break + } + if childRE == nil { + childRE = numNodesChildRE(subIndent) + } + if childRE.MatchString(sub) { + return true + } + } + } + return false +} + +// computeDomainDocsMissingNumNodes returns the 0-based indexes of YAML +// documents that declare kind: ComputeDomain without spec.numNodes. // -// A full YAML parse is unavailable — the manifests are Helm templates and -// contain {{ }} expressions that no YAML parser accepts. +// Scoped per document: a multi-document manifest where one ComputeDomain sets +// the key and a second omits it would satisfy a whole-file scan while still +// failing admission. No such manifest exists in the catalog today; the guard is +// document-scoped so adding one cannot silently bypass it. func computeDomainDocsMissingNumNodes(content string) []int { var missing []int for i, doc := range strings.Split(content, "\n---") { if !strings.Contains(doc, "kind: ComputeDomain") { continue } - var b strings.Builder - for _, line := range strings.Split(doc, "\n") { - if strings.HasPrefix(strings.TrimSpace(line), "#") { - continue - } - b.WriteString(line) - b.WriteString("\n") - } - if !numNodesKeyRE.MatchString(b.String()) { + if !specHasNumNodes(doc) { missing = append(missing, i) } } @@ -137,3 +170,79 @@ func TestComputeDomainManifestsSetNumNodes(t *testing.T) { } t.Logf("verified %d ComputeDomain manifest(s) set spec.numNodes", checked) } + +// TestComputeDomainScannerCases pins the scanner's behavior directly, so the +// catalog guard above cannot quietly stop discriminating if the catalog changes. +// Each case is a shape that has either fooled a previous version of this +// scanner or must keep working. +func TestComputeDomainScannerCases(t *testing.T) { + t.Parallel() + + const header = "apiVersion: resource.nvidia.com/v1beta1\nkind: ComputeDomain\n" + + tests := []struct { + name string + doc string + wantMissing bool + }{ + { + name: "spec.numNodes present", + doc: header + "metadata:\n name: cd\nspec:\n numNodes: 0\n channel:\n allocationMode: All\n", + }, + { + name: "spec.numNodes absent", + doc: header + "metadata:\n name: cd\nspec:\n channel:\n allocationMode: All\n", + wantMissing: true, + }, + { + // Regression: an earlier scanner matched numNodes anywhere in the + // document, so this passed while admission would still fail. + name: "numNodes under metadata, not spec", + doc: header + "metadata:\n name: cd\n numNodes: 0\nspec:\n channel:\n allocationMode: All\n", + wantMissing: true, + }, + { + // Regression: an earlier scanner did not strip comments, so the + // prose in the real manifest satisfied it even with the key gone. + name: "numNodes only mentioned in a comment", + doc: header + "metadata:\n name: cd\nspec:\n # numNodes: Required value\n channel:\n allocationMode: All\n", + wantMissing: true, + }, + { + name: "nested numNodes does not satisfy the direct-child rule", + doc: header + "metadata:\n name: cd\nspec:\n channel:\n numNodes: 0\n", + // numNodes exists but under spec.channel, not spec. + wantMissing: true, + }, + { + name: "templated value is acceptable", + doc: header + "metadata:\n name: cd\nspec:\n numNodes: {{ .Values.numNodes }}\n", + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + got := !specHasNumNodes(tt.doc) + if got != tt.wantMissing { + t.Errorf("specHasNumNodes reported missing=%v, want %v\ndoc:\n%s", + got, tt.wantMissing, tt.doc) + } + }) + } +} + +// TestComputeDomainMultiDocument covers the per-document scoping: a file where +// one ComputeDomain is valid and a second is not must report only the second. +func TestComputeDomainMultiDocument(t *testing.T) { + t.Parallel() + + content := "apiVersion: v1\nkind: ConfigMap\nmetadata:\n name: unrelated\n" + + "\n---\napiVersion: resource.nvidia.com/v1beta1\nkind: ComputeDomain\nmetadata:\n name: ok\nspec:\n numNodes: 0\n" + + "\n---\napiVersion: resource.nvidia.com/v1beta1\nkind: ComputeDomain\nmetadata:\n name: bad\nspec:\n channel:\n allocationMode: All\n" + + missing := computeDomainDocsMissingNumNodes(content) + if len(missing) != 1 || missing[0] != 2 { + t.Errorf("missing documents = %v, want [2] (only the third document lacks spec.numNodes)", missing) + } +} diff --git a/recipes/components/gpu-operator-ocp/values.yaml b/recipes/components/gpu-operator-ocp/values.yaml index 1f090147d..8b931edbc 100644 --- a/recipes/components/gpu-operator-ocp/values.yaml +++ b/recipes/components/gpu-operator-ocp/values.yaml @@ -160,7 +160,7 @@ daemonsets: # operator.upgradeCRD: Helm chart setting, not a ClusterPolicy field. # operator.resources: Helm chart setting for the operator Deployment, not CR. # driver.version: OCP operator manages driver version via the certified driver container. -# driver.useOpenKernelModules: OCP uses pre-built driver containers from the certified catalog. +# driver.kernelModuleType: OCP uses pre-built driver containers from the certified catalog. # driver.maxParallelUpgrades: Use daemonsets.rollingUpdate.maxUnavailable instead. # devicePlugin.env: OCP sets device plugin env via the operator's own defaults. # validator: OCP operator manages validation internally. diff --git a/tests/chainsaw/cli/cuj1-training/assert-bundle-scheduling.yaml b/tests/chainsaw/cli/cuj1-training/assert-bundle-scheduling.yaml index 858d25528..c38d7c7d3 100644 --- a/tests/chainsaw/cli/cuj1-training/assert-bundle-scheduling.yaml +++ b/tests/chainsaw/cli/cuj1-training/assert-bundle-scheduling.yaml @@ -46,7 +46,7 @@ driver: enabled: true rdma: enabled: false - useOpenKernelModules: true + kernelModuleType: auto # ── GDRCopy: GPU-direct memory for high-performance training ───────── gdrcopy: From 99f8cac965cb78ba2aeb1c3ba719b4752aec1b1d Mon Sep 17 00:00:00 2001 From: Yuan Chen Date: Fri, 28 Aug 2026 15:53:59 -0700 Subject: [PATCH 12/12] fix(recipes): hold the GPU driver at 580.173.02 for P6e EFA compatibility MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit NVIDIA documents R595 as backward incompatible with P6e-GB200 EFA. R595 drops NVreg_GrdmaPciTopoCheckOverride, the R580-era escape hatch, and substitutes a topology check requiring EFA in an IOMMU group or GPU and EFA under a shared root port. p6e-gb200/gb300 satisfies neither. Proven on p6e-gb300r.36xlarge, not inferred: running the NCCL NET workload under 595 with the preflight bypassed, both nodes selected aws-ofi-nccl/EFA/RDMA and then aborted, with the kernel logging "NVRM: dma-buf attach failed: topology not supported for mapping type FORCE_PCIE". The same two nodes passed NCCL NET at 43.16 GB/s under 580.173.02. Scope is the AWS P6e Grace-Blackwell EFA GPUDirect path. NVLS passed under 595 at 840 GB/s, as did deployment and conformance — NVLink never touches EFA. Holding here is a supported combination rather than an untested one: GPU Operator v26.7.0 lists 580.173.02 as supported (595.91.07 is only its default), and DRA driver 0.5.0 requires >= 580. The chart and DRA bumps are retained. This reverses an argument made earlier on this branch, that driver and chart should move together because holding one back invents an untested pairing. That reasoning did not account for the new driver being unsupported on hardware we ship. Signed-off-by: Yuan Chen --- docs/user/container-images.md | 2 +- pkg/bundler/testdata/stock_render_golden.yaml | 92 +++++++++---------- recipes/components/gpu-operator/values.yaml | 41 ++++++--- 3 files changed, 77 insertions(+), 58 deletions(-) diff --git a/docs/user/container-images.md b/docs/user/container-images.md index 351730f98..4098434d0 100644 --- a/docs/user/container-images.md +++ b/docs/user/container-images.md @@ -159,7 +159,7 @@ _No images extracted._ - `nvcr.io/nvidia/cloud-native/nvidia-fs:2.29.4` - `nvcr.io/nvidia/cloud-native/nvidia-sandbox-device-plugin:v0.0.5` - `nvcr.io/nvidia/cloud-native/vgpu-device-manager:v0.5.0` -- `nvcr.io/nvidia/driver:595.91.07` +- `nvcr.io/nvidia/driver:580.173.02` - `nvcr.io/nvidia/gpu-operator:v26.7.0` - `nvcr.io/nvidia/k8s-device-plugin:v0.20.0` - `nvcr.io/nvidia/k8s/container-toolkit:v1.20.0` diff --git a/pkg/bundler/testdata/stock_render_golden.yaml b/pkg/bundler/testdata/stock_render_golden.yaml index 9ce4a1e59..02c850371 100644 --- a/pkg/bundler/testdata/stock_render_golden.yaml +++ b/pkg/bundler/testdata/stock_render_golden.yaml @@ -3,51 +3,51 @@ # # One entry per leaf overlay: a digest over its fully rendered helm-deployer # bundle tree (sorted relative paths paired with per-file content hashes). -a100-aks-ubuntu-training-kubeflow: ba4809f205a8ddf94afa75a8a3623101d69874e477edcf3140890b35015fee46 -a100-any: 67228bd14328a01b06335b6b805ba8f66696cd18c3e6b80a24d9b0b4ea12b551 -a100-eks-ubuntu-training-kubeflow: 77ba19db6c1a621dc703b7efcb74e678e807c11d25964c119e61637780847c00 -a100-gke-cos-training-kubeflow: b00d861db41fb6b3d67c97706ab6ddf7b2ddc7f31d718f868b4e88586d918229 -a100-oke-ubuntu-training-kubeflow: 9e20629fee09306cd88265ad42c838ea8b3df9ad5b4a76e93adaa09552a49809 -b200-any: a0f3f6e9dc5ffcb4d0c96c32b735b885767e99ddd3e0a2447c8ade25a8d8d930 -b200-gke-cos-inference-dynamo: 2402e86bcd3453bc9604693879b7056c938d2b05911104fcca1aeec3413f03be -b200-gke-cos-training-kubeflow: cc5df420bee36fa2af8ab3dc5a690f38ca11ab8d0d3f3bba32d3889c6867a98c -bcm-inference: f06817c3866ae75d3c46c73fae1d121831f7a22ae4cb1edc2e18f9854e1f53cf -gb200-any: 580846f807cf370252c28428bc46ef13293f2303be04795e956d401ae145e108 -gb200-eks-ubuntu-inference-dynamo: 966c6929259615769e2bc887e0952f40ab5e6e50174f97d7409b6d589fe8c81f -gb200-eks-ubuntu-training-kubeflow: 98a44bea88ea04692e3e64e8ac4682095fb2e8f7141473a5bc1b21b2e5d55e50 -gb200-eks-ubuntu-training-slurm: d5f7ec4abdf22825cddf5beaa9410b090add7195b21b6f3eaa3e47f98fba0954 -gb200-oke-ubuntu-inference-dynamo: 1741a806e5b94997adfaebe2224128d78e1741bc94b5e0887f7ef49f0b0a0a4b -gb200-oke-ubuntu-training-kubeflow: ea86c42a1a9a4b51353b0f69b9abe80eaa881f0e0f640e384e9487c199127a8a -gb300-any: e703f2cd97e2aaec3b5f920a795e331e0d8a268e17da2450d688f853280db7a8 -gb300-eks-ubuntu-inference-dynamo: fa2fe0191625a7a5e9498406933154bbbc2f5de8ddf3233a9b64f60b1403070f -gb300-eks-ubuntu-training-kubeflow: ac3dd6280831114cdd367ae37780e12ee16c10b2c575ab67afb334badfe62e0b -h100-aks-ubuntu-inference-dynamo: ccac7522740d262e3ab12af286e7c3d7f58231673a7ded30018cecc5154cfa10 -h100-aks-ubuntu-training-kubeflow: fcd0b8598fade1c7086dd85c7ba301cad67b187294704b456294feac1ad59804 -h100-aks-ubuntu-training-slurm: 983f86e00d4b1423c308604034e3a05f068c3e6dc7cb74426ddeef5f644d1df3 -h100-any: 2c0e0386ec460ca8eeb3593bb22b68db7b95abe6fba51aea50df7f277fdb5503 -h100-bcm-ubuntu-training: eb3eb1572c56f89e7d8108915879f5debf6eb3892563bd12a366d05f5e52e120 -h100-eks-ubuntu-inference-dynamo: 82c84d76b24b45cf3845933cd567f3bbd84d92e5e7105ce3c46396d65000db8f -h100-eks-ubuntu-inference-nim: 415292845ee10056c61890ed5c5eb469c072e0786709c59173d9d7ed7555ecfb -h100-eks-ubuntu-training-kubeflow: 478fc650545d1b0caa9761373fd6ab7765f1c5fe09f6ddecf6ebb71d976f3e05 -h100-eks-ubuntu-training-slurm: 9a444f808d45439f6a8903a03e8610a94444c9fa4783634f0c20f72077e7db5c -h100-gke-cos-inference-dynamo: cb6f151bcd7f5ad7b81ceea35649320ee8c8842236d749d9f6837a833bc1dd60 -h100-gke-cos-training-kubeflow: feae5f125622fa0c472d2cf64cf8c6088cec1160cf95501b98c8fecd404fd734 -h100-gke-cos-training-slurm: 698bbed35f699b9f28abfa13863f38d6dc36fd62180027db0b87cad89cfe13ae -h100-kind-inference-dynamo: b7b33b07faaed415c9a120fb43bf78d58cf606783103b26d3c5edd4f0ed51a18 -h100-kind-training-kubeflow: 4412529fcd051ad8bc389993ca52d9526e1180805ede6e42b29fcbc85a64fd99 -h100-kind-training-slurm: a4f8479c9c210735ac66953e77e1fe4a1c5bd5ae8262178e300ce78e496a44e7 -h200-any: 737f444bf832d184f91f768cf3b58839c1ba36abf5578da79cf4c0938e2f1f07 -h200-eks-inference: d1fac6d45f2eae71ab0a0e51f2e942176b5f6e28a22132de8b9171b74e8e078a -h200-eks-training: cf5add53acd55c0275f0de66589b13bf5943b2c9d48d13f8f389936ddcaad52d -l40s-any: a4385e2dc809867437b44748b26a57ed2bc463d15254f73a532c8fb9dcdcb872 -l40s-oke-inference: 8e7fa89c8a94ddf517f1171cb2b6775ce37d0ab3a86fbc56fdb5324b99a2b493 -l40s-oke-training: 1f3ce9538778d67b32f1ef068000f1c336daaa812c61eb49ba03139c207500f6 -monitoring-hpa: 19bc810c681b40bc40d5f90821feba53cb1a60805f9f47ddcf79e2388e582c01 +a100-aks-ubuntu-training-kubeflow: 179e890bbea38115389a4e16ca19854c0a8e1a8dacd7dd62cae17a57e810aed1 +a100-any: 44c73314feb82b7dcc5c22cfd16d78d928e2dc47d6bdca334968dd9b39d78d0b +a100-eks-ubuntu-training-kubeflow: cd2be5a192926d9f91e439167477a3fe3db6d449c1ea5bf9be65a939f343a347 +a100-gke-cos-training-kubeflow: c8a208cd7df159bc6613d6737b787db3769a6b1255a73c969a98e0f28bff7387 +a100-oke-ubuntu-training-kubeflow: 6705797efa62ae99242fe45309aae39f2c02b44dfc17f536f2f7597b2fd11a7c +b200-any: 83c8acdda32fad2afd441d66fbe7ba3d47565527bd4b630c8e9b669918bd8020 +b200-gke-cos-inference-dynamo: 03b5805d397f75d387b8e8b93fb073638d15e92e2e785f82b6a914d9a402cda2 +b200-gke-cos-training-kubeflow: 5bf0d4e35ccb60d9291db3ad91eae4a6ed1da3c6e3b04e5c8530be5947a25a2f +bcm-inference: 1c0a7bb58b9fe346a58e15ee59c49046c9a2003e1ba27dada7642b56f7419818 +gb200-any: fc0974eabd9041f7e29697303fca0358d9eece203d462e4a63fa21f4843d1d86 +gb200-eks-ubuntu-inference-dynamo: 5a0e6a53d4044c69369169e07c1907c09239d114b7399f0c851f8fcc5275f71a +gb200-eks-ubuntu-training-kubeflow: 8895cdb1981c7c008bc0f631a5c01c1a02f02730c836661f2d4a605ee71f597c +gb200-eks-ubuntu-training-slurm: 12923102f3c7e46670f17214ac5d0d03dc64ebd8b938e7952633ab07c3255c7b +gb200-oke-ubuntu-inference-dynamo: fa5a57601733e82bb763de3234166e3f72fc89543fc6a9721f8e762d75e9a714 +gb200-oke-ubuntu-training-kubeflow: 8499a0e45d1d446bf987a64b4612fba8705d7255dcee08d9e440408a8952fea4 +gb300-any: 6299dd58a7c29d668071ac97adcc6221005816368ecfeca676c74787e3126625 +gb300-eks-ubuntu-inference-dynamo: e0a01e5542dfc641b7ce2b7e4570b7a73efa951597d1af95b8e0b7a54f581b6b +gb300-eks-ubuntu-training-kubeflow: 2300a31efb361f4c46302b1680202c8d63adb083b2c981f6ea5950996b0f6b81 +h100-aks-ubuntu-inference-dynamo: 90e70e9240cbc4f82b387b71804c3a34dcbdad4ad6cec4a765bb8c6a52a9f90d +h100-aks-ubuntu-training-kubeflow: e0b99b6e04b877ffce34635aa75cb52e1adc265962c2173ff52793690f072d94 +h100-aks-ubuntu-training-slurm: 432ddd1c8084e6515c428e6727496ced19f3136b41ba254702d3938c7baf266c +h100-any: b6d3c92581c3b0e86731c36cbcbd420cbd5cc34ac2ba87aeefd765b95a938c9d +h100-bcm-ubuntu-training: 1d143be546c4b2c5fb12eadec4beb8d7297f8706ee9642f83c74b3d600775352 +h100-eks-ubuntu-inference-dynamo: 917a36151dbbdb20145addda2abc9600aeaed3e475c9c99121d9c24437ac016b +h100-eks-ubuntu-inference-nim: eab44cbf067e48785f190e710cf9e4c21ec9b516b10d6a4ab808a7e626b59a1e +h100-eks-ubuntu-training-kubeflow: d473a308bd3399ada67498431ecba245c7a7c8ee1ad9c3a7229bdaac300137c2 +h100-eks-ubuntu-training-slurm: 3de4aa7e2d7589d945e0d5ebb9ea07dde2afb53422354bc5562a375ed3f6b8b9 +h100-gke-cos-inference-dynamo: 8b01f66e9dc3877c8349be6853914666b5ac2a0c901ca8d7f9f389ca7f918df3 +h100-gke-cos-training-kubeflow: 1812fe1059ac056cba9db7248d7ab18a37336acf138ea424a65e88c16693ce7b +h100-gke-cos-training-slurm: f62fbb6425fdaa606ec786f94889deaa15406e525033046bfdcf2e8c699e02c4 +h100-kind-inference-dynamo: 943ef4d74e8efeaf136c8b9f3771f19493c6bd595f4fc5ae1c9806b7d30c912e +h100-kind-training-kubeflow: 5420fbf7f27fa89df22cac52b37e41793b246932cc92ec5306abf91e63abf0fe +h100-kind-training-slurm: 9c1bb61f8fc2afcc772ff1a427611bb0b59732af751357ec7b0149d8d4a70d05 +h200-any: 378bd79e8f5eb801e52ef2d56e1f349c484367e60d8302ce64ace7654e8ef7c2 +h200-eks-inference: 7da11486decd1747a9fda05996be2e5584b6217f2acdf46637dfd9957d9944cd +h200-eks-training: d3c01bff1eace5f71df68470e60be0dac0ccd97b864c8f9ec3e4570117b9dfd4 +l40s-any: 34c6144a769834d1a491050ea4f21052ed00e28183e576eb3150c22888f90e78 +l40s-oke-inference: eb70336d93a39ea95c30275708047a90f8ab29117002735cb36df3740eab55ec +l40s-oke-training: 987818b884ea09e48335427178f30a28de5862ff1407334af56e99e77a31167b +monitoring-hpa: d563d7261ab44da64f6af4a7f9b33a6281797815232990d43014fc7aa360e8b2 ocp-inference-nim: a728bed625f568760bf291d54e066a443e2c141b239dc5fb7f4bc96dcac87c39 ocp-training: 32a047795bcb4c9ea6363e18de6d4f213e6b8c9003106cb4a54ccffd3a6b0ef1 -rtx-pro-6000-any: 39eabfa120d603d57963e4ebbf2ba500dae687dbb06c483d65d541b884f582bb -rtx-pro-6000-eks-ubuntu-inference-dynamo: f44ae470c5229ad94285f12076b4c9d9d3bab9021245bc8f5f85635bd773d857 -rtx-pro-6000-eks-ubuntu-inference-nim: 3fd68742176ba25430952efa41751fe092e16b9904566fa5f21ba4e1a0619605 -rtx-pro-6000-eks-ubuntu-training-kubeflow: 825f138e88d319e325226cfc1686e542f42a8809644e4d82817e9e5b2612a3e7 -rtx-pro-6000-lke-ubuntu-inference: 883e38b844d236e4bf233962a5a9f09fce8388a3966a77cc37d40f11248ee7d3 -rtx-pro-6000-lke-ubuntu-training: c61067119e33798f0e1b91a0f097daf1542b80846b34a99c30225654cefae1ca +rtx-pro-6000-any: dafcf7588df850699cd260e9ddeaff374bef6018e4a713fed1a72791d62ab931 +rtx-pro-6000-eks-ubuntu-inference-dynamo: 754541a260c92f495b17fe4a127829c76c2554a5bbdf58b45c253d444f864e3b +rtx-pro-6000-eks-ubuntu-inference-nim: eb6e7ae0b710db7570ddc12f79f0495bba16c36483a4b305fe5bc9061f314ec7 +rtx-pro-6000-eks-ubuntu-training-kubeflow: 32123c9c89208aa6b916bd0099326c890c8b41697a9339bf2170455a2c87a801 +rtx-pro-6000-lke-ubuntu-inference: 7e3ce1dead651a89ba35bfbf40a43e69a5f37f467df9601727e51ca181beedba +rtx-pro-6000-lke-ubuntu-training: a3e7f3a094d0da80f80496b0da88d81c0678746c272b46ec97f12e3ead4d845b diff --git a/recipes/components/gpu-operator/values.yaml b/recipes/components/gpu-operator/values.yaml index e71d8a7e5..569d102d2 100644 --- a/recipes/components/gpu-operator/values.yaml +++ b/recipes/components/gpu-operator/values.yaml @@ -147,18 +147,37 @@ gfd: enabled: true driver: - # Single global pin covering H100/B200/GB200 EKS. Pinned rather than - # inherited from the chart default so a future chart bump is an explicit - # driver decision instead of a silent fleet-wide driver change, and so - # BOM/attestation inputs stay deterministic. + # HELD at 580.173.02 — deliberately BEHIND the v26.7.0 chart default of + # 595.91.07. Do not advance without EFA validation on P6e hardware. # - # History: the previous pin (580.173.02) was deliberately AHEAD of the - # v26.3.3 chart default (580.126.20) to clear a GB200+EFA floor. This - # value is the v26.7.0 chart default and is far above that floor, so the - # pin and the chart recommendation coincide again — but they are not the - # same thing. If a future chart default regresses below a GB200+EFA - # requirement, hold the pin ahead rather than following the chart. - version: 595.91.07 + # NVIDIA documents R595 as backward incompatible with P6e-GB200 EFA. R595 + # drops NVreg_GrdmaPciTopoCheckOverride (the R580-era escape hatch) and + # substitutes a topology check: EFA must sit in an IOMMU group, or GPU and + # EFA must share a root port. p6e-gb200/gb300 satisfies neither — measured + # on p6e-gb300r.36xlarge, where EFA (0000:97:00.0) has no iommu_group and + # sits under pci0000:84 while the GPU (0000:9d:00.0) is in group 0 under + # pci0000:98. + # + # The failure is proven, not inferred. Running the NCCL NET workload under + # 595 with the preflight bypassed, both nodes selected aws-ofi-nccl/EFA/RDMA + # and then aborted, with the kernel logging: + # NVRM: dma-buf attach failed: topology not supported for mapping type FORCE_PCIE + # Under 580.173.02 on the same two nodes, NCCL NET passed at 43.16 GB/s. + # + # Scope: the AWS P6e Grace-Blackwell + EFA GPUDirect path specifically. NVLS + # passed under 595 (840 GB/s), as did deployment and conformance; NVLink + # never touches EFA. + # + # Holding here is a supported combination, not an untested one: GPU Operator + # v26.7.0 lists 580.173.02 as supported (595.91.07 is merely its default), + # and DRA driver 0.5.0 requires >= 580. Pinned rather than inherited so a + # chart bump is an explicit driver decision, and so BOM/attestation inputs + # stay deterministic. + # + # Do not substitute a newer R580 patch without re-validating EFA — + # 580.173.02 is the exact measured baseline. Per-family overrides, so + # non-P6e families can advance independently, are tracked separately. + version: 580.173.02 enabled: true # Replaces useOpenKernelModules, which the chart deprecated and made a # no-op (identical NOTE in v26.3.3 and v26.7.0 values.yaml), so the old