diff --git a/.github/renovate.json5 b/.github/renovate.json5 index b57b78ebe..bb78c5998 100644 --- a/.github/renovate.json5 +++ b/.github/renovate.json5 @@ -363,9 +363,17 @@ // workload-side daemon (tcpgpudmarxd-dev) must move together; an // independent Renovate bump on this image alone would create // stack version skew. See docs/integrator/gke-tcpxo-networking.md. + // - tcpgpudmarxd-dev: the workload-side half of the same TCPXO pair. + // Pre-emptive rather than an active guard today: the kubernetes manager + // above is narrowed to recipes/components/*/manifests, and the daemon + // appears only in demos/ and validators/**/testdata/, so Renovate does not + // extract it at present. Listed so that widening those patterns cannot + // silently let the daemon move on its own — the same stack version skew as + // bumping the installer alone, from the opposite direction. ignoreDeps: [ "602401143452.dkr.ecr.us-west-2.amazonaws.com/eks/aws-efa-k8s-device-plugin", "us-docker.pkg.dev/gce-ai-infra/gpudirect-tcpxo/nccl-plugin-gpudirecttcpx-dev", + "us-docker.pkg.dev/gce-ai-infra/gpudirect-tcpxo/tcpgpudmarxd-dev", ], // Paths Renovate should not scan. diff --git a/demos/workloads/training/gke-nccl-test-tcpxo.yaml b/demos/workloads/training/gke-nccl-test-tcpxo.yaml index 48d936e26..3bfab4c94 100644 --- a/demos/workloads/training/gke-nccl-test-tcpxo.yaml +++ b/demos/workloads/training/gke-nccl-test-tcpxo.yaml @@ -32,11 +32,17 @@ # - GPU Operator with driver.enabled=false (GKE COS manages drivers) # # Image version note: -# The nccl-test container image must match the cluster's installed TCPXO -# plugin version. Check with: +# Google ships the plugin installer and the tcpxo-daemon as a coupled release +# pair; running a mismatched pair is unsupported. This manifest is pinned to +# v1.0.15 (nccl-plugin-gpudirecttcpx-dev) with v1.0.21 (tcpgpudmarxd-dev), +# matching the installer the AICR recipe deploys. +# +# If your cluster runs a different installer version, check it with: # kubectl get ds nccl-tcpxo-installer -n kube-system \ -# -o jsonpath='{.spec.template.spec.containers[?(@.name=="nccl-tcpxo-installer")].image}' -# Then update the nccl-plugin-gpudirecttcpx-dev tag below to match. +# -o jsonpath='{.spec.template.spec.initContainers[?(@.name=="nccl-tcpxo-installer")].image}' +# then update BOTH images below to that installer's pair, per Google's +# release notes: https://github.com/GoogleCloudPlatform/container-engine-accelerators/blob/master/gpudirect-tcpxo/README.md +# Updating only one half recreates the mismatch this pairing exists to avoid. # # Usage: # kubectl create ns nccl-test @@ -127,7 +133,7 @@ spec: cloud.google.com/gke-accelerator: nvidia-h100-mega-80gb containers: - name: tcpxo-daemon - image: us-docker.pkg.dev/gce-ai-infra/gpudirect-tcpxo/tcpgpudmarxd-dev:v1.0.20 + image: us-docker.pkg.dev/gce-ai-infra/gpudirect-tcpxo/tcpgpudmarxd-dev:v1.0.21 imagePullPolicy: Always command: ["/bin/sh", "-c"] args: @@ -150,7 +156,7 @@ spec: - name: LD_LIBRARY_PATH value: /usr/local/nvidia/lib64 - name: nccl-test - image: us-docker.pkg.dev/gce-ai-infra/gpudirect-tcpxo/nccl-plugin-gpudirecttcpx-dev:v1.0.14 + image: us-docker.pkg.dev/gce-ai-infra/gpudirect-tcpxo/nccl-plugin-gpudirecttcpx-dev:v1.0.15 imagePullPolicy: Always command: - /bin/sh @@ -242,7 +248,7 @@ spec: cloud.google.com/gke-accelerator: nvidia-h100-mega-80gb containers: - name: tcpxo-daemon - image: us-docker.pkg.dev/gce-ai-infra/gpudirect-tcpxo/tcpgpudmarxd-dev:v1.0.20 + image: us-docker.pkg.dev/gce-ai-infra/gpudirect-tcpxo/tcpgpudmarxd-dev:v1.0.21 imagePullPolicy: Always command: ["/bin/sh", "-c"] args: @@ -265,7 +271,7 @@ spec: - name: LD_LIBRARY_PATH value: /usr/local/nvidia/lib64 - name: nccl-test - image: us-docker.pkg.dev/gce-ai-infra/gpudirect-tcpxo/nccl-plugin-gpudirecttcpx-dev:v1.0.14 + image: us-docker.pkg.dev/gce-ai-infra/gpudirect-tcpxo/nccl-plugin-gpudirecttcpx-dev:v1.0.15 imagePullPolicy: Always command: - /bin/sh diff --git a/docs/integrator/gke-tcpxo-networking.md b/docs/integrator/gke-tcpxo-networking.md index 64990ed66..5c21a5a9e 100644 --- a/docs/integrator/gke-tcpxo-networking.md +++ b/docs/integrator/gke-tcpxo-networking.md @@ -179,7 +179,7 @@ Running a **Kubeflow TrainJob** rather than a bare Pod? A TrainJob cannot add the `tcpxo-daemon` sidecar, so the wiring must live in a `TrainingRuntime` — see [Attaching a Training Workload to the Cluster Fabric](../user/fabric-attached-training.md). -See [`demos/workloads/training/gke-nccl-test-tcpxo.yaml`](https://github.com/NVIDIA/aicr/blob/main/demos/workloads/training/gke-nccl-test-tcpxo.yaml) for a complete 2-node NCCL benchmark example. (pinned to an earlier coupled pair, plugin `v1.0.14` with daemon `v1.0.20`) +See [`demos/workloads/training/gke-nccl-test-tcpxo.yaml`](https://github.com/NVIDIA/aicr/blob/main/demos/workloads/training/gke-nccl-test-tcpxo.yaml) for a complete 2-node NCCL benchmark example. (pinned to the same coupled pair the recipe ships, plugin `v1.0.15` with daemon `v1.0.21`) ## NCCL Plugin Version Matching @@ -258,8 +258,9 @@ NRI profile (recommended, no `hostNetwork`): ```shell kubectl create ns nccl-test -# Note: this manifest is pinned to the earlier v1.0.14 / v1.0.20 pair. -# Update both images to your cluster's pair before applying. +# Note: this manifest is pinned to the v1.0.15 / v1.0.21 pair, matching the +# installer the recipe deploys. If your cluster runs a different installer +# version, update both images to that cluster's pair before applying. kubectl apply -f demos/workloads/training/gke-nccl-test-tcpxo.yaml -n nccl-test # Wait for pods to be 2/2 Running diff --git a/validators/performance/testdata/h100/gke/runtime.yaml b/validators/performance/testdata/h100/gke/runtime.yaml index fc034c390..58c7c85b6 100644 --- a/validators/performance/testdata/h100/gke/runtime.yaml +++ b/validators/performance/testdata/h100/gke/runtime.yaml @@ -215,7 +215,7 @@ spec: # alongside the worker. Uses NRI device injection (devices.gke.io # annotation) for GPU device access instead of privileged mode. - name: tcpxo-daemon - image: us-docker.pkg.dev/gce-ai-infra/gpudirect-tcpxo/tcpgpudmarxd-dev:v1.0.20 + image: us-docker.pkg.dev/gce-ai-infra/gpudirect-tcpxo/tcpgpudmarxd-dev:v1.0.21 imagePullPolicy: Always restartPolicy: Always command: ["/bin/sh", "-c"]