From 9f6fd4cf949e1554c3b395377c6f7e9e1815cb56 Mon Sep 17 00:00:00 2001 From: Yuan Chen Date: Tue, 25 Aug 2026 10:52:08 -0700 Subject: [PATCH 1/3] fix(validators): pair the TCPXO sidecar with the installer AICR ships The performance validator's GKE H100 TrainingRuntime pinned tcpgpudmarxd-dev v1.0.20, which Google pairs with plugin installer v1.0.14. AICR's recipe deploys installer v1.0.15, whose partner is v1.0.21. That runtime is rendered and run on a live cluster during the performance phase, so the benchmark executed against a pair Google documents as unsupported. Renovate excludes **/testdata/** so the pin could never be updated automatically, and ignoreDeps listed only the installer half of the pair, leaving the daemon free to move on its own. Add the daemon so the coupling is symmetric. Also move the demo workload from the v1.0.14/v1.0.20 pair to v1.0.15/v1.0.21 so it matches the installer the recipe deploys, and update the two doc notes that described it as pinned to the earlier pair. Signed-off-by: Yuan Chen --- .github/renovate.json5 | 5 +++++ demos/workloads/training/gke-nccl-test-tcpxo.yaml | 8 ++++---- docs/integrator/gke-tcpxo-networking.md | 7 ++++--- validators/performance/testdata/h100/gke/runtime.yaml | 2 +- 4 files changed, 14 insertions(+), 8 deletions(-) diff --git a/.github/renovate.json5 b/.github/renovate.json5 index b57b78ebe..723ca33a6 100644 --- a/.github/renovate.json5 +++ b/.github/renovate.json5 @@ -363,9 +363,14 @@ // workload-side daemon (tcpgpudmarxd-dev) must move together; an // independent Renovate bump on this image alone would create // stack version skew. See docs/integrator/gke-tcpxo-networking.md. + // - tcpgpudmarxd-dev: the workload-side half of the same TCPXO pair. Listed + // alongside the installer so the coupling is symmetric — excluding only + // the installer would leave the daemon free to move on its own, which is + // the same stack version skew from the opposite direction. ignoreDeps: [ "602401143452.dkr.ecr.us-west-2.amazonaws.com/eks/aws-efa-k8s-device-plugin", "us-docker.pkg.dev/gce-ai-infra/gpudirect-tcpxo/nccl-plugin-gpudirecttcpx-dev", + "us-docker.pkg.dev/gce-ai-infra/gpudirect-tcpxo/tcpgpudmarxd-dev", ], // Paths Renovate should not scan. diff --git a/demos/workloads/training/gke-nccl-test-tcpxo.yaml b/demos/workloads/training/gke-nccl-test-tcpxo.yaml index 48d936e26..a9f504f6e 100644 --- a/demos/workloads/training/gke-nccl-test-tcpxo.yaml +++ b/demos/workloads/training/gke-nccl-test-tcpxo.yaml @@ -127,7 +127,7 @@ spec: cloud.google.com/gke-accelerator: nvidia-h100-mega-80gb containers: - name: tcpxo-daemon - image: us-docker.pkg.dev/gce-ai-infra/gpudirect-tcpxo/tcpgpudmarxd-dev:v1.0.20 + image: us-docker.pkg.dev/gce-ai-infra/gpudirect-tcpxo/tcpgpudmarxd-dev:v1.0.21 imagePullPolicy: Always command: ["/bin/sh", "-c"] args: @@ -150,7 +150,7 @@ spec: - name: LD_LIBRARY_PATH value: /usr/local/nvidia/lib64 - name: nccl-test - image: us-docker.pkg.dev/gce-ai-infra/gpudirect-tcpxo/nccl-plugin-gpudirecttcpx-dev:v1.0.14 + image: us-docker.pkg.dev/gce-ai-infra/gpudirect-tcpxo/nccl-plugin-gpudirecttcpx-dev:v1.0.15 imagePullPolicy: Always command: - /bin/sh @@ -242,7 +242,7 @@ spec: cloud.google.com/gke-accelerator: nvidia-h100-mega-80gb containers: - name: tcpxo-daemon - image: us-docker.pkg.dev/gce-ai-infra/gpudirect-tcpxo/tcpgpudmarxd-dev:v1.0.20 + image: us-docker.pkg.dev/gce-ai-infra/gpudirect-tcpxo/tcpgpudmarxd-dev:v1.0.21 imagePullPolicy: Always command: ["/bin/sh", "-c"] args: @@ -265,7 +265,7 @@ spec: - name: LD_LIBRARY_PATH value: /usr/local/nvidia/lib64 - name: nccl-test - image: us-docker.pkg.dev/gce-ai-infra/gpudirect-tcpxo/nccl-plugin-gpudirecttcpx-dev:v1.0.14 + image: us-docker.pkg.dev/gce-ai-infra/gpudirect-tcpxo/nccl-plugin-gpudirecttcpx-dev:v1.0.15 imagePullPolicy: Always command: - /bin/sh diff --git a/docs/integrator/gke-tcpxo-networking.md b/docs/integrator/gke-tcpxo-networking.md index 64990ed66..5c21a5a9e 100644 --- a/docs/integrator/gke-tcpxo-networking.md +++ b/docs/integrator/gke-tcpxo-networking.md @@ -179,7 +179,7 @@ Running a **Kubeflow TrainJob** rather than a bare Pod? A TrainJob cannot add the `tcpxo-daemon` sidecar, so the wiring must live in a `TrainingRuntime` — see [Attaching a Training Workload to the Cluster Fabric](../user/fabric-attached-training.md). -See [`demos/workloads/training/gke-nccl-test-tcpxo.yaml`](https://github.com/NVIDIA/aicr/blob/main/demos/workloads/training/gke-nccl-test-tcpxo.yaml) for a complete 2-node NCCL benchmark example. (pinned to an earlier coupled pair, plugin `v1.0.14` with daemon `v1.0.20`) +See [`demos/workloads/training/gke-nccl-test-tcpxo.yaml`](https://github.com/NVIDIA/aicr/blob/main/demos/workloads/training/gke-nccl-test-tcpxo.yaml) for a complete 2-node NCCL benchmark example. (pinned to the same coupled pair the recipe ships, plugin `v1.0.15` with daemon `v1.0.21`) ## NCCL Plugin Version Matching @@ -258,8 +258,9 @@ NRI profile (recommended, no `hostNetwork`): ```shell kubectl create ns nccl-test -# Note: this manifest is pinned to the earlier v1.0.14 / v1.0.20 pair. -# Update both images to your cluster's pair before applying. +# Note: this manifest is pinned to the v1.0.15 / v1.0.21 pair, matching the +# installer the recipe deploys. If your cluster runs a different installer +# version, update both images to that cluster's pair before applying. kubectl apply -f demos/workloads/training/gke-nccl-test-tcpxo.yaml -n nccl-test # Wait for pods to be 2/2 Running diff --git a/validators/performance/testdata/h100/gke/runtime.yaml b/validators/performance/testdata/h100/gke/runtime.yaml index fc034c390..58c7c85b6 100644 --- a/validators/performance/testdata/h100/gke/runtime.yaml +++ b/validators/performance/testdata/h100/gke/runtime.yaml @@ -215,7 +215,7 @@ spec: # alongside the worker. Uses NRI device injection (devices.gke.io # annotation) for GPU device access instead of privileged mode. - name: tcpxo-daemon - image: us-docker.pkg.dev/gce-ai-infra/gpudirect-tcpxo/tcpgpudmarxd-dev:v1.0.20 + image: us-docker.pkg.dev/gce-ai-infra/gpudirect-tcpxo/tcpgpudmarxd-dev:v1.0.21 imagePullPolicy: Always restartPolicy: Always command: ["/bin/sh", "-c"] From d5110a690d3d603690dd720d39bd5fdc2c57dd6d Mon Sep 17 00:00:00 2001 From: Yuan Chen Date: Tue, 25 Aug 2026 11:10:42 -0700 Subject: [PATCH 2/3] docs(demos): tell operators to update both halves of the TCPXO pair The manifest header told the reader to update only the plugin tag when their cluster runs a different installer version. Following it leaves both daemon references untouched, recreating the mismatched pair this change exists to fix. State the coupling, name the pinned pair, and say to update both images, so the manifest carries the same guidance the integrator doc already gives. Also correct the lookup: the installer image lives in initContainers, not containers, so the previous jsonpath returned empty. Signed-off-by: Yuan Chen --- demos/workloads/training/gke-nccl-test-tcpxo.yaml | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/demos/workloads/training/gke-nccl-test-tcpxo.yaml b/demos/workloads/training/gke-nccl-test-tcpxo.yaml index a9f504f6e..af90e6b1f 100644 --- a/demos/workloads/training/gke-nccl-test-tcpxo.yaml +++ b/demos/workloads/training/gke-nccl-test-tcpxo.yaml @@ -32,11 +32,17 @@ # - GPU Operator with driver.enabled=false (GKE COS manages drivers) # # Image version note: -# The nccl-test container image must match the cluster's installed TCPXO -# plugin version. Check with: +# Google ships the plugin installer and the tcpxo-daemon as a coupled release +# pair; running a mismatched pair is unsupported. This manifest is pinned to +# v1.0.15 (nccl-plugin-gpudirecttcpx-dev) with v1.0.21 (tcpgpudmarxd-dev), +# matching the installer the AICR recipe deploys. +# +# If your cluster runs a different installer version, check it with: # kubectl get ds nccl-tcpxo-installer -n kube-system \ -# -o jsonpath='{.spec.template.spec.containers[?(@.name=="nccl-tcpxo-installer")].image}' -# Then update the nccl-plugin-gpudirecttcpx-dev tag below to match. +# -o jsonpath='{.spec.template.spec.initContainers[?(@.name=="nccl-tcpxo-installer")].image}' +# then update BOTH images below to that installer's pair, per Google's +# release table: https://github.com/GoogleCloudPlatform/container-engine-accelerators/blob/master/gpudirect-tcpxo/README.md +# Updating only one half recreates the mismatch this pairing exists to avoid. # # Usage: # kubectl create ns nccl-test From 90354855fc4659cdb774f38e3e88b4bcc1ec7c63 Mon Sep 17 00:00:00 2001 From: Yuan Chen Date: Tue, 25 Aug 2026 15:14:37 -0700 Subject: [PATCH 3/3] chore(renovate): describe the daemon ignoreDeps entry as pre-emptive Review correction: the entry is not an active drift guard today. The kubernetes manager is narrowed to recipes/components/*/manifests, and tcpgpudmarxd-dev appears only in demos/ and validators/**/testdata/, so Renovate never extracts it. The previous comment claimed symmetry with the installer entry, which overstates what the config does now. Keep the entry -- it prevents a future widening of those patterns from letting the daemon move alone -- but say what it actually is. Also unify the wording for the same Google URL: the demo header called it a 'release table' while the integrator doc calls it 'release notes'. Signed-off-by: Yuan Chen --- .github/renovate.json5 | 11 +++++++---- demos/workloads/training/gke-nccl-test-tcpxo.yaml | 2 +- 2 files changed, 8 insertions(+), 5 deletions(-) diff --git a/.github/renovate.json5 b/.github/renovate.json5 index 723ca33a6..bb78c5998 100644 --- a/.github/renovate.json5 +++ b/.github/renovate.json5 @@ -363,10 +363,13 @@ // workload-side daemon (tcpgpudmarxd-dev) must move together; an // independent Renovate bump on this image alone would create // stack version skew. See docs/integrator/gke-tcpxo-networking.md. - // - tcpgpudmarxd-dev: the workload-side half of the same TCPXO pair. Listed - // alongside the installer so the coupling is symmetric — excluding only - // the installer would leave the daemon free to move on its own, which is - // the same stack version skew from the opposite direction. + // - tcpgpudmarxd-dev: the workload-side half of the same TCPXO pair. + // Pre-emptive rather than an active guard today: the kubernetes manager + // above is narrowed to recipes/components/*/manifests, and the daemon + // appears only in demos/ and validators/**/testdata/, so Renovate does not + // extract it at present. Listed so that widening those patterns cannot + // silently let the daemon move on its own — the same stack version skew as + // bumping the installer alone, from the opposite direction. ignoreDeps: [ "602401143452.dkr.ecr.us-west-2.amazonaws.com/eks/aws-efa-k8s-device-plugin", "us-docker.pkg.dev/gce-ai-infra/gpudirect-tcpxo/nccl-plugin-gpudirecttcpx-dev", diff --git a/demos/workloads/training/gke-nccl-test-tcpxo.yaml b/demos/workloads/training/gke-nccl-test-tcpxo.yaml index af90e6b1f..3bfab4c94 100644 --- a/demos/workloads/training/gke-nccl-test-tcpxo.yaml +++ b/demos/workloads/training/gke-nccl-test-tcpxo.yaml @@ -41,7 +41,7 @@ # kubectl get ds nccl-tcpxo-installer -n kube-system \ # -o jsonpath='{.spec.template.spec.initContainers[?(@.name=="nccl-tcpxo-installer")].image}' # then update BOTH images below to that installer's pair, per Google's -# release table: https://github.com/GoogleCloudPlatform/container-engine-accelerators/blob/master/gpudirect-tcpxo/README.md +# release notes: https://github.com/GoogleCloudPlatform/container-engine-accelerators/blob/master/gpudirect-tcpxo/README.md # Updating only one half recreates the mismatch this pairing exists to avoid. # # Usage: