test(bdd): fix secure multi-region coverage

tests/bdd/features/multi-cluster-helmfile-llm-registration-multiregion.feature

@@ -23,7 +23,8 @@ Feature: Register an LLM worker securely with routers in two local regions
         | addons.llm.requestRouter.discovery.remoteWatchUrls[0]             | https://region-b-watch.nvcf.svc.cluster.local:50071                                    |
         | addons.llm.requestRouter.grpcTls.dnsNames[1]                      | region-b-watch.nvcf.svc.cluster.local                                                  |
         | addons.llm.requestRouter.backendRouter.pylonGrpcDialAddress       | https://llm-request-router.nvcf.svc.cluster.local:50071                                |
-        | addons.llm.pki.dnsNames[2]                                        | *.llm-request-router-region-b-headless.nvcf.svc.cluster.local                          |
+        | addons.llm.pki.dnsNames[2]                                        | region-b-watch.nvcf.svc.cluster.local                                                  |
+        | addons.llm.pki.dnsNames[3]                                        | *.llm-request-router-region-b-headless.nvcf.svc.cluster.local                          |
         | observability.profile                                             | disabled                                                                               |
       And I prepare Helmfile environment "local-bdd-registration-multiregion" for stack "nvcf-compute-plane" from fixture "tests/bdd/fixtures/nvcf-compute-plane-local-bdd-multi.yaml" with values:
         | global.imagePullSecrets[0].name | nvcr-pull-secret                     |
@@ -63,6 +64,9 @@ Feature: Register an LLM worker securely with routers in two local regions
       Then the command exit code should be 0
       When I run command "kubectl --context k3d-ncp-local-cp wait certificate llm-request-router-grpc-tls -n envoy-gateway-system --for=condition=Ready --timeout=5m"
       Then the command exit code should be 0
+      When I run command "kubectl --context k3d-ncp-local-cp get certificate llm-request-router-grpc-tls -n envoy-gateway-system -o jsonpath={.spec.dnsNames}"
+      Then the command exit code should be 0
+      And the command output should contain "region-b-watch.nvcf.svc.cluster.local"
       When I run command "kubectl --context k3d-ncp-local-cp rollout status deployment/llm-request-router -n nvcf --timeout=10m"
       Then the command exit code should be 0
 
@@ -82,7 +86,7 @@ Feature: Register an LLM worker securely with routers in two local regions
       # identities and never relies on a dashed-IP SRV alias.
       When I run command:
         """
-        /bin/bash -c 'set -eu; output=$(grpcurl -max-time 3 -cacert <(kubectl --context k3d-ncp-local-cp get secret stargate-quic-tls -n nvcf -o jsonpath="{.data.ca\.crt}" | base64 -d) -authority region-b-watch.nvcf.svc.cluster.local -import-path src/libraries/rust/stargate/crates/proto/proto -proto stargate.proto 127.0.0.1:50071 stargate.StargateControlPlane/WatchStargates 2>&1 || true); printf "%s" "$output" | grep -Fq "llm-request-router-region-b-0"; printf "%s" "$output" | grep -Fq "llm-request-router-region-b-1"; printf "region-b-statefulset=2 tls=https\n"'
+        /bin/bash -c 'set -eu; output=$(grpcurl -max-time 3 -cacert <(kubectl --context k3d-ncp-local-cp get secret stargate-quic-tls -n nvcf -o jsonpath="{.data.ca\.crt}" | base64 -d) -authority region-b-watch.nvcf.svc.cluster.local -import-path src/libraries/rust/stargate/crates/proto/proto -proto stargate.proto 127.0.0.1:50071 stargate.StargateControlPlane/WatchStargates 2>&1 || true); identities=$(printf "%s\n" "$output" | grep -Eo "llm-request-router-region-b-[0-9]+" | sort -u || true); expected=$(printf "llm-request-router-region-b-0\nllm-request-router-region-b-1\n"); [ "$identities" = "$expected" ]; count=$(printf "%s\n" "$identities" | grep -c .); [ "$count" -eq 2 ]; ! printf "%s" "$output" | grep -Eq "([0-9]{1,3}-){3}[0-9]{1,3}\."; printf "region-b-statefulset=%s tls=https\n" "$count"'
         """
       Then the command exit code should be 0
       And the command output should contain "region-b-statefulset=2 tls=https"

tests/bdd/godog_test.go

@@ -1169,14 +1169,18 @@ func TestMultiClusterHelmfileLLMRegistrationMultiregionFeatureFileWiresToSteps(t
 	t.Setenv("NVCF_CLI", "/usr/bin/nvcf-cli")
 	t.Setenv("REPO_ROOT", "/repo-root-placeholder")
 
+	//revive:disable:line-length-limit Exact feature commands must remain byte-for-byte identical.
 	const (
-		regionAWatchCommand = `/bin/bash -c 'set -eu; output=$(grpcurl -max-time 3 -cacert <(kubectl --context k3d-ncp-local-cp get secret stargate-quic-tls -n nvcf -o jsonpath="{.data.ca\.crt}" | base64 -d) -authority llm-request-router.nvcf.svc.cluster.local -import-path src/libraries/rust/stargate/crates/proto/proto -proto stargate.proto 127.0.0.1:50071 stargate.StargateControlPlane/WatchStargates 2>&1 || true); pods=$(kubectl --context k3d-ncp-local-cp get pods -n nvcf -l app.kubernetes.io/instance=llm-request-router,app.kubernetes.io/name=llm-request-router -o jsonpath="{range .items[*]}{.metadata.name}{\"\\n\"}{end}"); count=0; while IFS= read -r pod; do [ -z "$pod" ] && continue; printf "%s" "$output" | grep -Fq "$pod"; count=$((count + 1)); done <<<"$pods"; [ "$count" -eq 3 ]; printf "%s" "$output" | grep -Fq "https://region-b-watch.nvcf.svc.cluster.local:50071"; printf "region-a-deployment=%s remote-watch=https\n" "$count"'`
-		regionBWatchCommand = `/bin/bash -c 'set -eu; output=$(grpcurl -max-time 3 -cacert <(kubectl --context k3d-ncp-local-cp get secret stargate-quic-tls -n nvcf -o jsonpath="{.data.ca\.crt}" | base64 -d) -authority region-b-watch.nvcf.svc.cluster.local -import-path src/libraries/rust/stargate/crates/proto/proto -proto stargate.proto 127.0.0.1:50071 stargate.StargateControlPlane/WatchStargates 2>&1 || true); printf "%s" "$output" | grep -Fq "llm-request-router-region-b-0"; printf "%s" "$output" | grep -Fq "llm-request-router-region-b-1"; printf "region-b-statefulset=2 tls=https\n"'`
-		pylonMetricsCommand = `/bin/sh -c 'set -eu; for attempt in $(seq 1 120); do row=$(kubectl --context k3d-ncp-local-compute-1 get pods -A -o json | jq -r "[.items[] | select(any(.spec.containers[]?; .name == \"llm-worker\")) | [.metadata.namespace,.metadata.name] | @tsv] | first // empty"); if [ -n "$row" ]; then ns=$(printf "%s" "$row" | cut -f1); pod=$(printf "%s" "$row" | cut -f2); metrics=$(kubectl --context k3d-ncp-local-compute-1 get --raw "/api/v1/namespaces/$ns/pods/$pod:9089/proxy/metrics" 2>/dev/null || true); registration=$(printf "%s\n" "$metrics" | grep -c "^pylon_registration_stream_connected.* 1$" || true); reverse=$(printf "%s\n" "$metrics" | grep -c "^pylon_reverse_tunnel_connected.* 1$" || true); if [ "$registration" -eq 5 ] && [ "$reverse" -ge 3 ]; then printf "registration=%s reverse=%s regions=2\n" "$registration" "$reverse"; exit 0; fi; fi; sleep 5; done; exit 1'`
-		invokeCommand       = "/usr/bin/nvcf-cli --config /repo-root-placeholder/tests/bdd/fixtures/nvcf-cli-local.yaml function invoke" +
+		regionAWatchCommand    = `/bin/bash -c 'set -eu; output=$(grpcurl -max-time 3 -cacert <(kubectl --context k3d-ncp-local-cp get secret stargate-quic-tls -n nvcf -o jsonpath="{.data.ca\.crt}" | base64 -d) -authority llm-request-router.nvcf.svc.cluster.local -import-path src/libraries/rust/stargate/crates/proto/proto -proto stargate.proto 127.0.0.1:50071 stargate.StargateControlPlane/WatchStargates 2>&1 || true); pods=$(kubectl --context k3d-ncp-local-cp get pods -n nvcf -l app.kubernetes.io/instance=llm-request-router,app.kubernetes.io/name=llm-request-router -o jsonpath="{range .items[*]}{.metadata.name}{\"\\n\"}{end}"); count=0; while IFS= read -r pod; do [ -z "$pod" ] && continue; printf "%s" "$output" | grep -Fq "$pod"; count=$((count + 1)); done <<<"$pods"; [ "$count" -eq 3 ]; printf "%s" "$output" | grep -Fq "https://region-b-watch.nvcf.svc.cluster.local:50071"; printf "region-a-deployment=%s remote-watch=https\n" "$count"'`
+		regionBWatchCommand    = `/bin/bash -c 'set -eu; output=$(grpcurl -max-time 3 -cacert <(kubectl --context k3d-ncp-local-cp get secret stargate-quic-tls -n nvcf -o jsonpath="{.data.ca\.crt}" | base64 -d) -authority region-b-watch.nvcf.svc.cluster.local -import-path src/libraries/rust/stargate/crates/proto/proto -proto stargate.proto 127.0.0.1:50071 stargate.StargateControlPlane/WatchStargates 2>&1 || true); identities=$(printf "%s\n" "$output" | grep -Eo "llm-request-router-region-b-[0-9]+" | sort -u || true); expected=$(printf "llm-request-router-region-b-0\nllm-request-router-region-b-1\n"); [ "$identities" = "$expected" ]; count=$(printf "%s\n" "$identities" | grep -c .); [ "$count" -eq 2 ]; ! printf "%s" "$output" | grep -Eq "([0-9]{1,3}-){3}[0-9]{1,3}\."; printf "region-b-statefulset=%s tls=https\n" "$count"'`
+		pylonMetricsCommand    = `/bin/sh -c 'set -eu; for attempt in $(seq 1 120); do row=$(kubectl --context k3d-ncp-local-compute-1 get pods -A -o json | jq -r "[.items[] | select(any(.spec.containers[]?; .name == \"llm-worker\")) | [.metadata.namespace,.metadata.name] | @tsv] | first // empty"); if [ -n "$row" ]; then ns=$(printf "%s" "$row" | cut -f1); pod=$(printf "%s" "$row" | cut -f2); metrics=$(kubectl --context k3d-ncp-local-compute-1 get --raw "/api/v1/namespaces/$ns/pods/$pod:9089/proxy/metrics" 2>/dev/null || true); registration=$(printf "%s\n" "$metrics" | grep -c "^pylon_registration_stream_connected.* 1$" || true); reverse=$(printf "%s\n" "$metrics" | grep -c "^pylon_reverse_tunnel_connected.* 1$" || true); if [ "$registration" -eq 5 ] && [ "$reverse" -ge 3 ]; then printf "registration=%s reverse=%s regions=2\n" "$registration" "$reverse"; exit 0; fi; fi; sleep 5; done; exit 1'`
+		grpcCertificateCommand = "kubectl --context k3d-ncp-local-cp get certificate llm-request-router-grpc-tls" +
+			" -n envoy-gateway-system -o jsonpath={.spec.dnsNames}"
+		invokeCommand = "/usr/bin/nvcf-cli --config /repo-root-placeholder/tests/bdd/fixtures/nvcf-cli-local.yaml function invoke" +
 			" --inference-url /v1/chat/completions --model-name openai-compatible-sample" +
 			" --request-body '{\"messages\":[{\"role\":\"user\",\"content\":\"bdd-registration-multiregion\"}]}' --timeout 120"
 	)
+	//revive:enable:line-length-limit
 
 	suite := newWiringSuite(t, newFakeRunner(map[string]harness.Result{
 		"k3d cluster get ncp-local": {ExitCode: 1},
@@ -1188,6 +1192,10 @@ func TestMultiClusterHelmfileLLMRegistrationMultiregionFeatureFileWiresToSteps(t
 			ExitCode: 0,
 			Stdout:   "region-b-statefulset=2 tls=https\n",
 		},
+		grpcCertificateCommand: {
+			ExitCode: 0,
+			Stdout:   "[llm-request-router.nvcf.svc.cluster.local region-b-watch.nvcf.svc.cluster.local]",
+		},
 		pylonMetricsCommand: {ExitCode: 0, Stdout: "registration=5 reverse=3 regions=2\n"},
 		invokeCommand: {
 			ExitCode: 0,
@@ -1228,7 +1236,12 @@ func TestMultiClusterHelmfileLLMRegistrationMultiregionFeatureFileWiresToSteps(t
 	if status != 0 {
 		t.Fatalf("godog suite status = %d\n%s", status, out.String())
 	}
-	for _, command := range []string{regionAWatchCommand, regionBWatchCommand, pylonMetricsCommand} {
+	for _, command := range []string{
+		grpcCertificateCommand,
+		regionAWatchCommand,
+		regionBWatchCommand,
+		pylonMetricsCommand,
+	} {
 		if !commandRanExactly(suite.Runner.(*fakeRunner).runs, command) {
 			t.Fatalf("exact multi-region observation command was not invoked: %s", command)
 		}
@@ -1241,6 +1254,31 @@ func TestMultiClusterHelmfileLLMRegistrationMultiregionFeatureFileWiresToSteps(t
 	) {
 		t.Fatal("multi-region sample was not created as an LLM function")
 	}
+
+	environmentPath, err := dsl.HelmfileEnvironmentPath(
+		suite.Config.RepoRoot,
+		"self-managed",
+		"local-bdd-registration-multiregion",
+	)
+	if err != nil {
+		t.Fatalf("resolve multi-region environment: %v", err)
+	}
+	for _, expectation := range []struct {
+		key  string
+		want string
+	}{
+		{key: "addons.llm.requestRouter.grpcTls.dnsNames[1]", want: "region-b-watch.nvcf.svc.cluster.local"},
+		{key: "addons.llm.pki.dnsNames[2]", want: "region-b-watch.nvcf.svc.cluster.local"},
+		{key: "addons.llm.pki.dnsNames[3]", want: "*.llm-request-router-region-b-headless.nvcf.svc.cluster.local"},
+	} {
+		got, found, readErr := dsl.ReadYAMLKey(environmentPath, expectation.key)
+		if readErr != nil {
+			t.Fatalf("read %s: %v", expectation.key, readErr)
+		}
+		if !found || got != expectation.want {
+			t.Fatalf("%s = %q, found %t, want %q", expectation.key, got, found, expectation.want)
+		}
+	}
 }
 
 // TestSingleClusterHelmfileUpstreamImagesFeatureFileWiresToSteps runs the

tests/bdd/region_b_script_test.go

@@ -0,0 +1,101 @@
+/*
+SPDX-FileCopyrightText: Copyright (c) NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+SPDX-License-Identifier: Apache-2.0
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package bdd_tmp
+
+import (
+	"os"
+	"os/exec"
+	"path/filepath"
+	"strings"
+	"testing"
+)
+
+func TestInstallLLMRegionBCreatesWatchAliasInBothClusters(t *testing.T) {
+	binDir := t.TempDir()
+	applyDir := t.TempDir()
+
+	helmScript := `#!/usr/bin/env bash
+set -euo pipefail
+case " $* " in
+  *" get values "*) printf '{"llmRequestRouter":{}}\n' ;;
+  *) cat >/dev/null ;;
+esac
+`
+	kubectlScript := `#!/usr/bin/env bash
+set -euo pipefail
+context=""
+previous=""
+for argument in "$@"; do
+  if [[ "${previous}" == "--context" ]]; then
+    context="${argument}"
+  fi
+  previous="${argument}"
+done
+case " $* " in
+  *" get endpoints llm-request-router "*) printf '192.0.2.10' ;;
+  *" apply -f - "*)
+    cat >>"${FAKE_APPLY_DIR}/${context}.yaml"
+    printf '\n---\n' >>"${FAKE_APPLY_DIR}/${context}.yaml"
+    ;;
+esac
+`
+	jqScript := `#!/usr/bin/env bash
+set -euo pipefail
+cat
+`
+	for name, body := range map[string]string{
+		"helm":    helmScript,
+		"jq":      jqScript,
+		"kubectl": kubectlScript,
+	} {
+		if err := os.WriteFile(filepath.Join(binDir, name), []byte(body), 0o755); err != nil {
+			t.Fatalf("write fake %s: %v", name, err)
+		}
+	}
+
+	cmd := exec.Command("bash", "scripts/install-llm-region-b.sh")
+	cmd.Env = append(os.Environ(),
+		"CONTROL_CONTEXT=bdd-control",
+		"COMPUTE_CONTEXT=bdd-compute",
+		"FAKE_APPLY_DIR="+applyDir,
+		"PATH="+binDir+":"+os.Getenv("PATH"),
+		"REPO_ROOT="+t.TempDir(),
+	)
+	if output, err := cmd.CombinedOutput(); err != nil {
+		t.Fatalf("install region B: %v\n%s", err, output)
+	}
+
+	for _, context := range []string{"bdd-control", "bdd-compute"} {
+		manifestPath := filepath.Join(applyDir, context+".yaml")
+		manifest, err := os.ReadFile(manifestPath)
+		if err != nil {
+			t.Fatalf("read %s aliases: %v", context, err)
+		}
+		for _, want := range []string{
+			"kind: Service\nmetadata:\n  name: region-b-watch",
+			"kind: Endpoints\nmetadata:\n  name: region-b-watch",
+			"- ip: 192.0.2.10",
+			"name: llm-grpc",
+			"name: llm-quic",
+		} {
+			if !strings.Contains(string(manifest), want) {
+				t.Fatalf("%s aliases missing %q:\n%s", context, want, manifest)
+			}
+		}
+	}
+}

tests/bdd/scripts/install-llm-region-b.sh

@@ -98,7 +98,8 @@ if [[ -z "${control_plane_ip}" ]]; then
   exit 1
 fi
 
-kubectl --context "${compute_context}" apply -f - <<YAML
+for alias_context in "${control_context}" "${compute_context}"; do
+  kubectl --context "${alias_context}" apply -f - <<YAML
 apiVersion: v1
 kind: Service
 metadata:
@@ -131,6 +132,7 @@ subsets:
         port: 50072
         protocol: UDP
 YAML
+done
 
 kubectl --context "${control_context}" rollout status \
   statefulset/llm-request-router-region-b --namespace "${namespace}" --timeout=10m