From ada50828cdb228a0967496354b6a3dc3e3f5cbda Mon Sep 17 00:00:00 2001 From: Mike Camp Date: Fri, 28 Aug 2026 14:41:06 -0400 Subject: [PATCH 1/2] test(bdd): fix multi-region registration coverage Relates to #1307 Signed-off-by: Mike Camp --- ...mfile-llm-registration-multiregion.feature | 6 +- tests/bdd/godog_test.go | 36 ++++++- tests/bdd/region_b_script_test.go | 101 ++++++++++++++++++ tests/bdd/scripts/install-llm-region-b.sh | 4 +- 4 files changed, 142 insertions(+), 5 deletions(-) create mode 100644 tests/bdd/region_b_script_test.go diff --git a/tests/bdd/features/multi-cluster-helmfile-llm-registration-multiregion.feature b/tests/bdd/features/multi-cluster-helmfile-llm-registration-multiregion.feature index 011c936f1..a493bd217 100644 --- a/tests/bdd/features/multi-cluster-helmfile-llm-registration-multiregion.feature +++ b/tests/bdd/features/multi-cluster-helmfile-llm-registration-multiregion.feature @@ -21,9 +21,9 @@ Feature: Register an LLM worker securely with routers in two local regions | global.workerEndpoints.llmRequestRouterAddress | https://llm-request-router.nvcf.svc.cluster.local:50071 | | addons.llm.requestRouter.workload.kind | Deployment | | addons.llm.requestRouter.discovery.remoteWatchUrls[0] | https://region-b-watch.nvcf.svc.cluster.local:50071 | - | addons.llm.requestRouter.grpcTls.dnsNames[1] | region-b-watch.nvcf.svc.cluster.local | | addons.llm.requestRouter.backendRouter.pylonGrpcDialAddress | https://llm-request-router.nvcf.svc.cluster.local:50071 | - | addons.llm.pki.dnsNames[2] | *.llm-request-router-region-b-headless.nvcf.svc.cluster.local | + | addons.llm.pki.dnsNames[2] | region-b-watch.nvcf.svc.cluster.local | + | addons.llm.pki.dnsNames[3] | *.llm-request-router-region-b-headless.nvcf.svc.cluster.local | | observability.profile | disabled | And I prepare Helmfile environment "local-bdd-registration-multiregion" for stack "nvcf-compute-plane" from fixture "tests/bdd/fixtures/nvcf-compute-plane-local-bdd-multi.yaml" with values: | global.imagePullSecrets[0].name | nvcr-pull-secret | @@ -82,7 +82,7 @@ Feature: Register an LLM worker securely with routers in two local regions # identities and never relies on a dashed-IP SRV alias. When I run command: """ - /bin/bash -c 'set -eu; output=$(grpcurl -max-time 3 -cacert <(kubectl --context k3d-ncp-local-cp get secret stargate-quic-tls -n nvcf -o jsonpath="{.data.ca\.crt}" | base64 -d) -authority region-b-watch.nvcf.svc.cluster.local -import-path src/libraries/rust/stargate/crates/proto/proto -proto stargate.proto 127.0.0.1:50071 stargate.StargateControlPlane/WatchStargates 2>&1 || true); printf "%s" "$output" | grep -Fq "llm-request-router-region-b-0"; printf "%s" "$output" | grep -Fq "llm-request-router-region-b-1"; printf "region-b-statefulset=2 tls=https\n"' + /bin/bash -c 'set -eu; output=$(grpcurl -max-time 3 -cacert <(kubectl --context k3d-ncp-local-cp get secret stargate-quic-tls -n nvcf -o jsonpath="{.data.ca\.crt}" | base64 -d) -authority region-b-watch.nvcf.svc.cluster.local -import-path src/libraries/rust/stargate/crates/proto/proto -proto stargate.proto 127.0.0.1:50071 stargate.StargateControlPlane/WatchStargates 2>&1 || true); identities=$(printf "%s\n" "$output" | grep -Eo "llm-request-router-region-b-[0-9]+" | sort -u || true); expected=$(printf "llm-request-router-region-b-0\nllm-request-router-region-b-1\n"); [ "$identities" = "$expected" ]; count=$(printf "%s\n" "$identities" | grep -c .); [ "$count" -eq 2 ]; ! printf "%s" "$output" | grep -Eq "([0-9]{1,3}-){3}[0-9]{1,3}\."; printf "region-b-statefulset=%s tls=https\n" "$count"' """ Then the command exit code should be 0 And the command output should contain "region-b-statefulset=2 tls=https" diff --git a/tests/bdd/godog_test.go b/tests/bdd/godog_test.go index b9db57dcb..6a903b191 100644 --- a/tests/bdd/godog_test.go +++ b/tests/bdd/godog_test.go @@ -1169,14 +1169,16 @@ func TestMultiClusterHelmfileLLMRegistrationMultiregionFeatureFileWiresToSteps(t t.Setenv("NVCF_CLI", "/usr/bin/nvcf-cli") t.Setenv("REPO_ROOT", "/repo-root-placeholder") + //revive:disable:line-length-limit Exact feature commands must remain byte-for-byte identical. const ( regionAWatchCommand = `/bin/bash -c 'set -eu; output=$(grpcurl -max-time 3 -cacert <(kubectl --context k3d-ncp-local-cp get secret stargate-quic-tls -n nvcf -o jsonpath="{.data.ca\.crt}" | base64 -d) -authority llm-request-router.nvcf.svc.cluster.local -import-path src/libraries/rust/stargate/crates/proto/proto -proto stargate.proto 127.0.0.1:50071 stargate.StargateControlPlane/WatchStargates 2>&1 || true); pods=$(kubectl --context k3d-ncp-local-cp get pods -n nvcf -l app.kubernetes.io/instance=llm-request-router,app.kubernetes.io/name=llm-request-router -o jsonpath="{range .items[*]}{.metadata.name}{\"\\n\"}{end}"); count=0; while IFS= read -r pod; do [ -z "$pod" ] && continue; printf "%s" "$output" | grep -Fq "$pod"; count=$((count + 1)); done <<<"$pods"; [ "$count" -eq 3 ]; printf "%s" "$output" | grep -Fq "https://region-b-watch.nvcf.svc.cluster.local:50071"; printf "region-a-deployment=%s remote-watch=https\n" "$count"'` - regionBWatchCommand = `/bin/bash -c 'set -eu; output=$(grpcurl -max-time 3 -cacert <(kubectl --context k3d-ncp-local-cp get secret stargate-quic-tls -n nvcf -o jsonpath="{.data.ca\.crt}" | base64 -d) -authority region-b-watch.nvcf.svc.cluster.local -import-path src/libraries/rust/stargate/crates/proto/proto -proto stargate.proto 127.0.0.1:50071 stargate.StargateControlPlane/WatchStargates 2>&1 || true); printf "%s" "$output" | grep -Fq "llm-request-router-region-b-0"; printf "%s" "$output" | grep -Fq "llm-request-router-region-b-1"; printf "region-b-statefulset=2 tls=https\n"'` + regionBWatchCommand = `/bin/bash -c 'set -eu; output=$(grpcurl -max-time 3 -cacert <(kubectl --context k3d-ncp-local-cp get secret stargate-quic-tls -n nvcf -o jsonpath="{.data.ca\.crt}" | base64 -d) -authority region-b-watch.nvcf.svc.cluster.local -import-path src/libraries/rust/stargate/crates/proto/proto -proto stargate.proto 127.0.0.1:50071 stargate.StargateControlPlane/WatchStargates 2>&1 || true); identities=$(printf "%s\n" "$output" | grep -Eo "llm-request-router-region-b-[0-9]+" | sort -u || true); expected=$(printf "llm-request-router-region-b-0\nllm-request-router-region-b-1\n"); [ "$identities" = "$expected" ]; count=$(printf "%s\n" "$identities" | grep -c .); [ "$count" -eq 2 ]; ! printf "%s" "$output" | grep -Eq "([0-9]{1,3}-){3}[0-9]{1,3}\."; printf "region-b-statefulset=%s tls=https\n" "$count"'` pylonMetricsCommand = `/bin/sh -c 'set -eu; for attempt in $(seq 1 120); do row=$(kubectl --context k3d-ncp-local-compute-1 get pods -A -o json | jq -r "[.items[] | select(any(.spec.containers[]?; .name == \"llm-worker\")) | [.metadata.namespace,.metadata.name] | @tsv] | first // empty"); if [ -n "$row" ]; then ns=$(printf "%s" "$row" | cut -f1); pod=$(printf "%s" "$row" | cut -f2); metrics=$(kubectl --context k3d-ncp-local-compute-1 get --raw "/api/v1/namespaces/$ns/pods/$pod:9089/proxy/metrics" 2>/dev/null || true); registration=$(printf "%s\n" "$metrics" | grep -c "^pylon_registration_stream_connected.* 1$" || true); reverse=$(printf "%s\n" "$metrics" | grep -c "^pylon_reverse_tunnel_connected.* 1$" || true); if [ "$registration" -eq 5 ] && [ "$reverse" -ge 3 ]; then printf "registration=%s reverse=%s regions=2\n" "$registration" "$reverse"; exit 0; fi; fi; sleep 5; done; exit 1'` invokeCommand = "/usr/bin/nvcf-cli --config /repo-root-placeholder/tests/bdd/fixtures/nvcf-cli-local.yaml function invoke" + " --inference-url /v1/chat/completions --model-name openai-compatible-sample" + " --request-body '{\"messages\":[{\"role\":\"user\",\"content\":\"bdd-registration-multiregion\"}]}' --timeout 120" ) + //revive:enable:line-length-limit suite := newWiringSuite(t, newFakeRunner(map[string]harness.Result{ "k3d cluster get ncp-local": {ExitCode: 1}, @@ -1241,6 +1243,38 @@ func TestMultiClusterHelmfileLLMRegistrationMultiregionFeatureFileWiresToSteps(t ) { t.Fatal("multi-region sample was not created as an LLM function") } + + environmentPath, err := dsl.HelmfileEnvironmentPath( + suite.Config.RepoRoot, + "self-managed", + "local-bdd-registration-multiregion", + ) + if err != nil { + t.Fatalf("resolve multi-region environment: %v", err) + } + for _, expectation := range []struct { + key string + want string + }{ + {key: "addons.llm.pki.dnsNames[2]", want: "region-b-watch.nvcf.svc.cluster.local"}, + {key: "addons.llm.pki.dnsNames[3]", want: "*.llm-request-router-region-b-headless.nvcf.svc.cluster.local"}, + } { + got, found, readErr := dsl.ReadYAMLKey(environmentPath, expectation.key) + if readErr != nil { + t.Fatalf("read %s: %v", expectation.key, readErr) + } + if !found || got != expectation.want { + t.Fatalf("%s = %q, found %t, want %q", expectation.key, got, found, expectation.want) + } + } + if _, found, readErr := dsl.ReadYAMLKey( + environmentPath, + "addons.llm.requestRouter.grpcTls.dnsNames[1]", + ); readErr != nil { + t.Fatalf("read unused request-router gRPC TLS key: %v", readErr) + } else if found { + t.Fatal("multi-region environment populated the unused request-router gRPC TLS SAN key") + } } // TestSingleClusterHelmfileUpstreamImagesFeatureFileWiresToSteps runs the diff --git a/tests/bdd/region_b_script_test.go b/tests/bdd/region_b_script_test.go new file mode 100644 index 000000000..23b6e3044 --- /dev/null +++ b/tests/bdd/region_b_script_test.go @@ -0,0 +1,101 @@ +/* +SPDX-FileCopyrightText: Copyright (c) NVIDIA CORPORATION & AFFILIATES. All rights reserved. +SPDX-License-Identifier: Apache-2.0 + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package bdd_tmp + +import ( + "os" + "os/exec" + "path/filepath" + "strings" + "testing" +) + +func TestInstallLLMRegionBCreatesWatchAliasInBothClusters(t *testing.T) { + binDir := t.TempDir() + applyDir := t.TempDir() + + helmScript := `#!/usr/bin/env bash +set -euo pipefail +case " $* " in + *" get values "*) printf '{"llmRequestRouter":{}}\n' ;; + *) cat >/dev/null ;; +esac +` + kubectlScript := `#!/usr/bin/env bash +set -euo pipefail +context="" +previous="" +for argument in "$@"; do + if [[ "${previous}" == "--context" ]]; then + context="${argument}" + fi + previous="${argument}" +done +case " $* " in + *" get endpoints llm-request-router "*) printf '192.0.2.10' ;; + *" apply -f - "*) + cat >>"${FAKE_APPLY_DIR}/${context}.yaml" + printf '\n---\n' >>"${FAKE_APPLY_DIR}/${context}.yaml" + ;; +esac +` + jqScript := `#!/usr/bin/env bash +set -euo pipefail +cat +` + for name, body := range map[string]string{ + "helm": helmScript, + "jq": jqScript, + "kubectl": kubectlScript, + } { + if err := os.WriteFile(filepath.Join(binDir, name), []byte(body), 0o755); err != nil { + t.Fatalf("write fake %s: %v", name, err) + } + } + + cmd := exec.Command("bash", "scripts/install-llm-region-b.sh") + cmd.Env = append(os.Environ(), + "CONTROL_CONTEXT=bdd-control", + "COMPUTE_CONTEXT=bdd-compute", + "FAKE_APPLY_DIR="+applyDir, + "PATH="+binDir+":"+os.Getenv("PATH"), + "REPO_ROOT="+t.TempDir(), + ) + if output, err := cmd.CombinedOutput(); err != nil { + t.Fatalf("install region B: %v\n%s", err, output) + } + + for _, context := range []string{"bdd-control", "bdd-compute"} { + manifestPath := filepath.Join(applyDir, context+".yaml") + manifest, err := os.ReadFile(manifestPath) + if err != nil { + t.Fatalf("read %s aliases: %v", context, err) + } + for _, want := range []string{ + "kind: Service\nmetadata:\n name: region-b-watch", + "kind: Endpoints\nmetadata:\n name: region-b-watch", + "- ip: 192.0.2.10", + "name: llm-grpc", + "name: llm-quic", + } { + if !strings.Contains(string(manifest), want) { + t.Fatalf("%s aliases missing %q:\n%s", context, want, manifest) + } + } + } +} diff --git a/tests/bdd/scripts/install-llm-region-b.sh b/tests/bdd/scripts/install-llm-region-b.sh index 9b1ae9931..26a7e03f0 100755 --- a/tests/bdd/scripts/install-llm-region-b.sh +++ b/tests/bdd/scripts/install-llm-region-b.sh @@ -98,7 +98,8 @@ if [[ -z "${control_plane_ip}" ]]; then exit 1 fi -kubectl --context "${compute_context}" apply -f - < Date: Fri, 28 Aug 2026 15:17:28 -0400 Subject: [PATCH 2/2] test(bdd): verify region watch gateway SAN Relates to #1307 Signed-off-by: Mike Camp --- ...mfile-llm-registration-multiregion.feature | 4 +++ tests/bdd/godog_test.go | 30 +++++++++++-------- 2 files changed, 21 insertions(+), 13 deletions(-) diff --git a/tests/bdd/features/multi-cluster-helmfile-llm-registration-multiregion.feature b/tests/bdd/features/multi-cluster-helmfile-llm-registration-multiregion.feature index a493bd217..d06b58af0 100644 --- a/tests/bdd/features/multi-cluster-helmfile-llm-registration-multiregion.feature +++ b/tests/bdd/features/multi-cluster-helmfile-llm-registration-multiregion.feature @@ -21,6 +21,7 @@ Feature: Register an LLM worker securely with routers in two local regions | global.workerEndpoints.llmRequestRouterAddress | https://llm-request-router.nvcf.svc.cluster.local:50071 | | addons.llm.requestRouter.workload.kind | Deployment | | addons.llm.requestRouter.discovery.remoteWatchUrls[0] | https://region-b-watch.nvcf.svc.cluster.local:50071 | + | addons.llm.requestRouter.grpcTls.dnsNames[1] | region-b-watch.nvcf.svc.cluster.local | | addons.llm.requestRouter.backendRouter.pylonGrpcDialAddress | https://llm-request-router.nvcf.svc.cluster.local:50071 | | addons.llm.pki.dnsNames[2] | region-b-watch.nvcf.svc.cluster.local | | addons.llm.pki.dnsNames[3] | *.llm-request-router-region-b-headless.nvcf.svc.cluster.local | @@ -63,6 +64,9 @@ Feature: Register an LLM worker securely with routers in two local regions Then the command exit code should be 0 When I run command "kubectl --context k3d-ncp-local-cp wait certificate llm-request-router-grpc-tls -n envoy-gateway-system --for=condition=Ready --timeout=5m" Then the command exit code should be 0 + When I run command "kubectl --context k3d-ncp-local-cp get certificate llm-request-router-grpc-tls -n envoy-gateway-system -o jsonpath={.spec.dnsNames}" + Then the command exit code should be 0 + And the command output should contain "region-b-watch.nvcf.svc.cluster.local" When I run command "kubectl --context k3d-ncp-local-cp rollout status deployment/llm-request-router -n nvcf --timeout=10m" Then the command exit code should be 0 diff --git a/tests/bdd/godog_test.go b/tests/bdd/godog_test.go index 6a903b191..6ce8ae8c6 100644 --- a/tests/bdd/godog_test.go +++ b/tests/bdd/godog_test.go @@ -1171,10 +1171,12 @@ func TestMultiClusterHelmfileLLMRegistrationMultiregionFeatureFileWiresToSteps(t //revive:disable:line-length-limit Exact feature commands must remain byte-for-byte identical. const ( - regionAWatchCommand = `/bin/bash -c 'set -eu; output=$(grpcurl -max-time 3 -cacert <(kubectl --context k3d-ncp-local-cp get secret stargate-quic-tls -n nvcf -o jsonpath="{.data.ca\.crt}" | base64 -d) -authority llm-request-router.nvcf.svc.cluster.local -import-path src/libraries/rust/stargate/crates/proto/proto -proto stargate.proto 127.0.0.1:50071 stargate.StargateControlPlane/WatchStargates 2>&1 || true); pods=$(kubectl --context k3d-ncp-local-cp get pods -n nvcf -l app.kubernetes.io/instance=llm-request-router,app.kubernetes.io/name=llm-request-router -o jsonpath="{range .items[*]}{.metadata.name}{\"\\n\"}{end}"); count=0; while IFS= read -r pod; do [ -z "$pod" ] && continue; printf "%s" "$output" | grep -Fq "$pod"; count=$((count + 1)); done <<<"$pods"; [ "$count" -eq 3 ]; printf "%s" "$output" | grep -Fq "https://region-b-watch.nvcf.svc.cluster.local:50071"; printf "region-a-deployment=%s remote-watch=https\n" "$count"'` - regionBWatchCommand = `/bin/bash -c 'set -eu; output=$(grpcurl -max-time 3 -cacert <(kubectl --context k3d-ncp-local-cp get secret stargate-quic-tls -n nvcf -o jsonpath="{.data.ca\.crt}" | base64 -d) -authority region-b-watch.nvcf.svc.cluster.local -import-path src/libraries/rust/stargate/crates/proto/proto -proto stargate.proto 127.0.0.1:50071 stargate.StargateControlPlane/WatchStargates 2>&1 || true); identities=$(printf "%s\n" "$output" | grep -Eo "llm-request-router-region-b-[0-9]+" | sort -u || true); expected=$(printf "llm-request-router-region-b-0\nllm-request-router-region-b-1\n"); [ "$identities" = "$expected" ]; count=$(printf "%s\n" "$identities" | grep -c .); [ "$count" -eq 2 ]; ! printf "%s" "$output" | grep -Eq "([0-9]{1,3}-){3}[0-9]{1,3}\."; printf "region-b-statefulset=%s tls=https\n" "$count"'` - pylonMetricsCommand = `/bin/sh -c 'set -eu; for attempt in $(seq 1 120); do row=$(kubectl --context k3d-ncp-local-compute-1 get pods -A -o json | jq -r "[.items[] | select(any(.spec.containers[]?; .name == \"llm-worker\")) | [.metadata.namespace,.metadata.name] | @tsv] | first // empty"); if [ -n "$row" ]; then ns=$(printf "%s" "$row" | cut -f1); pod=$(printf "%s" "$row" | cut -f2); metrics=$(kubectl --context k3d-ncp-local-compute-1 get --raw "/api/v1/namespaces/$ns/pods/$pod:9089/proxy/metrics" 2>/dev/null || true); registration=$(printf "%s\n" "$metrics" | grep -c "^pylon_registration_stream_connected.* 1$" || true); reverse=$(printf "%s\n" "$metrics" | grep -c "^pylon_reverse_tunnel_connected.* 1$" || true); if [ "$registration" -eq 5 ] && [ "$reverse" -ge 3 ]; then printf "registration=%s reverse=%s regions=2\n" "$registration" "$reverse"; exit 0; fi; fi; sleep 5; done; exit 1'` - invokeCommand = "/usr/bin/nvcf-cli --config /repo-root-placeholder/tests/bdd/fixtures/nvcf-cli-local.yaml function invoke" + + regionAWatchCommand = `/bin/bash -c 'set -eu; output=$(grpcurl -max-time 3 -cacert <(kubectl --context k3d-ncp-local-cp get secret stargate-quic-tls -n nvcf -o jsonpath="{.data.ca\.crt}" | base64 -d) -authority llm-request-router.nvcf.svc.cluster.local -import-path src/libraries/rust/stargate/crates/proto/proto -proto stargate.proto 127.0.0.1:50071 stargate.StargateControlPlane/WatchStargates 2>&1 || true); pods=$(kubectl --context k3d-ncp-local-cp get pods -n nvcf -l app.kubernetes.io/instance=llm-request-router,app.kubernetes.io/name=llm-request-router -o jsonpath="{range .items[*]}{.metadata.name}{\"\\n\"}{end}"); count=0; while IFS= read -r pod; do [ -z "$pod" ] && continue; printf "%s" "$output" | grep -Fq "$pod"; count=$((count + 1)); done <<<"$pods"; [ "$count" -eq 3 ]; printf "%s" "$output" | grep -Fq "https://region-b-watch.nvcf.svc.cluster.local:50071"; printf "region-a-deployment=%s remote-watch=https\n" "$count"'` + regionBWatchCommand = `/bin/bash -c 'set -eu; output=$(grpcurl -max-time 3 -cacert <(kubectl --context k3d-ncp-local-cp get secret stargate-quic-tls -n nvcf -o jsonpath="{.data.ca\.crt}" | base64 -d) -authority region-b-watch.nvcf.svc.cluster.local -import-path src/libraries/rust/stargate/crates/proto/proto -proto stargate.proto 127.0.0.1:50071 stargate.StargateControlPlane/WatchStargates 2>&1 || true); identities=$(printf "%s\n" "$output" | grep -Eo "llm-request-router-region-b-[0-9]+" | sort -u || true); expected=$(printf "llm-request-router-region-b-0\nllm-request-router-region-b-1\n"); [ "$identities" = "$expected" ]; count=$(printf "%s\n" "$identities" | grep -c .); [ "$count" -eq 2 ]; ! printf "%s" "$output" | grep -Eq "([0-9]{1,3}-){3}[0-9]{1,3}\."; printf "region-b-statefulset=%s tls=https\n" "$count"'` + pylonMetricsCommand = `/bin/sh -c 'set -eu; for attempt in $(seq 1 120); do row=$(kubectl --context k3d-ncp-local-compute-1 get pods -A -o json | jq -r "[.items[] | select(any(.spec.containers[]?; .name == \"llm-worker\")) | [.metadata.namespace,.metadata.name] | @tsv] | first // empty"); if [ -n "$row" ]; then ns=$(printf "%s" "$row" | cut -f1); pod=$(printf "%s" "$row" | cut -f2); metrics=$(kubectl --context k3d-ncp-local-compute-1 get --raw "/api/v1/namespaces/$ns/pods/$pod:9089/proxy/metrics" 2>/dev/null || true); registration=$(printf "%s\n" "$metrics" | grep -c "^pylon_registration_stream_connected.* 1$" || true); reverse=$(printf "%s\n" "$metrics" | grep -c "^pylon_reverse_tunnel_connected.* 1$" || true); if [ "$registration" -eq 5 ] && [ "$reverse" -ge 3 ]; then printf "registration=%s reverse=%s regions=2\n" "$registration" "$reverse"; exit 0; fi; fi; sleep 5; done; exit 1'` + grpcCertificateCommand = "kubectl --context k3d-ncp-local-cp get certificate llm-request-router-grpc-tls" + + " -n envoy-gateway-system -o jsonpath={.spec.dnsNames}" + invokeCommand = "/usr/bin/nvcf-cli --config /repo-root-placeholder/tests/bdd/fixtures/nvcf-cli-local.yaml function invoke" + " --inference-url /v1/chat/completions --model-name openai-compatible-sample" + " --request-body '{\"messages\":[{\"role\":\"user\",\"content\":\"bdd-registration-multiregion\"}]}' --timeout 120" ) @@ -1190,6 +1192,10 @@ func TestMultiClusterHelmfileLLMRegistrationMultiregionFeatureFileWiresToSteps(t ExitCode: 0, Stdout: "region-b-statefulset=2 tls=https\n", }, + grpcCertificateCommand: { + ExitCode: 0, + Stdout: "[llm-request-router.nvcf.svc.cluster.local region-b-watch.nvcf.svc.cluster.local]", + }, pylonMetricsCommand: {ExitCode: 0, Stdout: "registration=5 reverse=3 regions=2\n"}, invokeCommand: { ExitCode: 0, @@ -1230,7 +1236,12 @@ func TestMultiClusterHelmfileLLMRegistrationMultiregionFeatureFileWiresToSteps(t if status != 0 { t.Fatalf("godog suite status = %d\n%s", status, out.String()) } - for _, command := range []string{regionAWatchCommand, regionBWatchCommand, pylonMetricsCommand} { + for _, command := range []string{ + grpcCertificateCommand, + regionAWatchCommand, + regionBWatchCommand, + pylonMetricsCommand, + } { if !commandRanExactly(suite.Runner.(*fakeRunner).runs, command) { t.Fatalf("exact multi-region observation command was not invoked: %s", command) } @@ -1256,6 +1267,7 @@ func TestMultiClusterHelmfileLLMRegistrationMultiregionFeatureFileWiresToSteps(t key string want string }{ + {key: "addons.llm.requestRouter.grpcTls.dnsNames[1]", want: "region-b-watch.nvcf.svc.cluster.local"}, {key: "addons.llm.pki.dnsNames[2]", want: "region-b-watch.nvcf.svc.cluster.local"}, {key: "addons.llm.pki.dnsNames[3]", want: "*.llm-request-router-region-b-headless.nvcf.svc.cluster.local"}, } { @@ -1267,14 +1279,6 @@ func TestMultiClusterHelmfileLLMRegistrationMultiregionFeatureFileWiresToSteps(t t.Fatalf("%s = %q, found %t, want %q", expectation.key, got, found, expectation.want) } } - if _, found, readErr := dsl.ReadYAMLKey( - environmentPath, - "addons.llm.requestRouter.grpcTls.dnsNames[1]", - ); readErr != nil { - t.Fatalf("read unused request-router gRPC TLS key: %v", readErr) - } else if found { - t.Fatal("multi-region environment populated the unused request-router gRPC TLS SAN key") - } } // TestSingleClusterHelmfileUpstreamImagesFeatureFileWiresToSteps runs the