From 496344b9cdd2e05509e6a89203e98c275d112276 Mon Sep 17 00:00:00 2001 From: Stephanie Baum Date: Fri, 28 Aug 2026 03:10:41 -0700 Subject: [PATCH 1/2] test(bdd): cover secure multi-region registration --- ...mfile-llm-registration-multiregion.feature | 162 ++++++++++++++++++ tests/bdd/godog_test.go | 95 ++++++++++ tests/bdd/scripts/install-llm-region-b.sh | 138 +++++++++++++++ 3 files changed, 395 insertions(+) create mode 100644 tests/bdd/features/multi-cluster-helmfile-llm-registration-multiregion.feature create mode 100755 tests/bdd/scripts/install-llm-region-b.sh diff --git a/tests/bdd/features/multi-cluster-helmfile-llm-registration-multiregion.feature b/tests/bdd/features/multi-cluster-helmfile-llm-registration-multiregion.feature new file mode 100644 index 000000000..011c936f1 --- /dev/null +++ b/tests/bdd/features/multi-cluster-helmfile-llm-registration-multiregion.feature @@ -0,0 +1,162 @@ +@ncp-local @multi-cluster @helmfile @pki @llm-registration @multi-region +Feature: Register an LLM worker securely with routers in two local regions + As a self-managed NVCF operator, + I want recursive router discovery to retain explicit HTTPS transport, + so that one worker can register with routable Deployment and StatefulSet routers across regions. + + Rule: A secure remote Watch URI expands the registered router topology + + Background: + Given these environment variables are set: + | name | + | NGC_API_KEY | + | NVCF_CLI | + | REPO_ROOT | + | SAMPLE_NGC_ORG | + | SAMPLE_NGC_TEAM | + And I prepare Helmfile environment "local-bdd-registration-multiregion" for stack "self-managed" from fixture "tests/bdd/fixtures/self-managed-local-bdd-multi.yaml" with values: + | global.imagePullSecrets[0].name | nvcr-pull-secret | + | global.helm.sources.repository | ${SAMPLE_NGC_ORG}/${SAMPLE_NGC_TEAM} | + | global.image.repository | ${SAMPLE_NGC_ORG}/${SAMPLE_NGC_TEAM} | + | global.workerEndpoints.llmRequestRouterAddress | https://llm-request-router.nvcf.svc.cluster.local:50071 | + | addons.llm.requestRouter.workload.kind | Deployment | + | addons.llm.requestRouter.discovery.remoteWatchUrls[0] | https://region-b-watch.nvcf.svc.cluster.local:50071 | + | addons.llm.requestRouter.grpcTls.dnsNames[1] | region-b-watch.nvcf.svc.cluster.local | + | addons.llm.requestRouter.backendRouter.pylonGrpcDialAddress | https://llm-request-router.nvcf.svc.cluster.local:50071 | + | addons.llm.pki.dnsNames[2] | *.llm-request-router-region-b-headless.nvcf.svc.cluster.local | + | observability.profile | disabled | + And I prepare Helmfile environment "local-bdd-registration-multiregion" for stack "nvcf-compute-plane" from fixture "tests/bdd/fixtures/nvcf-compute-plane-local-bdd-multi.yaml" with values: + | global.imagePullSecrets[0].name | nvcr-pull-secret | + | global.helm.sources.repository | ${SAMPLE_NGC_ORG}/${SAMPLE_NGC_TEAM} | + | global.image.repository | ${SAMPLE_NGC_ORG}/${SAMPLE_NGC_TEAM} | + | observability.profile | disabled | + And I prepare self-managed secrets file "deploy/stacks/self-managed/secrets/local-bdd-registration-multiregion-secrets.yaml" from template "deploy/stacks/self-managed/secrets/secrets.yaml.template" using the current NGC registry credential + When I run command "k3d cluster get ncp-local" + Then the command exit code should be 1 + And multi-cluster ncp-local compute clusters are running: + | ncp-local-compute-1 | + And command has succeeded: + """ + kubectl config use-context k3d-ncp-local-cp + """ + And the "nvcr-pull-secret" image pull secret exists in namespaces: + | cassandra-system | + | nats-system | + | nvcf | + | api-keys | + | ess | + | sis | + | vault-system | + | nvca-operator | + | cert-manager | + + @llm-registration-multiregion-install + Scenario: Operator installs two secure regions with distinct router workload identities + When I run command "make -C deploy/stacks/self-managed template HELMFILE_ENV=local-bdd-registration-multiregion" + Then the command exit code should be 0 + And the rendered manifests in "deploy/stacks/self-managed/out" should contain: + | text | + | kind: Deployment | + | --remote-stargate-url=https://region-b-watch.nvcf.svc.cluster.local:50071 | + + When I run command "make -C deploy/stacks/self-managed install HELMFILE_ENV=local-bdd-registration-multiregion" + Then the command exit code should be 0 + When I run command "kubectl --context k3d-ncp-local-cp wait certificate llm-request-router-grpc-tls -n envoy-gateway-system --for=condition=Ready --timeout=5m" + Then the command exit code should be 0 + When I run command "kubectl --context k3d-ncp-local-cp rollout status deployment/llm-request-router -n nvcf --timeout=10m" + Then the command exit code should be 0 + + When I run command "tests/bdd/scripts/install-llm-region-b.sh" + Then the command exit code should be 0 + + # The initial region advertises an explicit HTTPS recursive seed while + # retaining every concrete Deployment pod identity. + When I run command: + """ + /bin/bash -c 'set -eu; output=$(grpcurl -max-time 3 -cacert <(kubectl --context k3d-ncp-local-cp get secret stargate-quic-tls -n nvcf -o jsonpath="{.data.ca\.crt}" | base64 -d) -authority llm-request-router.nvcf.svc.cluster.local -import-path src/libraries/rust/stargate/crates/proto/proto -proto stargate.proto 127.0.0.1:50071 stargate.StargateControlPlane/WatchStargates 2>&1 || true); pods=$(kubectl --context k3d-ncp-local-cp get pods -n nvcf -l app.kubernetes.io/instance=llm-request-router,app.kubernetes.io/name=llm-request-router -o jsonpath="{range .items[*]}{.metadata.name}{\"\\n\"}{end}"); count=0; while IFS= read -r pod; do [ -z "$pod" ] && continue; printf "%s" "$output" | grep -Fq "$pod"; count=$((count + 1)); done <<<"$pods"; [ "$count" -eq 3 ]; printf "%s" "$output" | grep -Fq "https://region-b-watch.nvcf.svc.cluster.local:50071"; printf "region-a-deployment=%s remote-watch=https\n" "$count"' + """ + Then the command exit code should be 0 + And the command output should contain "region-a-deployment=3 remote-watch=https" + + # The remote HTTPS authority resolves to two stable StatefulSet router + # identities and never relies on a dashed-IP SRV alias. + When I run command: + """ + /bin/bash -c 'set -eu; output=$(grpcurl -max-time 3 -cacert <(kubectl --context k3d-ncp-local-cp get secret stargate-quic-tls -n nvcf -o jsonpath="{.data.ca\.crt}" | base64 -d) -authority region-b-watch.nvcf.svc.cluster.local -import-path src/libraries/rust/stargate/crates/proto/proto -proto stargate.proto 127.0.0.1:50071 stargate.StargateControlPlane/WatchStargates 2>&1 || true); printf "%s" "$output" | grep -Fq "llm-request-router-region-b-0"; printf "%s" "$output" | grep -Fq "llm-request-router-region-b-1"; printf "region-b-statefulset=2 tls=https\n"' + """ + Then the command exit code should be 0 + And the command output should contain "region-b-statefulset=2 tls=https" + + When I run command: + """ + ${NVCF_CLI} --config ${REPO_ROOT}/tests/bdd/fixtures/nvcf-cli-local.yaml self-hosted --control-plane-stack deploy/stacks/self-managed --env local-bdd-registration-multiregion --control-plane-context k3d-ncp-local-cp --compute-plane-context k3d-ncp-local-compute-1 control-plane profile export --cluster-name ncp-local-cp + """ + Then the command exit code should be 0 + And file "deploy/stacks/self-managed/out/control-plane-profile.yaml" should exist + And yaml file "deploy/stacks/self-managed/out/control-plane-profile.yaml" should have non-empty keys: + | key | + | managementTls.caBundlePem | + | transportTls.trustBundleFingerprint | + | transportTls.trustBundlePem | + + And command has succeeded: + """ + /bin/sh -c '${NVCF_CLI} --config ${REPO_ROOT}/tests/bdd/fixtures/nvcf-cli-local.yaml init >/dev/null' + """ + When I run command "kubectl config use-context k3d-ncp-local-compute-1" + Then the command exit code should be 0 + When I run command: + """ + make -C deploy/stacks/nvcf-compute-plane register-cluster CLUSTER_NAME=ncp-local-compute-1 CONTROL_PLANE_PROFILE=${REPO_ROOT}/deploy/stacks/self-managed/out/control-plane-profile.yaml COMPUTE_KUBE_CONTEXT=k3d-ncp-local-compute-1 NVCF_CLI=${NVCF_CLI} NVCF_CLI_CONFIG=${REPO_ROOT}/tests/bdd/fixtures/nvcf-cli-local.yaml + """ + Then the command exit code should be 0 + And the "nvcr-pull-secret" image pull secret exists in namespaces: + | nvca-operator | + When I run command: + """ + make -C deploy/stacks/nvcf-compute-plane install CLUSTER_NAME=ncp-local-compute-1 HELMFILE_ENV=local-bdd-registration-multiregion COMPUTE_KUBE_CONTEXT=k3d-ncp-local-compute-1 NVCF_CLI=${NVCF_CLI} + """ + Then the command exit code should be 0 + Then NVCFBackend "ncp-local-compute-1" in namespace "nvca-operator" using context "k3d-ncp-local-compute-1" should report agent status "healthy" within "10m" + + @llm-registration-multiregion-runtime + Scenario: Pylon recursively registers with both regions and serves an authenticated request + Given I use NVCF CLI config "${REPO_ROOT}/tests/bdd/fixtures/nvcf-cli-local.yaml" + When I successfully create function "bdd-registration-multiregion" from image "nvcr.io/${SAMPLE_NGC_ORG}/${SAMPLE_NGC_TEAM}/nvcf-openai-compatible-sample:local" with CLI options: + | option | value | + | --function-type | LLM | + | --inference-url | /v1/chat/completions | + | --inference-port | 8000 | + | --health-uri | /health | + | --health-port | 8000 | + | --health-timeout | PT30S | + | --llm-model | name=openai-compatible-sample,uris=/v1/chat/completions\|/v1/embeddings,routingMethod=round_robin | + And I successfully deploy the function selected by NVCF CLI with options: + | option | value | + | --gpu | H100 | + | --instance-type | NCP.GPU.H100_1x | + | --backend | ncp-local-compute-1 | + | --regions | us-west-1 | + | --min-instances | 1 | + | --max-instances | 1 | + | --timeout | 900 | + And I successfully generate a function API key with CLI options: + | option | value | + | --description | bdd-registration-multiregion | + | --scopes | invoke_function,list_functions,queue_details,list_functions_details | + + When I run command: + """ + /bin/sh -c 'set -eu; for attempt in $(seq 1 120); do row=$(kubectl --context k3d-ncp-local-compute-1 get pods -A -o json | jq -r "[.items[] | select(any(.spec.containers[]?; .name == \"llm-worker\")) | [.metadata.namespace,.metadata.name] | @tsv] | first // empty"); if [ -n "$row" ]; then ns=$(printf "%s" "$row" | cut -f1); pod=$(printf "%s" "$row" | cut -f2); metrics=$(kubectl --context k3d-ncp-local-compute-1 get --raw "/api/v1/namespaces/$ns/pods/$pod:9089/proxy/metrics" 2>/dev/null || true); registration=$(printf "%s\n" "$metrics" | grep -c "^pylon_registration_stream_connected.* 1$" || true); reverse=$(printf "%s\n" "$metrics" | grep -c "^pylon_reverse_tunnel_connected.* 1$" || true); if [ "$registration" -eq 5 ] && [ "$reverse" -ge 3 ]; then printf "registration=%s reverse=%s regions=2\n" "$registration" "$reverse"; exit 0; fi; fi; sleep 5; done; exit 1' + """ + Then the command exit code should be 0 + And the command output should contain "registration=5" + And the command output should contain "regions=2" + + When I successfully invoke model "openai-compatible-sample" at "/v1/chat/completions" with timeout "120" seconds: + """ + {"messages":[{"role":"user","content":"bdd-registration-multiregion"}]} + """ + Then the command output should contain "chat.completion" + And the command output should contain "fixed 128-byte response" + And I successfully undeploy the function selected by NVCF CLI diff --git a/tests/bdd/godog_test.go b/tests/bdd/godog_test.go index eedb40e1f..afb162a91 100644 --- a/tests/bdd/godog_test.go +++ b/tests/bdd/godog_test.go @@ -1194,6 +1194,91 @@ func TestMultiClusterHelmfileFeatureFileWiresToSteps(t *testing.T) { assertFunctionDeploymentsUseInstanceType(t, suite.Runner.(*fakeRunner).runs, "NCP.GPU.H100_1x", 3) } +// TestMultiClusterHelmfileLLMRegistrationMultiregionFeatureFileWiresToSteps +// runs the secure recursive-discovery feature against a fake runner. The +// observations cover distinct Deployment and StatefulSet router identities, +// the HTTPS remote Watch URI, and Pylon's combined registration topology. +func TestMultiClusterHelmfileLLMRegistrationMultiregionFeatureFileWiresToSteps(t *testing.T) { + t.Setenv("NGC_API_KEY", "test-key") + t.Setenv("SAMPLE_NGC_ORG", "test-org") + t.Setenv("SAMPLE_NGC_TEAM", "test-team") + t.Setenv("NVCF_CLI", "/usr/bin/nvcf-cli") + t.Setenv("REPO_ROOT", "/repo-root-placeholder") + + const ( + regionAWatchCommand = `/bin/bash -c 'set -eu; output=$(grpcurl -max-time 3 -cacert <(kubectl --context k3d-ncp-local-cp get secret stargate-quic-tls -n nvcf -o jsonpath="{.data.ca\.crt}" | base64 -d) -authority llm-request-router.nvcf.svc.cluster.local -import-path src/libraries/rust/stargate/crates/proto/proto -proto stargate.proto 127.0.0.1:50071 stargate.StargateControlPlane/WatchStargates 2>&1 || true); pods=$(kubectl --context k3d-ncp-local-cp get pods -n nvcf -l app.kubernetes.io/instance=llm-request-router,app.kubernetes.io/name=llm-request-router -o jsonpath="{range .items[*]}{.metadata.name}{\"\\n\"}{end}"); count=0; while IFS= read -r pod; do [ -z "$pod" ] && continue; printf "%s" "$output" | grep -Fq "$pod"; count=$((count + 1)); done <<<"$pods"; [ "$count" -eq 3 ]; printf "%s" "$output" | grep -Fq "https://region-b-watch.nvcf.svc.cluster.local:50071"; printf "region-a-deployment=%s remote-watch=https\n" "$count"'` + regionBWatchCommand = `/bin/bash -c 'set -eu; output=$(grpcurl -max-time 3 -cacert <(kubectl --context k3d-ncp-local-cp get secret stargate-quic-tls -n nvcf -o jsonpath="{.data.ca\.crt}" | base64 -d) -authority region-b-watch.nvcf.svc.cluster.local -import-path src/libraries/rust/stargate/crates/proto/proto -proto stargate.proto 127.0.0.1:50071 stargate.StargateControlPlane/WatchStargates 2>&1 || true); printf "%s" "$output" | grep -Fq "llm-request-router-region-b-0"; printf "%s" "$output" | grep -Fq "llm-request-router-region-b-1"; printf "region-b-statefulset=2 tls=https\n"'` + pylonMetricsCommand = `/bin/sh -c 'set -eu; for attempt in $(seq 1 120); do row=$(kubectl --context k3d-ncp-local-compute-1 get pods -A -o json | jq -r "[.items[] | select(any(.spec.containers[]?; .name == \"llm-worker\")) | [.metadata.namespace,.metadata.name] | @tsv] | first // empty"); if [ -n "$row" ]; then ns=$(printf "%s" "$row" | cut -f1); pod=$(printf "%s" "$row" | cut -f2); metrics=$(kubectl --context k3d-ncp-local-compute-1 get --raw "/api/v1/namespaces/$ns/pods/$pod:9089/proxy/metrics" 2>/dev/null || true); registration=$(printf "%s\n" "$metrics" | grep -c "^pylon_registration_stream_connected.* 1$" || true); reverse=$(printf "%s\n" "$metrics" | grep -c "^pylon_reverse_tunnel_connected.* 1$" || true); if [ "$registration" -eq 5 ] && [ "$reverse" -ge 3 ]; then printf "registration=%s reverse=%s regions=2\n" "$registration" "$reverse"; exit 0; fi; fi; sleep 5; done; exit 1'` + invokeCommand = "/usr/bin/nvcf-cli --config /repo-root-placeholder/tests/bdd/fixtures/nvcf-cli-local.yaml function invoke" + + " --inference-url /v1/chat/completions --model-name openai-compatible-sample" + + " --request-body '{\"messages\":[{\"role\":\"user\",\"content\":\"bdd-registration-multiregion\"}]}' --timeout 120" + ) + + suite := newWiringSuite(t, newFakeRunner(map[string]harness.Result{ + "k3d cluster get ncp-local": {ExitCode: 1}, + regionAWatchCommand: { + ExitCode: 0, + Stdout: "region-a-deployment=3 remote-watch=https\n", + }, + regionBWatchCommand: { + ExitCode: 0, + Stdout: "region-b-statefulset=2 tls=https\n", + }, + pylonMetricsCommand: {ExitCode: 0, Stdout: "registration=5 reverse=3 regions=2\n"}, + invokeCommand: { + ExitCode: 0, + Stdout: "Function invocation completed!\n\nResponse:\n" + + `{"object":"chat.completion","choices":[{"message":{"content":"This is a fixed 128-byte response for routing and contract validation."}}]}` + + "\n", + }, + })) + seedHelmfileLocalBDDMultiFixture(t, suite.Config.RepoRoot) + seedComputePlaneLocalBDDMultiFixture(t, suite.Config.RepoRoot) + seedStackSecretsTemplate(t, suite.Config.RepoRoot) + writeProfileHandoffArtifact(t, suite.Config.RepoRoot) + writeMulticlusterComputeRegisterValues(t, suite.Config.RepoRoot, "nvcf-compute-plane", "ncp-local-compute-1") + writeArtifact( + t, + suite.Config.RepoRoot, + "self-managed", + "registration-multiregion-rendered.yaml", + "kind: Deployment\n"+ + "--remote-stargate-url=https://region-b-watch.nvcf.svc.cluster.local:50071\n", + ) + + sc := steps.NewScenarioContext(suite) + featurePath := mustResolveFeaturePath(t, "multi-cluster-helmfile-llm-registration-multiregion.feature") + var out strings.Builder + status := godog.TestSuite{ + Name: "multi-cluster-helmfile-llm-registration-multiregion-wiring", + ScenarioInitializer: func(ctx *godog.ScenarioContext) { + steps.RegisterAll(ctx, sc) + }, + Options: &godog.Options{ + Format: "pretty", + Paths: []string{featurePath}, + Strict: true, + Output: &out, + }, + }.Run() + if status != 0 { + t.Fatalf("godog suite status = %d\n%s", status, out.String()) + } + for _, command := range []string{regionAWatchCommand, regionBWatchCommand, pylonMetricsCommand} { + if !commandRanExactly(suite.Runner.(*fakeRunner).runs, command) { + t.Fatalf("exact multi-region observation command was not invoked: %s", command) + } + } + if !commandRanThatContainsAll( + suite.Runner.(*fakeRunner).runs, + "function create --name bdd-registration-multiregion", + "--function-type LLM", + "--llm-model", + ) { + t.Fatal("multi-region sample was not created as an LLM function") + } +} + // TestSingleClusterHelmfileUpstreamImagesFeatureFileWiresToSteps runs the // focused upstream-image feature against a fake runner. The seeded global // template contains the exact documentation blocks so the ledger-backed @@ -1959,6 +2044,16 @@ func TestMultiClusterHelmfile(t *testing.T) { runLiveFeature(t, "multi-cluster-helmfile.feature") } +// TestMultiClusterHelmfileLLMRegistrationMultiregion is the live entry point +// for secure recursive registration across two local logical regions. +// Skipped under -short. +func TestMultiClusterHelmfileLLMRegistrationMultiregion(t *testing.T) { + if testing.Short() { + t.Skip("live run skipped under -short") + } + runLiveFeature(t, "multi-cluster-helmfile-llm-registration-multiregion.feature") +} + // TestSingleClusterEKSHelmfile is the live entry point for the // single-cluster EKS Helmfile feature. Skipped under -short. func TestSingleClusterEKSHelmfile(t *testing.T) { diff --git a/tests/bdd/scripts/install-llm-region-b.sh b/tests/bdd/scripts/install-llm-region-b.sh new file mode 100755 index 000000000..9b1ae9931 --- /dev/null +++ b/tests/bdd/scripts/install-llm-region-b.sh @@ -0,0 +1,138 @@ +#!/usr/bin/env bash +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +set -euo pipefail + +control_context="${CONTROL_CONTEXT:-k3d-ncp-local-cp}" +compute_context="${COMPUTE_CONTEXT:-k3d-ncp-local-compute-1}" +namespace="nvcf" +region_b_release="llm-request-router-region-b" +region_b_watch_host="region-b-watch.nvcf.svc.cluster.local" +region_b_headless_host="*.llm-request-router-region-b-headless.nvcf.svc.cluster.local" +chart="${REPO_ROOT:?REPO_ROOT is required}/deploy/helm/llm-request-router/llm-request-router" + +values_json="$(helm --kube-context "${control_context}" get values llm-request-router \ + --namespace "${namespace}" --output json)" + +printf '%s' "${values_json}" | jq --arg watch_host "${region_b_watch_host}" ' + { + llmRequestRouter: ( + .llmRequestRouter + | .fullnameOverride = "llm-request-router-region-b" + | .replicaCount = 2 + | .workload.kind = "StatefulSet" + | .service.headlessName = "llm-request-router-region-b-headless" + | .kubernetes.advertisedHostnameTemplate = "{pod_name}.llm-request-router-region-b-headless.nvcf.svc.cluster.local" + | .discovery.remoteWatchUrls = [] + | .backendRouter.enabled = true + | .backendRouter.pylonGrpcDialAddress = ("https://" + $watch_host + ":50071") + | .backendRouter.pylonReverseTunnelDialAddress = ($watch_host + ":50072") + | .serviceAccount.create = false + | .serviceAccount.name = "llm-request-router" + | .pki.enabled = false + | .certificate.enabled = false + | .tls.mode = "existingSecret" + | .tls.secretName = "stargate-quic-tls" + | .image.pullPolicy = "IfNotPresent" + | .backendRouter.image.pullPolicy = "IfNotPresent" + ) + } +' | helm --kube-context "${control_context}" upgrade --install "${region_b_release}" "${chart}" \ + --namespace "${namespace}" --values - --wait --timeout 10m + +kubectl --context "${control_context}" apply -f - <&2 + exit 1 +fi + +kubectl --context "${compute_context}" apply -f - < Date: Fri, 28 Aug 2026 16:15:52 -0400 Subject: [PATCH 2/2] test(bdd): fix secure multi-region coverage (#1322) Signed-off-by: Mike Camp --- ...mfile-llm-registration-multiregion.feature | 8 +- tests/bdd/godog_test.go | 48 ++++++++- tests/bdd/region_b_script_test.go | 101 ++++++++++++++++++ tests/bdd/scripts/install-llm-region-b.sh | 4 +- 4 files changed, 153 insertions(+), 8 deletions(-) create mode 100644 tests/bdd/region_b_script_test.go diff --git a/tests/bdd/features/multi-cluster-helmfile-llm-registration-multiregion.feature b/tests/bdd/features/multi-cluster-helmfile-llm-registration-multiregion.feature index 011c936f1..d06b58af0 100644 --- a/tests/bdd/features/multi-cluster-helmfile-llm-registration-multiregion.feature +++ b/tests/bdd/features/multi-cluster-helmfile-llm-registration-multiregion.feature @@ -23,7 +23,8 @@ Feature: Register an LLM worker securely with routers in two local regions | addons.llm.requestRouter.discovery.remoteWatchUrls[0] | https://region-b-watch.nvcf.svc.cluster.local:50071 | | addons.llm.requestRouter.grpcTls.dnsNames[1] | region-b-watch.nvcf.svc.cluster.local | | addons.llm.requestRouter.backendRouter.pylonGrpcDialAddress | https://llm-request-router.nvcf.svc.cluster.local:50071 | - | addons.llm.pki.dnsNames[2] | *.llm-request-router-region-b-headless.nvcf.svc.cluster.local | + | addons.llm.pki.dnsNames[2] | region-b-watch.nvcf.svc.cluster.local | + | addons.llm.pki.dnsNames[3] | *.llm-request-router-region-b-headless.nvcf.svc.cluster.local | | observability.profile | disabled | And I prepare Helmfile environment "local-bdd-registration-multiregion" for stack "nvcf-compute-plane" from fixture "tests/bdd/fixtures/nvcf-compute-plane-local-bdd-multi.yaml" with values: | global.imagePullSecrets[0].name | nvcr-pull-secret | @@ -63,6 +64,9 @@ Feature: Register an LLM worker securely with routers in two local regions Then the command exit code should be 0 When I run command "kubectl --context k3d-ncp-local-cp wait certificate llm-request-router-grpc-tls -n envoy-gateway-system --for=condition=Ready --timeout=5m" Then the command exit code should be 0 + When I run command "kubectl --context k3d-ncp-local-cp get certificate llm-request-router-grpc-tls -n envoy-gateway-system -o jsonpath={.spec.dnsNames}" + Then the command exit code should be 0 + And the command output should contain "region-b-watch.nvcf.svc.cluster.local" When I run command "kubectl --context k3d-ncp-local-cp rollout status deployment/llm-request-router -n nvcf --timeout=10m" Then the command exit code should be 0 @@ -82,7 +86,7 @@ Feature: Register an LLM worker securely with routers in two local regions # identities and never relies on a dashed-IP SRV alias. When I run command: """ - /bin/bash -c 'set -eu; output=$(grpcurl -max-time 3 -cacert <(kubectl --context k3d-ncp-local-cp get secret stargate-quic-tls -n nvcf -o jsonpath="{.data.ca\.crt}" | base64 -d) -authority region-b-watch.nvcf.svc.cluster.local -import-path src/libraries/rust/stargate/crates/proto/proto -proto stargate.proto 127.0.0.1:50071 stargate.StargateControlPlane/WatchStargates 2>&1 || true); printf "%s" "$output" | grep -Fq "llm-request-router-region-b-0"; printf "%s" "$output" | grep -Fq "llm-request-router-region-b-1"; printf "region-b-statefulset=2 tls=https\n"' + /bin/bash -c 'set -eu; output=$(grpcurl -max-time 3 -cacert <(kubectl --context k3d-ncp-local-cp get secret stargate-quic-tls -n nvcf -o jsonpath="{.data.ca\.crt}" | base64 -d) -authority region-b-watch.nvcf.svc.cluster.local -import-path src/libraries/rust/stargate/crates/proto/proto -proto stargate.proto 127.0.0.1:50071 stargate.StargateControlPlane/WatchStargates 2>&1 || true); identities=$(printf "%s\n" "$output" | grep -Eo "llm-request-router-region-b-[0-9]+" | sort -u || true); expected=$(printf "llm-request-router-region-b-0\nllm-request-router-region-b-1\n"); [ "$identities" = "$expected" ]; count=$(printf "%s\n" "$identities" | grep -c .); [ "$count" -eq 2 ]; ! printf "%s" "$output" | grep -Eq "([0-9]{1,3}-){3}[0-9]{1,3}\."; printf "region-b-statefulset=%s tls=https\n" "$count"' """ Then the command exit code should be 0 And the command output should contain "region-b-statefulset=2 tls=https" diff --git a/tests/bdd/godog_test.go b/tests/bdd/godog_test.go index afb162a91..a4608b3d1 100644 --- a/tests/bdd/godog_test.go +++ b/tests/bdd/godog_test.go @@ -1205,14 +1205,18 @@ func TestMultiClusterHelmfileLLMRegistrationMultiregionFeatureFileWiresToSteps(t t.Setenv("NVCF_CLI", "/usr/bin/nvcf-cli") t.Setenv("REPO_ROOT", "/repo-root-placeholder") + //revive:disable:line-length-limit Exact feature commands must remain byte-for-byte identical. const ( - regionAWatchCommand = `/bin/bash -c 'set -eu; output=$(grpcurl -max-time 3 -cacert <(kubectl --context k3d-ncp-local-cp get secret stargate-quic-tls -n nvcf -o jsonpath="{.data.ca\.crt}" | base64 -d) -authority llm-request-router.nvcf.svc.cluster.local -import-path src/libraries/rust/stargate/crates/proto/proto -proto stargate.proto 127.0.0.1:50071 stargate.StargateControlPlane/WatchStargates 2>&1 || true); pods=$(kubectl --context k3d-ncp-local-cp get pods -n nvcf -l app.kubernetes.io/instance=llm-request-router,app.kubernetes.io/name=llm-request-router -o jsonpath="{range .items[*]}{.metadata.name}{\"\\n\"}{end}"); count=0; while IFS= read -r pod; do [ -z "$pod" ] && continue; printf "%s" "$output" | grep -Fq "$pod"; count=$((count + 1)); done <<<"$pods"; [ "$count" -eq 3 ]; printf "%s" "$output" | grep -Fq "https://region-b-watch.nvcf.svc.cluster.local:50071"; printf "region-a-deployment=%s remote-watch=https\n" "$count"'` - regionBWatchCommand = `/bin/bash -c 'set -eu; output=$(grpcurl -max-time 3 -cacert <(kubectl --context k3d-ncp-local-cp get secret stargate-quic-tls -n nvcf -o jsonpath="{.data.ca\.crt}" | base64 -d) -authority region-b-watch.nvcf.svc.cluster.local -import-path src/libraries/rust/stargate/crates/proto/proto -proto stargate.proto 127.0.0.1:50071 stargate.StargateControlPlane/WatchStargates 2>&1 || true); printf "%s" "$output" | grep -Fq "llm-request-router-region-b-0"; printf "%s" "$output" | grep -Fq "llm-request-router-region-b-1"; printf "region-b-statefulset=2 tls=https\n"'` - pylonMetricsCommand = `/bin/sh -c 'set -eu; for attempt in $(seq 1 120); do row=$(kubectl --context k3d-ncp-local-compute-1 get pods -A -o json | jq -r "[.items[] | select(any(.spec.containers[]?; .name == \"llm-worker\")) | [.metadata.namespace,.metadata.name] | @tsv] | first // empty"); if [ -n "$row" ]; then ns=$(printf "%s" "$row" | cut -f1); pod=$(printf "%s" "$row" | cut -f2); metrics=$(kubectl --context k3d-ncp-local-compute-1 get --raw "/api/v1/namespaces/$ns/pods/$pod:9089/proxy/metrics" 2>/dev/null || true); registration=$(printf "%s\n" "$metrics" | grep -c "^pylon_registration_stream_connected.* 1$" || true); reverse=$(printf "%s\n" "$metrics" | grep -c "^pylon_reverse_tunnel_connected.* 1$" || true); if [ "$registration" -eq 5 ] && [ "$reverse" -ge 3 ]; then printf "registration=%s reverse=%s regions=2\n" "$registration" "$reverse"; exit 0; fi; fi; sleep 5; done; exit 1'` - invokeCommand = "/usr/bin/nvcf-cli --config /repo-root-placeholder/tests/bdd/fixtures/nvcf-cli-local.yaml function invoke" + + regionAWatchCommand = `/bin/bash -c 'set -eu; output=$(grpcurl -max-time 3 -cacert <(kubectl --context k3d-ncp-local-cp get secret stargate-quic-tls -n nvcf -o jsonpath="{.data.ca\.crt}" | base64 -d) -authority llm-request-router.nvcf.svc.cluster.local -import-path src/libraries/rust/stargate/crates/proto/proto -proto stargate.proto 127.0.0.1:50071 stargate.StargateControlPlane/WatchStargates 2>&1 || true); pods=$(kubectl --context k3d-ncp-local-cp get pods -n nvcf -l app.kubernetes.io/instance=llm-request-router,app.kubernetes.io/name=llm-request-router -o jsonpath="{range .items[*]}{.metadata.name}{\"\\n\"}{end}"); count=0; while IFS= read -r pod; do [ -z "$pod" ] && continue; printf "%s" "$output" | grep -Fq "$pod"; count=$((count + 1)); done <<<"$pods"; [ "$count" -eq 3 ]; printf "%s" "$output" | grep -Fq "https://region-b-watch.nvcf.svc.cluster.local:50071"; printf "region-a-deployment=%s remote-watch=https\n" "$count"'` + regionBWatchCommand = `/bin/bash -c 'set -eu; output=$(grpcurl -max-time 3 -cacert <(kubectl --context k3d-ncp-local-cp get secret stargate-quic-tls -n nvcf -o jsonpath="{.data.ca\.crt}" | base64 -d) -authority region-b-watch.nvcf.svc.cluster.local -import-path src/libraries/rust/stargate/crates/proto/proto -proto stargate.proto 127.0.0.1:50071 stargate.StargateControlPlane/WatchStargates 2>&1 || true); identities=$(printf "%s\n" "$output" | grep -Eo "llm-request-router-region-b-[0-9]+" | sort -u || true); expected=$(printf "llm-request-router-region-b-0\nllm-request-router-region-b-1\n"); [ "$identities" = "$expected" ]; count=$(printf "%s\n" "$identities" | grep -c .); [ "$count" -eq 2 ]; ! printf "%s" "$output" | grep -Eq "([0-9]{1,3}-){3}[0-9]{1,3}\."; printf "region-b-statefulset=%s tls=https\n" "$count"'` + pylonMetricsCommand = `/bin/sh -c 'set -eu; for attempt in $(seq 1 120); do row=$(kubectl --context k3d-ncp-local-compute-1 get pods -A -o json | jq -r "[.items[] | select(any(.spec.containers[]?; .name == \"llm-worker\")) | [.metadata.namespace,.metadata.name] | @tsv] | first // empty"); if [ -n "$row" ]; then ns=$(printf "%s" "$row" | cut -f1); pod=$(printf "%s" "$row" | cut -f2); metrics=$(kubectl --context k3d-ncp-local-compute-1 get --raw "/api/v1/namespaces/$ns/pods/$pod:9089/proxy/metrics" 2>/dev/null || true); registration=$(printf "%s\n" "$metrics" | grep -c "^pylon_registration_stream_connected.* 1$" || true); reverse=$(printf "%s\n" "$metrics" | grep -c "^pylon_reverse_tunnel_connected.* 1$" || true); if [ "$registration" -eq 5 ] && [ "$reverse" -ge 3 ]; then printf "registration=%s reverse=%s regions=2\n" "$registration" "$reverse"; exit 0; fi; fi; sleep 5; done; exit 1'` + grpcCertificateCommand = "kubectl --context k3d-ncp-local-cp get certificate llm-request-router-grpc-tls" + + " -n envoy-gateway-system -o jsonpath={.spec.dnsNames}" + invokeCommand = "/usr/bin/nvcf-cli --config /repo-root-placeholder/tests/bdd/fixtures/nvcf-cli-local.yaml function invoke" + " --inference-url /v1/chat/completions --model-name openai-compatible-sample" + " --request-body '{\"messages\":[{\"role\":\"user\",\"content\":\"bdd-registration-multiregion\"}]}' --timeout 120" ) + //revive:enable:line-length-limit suite := newWiringSuite(t, newFakeRunner(map[string]harness.Result{ "k3d cluster get ncp-local": {ExitCode: 1}, @@ -1224,6 +1228,10 @@ func TestMultiClusterHelmfileLLMRegistrationMultiregionFeatureFileWiresToSteps(t ExitCode: 0, Stdout: "region-b-statefulset=2 tls=https\n", }, + grpcCertificateCommand: { + ExitCode: 0, + Stdout: "[llm-request-router.nvcf.svc.cluster.local region-b-watch.nvcf.svc.cluster.local]", + }, pylonMetricsCommand: {ExitCode: 0, Stdout: "registration=5 reverse=3 regions=2\n"}, invokeCommand: { ExitCode: 0, @@ -1264,7 +1272,12 @@ func TestMultiClusterHelmfileLLMRegistrationMultiregionFeatureFileWiresToSteps(t if status != 0 { t.Fatalf("godog suite status = %d\n%s", status, out.String()) } - for _, command := range []string{regionAWatchCommand, regionBWatchCommand, pylonMetricsCommand} { + for _, command := range []string{ + grpcCertificateCommand, + regionAWatchCommand, + regionBWatchCommand, + pylonMetricsCommand, + } { if !commandRanExactly(suite.Runner.(*fakeRunner).runs, command) { t.Fatalf("exact multi-region observation command was not invoked: %s", command) } @@ -1277,6 +1290,31 @@ func TestMultiClusterHelmfileLLMRegistrationMultiregionFeatureFileWiresToSteps(t ) { t.Fatal("multi-region sample was not created as an LLM function") } + + environmentPath, err := dsl.HelmfileEnvironmentPath( + suite.Config.RepoRoot, + "self-managed", + "local-bdd-registration-multiregion", + ) + if err != nil { + t.Fatalf("resolve multi-region environment: %v", err) + } + for _, expectation := range []struct { + key string + want string + }{ + {key: "addons.llm.requestRouter.grpcTls.dnsNames[1]", want: "region-b-watch.nvcf.svc.cluster.local"}, + {key: "addons.llm.pki.dnsNames[2]", want: "region-b-watch.nvcf.svc.cluster.local"}, + {key: "addons.llm.pki.dnsNames[3]", want: "*.llm-request-router-region-b-headless.nvcf.svc.cluster.local"}, + } { + got, found, readErr := dsl.ReadYAMLKey(environmentPath, expectation.key) + if readErr != nil { + t.Fatalf("read %s: %v", expectation.key, readErr) + } + if !found || got != expectation.want { + t.Fatalf("%s = %q, found %t, want %q", expectation.key, got, found, expectation.want) + } + } } // TestSingleClusterHelmfileUpstreamImagesFeatureFileWiresToSteps runs the diff --git a/tests/bdd/region_b_script_test.go b/tests/bdd/region_b_script_test.go new file mode 100644 index 000000000..23b6e3044 --- /dev/null +++ b/tests/bdd/region_b_script_test.go @@ -0,0 +1,101 @@ +/* +SPDX-FileCopyrightText: Copyright (c) NVIDIA CORPORATION & AFFILIATES. All rights reserved. +SPDX-License-Identifier: Apache-2.0 + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package bdd_tmp + +import ( + "os" + "os/exec" + "path/filepath" + "strings" + "testing" +) + +func TestInstallLLMRegionBCreatesWatchAliasInBothClusters(t *testing.T) { + binDir := t.TempDir() + applyDir := t.TempDir() + + helmScript := `#!/usr/bin/env bash +set -euo pipefail +case " $* " in + *" get values "*) printf '{"llmRequestRouter":{}}\n' ;; + *) cat >/dev/null ;; +esac +` + kubectlScript := `#!/usr/bin/env bash +set -euo pipefail +context="" +previous="" +for argument in "$@"; do + if [[ "${previous}" == "--context" ]]; then + context="${argument}" + fi + previous="${argument}" +done +case " $* " in + *" get endpoints llm-request-router "*) printf '192.0.2.10' ;; + *" apply -f - "*) + cat >>"${FAKE_APPLY_DIR}/${context}.yaml" + printf '\n---\n' >>"${FAKE_APPLY_DIR}/${context}.yaml" + ;; +esac +` + jqScript := `#!/usr/bin/env bash +set -euo pipefail +cat +` + for name, body := range map[string]string{ + "helm": helmScript, + "jq": jqScript, + "kubectl": kubectlScript, + } { + if err := os.WriteFile(filepath.Join(binDir, name), []byte(body), 0o755); err != nil { + t.Fatalf("write fake %s: %v", name, err) + } + } + + cmd := exec.Command("bash", "scripts/install-llm-region-b.sh") + cmd.Env = append(os.Environ(), + "CONTROL_CONTEXT=bdd-control", + "COMPUTE_CONTEXT=bdd-compute", + "FAKE_APPLY_DIR="+applyDir, + "PATH="+binDir+":"+os.Getenv("PATH"), + "REPO_ROOT="+t.TempDir(), + ) + if output, err := cmd.CombinedOutput(); err != nil { + t.Fatalf("install region B: %v\n%s", err, output) + } + + for _, context := range []string{"bdd-control", "bdd-compute"} { + manifestPath := filepath.Join(applyDir, context+".yaml") + manifest, err := os.ReadFile(manifestPath) + if err != nil { + t.Fatalf("read %s aliases: %v", context, err) + } + for _, want := range []string{ + "kind: Service\nmetadata:\n name: region-b-watch", + "kind: Endpoints\nmetadata:\n name: region-b-watch", + "- ip: 192.0.2.10", + "name: llm-grpc", + "name: llm-quic", + } { + if !strings.Contains(string(manifest), want) { + t.Fatalf("%s aliases missing %q:\n%s", context, want, manifest) + } + } + } +} diff --git a/tests/bdd/scripts/install-llm-region-b.sh b/tests/bdd/scripts/install-llm-region-b.sh index 9b1ae9931..26a7e03f0 100755 --- a/tests/bdd/scripts/install-llm-region-b.sh +++ b/tests/bdd/scripts/install-llm-region-b.sh @@ -98,7 +98,8 @@ if [[ -z "${control_plane_ip}" ]]; then exit 1 fi -kubectl --context "${compute_context}" apply -f - <