From 2ef166f596a58a39fe3d81d7be60d40a4243b7e4 Mon Sep 17 00:00:00 2001 From: Stephanie Baum Date: Sat, 29 Aug 2026 23:53:15 -0700 Subject: [PATCH 01/10] test(bdd): add shared Pylon observability DSL Keep TLS WatchStargates targets and Pylon metric expectations visible while sharing repeated command, polling, and output parsing mechanics. Refs #1358 Signed-off-by: Stephanie Baum --- tests/bdd/PLAN.md | 30 +++ tests/bdd/dsl/registration.go | 106 +++++++++++ tests/bdd/dsl/registration_test.go | 189 +++++++++++++++++++ tests/bdd/scripts/observe-watch-stargates.sh | 75 ++++++++ tests/bdd/scripts/wait-pylon-metrics.sh | 132 +++++++++++++ tests/bdd/steps/assertion_steps.go | 37 ++++ tests/bdd/steps/context.go | 1 + tests/bdd/steps/registration_steps.go | 105 +++++++++++ tests/bdd/steps/steps_test.go | 103 ++++++++++ 9 files changed, 778 insertions(+) create mode 100644 tests/bdd/dsl/registration.go create mode 100644 tests/bdd/dsl/registration_test.go create mode 100644 tests/bdd/scripts/observe-watch-stargates.sh create mode 100644 tests/bdd/scripts/wait-pylon-metrics.sh create mode 100644 tests/bdd/steps/registration_steps.go diff --git a/tests/bdd/PLAN.md b/tests/bdd/PLAN.md index e53c0aecf..8b940a83b 100644 --- a/tests/bdd/PLAN.md +++ b/tests/bdd/PLAN.md @@ -122,6 +122,16 @@ refactor in every consumer; that is a feature. | `When I run command with a terminal:` (docstring) | Same as the docstring form, but stdin is attached to a pseudo-terminal so the child sees a TTY on fd 0. For commands that gate interactive-only behavior on a TTY, such as `nvcf-cli self-hosted up` (its auth-gate mints the admin token only when stdin is a terminal). No input is written; stdout and stderr are captured separately as usual. | | `When I export command output to environment variable {string}` | Exports the previous command's trimmed stdout under the named env var. Fails the step unless the prior command exited 0 and produced non-empty stdout. Snapshotted by the env Ledger; restored at suite teardown. | +#### Registration observability command adapters + +These steps wrap repeated client, trust-material, polling, and output-format +mechanics while keeping every operator-selected target and expectation visible. +They preserve the real command output for subsequent assertions. + +| Step | Command | +|------|---------| +| `When I successfully observe WatchStargates at {string} with TLS authority {string} using CA secret {string} in namespace {string} and context {string} for {string} seconds` | Reads the named CA certificate from the explicit Kubernetes secret and context, runs the public `WatchStargates` gRPC method against the visible endpoint and TLS authority, and requires a streamed response before accepting the expected client deadline. | + #### Function lifecycle command adapters These steps hide the repeated executable, config prefix, fixed subcommand, shell @@ -156,8 +166,11 @@ original order. Repeated options and empty values are preserved. | Step | Notes | |------|-------| | `Then the command exit code should be {int}` | Last-run exit code. | +| `Then the command should fail` | Requires a non-zero last-run exit code. It does not accept a runner error that prevented command execution and never records the failed command in the successful-command cache. | | `Then the command output should contain {string}` | Substring match on combined stdout + stderr. | | `Then the command output should not contain {string}` | Negative substring match. | +| `Then the command output should contain all:` (table) | Requires a `text` header and one or more strings. Every interpolated string must appear in combined stdout + stderr. | +| `Then the command output should contain one of:` (table) | Requires a `text` header and one or more strings. At least one interpolated string must appear in combined stdout + stderr. | | `Then file {string} should exist` | | | `Then yaml file {string} key {string} should equal {string}` | Reads the YAML file, walks the dotted key path, compares to the value (with `${VAR}` expansion). | | `Then yaml file {string} key {string} should not be empty` | Same key resolution; passes if the resolved value is non-empty. Use for non-deterministic outputs (cluster IDs, identity sources) where exact-value assertions are wrong. | @@ -179,6 +192,7 @@ original order. Repeated options and empty values are preserved. | `Then deployment {string} in namespace {string} using context {string} should complete rollout within {string}` | Runs `kubectl rollout status` for the named deployment with the explicit namespace, context, and timeout. Failure messages name the deployment without printing command output. | | `Then NVCFBackend {string} in namespace {string} using context {string} should report agent status {string} within {string}` | Waits for the named backend's `status.agentStatus` to equal the visible value using the explicit namespace, context, and timeout. Failure messages name the backend without printing resource output. | | `Then these Gateway API routes should be accepted and resolved using context {string} within {string}:` (table) | Requires `kind`, `name`, `namespace`, and `parent` headers. Waits for every named route to report both `Accepted=True` and `ResolvedRefs=True` for the named Gateway parent using the explicit context and timeout. The route kind is passed through without an allowlist. Failures name the table row, route, namespace, parent, and unmet condition without printing resource output. | +| `Then a pod containing container {string} using context {string} should report Pylon metrics within {string}:` (table) | Requires `metric`, `comparison`, and `count` headers. Polls the Pylon metrics endpoint of a running pod containing the visible container name. Each metric row counts connected series with value `1`; `comparison` is `exactly` or `at least`, and the expected non-negative count remains visible. | #### YAML comparison semantics @@ -514,6 +528,22 @@ func SubstituteFile(path, placeholder, replacement string) error // exists in the array. Extra objects in the array are fine. func JSONContainsRows(raw string, rows []map[string]string) error +// WatchStargatesCommand builds a TLS WatchStargates observation with an +// explicit endpoint, authority, CA source, Kubernetes context, and duration. +func WatchStargatesCommand(endpoint, authority, caSecret, namespace, kubeContext, durationSeconds string) (string, error) + +// PylonMetricExpectation describes an expected count of connected metric +// series exposed by one Pylon sidecar. +type PylonMetricExpectation struct { + Metric string + Comparison string + Count int +} + +// PylonMetricsCommand builds a bounded Pylon metrics observation for a pod +// selected by container name in an explicit Kubernetes context. +func PylonMetricsCommand(containerName, kubeContext, timeout string, expectations []PylonMetricExpectation) (string, error) + // FilesDoNotContain recursively inspects regular files under root and // fails if any interpolated fixed string appears. func FilesDoNotContain(root string, needles []string) error diff --git a/tests/bdd/dsl/registration.go b/tests/bdd/dsl/registration.go new file mode 100644 index 000000000..4b546bab9 --- /dev/null +++ b/tests/bdd/dsl/registration.go @@ -0,0 +1,106 @@ +/* +SPDX-FileCopyrightText: Copyright (c) NVIDIA CORPORATION & AFFILIATES. All rights reserved. +SPDX-License-Identifier: Apache-2.0 + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package dsl + +import ( + "fmt" + "regexp" + "strconv" + "strings" +) + +const ( + observeWatchStargatesScript = "tests/bdd/scripts/observe-watch-stargates.sh" + waitPylonMetricsScript = "tests/bdd/scripts/wait-pylon-metrics.sh" +) + +var prometheusMetricNameRE = regexp.MustCompile(`^[a-zA-Z_:][a-zA-Z0-9_:]*$`) + +// PylonMetricExpectation describes an expected count of connected metric +// series exposed by one Pylon sidecar. +type PylonMetricExpectation struct { + Metric string + Comparison string + Count int +} + +// WatchStargatesCommand builds a TLS WatchStargates observation with an +// explicit endpoint, authority, CA source, Kubernetes context, and duration. +func WatchStargatesCommand(endpoint, authority, caSecret, namespace, kubeContext, durationSeconds string) (string, error) { + values := []*string{&endpoint, &authority, &caSecret, &namespace, &kubeContext, &durationSeconds} + labels := []string{"endpoint", "TLS authority", "CA secret", "namespace", "kube context", "duration seconds"} + for index := range values { + *values[index] = strings.TrimSpace(Interpolate(*values[index])) + if *values[index] == "" { + return "", fmt.Errorf("%s is empty", labels[index]) + } + } + duration, err := strconv.Atoi(durationSeconds) + if err != nil || duration <= 0 { + return "", fmt.Errorf("duration seconds must be a positive integer") + } + + return BuildCommand( + "bash", + observeWatchStargatesScript, + endpoint, + authority, + caSecret, + namespace, + kubeContext, + durationSeconds, + ), nil +} + +// PylonMetricsCommand builds a bounded Pylon metrics observation for a pod +// selected by container name in an explicit Kubernetes context. +func PylonMetricsCommand(containerName, kubeContext, timeout string, expectations []PylonMetricExpectation) (string, error) { + containerName = strings.TrimSpace(Interpolate(containerName)) + kubeContext = strings.TrimSpace(Interpolate(kubeContext)) + timeout = strings.TrimSpace(Interpolate(timeout)) + if containerName == "" { + return "", fmt.Errorf("container name is empty") + } + if kubeContext == "" { + return "", fmt.Errorf("kube context is empty") + } + if timeout == "" { + return "", fmt.Errorf("timeout is empty") + } + if len(expectations) == 0 { + return "", fmt.Errorf("Pylon metric expectations are empty") + } + + args := []string{"bash", waitPylonMetricsScript, containerName, kubeContext, timeout} + for index, expectation := range expectations { + expectation.Metric = strings.TrimSpace(Interpolate(expectation.Metric)) + expectation.Comparison = strings.TrimSpace(Interpolate(expectation.Comparison)) + if !prometheusMetricNameRE.MatchString(expectation.Metric) { + return "", fmt.Errorf("metric row %d has invalid metric name %q", index+1, expectation.Metric) + } + if expectation.Comparison != "exactly" && expectation.Comparison != "at least" { + return "", fmt.Errorf("metric row %d comparison must be exactly or at least", index+1) + } + if expectation.Count < 0 { + return "", fmt.Errorf("metric row %d count must be non-negative", index+1) + } + args = append(args, expectation.Metric, expectation.Comparison, strconv.Itoa(expectation.Count)) + } + + return BuildCommand(args...), nil +} diff --git a/tests/bdd/dsl/registration_test.go b/tests/bdd/dsl/registration_test.go new file mode 100644 index 000000000..606765864 --- /dev/null +++ b/tests/bdd/dsl/registration_test.go @@ -0,0 +1,189 @@ +/* +SPDX-FileCopyrightText: Copyright (c) NVIDIA CORPORATION & AFFILIATES. All rights reserved. +SPDX-License-Identifier: Apache-2.0 + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package dsl + +import ( + "os" + "os/exec" + "path/filepath" + "strings" + "testing" +) + +func TestWatchStargatesCommandKeepsObservationInputsExplicit(t *testing.T) { + t.Setenv("BDD_WATCH_CONTEXT", "k3d-ncp-local-cp") + got, err := WatchStargatesCommand( + "127.0.0.1:50071", + "llm-request-router.nvcf.svc.cluster.local", + "stargate-quic-tls", + "nvcf", + "${BDD_WATCH_CONTEXT}", + "3", + ) + if err != nil { + t.Fatalf("build WatchStargates command: %v", err) + } + want := "bash tests/bdd/scripts/observe-watch-stargates.sh 127.0.0.1:50071 llm-request-router.nvcf.svc.cluster.local stargate-quic-tls nvcf k3d-ncp-local-cp 3" + if got != want { + t.Fatalf("command = %q, want %q", got, want) + } +} + +func TestWatchStargatesCommandRejectsMissingOrInvalidInputs(t *testing.T) { + tests := []struct { + name string + endpoint string + authority string + duration string + }{ + {name: "empty endpoint", endpoint: "", authority: "router.nvcf.svc", duration: "3"}, + {name: "empty authority", endpoint: "127.0.0.1:50071", authority: "", duration: "3"}, + {name: "invalid duration", endpoint: "127.0.0.1:50071", authority: "router.nvcf.svc", duration: "3s"}, + } + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + if _, err := WatchStargatesCommand(test.endpoint, test.authority, "tls", "nvcf", "context", test.duration); err == nil { + t.Fatal("expected validation error") + } + }) + } +} + +func TestPylonMetricsCommandKeepsExpectationsExplicit(t *testing.T) { + got, err := PylonMetricsCommand("llm-worker", "k3d-ncp-local-compute-1", "10m", []PylonMetricExpectation{ + {Metric: "pylon_registration_stream_connected", Comparison: "exactly", Count: 5}, + {Metric: "pylon_reverse_tunnel_connected", Comparison: "at least", Count: 3}, + }) + if err != nil { + t.Fatalf("build Pylon metrics command: %v", err) + } + want := "bash tests/bdd/scripts/wait-pylon-metrics.sh llm-worker k3d-ncp-local-compute-1 10m pylon_registration_stream_connected exactly 5 pylon_reverse_tunnel_connected 'at least' 3" + if got != want { + t.Fatalf("command = %q, want %q", got, want) + } +} + +func TestPylonMetricsCommandRejectsInvalidExpectations(t *testing.T) { + tests := []struct { + name string + expectations []PylonMetricExpectation + }{ + {name: "empty", expectations: nil}, + {name: "invalid metric", expectations: []PylonMetricExpectation{{Metric: "metric name", Comparison: "exactly", Count: 1}}}, + {name: "invalid comparison", expectations: []PylonMetricExpectation{{Metric: "metric_name", Comparison: "more than", Count: 1}}}, + {name: "negative count", expectations: []PylonMetricExpectation{{Metric: "metric_name", Comparison: "exactly", Count: -1}}}, + } + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + if _, err := PylonMetricsCommand("llm-worker", "context", "10m", test.expectations); err == nil { + t.Fatal("expected validation error") + } + }) + } +} + +func TestObserveWatchStargatesScriptAcceptsSnapshotThenDeadline(t *testing.T) { + fakeBin := t.TempDir() + writeExecutable(t, filepath.Join(fakeBin, "kubectl"), "#!/bin/sh\nprintf 'dGVzdC1jYQ=='\n") + writeExecutable(t, filepath.Join(fakeBin, "grpcurl"), `#!/bin/sh +printf '{\n "stargates": []\n}\n' +printf 'ERROR:\n Code: DeadlineExceeded\n Message: context deadline exceeded\n' >&2 +exit 1 +`) + + output, err := runRegistrationScript(t, fakeBin, "observe-watch-stargates.sh", "127.0.0.1:50071", "router.nvcf.svc", "tls", "nvcf", "context", "3") + if err != nil { + t.Fatalf("observe WatchStargates: %v\n%s", err, output) + } + for _, want := range []string{`"stargates"`, "DeadlineExceeded"} { + if !strings.Contains(output, want) { + t.Fatalf("output = %q, want %q", output, want) + } + } +} + +func TestObserveWatchStargatesScriptRejectsDeadlineWithoutSnapshot(t *testing.T) { + fakeBin := t.TempDir() + writeExecutable(t, filepath.Join(fakeBin, "kubectl"), "#!/bin/sh\nprintf 'dGVzdC1jYQ=='\n") + writeExecutable(t, filepath.Join(fakeBin, "grpcurl"), "#!/bin/sh\nprintf 'context deadline exceeded\\n' >&2\nexit 1\n") + + output, err := runRegistrationScript(t, fakeBin, "observe-watch-stargates.sh", "127.0.0.1:50071", "router.nvcf.svc", "tls", "nvcf", "context", "3") + if err == nil { + t.Fatal("expected missing snapshot failure") + } + if !strings.Contains(output, "did not return a streamed snapshot") { + t.Fatalf("output = %q, want missing snapshot diagnostic", output) + } +} + +func TestWaitPylonMetricsScriptCountsConnectedSeries(t *testing.T) { + fakeBin := t.TempDir() + writeExecutable(t, filepath.Join(fakeBin, "kubectl"), `#!/bin/sh +case "$*" in + *"get pods -A -o json"*) printf '{"items":[]}' ;; + *) + printf 'pylon_registration_stream_connected{router="a"} 1\n' + printf 'pylon_registration_stream_connected{router="b"} 1\n' + printf 'pylon_registration_stream_connected{router="c"} 1\n' + printf 'pylon_reverse_tunnel_connected{router="a"} 1\n' + printf 'pylon_reverse_tunnel_connected{router="b"} 1\n' + printf 'pylon_reverse_tunnel_connected{router="c"} 1\n' + ;; +esac +`) + writeExecutable(t, filepath.Join(fakeBin, "jq"), "#!/bin/sh\nprintf 'functions\\tworker-0\\n'\n") + + output, err := runRegistrationScript( + t, + fakeBin, + "wait-pylon-metrics.sh", + "llm-worker", + "k3d-ncp-local-compute-1", + "1s", + "pylon_registration_stream_connected", + "exactly", + "3", + "pylon_reverse_tunnel_connected", + "at least", + "3", + ) + if err != nil { + t.Fatalf("wait for Pylon metrics: %v\n%s", err, output) + } + for _, want := range []string{"pylon_registration_stream_connected=3", "pylon_reverse_tunnel_connected=3"} { + if !strings.Contains(output, want) { + t.Fatalf("output = %q, want %q", output, want) + } + } +} + +func runRegistrationScript(t *testing.T, fakeBin, scriptName string, args ...string) (string, error) { + t.Helper() + script := filepath.Join("..", "scripts", scriptName) + command := exec.Command("bash", append([]string{script}, args...)...) + command.Env = append(os.Environ(), "PATH="+fakeBin+string(os.PathListSeparator)+os.Getenv("PATH")) + output, err := command.CombinedOutput() + return string(output), err +} + +func writeExecutable(t *testing.T, path, body string) { + t.Helper() + if err := os.WriteFile(path, []byte(body), 0o755); err != nil { + t.Fatalf("write executable %s: %v", path, err) + } +} diff --git a/tests/bdd/scripts/observe-watch-stargates.sh b/tests/bdd/scripts/observe-watch-stargates.sh new file mode 100644 index 000000000..139ed8706 --- /dev/null +++ b/tests/bdd/scripts/observe-watch-stargates.sh @@ -0,0 +1,75 @@ +#!/usr/bin/env bash +# SPDX-FileCopyrightText: Copyright (c) NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +set -euo pipefail + +if [[ $# -ne 6 ]]; then + echo "usage: $0 " >&2 + exit 64 +fi + +endpoint="$1" +tls_authority="$2" +ca_secret="$3" +namespace="$4" +kube_context="$5" +duration_seconds="$6" + +for value_name in endpoint tls_authority ca_secret namespace kube_context; do + if [[ -z "${!value_name}" ]]; then + echo "$value_name must be non-empty" >&2 + exit 64 + fi +done +if ! [[ "$duration_seconds" =~ ^[1-9][0-9]*$ ]]; then + echo "duration-seconds must be a positive integer, got: $duration_seconds" >&2 + exit 64 +fi +for tool in kubectl base64 grpcurl; do + if ! command -v "$tool" >/dev/null 2>&1; then + echo "required tool not found: $tool" >&2 + exit 127 + fi +done + +script_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd -P)" +repo_root="$(cd "$script_dir/../../.." && pwd -P)" +proto_path="$repo_root/src/libraries/rust/stargate/crates/proto/proto" +ca_file="$(mktemp "${TMPDIR:-/tmp}/nvcf-bdd-watch-ca.XXXXXX")" +trap 'rm -f "$ca_file"' EXIT + +kubectl --context "$kube_context" get secret "$ca_secret" -n "$namespace" \ + -o 'jsonpath={.data.ca\.crt}' | base64 -d >"$ca_file" +if [[ ! -s "$ca_file" ]]; then + echo "CA secret $namespace/$ca_secret did not contain ca.crt" >&2 + exit 1 +fi + +set +e +output="$(grpcurl \ + -max-time "$duration_seconds" \ + -cacert "$ca_file" \ + -authority "$tls_authority" \ + -import-path "$proto_path" \ + -proto stargate.proto \ + "$endpoint" \ + stargate.StargateControlPlane/WatchStargates 2>&1)" +grpcurl_status=$? +set -e + +printf '%s\n' "$output" + +if [[ "$grpcurl_status" -eq 0 ]]; then + echo "WatchStargates ended before the observation deadline" >&2 + exit 1 +fi +if ! grep -Eq '^[[:space:]]*\{' <<<"$output"; then + echo "WatchStargates did not return a streamed snapshot" >&2 + exit 1 +fi +if ! grep -Eiq 'DeadlineExceeded|context deadline exceeded' <<<"$output"; then + echo "WatchStargates failed before the expected observation deadline" >&2 + exit 1 +fi + diff --git a/tests/bdd/scripts/wait-pylon-metrics.sh b/tests/bdd/scripts/wait-pylon-metrics.sh new file mode 100644 index 000000000..2eb697598 --- /dev/null +++ b/tests/bdd/scripts/wait-pylon-metrics.sh @@ -0,0 +1,132 @@ +#!/usr/bin/env bash +# SPDX-FileCopyrightText: Copyright (c) NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +set -euo pipefail + +if [[ $# -lt 6 || $(( ( $# - 3 ) % 3 )) -ne 0 ]]; then + echo "usage: $0 [...]" >&2 + exit 64 +fi + +container_name="$1" +kube_context="$2" +timeout="$3" +shift 3 + +if [[ -z "$container_name" || -z "$kube_context" ]]; then + echo "container and kube-context must be non-empty" >&2 + exit 64 +fi +if ! [[ "$timeout" =~ ^([1-9][0-9]*)(s|m|h)$ ]]; then + echo "timeout must be a positive duration ending in s, m, or h, got: $timeout" >&2 + exit 64 +fi +case "${BASH_REMATCH[2]}" in + s) timeout_seconds="${BASH_REMATCH[1]}" ;; + m) timeout_seconds=$(( BASH_REMATCH[1] * 60 )) ;; + h) timeout_seconds=$(( BASH_REMATCH[1] * 3600 )) ;; +esac +for tool in kubectl jq awk; do + if ! command -v "$tool" >/dev/null 2>&1; then + echo "required tool not found: $tool" >&2 + exit 127 + fi +done + +metric_names=() +comparisons=() +expected_counts=() +while [[ $# -gt 0 ]]; do + metric="$1" + comparison="$2" + expected_count="$3" + shift 3 + + if ! [[ "$metric" =~ ^[a-zA-Z_:][a-zA-Z0-9_:]*$ ]]; then + echo "invalid Prometheus metric name: $metric" >&2 + exit 64 + fi + if [[ "$comparison" != "exactly" && "$comparison" != "at least" ]]; then + echo "comparison must be exactly or at least, got: $comparison" >&2 + exit 64 + fi + if ! [[ "$expected_count" =~ ^[0-9]+$ ]]; then + echo "count must be a non-negative integer, got: $expected_count" >&2 + exit 64 + fi + + metric_names+=("$metric") + comparisons+=("$comparison") + expected_counts+=("$expected_count") +done + +connected_series_count() { + local metric="$1" + awk -v metric="$metric" ' + { + series = $1 + if ((series == metric || index(series, metric "{") == 1) && $NF == "1") { + count++ + } + } + END { print count + 0 } + ' +} + +deadline=$(( $(date +%s) + timeout_seconds )) +last_summary="no running pod containing container $container_name" + +while true; do + pod_row="$(kubectl --context "$kube_context" get pods -A -o json | jq -r --arg container "$container_name" ' + [.items[] + | select(.metadata.deletionTimestamp == null) + | select(.status.phase == "Running") + | select(any(.spec.containers[]?; .name == $container)) + | [.metadata.namespace, .metadata.name] + | @tsv] + | first // empty + ' || true)" + + if [[ -n "$pod_row" ]]; then + read -r namespace pod_name <<<"$pod_row" + metrics="$(kubectl --context "$kube_context" get --raw "/api/v1/namespaces/$namespace/pods/$pod_name:9089/proxy/metrics" 2>/dev/null || true)" + all_match=true + summaries=() + + for index in "${!metric_names[@]}"; do + metric="${metric_names[$index]}" + comparison="${comparisons[$index]}" + expected_count="${expected_counts[$index]}" + observed_count="$(printf '%s\n' "$metrics" | connected_series_count "$metric")" + summaries+=("$metric=$observed_count") + + case "$comparison" in + exactly) + [[ "$observed_count" -eq "$expected_count" ]] || all_match=false + ;; + "at least") + [[ "$observed_count" -ge "$expected_count" ]] || all_match=false + ;; + esac + done + + last_summary="${summaries[*]}" + if [[ "$all_match" == true ]]; then + printf '%s\n' "${summaries[@]}" + exit 0 + fi + fi + + now="$(date +%s)" + if [[ "$now" -ge "$deadline" ]]; then + echo "timed out after $timeout waiting for Pylon metrics in context $kube_context ($last_summary)" >&2 + exit 1 + fi + remaining=$(( deadline - now )) + sleep_seconds=5 + if [[ "$remaining" -lt "$sleep_seconds" ]]; then + sleep_seconds="$remaining" + fi + sleep "$sleep_seconds" +done diff --git a/tests/bdd/steps/assertion_steps.go b/tests/bdd/steps/assertion_steps.go index dcd5bac6a..0b036b912 100644 --- a/tests/bdd/steps/assertion_steps.go +++ b/tests/bdd/steps/assertion_steps.go @@ -32,8 +32,11 @@ import ( // values from the feature file. func registerAssertionSteps(ctx *godog.ScenarioContext, sc *ScenarioContext) { ctx.Step(`^the command exit code should be (\d+)$`, sc.commandExitCodeShouldBe) + ctx.Step(`^the command should fail$`, sc.commandShouldFail) ctx.Step(`^the command output should contain "([^"]*)"$`, sc.commandOutputShouldContain) ctx.Step(`^the command output should not contain "([^"]*)"$`, sc.commandOutputShouldNotContain) + ctx.Step(`^the command output should contain all:$`, sc.commandOutputShouldContainAll) + ctx.Step(`^the command output should contain one of:$`, sc.commandOutputShouldContainOneOf) ctx.Step(`^file "([^"]*)" should exist$`, sc.fileShouldExist) ctx.Step(`^yaml file "([^"]*)" key "([^"]*)" should equal "([^"]*)"$`, sc.yamlFileKeyShouldEqual) ctx.Step(`^yaml file "([^"]*)" key "([^"]*)" should not be empty$`, sc.yamlFileKeyShouldNotBeEmpty) @@ -87,6 +90,13 @@ func (sc *ScenarioContext) commandExitCodeShouldBe(expected int) error { return nil } +func (sc *ScenarioContext) commandShouldFail() error { + if sc.LastResult.ExitCode == 0 { + return fmt.Errorf("exit code = 0, want non-zero (see %s for stdout/stderr)", sc.Suite.Config.CommandLogDir) + } + return nil +} + func (sc *ScenarioContext) commandOutputShouldContain(needle string) error { combined := combinedOutput(sc.LastResult) resolved := dsl.Interpolate(needle) @@ -105,6 +115,33 @@ func (sc *ScenarioContext) commandOutputShouldNotContain(needle string) error { return nil } +func (sc *ScenarioContext) commandOutputShouldContainAll(table *godog.Table) error { + needles, err := tableToSingleColumn(table, "text") + if err != nil { + return err + } + for _, needle := range needles { + if err := sc.commandOutputShouldContain(needle); err != nil { + return err + } + } + return nil +} + +func (sc *ScenarioContext) commandOutputShouldContainOneOf(table *godog.Table) error { + needles, err := tableToSingleColumn(table, "text") + if err != nil { + return err + } + combined := combinedOutput(sc.LastResult) + for _, needle := range needles { + if strings.Contains(combined, dsl.Interpolate(needle)) { + return nil + } + } + return fmt.Errorf("output does not contain any of the %d expected values", len(needles)) +} + func (sc *ScenarioContext) yamlFileKeyShouldEqual(path, key, expected string) error { resolvedPath := sc.resolvePath(dsl.Interpolate(path)) got, found, err := dsl.ReadYAMLKey(resolvedPath, key) diff --git a/tests/bdd/steps/context.go b/tests/bdd/steps/context.go index 55acfc397..a88bed910 100644 --- a/tests/bdd/steps/context.go +++ b/tests/bdd/steps/context.go @@ -80,6 +80,7 @@ func RegisterAll(ctx *godog.ScenarioContext, sc *ScenarioContext) { registerFileSteps(ctx, sc) registerCommandSteps(ctx, sc) registerNVCFCLISteps(ctx, sc) + registerRegistrationSteps(ctx, sc) registerAssertionSteps(ctx, sc) registerInfraSteps(ctx, sc) } diff --git a/tests/bdd/steps/registration_steps.go b/tests/bdd/steps/registration_steps.go new file mode 100644 index 000000000..74f28dae4 --- /dev/null +++ b/tests/bdd/steps/registration_steps.go @@ -0,0 +1,105 @@ +/* +SPDX-FileCopyrightText: Copyright (c) NVIDIA CORPORATION & AFFILIATES. All rights reserved. +SPDX-License-Identifier: Apache-2.0 + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package steps + +import ( + "context" + "fmt" + "strconv" + "strings" + + "github.com/cucumber/godog" + + "nvcf-bdd/dsl" +) + +func registerRegistrationSteps(ctx *godog.ScenarioContext, sc *ScenarioContext) { + ctx.Step(`^I successfully observe WatchStargates at "([^"]*)" with TLS authority "([^"]*)" using CA secret "([^"]*)" in namespace "([^"]*)" and context "([^"]*)" for "([^"]*)" seconds$`, sc.iSuccessfullyObserveWatchStargates) + ctx.Step(`^a pod containing container "([^"]*)" using context "([^"]*)" should report Pylon metrics within "([^"]*)":$`, sc.podShouldReportPylonMetrics) +} + +func (sc *ScenarioContext) iSuccessfullyObserveWatchStargates( + ctx context.Context, + endpoint, + authority, + caSecret, + namespace, + kubeContext, + durationSeconds string, +) error { + command, err := dsl.WatchStargatesCommand(endpoint, authority, caSecret, namespace, kubeContext, durationSeconds) + if err != nil { + return err + } + if err := sc.runResolvedSuccessfully(ctx, command); err != nil { + return fmt.Errorf("WatchStargates at %q with TLS authority %q failed: %w", dsl.Interpolate(endpoint), dsl.Interpolate(authority), err) + } + return nil +} + +func (sc *ScenarioContext) podShouldReportPylonMetrics( + ctx context.Context, + containerName, + kubeContext, + timeout string, + table *godog.Table, +) error { + expectations, err := tableToPylonMetricExpectations(table) + if err != nil { + return err + } + command, err := dsl.PylonMetricsCommand(containerName, kubeContext, timeout, expectations) + if err != nil { + return err + } + if err := sc.runResolvedSuccessfully(ctx, command); err != nil { + return fmt.Errorf("pod containing container %q did not report expected Pylon metrics: %w", dsl.Interpolate(containerName), err) + } + return nil +} + +func tableToPylonMetricExpectations(table *godog.Table) ([]dsl.PylonMetricExpectation, error) { + if table == nil || len(table.Rows) < 2 { + return nil, fmt.Errorf("table must have metric, comparison, and count headers and at least one data row") + } + headers := table.Rows[0].Cells + if len(headers) != 3 || + strings.TrimSpace(headers[0].Value) != "metric" || + strings.TrimSpace(headers[1].Value) != "comparison" || + strings.TrimSpace(headers[2].Value) != "count" { + return nil, fmt.Errorf("table headers must be metric, comparison, and count") + } + + expectations := make([]dsl.PylonMetricExpectation, 0, len(table.Rows)-1) + for index, row := range table.Rows[1:] { + if len(row.Cells) != len(headers) { + return nil, fmt.Errorf("row %d has %d cells, expected %d", index+1, len(row.Cells), len(headers)) + } + countText := strings.TrimSpace(dsl.Interpolate(row.Cells[2].Value)) + count, err := strconv.Atoi(countText) + if err != nil || count < 0 { + return nil, fmt.Errorf("row %d count must be a non-negative integer", index+1) + } + expectations = append(expectations, dsl.PylonMetricExpectation{ + Metric: row.Cells[0].Value, + Comparison: row.Cells[1].Value, + Count: count, + }) + } + return expectations, nil +} diff --git a/tests/bdd/steps/steps_test.go b/tests/bdd/steps/steps_test.go index 1189730d8..af21d385f 100644 --- a/tests/bdd/steps/steps_test.go +++ b/tests/bdd/steps/steps_test.go @@ -1418,6 +1418,109 @@ func TestCommandOutputContainsAssertion(t *testing.T) { } } +func TestCommandShouldFailAcceptsNonZeroWithoutCaching(t *testing.T) { + sc, _ := newScenarioContext(t) + sc.LastCommand = "grpcurl rejected-call" + sc.LastResult = harness.Result{ExitCode: 1, Stderr: "certificate is not trusted"} + + if err := sc.commandShouldFail(); err != nil { + t.Fatalf("assert command failure: %v", err) + } + if sc.Suite.Cache.Has(sc.LastCommand) { + t.Fatal("failed command should not enter the successful-command cache") + } + + sc.LastResult = harness.Result{ExitCode: 0} + if err := sc.commandShouldFail(); err == nil { + t.Fatal("expected successful command to fail the negative assertion") + } +} + +func TestCommandOutputTableAssertionsInterpolateExpectedText(t *testing.T) { + sc, _ := newScenarioContext(t) + t.Setenv("BDD_EXPECTED_DIAGNOSTIC", "certificate is not trusted") + sc.LastResult = harness.Result{ + Stdout: "request rejected\n", + Stderr: "certificate is not trusted\ncontext deadline exceeded\n", + } + + all := docTable(t, [][]string{ + {"text"}, + {"${BDD_EXPECTED_DIAGNOSTIC}"}, + {"context deadline exceeded"}, + }) + if err := sc.commandOutputShouldContainAll(all); err != nil { + t.Fatalf("contain all: %v", err) + } + + oneOf := docTable(t, [][]string{ + {"text"}, + {"certificate signed by unknown authority"}, + {"${BDD_EXPECTED_DIAGNOSTIC}"}, + }) + if err := sc.commandOutputShouldContainOneOf(oneOf); err != nil { + t.Fatalf("contain one of: %v", err) + } +} + +func TestISuccessfullyObserveWatchStargatesRunsExplicitCommand(t *testing.T) { + sc, fake := newScenarioContext(t) + fake.result = harness.Result{ExitCode: 0, Stdout: "{\n \"stargates\": []\n}\n"} + + err := sc.iSuccessfullyObserveWatchStargates( + context.Background(), + "127.0.0.1:50071", + "llm-request-router.nvcf.svc.cluster.local", + "stargate-quic-tls", + "nvcf", + "k3d-ncp-local-cp", + "3", + ) + if err != nil { + t.Fatalf("observe WatchStargates: %v", err) + } + want := "bash tests/bdd/scripts/observe-watch-stargates.sh 127.0.0.1:50071 llm-request-router.nvcf.svc.cluster.local stargate-quic-tls nvcf k3d-ncp-local-cp 3" + if len(fake.runs) != 1 || fake.runs[0].command != want { + t.Fatalf("runs = %#v, want %q", fake.runs, want) + } + if !strings.Contains(sc.LastResult.Stdout, "stargates") { + t.Fatalf("last result = %#v, want preserved WatchStargates output", sc.LastResult) + } +} + +func TestPodShouldReportPylonMetricsRunsVisibleExpectations(t *testing.T) { + sc, fake := newScenarioContext(t) + fake.result = harness.Result{ExitCode: 0} + table := docTable(t, [][]string{ + {"metric", "comparison", "count"}, + {"pylon_registration_stream_connected", "exactly", "5"}, + {"pylon_reverse_tunnel_connected", "at least", "3"}, + }) + + if err := sc.podShouldReportPylonMetrics(context.Background(), "llm-worker", "k3d-ncp-local-compute-1", "10m", table); err != nil { + t.Fatalf("observe Pylon metrics: %v", err) + } + want := "bash tests/bdd/scripts/wait-pylon-metrics.sh llm-worker k3d-ncp-local-compute-1 10m pylon_registration_stream_connected exactly 5 pylon_reverse_tunnel_connected 'at least' 3" + if len(fake.runs) != 1 || fake.runs[0].command != want { + t.Fatalf("runs = %#v, want %q", fake.runs, want) + } +} + +func TestPylonMetricTableRejectsInvalidStructureBeforeRunning(t *testing.T) { + sc, fake := newScenarioContext(t) + table := docTable(t, [][]string{ + {"metric", "comparison", "count"}, + {"pylon_registration_stream_connected", "exactly", "not-a-count"}, + }) + + if err := sc.podShouldReportPylonMetrics(context.Background(), "llm-worker", "context", "10m", table); err == nil { + t.Fatal("expected invalid count error") + } + if len(fake.runs) != 0 { + t.Fatalf("runs = %d, want 0 before table validation", len(fake.runs)) + } +} + func TestJSONOutputContainsRowsAssertion(t *testing.T) { sc, _ := newScenarioContext(t) sc.LastResult = harness.Result{Stdout: `[{"name":"api","namespace":"nvcf"}]`} From e9c456092e474bfa21ce9f339bba6e6f5d57e8a6 Mon Sep 17 00:00:00 2001 From: Stephanie Baum Date: Sun, 30 Aug 2026 00:39:24 -0700 Subject: [PATCH 02/10] fix(bdd): harden registration observability assertions Reject transport failures, premature watch termination, ambiguous Pylon targets, empty resolved output expectations, and timestamped sample miscounts. Refs #1358 Signed-off-by: Stephanie Baum --- tests/bdd/PLAN.md | 17 +-- tests/bdd/dsl/registration.go | 15 ++- tests/bdd/dsl/registration_test.go | 106 +++++++++++++++++-- tests/bdd/scripts/observe-watch-stargates.sh | 26 +++-- tests/bdd/scripts/wait-pylon-metrics.sh | 84 +++++++++------ tests/bdd/steps/assertion_steps.go | 41 +++++-- tests/bdd/steps/registration_steps.go | 9 +- tests/bdd/steps/steps_test.go | 37 ++++++- 8 files changed, 256 insertions(+), 79 deletions(-) diff --git a/tests/bdd/PLAN.md b/tests/bdd/PLAN.md index 8b940a83b..e1b7ce77b 100644 --- a/tests/bdd/PLAN.md +++ b/tests/bdd/PLAN.md @@ -167,10 +167,10 @@ original order. Repeated options and empty values are preserved. |------|-------| | `Then the command exit code should be {int}` | Last-run exit code. | | `Then the command should fail` | Requires a non-zero last-run exit code. It does not accept a runner error that prevented command execution and never records the failed command in the successful-command cache. | -| `Then the command output should contain {string}` | Substring match on combined stdout + stderr. | -| `Then the command output should not contain {string}` | Negative substring match. | -| `Then the command output should contain all:` (table) | Requires a `text` header and one or more strings. Every interpolated string must appear in combined stdout + stderr. | -| `Then the command output should contain one of:` (table) | Requires a `text` header and one or more strings. At least one interpolated string must appear in combined stdout + stderr. | +| `Then the command output should contain {string}` | Substring match on combined stdout + stderr. The interpolated value must not be empty or whitespace-only. | +| `Then the command output should not contain {string}` | Negative substring match. The interpolated value must not be empty or whitespace-only. | +| `Then the command output should contain all:` (table) | Requires a `text` header and one or more strings. Every interpolated string must be non-empty and appear in combined stdout + stderr. | +| `Then the command output should contain one of:` (table) | Requires a `text` header and one or more strings. Every interpolated candidate must be non-empty, and at least one must appear in combined stdout + stderr. | | `Then file {string} should exist` | | | `Then yaml file {string} key {string} should equal {string}` | Reads the YAML file, walks the dotted key path, compares to the value (with `${VAR}` expansion). | | `Then yaml file {string} key {string} should not be empty` | Same key resolution; passes if the resolved value is non-empty. Use for non-deterministic outputs (cluster IDs, identity sources) where exact-value assertions are wrong. | @@ -192,7 +192,7 @@ original order. Repeated options and empty values are preserved. | `Then deployment {string} in namespace {string} using context {string} should complete rollout within {string}` | Runs `kubectl rollout status` for the named deployment with the explicit namespace, context, and timeout. Failure messages name the deployment without printing command output. | | `Then NVCFBackend {string} in namespace {string} using context {string} should report agent status {string} within {string}` | Waits for the named backend's `status.agentStatus` to equal the visible value using the explicit namespace, context, and timeout. Failure messages name the backend without printing resource output. | | `Then these Gateway API routes should be accepted and resolved using context {string} within {string}:` (table) | Requires `kind`, `name`, `namespace`, and `parent` headers. Waits for every named route to report both `Accepted=True` and `ResolvedRefs=True` for the named Gateway parent using the explicit context and timeout. The route kind is passed through without an allowlist. Failures name the table row, route, namespace, parent, and unmet condition without printing resource output. | -| `Then a pod containing container {string} using context {string} should report Pylon metrics within {string}:` (table) | Requires `metric`, `comparison`, and `count` headers. Polls the Pylon metrics endpoint of a running pod containing the visible container name. Each metric row counts connected series with value `1`; `comparison` is `exactly` or `at least`, and the expected non-negative count remains visible. | +| `Then every Pylon for function {string} using container {string} and context {string} should report metrics within {string}:` (table) | Requires `metric`, `comparison`, and `count` headers. Polls every running pod selected by the visible `function-name` annotation and container name. Each pod must expose non-empty metrics, and each metric row counts connected series whose sample value is `1`; `comparison` is `exactly` or `at least`, and the expected non-negative count remains visible. Discovery, parsing, and scrape failures remain failures rather than zero metric counts. | #### YAML comparison semantics @@ -540,9 +540,10 @@ type PylonMetricExpectation struct { Count int } -// PylonMetricsCommand builds a bounded Pylon metrics observation for a pod -// selected by container name in an explicit Kubernetes context. -func PylonMetricsCommand(containerName, kubeContext, timeout string, expectations []PylonMetricExpectation) (string, error) +// PylonMetricsCommand builds a Pylon metrics observation for every running +// Pylon pod selected by function name and container name in an explicit +// Kubernetes context and polling window. +func PylonMetricsCommand(functionName, containerName, kubeContext, timeout string, expectations []PylonMetricExpectation) (string, error) // FilesDoNotContain recursively inspects regular files under root and // fails if any interpolated fixed string appears. diff --git a/tests/bdd/dsl/registration.go b/tests/bdd/dsl/registration.go index 4b546bab9..1a2102b6f 100644 --- a/tests/bdd/dsl/registration.go +++ b/tests/bdd/dsl/registration.go @@ -67,12 +67,17 @@ func WatchStargatesCommand(endpoint, authority, caSecret, namespace, kubeContext ), nil } -// PylonMetricsCommand builds a bounded Pylon metrics observation for a pod -// selected by container name in an explicit Kubernetes context. -func PylonMetricsCommand(containerName, kubeContext, timeout string, expectations []PylonMetricExpectation) (string, error) { +// PylonMetricsCommand builds a Pylon metrics observation for every running +// Pylon pod selected by function name and container name in an explicit +// Kubernetes context and polling window. +func PylonMetricsCommand(functionName, containerName, kubeContext, timeout string, expectations []PylonMetricExpectation) (string, error) { + functionName = strings.TrimSpace(Interpolate(functionName)) containerName = strings.TrimSpace(Interpolate(containerName)) kubeContext = strings.TrimSpace(Interpolate(kubeContext)) timeout = strings.TrimSpace(Interpolate(timeout)) + if functionName == "" { + return "", fmt.Errorf("function name is empty") + } if containerName == "" { return "", fmt.Errorf("container name is empty") } @@ -83,10 +88,10 @@ func PylonMetricsCommand(containerName, kubeContext, timeout string, expectation return "", fmt.Errorf("timeout is empty") } if len(expectations) == 0 { - return "", fmt.Errorf("Pylon metric expectations are empty") + return "", fmt.Errorf("pylon metric expectations are empty") } - args := []string{"bash", waitPylonMetricsScript, containerName, kubeContext, timeout} + args := []string{"bash", waitPylonMetricsScript, functionName, containerName, kubeContext, timeout} for index, expectation := range expectations { expectation.Metric = strings.TrimSpace(Interpolate(expectation.Metric)) expectation.Comparison = strings.TrimSpace(Interpolate(expectation.Comparison)) diff --git a/tests/bdd/dsl/registration_test.go b/tests/bdd/dsl/registration_test.go index 606765864..e52a33bdd 100644 --- a/tests/bdd/dsl/registration_test.go +++ b/tests/bdd/dsl/registration_test.go @@ -65,14 +65,14 @@ func TestWatchStargatesCommandRejectsMissingOrInvalidInputs(t *testing.T) { } func TestPylonMetricsCommandKeepsExpectationsExplicit(t *testing.T) { - got, err := PylonMetricsCommand("llm-worker", "k3d-ncp-local-compute-1", "10m", []PylonMetricExpectation{ + got, err := PylonMetricsCommand("bdd-registration-tls", "llm-worker", "k3d-ncp-local-compute-1", "10m", []PylonMetricExpectation{ {Metric: "pylon_registration_stream_connected", Comparison: "exactly", Count: 5}, {Metric: "pylon_reverse_tunnel_connected", Comparison: "at least", Count: 3}, }) if err != nil { t.Fatalf("build Pylon metrics command: %v", err) } - want := "bash tests/bdd/scripts/wait-pylon-metrics.sh llm-worker k3d-ncp-local-compute-1 10m pylon_registration_stream_connected exactly 5 pylon_reverse_tunnel_connected 'at least' 3" + want := "bash tests/bdd/scripts/wait-pylon-metrics.sh bdd-registration-tls llm-worker k3d-ncp-local-compute-1 10m pylon_registration_stream_connected exactly 5 pylon_reverse_tunnel_connected 'at least' 3" if got != want { t.Fatalf("command = %q, want %q", got, want) } @@ -90,7 +90,7 @@ func TestPylonMetricsCommandRejectsInvalidExpectations(t *testing.T) { } for _, test := range tests { t.Run(test.name, func(t *testing.T) { - if _, err := PylonMetricsCommand("llm-worker", "context", "10m", test.expectations); err == nil { + if _, err := PylonMetricsCommand("function", "llm-worker", "context", "10m", test.expectations); err == nil { t.Fatal("expected validation error") } }) @@ -102,11 +102,12 @@ func TestObserveWatchStargatesScriptAcceptsSnapshotThenDeadline(t *testing.T) { writeExecutable(t, filepath.Join(fakeBin, "kubectl"), "#!/bin/sh\nprintf 'dGVzdC1jYQ=='\n") writeExecutable(t, filepath.Join(fakeBin, "grpcurl"), `#!/bin/sh printf '{\n "stargates": []\n}\n' +sleep 1 printf 'ERROR:\n Code: DeadlineExceeded\n Message: context deadline exceeded\n' >&2 exit 1 `) - output, err := runRegistrationScript(t, fakeBin, "observe-watch-stargates.sh", "127.0.0.1:50071", "router.nvcf.svc", "tls", "nvcf", "context", "3") + output, err := runRegistrationScript(t, fakeBin, "observe-watch-stargates.sh", "127.0.0.1:50071", "router.nvcf.svc", "tls", "nvcf", "context", "1") if err != nil { t.Fatalf("observe WatchStargates: %v\n%s", err, output) } @@ -117,6 +118,24 @@ exit 1 } } +func TestObserveWatchStargatesScriptRejectsImmediateProductDeadline(t *testing.T) { + fakeBin := t.TempDir() + writeExecutable(t, filepath.Join(fakeBin, "kubectl"), "#!/bin/sh\nprintf 'dGVzdC1jYQ=='\n") + writeExecutable(t, filepath.Join(fakeBin, "grpcurl"), `#!/bin/sh +printf '{\n "stargates": []\n}\n' +printf 'ERROR:\n Code: DeadlineExceeded\n Message: product returned deadline exceeded\n' >&2 +exit 1 +`) + + output, err := runRegistrationScript(t, fakeBin, "observe-watch-stargates.sh", "127.0.0.1:50071", "router.nvcf.svc", "tls", "nvcf", "context", "3") + if err == nil { + t.Fatal("expected immediate product deadline failure") + } + if !strings.Contains(output, "before the 3s observation deadline") { + t.Fatalf("output = %q, want early deadline diagnostic", output) + } +} + func TestObserveWatchStargatesScriptRejectsDeadlineWithoutSnapshot(t *testing.T) { fakeBin := t.TempDir() writeExecutable(t, filepath.Join(fakeBin, "kubectl"), "#!/bin/sh\nprintf 'dGVzdC1jYQ=='\n") @@ -131,13 +150,19 @@ func TestObserveWatchStargatesScriptRejectsDeadlineWithoutSnapshot(t *testing.T) } } -func TestWaitPylonMetricsScriptCountsConnectedSeries(t *testing.T) { +func TestWaitPylonMetricsScriptChecksEverySelectedPodAndCountsTimestampedSeries(t *testing.T) { fakeBin := t.TempDir() writeExecutable(t, filepath.Join(fakeBin, "kubectl"), `#!/bin/sh case "$*" in - *"get pods -A -o json"*) printf '{"items":[]}' ;; + *"get pods -A -o json"*) + printf '%s\n' '{"items":[' + printf '%s\n' '{"metadata":{"namespace":"functions","name":"worker-0","annotations":{"function-name":"bdd-registration-tls"}},"status":{"phase":"Running"},"spec":{"containers":[{"name":"llm-worker"}]}},' + printf '%s\n' '{"metadata":{"namespace":"functions","name":"worker-1","annotations":{"function-name":"bdd-registration-tls"}},"status":{"phase":"Running"},"spec":{"containers":[{"name":"llm-worker"}]}},' + printf '%s\n' '{"metadata":{"namespace":"other","name":"worker-other","annotations":{"function-name":"another-function"}},"status":{"phase":"Running"},"spec":{"containers":[{"name":"llm-worker"}]}}' + printf '%s\n' ']}' + ;; *) - printf 'pylon_registration_stream_connected{router="a"} 1\n' + printf 'pylon_registration_stream_connected{router="a"} 1 1712345678\n' printf 'pylon_registration_stream_connected{router="b"} 1\n' printf 'pylon_registration_stream_connected{router="c"} 1\n' printf 'pylon_reverse_tunnel_connected{router="a"} 1\n' @@ -146,12 +171,12 @@ case "$*" in ;; esac `) - writeExecutable(t, filepath.Join(fakeBin, "jq"), "#!/bin/sh\nprintf 'functions\\tworker-0\\n'\n") output, err := runRegistrationScript( t, fakeBin, "wait-pylon-metrics.sh", + "bdd-registration-tls", "llm-worker", "k3d-ncp-local-compute-1", "1s", @@ -165,11 +190,74 @@ esac if err != nil { t.Fatalf("wait for Pylon metrics: %v\n%s", err, output) } - for _, want := range []string{"pylon_registration_stream_connected=3", "pylon_reverse_tunnel_connected=3"} { + for _, want := range []string{ + "functions/worker-0 pylon_registration_stream_connected=3", + "functions/worker-0 pylon_reverse_tunnel_connected=3", + "functions/worker-1 pylon_registration_stream_connected=3", + "functions/worker-1 pylon_reverse_tunnel_connected=3", + } { if !strings.Contains(output, want) { t.Fatalf("output = %q, want %q", output, want) } } + if strings.Contains(output, "worker-other") { + t.Fatalf("output = %q, did not want metrics from another function", output) + } +} + +func TestWaitPylonMetricsScriptDoesNotTreatScrapeFailureAsZero(t *testing.T) { + fakeBin := t.TempDir() + writeExecutable(t, filepath.Join(fakeBin, "kubectl"), `#!/bin/sh +case "$*" in + *"get pods -A -o json"*) + printf '%s\n' '{"items":[{"metadata":{"namespace":"functions","name":"worker-0","annotations":{"function-name":"bdd-registration-tls"}},"status":{"phase":"Running"},"spec":{"containers":[{"name":"llm-worker"}]}}]}' + ;; + *) printf 'metrics endpoint unavailable\n' >&2; exit 1 ;; +esac +`) + + output, err := runRegistrationScript( + t, + fakeBin, + "wait-pylon-metrics.sh", + "bdd-registration-tls", + "llm-worker", + "k3d-ncp-local-compute-1", + "1s", + "pylon_registration_stream_connected", + "exactly", + "0", + ) + if err == nil { + t.Fatal("expected metrics scrape failure") + } + if !strings.Contains(output, "metrics scrape failed: metrics endpoint unavailable") { + t.Fatalf("output = %q, want preserved scrape failure", output) + } +} + +func TestWaitPylonMetricsScriptPreservesPodDiscoveryFailure(t *testing.T) { + fakeBin := t.TempDir() + writeExecutable(t, filepath.Join(fakeBin, "kubectl"), "#!/bin/sh\nprintf 'API server unavailable\\n' >&2\nexit 1\n") + + output, err := runRegistrationScript( + t, + fakeBin, + "wait-pylon-metrics.sh", + "bdd-registration-tls", + "llm-worker", + "k3d-ncp-local-compute-1", + "1s", + "pylon_registration_stream_connected", + "exactly", + "0", + ) + if err == nil { + t.Fatal("expected pod discovery failure") + } + if !strings.Contains(output, "pod discovery failed: API server unavailable") { + t.Fatalf("output = %q, want preserved discovery failure", output) + } } func runRegistrationScript(t *testing.T, fakeBin, scriptName string, args ...string) (string, error) { diff --git a/tests/bdd/scripts/observe-watch-stargates.sh b/tests/bdd/scripts/observe-watch-stargates.sh index 139ed8706..9813b3fcb 100644 --- a/tests/bdd/scripts/observe-watch-stargates.sh +++ b/tests/bdd/scripts/observe-watch-stargates.sh @@ -26,7 +26,7 @@ if ! [[ "$duration_seconds" =~ ^[1-9][0-9]*$ ]]; then echo "duration-seconds must be a positive integer, got: $duration_seconds" >&2 exit 64 fi -for tool in kubectl base64 grpcurl; do +for tool in kubectl base64 grpcurl jq; do if ! command -v "$tool" >/dev/null 2>&1; then echo "required tool not found: $tool" >&2 exit 127 @@ -37,7 +37,9 @@ script_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd -P)" repo_root="$(cd "$script_dir/../../.." && pwd -P)" proto_path="$repo_root/src/libraries/rust/stargate/crates/proto/proto" ca_file="$(mktemp "${TMPDIR:-/tmp}/nvcf-bdd-watch-ca.XXXXXX")" -trap 'rm -f "$ca_file"' EXIT +stdout_file="$(mktemp "${TMPDIR:-/tmp}/nvcf-bdd-watch-stdout.XXXXXX")" +stderr_file="$(mktemp "${TMPDIR:-/tmp}/nvcf-bdd-watch-stderr.XXXXXX")" +trap 'rm -f "$ca_file" "$stdout_file" "$stderr_file"' EXIT kubectl --context "$kube_context" get secret "$ca_secret" -n "$namespace" \ -o 'jsonpath={.data.ca\.crt}' | base64 -d >"$ca_file" @@ -47,29 +49,37 @@ if [[ ! -s "$ca_file" ]]; then fi set +e -output="$(grpcurl \ +started_at="$(date +%s)" +grpcurl \ -max-time "$duration_seconds" \ + -emit-defaults \ -cacert "$ca_file" \ -authority "$tls_authority" \ -import-path "$proto_path" \ -proto stargate.proto \ "$endpoint" \ - stargate.StargateControlPlane/WatchStargates 2>&1)" + stargate.StargateControlPlane/WatchStargates >"$stdout_file" 2>"$stderr_file" grpcurl_status=$? +finished_at="$(date +%s)" set -e -printf '%s\n' "$output" +cat "$stdout_file" +cat "$stderr_file" >&2 if [[ "$grpcurl_status" -eq 0 ]]; then echo "WatchStargates ended before the observation deadline" >&2 exit 1 fi -if ! grep -Eq '^[[:space:]]*\{' <<<"$output"; then +if ! jq -se 'length > 0 and all(.[]; type == "object" and (has("stargates") or has("watchStargateUrls")))' "$stdout_file" >/dev/null; then echo "WatchStargates did not return a streamed snapshot" >&2 exit 1 fi -if ! grep -Eiq 'DeadlineExceeded|context deadline exceeded' <<<"$output"; then +if ! grep -Eiq 'DeadlineExceeded|context deadline exceeded' "$stderr_file"; then echo "WatchStargates failed before the expected observation deadline" >&2 exit 1 fi - +elapsed_seconds=$(( finished_at - started_at )) +if [[ "$elapsed_seconds" -lt "$duration_seconds" ]]; then + echo "WatchStargates ended after ${elapsed_seconds}s before the ${duration_seconds}s observation deadline" >&2 + exit 1 +fi diff --git a/tests/bdd/scripts/wait-pylon-metrics.sh b/tests/bdd/scripts/wait-pylon-metrics.sh index 2eb697598..56cf1e1ea 100644 --- a/tests/bdd/scripts/wait-pylon-metrics.sh +++ b/tests/bdd/scripts/wait-pylon-metrics.sh @@ -4,18 +4,19 @@ set -euo pipefail -if [[ $# -lt 6 || $(( ( $# - 3 ) % 3 )) -ne 0 ]]; then - echo "usage: $0 [...]" >&2 +if [[ $# -lt 7 || $(( ( $# - 4 ) % 3 )) -ne 0 ]]; then + echo "usage: $0 [...]" >&2 exit 64 fi -container_name="$1" -kube_context="$2" -timeout="$3" -shift 3 +function_name="$1" +container_name="$2" +kube_context="$3" +timeout="$4" +shift 4 -if [[ -z "$container_name" || -z "$kube_context" ]]; then - echo "container and kube-context must be non-empty" >&2 +if [[ -z "$function_name" || -z "$container_name" || -z "$kube_context" ]]; then + echo "function-name, container, and kube-context must be non-empty" >&2 exit 64 fi if ! [[ "$timeout" =~ ^([1-9][0-9]*)(s|m|h)$ ]]; then @@ -66,7 +67,7 @@ connected_series_count() { awk -v metric="$metric" ' { series = $1 - if ((series == metric || index(series, metric "{") == 1) && $NF == "1") { + if ((series == metric || index(series, metric "{") == 1) && $2 == "1") { count++ } } @@ -75,41 +76,56 @@ connected_series_count() { } deadline=$(( $(date +%s) + timeout_seconds )) -last_summary="no running pod containing container $container_name" +last_summary="no running pod for function $function_name containing container $container_name" while true; do - pod_row="$(kubectl --context "$kube_context" get pods -A -o json | jq -r --arg container "$container_name" ' - [.items[] + if ! pods_json="$(kubectl --context "$kube_context" get pods -A -o json 2>&1)"; then + last_summary="pod discovery failed: $pods_json" + elif ! pod_rows="$(printf '%s\n' "$pods_json" | jq -r --arg function "$function_name" --arg container "$container_name" ' + .items[]? | select(.metadata.deletionTimestamp == null) | select(.status.phase == "Running") + | select(.metadata.annotations["function-name"] == $function) | select(any(.spec.containers[]?; .name == $container)) | [.metadata.namespace, .metadata.name] - | @tsv] - | first // empty - ' || true)" - - if [[ -n "$pod_row" ]]; then - read -r namespace pod_name <<<"$pod_row" - metrics="$(kubectl --context "$kube_context" get --raw "/api/v1/namespaces/$namespace/pods/$pod_name:9089/proxy/metrics" 2>/dev/null || true)" + | @tsv + ' 2>&1)"; then + last_summary="pod discovery response could not be parsed: $pod_rows" + elif [[ -z "$pod_rows" ]]; then + last_summary="no running pod for function $function_name containing container $container_name" + else all_match=true summaries=() - for index in "${!metric_names[@]}"; do - metric="${metric_names[$index]}" - comparison="${comparisons[$index]}" - expected_count="${expected_counts[$index]}" - observed_count="$(printf '%s\n' "$metrics" | connected_series_count "$metric")" - summaries+=("$metric=$observed_count") + while IFS=$'\t' read -r namespace pod_name; do + if ! metrics="$(kubectl --context "$kube_context" get --raw "/api/v1/namespaces/$namespace/pods/$pod_name:9089/proxy/metrics" 2>&1)"; then + summaries+=("$namespace/$pod_name metrics scrape failed: $metrics") + all_match=false + continue + fi + if [[ -z "$metrics" ]]; then + summaries+=("$namespace/$pod_name metrics scrape returned an empty response") + all_match=false + continue + fi + + for index in "${!metric_names[@]}"; do + metric="${metric_names[$index]}" + comparison="${comparisons[$index]}" + expected_count="${expected_counts[$index]}" + observed_count="$(printf '%s\n' "$metrics" | connected_series_count "$metric")" + summaries+=("$namespace/$pod_name $metric=$observed_count") - case "$comparison" in - exactly) - [[ "$observed_count" -eq "$expected_count" ]] || all_match=false - ;; - "at least") - [[ "$observed_count" -ge "$expected_count" ]] || all_match=false - ;; - esac - done + case "$comparison" in + exactly) + [[ "$observed_count" -eq "$expected_count" ]] || all_match=false + ;; + "at least") + [[ "$observed_count" -ge "$expected_count" ]] || all_match=false + ;; + esac + done + done <<<"$pod_rows" last_summary="${summaries[*]}" if [[ "$all_match" == true ]]; then diff --git a/tests/bdd/steps/assertion_steps.go b/tests/bdd/steps/assertion_steps.go index 0b036b912..f651efed9 100644 --- a/tests/bdd/steps/assertion_steps.go +++ b/tests/bdd/steps/assertion_steps.go @@ -99,7 +99,10 @@ func (sc *ScenarioContext) commandShouldFail() error { func (sc *ScenarioContext) commandOutputShouldContain(needle string) error { combined := combinedOutput(sc.LastResult) - resolved := dsl.Interpolate(needle) + resolved, err := resolveOutputNeedle(needle) + if err != nil { + return err + } if !strings.Contains(combined, resolved) { return fmt.Errorf("output does not contain %q", resolved) } @@ -108,7 +111,10 @@ func (sc *ScenarioContext) commandOutputShouldContain(needle string) error { func (sc *ScenarioContext) commandOutputShouldNotContain(needle string) error { combined := combinedOutput(sc.LastResult) - resolved := dsl.Interpolate(needle) + resolved, err := resolveOutputNeedle(needle) + if err != nil { + return err + } if strings.Contains(combined, resolved) { return fmt.Errorf("output contains %q", resolved) } @@ -120,9 +126,14 @@ func (sc *ScenarioContext) commandOutputShouldContainAll(table *godog.Table) err if err != nil { return err } - for _, needle := range needles { - if err := sc.commandOutputShouldContain(needle); err != nil { - return err + combined := combinedOutput(sc.LastResult) + for index, needle := range needles { + resolved, err := resolveOutputNeedle(needle) + if err != nil { + return fmt.Errorf("row %d: %w", index+1, err) + } + if !strings.Contains(combined, resolved) { + return fmt.Errorf("output does not contain %q", resolved) } } return nil @@ -133,15 +144,31 @@ func (sc *ScenarioContext) commandOutputShouldContainOneOf(table *godog.Table) e if err != nil { return err } + resolvedNeedles := make([]string, 0, len(needles)) + for index, needle := range needles { + resolved, err := resolveOutputNeedle(needle) + if err != nil { + return fmt.Errorf("row %d: %w", index+1, err) + } + resolvedNeedles = append(resolvedNeedles, resolved) + } combined := combinedOutput(sc.LastResult) - for _, needle := range needles { - if strings.Contains(combined, dsl.Interpolate(needle)) { + for _, resolved := range resolvedNeedles { + if strings.Contains(combined, resolved) { return nil } } return fmt.Errorf("output does not contain any of the %d expected values", len(needles)) } +func resolveOutputNeedle(needle string) (string, error) { + resolved := dsl.Interpolate(needle) + if strings.TrimSpace(resolved) == "" { + return "", fmt.Errorf("expected output text resolves to an empty value") + } + return resolved, nil +} + func (sc *ScenarioContext) yamlFileKeyShouldEqual(path, key, expected string) error { resolvedPath := sc.resolvePath(dsl.Interpolate(path)) got, found, err := dsl.ReadYAMLKey(resolvedPath, key) diff --git a/tests/bdd/steps/registration_steps.go b/tests/bdd/steps/registration_steps.go index 74f28dae4..4dd957827 100644 --- a/tests/bdd/steps/registration_steps.go +++ b/tests/bdd/steps/registration_steps.go @@ -30,7 +30,7 @@ import ( func registerRegistrationSteps(ctx *godog.ScenarioContext, sc *ScenarioContext) { ctx.Step(`^I successfully observe WatchStargates at "([^"]*)" with TLS authority "([^"]*)" using CA secret "([^"]*)" in namespace "([^"]*)" and context "([^"]*)" for "([^"]*)" seconds$`, sc.iSuccessfullyObserveWatchStargates) - ctx.Step(`^a pod containing container "([^"]*)" using context "([^"]*)" should report Pylon metrics within "([^"]*)":$`, sc.podShouldReportPylonMetrics) + ctx.Step(`^every Pylon for function "([^"]*)" using container "([^"]*)" and context "([^"]*)" should report metrics within "([^"]*)":$`, sc.everyPylonForFunctionShouldReportMetrics) } func (sc *ScenarioContext) iSuccessfullyObserveWatchStargates( @@ -52,8 +52,9 @@ func (sc *ScenarioContext) iSuccessfullyObserveWatchStargates( return nil } -func (sc *ScenarioContext) podShouldReportPylonMetrics( +func (sc *ScenarioContext) everyPylonForFunctionShouldReportMetrics( ctx context.Context, + functionName, containerName, kubeContext, timeout string, @@ -63,12 +64,12 @@ func (sc *ScenarioContext) podShouldReportPylonMetrics( if err != nil { return err } - command, err := dsl.PylonMetricsCommand(containerName, kubeContext, timeout, expectations) + command, err := dsl.PylonMetricsCommand(functionName, containerName, kubeContext, timeout, expectations) if err != nil { return err } if err := sc.runResolvedSuccessfully(ctx, command); err != nil { - return fmt.Errorf("pod containing container %q did not report expected Pylon metrics: %w", dsl.Interpolate(containerName), err) + return fmt.Errorf("pylon pods for function %q using container %q did not report expected metrics: %w", dsl.Interpolate(functionName), dsl.Interpolate(containerName), err) } return nil } diff --git a/tests/bdd/steps/steps_test.go b/tests/bdd/steps/steps_test.go index af21d385f..bcfed0cdb 100644 --- a/tests/bdd/steps/steps_test.go +++ b/tests/bdd/steps/steps_test.go @@ -1463,6 +1463,35 @@ func TestCommandOutputTableAssertionsInterpolateExpectedText(t *testing.T) { } } +func TestCommandOutputAssertionsRejectValuesThatInterpolateToEmpty(t *testing.T) { + sc, _ := newScenarioContext(t) + t.Setenv("BDD_EMPTY_EXPECTATION", "") + sc.LastResult = harness.Result{Stdout: "any output contains the empty string"} + + if err := sc.commandOutputShouldContain("${BDD_EMPTY_EXPECTATION}"); err == nil { + t.Fatal("expected empty single-value expectation to fail") + } + if err := sc.commandOutputShouldNotContain("${BDD_EMPTY_EXPECTATION}"); err == nil { + t.Fatal("expected empty negative expectation to fail validation") + } + + containAll := docTable(t, [][]string{ + {"text"}, + {"${BDD_EMPTY_EXPECTATION}"}, + }) + if err := sc.commandOutputShouldContainAll(containAll); err == nil { + t.Fatal("expected contain-all table with an empty resolved value to fail") + } + containOneOf := docTable(t, [][]string{ + {"text"}, + {"any output"}, + {"${BDD_EMPTY_EXPECTATION}"}, + }) + if err := sc.commandOutputShouldContainOneOf(containOneOf); err == nil { + t.Fatal("expected contain-one-of table with an empty resolved value to fail") + } +} + func TestISuccessfullyObserveWatchStargatesRunsExplicitCommand(t *testing.T) { sc, fake := newScenarioContext(t) fake.result = harness.Result{ExitCode: 0, Stdout: "{\n \"stargates\": []\n}\n"} @@ -1488,7 +1517,7 @@ func TestISuccessfullyObserveWatchStargatesRunsExplicitCommand(t *testing.T) { } } -func TestPodShouldReportPylonMetricsRunsVisibleExpectations(t *testing.T) { +func TestEveryPylonForFunctionShouldReportMetricsRunsVisibleExpectations(t *testing.T) { sc, fake := newScenarioContext(t) fake.result = harness.Result{ExitCode: 0} table := docTable(t, [][]string{ @@ -1497,10 +1526,10 @@ func TestPodShouldReportPylonMetricsRunsVisibleExpectations(t *testing.T) { {"pylon_reverse_tunnel_connected", "at least", "3"}, }) - if err := sc.podShouldReportPylonMetrics(context.Background(), "llm-worker", "k3d-ncp-local-compute-1", "10m", table); err != nil { + if err := sc.everyPylonForFunctionShouldReportMetrics(context.Background(), "bdd-registration-tls", "llm-worker", "k3d-ncp-local-compute-1", "10m", table); err != nil { t.Fatalf("observe Pylon metrics: %v", err) } - want := "bash tests/bdd/scripts/wait-pylon-metrics.sh llm-worker k3d-ncp-local-compute-1 10m pylon_registration_stream_connected exactly 5 pylon_reverse_tunnel_connected 'at least' 3" + want := "bash tests/bdd/scripts/wait-pylon-metrics.sh bdd-registration-tls llm-worker k3d-ncp-local-compute-1 10m pylon_registration_stream_connected exactly 5 pylon_reverse_tunnel_connected 'at least' 3" if len(fake.runs) != 1 || fake.runs[0].command != want { t.Fatalf("runs = %#v, want %q", fake.runs, want) } @@ -1513,7 +1542,7 @@ func TestPylonMetricTableRejectsInvalidStructureBeforeRunning(t *testing.T) { {"pylon_registration_stream_connected", "exactly", "not-a-count"}, }) - if err := sc.podShouldReportPylonMetrics(context.Background(), "llm-worker", "context", "10m", table); err == nil { + if err := sc.everyPylonForFunctionShouldReportMetrics(context.Background(), "function", "llm-worker", "context", "10m", table); err == nil { t.Fatal("expected invalid count error") } if len(fake.runs) != 0 { From 0be3b470041213479980e319fe9f152c5d4c30fa Mon Sep 17 00:00:00 2001 From: Stephanie Baum Date: Fri, 28 Aug 2026 01:20:23 -0700 Subject: [PATCH 03/10] test(bdd): cover secure multi-cluster registration --- ...ster-helmfile-llm-registration-tls.feature | 179 ++++++++++++++++++ tests/bdd/godog_test.go | 131 +++++++++++++ 2 files changed, 310 insertions(+) create mode 100644 tests/bdd/features/multi-cluster-helmfile-llm-registration-tls.feature diff --git a/tests/bdd/features/multi-cluster-helmfile-llm-registration-tls.feature b/tests/bdd/features/multi-cluster-helmfile-llm-registration-tls.feature new file mode 100644 index 000000000..726c34053 --- /dev/null +++ b/tests/bdd/features/multi-cluster-helmfile-llm-registration-tls.feature @@ -0,0 +1,179 @@ +@ncp-local @multi-cluster @helmfile @pki @llm-registration +Feature: Register an LLM worker securely with every router in a local split-cluster stack + As a self-managed NVCF operator, + I want Pylon registration to use the stack-issued TLS identity across clusters, + so that plaintext or untrusted registration cannot silently enter the routing plane. + + Rule: Secure registration is observable from the operator boundary + + Background: + Given these environment variables are set: + | name | + | NGC_API_KEY | + | NVCF_CLI | + | REPO_ROOT | + | SAMPLE_NGC_ORG | + | SAMPLE_NGC_TEAM | + And I prepare Helmfile environment "local-bdd-registration-tls" for stack "self-managed" from fixture "tests/bdd/fixtures/self-managed-local-bdd-multi.yaml" with values: + | global.imagePullSecrets[0].name | nvcr-pull-secret | + | global.helm.sources.repository | ${SAMPLE_NGC_ORG}/${SAMPLE_NGC_TEAM} | + | global.image.repository | ${SAMPLE_NGC_ORG}/${SAMPLE_NGC_TEAM} | + | global.workerEndpoints.llmRequestRouterAddress | https://llm-request-router.nvcf.svc.cluster.local:50071 | + | api.env.NVCF_SIDECARS_LLM_ROUTER_CLIENT_IMAGE | nvcr.io/${SAMPLE_NGC_ORG}/${SAMPLE_NGC_TEAM}/stargate-client:0.2.0 | + | addons.llm.requestRouter.backendRouter.pylonGrpcDialAddress | https://llm-request-router.nvcf.svc.cluster.local:50071 | + | observability.profile | disabled | + And I prepare Helmfile environment "local-bdd-registration-tls" for stack "nvcf-compute-plane" from fixture "tests/bdd/fixtures/nvcf-compute-plane-local-bdd-multi.yaml" with values: + | global.imagePullSecrets[0].name | nvcr-pull-secret | + | global.helm.sources.repository | ${SAMPLE_NGC_ORG}/${SAMPLE_NGC_TEAM} | + | global.image.repository | ${SAMPLE_NGC_ORG}/${SAMPLE_NGC_TEAM} | + | observability.profile | disabled | + And I prepare self-managed secrets file "deploy/stacks/self-managed/secrets/local-bdd-registration-tls-secrets.yaml" from template "deploy/stacks/self-managed/secrets/secrets.yaml.template" using the current NGC registry credential + # Conflict precheck: the single-cluster topology owns the same host + # ports. Run make -C tools/ncp-local-cluster destroy CLUSTER_NAME=ncp-local + # before retrying. k3d v5 exits 1 when the cluster is absent. + When I run command "k3d cluster get ncp-local" + Then the command exit code should be 1 + And multi-cluster ncp-local compute clusters are running: + | ncp-local-compute-1 | + And command has succeeded: + """ + kubectl config use-context k3d-ncp-local-cp + """ + And the "nvcr-pull-secret" image pull secret exists in namespaces: + | cassandra-system | + | nats-system | + | nvcf | + | api-keys | + | ess | + | sis | + | vault-system | + | nvca-operator | + | cert-manager | + + @llm-registration-tls-install + Scenario: Operator installs a TLS-only registration endpoint with three concrete routers + When I run command "make -C deploy/stacks/self-managed template HELMFILE_ENV=local-bdd-registration-tls" + Then the command exit code should be 0 + And the rendered manifests in "deploy/stacks/self-managed/out" should contain: + | text | + | https://llm-request-router.nvcf.svc.cluster.local:50071 | + | --grpc-pylon-dial-addr=https://llm-request-router.nvcf.svc.cluster.local:50071 | + + When I run command "make -C deploy/stacks/self-managed install HELMFILE_ENV=local-bdd-registration-tls" + Then the command exit code should be 0 + + When I run command "kubectl --context k3d-ncp-local-cp wait clusterissuer nvcf-openbao-pki --for=condition=Ready --timeout=5m" + Then the command exit code should be 0 + When I run command "kubectl --context k3d-ncp-local-cp wait certificate stargate-quic-tls -n nvcf --for=condition=Ready --timeout=5m" + Then the command exit code should be 0 + When I run command "kubectl --context k3d-ncp-local-cp rollout status statefulset/llm-request-router -n nvcf --timeout=10m" + Then the command exit code should be 0 + + When I run command "kubectl --context k3d-ncp-local-cp get configmap/nvcf-api-remote-config -n nvcf -o yaml" + Then the command exit code should be 0 + And the command output should contain "worker-address: https://llm-request-router.nvcf.svc.cluster.local:50071" + + # openssl verifies the externally reachable listener against the same + # stack-issued CA and DNS identity that a compute-plane Pylon uses. + When I run command: + """ + /bin/bash -c 'openssl s_client -connect 127.0.0.1:50071 -servername llm-request-router.nvcf.svc.cluster.local -alpn h2 -verify_return_error -CAfile <(kubectl --context k3d-ncp-local-cp get secret stargate-quic-tls -n nvcf -o jsonpath="{.data.ca\.crt}" | base64 -d) &1' + """ + Then the command exit code should be 0 + And the command output should contain "Verify return code: 0 (ok)" + And the command output should contain "ALPN protocol: h2" + + When I run command: + """ + grpcurl -plaintext -max-time 5 -import-path src/libraries/rust/stargate/crates/proto/proto -proto stargate.proto 127.0.0.1:50071 stargate.StargateControlPlane/WatchStargates + """ + Then the command exit code should be 1 + + # WatchStargates is a long-lived stream. grpcurl exits on its explicit + # deadline after printing the initial snapshot, so exit 1 is expected. + When I run command: + """ + /bin/bash -c 'grpcurl -max-time 3 -cacert <(kubectl --context k3d-ncp-local-cp get secret stargate-quic-tls -n nvcf -o jsonpath="{.data.ca\.crt}" | base64 -d) -authority llm-request-router.nvcf.svc.cluster.local -import-path src/libraries/rust/stargate/crates/proto/proto -proto stargate.proto 127.0.0.1:50071 stargate.StargateControlPlane/WatchStargates' + """ + Then the command exit code should be 1 + And the command output should contain "llm-request-router-0" + And the command output should contain "llm-request-router-1" + And the command output should contain "llm-request-router-2" + And the command output should contain "https://llm-request-router.nvcf.svc.cluster.local:50071" + + When I run command: + """ + ${NVCF_CLI} --config ${REPO_ROOT}/tests/bdd/fixtures/nvcf-cli-local.yaml self-hosted --control-plane-stack deploy/stacks/self-managed --env local-bdd-registration-tls --control-plane-context k3d-ncp-local-cp --compute-plane-context k3d-ncp-local-compute-1 control-plane profile export --cluster-name ncp-local-cp + """ + Then the command exit code should be 0 + And file "deploy/stacks/self-managed/out/control-plane-profile.yaml" should exist + And yaml file "deploy/stacks/self-managed/out/control-plane-profile.yaml" should have non-empty keys: + | key | + | managementTls.caBundlePem | + | transportTls.trustBundleFingerprint | + | transportTls.trustBundlePem | + + And command has succeeded: + """ + /bin/sh -c '${NVCF_CLI} --config ${REPO_ROOT}/tests/bdd/fixtures/nvcf-cli-local.yaml init >/dev/null' + """ + When I run command "kubectl config use-context k3d-ncp-local-compute-1" + Then the command exit code should be 0 + When I run command: + """ + make -C deploy/stacks/nvcf-compute-plane register-cluster CLUSTER_NAME=ncp-local-compute-1 CONTROL_PLANE_PROFILE=${REPO_ROOT}/deploy/stacks/self-managed/out/control-plane-profile.yaml COMPUTE_KUBE_CONTEXT=k3d-ncp-local-compute-1 NVCF_CLI=${NVCF_CLI} NVCF_CLI_CONFIG=${REPO_ROOT}/tests/bdd/fixtures/nvcf-cli-local.yaml + """ + Then the command exit code should be 0 + And file "deploy/stacks/nvcf-compute-plane/registration/ncp-local-compute-1-register-values.yaml" should exist + And the "nvcr-pull-secret" image pull secret exists in namespaces: + | nvca-operator | + When I run command: + """ + make -C deploy/stacks/nvcf-compute-plane install CLUSTER_NAME=ncp-local-compute-1 HELMFILE_ENV=local-bdd-registration-tls COMPUTE_KUBE_CONTEXT=k3d-ncp-local-compute-1 NVCF_CLI=${NVCF_CLI} + """ + Then the command exit code should be 0 + Then NVCFBackend "ncp-local-compute-1" in namespace "nvca-operator" using context "k3d-ncp-local-compute-1" should report agent status "healthy" within "10m" + + @llm-registration-tls-runtime + Scenario: Pylon registers with every router and serves an authenticated LLM request + Given I use NVCF CLI config "${REPO_ROOT}/tests/bdd/fixtures/nvcf-cli-local.yaml" + When I successfully create function "bdd-registration-tls" from image "nvcr.io/${SAMPLE_NGC_ORG}/${SAMPLE_NGC_TEAM}/nvcf-openai-compatible-sample:local" with CLI options: + | option | value | + | --function-type | LLM | + | --inference-url | /v1/chat/completions | + | --inference-port | 8000 | + | --health-uri | /health | + | --health-port | 8000 | + | --health-timeout | PT30S | + | --llm-model | name=openai-compatible-sample,uris=/v1/chat/completions\|/v1/embeddings,routingMethod=round_robin | + And I successfully deploy the function selected by NVCF CLI with options: + | option | value | + | --gpu | H100 | + | --instance-type | NCP.GPU.H100_1x | + | --backend | ncp-local-compute-1 | + | --regions | us-west-1 | + | --min-instances | 1 | + | --max-instances | 1 | + | --timeout | 900 | + And I successfully generate a function API key with CLI options: + | option | value | + | --description | bdd-registration-tls | + | --scopes | invoke_function,list_functions,queue_details,list_functions_details | + + # Discover the workload pod by its public sidecar name, then poll the + # Pylon metrics endpoint until all three router streams and reverse + # tunnels are connected. + When I run command: + """ + /bin/sh -c 'set -eu; for attempt in $(seq 1 120); do row=$(kubectl --context k3d-ncp-local-compute-1 get pods -A -o json | jq -r "[.items[] | select(any(.spec.containers[]?; .name == \"llm-worker\")) | [.metadata.namespace,.metadata.name] | @tsv] | first // empty"); if [ -n "$row" ]; then ns=$(printf "%s" "$row" | cut -f1); pod=$(printf "%s" "$row" | cut -f2); metrics=$(kubectl --context k3d-ncp-local-compute-1 get --raw "/api/v1/namespaces/$ns/pods/$pod:9089/proxy/metrics" 2>/dev/null || true); registration=$(printf "%s\n" "$metrics" | grep -c "^pylon_registration_stream_connected.* 1$" || true); tunnels=$(printf "%s\n" "$metrics" | grep -c "^pylon_reverse_tunnel_connected.* 1$" || true); if [ "$registration" -eq 3 ] && [ "$tunnels" -eq 3 ]; then printf "registration=%s reverse=%s\n" "$registration" "$tunnels"; exit 0; fi; fi; sleep 5; done; exit 1' + """ + Then the command exit code should be 0 + And the command output should contain "registration=3 reverse=3" + + When I successfully invoke model "openai-compatible-sample" at "/v1/chat/completions" with timeout "120" seconds: + """ + {"messages":[{"role":"user","content":"bdd-registration-tls"}]} + """ + Then the command output should contain "chat.completion" + And the command output should contain "fixed 128-byte response" + And I successfully undeploy the function selected by NVCF CLI diff --git a/tests/bdd/godog_test.go b/tests/bdd/godog_test.go index 970c26657..4d1bf64fa 100644 --- a/tests/bdd/godog_test.go +++ b/tests/bdd/godog_test.go @@ -1206,6 +1206,127 @@ func TestMultiClusterHelmfileFeatureFileWiresToSteps(t *testing.T) { assertFunctionDeploymentsUseInstanceType(t, suite.Runner.(*fakeRunner).runs, "NCP.GPU.H100_1x", 3) } +// TestMultiClusterHelmfileLLMRegistrationTLSFeatureFileWiresToSteps runs the +// focused secure registration feature against a fake runner. The canned +// external observations cover the TLS listener, WatchStargates snapshot, +// Pylon metrics, and authenticated invocation. +func TestMultiClusterHelmfileLLMRegistrationTLSFeatureFileWiresToSteps(t *testing.T) { + t.Setenv("NGC_API_KEY", "test-key") + t.Setenv("SAMPLE_NGC_ORG", "test-org") + t.Setenv("SAMPLE_NGC_TEAM", "test-team") + t.Setenv("NVCF_CLI", "/usr/bin/nvcf-cli") + t.Setenv("REPO_ROOT", "/repo-root-placeholder") + + const ( + tlsHandshakeCommand = `/bin/bash -c 'openssl s_client -connect 127.0.0.1:50071 ` + + `-servername llm-request-router.nvcf.svc.cluster.local -alpn h2 -verify_return_error ` + + `-CAfile <(kubectl --context k3d-ncp-local-cp get secret stargate-quic-tls -n nvcf ` + + `-o jsonpath="{.data.ca\.crt}" | base64 -d) &1'` + plaintextWatchCommand = "grpcurl -plaintext -max-time 5 -import-path src/libraries/rust/stargate/crates/proto/proto -proto stargate.proto 127.0.0.1:50071 stargate.StargateControlPlane/WatchStargates" + tlsWatchCommand = `/bin/bash -c 'grpcurl -max-time 3 ` + + `-cacert <(kubectl --context k3d-ncp-local-cp get secret stargate-quic-tls -n nvcf ` + + `-o jsonpath="{.data.ca\.crt}" | base64 -d) ` + + `-authority llm-request-router.nvcf.svc.cluster.local ` + + `-import-path src/libraries/rust/stargate/crates/proto/proto -proto stargate.proto ` + + `127.0.0.1:50071 stargate.StargateControlPlane/WatchStargates'` + pylonMetricsCommand = `/bin/sh -c 'set -eu; for attempt in $(seq 1 120); do ` + + `row=$(kubectl --context k3d-ncp-local-compute-1 get pods -A -o json | ` + + `jq -r "[.items[] | select(any(.spec.containers[]?; .name == \"llm-worker\")) | ` + + `[.metadata.namespace,.metadata.name] | @tsv] | first // empty"); ` + + `if [ -n "$row" ]; then ns=$(printf "%s" "$row" | cut -f1); ` + + `pod=$(printf "%s" "$row" | cut -f2); ` + + `metrics=$(kubectl --context k3d-ncp-local-compute-1 get --raw ` + + `"/api/v1/namespaces/$ns/pods/$pod:9089/proxy/metrics" 2>/dev/null || true); ` + + `registration=$(printf "%s\n" "$metrics" | ` + + `grep -c "^pylon_registration_stream_connected.* 1$" || true); ` + + `tunnels=$(printf "%s\n" "$metrics" | ` + + `grep -c "^pylon_reverse_tunnel_connected.* 1$" || true); ` + + `if [ "$registration" -eq 3 ] && [ "$tunnels" -eq 3 ]; then ` + + `printf "registration=%s reverse=%s\n" "$registration" "$tunnels"; exit 0; fi; ` + + `fi; sleep 5; done; exit 1'` + invokeCommand = "/usr/bin/nvcf-cli --config /repo-root-placeholder/tests/bdd/fixtures/nvcf-cli-local.yaml function invoke" + + " --inference-url /v1/chat/completions --model-name openai-compatible-sample" + + " --request-body '{\"messages\":[{\"role\":\"user\",\"content\":\"bdd-registration-tls\"}]}' --timeout 120" + ) + + suite := newWiringSuite(t, newFakeRunner(map[string]harness.Result{ + "k3d cluster get ncp-local": {ExitCode: 1}, + "kubectl --context k3d-ncp-local-cp get configmap/nvcf-api-remote-config -n nvcf -o yaml": { + ExitCode: 0, + Stdout: "worker-address: https://llm-request-router.nvcf.svc.cluster.local:50071\n", + }, + tlsHandshakeCommand: { + ExitCode: 0, + Stdout: "ALPN protocol: h2\nVerify return code: 0 (ok)\n", + }, + plaintextWatchCommand: {ExitCode: 1, Stderr: "tls: first record does not look like a TLS handshake"}, + tlsWatchCommand: { + ExitCode: 1, + Stdout: `{ + "stargates": [ + {"stargateId": "llm-request-router-0", "grpcPylonDialAddr": "https://llm-request-router.nvcf.svc.cluster.local:50071"}, + {"stargateId": "llm-request-router-1", "grpcPylonDialAddr": "https://llm-request-router.nvcf.svc.cluster.local:50071"}, + {"stargateId": "llm-request-router-2", "grpcPylonDialAddr": "https://llm-request-router.nvcf.svc.cluster.local:50071"} + ] +}`, + Stderr: "ERROR: DeadlineExceeded", + }, + pylonMetricsCommand: {ExitCode: 0, Stdout: "registration=3 reverse=3\n"}, + invokeCommand: { + ExitCode: 0, + Stdout: "Function invocation completed!\n\nResponse:\n" + + `{"object":"chat.completion","choices":[{"message":{"content":"This is a fixed 128-byte response for routing and contract validation."}}]}` + + "\n", + }, + })) + seedHelmfileLocalBDDMultiFixture(t, suite.Config.RepoRoot) + seedComputePlaneLocalBDDMultiFixture(t, suite.Config.RepoRoot) + seedStackSecretsTemplate(t, suite.Config.RepoRoot) + writeProfileHandoffArtifact(t, suite.Config.RepoRoot) + writeMulticlusterComputeRegisterValues(t, suite.Config.RepoRoot, "nvcf-compute-plane", "ncp-local-compute-1") + writeArtifact( + t, + suite.Config.RepoRoot, + "self-managed", + "registration-tls-rendered.yaml", + "https://llm-request-router.nvcf.svc.cluster.local:50071\n"+ + "--grpc-pylon-dial-addr=https://llm-request-router.nvcf.svc.cluster.local:50071\n", + ) + + sc := steps.NewScenarioContext(suite) + featurePath := mustResolveFeaturePath(t, "multi-cluster-helmfile-llm-registration-tls.feature") + var out strings.Builder + status := godog.TestSuite{ + Name: "multi-cluster-helmfile-llm-registration-tls-wiring", + ScenarioInitializer: func(ctx *godog.ScenarioContext) { + steps.RegisterAll(ctx, sc) + }, + Options: &godog.Options{ + Format: "pretty", + Paths: []string{featurePath}, + Strict: true, + Output: &out, + }, + }.Run() + if status != 0 { + t.Fatalf("godog suite status = %d\n%s", status, out.String()) + } + if !commandRanThatContainsAll( + suite.Runner.(*fakeRunner).runs, + "function create --name bdd-registration-tls", + "--function-type LLM", + "--llm-model", + ) { + t.Fatal("secure registration sample was not created as an LLM function") + } + if !commandRanExactly(suite.Runner.(*fakeRunner).runs, pylonMetricsCommand) { + t.Fatal("Pylon registration and reverse-tunnel metrics were not observed") + } + if !commandRanExactly(suite.Runner.(*fakeRunner).runs, tlsWatchCommand) { + t.Fatal("WatchStargates was not observed over the trusted TLS listener") + } +} + // TestSingleClusterHelmfileUpstreamImagesFeatureFileWiresToSteps runs the // focused upstream-image feature against a fake runner. The seeded global // template contains the exact documentation blocks so the ledger-backed @@ -1971,6 +2092,16 @@ func TestMultiClusterHelmfile(t *testing.T) { runLiveFeature(t, "multi-cluster-helmfile.feature") } +// TestMultiClusterHelmfileLLMRegistrationTLS is the live entry point for the +// focused secure Pylon registration feature on local split-cluster k3d. +// Skipped under -short. +func TestMultiClusterHelmfileLLMRegistrationTLS(t *testing.T) { + if testing.Short() { + t.Skip("live run skipped under -short") + } + runLiveFeature(t, "multi-cluster-helmfile-llm-registration-tls.feature") +} + // TestSingleClusterEKSHelmfile is the live entry point for the // single-cluster EKS Helmfile feature. Skipped under -short. func TestSingleClusterEKSHelmfile(t *testing.T) { From 06857069752cc62e2c51a54b56c56259fb7b7645 Mon Sep 17 00:00:00 2001 From: Stephanie Baum Date: Fri, 28 Aug 2026 01:26:27 -0700 Subject: [PATCH 04/10] test(bdd): pin router workload for TLS registration --- .../features/multi-cluster-helmfile-llm-registration-tls.feature | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/bdd/features/multi-cluster-helmfile-llm-registration-tls.feature b/tests/bdd/features/multi-cluster-helmfile-llm-registration-tls.feature index 726c34053..58187c296 100644 --- a/tests/bdd/features/multi-cluster-helmfile-llm-registration-tls.feature +++ b/tests/bdd/features/multi-cluster-helmfile-llm-registration-tls.feature @@ -20,6 +20,7 @@ Feature: Register an LLM worker securely with every router in a local split-clus | global.image.repository | ${SAMPLE_NGC_ORG}/${SAMPLE_NGC_TEAM} | | global.workerEndpoints.llmRequestRouterAddress | https://llm-request-router.nvcf.svc.cluster.local:50071 | | api.env.NVCF_SIDECARS_LLM_ROUTER_CLIENT_IMAGE | nvcr.io/${SAMPLE_NGC_ORG}/${SAMPLE_NGC_TEAM}/stargate-client:0.2.0 | + | addons.llm.requestRouter.workload.kind | StatefulSet | | addons.llm.requestRouter.backendRouter.pylonGrpcDialAddress | https://llm-request-router.nvcf.svc.cluster.local:50071 | | observability.profile | disabled | And I prepare Helmfile environment "local-bdd-registration-tls" for stack "nvcf-compute-plane" from fixture "tests/bdd/fixtures/nvcf-compute-plane-local-bdd-multi.yaml" with values: From f54fedc06330eedad5e10694e2962212ace104ae Mon Sep 17 00:00:00 2001 From: Stephanie Baum Date: Fri, 28 Aug 2026 01:45:13 -0700 Subject: [PATCH 05/10] test(bdd): accept grpcurl stream timeout codes --- .../multi-cluster-helmfile-llm-registration-tls.feature | 4 ++-- tests/bdd/godog_test.go | 5 +++-- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/tests/bdd/features/multi-cluster-helmfile-llm-registration-tls.feature b/tests/bdd/features/multi-cluster-helmfile-llm-registration-tls.feature index 58187c296..68a24d7d0 100644 --- a/tests/bdd/features/multi-cluster-helmfile-llm-registration-tls.feature +++ b/tests/bdd/features/multi-cluster-helmfile-llm-registration-tls.feature @@ -94,9 +94,9 @@ Feature: Register an LLM worker securely with every router in a local split-clus # deadline after printing the initial snapshot, so exit 1 is expected. When I run command: """ - /bin/bash -c 'grpcurl -max-time 3 -cacert <(kubectl --context k3d-ncp-local-cp get secret stargate-quic-tls -n nvcf -o jsonpath="{.data.ca\.crt}" | base64 -d) -authority llm-request-router.nvcf.svc.cluster.local -import-path src/libraries/rust/stargate/crates/proto/proto -proto stargate.proto 127.0.0.1:50071 stargate.StargateControlPlane/WatchStargates' + /bin/bash -c 'grpcurl -max-time 3 -cacert <(kubectl --context k3d-ncp-local-cp get secret stargate-quic-tls -n nvcf -o jsonpath="{.data.ca\.crt}" | base64 -d) -authority llm-request-router.nvcf.svc.cluster.local -import-path src/libraries/rust/stargate/crates/proto/proto -proto stargate.proto 127.0.0.1:50071 stargate.StargateControlPlane/WatchStargates; rc=$?; [ "$rc" -ne 0 ]' """ - Then the command exit code should be 1 + Then the command exit code should be 0 And the command output should contain "llm-request-router-0" And the command output should contain "llm-request-router-1" And the command output should contain "llm-request-router-2" diff --git a/tests/bdd/godog_test.go b/tests/bdd/godog_test.go index 4d1bf64fa..542754337 100644 --- a/tests/bdd/godog_test.go +++ b/tests/bdd/godog_test.go @@ -1228,7 +1228,8 @@ func TestMultiClusterHelmfileLLMRegistrationTLSFeatureFileWiresToSteps(t *testin `-o jsonpath="{.data.ca\.crt}" | base64 -d) ` + `-authority llm-request-router.nvcf.svc.cluster.local ` + `-import-path src/libraries/rust/stargate/crates/proto/proto -proto stargate.proto ` + - `127.0.0.1:50071 stargate.StargateControlPlane/WatchStargates'` + `127.0.0.1:50071 stargate.StargateControlPlane/WatchStargates; ` + + `rc=$?; [ "$rc" -ne 0 ]'` pylonMetricsCommand = `/bin/sh -c 'set -eu; for attempt in $(seq 1 120); do ` + `row=$(kubectl --context k3d-ncp-local-compute-1 get pods -A -o json | ` + `jq -r "[.items[] | select(any(.spec.containers[]?; .name == \"llm-worker\")) | ` + @@ -1261,7 +1262,7 @@ func TestMultiClusterHelmfileLLMRegistrationTLSFeatureFileWiresToSteps(t *testin }, plaintextWatchCommand: {ExitCode: 1, Stderr: "tls: first record does not look like a TLS handshake"}, tlsWatchCommand: { - ExitCode: 1, + ExitCode: 0, Stdout: `{ "stargates": [ {"stargateId": "llm-request-router-0", "grpcPylonDialAddr": "https://llm-request-router.nvcf.svc.cluster.local:50071"}, From 052cc6eae3659d4bac8cb3e2440653046c422de9 Mon Sep 17 00:00:00 2001 From: Stephanie Baum Date: Fri, 28 Aug 2026 02:34:56 -0700 Subject: [PATCH 06/10] test(bdd): use stack Pylon artifact configuration --- .../features/multi-cluster-helmfile-llm-registration-tls.feature | 1 - 1 file changed, 1 deletion(-) diff --git a/tests/bdd/features/multi-cluster-helmfile-llm-registration-tls.feature b/tests/bdd/features/multi-cluster-helmfile-llm-registration-tls.feature index 68a24d7d0..6f18a784b 100644 --- a/tests/bdd/features/multi-cluster-helmfile-llm-registration-tls.feature +++ b/tests/bdd/features/multi-cluster-helmfile-llm-registration-tls.feature @@ -19,7 +19,6 @@ Feature: Register an LLM worker securely with every router in a local split-clus | global.helm.sources.repository | ${SAMPLE_NGC_ORG}/${SAMPLE_NGC_TEAM} | | global.image.repository | ${SAMPLE_NGC_ORG}/${SAMPLE_NGC_TEAM} | | global.workerEndpoints.llmRequestRouterAddress | https://llm-request-router.nvcf.svc.cluster.local:50071 | - | api.env.NVCF_SIDECARS_LLM_ROUTER_CLIENT_IMAGE | nvcr.io/${SAMPLE_NGC_ORG}/${SAMPLE_NGC_TEAM}/stargate-client:0.2.0 | | addons.llm.requestRouter.workload.kind | StatefulSet | | addons.llm.requestRouter.backendRouter.pylonGrpcDialAddress | https://llm-request-router.nvcf.svc.cluster.local:50071 | | observability.profile | disabled | From bf1c8086d34875c18ebba01bd6b298c5babe673a Mon Sep 17 00:00:00 2001 From: Stephanie Baum Date: Fri, 28 Aug 2026 03:10:35 -0700 Subject: [PATCH 07/10] test(bdd): clarify streaming Watch assertion --- .../multi-cluster-helmfile-llm-registration-tls.feature | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/bdd/features/multi-cluster-helmfile-llm-registration-tls.feature b/tests/bdd/features/multi-cluster-helmfile-llm-registration-tls.feature index 6f18a784b..2262904fc 100644 --- a/tests/bdd/features/multi-cluster-helmfile-llm-registration-tls.feature +++ b/tests/bdd/features/multi-cluster-helmfile-llm-registration-tls.feature @@ -89,8 +89,8 @@ Feature: Register an LLM worker securely with every router in a local split-clus """ Then the command exit code should be 1 - # WatchStargates is a long-lived stream. grpcurl exits on its explicit - # deadline after printing the initial snapshot, so exit 1 is expected. + # WatchStargates is a long-lived stream. Normalize grpcurl's deadline + # exit after it prints the initial snapshot. When I run command: """ /bin/bash -c 'grpcurl -max-time 3 -cacert <(kubectl --context k3d-ncp-local-cp get secret stargate-quic-tls -n nvcf -o jsonpath="{.data.ca\.crt}" | base64 -d) -authority llm-request-router.nvcf.svc.cluster.local -import-path src/libraries/rust/stargate/crates/proto/proto -proto stargate.proto 127.0.0.1:50071 stargate.StargateControlPlane/WatchStargates; rc=$?; [ "$rc" -ne 0 ]' From 177cba3e4b9821a76e4d01c8b75863ef019a3788 Mon Sep 17 00:00:00 2001 From: Mike Camp Date: Fri, 28 Aug 2026 16:16:09 -0400 Subject: [PATCH 08/10] test(bdd): assert plaintext TLS rejection (#1323) Signed-off-by: Mike Camp --- ...ster-helmfile-llm-registration-tls.feature | 8 +- tests/bdd/godog_test.go | 14 +++- .../plaintext_tls_rejection_script_test.go | 81 +++++++++++++++++++ .../assert-grpcurl-plaintext-tls-rejection.sh | 19 +++++ 4 files changed, 117 insertions(+), 5 deletions(-) create mode 100644 tests/bdd/plaintext_tls_rejection_script_test.go create mode 100755 tests/bdd/scripts/assert-grpcurl-plaintext-tls-rejection.sh diff --git a/tests/bdd/features/multi-cluster-helmfile-llm-registration-tls.feature b/tests/bdd/features/multi-cluster-helmfile-llm-registration-tls.feature index 2262904fc..1c890de8f 100644 --- a/tests/bdd/features/multi-cluster-helmfile-llm-registration-tls.feature +++ b/tests/bdd/features/multi-cluster-helmfile-llm-registration-tls.feature @@ -83,11 +83,15 @@ Feature: Register an LLM worker securely with every router in a local split-clus And the command output should contain "Verify return code: 0 (ok)" And the command output should contain "ALPN protocol: h2" + # grpcurl reports a client-side dial deadline when plaintext HTTP/2 is + # sent to this verified TLS listener. The trusted Watch below proves + # that the same endpoint remains healthy. When I run command: """ - grpcurl -plaintext -max-time 5 -import-path src/libraries/rust/stargate/crates/proto/proto -proto stargate.proto 127.0.0.1:50071 stargate.StargateControlPlane/WatchStargates + /bin/bash -c 'set -u; output=$(grpcurl -plaintext -max-time 5 -import-path src/libraries/rust/stargate/crates/proto/proto -proto stargate.proto 127.0.0.1:50071 stargate.StargateControlPlane/WatchStargates 2>&1); rc=$?; if [ "$rc" -eq 0 ]; then printf "%s\n" "plaintext Watch unexpectedly succeeded" >&2; exit 1; fi; printf "%s\n" "$output" | bash tests/bdd/scripts/assert-grpcurl-plaintext-tls-rejection.sh' """ - Then the command exit code should be 1 + Then the command exit code should be 0 + And the command output should contain "plaintext-watch-rejected=tls-listener-timeout" # WatchStargates is a long-lived stream. Normalize grpcurl's deadline # exit after it prints the initial snapshot. diff --git a/tests/bdd/godog_test.go b/tests/bdd/godog_test.go index 542754337..3f3d899c7 100644 --- a/tests/bdd/godog_test.go +++ b/tests/bdd/godog_test.go @@ -1222,8 +1222,13 @@ func TestMultiClusterHelmfileLLMRegistrationTLSFeatureFileWiresToSteps(t *testin `-servername llm-request-router.nvcf.svc.cluster.local -alpn h2 -verify_return_error ` + `-CAfile <(kubectl --context k3d-ncp-local-cp get secret stargate-quic-tls -n nvcf ` + `-o jsonpath="{.data.ca\.crt}" | base64 -d) &1'` - plaintextWatchCommand = "grpcurl -plaintext -max-time 5 -import-path src/libraries/rust/stargate/crates/proto/proto -proto stargate.proto 127.0.0.1:50071 stargate.StargateControlPlane/WatchStargates" - tlsWatchCommand = `/bin/bash -c 'grpcurl -max-time 3 ` + + plaintextWatchCommand = `/bin/bash -c 'set -u; output=$(grpcurl -plaintext -max-time 5 ` + + `-import-path src/libraries/rust/stargate/crates/proto/proto -proto stargate.proto ` + + `127.0.0.1:50071 stargate.StargateControlPlane/WatchStargates 2>&1); rc=$?; ` + + `if [ "$rc" -eq 0 ]; then printf "%s\n" "plaintext Watch unexpectedly succeeded" >&2; ` + + `exit 1; fi; printf "%s\n" "$output" | ` + + `bash tests/bdd/scripts/assert-grpcurl-plaintext-tls-rejection.sh'` + tlsWatchCommand = `/bin/bash -c 'grpcurl -max-time 3 ` + `-cacert <(kubectl --context k3d-ncp-local-cp get secret stargate-quic-tls -n nvcf ` + `-o jsonpath="{.data.ca\.crt}" | base64 -d) ` + `-authority llm-request-router.nvcf.svc.cluster.local ` + @@ -1260,7 +1265,7 @@ func TestMultiClusterHelmfileLLMRegistrationTLSFeatureFileWiresToSteps(t *testin ExitCode: 0, Stdout: "ALPN protocol: h2\nVerify return code: 0 (ok)\n", }, - plaintextWatchCommand: {ExitCode: 1, Stderr: "tls: first record does not look like a TLS handshake"}, + plaintextWatchCommand: {ExitCode: 0, Stdout: "plaintext-watch-rejected=tls-listener-timeout\n"}, tlsWatchCommand: { ExitCode: 0, Stdout: `{ @@ -1326,6 +1331,9 @@ func TestMultiClusterHelmfileLLMRegistrationTLSFeatureFileWiresToSteps(t *testin if !commandRanExactly(suite.Runner.(*fakeRunner).runs, tlsWatchCommand) { t.Fatal("WatchStargates was not observed over the trusted TLS listener") } + if !commandRanExactly(suite.Runner.(*fakeRunner).runs, plaintextWatchCommand) { + t.Fatal("plaintext WatchStargates rejection was not exercised") + } } // TestSingleClusterHelmfileUpstreamImagesFeatureFileWiresToSteps runs the diff --git a/tests/bdd/plaintext_tls_rejection_script_test.go b/tests/bdd/plaintext_tls_rejection_script_test.go new file mode 100644 index 000000000..fbb75ea79 --- /dev/null +++ b/tests/bdd/plaintext_tls_rejection_script_test.go @@ -0,0 +1,81 @@ +/* +SPDX-FileCopyrightText: Copyright (c) NVIDIA CORPORATION & AFFILIATES. All rights reserved. +SPDX-License-Identifier: Apache-2.0 + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package bdd_tmp + +import ( + "os/exec" + "strings" + "testing" +) + +func TestPlaintextTLSRejectionAcceptsObservedGrpcurlDeadline(t *testing.T) { + cmd := exec.Command("bash", "scripts/assert-grpcurl-plaintext-tls-rejection.sh") + cmd.Stdin = strings.NewReader( + `Failed to dial target host "127.0.0.1:50071": context deadline exceeded`, + ) + + output, err := cmd.CombinedOutput() + if err != nil { + t.Fatalf("classify grpcurl deadline: %v\n%s", err, output) + } + if got, want := strings.TrimSpace(string(output)), "plaintext-watch-rejected=tls-listener-timeout"; got != want { + t.Fatalf("normalized output = %q, want %q", got, want) + } +} + +func TestPlaintextTLSRejectionRejectsSnapshotThenRPCDeadline(t *testing.T) { + cmd := exec.Command("bash", "scripts/assert-grpcurl-plaintext-tls-rejection.sh") + cmd.Stdin = strings.NewReader(`{ + "stargates": [] +} +ERROR: + Code: DeadlineExceeded + Message: context deadline exceeded`) + + output, err := cmd.CombinedOutput() + if err == nil { + t.Fatalf("successful plaintext snapshot followed by RPC deadline was accepted: %s", output) + } + if strings.Contains(string(output), "plaintext-watch-rejected=") { + t.Fatalf("successful plaintext snapshot emitted success marker: %s", output) + } +} + +func TestPlaintextTLSRejectionRejectsUnrelatedGrpcurlFailures(t *testing.T) { + for name, diagnostic := range map[string]string{ + "binary missing": "bash: grpcurl: command not found", + "connection refused": "Failed to dial target host 127.0.0.1:50071: connection refused", + "dial timeout plus output": "Failed to dial target host \"127.0.0.1:50071\": context deadline exceeded\n{}", + "proto import": "Failed to process proto source files.: missing.proto does not reside in any import path", + "usage": "flag provided but not defined: -bad-flag", + "wrong target dial timeout": "Failed to dial target host \"127.0.0.1:50443\": context deadline exceeded", + } { + t.Run(name, func(t *testing.T) { + cmd := exec.Command("bash", "scripts/assert-grpcurl-plaintext-tls-rejection.sh") + cmd.Stdin = strings.NewReader(diagnostic) + + output, err := cmd.CombinedOutput() + if err == nil { + t.Fatalf("unrelated failure was accepted: %s", output) + } + if strings.Contains(string(output), "plaintext-watch-rejected=") { + t.Fatalf("unrelated failure emitted success marker: %s", output) + } + }) + } +} diff --git a/tests/bdd/scripts/assert-grpcurl-plaintext-tls-rejection.sh b/tests/bdd/scripts/assert-grpcurl-plaintext-tls-rejection.sh new file mode 100755 index 000000000..453cdbc12 --- /dev/null +++ b/tests/bdd/scripts/assert-grpcurl-plaintext-tls-rejection.sh @@ -0,0 +1,19 @@ +#!/usr/bin/env bash +# SPDX-FileCopyrightText: Copyright (c) NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Normalize only the complete grpcurl 1.9.3 dial-stage diagnostic observed when +# plaintext HTTP/2 is sent to the verified TLS listener. RPC stream deadlines, +# additional output, and unrelated command or endpoint failures remain errors. + +set -euo pipefail + +diagnostic="$(cat)" +expected_diagnostic='Failed to dial target host "127.0.0.1:50071": context deadline exceeded' +if [[ "${diagnostic}" == "${expected_diagnostic}" ]]; then + printf '%s\n' 'plaintext-watch-rejected=tls-listener-timeout' + exit 0 +fi + +printf '%s\n' 'plaintext Watch failed without the expected TLS-listener dial timeout' >&2 +exit 1 From f6a99dd5c82356a7a78ca55667858dd7fa0d8ba5 Mon Sep 17 00:00:00 2001 From: Stephanie Baum Date: Sun, 30 Aug 2026 00:03:36 -0700 Subject: [PATCH 09/10] test(bdd): use shared registration observability steps Reuse the shared WatchStargates and Pylon metrics assertions while keeping the endpoint, trust, topology, and expected counts visible in Gherkin. Refs #1305 Refs #1358 Signed-off-by: Stephanie Baum --- ...ster-helmfile-llm-registration-tls.feature | 37 ++++++++----------- tests/bdd/godog_test.go | 35 ++++++------------ 2 files changed, 27 insertions(+), 45 deletions(-) diff --git a/tests/bdd/features/multi-cluster-helmfile-llm-registration-tls.feature b/tests/bdd/features/multi-cluster-helmfile-llm-registration-tls.feature index 1c890de8f..f2e5cc931 100644 --- a/tests/bdd/features/multi-cluster-helmfile-llm-registration-tls.feature +++ b/tests/bdd/features/multi-cluster-helmfile-llm-registration-tls.feature @@ -93,17 +93,13 @@ Feature: Register an LLM worker securely with every router in a local split-clus Then the command exit code should be 0 And the command output should contain "plaintext-watch-rejected=tls-listener-timeout" - # WatchStargates is a long-lived stream. Normalize grpcurl's deadline - # exit after it prints the initial snapshot. - When I run command: - """ - /bin/bash -c 'grpcurl -max-time 3 -cacert <(kubectl --context k3d-ncp-local-cp get secret stargate-quic-tls -n nvcf -o jsonpath="{.data.ca\.crt}" | base64 -d) -authority llm-request-router.nvcf.svc.cluster.local -import-path src/libraries/rust/stargate/crates/proto/proto -proto stargate.proto 127.0.0.1:50071 stargate.StargateControlPlane/WatchStargates; rc=$?; [ "$rc" -ne 0 ]' - """ - Then the command exit code should be 0 - And the command output should contain "llm-request-router-0" - And the command output should contain "llm-request-router-1" - And the command output should contain "llm-request-router-2" - And the command output should contain "https://llm-request-router.nvcf.svc.cluster.local:50071" + When I successfully observe WatchStargates at "127.0.0.1:50071" with TLS authority "llm-request-router.nvcf.svc.cluster.local" using CA secret "stargate-quic-tls" in namespace "nvcf" and context "k3d-ncp-local-cp" for "3" seconds + Then the command output should contain all: + | text | + | llm-request-router-0 | + | llm-request-router-1 | + | llm-request-router-2 | + | https://llm-request-router.nvcf.svc.cluster.local:50071 | When I run command: """ @@ -164,20 +160,17 @@ Feature: Register an LLM worker securely with every router in a local split-clus | --description | bdd-registration-tls | | --scopes | invoke_function,list_functions,queue_details,list_functions_details | - # Discover the workload pod by its public sidecar name, then poll the - # Pylon metrics endpoint until all three router streams and reverse - # tunnels are connected. - When I run command: - """ - /bin/sh -c 'set -eu; for attempt in $(seq 1 120); do row=$(kubectl --context k3d-ncp-local-compute-1 get pods -A -o json | jq -r "[.items[] | select(any(.spec.containers[]?; .name == \"llm-worker\")) | [.metadata.namespace,.metadata.name] | @tsv] | first // empty"); if [ -n "$row" ]; then ns=$(printf "%s" "$row" | cut -f1); pod=$(printf "%s" "$row" | cut -f2); metrics=$(kubectl --context k3d-ncp-local-compute-1 get --raw "/api/v1/namespaces/$ns/pods/$pod:9089/proxy/metrics" 2>/dev/null || true); registration=$(printf "%s\n" "$metrics" | grep -c "^pylon_registration_stream_connected.* 1$" || true); tunnels=$(printf "%s\n" "$metrics" | grep -c "^pylon_reverse_tunnel_connected.* 1$" || true); if [ "$registration" -eq 3 ] && [ "$tunnels" -eq 3 ]; then printf "registration=%s reverse=%s\n" "$registration" "$tunnels"; exit 0; fi; fi; sleep 5; done; exit 1' - """ - Then the command exit code should be 0 - And the command output should contain "registration=3 reverse=3" + Then a pod containing container "llm-worker" using context "k3d-ncp-local-compute-1" should report Pylon metrics within "10m": + | metric | comparison | count | + | pylon_registration_stream_connected | exactly | 3 | + | pylon_reverse_tunnel_connected | exactly | 3 | When I successfully invoke model "openai-compatible-sample" at "/v1/chat/completions" with timeout "120" seconds: """ {"messages":[{"role":"user","content":"bdd-registration-tls"}]} """ - Then the command output should contain "chat.completion" - And the command output should contain "fixed 128-byte response" + Then the command output should contain all: + | text | + | chat.completion | + | fixed 128-byte response | And I successfully undeploy the function selected by NVCF CLI diff --git a/tests/bdd/godog_test.go b/tests/bdd/godog_test.go index 3f3d899c7..b585b8be5 100644 --- a/tests/bdd/godog_test.go +++ b/tests/bdd/godog_test.go @@ -1228,28 +1228,13 @@ func TestMultiClusterHelmfileLLMRegistrationTLSFeatureFileWiresToSteps(t *testin `if [ "$rc" -eq 0 ]; then printf "%s\n" "plaintext Watch unexpectedly succeeded" >&2; ` + `exit 1; fi; printf "%s\n" "$output" | ` + `bash tests/bdd/scripts/assert-grpcurl-plaintext-tls-rejection.sh'` - tlsWatchCommand = `/bin/bash -c 'grpcurl -max-time 3 ` + - `-cacert <(kubectl --context k3d-ncp-local-cp get secret stargate-quic-tls -n nvcf ` + - `-o jsonpath="{.data.ca\.crt}" | base64 -d) ` + - `-authority llm-request-router.nvcf.svc.cluster.local ` + - `-import-path src/libraries/rust/stargate/crates/proto/proto -proto stargate.proto ` + - `127.0.0.1:50071 stargate.StargateControlPlane/WatchStargates; ` + - `rc=$?; [ "$rc" -ne 0 ]'` - pylonMetricsCommand = `/bin/sh -c 'set -eu; for attempt in $(seq 1 120); do ` + - `row=$(kubectl --context k3d-ncp-local-compute-1 get pods -A -o json | ` + - `jq -r "[.items[] | select(any(.spec.containers[]?; .name == \"llm-worker\")) | ` + - `[.metadata.namespace,.metadata.name] | @tsv] | first // empty"); ` + - `if [ -n "$row" ]; then ns=$(printf "%s" "$row" | cut -f1); ` + - `pod=$(printf "%s" "$row" | cut -f2); ` + - `metrics=$(kubectl --context k3d-ncp-local-compute-1 get --raw ` + - `"/api/v1/namespaces/$ns/pods/$pod:9089/proxy/metrics" 2>/dev/null || true); ` + - `registration=$(printf "%s\n" "$metrics" | ` + - `grep -c "^pylon_registration_stream_connected.* 1$" || true); ` + - `tunnels=$(printf "%s\n" "$metrics" | ` + - `grep -c "^pylon_reverse_tunnel_connected.* 1$" || true); ` + - `if [ "$registration" -eq 3 ] && [ "$tunnels" -eq 3 ]; then ` + - `printf "registration=%s reverse=%s\n" "$registration" "$tunnels"; exit 0; fi; ` + - `fi; sleep 5; done; exit 1'` + tlsWatchCommand = "bash tests/bdd/scripts/observe-watch-stargates.sh" + + " 127.0.0.1:50071 llm-request-router.nvcf.svc.cluster.local" + + " stargate-quic-tls nvcf k3d-ncp-local-cp 3" + pylonMetricsCommand = "bash tests/bdd/scripts/wait-pylon-metrics.sh" + + " llm-worker k3d-ncp-local-compute-1 10m" + + " pylon_registration_stream_connected exactly 3" + + " pylon_reverse_tunnel_connected exactly 3" invokeCommand = "/usr/bin/nvcf-cli --config /repo-root-placeholder/tests/bdd/fixtures/nvcf-cli-local.yaml function invoke" + " --inference-url /v1/chat/completions --model-name openai-compatible-sample" + " --request-body '{\"messages\":[{\"role\":\"user\",\"content\":\"bdd-registration-tls\"}]}' --timeout 120" @@ -1277,7 +1262,11 @@ func TestMultiClusterHelmfileLLMRegistrationTLSFeatureFileWiresToSteps(t *testin }`, Stderr: "ERROR: DeadlineExceeded", }, - pylonMetricsCommand: {ExitCode: 0, Stdout: "registration=3 reverse=3\n"}, + pylonMetricsCommand: { + ExitCode: 0, + Stdout: "pylon_registration_stream_connected=3\n" + + "pylon_reverse_tunnel_connected=3\n", + }, invokeCommand: { ExitCode: 0, Stdout: "Function invocation completed!\n\nResponse:\n" + From 6cf1b304b093c108a3e26ab9f47a20ef7efe1e00 Mon Sep 17 00:00:00 2001 From: Stephanie Baum Date: Sun, 30 Aug 2026 00:41:56 -0700 Subject: [PATCH 10/10] test(bdd): target registration Pylons by function Refs #1292 Signed-off-by: Stephanie Baum --- .../multi-cluster-helmfile-llm-registration-tls.feature | 2 +- tests/bdd/godog_test.go | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/bdd/features/multi-cluster-helmfile-llm-registration-tls.feature b/tests/bdd/features/multi-cluster-helmfile-llm-registration-tls.feature index f2e5cc931..4979e59c2 100644 --- a/tests/bdd/features/multi-cluster-helmfile-llm-registration-tls.feature +++ b/tests/bdd/features/multi-cluster-helmfile-llm-registration-tls.feature @@ -160,7 +160,7 @@ Feature: Register an LLM worker securely with every router in a local split-clus | --description | bdd-registration-tls | | --scopes | invoke_function,list_functions,queue_details,list_functions_details | - Then a pod containing container "llm-worker" using context "k3d-ncp-local-compute-1" should report Pylon metrics within "10m": + Then every Pylon for function "bdd-registration-tls" using container "llm-worker" and context "k3d-ncp-local-compute-1" should report metrics within "10m": | metric | comparison | count | | pylon_registration_stream_connected | exactly | 3 | | pylon_reverse_tunnel_connected | exactly | 3 | diff --git a/tests/bdd/godog_test.go b/tests/bdd/godog_test.go index b585b8be5..8c62ad016 100644 --- a/tests/bdd/godog_test.go +++ b/tests/bdd/godog_test.go @@ -1232,7 +1232,7 @@ func TestMultiClusterHelmfileLLMRegistrationTLSFeatureFileWiresToSteps(t *testin " 127.0.0.1:50071 llm-request-router.nvcf.svc.cluster.local" + " stargate-quic-tls nvcf k3d-ncp-local-cp 3" pylonMetricsCommand = "bash tests/bdd/scripts/wait-pylon-metrics.sh" + - " llm-worker k3d-ncp-local-compute-1 10m" + + " bdd-registration-tls llm-worker k3d-ncp-local-compute-1 10m" + " pylon_registration_stream_connected exactly 3" + " pylon_reverse_tunnel_connected exactly 3" invokeCommand = "/usr/bin/nvcf-cli --config /repo-root-placeholder/tests/bdd/fixtures/nvcf-cli-local.yaml function invoke" +