diff --git a/tests/bdd/PLAN.md b/tests/bdd/PLAN.md index e53c0aecf..e1b7ce77b 100644 --- a/tests/bdd/PLAN.md +++ b/tests/bdd/PLAN.md @@ -122,6 +122,16 @@ refactor in every consumer; that is a feature. | `When I run command with a terminal:` (docstring) | Same as the docstring form, but stdin is attached to a pseudo-terminal so the child sees a TTY on fd 0. For commands that gate interactive-only behavior on a TTY, such as `nvcf-cli self-hosted up` (its auth-gate mints the admin token only when stdin is a terminal). No input is written; stdout and stderr are captured separately as usual. | | `When I export command output to environment variable {string}` | Exports the previous command's trimmed stdout under the named env var. Fails the step unless the prior command exited 0 and produced non-empty stdout. Snapshotted by the env Ledger; restored at suite teardown. | +#### Registration observability command adapters + +These steps wrap repeated client, trust-material, polling, and output-format +mechanics while keeping every operator-selected target and expectation visible. +They preserve the real command output for subsequent assertions. + +| Step | Command | +|------|---------| +| `When I successfully observe WatchStargates at {string} with TLS authority {string} using CA secret {string} in namespace {string} and context {string} for {string} seconds` | Reads the named CA certificate from the explicit Kubernetes secret and context, runs the public `WatchStargates` gRPC method against the visible endpoint and TLS authority, and requires a streamed response before accepting the expected client deadline. | + #### Function lifecycle command adapters These steps hide the repeated executable, config prefix, fixed subcommand, shell @@ -156,8 +166,11 @@ original order. Repeated options and empty values are preserved. | Step | Notes | |------|-------| | `Then the command exit code should be {int}` | Last-run exit code. | -| `Then the command output should contain {string}` | Substring match on combined stdout + stderr. | -| `Then the command output should not contain {string}` | Negative substring match. | +| `Then the command should fail` | Requires a non-zero last-run exit code. It does not accept a runner error that prevented command execution and never records the failed command in the successful-command cache. | +| `Then the command output should contain {string}` | Substring match on combined stdout + stderr. The interpolated value must not be empty or whitespace-only. | +| `Then the command output should not contain {string}` | Negative substring match. The interpolated value must not be empty or whitespace-only. | +| `Then the command output should contain all:` (table) | Requires a `text` header and one or more strings. Every interpolated string must be non-empty and appear in combined stdout + stderr. | +| `Then the command output should contain one of:` (table) | Requires a `text` header and one or more strings. Every interpolated candidate must be non-empty, and at least one must appear in combined stdout + stderr. | | `Then file {string} should exist` | | | `Then yaml file {string} key {string} should equal {string}` | Reads the YAML file, walks the dotted key path, compares to the value (with `${VAR}` expansion). | | `Then yaml file {string} key {string} should not be empty` | Same key resolution; passes if the resolved value is non-empty. Use for non-deterministic outputs (cluster IDs, identity sources) where exact-value assertions are wrong. | @@ -179,6 +192,7 @@ original order. Repeated options and empty values are preserved. | `Then deployment {string} in namespace {string} using context {string} should complete rollout within {string}` | Runs `kubectl rollout status` for the named deployment with the explicit namespace, context, and timeout. Failure messages name the deployment without printing command output. | | `Then NVCFBackend {string} in namespace {string} using context {string} should report agent status {string} within {string}` | Waits for the named backend's `status.agentStatus` to equal the visible value using the explicit namespace, context, and timeout. Failure messages name the backend without printing resource output. | | `Then these Gateway API routes should be accepted and resolved using context {string} within {string}:` (table) | Requires `kind`, `name`, `namespace`, and `parent` headers. Waits for every named route to report both `Accepted=True` and `ResolvedRefs=True` for the named Gateway parent using the explicit context and timeout. The route kind is passed through without an allowlist. Failures name the table row, route, namespace, parent, and unmet condition without printing resource output. | +| `Then every Pylon for function {string} using container {string} and context {string} should report metrics within {string}:` (table) | Requires `metric`, `comparison`, and `count` headers. Polls every running pod selected by the visible `function-name` annotation and container name. Each pod must expose non-empty metrics, and each metric row counts connected series whose sample value is `1`; `comparison` is `exactly` or `at least`, and the expected non-negative count remains visible. Discovery, parsing, and scrape failures remain failures rather than zero metric counts. | #### YAML comparison semantics @@ -514,6 +528,23 @@ func SubstituteFile(path, placeholder, replacement string) error // exists in the array. Extra objects in the array are fine. func JSONContainsRows(raw string, rows []map[string]string) error +// WatchStargatesCommand builds a TLS WatchStargates observation with an +// explicit endpoint, authority, CA source, Kubernetes context, and duration. +func WatchStargatesCommand(endpoint, authority, caSecret, namespace, kubeContext, durationSeconds string) (string, error) + +// PylonMetricExpectation describes an expected count of connected metric +// series exposed by one Pylon sidecar. +type PylonMetricExpectation struct { + Metric string + Comparison string + Count int +} + +// PylonMetricsCommand builds a Pylon metrics observation for every running +// Pylon pod selected by function name and container name in an explicit +// Kubernetes context and polling window. +func PylonMetricsCommand(functionName, containerName, kubeContext, timeout string, expectations []PylonMetricExpectation) (string, error) + // FilesDoNotContain recursively inspects regular files under root and // fails if any interpolated fixed string appears. func FilesDoNotContain(root string, needles []string) error diff --git a/tests/bdd/dsl/registration.go b/tests/bdd/dsl/registration.go new file mode 100644 index 000000000..1a2102b6f --- /dev/null +++ b/tests/bdd/dsl/registration.go @@ -0,0 +1,111 @@ +/* +SPDX-FileCopyrightText: Copyright (c) NVIDIA CORPORATION & AFFILIATES. All rights reserved. +SPDX-License-Identifier: Apache-2.0 + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package dsl + +import ( + "fmt" + "regexp" + "strconv" + "strings" +) + +const ( + observeWatchStargatesScript = "tests/bdd/scripts/observe-watch-stargates.sh" + waitPylonMetricsScript = "tests/bdd/scripts/wait-pylon-metrics.sh" +) + +var prometheusMetricNameRE = regexp.MustCompile(`^[a-zA-Z_:][a-zA-Z0-9_:]*$`) + +// PylonMetricExpectation describes an expected count of connected metric +// series exposed by one Pylon sidecar. +type PylonMetricExpectation struct { + Metric string + Comparison string + Count int +} + +// WatchStargatesCommand builds a TLS WatchStargates observation with an +// explicit endpoint, authority, CA source, Kubernetes context, and duration. +func WatchStargatesCommand(endpoint, authority, caSecret, namespace, kubeContext, durationSeconds string) (string, error) { + values := []*string{&endpoint, &authority, &caSecret, &namespace, &kubeContext, &durationSeconds} + labels := []string{"endpoint", "TLS authority", "CA secret", "namespace", "kube context", "duration seconds"} + for index := range values { + *values[index] = strings.TrimSpace(Interpolate(*values[index])) + if *values[index] == "" { + return "", fmt.Errorf("%s is empty", labels[index]) + } + } + duration, err := strconv.Atoi(durationSeconds) + if err != nil || duration <= 0 { + return "", fmt.Errorf("duration seconds must be a positive integer") + } + + return BuildCommand( + "bash", + observeWatchStargatesScript, + endpoint, + authority, + caSecret, + namespace, + kubeContext, + durationSeconds, + ), nil +} + +// PylonMetricsCommand builds a Pylon metrics observation for every running +// Pylon pod selected by function name and container name in an explicit +// Kubernetes context and polling window. +func PylonMetricsCommand(functionName, containerName, kubeContext, timeout string, expectations []PylonMetricExpectation) (string, error) { + functionName = strings.TrimSpace(Interpolate(functionName)) + containerName = strings.TrimSpace(Interpolate(containerName)) + kubeContext = strings.TrimSpace(Interpolate(kubeContext)) + timeout = strings.TrimSpace(Interpolate(timeout)) + if functionName == "" { + return "", fmt.Errorf("function name is empty") + } + if containerName == "" { + return "", fmt.Errorf("container name is empty") + } + if kubeContext == "" { + return "", fmt.Errorf("kube context is empty") + } + if timeout == "" { + return "", fmt.Errorf("timeout is empty") + } + if len(expectations) == 0 { + return "", fmt.Errorf("pylon metric expectations are empty") + } + + args := []string{"bash", waitPylonMetricsScript, functionName, containerName, kubeContext, timeout} + for index, expectation := range expectations { + expectation.Metric = strings.TrimSpace(Interpolate(expectation.Metric)) + expectation.Comparison = strings.TrimSpace(Interpolate(expectation.Comparison)) + if !prometheusMetricNameRE.MatchString(expectation.Metric) { + return "", fmt.Errorf("metric row %d has invalid metric name %q", index+1, expectation.Metric) + } + if expectation.Comparison != "exactly" && expectation.Comparison != "at least" { + return "", fmt.Errorf("metric row %d comparison must be exactly or at least", index+1) + } + if expectation.Count < 0 { + return "", fmt.Errorf("metric row %d count must be non-negative", index+1) + } + args = append(args, expectation.Metric, expectation.Comparison, strconv.Itoa(expectation.Count)) + } + + return BuildCommand(args...), nil +} diff --git a/tests/bdd/dsl/registration_test.go b/tests/bdd/dsl/registration_test.go new file mode 100644 index 000000000..e52a33bdd --- /dev/null +++ b/tests/bdd/dsl/registration_test.go @@ -0,0 +1,277 @@ +/* +SPDX-FileCopyrightText: Copyright (c) NVIDIA CORPORATION & AFFILIATES. All rights reserved. +SPDX-License-Identifier: Apache-2.0 + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package dsl + +import ( + "os" + "os/exec" + "path/filepath" + "strings" + "testing" +) + +func TestWatchStargatesCommandKeepsObservationInputsExplicit(t *testing.T) { + t.Setenv("BDD_WATCH_CONTEXT", "k3d-ncp-local-cp") + got, err := WatchStargatesCommand( + "127.0.0.1:50071", + "llm-request-router.nvcf.svc.cluster.local", + "stargate-quic-tls", + "nvcf", + "${BDD_WATCH_CONTEXT}", + "3", + ) + if err != nil { + t.Fatalf("build WatchStargates command: %v", err) + } + want := "bash tests/bdd/scripts/observe-watch-stargates.sh 127.0.0.1:50071 llm-request-router.nvcf.svc.cluster.local stargate-quic-tls nvcf k3d-ncp-local-cp 3" + if got != want { + t.Fatalf("command = %q, want %q", got, want) + } +} + +func TestWatchStargatesCommandRejectsMissingOrInvalidInputs(t *testing.T) { + tests := []struct { + name string + endpoint string + authority string + duration string + }{ + {name: "empty endpoint", endpoint: "", authority: "router.nvcf.svc", duration: "3"}, + {name: "empty authority", endpoint: "127.0.0.1:50071", authority: "", duration: "3"}, + {name: "invalid duration", endpoint: "127.0.0.1:50071", authority: "router.nvcf.svc", duration: "3s"}, + } + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + if _, err := WatchStargatesCommand(test.endpoint, test.authority, "tls", "nvcf", "context", test.duration); err == nil { + t.Fatal("expected validation error") + } + }) + } +} + +func TestPylonMetricsCommandKeepsExpectationsExplicit(t *testing.T) { + got, err := PylonMetricsCommand("bdd-registration-tls", "llm-worker", "k3d-ncp-local-compute-1", "10m", []PylonMetricExpectation{ + {Metric: "pylon_registration_stream_connected", Comparison: "exactly", Count: 5}, + {Metric: "pylon_reverse_tunnel_connected", Comparison: "at least", Count: 3}, + }) + if err != nil { + t.Fatalf("build Pylon metrics command: %v", err) + } + want := "bash tests/bdd/scripts/wait-pylon-metrics.sh bdd-registration-tls llm-worker k3d-ncp-local-compute-1 10m pylon_registration_stream_connected exactly 5 pylon_reverse_tunnel_connected 'at least' 3" + if got != want { + t.Fatalf("command = %q, want %q", got, want) + } +} + +func TestPylonMetricsCommandRejectsInvalidExpectations(t *testing.T) { + tests := []struct { + name string + expectations []PylonMetricExpectation + }{ + {name: "empty", expectations: nil}, + {name: "invalid metric", expectations: []PylonMetricExpectation{{Metric: "metric name", Comparison: "exactly", Count: 1}}}, + {name: "invalid comparison", expectations: []PylonMetricExpectation{{Metric: "metric_name", Comparison: "more than", Count: 1}}}, + {name: "negative count", expectations: []PylonMetricExpectation{{Metric: "metric_name", Comparison: "exactly", Count: -1}}}, + } + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + if _, err := PylonMetricsCommand("function", "llm-worker", "context", "10m", test.expectations); err == nil { + t.Fatal("expected validation error") + } + }) + } +} + +func TestObserveWatchStargatesScriptAcceptsSnapshotThenDeadline(t *testing.T) { + fakeBin := t.TempDir() + writeExecutable(t, filepath.Join(fakeBin, "kubectl"), "#!/bin/sh\nprintf 'dGVzdC1jYQ=='\n") + writeExecutable(t, filepath.Join(fakeBin, "grpcurl"), `#!/bin/sh +printf '{\n "stargates": []\n}\n' +sleep 1 +printf 'ERROR:\n Code: DeadlineExceeded\n Message: context deadline exceeded\n' >&2 +exit 1 +`) + + output, err := runRegistrationScript(t, fakeBin, "observe-watch-stargates.sh", "127.0.0.1:50071", "router.nvcf.svc", "tls", "nvcf", "context", "1") + if err != nil { + t.Fatalf("observe WatchStargates: %v\n%s", err, output) + } + for _, want := range []string{`"stargates"`, "DeadlineExceeded"} { + if !strings.Contains(output, want) { + t.Fatalf("output = %q, want %q", output, want) + } + } +} + +func TestObserveWatchStargatesScriptRejectsImmediateProductDeadline(t *testing.T) { + fakeBin := t.TempDir() + writeExecutable(t, filepath.Join(fakeBin, "kubectl"), "#!/bin/sh\nprintf 'dGVzdC1jYQ=='\n") + writeExecutable(t, filepath.Join(fakeBin, "grpcurl"), `#!/bin/sh +printf '{\n "stargates": []\n}\n' +printf 'ERROR:\n Code: DeadlineExceeded\n Message: product returned deadline exceeded\n' >&2 +exit 1 +`) + + output, err := runRegistrationScript(t, fakeBin, "observe-watch-stargates.sh", "127.0.0.1:50071", "router.nvcf.svc", "tls", "nvcf", "context", "3") + if err == nil { + t.Fatal("expected immediate product deadline failure") + } + if !strings.Contains(output, "before the 3s observation deadline") { + t.Fatalf("output = %q, want early deadline diagnostic", output) + } +} + +func TestObserveWatchStargatesScriptRejectsDeadlineWithoutSnapshot(t *testing.T) { + fakeBin := t.TempDir() + writeExecutable(t, filepath.Join(fakeBin, "kubectl"), "#!/bin/sh\nprintf 'dGVzdC1jYQ=='\n") + writeExecutable(t, filepath.Join(fakeBin, "grpcurl"), "#!/bin/sh\nprintf 'context deadline exceeded\\n' >&2\nexit 1\n") + + output, err := runRegistrationScript(t, fakeBin, "observe-watch-stargates.sh", "127.0.0.1:50071", "router.nvcf.svc", "tls", "nvcf", "context", "3") + if err == nil { + t.Fatal("expected missing snapshot failure") + } + if !strings.Contains(output, "did not return a streamed snapshot") { + t.Fatalf("output = %q, want missing snapshot diagnostic", output) + } +} + +func TestWaitPylonMetricsScriptChecksEverySelectedPodAndCountsTimestampedSeries(t *testing.T) { + fakeBin := t.TempDir() + writeExecutable(t, filepath.Join(fakeBin, "kubectl"), `#!/bin/sh +case "$*" in + *"get pods -A -o json"*) + printf '%s\n' '{"items":[' + printf '%s\n' '{"metadata":{"namespace":"functions","name":"worker-0","annotations":{"function-name":"bdd-registration-tls"}},"status":{"phase":"Running"},"spec":{"containers":[{"name":"llm-worker"}]}},' + printf '%s\n' '{"metadata":{"namespace":"functions","name":"worker-1","annotations":{"function-name":"bdd-registration-tls"}},"status":{"phase":"Running"},"spec":{"containers":[{"name":"llm-worker"}]}},' + printf '%s\n' '{"metadata":{"namespace":"other","name":"worker-other","annotations":{"function-name":"another-function"}},"status":{"phase":"Running"},"spec":{"containers":[{"name":"llm-worker"}]}}' + printf '%s\n' ']}' + ;; + *) + printf 'pylon_registration_stream_connected{router="a"} 1 1712345678\n' + printf 'pylon_registration_stream_connected{router="b"} 1\n' + printf 'pylon_registration_stream_connected{router="c"} 1\n' + printf 'pylon_reverse_tunnel_connected{router="a"} 1\n' + printf 'pylon_reverse_tunnel_connected{router="b"} 1\n' + printf 'pylon_reverse_tunnel_connected{router="c"} 1\n' + ;; +esac +`) + + output, err := runRegistrationScript( + t, + fakeBin, + "wait-pylon-metrics.sh", + "bdd-registration-tls", + "llm-worker", + "k3d-ncp-local-compute-1", + "1s", + "pylon_registration_stream_connected", + "exactly", + "3", + "pylon_reverse_tunnel_connected", + "at least", + "3", + ) + if err != nil { + t.Fatalf("wait for Pylon metrics: %v\n%s", err, output) + } + for _, want := range []string{ + "functions/worker-0 pylon_registration_stream_connected=3", + "functions/worker-0 pylon_reverse_tunnel_connected=3", + "functions/worker-1 pylon_registration_stream_connected=3", + "functions/worker-1 pylon_reverse_tunnel_connected=3", + } { + if !strings.Contains(output, want) { + t.Fatalf("output = %q, want %q", output, want) + } + } + if strings.Contains(output, "worker-other") { + t.Fatalf("output = %q, did not want metrics from another function", output) + } +} + +func TestWaitPylonMetricsScriptDoesNotTreatScrapeFailureAsZero(t *testing.T) { + fakeBin := t.TempDir() + writeExecutable(t, filepath.Join(fakeBin, "kubectl"), `#!/bin/sh +case "$*" in + *"get pods -A -o json"*) + printf '%s\n' '{"items":[{"metadata":{"namespace":"functions","name":"worker-0","annotations":{"function-name":"bdd-registration-tls"}},"status":{"phase":"Running"},"spec":{"containers":[{"name":"llm-worker"}]}}]}' + ;; + *) printf 'metrics endpoint unavailable\n' >&2; exit 1 ;; +esac +`) + + output, err := runRegistrationScript( + t, + fakeBin, + "wait-pylon-metrics.sh", + "bdd-registration-tls", + "llm-worker", + "k3d-ncp-local-compute-1", + "1s", + "pylon_registration_stream_connected", + "exactly", + "0", + ) + if err == nil { + t.Fatal("expected metrics scrape failure") + } + if !strings.Contains(output, "metrics scrape failed: metrics endpoint unavailable") { + t.Fatalf("output = %q, want preserved scrape failure", output) + } +} + +func TestWaitPylonMetricsScriptPreservesPodDiscoveryFailure(t *testing.T) { + fakeBin := t.TempDir() + writeExecutable(t, filepath.Join(fakeBin, "kubectl"), "#!/bin/sh\nprintf 'API server unavailable\\n' >&2\nexit 1\n") + + output, err := runRegistrationScript( + t, + fakeBin, + "wait-pylon-metrics.sh", + "bdd-registration-tls", + "llm-worker", + "k3d-ncp-local-compute-1", + "1s", + "pylon_registration_stream_connected", + "exactly", + "0", + ) + if err == nil { + t.Fatal("expected pod discovery failure") + } + if !strings.Contains(output, "pod discovery failed: API server unavailable") { + t.Fatalf("output = %q, want preserved discovery failure", output) + } +} + +func runRegistrationScript(t *testing.T, fakeBin, scriptName string, args ...string) (string, error) { + t.Helper() + script := filepath.Join("..", "scripts", scriptName) + command := exec.Command("bash", append([]string{script}, args...)...) + command.Env = append(os.Environ(), "PATH="+fakeBin+string(os.PathListSeparator)+os.Getenv("PATH")) + output, err := command.CombinedOutput() + return string(output), err +} + +func writeExecutable(t *testing.T, path, body string) { + t.Helper() + if err := os.WriteFile(path, []byte(body), 0o755); err != nil { + t.Fatalf("write executable %s: %v", path, err) + } +} diff --git a/tests/bdd/scripts/observe-watch-stargates.sh b/tests/bdd/scripts/observe-watch-stargates.sh new file mode 100644 index 000000000..9813b3fcb --- /dev/null +++ b/tests/bdd/scripts/observe-watch-stargates.sh @@ -0,0 +1,85 @@ +#!/usr/bin/env bash +# SPDX-FileCopyrightText: Copyright (c) NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +set -euo pipefail + +if [[ $# -ne 6 ]]; then + echo "usage: $0 " >&2 + exit 64 +fi + +endpoint="$1" +tls_authority="$2" +ca_secret="$3" +namespace="$4" +kube_context="$5" +duration_seconds="$6" + +for value_name in endpoint tls_authority ca_secret namespace kube_context; do + if [[ -z "${!value_name}" ]]; then + echo "$value_name must be non-empty" >&2 + exit 64 + fi +done +if ! [[ "$duration_seconds" =~ ^[1-9][0-9]*$ ]]; then + echo "duration-seconds must be a positive integer, got: $duration_seconds" >&2 + exit 64 +fi +for tool in kubectl base64 grpcurl jq; do + if ! command -v "$tool" >/dev/null 2>&1; then + echo "required tool not found: $tool" >&2 + exit 127 + fi +done + +script_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd -P)" +repo_root="$(cd "$script_dir/../../.." && pwd -P)" +proto_path="$repo_root/src/libraries/rust/stargate/crates/proto/proto" +ca_file="$(mktemp "${TMPDIR:-/tmp}/nvcf-bdd-watch-ca.XXXXXX")" +stdout_file="$(mktemp "${TMPDIR:-/tmp}/nvcf-bdd-watch-stdout.XXXXXX")" +stderr_file="$(mktemp "${TMPDIR:-/tmp}/nvcf-bdd-watch-stderr.XXXXXX")" +trap 'rm -f "$ca_file" "$stdout_file" "$stderr_file"' EXIT + +kubectl --context "$kube_context" get secret "$ca_secret" -n "$namespace" \ + -o 'jsonpath={.data.ca\.crt}' | base64 -d >"$ca_file" +if [[ ! -s "$ca_file" ]]; then + echo "CA secret $namespace/$ca_secret did not contain ca.crt" >&2 + exit 1 +fi + +set +e +started_at="$(date +%s)" +grpcurl \ + -max-time "$duration_seconds" \ + -emit-defaults \ + -cacert "$ca_file" \ + -authority "$tls_authority" \ + -import-path "$proto_path" \ + -proto stargate.proto \ + "$endpoint" \ + stargate.StargateControlPlane/WatchStargates >"$stdout_file" 2>"$stderr_file" +grpcurl_status=$? +finished_at="$(date +%s)" +set -e + +cat "$stdout_file" +cat "$stderr_file" >&2 + +if [[ "$grpcurl_status" -eq 0 ]]; then + echo "WatchStargates ended before the observation deadline" >&2 + exit 1 +fi +if ! jq -se 'length > 0 and all(.[]; type == "object" and (has("stargates") or has("watchStargateUrls")))' "$stdout_file" >/dev/null; then + echo "WatchStargates did not return a streamed snapshot" >&2 + exit 1 +fi +if ! grep -Eiq 'DeadlineExceeded|context deadline exceeded' "$stderr_file"; then + echo "WatchStargates failed before the expected observation deadline" >&2 + exit 1 +fi +elapsed_seconds=$(( finished_at - started_at )) +if [[ "$elapsed_seconds" -lt "$duration_seconds" ]]; then + echo "WatchStargates ended after ${elapsed_seconds}s before the ${duration_seconds}s observation deadline" >&2 + exit 1 +fi diff --git a/tests/bdd/scripts/wait-pylon-metrics.sh b/tests/bdd/scripts/wait-pylon-metrics.sh new file mode 100644 index 000000000..56cf1e1ea --- /dev/null +++ b/tests/bdd/scripts/wait-pylon-metrics.sh @@ -0,0 +1,148 @@ +#!/usr/bin/env bash +# SPDX-FileCopyrightText: Copyright (c) NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +set -euo pipefail + +if [[ $# -lt 7 || $(( ( $# - 4 ) % 3 )) -ne 0 ]]; then + echo "usage: $0 [...]" >&2 + exit 64 +fi + +function_name="$1" +container_name="$2" +kube_context="$3" +timeout="$4" +shift 4 + +if [[ -z "$function_name" || -z "$container_name" || -z "$kube_context" ]]; then + echo "function-name, container, and kube-context must be non-empty" >&2 + exit 64 +fi +if ! [[ "$timeout" =~ ^([1-9][0-9]*)(s|m|h)$ ]]; then + echo "timeout must be a positive duration ending in s, m, or h, got: $timeout" >&2 + exit 64 +fi +case "${BASH_REMATCH[2]}" in + s) timeout_seconds="${BASH_REMATCH[1]}" ;; + m) timeout_seconds=$(( BASH_REMATCH[1] * 60 )) ;; + h) timeout_seconds=$(( BASH_REMATCH[1] * 3600 )) ;; +esac +for tool in kubectl jq awk; do + if ! command -v "$tool" >/dev/null 2>&1; then + echo "required tool not found: $tool" >&2 + exit 127 + fi +done + +metric_names=() +comparisons=() +expected_counts=() +while [[ $# -gt 0 ]]; do + metric="$1" + comparison="$2" + expected_count="$3" + shift 3 + + if ! [[ "$metric" =~ ^[a-zA-Z_:][a-zA-Z0-9_:]*$ ]]; then + echo "invalid Prometheus metric name: $metric" >&2 + exit 64 + fi + if [[ "$comparison" != "exactly" && "$comparison" != "at least" ]]; then + echo "comparison must be exactly or at least, got: $comparison" >&2 + exit 64 + fi + if ! [[ "$expected_count" =~ ^[0-9]+$ ]]; then + echo "count must be a non-negative integer, got: $expected_count" >&2 + exit 64 + fi + + metric_names+=("$metric") + comparisons+=("$comparison") + expected_counts+=("$expected_count") +done + +connected_series_count() { + local metric="$1" + awk -v metric="$metric" ' + { + series = $1 + if ((series == metric || index(series, metric "{") == 1) && $2 == "1") { + count++ + } + } + END { print count + 0 } + ' +} + +deadline=$(( $(date +%s) + timeout_seconds )) +last_summary="no running pod for function $function_name containing container $container_name" + +while true; do + if ! pods_json="$(kubectl --context "$kube_context" get pods -A -o json 2>&1)"; then + last_summary="pod discovery failed: $pods_json" + elif ! pod_rows="$(printf '%s\n' "$pods_json" | jq -r --arg function "$function_name" --arg container "$container_name" ' + .items[]? + | select(.metadata.deletionTimestamp == null) + | select(.status.phase == "Running") + | select(.metadata.annotations["function-name"] == $function) + | select(any(.spec.containers[]?; .name == $container)) + | [.metadata.namespace, .metadata.name] + | @tsv + ' 2>&1)"; then + last_summary="pod discovery response could not be parsed: $pod_rows" + elif [[ -z "$pod_rows" ]]; then + last_summary="no running pod for function $function_name containing container $container_name" + else + all_match=true + summaries=() + + while IFS=$'\t' read -r namespace pod_name; do + if ! metrics="$(kubectl --context "$kube_context" get --raw "/api/v1/namespaces/$namespace/pods/$pod_name:9089/proxy/metrics" 2>&1)"; then + summaries+=("$namespace/$pod_name metrics scrape failed: $metrics") + all_match=false + continue + fi + if [[ -z "$metrics" ]]; then + summaries+=("$namespace/$pod_name metrics scrape returned an empty response") + all_match=false + continue + fi + + for index in "${!metric_names[@]}"; do + metric="${metric_names[$index]}" + comparison="${comparisons[$index]}" + expected_count="${expected_counts[$index]}" + observed_count="$(printf '%s\n' "$metrics" | connected_series_count "$metric")" + summaries+=("$namespace/$pod_name $metric=$observed_count") + + case "$comparison" in + exactly) + [[ "$observed_count" -eq "$expected_count" ]] || all_match=false + ;; + "at least") + [[ "$observed_count" -ge "$expected_count" ]] || all_match=false + ;; + esac + done + done <<<"$pod_rows" + + last_summary="${summaries[*]}" + if [[ "$all_match" == true ]]; then + printf '%s\n' "${summaries[@]}" + exit 0 + fi + fi + + now="$(date +%s)" + if [[ "$now" -ge "$deadline" ]]; then + echo "timed out after $timeout waiting for Pylon metrics in context $kube_context ($last_summary)" >&2 + exit 1 + fi + remaining=$(( deadline - now )) + sleep_seconds=5 + if [[ "$remaining" -lt "$sleep_seconds" ]]; then + sleep_seconds="$remaining" + fi + sleep "$sleep_seconds" +done diff --git a/tests/bdd/steps/assertion_steps.go b/tests/bdd/steps/assertion_steps.go index dcd5bac6a..f651efed9 100644 --- a/tests/bdd/steps/assertion_steps.go +++ b/tests/bdd/steps/assertion_steps.go @@ -32,8 +32,11 @@ import ( // values from the feature file. func registerAssertionSteps(ctx *godog.ScenarioContext, sc *ScenarioContext) { ctx.Step(`^the command exit code should be (\d+)$`, sc.commandExitCodeShouldBe) + ctx.Step(`^the command should fail$`, sc.commandShouldFail) ctx.Step(`^the command output should contain "([^"]*)"$`, sc.commandOutputShouldContain) ctx.Step(`^the command output should not contain "([^"]*)"$`, sc.commandOutputShouldNotContain) + ctx.Step(`^the command output should contain all:$`, sc.commandOutputShouldContainAll) + ctx.Step(`^the command output should contain one of:$`, sc.commandOutputShouldContainOneOf) ctx.Step(`^file "([^"]*)" should exist$`, sc.fileShouldExist) ctx.Step(`^yaml file "([^"]*)" key "([^"]*)" should equal "([^"]*)"$`, sc.yamlFileKeyShouldEqual) ctx.Step(`^yaml file "([^"]*)" key "([^"]*)" should not be empty$`, sc.yamlFileKeyShouldNotBeEmpty) @@ -87,9 +90,19 @@ func (sc *ScenarioContext) commandExitCodeShouldBe(expected int) error { return nil } +func (sc *ScenarioContext) commandShouldFail() error { + if sc.LastResult.ExitCode == 0 { + return fmt.Errorf("exit code = 0, want non-zero (see %s for stdout/stderr)", sc.Suite.Config.CommandLogDir) + } + return nil +} + func (sc *ScenarioContext) commandOutputShouldContain(needle string) error { combined := combinedOutput(sc.LastResult) - resolved := dsl.Interpolate(needle) + resolved, err := resolveOutputNeedle(needle) + if err != nil { + return err + } if !strings.Contains(combined, resolved) { return fmt.Errorf("output does not contain %q", resolved) } @@ -98,13 +111,64 @@ func (sc *ScenarioContext) commandOutputShouldContain(needle string) error { func (sc *ScenarioContext) commandOutputShouldNotContain(needle string) error { combined := combinedOutput(sc.LastResult) - resolved := dsl.Interpolate(needle) + resolved, err := resolveOutputNeedle(needle) + if err != nil { + return err + } if strings.Contains(combined, resolved) { return fmt.Errorf("output contains %q", resolved) } return nil } +func (sc *ScenarioContext) commandOutputShouldContainAll(table *godog.Table) error { + needles, err := tableToSingleColumn(table, "text") + if err != nil { + return err + } + combined := combinedOutput(sc.LastResult) + for index, needle := range needles { + resolved, err := resolveOutputNeedle(needle) + if err != nil { + return fmt.Errorf("row %d: %w", index+1, err) + } + if !strings.Contains(combined, resolved) { + return fmt.Errorf("output does not contain %q", resolved) + } + } + return nil +} + +func (sc *ScenarioContext) commandOutputShouldContainOneOf(table *godog.Table) error { + needles, err := tableToSingleColumn(table, "text") + if err != nil { + return err + } + resolvedNeedles := make([]string, 0, len(needles)) + for index, needle := range needles { + resolved, err := resolveOutputNeedle(needle) + if err != nil { + return fmt.Errorf("row %d: %w", index+1, err) + } + resolvedNeedles = append(resolvedNeedles, resolved) + } + combined := combinedOutput(sc.LastResult) + for _, resolved := range resolvedNeedles { + if strings.Contains(combined, resolved) { + return nil + } + } + return fmt.Errorf("output does not contain any of the %d expected values", len(needles)) +} + +func resolveOutputNeedle(needle string) (string, error) { + resolved := dsl.Interpolate(needle) + if strings.TrimSpace(resolved) == "" { + return "", fmt.Errorf("expected output text resolves to an empty value") + } + return resolved, nil +} + func (sc *ScenarioContext) yamlFileKeyShouldEqual(path, key, expected string) error { resolvedPath := sc.resolvePath(dsl.Interpolate(path)) got, found, err := dsl.ReadYAMLKey(resolvedPath, key) diff --git a/tests/bdd/steps/context.go b/tests/bdd/steps/context.go index 55acfc397..a88bed910 100644 --- a/tests/bdd/steps/context.go +++ b/tests/bdd/steps/context.go @@ -80,6 +80,7 @@ func RegisterAll(ctx *godog.ScenarioContext, sc *ScenarioContext) { registerFileSteps(ctx, sc) registerCommandSteps(ctx, sc) registerNVCFCLISteps(ctx, sc) + registerRegistrationSteps(ctx, sc) registerAssertionSteps(ctx, sc) registerInfraSteps(ctx, sc) } diff --git a/tests/bdd/steps/registration_steps.go b/tests/bdd/steps/registration_steps.go new file mode 100644 index 000000000..4dd957827 --- /dev/null +++ b/tests/bdd/steps/registration_steps.go @@ -0,0 +1,106 @@ +/* +SPDX-FileCopyrightText: Copyright (c) NVIDIA CORPORATION & AFFILIATES. All rights reserved. +SPDX-License-Identifier: Apache-2.0 + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package steps + +import ( + "context" + "fmt" + "strconv" + "strings" + + "github.com/cucumber/godog" + + "nvcf-bdd/dsl" +) + +func registerRegistrationSteps(ctx *godog.ScenarioContext, sc *ScenarioContext) { + ctx.Step(`^I successfully observe WatchStargates at "([^"]*)" with TLS authority "([^"]*)" using CA secret "([^"]*)" in namespace "([^"]*)" and context "([^"]*)" for "([^"]*)" seconds$`, sc.iSuccessfullyObserveWatchStargates) + ctx.Step(`^every Pylon for function "([^"]*)" using container "([^"]*)" and context "([^"]*)" should report metrics within "([^"]*)":$`, sc.everyPylonForFunctionShouldReportMetrics) +} + +func (sc *ScenarioContext) iSuccessfullyObserveWatchStargates( + ctx context.Context, + endpoint, + authority, + caSecret, + namespace, + kubeContext, + durationSeconds string, +) error { + command, err := dsl.WatchStargatesCommand(endpoint, authority, caSecret, namespace, kubeContext, durationSeconds) + if err != nil { + return err + } + if err := sc.runResolvedSuccessfully(ctx, command); err != nil { + return fmt.Errorf("WatchStargates at %q with TLS authority %q failed: %w", dsl.Interpolate(endpoint), dsl.Interpolate(authority), err) + } + return nil +} + +func (sc *ScenarioContext) everyPylonForFunctionShouldReportMetrics( + ctx context.Context, + functionName, + containerName, + kubeContext, + timeout string, + table *godog.Table, +) error { + expectations, err := tableToPylonMetricExpectations(table) + if err != nil { + return err + } + command, err := dsl.PylonMetricsCommand(functionName, containerName, kubeContext, timeout, expectations) + if err != nil { + return err + } + if err := sc.runResolvedSuccessfully(ctx, command); err != nil { + return fmt.Errorf("pylon pods for function %q using container %q did not report expected metrics: %w", dsl.Interpolate(functionName), dsl.Interpolate(containerName), err) + } + return nil +} + +func tableToPylonMetricExpectations(table *godog.Table) ([]dsl.PylonMetricExpectation, error) { + if table == nil || len(table.Rows) < 2 { + return nil, fmt.Errorf("table must have metric, comparison, and count headers and at least one data row") + } + headers := table.Rows[0].Cells + if len(headers) != 3 || + strings.TrimSpace(headers[0].Value) != "metric" || + strings.TrimSpace(headers[1].Value) != "comparison" || + strings.TrimSpace(headers[2].Value) != "count" { + return nil, fmt.Errorf("table headers must be metric, comparison, and count") + } + + expectations := make([]dsl.PylonMetricExpectation, 0, len(table.Rows)-1) + for index, row := range table.Rows[1:] { + if len(row.Cells) != len(headers) { + return nil, fmt.Errorf("row %d has %d cells, expected %d", index+1, len(row.Cells), len(headers)) + } + countText := strings.TrimSpace(dsl.Interpolate(row.Cells[2].Value)) + count, err := strconv.Atoi(countText) + if err != nil || count < 0 { + return nil, fmt.Errorf("row %d count must be a non-negative integer", index+1) + } + expectations = append(expectations, dsl.PylonMetricExpectation{ + Metric: row.Cells[0].Value, + Comparison: row.Cells[1].Value, + Count: count, + }) + } + return expectations, nil +} diff --git a/tests/bdd/steps/steps_test.go b/tests/bdd/steps/steps_test.go index 1189730d8..bcfed0cdb 100644 --- a/tests/bdd/steps/steps_test.go +++ b/tests/bdd/steps/steps_test.go @@ -1418,6 +1418,138 @@ func TestCommandOutputContainsAssertion(t *testing.T) { } } +func TestCommandShouldFailAcceptsNonZeroWithoutCaching(t *testing.T) { + sc, _ := newScenarioContext(t) + sc.LastCommand = "grpcurl rejected-call" + sc.LastResult = harness.Result{ExitCode: 1, Stderr: "certificate is not trusted"} + + if err := sc.commandShouldFail(); err != nil { + t.Fatalf("assert command failure: %v", err) + } + if sc.Suite.Cache.Has(sc.LastCommand) { + t.Fatal("failed command should not enter the successful-command cache") + } + + sc.LastResult = harness.Result{ExitCode: 0} + if err := sc.commandShouldFail(); err == nil { + t.Fatal("expected successful command to fail the negative assertion") + } +} + +func TestCommandOutputTableAssertionsInterpolateExpectedText(t *testing.T) { + sc, _ := newScenarioContext(t) + t.Setenv("BDD_EXPECTED_DIAGNOSTIC", "certificate is not trusted") + sc.LastResult = harness.Result{ + Stdout: "request rejected\n", + Stderr: "certificate is not trusted\ncontext deadline exceeded\n", + } + + all := docTable(t, [][]string{ + {"text"}, + {"${BDD_EXPECTED_DIAGNOSTIC}"}, + {"context deadline exceeded"}, + }) + if err := sc.commandOutputShouldContainAll(all); err != nil { + t.Fatalf("contain all: %v", err) + } + + oneOf := docTable(t, [][]string{ + {"text"}, + {"certificate signed by unknown authority"}, + {"${BDD_EXPECTED_DIAGNOSTIC}"}, + }) + if err := sc.commandOutputShouldContainOneOf(oneOf); err != nil { + t.Fatalf("contain one of: %v", err) + } +} + +func TestCommandOutputAssertionsRejectValuesThatInterpolateToEmpty(t *testing.T) { + sc, _ := newScenarioContext(t) + t.Setenv("BDD_EMPTY_EXPECTATION", "") + sc.LastResult = harness.Result{Stdout: "any output contains the empty string"} + + if err := sc.commandOutputShouldContain("${BDD_EMPTY_EXPECTATION}"); err == nil { + t.Fatal("expected empty single-value expectation to fail") + } + if err := sc.commandOutputShouldNotContain("${BDD_EMPTY_EXPECTATION}"); err == nil { + t.Fatal("expected empty negative expectation to fail validation") + } + + containAll := docTable(t, [][]string{ + {"text"}, + {"${BDD_EMPTY_EXPECTATION}"}, + }) + if err := sc.commandOutputShouldContainAll(containAll); err == nil { + t.Fatal("expected contain-all table with an empty resolved value to fail") + } + containOneOf := docTable(t, [][]string{ + {"text"}, + {"any output"}, + {"${BDD_EMPTY_EXPECTATION}"}, + }) + if err := sc.commandOutputShouldContainOneOf(containOneOf); err == nil { + t.Fatal("expected contain-one-of table with an empty resolved value to fail") + } +} + +func TestISuccessfullyObserveWatchStargatesRunsExplicitCommand(t *testing.T) { + sc, fake := newScenarioContext(t) + fake.result = harness.Result{ExitCode: 0, Stdout: "{\n \"stargates\": []\n}\n"} + + err := sc.iSuccessfullyObserveWatchStargates( + context.Background(), + "127.0.0.1:50071", + "llm-request-router.nvcf.svc.cluster.local", + "stargate-quic-tls", + "nvcf", + "k3d-ncp-local-cp", + "3", + ) + if err != nil { + t.Fatalf("observe WatchStargates: %v", err) + } + want := "bash tests/bdd/scripts/observe-watch-stargates.sh 127.0.0.1:50071 llm-request-router.nvcf.svc.cluster.local stargate-quic-tls nvcf k3d-ncp-local-cp 3" + if len(fake.runs) != 1 || fake.runs[0].command != want { + t.Fatalf("runs = %#v, want %q", fake.runs, want) + } + if !strings.Contains(sc.LastResult.Stdout, "stargates") { + t.Fatalf("last result = %#v, want preserved WatchStargates output", sc.LastResult) + } +} + +func TestEveryPylonForFunctionShouldReportMetricsRunsVisibleExpectations(t *testing.T) { + sc, fake := newScenarioContext(t) + fake.result = harness.Result{ExitCode: 0} + table := docTable(t, [][]string{ + {"metric", "comparison", "count"}, + {"pylon_registration_stream_connected", "exactly", "5"}, + {"pylon_reverse_tunnel_connected", "at least", "3"}, + }) + + if err := sc.everyPylonForFunctionShouldReportMetrics(context.Background(), "bdd-registration-tls", "llm-worker", "k3d-ncp-local-compute-1", "10m", table); err != nil { + t.Fatalf("observe Pylon metrics: %v", err) + } + want := "bash tests/bdd/scripts/wait-pylon-metrics.sh bdd-registration-tls llm-worker k3d-ncp-local-compute-1 10m pylon_registration_stream_connected exactly 5 pylon_reverse_tunnel_connected 'at least' 3" + if len(fake.runs) != 1 || fake.runs[0].command != want { + t.Fatalf("runs = %#v, want %q", fake.runs, want) + } +} + +func TestPylonMetricTableRejectsInvalidStructureBeforeRunning(t *testing.T) { + sc, fake := newScenarioContext(t) + table := docTable(t, [][]string{ + {"metric", "comparison", "count"}, + {"pylon_registration_stream_connected", "exactly", "not-a-count"}, + }) + + if err := sc.everyPylonForFunctionShouldReportMetrics(context.Background(), "function", "llm-worker", "context", "10m", table); err == nil { + t.Fatal("expected invalid count error") + } + if len(fake.runs) != 0 { + t.Fatalf("runs = %d, want 0 before table validation", len(fake.runs)) + } +} + func TestJSONOutputContainsRowsAssertion(t *testing.T) { sc, _ := newScenarioContext(t) sc.LastResult = harness.Result{Stdout: `[{"name":"api","namespace":"nvcf"}]`}