From 6a6fef01fe3f56baad43d0c15484e57ca03dd25b Mon Sep 17 00:00:00 2001 From: Mike Camp Date: Fri, 28 Aug 2026 14:58:48 -0400 Subject: [PATCH 1/3] test(bdd): assert plaintext TLS rejection Require the plaintext Watch probe to report the expected TLS transport failure, and assert that the wiring suite executes the exact probe. Relates-to: #1305 Signed-off-by: Mike Camp --- .../multi-cluster-helmfile-llm-registration-tls.feature | 1 + tests/bdd/godog_test.go | 3 +++ 2 files changed, 4 insertions(+) diff --git a/tests/bdd/features/multi-cluster-helmfile-llm-registration-tls.feature b/tests/bdd/features/multi-cluster-helmfile-llm-registration-tls.feature index 2262904fc..8aa2ef52a 100644 --- a/tests/bdd/features/multi-cluster-helmfile-llm-registration-tls.feature +++ b/tests/bdd/features/multi-cluster-helmfile-llm-registration-tls.feature @@ -88,6 +88,7 @@ Feature: Register an LLM worker securely with every router in a local split-clus grpcurl -plaintext -max-time 5 -import-path src/libraries/rust/stargate/crates/proto/proto -proto stargate.proto 127.0.0.1:50071 stargate.StargateControlPlane/WatchStargates """ Then the command exit code should be 1 + And the command output should contain "does not look like a TLS handshake" # WatchStargates is a long-lived stream. Normalize grpcurl's deadline # exit after it prints the initial snapshot. diff --git a/tests/bdd/godog_test.go b/tests/bdd/godog_test.go index 1e5c64ab1..118531833 100644 --- a/tests/bdd/godog_test.go +++ b/tests/bdd/godog_test.go @@ -1278,6 +1278,9 @@ func TestMultiClusterHelmfileLLMRegistrationTLSFeatureFileWiresToSteps(t *testin if !commandRanExactly(suite.Runner.(*fakeRunner).runs, tlsWatchCommand) { t.Fatal("WatchStargates was not observed over the trusted TLS listener") } + if !commandRanExactly(suite.Runner.(*fakeRunner).runs, plaintextWatchCommand) { + t.Fatal("plaintext WatchStargates rejection was not exercised") + } } // TestSingleClusterHelmfileUpstreamImagesFeatureFileWiresToSteps runs the From 760c73f5109b45fe818c250a29f4a2f042b8d1a1 Mon Sep 17 00:00:00 2001 From: Mike Camp Date: Fri, 28 Aug 2026 15:40:14 -0400 Subject: [PATCH 2/3] test(bdd): normalize plaintext TLS rejection Assert the client-observable grpcurl deadline from a verified TLS listener while rejecting unrelated command, proto, usage, and endpoint failures. Relates-to: #1305 Signed-off-by: Mike Camp --- ...ster-helmfile-llm-registration-tls.feature | 9 ++- tests/bdd/godog_test.go | 11 +++- .../plaintext_tls_rejection_script_test.go | 61 +++++++++++++++++++ .../assert-grpcurl-plaintext-tls-rejection.sh | 20 ++++++ 4 files changed, 95 insertions(+), 6 deletions(-) create mode 100644 tests/bdd/plaintext_tls_rejection_script_test.go create mode 100755 tests/bdd/scripts/assert-grpcurl-plaintext-tls-rejection.sh diff --git a/tests/bdd/features/multi-cluster-helmfile-llm-registration-tls.feature b/tests/bdd/features/multi-cluster-helmfile-llm-registration-tls.feature index 8aa2ef52a..1c890de8f 100644 --- a/tests/bdd/features/multi-cluster-helmfile-llm-registration-tls.feature +++ b/tests/bdd/features/multi-cluster-helmfile-llm-registration-tls.feature @@ -83,12 +83,15 @@ Feature: Register an LLM worker securely with every router in a local split-clus And the command output should contain "Verify return code: 0 (ok)" And the command output should contain "ALPN protocol: h2" + # grpcurl reports a client-side dial deadline when plaintext HTTP/2 is + # sent to this verified TLS listener. The trusted Watch below proves + # that the same endpoint remains healthy. When I run command: """ - grpcurl -plaintext -max-time 5 -import-path src/libraries/rust/stargate/crates/proto/proto -proto stargate.proto 127.0.0.1:50071 stargate.StargateControlPlane/WatchStargates + /bin/bash -c 'set -u; output=$(grpcurl -plaintext -max-time 5 -import-path src/libraries/rust/stargate/crates/proto/proto -proto stargate.proto 127.0.0.1:50071 stargate.StargateControlPlane/WatchStargates 2>&1); rc=$?; if [ "$rc" -eq 0 ]; then printf "%s\n" "plaintext Watch unexpectedly succeeded" >&2; exit 1; fi; printf "%s\n" "$output" | bash tests/bdd/scripts/assert-grpcurl-plaintext-tls-rejection.sh' """ - Then the command exit code should be 1 - And the command output should contain "does not look like a TLS handshake" + Then the command exit code should be 0 + And the command output should contain "plaintext-watch-rejected=tls-listener-timeout" # WatchStargates is a long-lived stream. Normalize grpcurl's deadline # exit after it prints the initial snapshot. diff --git a/tests/bdd/godog_test.go b/tests/bdd/godog_test.go index 118531833..fccfe9945 100644 --- a/tests/bdd/godog_test.go +++ b/tests/bdd/godog_test.go @@ -1174,8 +1174,13 @@ func TestMultiClusterHelmfileLLMRegistrationTLSFeatureFileWiresToSteps(t *testin `-servername llm-request-router.nvcf.svc.cluster.local -alpn h2 -verify_return_error ` + `-CAfile <(kubectl --context k3d-ncp-local-cp get secret stargate-quic-tls -n nvcf ` + `-o jsonpath="{.data.ca\.crt}" | base64 -d) &1'` - plaintextWatchCommand = "grpcurl -plaintext -max-time 5 -import-path src/libraries/rust/stargate/crates/proto/proto -proto stargate.proto 127.0.0.1:50071 stargate.StargateControlPlane/WatchStargates" - tlsWatchCommand = `/bin/bash -c 'grpcurl -max-time 3 ` + + plaintextWatchCommand = `/bin/bash -c 'set -u; output=$(grpcurl -plaintext -max-time 5 ` + + `-import-path src/libraries/rust/stargate/crates/proto/proto -proto stargate.proto ` + + `127.0.0.1:50071 stargate.StargateControlPlane/WatchStargates 2>&1); rc=$?; ` + + `if [ "$rc" -eq 0 ]; then printf "%s\n" "plaintext Watch unexpectedly succeeded" >&2; ` + + `exit 1; fi; printf "%s\n" "$output" | ` + + `bash tests/bdd/scripts/assert-grpcurl-plaintext-tls-rejection.sh'` + tlsWatchCommand = `/bin/bash -c 'grpcurl -max-time 3 ` + `-cacert <(kubectl --context k3d-ncp-local-cp get secret stargate-quic-tls -n nvcf ` + `-o jsonpath="{.data.ca\.crt}" | base64 -d) ` + `-authority llm-request-router.nvcf.svc.cluster.local ` + @@ -1212,7 +1217,7 @@ func TestMultiClusterHelmfileLLMRegistrationTLSFeatureFileWiresToSteps(t *testin ExitCode: 0, Stdout: "ALPN protocol: h2\nVerify return code: 0 (ok)\n", }, - plaintextWatchCommand: {ExitCode: 1, Stderr: "tls: first record does not look like a TLS handshake"}, + plaintextWatchCommand: {ExitCode: 0, Stdout: "plaintext-watch-rejected=tls-listener-timeout\n"}, tlsWatchCommand: { ExitCode: 0, Stdout: `{ diff --git a/tests/bdd/plaintext_tls_rejection_script_test.go b/tests/bdd/plaintext_tls_rejection_script_test.go new file mode 100644 index 000000000..af9155a66 --- /dev/null +++ b/tests/bdd/plaintext_tls_rejection_script_test.go @@ -0,0 +1,61 @@ +/* +SPDX-FileCopyrightText: Copyright (c) NVIDIA CORPORATION & AFFILIATES. All rights reserved. +SPDX-License-Identifier: Apache-2.0 + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package bdd_tmp + +import ( + "os/exec" + "strings" + "testing" +) + +func TestPlaintextTLSRejectionAcceptsObservedGrpcurlDeadline(t *testing.T) { + cmd := exec.Command("bash", "scripts/assert-grpcurl-plaintext-tls-rejection.sh") + cmd.Stdin = strings.NewReader( + `Failed to dial target host "127.0.0.1:50443": context deadline exceeded`, + ) + + output, err := cmd.CombinedOutput() + if err != nil { + t.Fatalf("classify grpcurl deadline: %v\n%s", err, output) + } + if got, want := strings.TrimSpace(string(output)), "plaintext-watch-rejected=tls-listener-timeout"; got != want { + t.Fatalf("normalized output = %q, want %q", got, want) + } +} + +func TestPlaintextTLSRejectionRejectsUnrelatedGrpcurlFailures(t *testing.T) { + for name, diagnostic := range map[string]string{ + "binary missing": "bash: grpcurl: command not found", + "connection refused": "Failed to dial target host 127.0.0.1:50071: connection refused", + "proto import": "Failed to process proto source files.: missing.proto does not reside in any import path", + "usage": "flag provided but not defined: -bad-flag", + } { + t.Run(name, func(t *testing.T) { + cmd := exec.Command("bash", "scripts/assert-grpcurl-plaintext-tls-rejection.sh") + cmd.Stdin = strings.NewReader(diagnostic) + + output, err := cmd.CombinedOutput() + if err == nil { + t.Fatalf("unrelated failure was accepted: %s", output) + } + if strings.Contains(string(output), "plaintext-watch-rejected=") { + t.Fatalf("unrelated failure emitted success marker: %s", output) + } + }) + } +} diff --git a/tests/bdd/scripts/assert-grpcurl-plaintext-tls-rejection.sh b/tests/bdd/scripts/assert-grpcurl-plaintext-tls-rejection.sh new file mode 100755 index 000000000..8282b14b0 --- /dev/null +++ b/tests/bdd/scripts/assert-grpcurl-plaintext-tls-rejection.sh @@ -0,0 +1,20 @@ +#!/usr/bin/env bash +# SPDX-FileCopyrightText: Copyright (c) NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Normalize the grpcurl 1.9.3 client-side diagnostic observed when plaintext +# HTTP/2 is sent to a healthy TLS listener. Unexpected command, proto, usage, +# binary, and endpoint failures remain errors instead of satisfying the test. + +set -euo pipefail + +diagnostic="$(cat)" +case "${diagnostic}" in + *"context deadline exceeded"*) + printf '%s\n' 'plaintext-watch-rejected=tls-listener-timeout' + ;; + *) + printf '%s\n' 'plaintext Watch failed without the expected TLS-listener transport timeout' >&2 + exit 1 + ;; +esac From 0a851365c5aa99b0092036cb5ceb0c6cc2540acd Mon Sep 17 00:00:00 2001 From: Mike Camp Date: Fri, 28 Aug 2026 15:51:14 -0400 Subject: [PATCH 3/3] test(bdd): require exact TLS dial rejection Match only the complete grpcurl dial-stage diagnostic for the expected listener so successful Watch snapshots followed by RPC deadlines cannot satisfy the plaintext rejection check. Relates-to: #1305 Signed-off-by: Mike Camp --- .../plaintext_tls_rejection_script_test.go | 30 +++++++++++++++---- .../assert-grpcurl-plaintext-tls-rejection.sh | 23 +++++++------- 2 files changed, 36 insertions(+), 17 deletions(-) diff --git a/tests/bdd/plaintext_tls_rejection_script_test.go b/tests/bdd/plaintext_tls_rejection_script_test.go index af9155a66..fbb75ea79 100644 --- a/tests/bdd/plaintext_tls_rejection_script_test.go +++ b/tests/bdd/plaintext_tls_rejection_script_test.go @@ -26,7 +26,7 @@ import ( func TestPlaintextTLSRejectionAcceptsObservedGrpcurlDeadline(t *testing.T) { cmd := exec.Command("bash", "scripts/assert-grpcurl-plaintext-tls-rejection.sh") cmd.Stdin = strings.NewReader( - `Failed to dial target host "127.0.0.1:50443": context deadline exceeded`, + `Failed to dial target host "127.0.0.1:50071": context deadline exceeded`, ) output, err := cmd.CombinedOutput() @@ -38,12 +38,32 @@ func TestPlaintextTLSRejectionAcceptsObservedGrpcurlDeadline(t *testing.T) { } } +func TestPlaintextTLSRejectionRejectsSnapshotThenRPCDeadline(t *testing.T) { + cmd := exec.Command("bash", "scripts/assert-grpcurl-plaintext-tls-rejection.sh") + cmd.Stdin = strings.NewReader(`{ + "stargates": [] +} +ERROR: + Code: DeadlineExceeded + Message: context deadline exceeded`) + + output, err := cmd.CombinedOutput() + if err == nil { + t.Fatalf("successful plaintext snapshot followed by RPC deadline was accepted: %s", output) + } + if strings.Contains(string(output), "plaintext-watch-rejected=") { + t.Fatalf("successful plaintext snapshot emitted success marker: %s", output) + } +} + func TestPlaintextTLSRejectionRejectsUnrelatedGrpcurlFailures(t *testing.T) { for name, diagnostic := range map[string]string{ - "binary missing": "bash: grpcurl: command not found", - "connection refused": "Failed to dial target host 127.0.0.1:50071: connection refused", - "proto import": "Failed to process proto source files.: missing.proto does not reside in any import path", - "usage": "flag provided but not defined: -bad-flag", + "binary missing": "bash: grpcurl: command not found", + "connection refused": "Failed to dial target host 127.0.0.1:50071: connection refused", + "dial timeout plus output": "Failed to dial target host \"127.0.0.1:50071\": context deadline exceeded\n{}", + "proto import": "Failed to process proto source files.: missing.proto does not reside in any import path", + "usage": "flag provided but not defined: -bad-flag", + "wrong target dial timeout": "Failed to dial target host \"127.0.0.1:50443\": context deadline exceeded", } { t.Run(name, func(t *testing.T) { cmd := exec.Command("bash", "scripts/assert-grpcurl-plaintext-tls-rejection.sh") diff --git a/tests/bdd/scripts/assert-grpcurl-plaintext-tls-rejection.sh b/tests/bdd/scripts/assert-grpcurl-plaintext-tls-rejection.sh index 8282b14b0..453cdbc12 100755 --- a/tests/bdd/scripts/assert-grpcurl-plaintext-tls-rejection.sh +++ b/tests/bdd/scripts/assert-grpcurl-plaintext-tls-rejection.sh @@ -2,19 +2,18 @@ # SPDX-FileCopyrightText: Copyright (c) NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 # -# Normalize the grpcurl 1.9.3 client-side diagnostic observed when plaintext -# HTTP/2 is sent to a healthy TLS listener. Unexpected command, proto, usage, -# binary, and endpoint failures remain errors instead of satisfying the test. +# Normalize only the complete grpcurl 1.9.3 dial-stage diagnostic observed when +# plaintext HTTP/2 is sent to the verified TLS listener. RPC stream deadlines, +# additional output, and unrelated command or endpoint failures remain errors. set -euo pipefail diagnostic="$(cat)" -case "${diagnostic}" in - *"context deadline exceeded"*) - printf '%s\n' 'plaintext-watch-rejected=tls-listener-timeout' - ;; - *) - printf '%s\n' 'plaintext Watch failed without the expected TLS-listener transport timeout' >&2 - exit 1 - ;; -esac +expected_diagnostic='Failed to dial target host "127.0.0.1:50071": context deadline exceeded' +if [[ "${diagnostic}" == "${expected_diagnostic}" ]]; then + printf '%s\n' 'plaintext-watch-rejected=tls-listener-timeout' + exit 0 +fi + +printf '%s\n' 'plaintext Watch failed without the expected TLS-listener dial timeout' >&2 +exit 1