diff --git a/tests/bdd/features/multi-cluster-helmfile-llm-registration-tls.feature b/tests/bdd/features/multi-cluster-helmfile-llm-registration-tls.feature index 2262904fc..1c890de8f 100644 --- a/tests/bdd/features/multi-cluster-helmfile-llm-registration-tls.feature +++ b/tests/bdd/features/multi-cluster-helmfile-llm-registration-tls.feature @@ -83,11 +83,15 @@ Feature: Register an LLM worker securely with every router in a local split-clus And the command output should contain "Verify return code: 0 (ok)" And the command output should contain "ALPN protocol: h2" + # grpcurl reports a client-side dial deadline when plaintext HTTP/2 is + # sent to this verified TLS listener. The trusted Watch below proves + # that the same endpoint remains healthy. When I run command: """ - grpcurl -plaintext -max-time 5 -import-path src/libraries/rust/stargate/crates/proto/proto -proto stargate.proto 127.0.0.1:50071 stargate.StargateControlPlane/WatchStargates + /bin/bash -c 'set -u; output=$(grpcurl -plaintext -max-time 5 -import-path src/libraries/rust/stargate/crates/proto/proto -proto stargate.proto 127.0.0.1:50071 stargate.StargateControlPlane/WatchStargates 2>&1); rc=$?; if [ "$rc" -eq 0 ]; then printf "%s\n" "plaintext Watch unexpectedly succeeded" >&2; exit 1; fi; printf "%s\n" "$output" | bash tests/bdd/scripts/assert-grpcurl-plaintext-tls-rejection.sh' """ - Then the command exit code should be 1 + Then the command exit code should be 0 + And the command output should contain "plaintext-watch-rejected=tls-listener-timeout" # WatchStargates is a long-lived stream. Normalize grpcurl's deadline # exit after it prints the initial snapshot. diff --git a/tests/bdd/godog_test.go b/tests/bdd/godog_test.go index 1e5c64ab1..fccfe9945 100644 --- a/tests/bdd/godog_test.go +++ b/tests/bdd/godog_test.go @@ -1174,8 +1174,13 @@ func TestMultiClusterHelmfileLLMRegistrationTLSFeatureFileWiresToSteps(t *testin `-servername llm-request-router.nvcf.svc.cluster.local -alpn h2 -verify_return_error ` + `-CAfile <(kubectl --context k3d-ncp-local-cp get secret stargate-quic-tls -n nvcf ` + `-o jsonpath="{.data.ca\.crt}" | base64 -d) &1'` - plaintextWatchCommand = "grpcurl -plaintext -max-time 5 -import-path src/libraries/rust/stargate/crates/proto/proto -proto stargate.proto 127.0.0.1:50071 stargate.StargateControlPlane/WatchStargates" - tlsWatchCommand = `/bin/bash -c 'grpcurl -max-time 3 ` + + plaintextWatchCommand = `/bin/bash -c 'set -u; output=$(grpcurl -plaintext -max-time 5 ` + + `-import-path src/libraries/rust/stargate/crates/proto/proto -proto stargate.proto ` + + `127.0.0.1:50071 stargate.StargateControlPlane/WatchStargates 2>&1); rc=$?; ` + + `if [ "$rc" -eq 0 ]; then printf "%s\n" "plaintext Watch unexpectedly succeeded" >&2; ` + + `exit 1; fi; printf "%s\n" "$output" | ` + + `bash tests/bdd/scripts/assert-grpcurl-plaintext-tls-rejection.sh'` + tlsWatchCommand = `/bin/bash -c 'grpcurl -max-time 3 ` + `-cacert <(kubectl --context k3d-ncp-local-cp get secret stargate-quic-tls -n nvcf ` + `-o jsonpath="{.data.ca\.crt}" | base64 -d) ` + `-authority llm-request-router.nvcf.svc.cluster.local ` + @@ -1212,7 +1217,7 @@ func TestMultiClusterHelmfileLLMRegistrationTLSFeatureFileWiresToSteps(t *testin ExitCode: 0, Stdout: "ALPN protocol: h2\nVerify return code: 0 (ok)\n", }, - plaintextWatchCommand: {ExitCode: 1, Stderr: "tls: first record does not look like a TLS handshake"}, + plaintextWatchCommand: {ExitCode: 0, Stdout: "plaintext-watch-rejected=tls-listener-timeout\n"}, tlsWatchCommand: { ExitCode: 0, Stdout: `{ @@ -1278,6 +1283,9 @@ func TestMultiClusterHelmfileLLMRegistrationTLSFeatureFileWiresToSteps(t *testin if !commandRanExactly(suite.Runner.(*fakeRunner).runs, tlsWatchCommand) { t.Fatal("WatchStargates was not observed over the trusted TLS listener") } + if !commandRanExactly(suite.Runner.(*fakeRunner).runs, plaintextWatchCommand) { + t.Fatal("plaintext WatchStargates rejection was not exercised") + } } // TestSingleClusterHelmfileUpstreamImagesFeatureFileWiresToSteps runs the diff --git a/tests/bdd/plaintext_tls_rejection_script_test.go b/tests/bdd/plaintext_tls_rejection_script_test.go new file mode 100644 index 000000000..fbb75ea79 --- /dev/null +++ b/tests/bdd/plaintext_tls_rejection_script_test.go @@ -0,0 +1,81 @@ +/* +SPDX-FileCopyrightText: Copyright (c) NVIDIA CORPORATION & AFFILIATES. All rights reserved. +SPDX-License-Identifier: Apache-2.0 + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package bdd_tmp + +import ( + "os/exec" + "strings" + "testing" +) + +func TestPlaintextTLSRejectionAcceptsObservedGrpcurlDeadline(t *testing.T) { + cmd := exec.Command("bash", "scripts/assert-grpcurl-plaintext-tls-rejection.sh") + cmd.Stdin = strings.NewReader( + `Failed to dial target host "127.0.0.1:50071": context deadline exceeded`, + ) + + output, err := cmd.CombinedOutput() + if err != nil { + t.Fatalf("classify grpcurl deadline: %v\n%s", err, output) + } + if got, want := strings.TrimSpace(string(output)), "plaintext-watch-rejected=tls-listener-timeout"; got != want { + t.Fatalf("normalized output = %q, want %q", got, want) + } +} + +func TestPlaintextTLSRejectionRejectsSnapshotThenRPCDeadline(t *testing.T) { + cmd := exec.Command("bash", "scripts/assert-grpcurl-plaintext-tls-rejection.sh") + cmd.Stdin = strings.NewReader(`{ + "stargates": [] +} +ERROR: + Code: DeadlineExceeded + Message: context deadline exceeded`) + + output, err := cmd.CombinedOutput() + if err == nil { + t.Fatalf("successful plaintext snapshot followed by RPC deadline was accepted: %s", output) + } + if strings.Contains(string(output), "plaintext-watch-rejected=") { + t.Fatalf("successful plaintext snapshot emitted success marker: %s", output) + } +} + +func TestPlaintextTLSRejectionRejectsUnrelatedGrpcurlFailures(t *testing.T) { + for name, diagnostic := range map[string]string{ + "binary missing": "bash: grpcurl: command not found", + "connection refused": "Failed to dial target host 127.0.0.1:50071: connection refused", + "dial timeout plus output": "Failed to dial target host \"127.0.0.1:50071\": context deadline exceeded\n{}", + "proto import": "Failed to process proto source files.: missing.proto does not reside in any import path", + "usage": "flag provided but not defined: -bad-flag", + "wrong target dial timeout": "Failed to dial target host \"127.0.0.1:50443\": context deadline exceeded", + } { + t.Run(name, func(t *testing.T) { + cmd := exec.Command("bash", "scripts/assert-grpcurl-plaintext-tls-rejection.sh") + cmd.Stdin = strings.NewReader(diagnostic) + + output, err := cmd.CombinedOutput() + if err == nil { + t.Fatalf("unrelated failure was accepted: %s", output) + } + if strings.Contains(string(output), "plaintext-watch-rejected=") { + t.Fatalf("unrelated failure emitted success marker: %s", output) + } + }) + } +} diff --git a/tests/bdd/scripts/assert-grpcurl-plaintext-tls-rejection.sh b/tests/bdd/scripts/assert-grpcurl-plaintext-tls-rejection.sh new file mode 100755 index 000000000..453cdbc12 --- /dev/null +++ b/tests/bdd/scripts/assert-grpcurl-plaintext-tls-rejection.sh @@ -0,0 +1,19 @@ +#!/usr/bin/env bash +# SPDX-FileCopyrightText: Copyright (c) NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Normalize only the complete grpcurl 1.9.3 dial-stage diagnostic observed when +# plaintext HTTP/2 is sent to the verified TLS listener. RPC stream deadlines, +# additional output, and unrelated command or endpoint failures remain errors. + +set -euo pipefail + +diagnostic="$(cat)" +expected_diagnostic='Failed to dial target host "127.0.0.1:50071": context deadline exceeded' +if [[ "${diagnostic}" == "${expected_diagnostic}" ]]; then + printf '%s\n' 'plaintext-watch-rejected=tls-listener-timeout' + exit 0 +fi + +printf '%s\n' 'plaintext Watch failed without the expected TLS-listener dial timeout' >&2 +exit 1