diff --git a/tests/bdd/features/multi-cluster-helmfile-llm-registration-tls.feature b/tests/bdd/features/multi-cluster-helmfile-llm-registration-tls.feature new file mode 100644 index 000000000..789fd9c63 --- /dev/null +++ b/tests/bdd/features/multi-cluster-helmfile-llm-registration-tls.feature @@ -0,0 +1,258 @@ +@ncp-local @multi-cluster @helmfile @pki @llm-registration +Feature: Register an LLM worker securely with a local split-cluster routing plane + As a self-managed NVCF operator, + I want Pylon registration to use the stack-issued TLS identity across clusters, + so that plaintext or untrusted registration cannot silently enter the routing plane. + + Background: + Given these environment variables are set: + | name | + | NGC_API_KEY | + | NVCF_CLI | + | REPO_ROOT | + | SAMPLE_NGC_ORG | + | SAMPLE_NGC_TEAM | + And I prepare Helmfile environment "local-bdd-registration-tls" for stack "self-managed" from fixture "tests/bdd/fixtures/self-managed-local-bdd-multi.yaml" with values: + | global.imagePullSecrets[0].name | nvcr-pull-secret | + | global.helm.sources.repository | ${SAMPLE_NGC_ORG}/${SAMPLE_NGC_TEAM} | + | global.image.repository | ${SAMPLE_NGC_ORG}/${SAMPLE_NGC_TEAM} | + | global.workerEndpoints.llmRequestRouterAddress | https://llm-request-router.nvcf.svc.cluster.local:50071 | + | addons.llm.requestRouter.workload.kind | StatefulSet | + | addons.llm.requestRouter.backendRouter.pylonGrpcDialAddress | https://llm-request-router.nvcf.svc.cluster.local:50071 | + | observability.profile | disabled | + And I prepare self-managed secrets file "deploy/stacks/self-managed/secrets/local-bdd-registration-tls-secrets.yaml" from template "deploy/stacks/self-managed/secrets/secrets.yaml.template" using the current NGC registry credential + When I run command "/bin/sh -c 'command -v grpcurl >/dev/null'" + Then the command exit code should be 0 + # Conflict precheck: the single-cluster topology owns the same host + # ports. Run make -C tools/ncp-local-cluster destroy CLUSTER_NAME=ncp-local + # before retrying. k3d v5 exits 1 when the cluster is absent. + When I run command "k3d cluster get ncp-local" + Then the command exit code should be 1 + And multi-cluster ncp-local compute clusters are running: + | ncp-local-compute-1 | + And command has succeeded: + """ + kubectl config use-context k3d-ncp-local-cp + """ + And the "nvcr-pull-secret" image pull secret exists in namespaces: + | cassandra-system | + | nats-system | + | nvcf | + | api-keys | + | ess | + | sis | + | vault-system | + | nvca-operator | + | cert-manager | + + Rule: Trusted registration is observable from the operator boundary + + @llm-registration-tls-install @llm-registration-tls-runtime + Scenario: A trusted Pylon registers with every router and serves an authenticated request + Given I prepare Helmfile environment "local-bdd-registration-tls" for stack "nvcf-compute-plane" from fixture "tests/bdd/fixtures/nvcf-compute-plane-local-bdd-multi.yaml" with values: + | global.imagePullSecrets[0].name | nvcr-pull-secret | + | global.helm.sources.repository | ${SAMPLE_NGC_ORG}/${SAMPLE_NGC_TEAM} | + | global.image.repository | ${SAMPLE_NGC_ORG}/${SAMPLE_NGC_TEAM} | + | observability.profile | disabled | + When I successfully run command "make -C deploy/stacks/self-managed template HELMFILE_ENV=local-bdd-registration-tls" + And the rendered manifests in "deploy/stacks/self-managed/out" should contain: + | text | + | https://llm-request-router.nvcf.svc.cluster.local:50071 | + | --grpc-pylon-dial-addr=https://llm-request-router.nvcf.svc.cluster.local:50071 | + + When I successfully run command "make -C deploy/stacks/self-managed install HELMFILE_ENV=local-bdd-registration-tls" + + When I successfully run command "kubectl --context k3d-ncp-local-cp wait clusterissuer nvcf-openbao-pki --for=condition=Ready --timeout=5m" + When I successfully run command "kubectl --context k3d-ncp-local-cp wait certificate stargate-quic-tls -n nvcf --for=condition=Ready --timeout=5m" + When I successfully run command "kubectl --context k3d-ncp-local-cp rollout status statefulset/llm-request-router -n nvcf --timeout=10m" + + When I run command "kubectl --context k3d-ncp-local-cp get configmap/nvcf-api-remote-config -n nvcf -o yaml" + Then the command exit code should be 0 + And the command output should contain "worker-address: https://llm-request-router.nvcf.svc.cluster.local:50071" + + # openssl verifies the externally reachable listener against the same + # stack-issued CA and DNS identity that a compute-plane Pylon uses. + When I run command: + """ + /bin/bash -c 'openssl s_client -connect 127.0.0.1:50071 -servername llm-request-router.nvcf.svc.cluster.local -alpn h2 -verify_return_error -CAfile <(kubectl --context k3d-ncp-local-cp get secret stargate-quic-tls -n nvcf -o jsonpath="{.data.ca\.crt}" | base64 -d) &1' + """ + Then the command exit code should be 0 + And the command output should contain "Verify return code: 0 (ok)" + And the command output should contain "ALPN protocol: h2" + + # grpcurl reports a client-side dial deadline when plaintext HTTP/2 is + # sent to this verified TLS listener. The trusted Watch below proves + # that the same endpoint remains healthy. + When I run command: + """ + /bin/bash -c 'set -u; output=$(grpcurl -plaintext -max-time 5 -import-path src/libraries/rust/stargate/crates/proto/proto -proto stargate.proto 127.0.0.1:50071 stargate.StargateControlPlane/WatchStargates 2>&1); rc=$?; if [ "$rc" -eq 0 ]; then printf "%s\n" "plaintext Watch unexpectedly succeeded" >&2; exit 1; fi; printf "%s\n" "$output" | bash tests/bdd/scripts/assert-grpcurl-plaintext-tls-rejection.sh' + """ + Then the command exit code should be 0 + And the command output should contain "plaintext-watch-rejected=tls-listener-timeout" + + When I successfully observe WatchStargates at "127.0.0.1:50071" with TLS authority "llm-request-router.nvcf.svc.cluster.local" using CA secret "stargate-quic-tls" in namespace "nvcf" and context "k3d-ncp-local-cp" for "3" seconds + Then the command output should contain all: + | text | + | llm-request-router-0 | + | llm-request-router-1 | + | llm-request-router-2 | + | https://llm-request-router.nvcf.svc.cluster.local:50071 | + + When I run command: + """ + ${NVCF_CLI} --config ${REPO_ROOT}/tests/bdd/fixtures/nvcf-cli-local.yaml self-hosted --control-plane-stack deploy/stacks/self-managed --env local-bdd-registration-tls --control-plane-context k3d-ncp-local-cp --compute-plane-context k3d-ncp-local-compute-1 control-plane profile export --cluster-name ncp-local-cp + """ + Then the command exit code should be 0 + And file "deploy/stacks/self-managed/out/control-plane-profile.yaml" should exist + And yaml file "deploy/stacks/self-managed/out/control-plane-profile.yaml" should have non-empty keys: + | key | + | managementTls.caBundlePem | + | transportTls.trustBundleFingerprint | + | transportTls.trustBundlePem | + + And command has succeeded: + """ + /bin/sh -c '${NVCF_CLI} --config ${REPO_ROOT}/tests/bdd/fixtures/nvcf-cli-local.yaml init >/dev/null' + """ + When I run command "kubectl config use-context k3d-ncp-local-compute-1" + Then the command exit code should be 0 + When I run command: + """ + make -C deploy/stacks/nvcf-compute-plane register-cluster CLUSTER_NAME=ncp-local-compute-1 CONTROL_PLANE_PROFILE=${REPO_ROOT}/deploy/stacks/self-managed/out/control-plane-profile.yaml COMPUTE_KUBE_CONTEXT=k3d-ncp-local-compute-1 NVCF_CLI=${NVCF_CLI} NVCF_CLI_CONFIG=${REPO_ROOT}/tests/bdd/fixtures/nvcf-cli-local.yaml + """ + Then the command exit code should be 0 + And file "deploy/stacks/nvcf-compute-plane/registration/ncp-local-compute-1-register-values.yaml" should exist + And the "nvcr-pull-secret" image pull secret exists in namespaces: + | nvca-operator | + When I run command: + """ + make -C deploy/stacks/nvcf-compute-plane install CLUSTER_NAME=ncp-local-compute-1 HELMFILE_ENV=local-bdd-registration-tls COMPUTE_KUBE_CONTEXT=k3d-ncp-local-compute-1 NVCF_CLI=${NVCF_CLI} + """ + Then the command exit code should be 0 + Then NVCFBackend "ncp-local-compute-1" in namespace "nvca-operator" using context "k3d-ncp-local-compute-1" should report agent status "healthy" within "10m" + + Given I use NVCF CLI config "${REPO_ROOT}/tests/bdd/fixtures/nvcf-cli-local.yaml" + When I successfully create function "bdd-registration-tls" from image "nvcr.io/${SAMPLE_NGC_ORG}/${SAMPLE_NGC_TEAM}/nvcf-openai-compatible-sample:local" with CLI options: + | option | value | + | --function-type | LLM | + | --inference-url | /v1/chat/completions | + | --inference-port | 8000 | + | --health-uri | /health | + | --health-port | 8000 | + | --health-timeout | PT30S | + | --llm-model | name=openai-compatible-sample,uris=/v1/chat/completions\|/v1/embeddings,routingMethod=round_robin | + And I successfully deploy the function selected by NVCF CLI with options: + | option | value | + | --gpu | H100 | + | --instance-type | NCP.GPU.H100_1x | + | --backend | ncp-local-compute-1 | + | --regions | us-west-1 | + | --min-instances | 1 | + | --max-instances | 1 | + | --timeout | 900 | + And I successfully generate a function API key with CLI options: + | option | value | + | --description | bdd-registration-tls | + | --scopes | invoke_function,list_functions,queue_details,list_functions_details | + + Then every Pylon for function "bdd-registration-tls" using container "llm-worker" and context "k3d-ncp-local-compute-1" should report metrics within "10m": + | metric | comparison | count | + | pylon_registration_stream_connected | exactly | 3 | + | pylon_reverse_tunnel_connected | exactly | 3 | + + When I successfully invoke model "openai-compatible-sample" at "/v1/chat/completions" with timeout "120" seconds: + """ + {"messages":[{"role":"user","content":"bdd-registration-tls"}]} + """ + Then the command output should contain all: + | text | + | chat.completion | + | fixed 128-byte response | + And I successfully undeploy the function selected by NVCF CLI + + @negative + Rule: Registration rejects clients that do not satisfy the TLS contract + + Background: + Given command has succeeded: + """ + make -C deploy/stacks/self-managed template HELMFILE_ENV=local-bdd-registration-tls + """ + And command has succeeded: + """ + make -C deploy/stacks/self-managed install HELMFILE_ENV=local-bdd-registration-tls + """ + And command has succeeded: + """ + kubectl --context k3d-ncp-local-cp wait clusterissuer nvcf-openbao-pki --for=condition=Ready --timeout=5m + """ + And command has succeeded: + """ + kubectl --context k3d-ncp-local-cp wait certificate stargate-quic-tls -n nvcf --for=condition=Ready --timeout=5m + """ + And command has succeeded: + """ + kubectl --context k3d-ncp-local-cp rollout status statefulset/llm-request-router -n nvcf --timeout=10m + """ + + Scenario: Registration rejects a client with an untrusted root + When I run command: + """ + /bin/bash -c 'set -u; cert_dir=$(mktemp -d); trap '\''rm -rf "$cert_dir"'\'' EXIT; openssl req -x509 -newkey rsa:2048 -nodes -subj /CN=wrong-root -keyout "$cert_dir/key.pem" -out "$cert_dir/ca.pem" -days 1 >/dev/null 2>&1 || exit; grpcurl -max-time 5 -cacert "$cert_dir/ca.pem" -authority llm-request-router.nvcf.svc.cluster.local -import-path src/libraries/rust/stargate/crates/proto/proto -proto stargate.proto 127.0.0.1:50071 stargate.StargateControlPlane/WatchStargates' + """ + Then the command should fail + And the command output should contain one of: + | text | + | certificate signed by unknown authority | + | certificate is not trusted | + + Scenario: Registration rejects a client using the wrong authority + When I run command: + """ + /bin/bash -c 'grpcurl -max-time 5 -cacert <(kubectl --context k3d-ncp-local-cp get secret stargate-quic-tls -n nvcf -o jsonpath="{.data.ca\.crt}" | base64 -d) -authority wrong-host.nvcf.svc.cluster.local -import-path src/libraries/rust/stargate/crates/proto/proto -proto stargate.proto 127.0.0.1:50071 stargate.StargateControlPlane/WatchStargates' + """ + Then the command should fail + And the command output should contain one of: + | text | + | not wrong-host.nvcf.svc.cluster.local | + + Scenario: Registration rejects a client without the issued root + When I run command: + """ + /bin/bash -c 'grpcurl -max-time 5 -authority llm-request-router.nvcf.svc.cluster.local -import-path src/libraries/rust/stargate/crates/proto/proto -proto stargate.proto 127.0.0.1:50071 stargate.StargateControlPlane/WatchStargates' + """ + Then the command should fail + And the command output should contain one of: + | text | + | certificate signed by unknown authority | + | certificate is not trusted | + + Scenario: Registration rejects a plaintext client + When I run command: + """ + /bin/bash -c 'grpcurl -plaintext -max-time 5 -import-path src/libraries/rust/stargate/crates/proto/proto -proto stargate.proto 127.0.0.1:50071 stargate.StargateControlPlane/WatchStargates' + """ + Then the command should fail + And the command output should contain one of: + | text | + | context deadline exceeded | + | error reading server preface | + + @negative + Rule: Invalid registration authorities fail before installation + + Scenario: Operator cannot render an invalid worker authority + Given I prepare Helmfile environment "local-bdd-registration-tls-invalid-authority" for stack "self-managed" from fixture "tests/bdd/fixtures/self-managed-local-bdd-multi.yaml" with values: + | global.imagePullSecrets[0].name | nvcr-pull-secret | + | global.helm.sources.repository | ${SAMPLE_NGC_ORG}/${SAMPLE_NGC_TEAM} | + | global.image.repository | ${SAMPLE_NGC_ORG}/${SAMPLE_NGC_TEAM} | + | global.workerEndpoints.llmRequestRouterAddress | https://llm_request_router.nvcf.svc.cluster.local:50071 | + | addons.llm.requestRouter.workload.kind | StatefulSet | + | addons.llm.requestRouter.backendRouter.pylonGrpcDialAddress | https://llm_request_router.nvcf.svc.cluster.local:50071 | + | observability.profile | disabled | + And I prepare self-managed secrets file "deploy/stacks/self-managed/secrets/local-bdd-registration-tls-invalid-authority-secrets.yaml" from template "deploy/stacks/self-managed/secrets/secrets.yaml.template" using the current NGC registry credential + When I run command "make -C deploy/stacks/self-managed template HELMFILE_ENV=local-bdd-registration-tls-invalid-authority" + Then the command should fail + And the command output should contain one of: + | text | + | global.workerEndpoints.llmRequestRouterAddress must use optional http:// or https:// followed by DNS-or-IPv4:port or [IPv6]:port with port 1-65535 | diff --git a/tests/bdd/godog_test.go b/tests/bdd/godog_test.go index 970c26657..082c19f75 100644 --- a/tests/bdd/godog_test.go +++ b/tests/bdd/godog_test.go @@ -1206,6 +1206,207 @@ func TestMultiClusterHelmfileFeatureFileWiresToSteps(t *testing.T) { assertFunctionDeploymentsUseInstanceType(t, suite.Runner.(*fakeRunner).runs, "NCP.GPU.H100_1x", 3) } +// TestMultiClusterHelmfileLLMRegistrationTLSFeatureFileWiresToSteps runs the +// focused secure registration feature against a fake runner. The canned +// external observations cover the TLS listener, WatchStargates snapshot, +// Pylon metrics, and authenticated invocation. +func TestMultiClusterHelmfileLLMRegistrationTLSFeatureFileWiresToSteps(t *testing.T) { + t.Setenv("NGC_API_KEY", "test-key") + t.Setenv("SAMPLE_NGC_ORG", "test-org") + t.Setenv("SAMPLE_NGC_TEAM", "test-team") + t.Setenv("NVCF_CLI", "/usr/bin/nvcf-cli") + t.Setenv("REPO_ROOT", "/repo-root-placeholder") + + const ( + grpcurlPreflightCommand = `/bin/sh -c 'command -v grpcurl >/dev/null'` + tlsHandshakeCommand = `/bin/bash -c 'openssl s_client -connect 127.0.0.1:50071 ` + + `-servername llm-request-router.nvcf.svc.cluster.local -alpn h2 -verify_return_error ` + + `-CAfile <(kubectl --context k3d-ncp-local-cp get secret stargate-quic-tls -n nvcf ` + + `-o jsonpath="{.data.ca\.crt}" | base64 -d) &1'` + plaintextWatchCommand = `/bin/bash -c 'set -u; output=$(grpcurl -plaintext -max-time 5 ` + + `-import-path src/libraries/rust/stargate/crates/proto/proto -proto stargate.proto ` + + `127.0.0.1:50071 stargate.StargateControlPlane/WatchStargates 2>&1); rc=$?; ` + + `if [ "$rc" -eq 0 ]; then printf "%s\n" "plaintext Watch unexpectedly succeeded" >&2; ` + + `exit 1; fi; printf "%s\n" "$output" | ` + + `bash tests/bdd/scripts/assert-grpcurl-plaintext-tls-rejection.sh'` + tlsWatchCommand = "bash tests/bdd/scripts/observe-watch-stargates.sh" + + " 127.0.0.1:50071 llm-request-router.nvcf.svc.cluster.local" + + " stargate-quic-tls nvcf k3d-ncp-local-cp 3" + pylonMetricsCommand = "bash tests/bdd/scripts/wait-pylon-metrics.sh" + + " bdd-registration-tls llm-worker k3d-ncp-local-compute-1 10m" + + " pylon_registration_stream_connected exactly 3" + + " pylon_reverse_tunnel_connected exactly 3" + invokeCommand = "/usr/bin/nvcf-cli --config /repo-root-placeholder/tests/bdd/fixtures/nvcf-cli-local.yaml function invoke" + + " --inference-url /v1/chat/completions --model-name openai-compatible-sample" + + " --request-body '{\"messages\":[{\"role\":\"user\",\"content\":\"bdd-registration-tls\"}]}' --timeout 120" + wrongRootCommand = `/bin/bash -c 'set -u; cert_dir=$(mktemp -d); ` + + `trap '\''rm -rf "$cert_dir"'\'' EXIT; openssl req -x509 -newkey rsa:2048 -nodes ` + + `-subj /CN=wrong-root -keyout "$cert_dir/key.pem" -out "$cert_dir/ca.pem" -days 1 ` + + `>/dev/null 2>&1 || exit; grpcurl -max-time 5 -cacert "$cert_dir/ca.pem" ` + + `-authority llm-request-router.nvcf.svc.cluster.local ` + + `-import-path src/libraries/rust/stargate/crates/proto/proto -proto stargate.proto ` + + `127.0.0.1:50071 stargate.StargateControlPlane/WatchStargates'` + wrongHostCommand = `/bin/bash -c 'grpcurl -max-time 5 ` + + `-cacert <(kubectl --context k3d-ncp-local-cp get secret stargate-quic-tls -n nvcf ` + + `-o jsonpath="{.data.ca\.crt}" | base64 -d) ` + + `-authority wrong-host.nvcf.svc.cluster.local ` + + `-import-path src/libraries/rust/stargate/crates/proto/proto -proto stargate.proto ` + + `127.0.0.1:50071 stargate.StargateControlPlane/WatchStargates'` + missingTrustCommand = `/bin/bash -c 'grpcurl -max-time 5 ` + + `-authority llm-request-router.nvcf.svc.cluster.local ` + + `-import-path src/libraries/rust/stargate/crates/proto/proto -proto stargate.proto ` + + `127.0.0.1:50071 stargate.StargateControlPlane/WatchStargates'` + plaintextCommand = `/bin/bash -c 'grpcurl -plaintext -max-time 5 ` + + `-import-path src/libraries/rust/stargate/crates/proto/proto -proto stargate.proto ` + + `127.0.0.1:50071 stargate.StargateControlPlane/WatchStargates'` + invalidAuthorityCommand = "make -C deploy/stacks/self-managed template " + + "HELMFILE_ENV=local-bdd-registration-tls-invalid-authority" + ) + + suite := newWiringSuite(t, newFakeRunner(map[string]harness.Result{ + "k3d cluster get ncp-local": {ExitCode: 1}, + grpcurlPreflightCommand: {ExitCode: 0}, + "kubectl --context k3d-ncp-local-cp get configmap/nvcf-api-remote-config -n nvcf -o yaml": { + ExitCode: 0, + Stdout: "worker-address: https://llm-request-router.nvcf.svc.cluster.local:50071\n", + }, + tlsHandshakeCommand: { + ExitCode: 0, + Stdout: "ALPN protocol: h2\nVerify return code: 0 (ok)\n", + }, + plaintextWatchCommand: {ExitCode: 0, Stdout: "plaintext-watch-rejected=tls-listener-timeout\n"}, + tlsWatchCommand: { + ExitCode: 0, + Stdout: `{ + "stargates": [ + {"stargateId": "llm-request-router-0", "grpcPylonDialAddr": "https://llm-request-router.nvcf.svc.cluster.local:50071"}, + {"stargateId": "llm-request-router-1", "grpcPylonDialAddr": "https://llm-request-router.nvcf.svc.cluster.local:50071"}, + {"stargateId": "llm-request-router-2", "grpcPylonDialAddr": "https://llm-request-router.nvcf.svc.cluster.local:50071"} + ] +}`, + Stderr: "ERROR: DeadlineExceeded", + }, + pylonMetricsCommand: { + ExitCode: 0, + Stdout: "pylon_registration_stream_connected=3\n" + + "pylon_reverse_tunnel_connected=3\n", + }, + invokeCommand: { + ExitCode: 0, + Stdout: "Function invocation completed!\n\nResponse:\n" + + `{"object":"chat.completion","choices":[{"message":{"content":"This is a fixed 128-byte response for routing and contract validation."}}]}` + + "\n", + }, + wrongRootCommand: { + ExitCode: 1, + Stderr: "certificate signed by unknown authority\n", + }, + wrongHostCommand: { + ExitCode: 1, + Stderr: "certificate is valid for another name, not wrong-host.nvcf.svc.cluster.local\n", + }, + missingTrustCommand: { + ExitCode: 1, + Stderr: "certificate is not trusted\n", + }, + plaintextCommand: { + ExitCode: 1, + Stderr: "context deadline exceeded\n", + }, + invalidAuthorityCommand: { + ExitCode: 1, + Stderr: "global.workerEndpoints.llmRequestRouterAddress must use " + + "optional http:// or https:// followed by DNS-or-IPv4:port or [IPv6]:port " + + "with port 1-65535\n", + }, + })) + seedHelmfileLocalBDDMultiFixture(t, suite.Config.RepoRoot) + seedComputePlaneLocalBDDMultiFixture(t, suite.Config.RepoRoot) + seedStackSecretsTemplate(t, suite.Config.RepoRoot) + writeProfileHandoffArtifact(t, suite.Config.RepoRoot) + writeMulticlusterComputeRegisterValues(t, suite.Config.RepoRoot, "nvcf-compute-plane", "ncp-local-compute-1") + writeArtifact( + t, + suite.Config.RepoRoot, + "self-managed", + "registration-tls-rendered.yaml", + "https://llm-request-router.nvcf.svc.cluster.local:50071\n"+ + "--grpc-pylon-dial-addr=https://llm-request-router.nvcf.svc.cluster.local:50071\n", + ) + + sc := steps.NewScenarioContext(suite) + featurePath := mustResolveFeaturePath(t, "multi-cluster-helmfile-llm-registration-tls.feature") + var out strings.Builder + status := godog.TestSuite{ + Name: "multi-cluster-helmfile-llm-registration-tls-wiring", + ScenarioInitializer: func(ctx *godog.ScenarioContext) { + steps.RegisterAll(ctx, sc) + }, + Options: &godog.Options{ + Format: "pretty", + Paths: []string{featurePath}, + Strict: true, + Output: &out, + }, + }.Run() + if status != 0 { + t.Fatalf("godog suite status = %d\n%s", status, out.String()) + } + if !commandRanThatContainsAll( + suite.Runner.(*fakeRunner).runs, + "function create --name bdd-registration-tls", + "--function-type LLM", + "--llm-model", + ) { + t.Fatal("secure registration sample was not created as an LLM function") + } + if !commandRanExactly(suite.Runner.(*fakeRunner).runs, pylonMetricsCommand) { + t.Fatal("Pylon registration and reverse-tunnel metrics were not observed") + } + if !commandRanExactly(suite.Runner.(*fakeRunner).runs, tlsWatchCommand) { + t.Fatal("WatchStargates was not observed over the trusted TLS listener") + } + if !commandRanExactly(suite.Runner.(*fakeRunner).runs, plaintextWatchCommand) { + t.Fatal("plaintext WatchStargates rejection was not exercised") + } + runs := suite.Runner.(*fakeRunner).runs + if !commandRanExactly(runs, grpcurlPreflightCommand) { + t.Fatal("grpcurl availability was not checked before the live probes") + } + for name, command := range map[string]string{ + "wrong root": wrongRootCommand, + "wrong host": wrongHostCommand, + "missing trust": missingTrustCommand, + "plaintext": plaintextCommand, + "invalid authority": invalidAuthorityCommand, + } { + if !commandRanExactly(runs, command) { + t.Fatalf("%s negative registration command was not invoked", name) + } + } + for _, assertion := range []struct { + path string + want string + }{ + { + path: filepath.Join(suite.Config.RepoRoot, "deploy", "stacks", "self-managed", "environments", "local-bdd-registration-tls.yaml"), + want: "https://llm-request-router.nvcf.svc.cluster.local:50071", + }, + { + path: filepath.Join(suite.Config.RepoRoot, "deploy", "stacks", "self-managed", "environments", "local-bdd-registration-tls-invalid-authority.yaml"), + want: "https://llm_request_router.nvcf.svc.cluster.local:50071", + }, + } { + got, found, err := dsl.ReadYAMLKey(assertion.path, "global.workerEndpoints.llmRequestRouterAddress") + if err != nil { + t.Fatalf("read worker endpoint override: %v", err) + } + if !found || got != assertion.want { + t.Fatalf("worker endpoint = %q, found = %t; want %q", got, found, assertion.want) + } + } +} + // TestSingleClusterHelmfileUpstreamImagesFeatureFileWiresToSteps runs the // focused upstream-image feature against a fake runner. The seeded global // template contains the exact documentation blocks so the ledger-backed @@ -1971,6 +2172,16 @@ func TestMultiClusterHelmfile(t *testing.T) { runLiveFeature(t, "multi-cluster-helmfile.feature") } +// TestMultiClusterHelmfileLLMRegistrationTLS is the live entry point for the +// focused secure Pylon registration feature on local split-cluster k3d. +// Skipped under -short. +func TestMultiClusterHelmfileLLMRegistrationTLS(t *testing.T) { + if testing.Short() { + t.Skip("live run skipped under -short") + } + runLiveFeature(t, "multi-cluster-helmfile-llm-registration-tls.feature") +} + // TestSingleClusterEKSHelmfile is the live entry point for the // single-cluster EKS Helmfile feature. Skipped under -short. func TestSingleClusterEKSHelmfile(t *testing.T) { diff --git a/tests/bdd/plaintext_tls_rejection_script_test.go b/tests/bdd/plaintext_tls_rejection_script_test.go new file mode 100644 index 000000000..fbb75ea79 --- /dev/null +++ b/tests/bdd/plaintext_tls_rejection_script_test.go @@ -0,0 +1,81 @@ +/* +SPDX-FileCopyrightText: Copyright (c) NVIDIA CORPORATION & AFFILIATES. All rights reserved. +SPDX-License-Identifier: Apache-2.0 + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package bdd_tmp + +import ( + "os/exec" + "strings" + "testing" +) + +func TestPlaintextTLSRejectionAcceptsObservedGrpcurlDeadline(t *testing.T) { + cmd := exec.Command("bash", "scripts/assert-grpcurl-plaintext-tls-rejection.sh") + cmd.Stdin = strings.NewReader( + `Failed to dial target host "127.0.0.1:50071": context deadline exceeded`, + ) + + output, err := cmd.CombinedOutput() + if err != nil { + t.Fatalf("classify grpcurl deadline: %v\n%s", err, output) + } + if got, want := strings.TrimSpace(string(output)), "plaintext-watch-rejected=tls-listener-timeout"; got != want { + t.Fatalf("normalized output = %q, want %q", got, want) + } +} + +func TestPlaintextTLSRejectionRejectsSnapshotThenRPCDeadline(t *testing.T) { + cmd := exec.Command("bash", "scripts/assert-grpcurl-plaintext-tls-rejection.sh") + cmd.Stdin = strings.NewReader(`{ + "stargates": [] +} +ERROR: + Code: DeadlineExceeded + Message: context deadline exceeded`) + + output, err := cmd.CombinedOutput() + if err == nil { + t.Fatalf("successful plaintext snapshot followed by RPC deadline was accepted: %s", output) + } + if strings.Contains(string(output), "plaintext-watch-rejected=") { + t.Fatalf("successful plaintext snapshot emitted success marker: %s", output) + } +} + +func TestPlaintextTLSRejectionRejectsUnrelatedGrpcurlFailures(t *testing.T) { + for name, diagnostic := range map[string]string{ + "binary missing": "bash: grpcurl: command not found", + "connection refused": "Failed to dial target host 127.0.0.1:50071: connection refused", + "dial timeout plus output": "Failed to dial target host \"127.0.0.1:50071\": context deadline exceeded\n{}", + "proto import": "Failed to process proto source files.: missing.proto does not reside in any import path", + "usage": "flag provided but not defined: -bad-flag", + "wrong target dial timeout": "Failed to dial target host \"127.0.0.1:50443\": context deadline exceeded", + } { + t.Run(name, func(t *testing.T) { + cmd := exec.Command("bash", "scripts/assert-grpcurl-plaintext-tls-rejection.sh") + cmd.Stdin = strings.NewReader(diagnostic) + + output, err := cmd.CombinedOutput() + if err == nil { + t.Fatalf("unrelated failure was accepted: %s", output) + } + if strings.Contains(string(output), "plaintext-watch-rejected=") { + t.Fatalf("unrelated failure emitted success marker: %s", output) + } + }) + } +} diff --git a/tests/bdd/scripts/assert-grpcurl-plaintext-tls-rejection.sh b/tests/bdd/scripts/assert-grpcurl-plaintext-tls-rejection.sh new file mode 100755 index 000000000..453cdbc12 --- /dev/null +++ b/tests/bdd/scripts/assert-grpcurl-plaintext-tls-rejection.sh @@ -0,0 +1,19 @@ +#!/usr/bin/env bash +# SPDX-FileCopyrightText: Copyright (c) NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Normalize only the complete grpcurl 1.9.3 dial-stage diagnostic observed when +# plaintext HTTP/2 is sent to the verified TLS listener. RPC stream deadlines, +# additional output, and unrelated command or endpoint failures remain errors. + +set -euo pipefail + +diagnostic="$(cat)" +expected_diagnostic='Failed to dial target host "127.0.0.1:50071": context deadline exceeded' +if [[ "${diagnostic}" == "${expected_diagnostic}" ]]; then + printf '%s\n' 'plaintext-watch-rejected=tls-listener-timeout' + exit 0 +fi + +printf '%s\n' 'plaintext Watch failed without the expected TLS-listener dial timeout' >&2 +exit 1