diff --git a/tests/bdd/features/multi-cluster-helmfile-llm-registration-tls-fail-closed.feature b/tests/bdd/features/multi-cluster-helmfile-llm-registration-tls-fail-closed.feature new file mode 100644 index 000000000..a170b357b --- /dev/null +++ b/tests/bdd/features/multi-cluster-helmfile-llm-registration-tls-fail-closed.feature @@ -0,0 +1,104 @@ +@ncp-local @multi-cluster @helmfile @pki @llm-registration @negative +Feature: Reject insecure or invalid LLM worker registration + As a self-managed NVCF operator, + I want the Pylon registration endpoint to fail closed, + so that invalid trust or authority cannot silently enter the routing plane. + + Rule: Every rejected path is observable at the operator boundary + + Background: + Given these environment variables are set: + | name | + | NGC_API_KEY | + | SAMPLE_NGC_ORG | + | SAMPLE_NGC_TEAM | + And I prepare Helmfile environment "local-bdd-registration-tls-fail-closed" for stack "self-managed" from fixture "tests/bdd/fixtures/self-managed-local-bdd-multi.yaml" with values: + | global.imagePullSecrets[0].name | nvcr-pull-secret | + | global.helm.sources.repository | ${SAMPLE_NGC_ORG}/${SAMPLE_NGC_TEAM} | + | global.image.repository | ${SAMPLE_NGC_ORG}/${SAMPLE_NGC_TEAM} | + | global.workerEndpoints.llmRequestRouterAddress | https://llm-request-router.nvcf.svc.cluster.local:50071 | + | addons.llm.requestRouter.workload.kind | StatefulSet | + | addons.llm.requestRouter.backendRouter.pylonGrpcDialAddress | https://llm-request-router.nvcf.svc.cluster.local:50071 | + | observability.profile | disabled | + And I prepare Helmfile environment "local-bdd-registration-tls-invalid-authority" for stack "self-managed" from fixture "tests/bdd/fixtures/self-managed-local-bdd-multi.yaml" with values: + | global.imagePullSecrets[0].name | nvcr-pull-secret | + | global.helm.sources.repository | ${SAMPLE_NGC_ORG}/${SAMPLE_NGC_TEAM} | + | global.image.repository | ${SAMPLE_NGC_ORG}/${SAMPLE_NGC_TEAM} | + | global.workerEndpoints.llmRequestRouterAddress | https://llm_request_router.nvcf.svc.cluster.local:50071 | + | addons.llm.requestRouter.workload.kind | StatefulSet | + | addons.llm.requestRouter.backendRouter.pylonGrpcDialAddress | https://llm_request_router.nvcf.svc.cluster.local:50071 | + | observability.profile | disabled | + And I prepare self-managed secrets file "deploy/stacks/self-managed/secrets/local-bdd-registration-tls-fail-closed-secrets.yaml" from template "deploy/stacks/self-managed/secrets/secrets.yaml.template" using the current NGC registry credential + And I prepare self-managed secrets file "deploy/stacks/self-managed/secrets/local-bdd-registration-tls-invalid-authority-secrets.yaml" from template "deploy/stacks/self-managed/secrets/secrets.yaml.template" using the current NGC registry credential + When I run command "/bin/sh -c 'command -v grpcurl >/dev/null'" + Then the command exit code should be 0 + When I run command "k3d cluster get ncp-local" + Then the command exit code should be 1 + And multi-cluster ncp-local compute clusters are running: + | ncp-local-compute-1 | + And command has succeeded: + """ + kubectl config use-context k3d-ncp-local-cp + """ + And the "nvcr-pull-secret" image pull secret exists in namespaces: + | cassandra-system | + | nats-system | + | nvcf | + | api-keys | + | ess | + | sis | + | vault-system | + | nvca-operator | + | cert-manager | + + Scenario: Registration rejects every untrusted client path + When I run command "make -C deploy/stacks/self-managed template HELMFILE_ENV=local-bdd-registration-tls-fail-closed" + Then the command exit code should be 0 + When I run command "make -C deploy/stacks/self-managed install HELMFILE_ENV=local-bdd-registration-tls-fail-closed" + Then the command exit code should be 0 + When I run command "kubectl --context k3d-ncp-local-cp wait clusterissuer nvcf-openbao-pki --for=condition=Ready --timeout=5m" + Then the command exit code should be 0 + When I run command "kubectl --context k3d-ncp-local-cp wait certificate stargate-quic-tls -n nvcf --for=condition=Ready --timeout=5m" + Then the command exit code should be 0 + When I run command "kubectl --context k3d-ncp-local-cp rollout status statefulset/llm-request-router -n nvcf --timeout=10m" + Then the command exit code should be 0 + + # Establish that the TLS listener is reachable with its issued root, + # expected DNS identity, and HTTP/2 application protocol. + When I run command: + """ + /bin/bash -c 'openssl s_client -connect 127.0.0.1:50071 -servername llm-request-router.nvcf.svc.cluster.local -verify_hostname llm-request-router.nvcf.svc.cluster.local -alpn h2 -verify_return_error -CAfile <(kubectl --context k3d-ncp-local-cp get secret stargate-quic-tls -n nvcf -o jsonpath="{.data.ca\.crt}" | base64 -d) &1' + """ + Then the command exit code should be 0 + And the command output should contain "Verify return code: 0 (ok)" + And the command output should contain "ALPN protocol: h2" + + When I run command: + """ + /bin/bash -c 'set -u; cert_dir=$(mktemp -d); trap '\''rm -rf "$cert_dir"'\'' EXIT; openssl req -x509 -newkey rsa:2048 -nodes -subj /CN=wrong-root -keyout "$cert_dir/key.pem" -out "$cert_dir/ca.pem" -days 1 >/dev/null 2>&1 || exit; if diagnostic=$(grpcurl -max-time 5 -cacert "$cert_dir/ca.pem" -authority llm-request-router.nvcf.svc.cluster.local -import-path src/libraries/rust/stargate/crates/proto/proto -proto stargate.proto 127.0.0.1:50071 stargate.StargateControlPlane/WatchStargates 2>&1); then printf "wrong root was trusted\n" >&2; exit 1; fi; case "$diagnostic" in *"certificate signed by unknown authority"*|*"certificate is not trusted"*) printf "wrong-root-rejected\n" ;; *) printf "%s\n" "$diagnostic" >&2; exit 1 ;; esac' + """ + Then the command exit code should be 0 + + When I run command: + """ + /bin/bash -c 'if diagnostic=$(grpcurl -max-time 5 -cacert <(kubectl --context k3d-ncp-local-cp get secret stargate-quic-tls -n nvcf -o jsonpath="{.data.ca\.crt}" | base64 -d) -authority wrong-host.nvcf.svc.cluster.local -import-path src/libraries/rust/stargate/crates/proto/proto -proto stargate.proto 127.0.0.1:50071 stargate.StargateControlPlane/WatchStargates 2>&1); then printf "wrong hostname was accepted\n" >&2; exit 1; fi; case "$diagnostic" in *"not wrong-host.nvcf.svc.cluster.local"*) printf "wrong-host-rejected\n" ;; *) printf "%s\n" "$diagnostic" >&2; exit 1 ;; esac' + """ + Then the command exit code should be 0 + + When I run command: + """ + /bin/bash -c 'if diagnostic=$(grpcurl -max-time 5 -authority llm-request-router.nvcf.svc.cluster.local -import-path src/libraries/rust/stargate/crates/proto/proto -proto stargate.proto 127.0.0.1:50071 stargate.StargateControlPlane/WatchStargates 2>&1); then printf "missing trust was accepted\n" >&2; exit 1; fi; case "$diagnostic" in *"certificate signed by unknown authority"*|*"certificate is not trusted"*) printf "missing-trust-rejected\n" ;; *) printf "%s\n" "$diagnostic" >&2; exit 1 ;; esac' + """ + Then the command exit code should be 0 + + When I run command: + """ + /bin/bash -c 'if diagnostic=$(grpcurl -plaintext -max-time 5 -import-path src/libraries/rust/stargate/crates/proto/proto -proto stargate.proto 127.0.0.1:50071 stargate.StargateControlPlane/WatchStargates 2>&1); then printf "plaintext was accepted\n" >&2; exit 1; fi; case "$diagnostic" in *"context deadline exceeded"*|*"error reading server preface"*) printf "plaintext-rejected\n" ;; *) printf "%s\n" "$diagnostic" >&2; exit 1 ;; esac' + """ + Then the command exit code should be 0 + + When I run command: + """ + /bin/sh -c 'if diagnostic=$(make -C deploy/stacks/self-managed template HELMFILE_ENV=local-bdd-registration-tls-invalid-authority 2>&1); then printf "invalid authority was accepted\n" >&2; exit 1; fi; case "$diagnostic" in *"global.workerEndpoints.llmRequestRouterAddress must use optional http:// or https:// followed by DNS-or-IPv4:port or [IPv6]:port with port 1-65535"*) printf "invalid-authority-rejected\n" ;; *) printf "%s\n" "$diagnostic" >&2; exit 1 ;; esac' + """ + Then the command exit code should be 0 diff --git a/tests/bdd/godog_test.go b/tests/bdd/godog_test.go index eedb40e1f..b757d0232 100644 --- a/tests/bdd/godog_test.go +++ b/tests/bdd/godog_test.go @@ -1194,6 +1194,102 @@ func TestMultiClusterHelmfileFeatureFileWiresToSteps(t *testing.T) { assertFunctionDeploymentsUseInstanceType(t, suite.Runner.(*fakeRunner).runs, "NCP.GPU.H100_1x", 3) } +// TestMultiClusterHelmfileLLMRegistrationTLSFailClosedFeatureFileWiresToSteps +// runs the negative TLS registration matrix against a fake runner. +func TestMultiClusterHelmfileLLMRegistrationTLSFailClosedFeatureFileWiresToSteps(t *testing.T) { + t.Setenv("NGC_API_KEY", "test-key") + t.Setenv("SAMPLE_NGC_ORG", "test-org") + t.Setenv("SAMPLE_NGC_TEAM", "test-team") + + const tlsHandshakeCommand = `/bin/bash -c 'openssl s_client -connect 127.0.0.1:50071 ` + + `-servername llm-request-router.nvcf.svc.cluster.local ` + + `-verify_hostname llm-request-router.nvcf.svc.cluster.local -alpn h2 -verify_return_error ` + + `-CAfile <(kubectl --context k3d-ncp-local-cp get secret stargate-quic-tls -n nvcf ` + + `-o jsonpath="{.data.ca\.crt}" | base64 -d) &1'` + const grpcurlPreflightCommand = `/bin/sh -c 'command -v grpcurl >/dev/null'` + suite := newWiringSuite(t, newFakeRunner(map[string]harness.Result{ + "k3d cluster get ncp-local": {ExitCode: 1}, + grpcurlPreflightCommand: {ExitCode: 0}, + tlsHandshakeCommand: { + ExitCode: 0, + Stdout: "ALPN protocol: h2\nVerify return code: 0 (ok)\n", + }, + })) + seedHelmfileLocalBDDMultiFixture(t, suite.Config.RepoRoot) + seedStackSecretsTemplate(t, suite.Config.RepoRoot) + + sc := steps.NewScenarioContext(suite) + featurePath := mustResolveFeaturePath(t, "multi-cluster-helmfile-llm-registration-tls-fail-closed.feature") + var out strings.Builder + status := godog.TestSuite{ + Name: "multi-cluster-helmfile-llm-registration-tls-fail-closed-wiring", + ScenarioInitializer: func(ctx *godog.ScenarioContext) { + steps.RegisterAll(ctx, sc) + }, + Options: &godog.Options{ + Format: "pretty", + Paths: []string{featurePath}, + Strict: true, + Output: &out, + }, + }.Run() + if status != 0 { + t.Fatalf("godog suite status = %d\n%s", status, out.String()) + } + runs := suite.Runner.(*fakeRunner).runs + if !commandRanExactly(runs, grpcurlPreflightCommand) { + t.Fatal("grpcurl availability was not checked before the live probes") + } + for _, assertion := range []struct { + marker string + diagnostic string + }{ + {marker: "wrong-root-rejected", diagnostic: "certificate signed by unknown authority"}, + {marker: "wrong-host-rejected", diagnostic: "not wrong-host.nvcf.svc.cluster.local"}, + {marker: "missing-trust-rejected", diagnostic: "certificate is not trusted"}, + {marker: "plaintext-rejected", diagnostic: "context deadline exceeded"}, + { + marker: "invalid-authority-rejected", + diagnostic: "global.workerEndpoints.llmRequestRouterAddress must use " + + "optional http:// or https:// followed by DNS-or-IPv4:port or [IPv6]:port " + + "with port 1-65535", + }, + } { + if !commandRanThatContainsAll(runs, assertion.marker, assertion.diagnostic) { + t.Fatalf( + "negative registration command containing %q did not require diagnostic %q", + assertion.marker, + assertion.diagnostic, + ) + } + } + validEnvironment := filepath.Join( + suite.Config.RepoRoot, + "deploy", "stacks", "self-managed", "environments", + "local-bdd-registration-tls-fail-closed.yaml", + ) + invalidEnvironment := filepath.Join( + suite.Config.RepoRoot, + "deploy", "stacks", "self-managed", "environments", + "local-bdd-registration-tls-invalid-authority.yaml", + ) + for _, assertion := range []struct { + path string + want string + }{ + {path: validEnvironment, want: "https://llm-request-router.nvcf.svc.cluster.local:50071"}, + {path: invalidEnvironment, want: "https://llm_request_router.nvcf.svc.cluster.local:50071"}, + } { + got, found, err := dsl.ReadYAMLKey(assertion.path, "global.workerEndpoints.llmRequestRouterAddress") + if err != nil { + t.Fatalf("read worker endpoint override: %v", err) + } + if !found || got != assertion.want { + t.Fatalf("worker endpoint = %q, found = %t; want %q", got, found, assertion.want) + } + } +} + // TestSingleClusterHelmfileUpstreamImagesFeatureFileWiresToSteps runs the // focused upstream-image feature against a fake runner. The seeded global // template contains the exact documentation blocks so the ledger-backed @@ -1959,6 +2055,15 @@ func TestMultiClusterHelmfile(t *testing.T) { runLiveFeature(t, "multi-cluster-helmfile.feature") } +// TestMultiClusterHelmfileLLMRegistrationTLSFailClosed is the live entry +// point for the negative TLS registration matrix. Skipped under -short. +func TestMultiClusterHelmfileLLMRegistrationTLSFailClosed(t *testing.T) { + if testing.Short() { + t.Skip("live run skipped under -short") + } + runLiveFeature(t, "multi-cluster-helmfile-llm-registration-tls-fail-closed.feature") +} + // TestSingleClusterEKSHelmfile is the live entry point for the // single-cluster EKS Helmfile feature. Skipped under -short. func TestSingleClusterEKSHelmfile(t *testing.T) {