From 0b15b121c717a2d5ee086d532f429f277684535d Mon Sep 17 00:00:00 2001 From: Mike Camp Date: Fri, 28 Aug 2026 15:48:01 -0400 Subject: [PATCH 1/2] fix(stargate-k8s-router): separate Raw QUIC upstream trust Signed-off-by: Mike Camp --- .../crates/stargate-k8s-router/src/main.rs | 76 +++++++++++-------- .../crates/stargate-k8s-router/src/quic.rs | 34 ++++++++- .../rust/stargate/docs/tunnel-transports.md | 13 ++-- 3 files changed, 84 insertions(+), 39 deletions(-) diff --git a/src/libraries/rust/stargate/crates/stargate-k8s-router/src/main.rs b/src/libraries/rust/stargate/crates/stargate-k8s-router/src/main.rs index b406269a4..4f6c6b912 100644 --- a/src/libraries/rust/stargate/crates/stargate-k8s-router/src/main.rs +++ b/src/libraries/rust/stargate/crates/stargate-k8s-router/src/main.rs @@ -114,6 +114,7 @@ struct Args { /// Tunnel protocol served by this UDP listener. HTTP/3 without WebTransport is unsupported. #[arg(long, default_value = "raw-quic", value_name = "PROTOCOL")] tunnel_protocol: RouterTunnelProtocol, + /// CA bundle used to verify the selected upstream Stargate pod. #[arg(long, env = "STARGATE_UPSTREAM_TLS_CERT_PATH", value_name = "PATH")] upstream_tls_cert_path: Option, } @@ -165,8 +166,9 @@ impl RouterStartupConfig { let server_identity_reloader = server_identity_reloader_from_args(&args)?; // Read the mounted pair once. The reloader validated and owns these // bytes, so the listener identity and the reload baseline cannot come - // from different projected generations. `tls_cert_pem` doubles as the - // startup-only outbound trust bundle. + // from different projected generations. Raw QUIC retains the serving + // certificate as a compatibility fallback when no dedicated upstream + // trust bundle is configured. let (tls_cert_pem, tls_key_pem) = match server_identity_reloader .as_ref() .map(stargate_tls::ServerIdentityReloader::current_identity) @@ -176,6 +178,7 @@ impl RouterStartupConfig { } Some(stargate_tls::ServerTlsIdentity::SelfSigned) | None => (None, None), }; + let upstream_tls_cert_pem = read_optional_file(args.upstream_tls_cert_path.as_deref())?; let grpc = GrpcRouterConfig { advertised_hostname_template: args.advertised_hostname_template.clone(), advertised_grpc_port: args.advertised_grpc_port, @@ -186,25 +189,20 @@ impl RouterStartupConfig { watch_heartbeat_interval: Duration::from_millis(args.watch_heartbeat_ms), }; let tunnel = match args.tunnel_protocol { - RouterTunnelProtocol::RawQuic => { - ensure!( - args.upstream_tls_cert_path.is_none(), - "--upstream-tls-cert-path is only supported with --tunnel-protocol=webtransport" - ); - RouterTunnelConfig::RawQuic(QuicRouterConfig { - listen_addr: args.reverse_tunnel_listen_addr, - advertised_hostname_template: grpc.advertised_hostname_template.clone(), - target_namespace: grpc.target_namespace.clone(), - connect_timeout: grpc.connect_timeout, - relay_max_idle_timeout: relay_endpoint_config.max_idle_timeout, - relay_keep_alive_interval: relay_endpoint_config.keep_alive_interval, - tls_cert_pem, - tls_key_pem, - server_identity_reloader, - tls_reload_interval: stargate_tls::DEFAULT_TLS_RELOAD_INTERVAL, - quic_insecure: args.quic_insecure, - }) - } + RouterTunnelProtocol::RawQuic => RouterTunnelConfig::RawQuic(QuicRouterConfig { + listen_addr: args.reverse_tunnel_listen_addr, + advertised_hostname_template: grpc.advertised_hostname_template.clone(), + target_namespace: grpc.target_namespace.clone(), + connect_timeout: grpc.connect_timeout, + relay_max_idle_timeout: relay_endpoint_config.max_idle_timeout, + relay_keep_alive_interval: relay_endpoint_config.keep_alive_interval, + tls_cert_pem, + tls_key_pem, + upstream_tls_cert_pem, + server_identity_reloader, + tls_reload_interval: stargate_tls::DEFAULT_TLS_RELOAD_INTERVAL, + quic_insecure: args.quic_insecure, + }), RouterTunnelProtocol::WebTransport => { RouterTunnelConfig::WebTransport(WebTransportRouterConfig { listen_addr: args.reverse_tunnel_listen_addr, @@ -217,9 +215,7 @@ impl RouterStartupConfig { tls_key_pem, server_identity_reloader, tls_reload_interval: stargate_tls::DEFAULT_TLS_RELOAD_INTERVAL, - upstream_tls_cert_pem: read_optional_file( - args.upstream_tls_cert_path.as_deref(), - )?, + upstream_tls_cert_pem, quic_insecure: args.quic_insecure, }) } @@ -609,16 +605,34 @@ mod tests { } #[test] - fn raw_quic_rejects_the_webtransport_only_upstream_trust_option() { + fn raw_quic_keeps_upstream_trust_separate_from_the_serving_identity() { + install_default_crypto_provider(); + let (cert_pem, key_pem) = + stargate_tls::generate_self_signed_cert().expect("test identity should generate"); + let cert = test_file(&cert_pem); + let key = test_file(&key_pem); let upstream_ca = test_file(b"upstream-ca-bytes"); - let args = router_args(&["--upstream-tls-cert-path", test_file_path(&upstream_ca)]); + let config = startup_config(&[ + "--tls-cert-path", + test_file_path(&cert), + "--tls-key-path", + test_file_path(&key), + "--upstream-tls-cert-path", + test_file_path(&upstream_ca), + ]); - let error = RouterStartupConfig::from_args(args) - .err() - .expect("Raw QUIC must reject a WebTransport-only trust option"); + let RouterTunnelConfig::RawQuic(tunnel) = config.tunnel else { + panic!("default startup configuration must use Raw QUIC"); + }; assert_eq!( - error.to_string(), - "--upstream-tls-cert-path is only supported with --tunnel-protocol=webtransport" + tunnel.tls_cert_pem.as_deref(), + Some(cert_pem.as_slice()), + "the serving identity certificate must stay worker-facing" + ); + assert_eq!( + tunnel.upstream_tls_cert_pem.as_deref(), + Some(&b"upstream-ca-bytes"[..]), + "the explicit CA bundle must configure the upstream Raw QUIC client" ); } diff --git a/src/libraries/rust/stargate/crates/stargate-k8s-router/src/quic.rs b/src/libraries/rust/stargate/crates/stargate-k8s-router/src/quic.rs index e3528bb97..a4bdb1588 100644 --- a/src/libraries/rust/stargate/crates/stargate-k8s-router/src/quic.rs +++ b/src/libraries/rust/stargate/crates/stargate-k8s-router/src/quic.rs @@ -41,6 +41,9 @@ pub struct QuicRouterConfig { pub relay_keep_alive_interval: Option, pub tls_cert_pem: Option>, pub tls_key_pem: Option>, + /// Dedicated upstream trust bundle. The serving certificate remains a + /// compatibility fallback when this is absent. + pub upstream_tls_cert_pem: Option>, pub server_identity_reloader: Option, pub tls_reload_interval: Duration, pub quic_insecure: bool, @@ -62,14 +65,20 @@ struct QuicRouterRuntime { tls_reload_interval: Duration, } +fn upstream_trust_pem(config: &QuicRouterConfig) -> Option<&[u8]> { + config + .upstream_tls_cert_pem + .as_deref() + .or(config.tls_cert_pem.as_deref()) +} + impl QuicRouterRuntime { fn bind(config: QuicRouterConfig, connection_tasks: TaskTracker) -> Result { let relay_config = RelayEndpointConfig { max_idle_timeout: config.relay_max_idle_timeout, keep_alive_interval: config.relay_keep_alive_interval, }; - let client_config = - build_client_config(config.tls_cert_pem.as_deref(), config.quic_insecure)?; + let client_config = build_client_config(upstream_trust_pem(&config), config.quic_insecure)?; let server_config = match &config.server_identity_reloader { // Serve the identity the reloader validated and owns. Reading the // mounted files a second time here could pick up a different @@ -345,12 +354,33 @@ mod tests { relay_keep_alive_interval: Some(Duration::from_secs(5)), tls_cert_pem: None, tls_key_pem: None, + upstream_tls_cert_pem: None, server_identity_reloader: None, tls_reload_interval: stargate_tls::DEFAULT_TLS_RELOAD_INTERVAL, quic_insecure: true, } } + #[test] + fn explicit_upstream_trust_precedes_serving_certificate() { + let mut config = test_config(); + config.tls_cert_pem = Some(b"serving certificate".to_vec()); + config.upstream_tls_cert_pem = Some(b"upstream CA".to_vec()); + + assert_eq!(upstream_trust_pem(&config), Some(b"upstream CA".as_slice())); + } + + #[test] + fn serving_certificate_remains_upstream_trust_fallback() { + let mut config = test_config(); + config.tls_cert_pem = Some(b"serving certificate".to_vec()); + + assert_eq!( + upstream_trust_pem(&config), + Some(b"serving certificate".as_slice()) + ); + } + fn snapshot_with_quic_target(pod_name: &str, quic_addr: SocketAddr) -> TargetSnapshot { TargetSnapshot::initialized([PodTarget { pod_name: pod_name.to_string(), diff --git a/src/libraries/rust/stargate/docs/tunnel-transports.md b/src/libraries/rust/stargate/docs/tunnel-transports.md index 959c518c6..2bdd7b7ae 100644 --- a/src/libraries/rust/stargate/docs/tunnel-transports.md +++ b/src/libraries/rust/stargate/docs/tunnel-transports.md @@ -67,14 +67,15 @@ new sessions from targeting that pod but does not reassign an existing session. The router preserves CONNECT request headers and propagates a non-success upstream CONNECT status to the downstream pylon. -The two TLS hops are independent in WebTransport mode: `--tls-cert-path` and -`--tls-key-path` identify the router to pylon, while +The two TLS hops can use independent material in both Raw QUIC and WebTransport +modes. `--tls-cert-path` and `--tls-key-path` identify the router to pylon, while `--upstream-tls-cert-path` (or `STARGATE_UPSTREAM_TLS_CERT_PATH`) is the trust anchor used to verify Stargate. `--quic-insecure` disables only upstream -verification; without it, the upstream trust anchor is required. -Raw QUIC retains its existing router TLS configuration: its certificate path is -also its upstream trust source when verification is enabled, and it rejects the -WebTransport-only `--upstream-tls-cert-path` option. +verification. Raw QUIC retains the serving certificate as a compatibility +fallback when no dedicated upstream trust bundle is configured. Configure the +dedicated bundle so serving-certificate rotation does not also change outbound +trust. WebTransport requires the dedicated upstream trust bundle when +verification is enabled. On GKE internal LoadBalancer Services, expose backend-facing gRPC/TCP and Raw QUIC/UDP with separate single-protocol Services. The active-development From db18097b89dc059ed6c2488286dd3100c71f3bfc Mon Sep 17 00:00:00 2001 From: Mike Camp Date: Sun, 30 Aug 2026 08:15:35 -0400 Subject: [PATCH 2/2] fix(stargate-k8s-router): validate upstream TLS trust bundle Signed-off-by: Mike Camp --- .../crates/stargate-k8s-router/src/main.rs | 68 ++++++++++++++++++- 1 file changed, 66 insertions(+), 2 deletions(-) diff --git a/src/libraries/rust/stargate/crates/stargate-k8s-router/src/main.rs b/src/libraries/rust/stargate/crates/stargate-k8s-router/src/main.rs index 4f6c6b912..fb318a7a4 100644 --- a/src/libraries/rust/stargate/crates/stargate-k8s-router/src/main.rs +++ b/src/libraries/rust/stargate/crates/stargate-k8s-router/src/main.rs @@ -179,6 +179,12 @@ impl RouterStartupConfig { Some(stargate_tls::ServerTlsIdentity::SelfSigned) | None => (None, None), }; let upstream_tls_cert_pem = read_optional_file(args.upstream_tls_cert_path.as_deref())?; + if !args.quic_insecure + && let Some(cert_pem) = upstream_tls_cert_pem.as_deref() + { + stargate_tls::build_trusted_quic_client_config_with_alpn(cert_pem, Vec::new()) + .context("validate upstream TLS trust bundle")?; + } let grpc = GrpcRouterConfig { advertised_hostname_template: args.advertised_hostname_template.clone(), advertised_grpc_port: args.advertised_grpc_port, @@ -609,9 +615,11 @@ mod tests { install_default_crypto_provider(); let (cert_pem, key_pem) = stargate_tls::generate_self_signed_cert().expect("test identity should generate"); + let (upstream_ca_pem, _) = + stargate_tls::generate_self_signed_cert().expect("test CA should generate"); let cert = test_file(&cert_pem); let key = test_file(&key_pem); - let upstream_ca = test_file(b"upstream-ca-bytes"); + let upstream_ca = test_file(&upstream_ca_pem); let config = startup_config(&[ "--tls-cert-path", test_file_path(&cert), @@ -631,11 +639,67 @@ mod tests { ); assert_eq!( tunnel.upstream_tls_cert_pem.as_deref(), - Some(&b"upstream-ca-bytes"[..]), + Some(upstream_ca_pem.as_slice()), "the explicit CA bundle must configure the upstream Raw QUIC client" ); } + #[test] + fn startup_config_rejects_malformed_upstream_trust_bundle() { + install_default_crypto_provider(); + let upstream_ca = + test_file(b"-----BEGIN CERTIFICATE-----\nnot base64\n-----END CERTIFICATE-----\n"); + + let error = RouterStartupConfig::from_args(router_args(&[ + "--upstream-tls-cert-path", + test_file_path(&upstream_ca), + ])) + .err() + .expect("malformed upstream trust bundle must be rejected at startup"); + + let error = format!("{error:#}"); + assert!(error.contains("validate upstream TLS trust bundle")); + assert!(error.contains("failed to parse cert PEM")); + } + + #[test] + fn startup_config_rejects_empty_upstream_trust_bundle() { + install_default_crypto_provider(); + let upstream_ca = test_file(b""); + + let error = RouterStartupConfig::from_args(router_args(&[ + "--upstream-tls-cert-path", + test_file_path(&upstream_ca), + ])) + .err() + .expect("empty upstream trust bundle must be rejected at startup"); + + let error = format!("{error:#}"); + assert!(error.contains("validate upstream TLS trust bundle")); + assert!(error.contains("TLS trust bundle contains no certificates")); + } + + #[test] + fn startup_config_reports_unreadable_upstream_trust_bundle_path() { + let upstream_ca = tempfile::NamedTempFile::new().expect("test file should be creatable"); + let upstream_ca_path = upstream_ca.path().to_owned(); + drop(upstream_ca); + + let upstream_ca_path_string = upstream_ca_path + .to_str() + .expect("test file path should be valid UTF-8"); + let error = RouterStartupConfig::from_args(router_args(&[ + "--upstream-tls-cert-path", + upstream_ca_path_string, + ])) + .err() + .expect("unreadable upstream trust bundle must be rejected at startup"); + + assert!( + format!("{error:#}").contains(&format!("failed to read {upstream_ca_path_string}")) + ); + } + #[test] fn startup_config_derives_runtime_configs_from_args() { // The server identity is validated at startup, so the fixture has to be a