Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
142 changes: 110 additions & 32 deletions src/libraries/rust/stargate/crates/stargate-k8s-router/src/main.rs
Original file line number Diff line number Diff line change
Expand Up @@ -114,6 +114,7 @@ struct Args {
/// Tunnel protocol served by this UDP listener. HTTP/3 without WebTransport is unsupported.
#[arg(long, default_value = "raw-quic", value_name = "PROTOCOL")]
tunnel_protocol: RouterTunnelProtocol,
/// CA bundle used to verify the selected upstream Stargate pod.
#[arg(long, env = "STARGATE_UPSTREAM_TLS_CERT_PATH", value_name = "PATH")]
upstream_tls_cert_path: Option<String>,
}
Expand Down Expand Up @@ -165,8 +166,9 @@ impl RouterStartupConfig {
let server_identity_reloader = server_identity_reloader_from_args(&args)?;
// Read the mounted pair once. The reloader validated and owns these
// bytes, so the listener identity and the reload baseline cannot come
// from different projected generations. `tls_cert_pem` doubles as the
// startup-only outbound trust bundle.
// from different projected generations. Raw QUIC retains the serving
// certificate as a compatibility fallback when no dedicated upstream
// trust bundle is configured.
let (tls_cert_pem, tls_key_pem) = match server_identity_reloader
.as_ref()
.map(stargate_tls::ServerIdentityReloader::current_identity)
Expand All @@ -176,6 +178,13 @@ impl RouterStartupConfig {
}
Some(stargate_tls::ServerTlsIdentity::SelfSigned) | None => (None, None),
};
let upstream_tls_cert_pem = read_optional_file(args.upstream_tls_cert_path.as_deref())?;
Comment thread
mikeyrcamp marked this conversation as resolved.
if !args.quic_insecure
&& let Some(cert_pem) = upstream_tls_cert_pem.as_deref()
{
stargate_tls::build_trusted_quic_client_config_with_alpn(cert_pem, Vec::new())
.context("validate upstream TLS trust bundle")?;
}
let grpc = GrpcRouterConfig {
advertised_hostname_template: args.advertised_hostname_template.clone(),
advertised_grpc_port: args.advertised_grpc_port,
Expand All @@ -186,25 +195,20 @@ impl RouterStartupConfig {
watch_heartbeat_interval: Duration::from_millis(args.watch_heartbeat_ms),
};
let tunnel = match args.tunnel_protocol {
RouterTunnelProtocol::RawQuic => {
ensure!(
args.upstream_tls_cert_path.is_none(),
"--upstream-tls-cert-path is only supported with --tunnel-protocol=webtransport"
);
RouterTunnelConfig::RawQuic(QuicRouterConfig {
listen_addr: args.reverse_tunnel_listen_addr,
advertised_hostname_template: grpc.advertised_hostname_template.clone(),
target_namespace: grpc.target_namespace.clone(),
connect_timeout: grpc.connect_timeout,
relay_max_idle_timeout: relay_endpoint_config.max_idle_timeout,
relay_keep_alive_interval: relay_endpoint_config.keep_alive_interval,
tls_cert_pem,
tls_key_pem,
server_identity_reloader,
tls_reload_interval: stargate_tls::DEFAULT_TLS_RELOAD_INTERVAL,
quic_insecure: args.quic_insecure,
})
}
RouterTunnelProtocol::RawQuic => RouterTunnelConfig::RawQuic(QuicRouterConfig {
listen_addr: args.reverse_tunnel_listen_addr,
advertised_hostname_template: grpc.advertised_hostname_template.clone(),
target_namespace: grpc.target_namespace.clone(),
connect_timeout: grpc.connect_timeout,
relay_max_idle_timeout: relay_endpoint_config.max_idle_timeout,
relay_keep_alive_interval: relay_endpoint_config.keep_alive_interval,
tls_cert_pem,
tls_key_pem,
upstream_tls_cert_pem,
server_identity_reloader,
tls_reload_interval: stargate_tls::DEFAULT_TLS_RELOAD_INTERVAL,
quic_insecure: args.quic_insecure,
}),
RouterTunnelProtocol::WebTransport => {
RouterTunnelConfig::WebTransport(WebTransportRouterConfig {
listen_addr: args.reverse_tunnel_listen_addr,
Expand All @@ -217,9 +221,7 @@ impl RouterStartupConfig {
tls_key_pem,
server_identity_reloader,
tls_reload_interval: stargate_tls::DEFAULT_TLS_RELOAD_INTERVAL,
upstream_tls_cert_pem: read_optional_file(
args.upstream_tls_cert_path.as_deref(),
)?,
upstream_tls_cert_pem,
quic_insecure: args.quic_insecure,
})
}
Expand Down Expand Up @@ -609,16 +611,92 @@ mod tests {
}

#[test]
fn raw_quic_rejects_the_webtransport_only_upstream_trust_option() {
let upstream_ca = test_file(b"upstream-ca-bytes");
let args = router_args(&["--upstream-tls-cert-path", test_file_path(&upstream_ca)]);
fn raw_quic_keeps_upstream_trust_separate_from_the_serving_identity() {
install_default_crypto_provider();
let (cert_pem, key_pem) =
stargate_tls::generate_self_signed_cert().expect("test identity should generate");
let (upstream_ca_pem, _) =
stargate_tls::generate_self_signed_cert().expect("test CA should generate");
let cert = test_file(&cert_pem);
let key = test_file(&key_pem);
let upstream_ca = test_file(&upstream_ca_pem);
let config = startup_config(&[
"--tls-cert-path",
test_file_path(&cert),
"--tls-key-path",
test_file_path(&key),
"--upstream-tls-cert-path",
test_file_path(&upstream_ca),
]);

let error = RouterStartupConfig::from_args(args)
.err()
.expect("Raw QUIC must reject a WebTransport-only trust option");
let RouterTunnelConfig::RawQuic(tunnel) = config.tunnel else {
panic!("default startup configuration must use Raw QUIC");
};
assert_eq!(
error.to_string(),
"--upstream-tls-cert-path is only supported with --tunnel-protocol=webtransport"
tunnel.tls_cert_pem.as_deref(),
Some(cert_pem.as_slice()),
"the serving identity certificate must stay worker-facing"
);
assert_eq!(
tunnel.upstream_tls_cert_pem.as_deref(),
Some(upstream_ca_pem.as_slice()),
"the explicit CA bundle must configure the upstream Raw QUIC client"
);
}

#[test]
fn startup_config_rejects_malformed_upstream_trust_bundle() {
install_default_crypto_provider();
let upstream_ca =
test_file(b"-----BEGIN CERTIFICATE-----\nnot base64\n-----END CERTIFICATE-----\n");

let error = RouterStartupConfig::from_args(router_args(&[
"--upstream-tls-cert-path",
test_file_path(&upstream_ca),
]))
.err()
.expect("malformed upstream trust bundle must be rejected at startup");

let error = format!("{error:#}");
assert!(error.contains("validate upstream TLS trust bundle"));
assert!(error.contains("failed to parse cert PEM"));
}

#[test]
fn startup_config_rejects_empty_upstream_trust_bundle() {
install_default_crypto_provider();
let upstream_ca = test_file(b"");

let error = RouterStartupConfig::from_args(router_args(&[
"--upstream-tls-cert-path",
test_file_path(&upstream_ca),
]))
.err()
.expect("empty upstream trust bundle must be rejected at startup");

let error = format!("{error:#}");
assert!(error.contains("validate upstream TLS trust bundle"));
assert!(error.contains("TLS trust bundle contains no certificates"));
}

#[test]
fn startup_config_reports_unreadable_upstream_trust_bundle_path() {
let upstream_ca = tempfile::NamedTempFile::new().expect("test file should be creatable");
let upstream_ca_path = upstream_ca.path().to_owned();
drop(upstream_ca);

let upstream_ca_path_string = upstream_ca_path
.to_str()
.expect("test file path should be valid UTF-8");
let error = RouterStartupConfig::from_args(router_args(&[
"--upstream-tls-cert-path",
upstream_ca_path_string,
]))
.err()
.expect("unreadable upstream trust bundle must be rejected at startup");

assert!(
format!("{error:#}").contains(&format!("failed to read {upstream_ca_path_string}"))
);
}

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -41,6 +41,9 @@ pub struct QuicRouterConfig {
pub relay_keep_alive_interval: Option<Duration>,
pub tls_cert_pem: Option<Vec<u8>>,
pub tls_key_pem: Option<Vec<u8>>,
/// Dedicated upstream trust bundle. The serving certificate remains a
/// compatibility fallback when this is absent.
pub upstream_tls_cert_pem: Option<Vec<u8>>,
pub server_identity_reloader: Option<stargate_tls::ServerIdentityReloader>,
pub tls_reload_interval: Duration,
pub quic_insecure: bool,
Expand All @@ -62,14 +65,20 @@ struct QuicRouterRuntime {
tls_reload_interval: Duration,
}

fn upstream_trust_pem(config: &QuicRouterConfig) -> Option<&[u8]> {
config
.upstream_tls_cert_pem
.as_deref()
.or(config.tls_cert_pem.as_deref())
}

impl QuicRouterRuntime {
fn bind(config: QuicRouterConfig, connection_tasks: TaskTracker) -> Result<Self> {
let relay_config = RelayEndpointConfig {
max_idle_timeout: config.relay_max_idle_timeout,
keep_alive_interval: config.relay_keep_alive_interval,
};
let client_config =
build_client_config(config.tls_cert_pem.as_deref(), config.quic_insecure)?;
let client_config = build_client_config(upstream_trust_pem(&config), config.quic_insecure)?;
let server_config = match &config.server_identity_reloader {
// Serve the identity the reloader validated and owns. Reading the
// mounted files a second time here could pick up a different
Expand Down Expand Up @@ -345,12 +354,33 @@ mod tests {
relay_keep_alive_interval: Some(Duration::from_secs(5)),
tls_cert_pem: None,
tls_key_pem: None,
upstream_tls_cert_pem: None,
server_identity_reloader: None,
tls_reload_interval: stargate_tls::DEFAULT_TLS_RELOAD_INTERVAL,
quic_insecure: true,
}
}

#[test]
fn explicit_upstream_trust_precedes_serving_certificate() {
let mut config = test_config();
config.tls_cert_pem = Some(b"serving certificate".to_vec());
config.upstream_tls_cert_pem = Some(b"upstream CA".to_vec());

assert_eq!(upstream_trust_pem(&config), Some(b"upstream CA".as_slice()));
}

#[test]
fn serving_certificate_remains_upstream_trust_fallback() {
let mut config = test_config();
config.tls_cert_pem = Some(b"serving certificate".to_vec());

assert_eq!(
upstream_trust_pem(&config),
Some(b"serving certificate".as_slice())
);
}

fn snapshot_with_quic_target(pod_name: &str, quic_addr: SocketAddr) -> TargetSnapshot {
TargetSnapshot::initialized([PodTarget {
pod_name: pod_name.to_string(),
Expand Down
13 changes: 7 additions & 6 deletions src/libraries/rust/stargate/docs/tunnel-transports.md
Original file line number Diff line number Diff line change
Expand Up @@ -67,14 +67,15 @@ new sessions from targeting that pod but does not reassign an existing session.
The router preserves CONNECT request headers and propagates a non-success
upstream CONNECT status to the downstream pylon.

The two TLS hops are independent in WebTransport mode: `--tls-cert-path` and
`--tls-key-path` identify the router to pylon, while
The two TLS hops can use independent material in both Raw QUIC and WebTransport
modes. `--tls-cert-path` and `--tls-key-path` identify the router to pylon, while
`--upstream-tls-cert-path` (or `STARGATE_UPSTREAM_TLS_CERT_PATH`) is the trust
anchor used to verify Stargate. `--quic-insecure` disables only upstream
verification; without it, the upstream trust anchor is required.
Raw QUIC retains its existing router TLS configuration: its certificate path is
also its upstream trust source when verification is enabled, and it rejects the
WebTransport-only `--upstream-tls-cert-path` option.
verification. Raw QUIC retains the serving certificate as a compatibility
fallback when no dedicated upstream trust bundle is configured. Configure the
dedicated bundle so serving-certificate rotation does not also change outbound
trust. WebTransport requires the dedicated upstream trust bundle when
verification is enabled.

On GKE internal LoadBalancer Services, expose backend-facing gRPC/TCP and
Raw QUIC/UDP with separate single-protocol Services. The active-development
Expand Down
Loading