From a4190fd3aad712a5c93bbb150e092f134cdd7ef5 Mon Sep 17 00:00:00 2001 From: Justin Kaseman Date: Thu, 27 Aug 2026 09:53:37 -0700 Subject: [PATCH 1/2] Add outbound HTTP action limits --- pkg/settings/cresettings/README.md | 2 ++ pkg/settings/cresettings/defaults.json | 2 ++ pkg/settings/cresettings/defaults.toml | 2 ++ pkg/settings/cresettings/settings.go | 11 ++++++++++- 4 files changed, 16 insertions(+), 1 deletion(-) diff --git a/pkg/settings/cresettings/README.md b/pkg/settings/cresettings/README.md index 50776b3523..8a01fa095b 100644 --- a/pkg/settings/cresettings/README.md +++ b/pkg/settings/cresettings/README.md @@ -69,6 +69,8 @@ flowchart GatewayConfidentialRelayPerNodeRate[\GatewayConfidentialRelayPerNodeRate/]:::rate GatewayHTTPActionMtlsRequestRate[\GatewayHTTPActionMtlsRequestRate/]:::rate GatewayHTTPActionMtlsConcurrencyLimit([GatewayHTTPActionMtlsConcurrencyLimit]):::resource + GatewayHTTPActionOutboundConcurrencyLimit([GatewayHTTPActionOutboundConcurrencyLimit]):::resource + GatewayHTTPActionOutboundPerNodeConcurrencyLimit([GatewayHTTPActionOutboundPerNodeConcurrencyLimit]):::resource end %% TODO unused %% PerOrg.ZeroBalancePruningTimeout diff --git a/pkg/settings/cresettings/defaults.json b/pkg/settings/cresettings/defaults.json index d49ec51ca1..01e0cd5a7d 100644 --- a/pkg/settings/cresettings/defaults.json +++ b/pkg/settings/cresettings/defaults.json @@ -26,6 +26,8 @@ "GatewayConfidentialRelayPerNodeRate": "10rps:10", "GatewayHTTPActionMtlsRequestRate": "every30s:0", "GatewayHTTPActionMtlsConcurrencyLimit": "50", + "GatewayHTTPActionOutboundConcurrencyLimit": "256", + "GatewayHTTPActionOutboundPerNodeConcurrencyLimit": "32", "TriggerRegistrationStatusUpdateTimeout": "0s", "BaseTriggerRetryInterval": "30s", "BaseTriggerMaxRetries": "20", diff --git a/pkg/settings/cresettings/defaults.toml b/pkg/settings/cresettings/defaults.toml index c6065f3d8e..fdbf851ad7 100644 --- a/pkg/settings/cresettings/defaults.toml +++ b/pkg/settings/cresettings/defaults.toml @@ -25,6 +25,8 @@ GatewayConfidentialRelayGlobalRate = '50rps:10' GatewayConfidentialRelayPerNodeRate = '10rps:10' GatewayHTTPActionMtlsRequestRate = 'every30s:0' GatewayHTTPActionMtlsConcurrencyLimit = '50' +GatewayHTTPActionOutboundConcurrencyLimit = '256' +GatewayHTTPActionOutboundPerNodeConcurrencyLimit = '32' TriggerRegistrationStatusUpdateTimeout = '0s' BaseTriggerRetryInterval = '30s' BaseTriggerMaxRetries = '20' diff --git a/pkg/settings/cresettings/settings.go b/pkg/settings/cresettings/settings.go index 652a79c876..4871db6e06 100644 --- a/pkg/settings/cresettings/settings.go +++ b/pkg/settings/cresettings/settings.go @@ -80,6 +80,8 @@ var Default = Schema{ GatewayConfidentialRelayPerNodeRate: Rate(rate.Limit(10), 10), GatewayHTTPActionMtlsRequestRate: Rate(rate.Every(30*time.Second), 0), GatewayHTTPActionMtlsConcurrencyLimit: Int(50), + GatewayHTTPActionOutboundConcurrencyLimit: Int(256), + GatewayHTTPActionOutboundPerNodeConcurrencyLimit: Int(32), TriggerRegistrationStatusUpdateTimeout: Duration(0 * time.Second), BaseTriggerRetryInterval: Duration(30 * time.Second), BaseTriggerMaxRetries: Int(20), @@ -378,7 +380,14 @@ type Schema struct { GatewayConfidentialRelayPerNodeRate Setting[config.Rate] GatewayHTTPActionMtlsRequestRate Setting[config.Rate] GatewayHTTPActionMtlsConcurrencyLimit Setting[int] `unit:"{request}"` - TriggerRegistrationStatusUpdateTimeout Setting[time.Duration] + // GatewayHTTPActionOutboundConcurrencyLimit bounds the number of outbound HTTP action + // requests the gateway will have in flight at once, across all nodes. + GatewayHTTPActionOutboundConcurrencyLimit Setting[int] `unit:"{request}"` + // GatewayHTTPActionOutboundPerNodeConcurrencyLimit bounds the number of outbound HTTP + // action requests the gateway will have in flight for a single node, so one node cannot + // occupy every slot in GatewayHTTPActionOutboundConcurrencyLimit. + GatewayHTTPActionOutboundPerNodeConcurrencyLimit Setting[int] `unit:"{request}"` + TriggerRegistrationStatusUpdateTimeout Setting[time.Duration] BaseTriggerRetryInterval Setting[time.Duration] BaseTriggerMaxRetries Setting[int] `unit:"{attempt}"` From 0e1bbad2a28648ce65e3cfb71791f55c6980fcf4 Mon Sep 17 00:00:00 2001 From: Justin Kaseman Date: Mon, 31 Aug 2026 12:15:56 -0700 Subject: [PATCH 2/2] Tune default to current global rate limit --- pkg/settings/cresettings/defaults.json | 4 ++-- pkg/settings/cresettings/defaults.toml | 4 ++-- pkg/settings/cresettings/settings.go | 13 +++++++++---- 3 files changed, 13 insertions(+), 8 deletions(-) diff --git a/pkg/settings/cresettings/defaults.json b/pkg/settings/cresettings/defaults.json index 01e0cd5a7d..c200204866 100644 --- a/pkg/settings/cresettings/defaults.json +++ b/pkg/settings/cresettings/defaults.json @@ -26,8 +26,8 @@ "GatewayConfidentialRelayPerNodeRate": "10rps:10", "GatewayHTTPActionMtlsRequestRate": "every30s:0", "GatewayHTTPActionMtlsConcurrencyLimit": "50", - "GatewayHTTPActionOutboundConcurrencyLimit": "256", - "GatewayHTTPActionOutboundPerNodeConcurrencyLimit": "32", + "GatewayHTTPActionOutboundConcurrencyLimit": "875", + "GatewayHTTPActionOutboundPerNodeConcurrencyLimit": "175", "TriggerRegistrationStatusUpdateTimeout": "0s", "BaseTriggerRetryInterval": "30s", "BaseTriggerMaxRetries": "20", diff --git a/pkg/settings/cresettings/defaults.toml b/pkg/settings/cresettings/defaults.toml index fdbf851ad7..0ab52487e2 100644 --- a/pkg/settings/cresettings/defaults.toml +++ b/pkg/settings/cresettings/defaults.toml @@ -25,8 +25,8 @@ GatewayConfidentialRelayGlobalRate = '50rps:10' GatewayConfidentialRelayPerNodeRate = '10rps:10' GatewayHTTPActionMtlsRequestRate = 'every30s:0' GatewayHTTPActionMtlsConcurrencyLimit = '50' -GatewayHTTPActionOutboundConcurrencyLimit = '256' -GatewayHTTPActionOutboundPerNodeConcurrencyLimit = '32' +GatewayHTTPActionOutboundConcurrencyLimit = '875' +GatewayHTTPActionOutboundPerNodeConcurrencyLimit = '175' TriggerRegistrationStatusUpdateTimeout = '0s' BaseTriggerRetryInterval = '30s' BaseTriggerMaxRetries = '20' diff --git a/pkg/settings/cresettings/settings.go b/pkg/settings/cresettings/settings.go index 4871db6e06..b132dc2806 100644 --- a/pkg/settings/cresettings/settings.go +++ b/pkg/settings/cresettings/settings.go @@ -80,8 +80,8 @@ var Default = Schema{ GatewayConfidentialRelayPerNodeRate: Rate(rate.Limit(10), 10), GatewayHTTPActionMtlsRequestRate: Rate(rate.Every(30*time.Second), 0), GatewayHTTPActionMtlsConcurrencyLimit: Int(50), - GatewayHTTPActionOutboundConcurrencyLimit: Int(256), - GatewayHTTPActionOutboundPerNodeConcurrencyLimit: Int(32), + GatewayHTTPActionOutboundConcurrencyLimit: Int(875), + GatewayHTTPActionOutboundPerNodeConcurrencyLimit: Int(175), TriggerRegistrationStatusUpdateTimeout: Duration(0 * time.Second), BaseTriggerRetryInterval: Duration(30 * time.Second), BaseTriggerMaxRetries: Int(20), @@ -381,11 +381,16 @@ type Schema struct { GatewayHTTPActionMtlsRequestRate Setting[config.Rate] GatewayHTTPActionMtlsConcurrencyLimit Setting[int] `unit:"{request}"` // GatewayHTTPActionOutboundConcurrencyLimit bounds the number of outbound HTTP action - // requests the gateway will have in flight at once, across all nodes. + // requests the gateway will have in flight at once, across all nodes. Sized to + // GatewayHTTPGlobalRate's ceiling (500rps burst) times observed p99.9 outbound latency + // (~1.75s in production-mainnet), so this never binds tighter than the rate limit already + // permits: 500 * 1.75 ~= 875. GatewayHTTPActionOutboundConcurrencyLimit Setting[int] `unit:"{request}"` // GatewayHTTPActionOutboundPerNodeConcurrencyLimit bounds the number of outbound HTTP // action requests the gateway will have in flight for a single node, so one node cannot - // occupy every slot in GatewayHTTPActionOutboundConcurrencyLimit. + // occupy every slot in GatewayHTTPActionOutboundConcurrencyLimit. Sized the same way as + // GatewayHTTPActionOutboundConcurrencyLimit, against GatewayHTTPPerNodeRate's ceiling + // (100rps burst): 100 * 1.75 ~= 175. GatewayHTTPActionOutboundPerNodeConcurrencyLimit Setting[int] `unit:"{request}"` TriggerRegistrationStatusUpdateTimeout Setting[time.Duration]