client connection issues

crates/beacon_state/tile/src/counters.rs

@@ -9,6 +9,10 @@ silver_common::declare_counters! {
         // gossip-admission structures at capacity (attestations still
         // accepted and relayed; aggregation/shedding coverage degrades)
         AttestationPoolFull,
+        // Gossip attestation ignored (never relayed): beacon_block_root not
+        // in fork choice — the reprocess-queue gap vs lighthouse, which
+        // parks and replays these.
+        AttestationUnknownRoot,
         SeenAggregatesFull,
         AttestationRootMemoFull,
         // attestation-root memo effectiveness (hit rate is the memo's

crates/beacon_state/tile/src/tile/gossip.rs

@@ -70,7 +70,10 @@ impl BeaconStateTile {
         data: &[u8],
         subnet: u64,
     ) -> Result<PreparedAttestation, Feedback> {
-        if data.len() < SINGLE_ATT_SIZE {
+        // Exact size: trailing bytes parse fine here (fixed-size prefix) but
+        // strict-SSZ peers reject the relayed message — their P4 lands on us,
+        // not the originator.
+        if data.len() != SINGLE_ATT_SIZE {
             return Err(Feedback::Reject(None));
         }
         let buf: &[u8; SINGLE_ATT_SIZE] = data[..SINGLE_ATT_SIZE].try_into().unwrap();
@@ -576,6 +579,7 @@ impl BeaconStateTile {
             return Err(Feedback::Reject(None));
         }
         let Some(idx) = self.fork_choice.find_node_idx(data.beacon_block_root()) else {
+            BeaconStateCounters::AttestationUnknownRoot.inc();
             return Err(Feedback::Ignore);
         };
         match self.fork_choice.checkpoint_block_of(idx, target_epoch * SLOTS_PER_EPOCH) {

crates/bin/Cargo.toml

@@ -36,4 +36,4 @@ alloc-profile = ["silver_common/alloc-profile"]
 # perf-{fn} queues (dev; needs `sudo sysctl kernel.perf_event_paranoid=2`).
 # Surfer picks the queues up when present; non-perf builds are unaffected.
 perf = ["silver_common/perf"]
-thread_park = ["flux/park", "silver_common/thread_park"]
+thread_park = ["flux/park", "silver_common/thread_park", "silver_network/thread_park"]

crates/bin/src/main.rs

@@ -134,6 +134,7 @@ fn main() -> Result<(), Box<dyn Error>> {
             false,
             None,
         ),
+        config.max_connections(),
     );
     let p2p_context = Context {
         gossip_producer: incoming_gossip_producer,

crates/common/src/spine.rs

@@ -36,18 +36,18 @@ pub struct SilverSpine {
     pub tile_info: ShmemData<TileInfo>,
 
     /// New incoming network gossip messages
-    #[queue(size(2usize.pow(16)))]
+    #[queue(size(2usize.pow(19)))]
     pub gossip_in: SpineQueue<GossipMsgIn>,
     /// New incoming gossip messages
-    #[queue(size(2usize.pow(16)))]
+    #[queue(size(2usize.pow(18)))]
     pub new_gossip: SpineQueue<NewGossipMsg>,
     /// P2p send messages.
-    #[queue(size(2usize.pow(16)))]
+    #[queue(size(2usize.pow(19)))]
     pub p2p_send: SpineQueue<P2pSend>,
     /// RPC recv messages.
     #[queue(size(2usize.pow(14)))]
     pub rpc_inbound: SpineQueue<RpcInbound>,
-    #[queue(size(2usize.pow(14)))]
+    #[queue(size(2usize.pow(16)))]
     pub peer_events: SpineQueue<PeerEvent>,
     #[queue(size(2usize.pow(14)))]
     pub peer_control: SpineQueue<PeerControl>,

crates/common/src/spine/messages.rs

@@ -314,10 +314,26 @@ pub enum PeerEvent {
         /// Failed send was an outbound RPC request: the PM must release the
         /// `outbound_in_flight` slot admitted for it, else it leaks.
         rpc_request: bool,
+        /// Response targeted a stream already closed/reset, as opposed to
+        /// stream-credit exhaustion opening a new request stream.
+        stream_gone: bool,
     },
     P2pStreamClosed {
         stream_id: P2pStreamId,
     },
+    /// Storage tile finished (or aborted) serving an inbound RPC request;
+    /// the PM logs it with peer identity.
+    RpcServeOutcome {
+        p2p_peer: usize,
+        protocol: StreamProtocol,
+        units_total: u32,
+        units_sent: u32,
+        /// Terminated with ResourceUnavailable on a missing unit rather than
+        /// draining to `Complete`.
+        missing: bool,
+        first_chunk_ms: u64,
+        elapsed_ms: u64,
+    },
     P2pOutboundMessageDropped {
         p2p_peer: usize,
         protocol: StreamProtocol,
@@ -646,6 +662,10 @@ pub enum PeerControl {
         p2p: PeerId,
         p2p_connection: usize,
     },
+    P2pPeerGoodbye {
+        p2p_connection: usize,
+        code: u32,
+    },
     /// Peer-level ban has timed out — counterpart to `Ban`. Network tile
     /// removes the peer from any deny-list / discv5 routing-table eviction
     /// state. Emitted from `tick` when the per-peer ban TTL expires.
@@ -1104,6 +1124,11 @@ pub struct P2pConnectionStats {
     pub tx_blocking: u64,
     pub rx_datagrams: u64,
     pub tx_datagrams: u64,
+    /// Live stream states on the connection. Climbing toward the remote's
+    /// MAX_STREAMS limit precedes "cannot create stream" bursts.
+    pub streams: u64,
+    /// Peer dialed us (QUIC server side), as opposed to us dialing them.
+    pub inbound: bool,
 }
 
 #[derive(Clone, Copy, Debug)]

crates/config/src/lib.rs

@@ -1,4 +1,7 @@
-use std::net::{IpAddr, Ipv4Addr, Ipv6Addr, SocketAddr, SocketAddrV4, SocketAddrV6};
+use std::{
+    net::{IpAddr, Ipv4Addr, Ipv6Addr, SocketAddr, SocketAddrV4, SocketAddrV6},
+    time::{SystemTime, UNIX_EPOCH},
+};
 
 pub use chain_config::ChainConfig;
 pub use discovery_config::DiscoveryConfig;
@@ -230,6 +233,15 @@ impl Config {
 
     pub fn enr(&self) -> Result<Enr, Error> {
         let mut builder = Enr::builder();
+        // Remotes only replace a cached record on a strictly higher seq, and
+        // the node key (= node_id) is stable across restarts — a constant
+        // seed pins the network to whatever record it saw first, so record
+        // changes (tcp, attnets) never propagate. Boot time is monotonic
+        // across restarts; in-boot `set_*` bumps of +1 stay far below the
+        // next boot's seed.
+        let boot_seq =
+            SystemTime::now().duration_since(UNIX_EPOCH).unwrap_or_default().as_millis() as u64;
+        builder.seq(boot_seq);
         let mut eth2 = [0u8; 16];
         eth2[..4].copy_from_slice(&self.fork_digest);
         eth2[4..8].copy_from_slice(&self.next_fork_version);
@@ -250,6 +262,11 @@ impl Config {
         }
         if let Some(qp) = self.quic_port {
             builder.quic4(qp).quic6(qp);
+            // Not served: lighthouse's discovery predicate (v8.x
+            // `start_query`) drops ENRs without a tcp port, so QUIC-only
+            // nodes are never dialed by it. Its dialer tries the quic
+            // address first, so advertising tcp gets us QUIC-dialed.
+            builder.tcp4(qp).tcp6(qp);
         }
         Ok(builder.build(self.keypair()?.secret_key())?)
     }
@@ -296,6 +313,14 @@ impl Config {
         self.discovery_config.clone()
     }
 
+    /// Hard cap on transport connections — inbound accepts are refused at
+    /// the QUIC layer beyond it. Sits above the peer manager's trim band
+    /// (`max_priority_peers` + 10%) so score-based trimming has room to
+    /// work inside it.
+    pub fn max_connections(&self) -> usize {
+        self.peer_score_params.max_priority_peers * 12 / 10
+    }
+
     pub fn peer_score_params(&self) -> ScoreParams {
         self.peer_score_params.clone()
     }
@@ -376,6 +401,22 @@ mod tests {
         assert_eq!(cfg.gossip_topics().unwrap().len(), 8);
     }
 
+    /// discv5 peers silently drop records over 300 bytes — an oversized ENR
+    /// makes the node invisible to discovery, not degraded.
+    #[test]
+    fn production_enr_fits_discv5_record_cap() {
+        let cfg = Config::new([1u8; 32], [1, 2, 3, 4], [5, 6, 7, 8], 123_456)
+            .with_external_ip_v4(Ipv4Addr::new(203, 0, 113, 7))
+            .with_discovery_port(9000)
+            .with_quic_port(9001);
+        let mut enr = cfg.enr().unwrap();
+        let key = cfg.keypair().unwrap();
+        enr.set_attnets([0xff; 8], key.secret_key()).unwrap();
+        // Unpadded base64: 4 chars per 3 bytes.
+        let bytes = enr.to_base64().trim_start_matches("enr:").len() * 3 / 4;
+        assert!(bytes <= 300, "ENR is {bytes} bytes, discv5 caps records at 300");
+    }
+
     #[test]
     fn builders_set_external_ip_and_genesis() {
         let cfg = Config::new([1u8; 32], [0u8; 4], [0u8; 4], 0)

crates/config/src/peer_score_params.rs

@@ -161,8 +161,12 @@ impl Default for ScoreParams {
             // once per slot to accumulate toward the −80 graylist floor.
             application_score_decay: 0.95,
 
-            // P7 — behaviour penalty
-            behaviour_penalty_threshold: 0.0,
+            // P7 — behaviour penalty. Squared excess over the threshold:
+            // with the free budget below, isolated events (a stream race, a
+            // one-off bad frame) cost nothing; only sustained misbehaviour
+            // gates. Threshold 0 made a single benign event = -10 = the
+            // gossip gate, which starved the peer's P3 view of us.
+            behaviour_penalty_threshold: 5.0,
             behaviour_penalty_weight: -10.0,
             behaviour_penalty_decay: 0.999,
 

crates/e2e/src/stack.rs

@@ -249,7 +249,7 @@ impl PublisherStack {
         );
 
         let endpoint = quic_endpoint(&keypair, /* is_server= */ true);
-        let p2p = P2p::new(keypair, endpoint);
+        let p2p = P2p::new(keypair, endpoint, 1024);
         let network = NetworkTile::new(disc_addr, discovery, addr, p2p, context)
             .map_err(std::io::Error::other)?;
 
@@ -363,7 +363,7 @@ impl EchoStack {
         );
 
         let endpoint = quic_endpoint(&keypair, /* is_server= */ true);
-        let p2p = P2p::new(keypair, endpoint);
+        let p2p = P2p::new(keypair, endpoint, 1024);
         let network = NetworkTile::new(disc_addr, discovery, addr, p2p, context)
             .map_err(std::io::Error::other)?;
 

crates/network/benches/quic_basic.rs

@@ -69,7 +69,7 @@ pub fn broadcast(c: &mut Criterion) {
                                 identify: None,
                             };
 
-                            let p2p = P2p::new(keypair, server_endpoint);
+                            let p2p = P2p::new(keypair, server_endpoint, 1024);
                             (
                                 NetworkTileInner::new(
                                     "0.0.0.0:20001".parse().unwrap(),
@@ -129,7 +129,7 @@ pub fn broadcast(c: &mut Criterion) {
                             };
 
                             let addr = format!("127.0.0.1:{}", 20002 + n);
-                            let mut p2p = P2p::new(keypair, client_endpoint);
+                            let mut p2p = P2p::new(keypair, client_endpoint, 1024);
                             p2p.connect(
                                 server_id.clone(),
                                 "127.0.0.1:20001".parse().unwrap(),

crates/network/benches/quic_pingpong.rs

@@ -59,7 +59,7 @@ pub fn broadcast(c: &mut Criterion) {
                             false,
                             None,
                         );
-                        let p2p = P2p::new(keypair, server_endpoint);
+                        let p2p = P2p::new(keypair, server_endpoint, 1024);
 
                         let context = Context {
                             gossip_producer: gi_producer,
@@ -136,7 +136,7 @@ pub fn broadcast(c: &mut Criterion) {
                     };
 
                     let addr = "127.0.0.1:20002";
-                    let mut p2p = P2p::new(keypair, client_endpoint);
+                    let mut p2p = P2p::new(keypair, client_endpoint, 1024);
                     p2p.connect(
                         server_id.clone(),
                         "127.0.0.1:20001".parse().unwrap(),

crates/network/src/lib.rs

@@ -22,13 +22,17 @@ silver_common::declare_counters! {
         // (the "zombie": peer never responded).
         DialTimeoutZombie,
         InboundAccepted,
+        InboundRefused,
         InboundHandshakeOk,
         // Disconnect reason buckets (ConnectionError variants).
         DisconnectTimedOut,
         DisconnectReset,
         DisconnectAppClosed,
         DisconnectLocal,
         DisconnectOther,
+        // A peer's read-timeout gave up on our response (their reset carried
+        // the response-timeout code): direct we-are-slow signal.
+        RemoteResponseTimeout,
     }
 }
 

crates/network/src/p2p/mod.rs

@@ -111,10 +111,13 @@ pub struct P2p {
     timeout: Option<Duration>,
     recv_count: usize,
     stats_cursor: usize,
+    /// Hard transport-level connection cap; inbound accepts refused beyond
+    /// it. Outbound dials are bounded separately by the peer manager.
+    max_connections: usize,
 }
 
 impl P2p {
-    pub fn new(keypair: Keypair, endpoint: Endpoint) -> Self {
+    pub fn new(keypair: Keypair, endpoint: Endpoint, max_connections: usize) -> Self {
         Self {
             keypair,
             endpoint,
@@ -123,6 +126,7 @@ impl P2p {
             timeout: Some(Duration::ZERO),
             recv_count: 0,
             stats_cursor: 0,
+            max_connections,
         }
     }
 
@@ -137,14 +141,16 @@ impl P2p {
         for _ in 0..batch.min(self.peers.len()) {
             let next = self
                 .peers
-                .iter()
-                .filter(|(h, _)| h.0 > self.stats_cursor)
-                .min_by_key(|(h, _)| h.0)
-                .or_else(|| self.peers.iter().min_by_key(|(h, _)| h.0));
-            let Some((handle, peer)) = next else {
+                .keys()
+                .filter(|h| h.0 > self.stats_cursor)
+                .min_by_key(|h| h.0)
+                .or_else(|| self.peers.keys().min_by_key(|h| h.0))
+                .copied();
+            let Some(handle) = next else {
                 return;
             };
             self.stats_cursor = handle.0;
+            let Some(peer) = self.peers.get_mut(&handle) else { return };
             if let Some(stats) = peer.stats(now) {
                 emit(stats);
             }
@@ -221,6 +227,15 @@ impl P2p {
                 }
             }
             DatagramEvent::NewConnection(incoming) => {
+                // Hard population cap: refuse at the QUIC layer (pre-TLS,
+                // cheap) — the peer manager's score-based trim only polices
+                // quality inside the band below this.
+                if self.peers.len() >= self.max_connections {
+                    crate::NetworkCounters::InboundRefused.inc();
+                    let rsp = self.endpoint.refuse(incoming, scratch);
+                    let _ = socket.send_to(&scratch[..rsp.size], rsp.destination);
+                    return true;
+                }
                 match self.endpoint.accept(incoming, now, scratch, None) {
                     Ok((handle, conn)) => {
                         crate::NetworkCounters::InboundAccepted.inc();

crates/network/src/p2p/quic/mod.rs

@@ -21,27 +21,50 @@ pub fn create_endpoint(server_config: Option<Arc<ServerConfig>>) -> Result<Endpo
     Ok(endpoint)
 }
 
+/// Quinn defaults are 10s idle with keep-alive off: quieter than 10s —
+/// e.g. a connection holding no mesh slots — died `TimedOut`, since both
+/// our 17s app ping and libp2p's typical 15s keep-alive are slower.
+/// 30s idle matches the libp2p ecosystem; the transport-level keep-alive
+/// makes quiet connections self-sustaining.
+fn transport_config() -> Arc<quinn_proto::TransportConfig> {
+    let mut tc = quinn_proto::TransportConfig::default();
+    tc.max_idle_timeout(Some(quinn_proto::IdleTimeout::try_from(IDLE_TIMEOUT).unwrap()));
+    tc.keep_alive_interval(Some(KEEP_ALIVE_INTERVAL));
+    Arc::new(tc)
+}
+
+const IDLE_TIMEOUT: std::time::Duration = std::time::Duration::from_secs(30);
+const KEEP_ALIVE_INTERVAL: std::time::Duration = std::time::Duration::from_secs(10);
+
 /// QUIC client config with libp2p TLS authentication.
 pub fn create_client_config(
     keypair: &Keypair,
     remote_peer_id: Option<PeerId>,
 ) -> Result<ClientConfig, Error> {
     let rustls_cfg = tls::make_client_config(keypair, remote_peer_id).map_err(Error::other)?;
-    Ok(ClientConfig::new(Arc::new(QuicClientConfig::try_from(rustls_cfg).map_err(Error::other)?)))
+    let mut config =
+        ClientConfig::new(Arc::new(QuicClientConfig::try_from(rustls_cfg).map_err(Error::other)?));
+    config.transport_config(transport_config());
+    Ok(config)
 }
 
 /// QUIC server config with libp2p TLS authentication.
 pub fn create_server_config(keypair: &Keypair) -> Result<ServerConfig, Error> {
     let rustls_cfg = tls::make_server_config(keypair).map_err(Error::other)?;
-    Ok(ServerConfig::with_crypto(Arc::new(
+    let mut config = ServerConfig::with_crypto(Arc::new(
         QuicServerConfig::try_from(rustls_cfg).map_err(Error::other)?,
-    )))
+    ));
+    config.transport_config(transport_config());
+    Ok(config)
 }
 
 #[derive(Clone, Copy, Debug, Eq, PartialEq)]
 pub enum SendResult {
     Ok,
     StreamCreationError,
+    /// RPC response targeted a stream no longer in the map (closed/reset
+    /// before the response was enqueued).
+    StreamGone,
     MessageDropped,
     UnknownPeer,
 }