From e3861395c1f7e1e7b6415be2818b9a2b214cf260 Mon Sep 17 00:00:00 2001 From: vladimir-ea Date: Fri, 21 Aug 2026 16:45:59 +0100 Subject: [PATCH 01/11] markups --- crates/beacon_state/tile/src/tile/gossip.rs | 5 +- crates/bin/Cargo.toml | 2 +- crates/common/src/spine/messages.rs | 21 + crates/config/src/lib.rs | 35 +- crates/network/src/lib.rs | 3 + crates/network/src/p2p/mod.rs | 12 +- crates/network/src/p2p/quic/mod.rs | 3 + crates/network/src/p2p/quic/peer.rs | 267 +- .../network/src/p2p/streams/rpc/request_in.rs | 113 +- crates/network/src/p2p/streams/state.rs | 10 + crates/network/src/tile.rs | 7 +- crates/peer/src/lib.rs | 8 + crates/peer/src/manager.rs | 2188 ++--------------- crates/peer/src/manager/gossip.rs | 2084 ++++++++++++++++ crates/peer/src/manager/rpc.rs | 9 +- crates/peer/src/state.rs | 13 +- crates/ssz/src/ssz_view.rs | 60 + crates/storage/src/store.rs | 171 +- crates/storage/src/store/io.rs | 5 + crates/surfer/src/render/gossip_pane.rs | 96 +- crates/surfer/src/render/peers_pane.rs | 30 +- 21 files changed, 3062 insertions(+), 2080 deletions(-) create mode 100644 crates/peer/src/manager/gossip.rs diff --git a/crates/beacon_state/tile/src/tile/gossip.rs b/crates/beacon_state/tile/src/tile/gossip.rs index 70a08690..9a4da3bd 100644 --- a/crates/beacon_state/tile/src/tile/gossip.rs +++ b/crates/beacon_state/tile/src/tile/gossip.rs @@ -70,7 +70,10 @@ impl BeaconStateTile { data: &[u8], subnet: u64, ) -> Result { - if data.len() < SINGLE_ATT_SIZE { + // Exact size: trailing bytes parse fine here (fixed-size prefix) but + // strict-SSZ peers reject the relayed message — their P4 lands on us, + // not the originator. + if data.len() != SINGLE_ATT_SIZE { return Err(Feedback::Reject(None)); } let buf: &[u8; SINGLE_ATT_SIZE] = data[..SINGLE_ATT_SIZE].try_into().unwrap(); diff --git a/crates/bin/Cargo.toml b/crates/bin/Cargo.toml index 79f9f6cb..e7380f65 100644 --- a/crates/bin/Cargo.toml +++ b/crates/bin/Cargo.toml @@ -36,4 +36,4 @@ alloc-profile = ["silver_common/alloc-profile"] # perf-{fn} queues (dev; needs `sudo sysctl kernel.perf_event_paranoid=2`). # Surfer picks the queues up when present; non-perf builds are unaffected. perf = ["silver_common/perf"] -thread_park = ["flux/park", "silver_common/thread_park"] +thread_park = ["flux/park", "silver_common/thread_park", "silver_network/thread_park"] diff --git a/crates/common/src/spine/messages.rs b/crates/common/src/spine/messages.rs index a14e0029..50cada94 100644 --- a/crates/common/src/spine/messages.rs +++ b/crates/common/src/spine/messages.rs @@ -314,10 +314,26 @@ pub enum PeerEvent { /// Failed send was an outbound RPC request: the PM must release the /// `outbound_in_flight` slot admitted for it, else it leaks. rpc_request: bool, + /// Response targeted a stream already closed/reset, as opposed to + /// stream-credit exhaustion opening a new request stream. + stream_gone: bool, }, P2pStreamClosed { stream_id: P2pStreamId, }, + /// Storage tile finished (or aborted) serving an inbound RPC request; + /// the PM logs it with peer identity. + RpcServeOutcome { + p2p_peer: usize, + protocol: StreamProtocol, + units_total: u32, + units_sent: u32, + /// Terminated with ResourceUnavailable on a missing unit rather than + /// draining to `Complete`. + missing: bool, + first_chunk_ms: u64, + elapsed_ms: u64, + }, P2pOutboundMessageDropped { p2p_peer: usize, protocol: StreamProtocol, @@ -1104,6 +1120,11 @@ pub struct P2pConnectionStats { pub tx_blocking: u64, pub rx_datagrams: u64, pub tx_datagrams: u64, + /// Live stream states on the connection. Climbing toward the remote's + /// MAX_STREAMS limit precedes "cannot create stream" bursts. + pub streams: u64, + /// Peer dialed us (QUIC server side), as opposed to us dialing them. + pub inbound: bool, } #[derive(Clone, Copy, Debug)] diff --git a/crates/config/src/lib.rs b/crates/config/src/lib.rs index c1f0a6f8..2d6fa0a1 100644 --- a/crates/config/src/lib.rs +++ b/crates/config/src/lib.rs @@ -1,4 +1,7 @@ -use std::net::{IpAddr, Ipv4Addr, Ipv6Addr, SocketAddr, SocketAddrV4, SocketAddrV6}; +use std::{ + net::{IpAddr, Ipv4Addr, Ipv6Addr, SocketAddr, SocketAddrV4, SocketAddrV6}, + time::{SystemTime, UNIX_EPOCH}, +}; pub use chain_config::ChainConfig; pub use discovery_config::DiscoveryConfig; @@ -230,6 +233,15 @@ impl Config { pub fn enr(&self) -> Result { let mut builder = Enr::builder(); + // Remotes only replace a cached record on a strictly higher seq, and + // the node key (= node_id) is stable across restarts — a constant + // seed pins the network to whatever record it saw first, so record + // changes (tcp, attnets) never propagate. Boot time is monotonic + // across restarts; in-boot `set_*` bumps of +1 stay far below the + // next boot's seed. + let boot_seq = + SystemTime::now().duration_since(UNIX_EPOCH).unwrap_or_default().as_millis() as u64; + builder.seq(boot_seq); let mut eth2 = [0u8; 16]; eth2[..4].copy_from_slice(&self.fork_digest); eth2[4..8].copy_from_slice(&self.next_fork_version); @@ -250,6 +262,11 @@ impl Config { } if let Some(qp) = self.quic_port { builder.quic4(qp).quic6(qp); + // Not served: lighthouse's discovery predicate (v8.x + // `start_query`) drops ENRs without a tcp port, so QUIC-only + // nodes are never dialed by it. Its dialer tries the quic + // address first, so advertising tcp gets us QUIC-dialed. + builder.tcp4(qp).tcp6(qp); } Ok(builder.build(self.keypair()?.secret_key())?) } @@ -376,6 +393,22 @@ mod tests { assert_eq!(cfg.gossip_topics().unwrap().len(), 8); } + /// discv5 peers silently drop records over 300 bytes — an oversized ENR + /// makes the node invisible to discovery, not degraded. + #[test] + fn production_enr_fits_discv5_record_cap() { + let cfg = Config::new([1u8; 32], [1, 2, 3, 4], [5, 6, 7, 8], 123_456) + .with_external_ip_v4(Ipv4Addr::new(203, 0, 113, 7)) + .with_discovery_port(9000) + .with_quic_port(9001); + let mut enr = cfg.enr().unwrap(); + let key = cfg.keypair().unwrap(); + enr.set_attnets([0xff; 8], key.secret_key()).unwrap(); + // Unpadded base64: 4 chars per 3 bytes. + let bytes = enr.to_base64().trim_start_matches("enr:").len() * 3 / 4; + assert!(bytes <= 300, "ENR is {bytes} bytes, discv5 caps records at 300"); + } + #[test] fn builders_set_external_ip_and_genesis() { let cfg = Config::new([1u8; 32], [0u8; 4], [0u8; 4], 0) diff --git a/crates/network/src/lib.rs b/crates/network/src/lib.rs index 0a100bb9..c351c052 100644 --- a/crates/network/src/lib.rs +++ b/crates/network/src/lib.rs @@ -29,6 +29,9 @@ silver_common::declare_counters! { DisconnectAppClosed, DisconnectLocal, DisconnectOther, + // A peer's read-timeout gave up on our response (their reset carried + // the response-timeout code): direct we-are-slow signal. + RemoteResponseTimeout, } } diff --git a/crates/network/src/p2p/mod.rs b/crates/network/src/p2p/mod.rs index 3b3ebccb..ec9971dc 100644 --- a/crates/network/src/p2p/mod.rs +++ b/crates/network/src/p2p/mod.rs @@ -137,14 +137,16 @@ impl P2p { for _ in 0..batch.min(self.peers.len()) { let next = self .peers - .iter() - .filter(|(h, _)| h.0 > self.stats_cursor) - .min_by_key(|(h, _)| h.0) - .or_else(|| self.peers.iter().min_by_key(|(h, _)| h.0)); - let Some((handle, peer)) = next else { + .keys() + .filter(|h| h.0 > self.stats_cursor) + .min_by_key(|h| h.0) + .or_else(|| self.peers.keys().min_by_key(|h| h.0)) + .copied(); + let Some(handle) = next else { return; }; self.stats_cursor = handle.0; + let Some(peer) = self.peers.get_mut(&handle) else { return }; if let Some(stats) = peer.stats(now) { emit(stats); } diff --git a/crates/network/src/p2p/quic/mod.rs b/crates/network/src/p2p/quic/mod.rs index 165d87f4..cf37a6c9 100644 --- a/crates/network/src/p2p/quic/mod.rs +++ b/crates/network/src/p2p/quic/mod.rs @@ -42,6 +42,9 @@ pub fn create_server_config(keypair: &Keypair) -> Result { pub enum SendResult { Ok, StreamCreationError, + /// RPC response targeted a stream no longer in the map (closed/reset + /// before the response was enqueued). + StreamGone, MessageDropped, UnknownPeer, } diff --git a/crates/network/src/p2p/quic/peer.rs b/crates/network/src/p2p/quic/peer.rs index 59f5aaa6..ee3a3ae8 100644 --- a/crates/network/src/p2p/quic/peer.rs +++ b/crates/network/src/p2p/quic/peer.rs @@ -22,11 +22,23 @@ use crate::{ NetEvent, context::Context, quic::{SendResult, stream::StreamIoImpl}, - streams::{AcquiredRpcOutbound, StreamState}, + streams::{AcquiredRpcOutbound, StreamError, StreamState}, tls::peer_id_from_certificate, }, }; +const STREAM_SETUP_TIMEOUT: Duration = Duration::from_secs(10); + +/// Application error codes on abnormal stream teardown, so the remote can +/// tell a protocol violation from our read-timeout giving up on its slow +/// response (the latter is a they-are-slow signal at the receiver, counted +/// as `RemoteResponseTimeout`). +const STREAM_ERR_CODE_PROTOCOL: u32 = 1; +const STREAM_ERR_CODE_RESPONSE_TIMEOUT: u32 = 2; +/// Whole-life budget for an inbound RPC stream (negotiate + request + +/// serve). Generous so a large by-range serve to a slow peer survives. +const INBOUND_RPC_TIMEOUT: Duration = Duration::from_secs(30); + pub(crate) struct Peer { id: RemotePeer, handle: ConnectionHandle, @@ -135,18 +147,31 @@ impl Peer { tracing::debug!(id=?self.id, protocol=?msg.protocol(), "outbound rpc"); self.dirty = true; - if let Some(stream) = match &msg { + let stream = match &msg { AcquiredRpcOutbound::Request(req) => { - self.open_stream(req.request.protocol()).and_then(|id| self.streams.get_mut(&id)) + match self + .open_stream(req.request.protocol()) + .and_then(|id| self.streams.get_mut(&id)) + { + Some(stream) => stream, + None => return SendResult::StreamCreationError, + } } - AcquiredRpcOutbound::Response(rsp) => self.streams.get_mut(&rsp.stream_id.stream_id()), - } { - if let OutboundBuffer::Rpc(buffer) = &mut stream.out_buffer { - let dropped = buffer.add_msg(msg); - stream.needs_spin = true; - return if dropped { SendResult::MessageDropped } else { SendResult::Ok }; + AcquiredRpcOutbound::Response(rsp) => { + match self.streams.get_mut(&rsp.stream_id.stream_id()) { + Some(stream) => stream, + None => { + tracing::debug!(stream_id = ?rsp.stream_id, "rpc response: stream gone"); + return SendResult::StreamGone; + } + } } }; + if let OutboundBuffer::Rpc(buffer) = &mut stream.out_buffer { + let dropped = buffer.add_msg(msg); + stream.needs_spin = true; + return if dropped { SendResult::MessageDropped } else { SendResult::Ok }; + } SendResult::StreamCreationError } @@ -171,6 +196,38 @@ impl Peer { self.connection.close(now, VarInt::from_u32(0), Bytes::new()); } + /// Stream-leak diagnostics: one line per over-populated connection + /// listing each stream's protocol, state, direction, age and queue, so + /// a stuck state names itself. Steady state is 2 gossip + transient + /// RPC. + #[allow(dead_code)] + pub(crate) fn log_stream_census(&mut self, now: Instant) { + const CENSUS_MIN_STREAMS: usize = 5; + if self.streams.len() < CENSUS_MIN_STREAMS { + return; + } + let census: Vec = self + .streams + .values_mut() + .map(|s| { + format!( + "{:?}:{}:{}:{}s:q{}", + s.p2p_id.protocol(), + s.state.get_mut().name(), + if s.p2p_id.is_incoming() { "in" } else { "out" }, + now.saturating_duration_since(s.created_at).as_secs(), + s.out_buffer.len(), + ) + }) + .collect(); + tracing::info!( + id = ?self.id, + count = census.len(), + streams = census.join(" | "), + "stream census" + ); + } + /// Returns connection stats for peers connected > 30 seconds. pub(crate) fn stats(&self, now: Instant) -> Option { (now - self.created_at > Duration::from_secs(10)).then(|| { @@ -186,6 +243,8 @@ impl Peer { tx_blocking: stats.frame_tx.data_blocked, rx_datagrams: stats.udp_rx.datagrams, tx_datagrams: stats.udp_tx.datagrams, + streams: self.streams.len() as u64, + inbound: self.connection.side() == Side::Server, } }) } @@ -215,6 +274,8 @@ impl Peer { state: Cell::new(stream), out_buffer, needs_spin: true, + setup_deadline: None, + created_at: Instant::now(), }); Some(id) } @@ -387,7 +448,7 @@ impl Peer { let state = stream.state.get_mut(); stream.needs_spin = state.awaiting_alloc() || (state.write_idle() && !stream.out_buffer.is_empty()); - if let Some(d) = state.deadline() { + if let Some(d) = stream.wake_deadline() { self.next_deadline = Some(self.next_deadline.map_or(d, |cur| cur.min(d))); } @@ -425,6 +486,8 @@ impl Peer { state: Cell::new(StreamState::new_inbound()), out_buffer: OutboundBuffer::Unset, needs_spin: true, + setup_deadline: None, + created_at: Instant::now(), }); on_event(NetEvent::StreamReady { stream: p2p_id }); tracing::debug!(?p2p_id, "stream open"); @@ -438,6 +501,17 @@ impl Peer { // This event is emitted after we call 'finish()' on the send side of // the stream. Indicates that data was sent and acked. tracing::debug!(?id, "send half finished"); + // Fully-served inbound stream: response acked, nothing left + // to write. Reclaim here — quinn requesters reclaim it for + // us via a spurious STOP_SENDING (`stop_send`), but netty + // (teku) just FINs and the state would leak. + if let Some(stream) = self.streams.get_mut(&id) && + stream.p2p_id.is_incoming() && + stream.is_complete() && + stream.out_buffer.is_empty() + { + self.remove_stream(id); + } } quinn_proto::StreamEvent::Stopped { id, error_code } => { if let Some(stream) = self.streams.get_mut(&id) { @@ -511,7 +585,7 @@ where let state = stream.state.get_mut(); stream.needs_spin = state.awaiting_alloc() || (state.write_idle() && !stream.out_buffer.is_empty()); - if let Some(d) = state.deadline() { + if let Some(d) = stream.wake_deadline() { *next_deadline = Some(next_deadline.map_or(d, |cur| cur.min(d))); } @@ -560,7 +634,7 @@ fn id_from_connection(conn: &Connection) -> Option { fn out_buffer(id: &P2pStreamId, incoming: bool) -> OutboundBuffer { match id.protocol() { - StreamProtocol::GossipSub => OutboundBuffer::Gossip(OutBuffer::new(1024)), + StreamProtocol::GossipSub => OutboundBuffer::Gossip(OutBuffer::new(4096)), StreamProtocol::BeaconBlocksByRange | StreamProtocol::BeaconBlocksByRoot | StreamProtocol::DataColumnSidecarsByRange | @@ -583,6 +657,8 @@ struct Stream { /// credit). Fresh streams are born flagged — the initial negotiate /// write, and bytes delivered with `Opened`, emit no event. needs_spin: bool, + setup_deadline: Option, + created_at: Instant, } enum SpinResult { @@ -603,6 +679,23 @@ impl Stream { where E: FnMut(crate::NetEvent), { + if self.state.get_mut().in_setup() { + let deadline = *self.setup_deadline.get_or_insert(now + STREAM_SETUP_TIMEOUT); + if now >= deadline { + return self.timed_out("stream setup timeout", connection, on_event); + } + } + + // Inbound RPC streams have no state-machine deadline of their own: + // a starved request read, a response lost before enqueue, or an + // unacked FIN would otherwise park the stream forever (observed as + // aged `IncomingRpc` census entries from teku). + if matches!(self.state.get_mut(), StreamState::IncomingRpc { .. }) && + now >= self.created_at + INBOUND_RPC_TIMEOUT + { + return self.timed_out("inbound rpc timeout", connection, on_event); + } + let state = self.state.take(); // Capture the phase before `spin` consumes `state`, so a stream // error/teardown log can report which protocol phase it failed in. @@ -666,8 +759,14 @@ impl Stream { "stream error" ); let id = self.p2p_id.stream_id(); - let _ = connection.send_stream(id).finish(); - let _ = connection.recv_stream(id).stop(VarInt::from_u32(1)); + let code = if matches!(e, StreamError::ReadResponseTimeout) { + STREAM_ERR_CODE_RESPONSE_TIMEOUT + } else { + STREAM_ERR_CODE_PROTOCOL + }; + // reset, not finish: see the setup-timeout teardown above. + let _ = connection.send_stream(id).reset(VarInt::from_u32(code)); + let _ = connection.recv_stream(id).stop(VarInt::from_u32(code)); // TODO error info. on_event(NetEvent::StreamClosed { stream: self.p2p_id }); @@ -677,6 +776,45 @@ impl Stream { } /// Remote peer has called 'stop' on their recv stream (our send side). + fn timed_out( + &mut self, + reason: &'static str, + connection: &mut Connection, + on_event: &mut E, + ) -> SpinResult + where + E: FnMut(crate::NetEvent), + { + tracing::warn!( + id = ?self.p2p_id, + protocol = ?self.p2p_id.protocol(), + state = self.state.get_mut().name(), + reason, + "stream timeout" + ); + let id = self.p2p_id.stream_id(); + // reset, not finish: FIN queues behind buffered data the stalled + // peer isn't draining, so the stream — and its MAX_STREAMS credit — + // would leak at the QUIC layer. + let _ = connection.send_stream(id).reset(VarInt::from_u32(1)); + let _ = connection.recv_stream(id).stop(VarInt::from_u32(1)); + on_event(NetEvent::StreamClosed { stream: self.p2p_id }); + SpinResult::End + } + + /// Earliest instant this stream needs an unprompted poll: setup and + /// inbound-RPC lifetime budgets, or the state machine's own deadline. + fn wake_deadline(&mut self) -> Option { + let state = self.state.get_mut(); + if state.in_setup() { + self.setup_deadline + } else if matches!(state, StreamState::IncomingRpc { .. }) { + Some(self.created_at + INBOUND_RPC_TIMEOUT) + } else { + state.deadline() + } + } + fn stop_send( &mut self, error_code: VarInt, @@ -687,6 +825,9 @@ impl Stream { E: FnMut(crate::NetEvent), { let _ = connection.send_stream(self.p2p_id.stream_id()).reset(VarInt::from_u32(0)); + if error_code.into_inner() == STREAM_ERR_CODE_RESPONSE_TIMEOUT as u64 { + crate::NetworkCounters::RemoteResponseTimeout.inc(); + } if self.state.get_mut().is_receive_only(self.p2p_id.protocol()) { return SpinResult::Ok; } @@ -1111,6 +1252,104 @@ mod tests { } } + #[test] + fn stream_setup_timeout_reaps_unnegotiated_stream() { + let mut pair = PeerPair::new(); + let mut client_h = PeerHarness::new(); + + let t0 = Instant::now(); + pair.client_peer.open_stream(StreamProtocol::Ping).unwrap(); + + let PeerPair { client_ep, client_peer, .. } = &mut pair; + let mut cb = |h, e| client_ep.handle_event(h, e); + let closed = Cell::new(0usize); + let mut on_event = |e: NetEvent| { + if matches!(e, NetEvent::StreamClosed { .. }) { + closed.set(closed.get() + 1); + } + }; + + client_peer.spin(t0, &mut cb, &mut client_h.context, &mut on_event, &FxHashSet::default()); + assert_eq!(client_peer.streams.len(), 1); + + client_peer.spin( + t0 + STREAM_SETUP_TIMEOUT / 2, + &mut cb, + &mut client_h.context, + &mut on_event, + &FxHashSet::default(), + ); + assert_eq!(client_peer.streams.len(), 1); + assert_eq!(closed.get(), 0); + + client_peer.spin( + t0 + STREAM_SETUP_TIMEOUT + Duration::from_secs(1), + &mut cb, + &mut client_h.context, + &mut on_event, + &FxHashSet::default(), + ); + assert!(client_peer.streams.is_empty()); + assert_eq!(closed.get(), 1); + } + + /// A negotiated inbound RPC stream whose request never arrives must be + /// reaped by `INBOUND_RPC_TIMEOUT` — only the server is spun past the + /// deadline, so the reap can't be masked by the client's own teardown. + #[test] + fn inbound_rpc_timeout_reaps_unanswered_stream() { + use silver_common::{RpcInbound, RpcOutbound, RpcRequest, RpcRequestOutbound}; + + let mut pair = PeerPair::new(); + let mut client_h = PeerHarness::new(); + let mut server_h = PeerHarness::new(); + + let t0 = Instant::now(); + let request = RpcOutbound::Request(RpcRequestOutbound { + application_id: 7, + peer: pair.client_peer.id.connection, + request: RpcRequest::Ping([0u8; 8]), + }); + let msg = AcquiredRpcOutbound::from((request, &mut client_h.context.rpc_consumer)); + assert!(matches!(pair.client_peer.send_rpc(msg), SendResult::Ok)); + + // Step until the server has read the request; nothing ever responds, + // so its stream parks in `WriteResponse(Idle)` — the leak shape. + let mut server_got_request = false; + for _ in 0..300 { + let mut noop_c = |_: NetEvent| {}; + let mut scb = |e: NetEvent| { + if matches!(e, NetEvent::RpcInbound(RpcInbound::Request(_))) { + server_got_request = true; + } + }; + pair.step(t0, &mut client_h, &mut server_h, &mut noop_c, &mut scb); + if server_got_request { + break; + } + } + assert!(server_got_request, "server never read the ping request"); + assert!( + pair.server_peer.streams.values().any(|s| s.p2p_id.protocol() == StreamProtocol::Ping), + "server should hold the unanswered inbound ping stream" + ); + + let PeerPair { server_ep, server_peer, .. } = &mut pair; + let mut cb = |h, e| server_ep.handle_event(h, e); + let mut on_event = |_: NetEvent| {}; + server_peer.spin( + t0 + INBOUND_RPC_TIMEOUT + Duration::from_secs(1), + &mut cb, + &mut server_h.context, + &mut on_event, + &FxHashSet::default(), + ); + assert!( + !server_peer.streams.values().any(|s| s.p2p_id.protocol() == StreamProtocol::Ping), + "unanswered inbound rpc stream survived its timeout" + ); + } + /// Negotiation completion is observed indirectly by sending a tiny /// payload and waiting for it to arrive — only possible after the /// gossip-write state machine has crossed out of `NegotiateState`. diff --git a/crates/network/src/p2p/streams/rpc/request_in.rs b/crates/network/src/p2p/streams/rpc/request_in.rs index 94ccaa15..b17eec56 100644 --- a/crates/network/src/p2p/streams/rpc/request_in.rs +++ b/crates/network/src/p2p/streams/rpc/request_in.rs @@ -66,6 +66,13 @@ impl RpcReadRequest { if buf[pos] & 0x80 == 0 { // last byte of varint. let (length, offset) = decode_varint(&buf[..read], 0)?; + if length == 0 { + tracing::warn!( + ?p2p_id, + prefix = ?&buf[..read], + "rpc request chunk with zero varint length" + ); + } return Ok(Spin::Next(Self::AllocBody { length: length as usize, buf, @@ -119,8 +126,14 @@ impl RpcReadRequest { if written == 0 { return Ok(Spin::Ok(Self::ReadingBody { reservation, remaining })); } - remaining -= + let decoded = decoder.decompress_written(written, reservation.remaining_buffer()?)?; + // Advance the reservation past the decoded bytes — a + // tcache-backed request (by-root) only commits via this, + // and without the commit the consumer acquires an empty + // buffer. + reservation.increment_offset(decoded)?; + remaining -= decoded; } Ok(Spin::Next(Self::ReadingBody { reservation, remaining })) } @@ -128,3 +141,101 @@ impl RpcReadRequest { } } } + +#[cfg(test)] +mod tests { + use std::net::SocketAddr; + + use quinn_proto::StreamId; + use silver_common::{StreamProtocol, TCache, TCacheProducer, TRead}; + + use super::*; + use crate::p2p::streams::{ + StreamIo, + rpc::AcquiredRpcOutbound, + snappy::{SnappyDecoder, SnappyEncoder}, + }; + + /// Serves a fixed byte stream in `cap`-sized reads; sinks writes. + struct WireIo { + data: Vec, + pos: usize, + cap: usize, + } + + impl StreamIo for WireIo { + fn write_to_stream(&mut self, _id: StreamId, data: &[u8]) -> Result { + Ok(data.len()) + } + + fn read_from_stream( + &mut self, + _id: StreamId, + out: &mut [u8], + ) -> Result { + let n = out.len().min(self.cap).min(self.data.len() - self.pos); + out[..n].copy_from_slice(&self.data[self.pos..self.pos + n]); + self.pos += n; + Ok(n) + } + + fn close_write(&mut self, _id: StreamId) -> Result<(), StreamError> { + Ok(()) + } + + fn rpc_next(&mut self) -> Option { + None + } + + fn gossip_next(&mut self) -> Option { + None + } + + fn remote_addr(&self) -> SocketAddr { + "127.0.0.1:0".parse().unwrap() + } + } + + /// Regression: a by-root request body flows through `ReadingBody` into a + /// tcache reservation; without `increment_offset` per decode the + /// reservation never commits and the consumer acquires an empty buffer — + /// live symptom was count=0 for every by-root request from every client. + #[test] + fn by_root_request_body_commits_tcache() { + let mut ssz = Vec::new(); + ssz.extend_from_slice(&[0xAB; 32]); + ssz.extend_from_slice(&[0xCD; 32]); + + let mut body = Vec::new(); + let mut enc = SnappyEncoder::new(); + let (consumed, pending) = enc.compress(&ssz, &mut body).unwrap(); + assert_eq!((consumed, pending), (ssz.len(), 0)); + let mut wire = vec![ssz.len() as u8]; // single-byte varint (64) + wire.extend_from_slice(&body); + + let mut producer = TCache::producer("test_rpc_by_root_req", 1 << 16); + let mut consumer = + producer.cache_ref().random_access("test_rpc_by_root_req", false).unwrap(); + let p2p_id = P2pStreamId::new(0, 16, StreamProtocol::BeaconBlocksByRoot, true); + + // Full-buffer reads, byte-by-byte, and odd-sized. + for cap in [usize::MAX, 1, 7] { + let mut dec = SnappyDecoder::default(); + let mut io = WireIo { data: wire.clone(), pos: 0, cap }; + let mut state = RpcReadRequest::default(); + let mut spins = 0; + let msg = loop { + state = state.spin(&mut io, &p2p_id, &mut producer, &mut dec).unwrap(); + if let RpcReadRequest::Complete { msg } = state { + break msg; + } + spins += 1; + assert!(spins < 1000, "cap {cap}: state machine did not complete"); + }; + let RpcRequest::BlockByRoot(read) = msg else { panic!("wrong request kind") }; + let acquired = consumer.acquire(read); + let (buf, _) = acquired.buffer().unwrap(); + assert_eq!(buf, &ssz[..], "cap {cap}: committed body mismatch"); + } + } +} diff --git a/crates/network/src/p2p/streams/state.rs b/crates/network/src/p2p/streams/state.rs index 09354348..0cf78cd5 100644 --- a/crates/network/src/p2p/streams/state.rs +++ b/crates/network/src/p2p/streams/state.rs @@ -166,6 +166,16 @@ impl StreamState { } } + pub fn in_setup(&self) -> bool { + matches!( + self, + StreamState::Negotiate(_) | + StreamState::OutgoingIdentify(_) | + StreamState::IncomingIdentify(_) | + StreamState::OutgoingRpc { rpc: RpcOut::WriteRequest(_), .. } + ) + } + pub fn is_receive_only(&self, protocol: StreamProtocol) -> bool { match self { StreamState::Negotiate(state) => matches!(state, NegotiateState::OutReading { .. }), diff --git a/crates/network/src/tile.rs b/crates/network/src/tile.rs index ff671f9c..13176ac5 100644 --- a/crates/network/src/tile.rs +++ b/crates/network/src/tile.rs @@ -84,7 +84,7 @@ impl NetworkTile { let addr = enr.quic4_socket().or(enr.quic6_socket()); if let Some(addr) = addr { crate::NetworkCounters::DialAttempts.inc(); - tracing::info!(peer_id=?p2p, ?addr, "dialling p2p peer"); + tracing::debug!(peer_id=?p2p, ?addr, "dialling p2p peer"); if let Err(e) = self.inner.p2p_endpoint.connect(p2p, addr, now) { tracing::error!(?e, ?p2p, ?addr, "failed to initiate p2p to peer"); } @@ -200,12 +200,13 @@ impl NetworkTile { }; match result { p2p::SendResult::Ok => {} - p2p::SendResult::StreamCreationError => { + p2p::SendResult::StreamCreationError | p2p::SendResult::StreamGone => { producers.peer_events.produce( &(PeerEvent::P2pCannotCreateStream { p2p_peer: msg.peer_id(), protocol: msg.protocol(), rpc_request, + stream_gone: matches!(result, p2p::SendResult::StreamGone), } .into()), ); @@ -222,7 +223,7 @@ impl NetworkTile { } p2p::SendResult::UnknownPeer => { // Can happen if peer has disconnected. - tracing::warn!(peer=msg.peer_id(), protocol=?msg.protocol(), "Tried to send to unknown peer"); + tracing::debug!(peer=msg.peer_id(), protocol=?msg.protocol(), "Tried to send to unknown peer"); }, } }) { diff --git a/crates/peer/src/lib.rs b/crates/peer/src/lib.rs index 035c4886..2d810399 100644 --- a/crates/peer/src/lib.rs +++ b/crates/peer/src/lib.rs @@ -26,5 +26,13 @@ silver_common::declare_counters! { GossipInvalidFrame, GossipInvalidControl, GossipInvalidMsg, + // Mesh churn direction. + MeshPrunedByRemote, + MeshPrunedByUs, + MeshGraftAcceptedByUs, + MeshGraftRefusedByUs, + // "cannot create stream" split by cause. + StreamCreditExhausted, + ResponseStreamGone, } } diff --git a/crates/peer/src/manager.rs b/crates/peer/src/manager.rs index ffefdf31..df623bb9 100644 --- a/crates/peer/src/manager.rs +++ b/crates/peer/src/manager.rs @@ -10,11 +10,10 @@ use std::{ use flux_profiler::timed; use fxhash::FxHashSet; -use rand::seq::SliceRandom; use silver_common::{ - AgentString, BlockSource, Enr, GossipMsgOut, GossipTopic, IpBytes, MessageId, Nanos, P2pSend, - PeerControl, PeerEvent, PeerId, PeerScores, PeerStatus, PeerTopicScores, RpcRequest, - RpcRequestOutbound, RpcSeverity, StreamProtocol, SyncUpdate, TCacheRead, + AgentString, BlockSource, Enr, GossipTopic, IpBytes, P2pSend, PeerControl, PeerEvent, PeerId, + PeerScores, PeerStatus, PeerTopicScores, RpcRequest, RpcRequestOutbound, RpcSeverity, + StreamProtocol, SyncUpdate, rpc_rate_limit::RpcRateLimit, ssz_view::{METADATA_SIZE, STATUS_V2_SIZE, StatusView}, }; @@ -26,8 +25,11 @@ use crate::{ state::{ArchivedState, IpPrefix, MsgIdMap, PeerState, TopicScore}, }; +mod gossip; pub(crate) mod rpc; +use gossip::RecentDelivery; + /// Initial capacity hints — chosen so normal steady-state activity doesn't /// rehash. Undersizing is fine correctness-wise; this is a perf nudge. const PEERS_CAP: usize = 256; @@ -45,42 +47,6 @@ const SHORT_LIVED_CONNECTION: Duration = Duration::from_secs(30); const SHORT_LIVED_DIAL_BACKOFF: Duration = Duration::from_secs(15 * 60); const GOODBYE_CLIENT_SHUTDOWN: u64 = 1; -const MESH_MESSAGE_DELIVERIES_WINDOW_NS: u64 = 2_000_000_000; -const MESH_RETAIN_SCORES: usize = 4; -const OPPORTUNISTIC_GRAFT_INTERVAL: Duration = Duration::from_secs(60); -const OPPORTUNISTIC_GRAFT_PEERS: usize = 2; - -struct RecentDelivery { - topic: GossipTopic, - received_at: Nanos, - first_credited_peer: Option, - additional_credited_peers: Vec, -} - -impl RecentDelivery { - fn new(topic: GossipTopic, received_at: Nanos, credited_peer: Option) -> Self { - Self { - topic, - received_at, - first_credited_peer: credited_peer, - additional_credited_peers: Vec::new(), - } - } - - fn credit(&mut self, peer_id: PeerId) -> bool { - if self.first_credited_peer == Some(peer_id) || - self.additional_credited_peers.contains(&peer_id) - { - return false; - } - if self.first_credited_peer.is_none() { - self.first_credited_peer = Some(peer_id); - } else { - self.additional_credited_peers.push(peer_id); - } - true - } -} pub struct PeerManager { local_peer_id: PeerId, @@ -106,7 +72,8 @@ pub struct PeerManager { /// peers and mesh-management decisions. our_topics: Vec, - /// Our mesh per topic: connections we've grafted onto. Bounded by d_high. + /// Our mesh per topic: connections we've grafted onto. May exceed d_high + /// between heartbeats; trimmed back to d by `ensure_mesh_capped`. mesh: HashMap>, /// Outstanding IHAVE→IWANT promises, keyed by `MessageId`. Each entry @@ -474,44 +441,6 @@ impl PeerManager { pub fn live_peers_with_status(&self) -> impl Iterator { self.database.live_peers_with_status() } - - /// Announce our topic subscriptions to every currently-connected peer. - /// Add topics at runtime (deferred long-lived subnets): extends - /// `our_topics` + mesh bookkeeping + subnet masks, and announces - /// SUBSCRIBE to every connected peer. New connections pick the - /// topics up via the normal `on_connected` fan-out. - pub fn activate_topics(&mut self, topics: &[GossipTopic], emit: &mut impl FnMut(PeerControl)) { - for &topic in topics { - if self.our_topics.contains(&topic) { - continue; - } - self.our_topics.push(topic); - self.mesh.insert(topic, Vec::with_capacity(self.params.d_high as usize)); - for (&conn, peer) in &self.peers { - emit(PeerControl::P2pGossipSubscribe { - p2p: peer.peer_id, - p2p_connection: conn, - topic, - }); - } - } - let (attnets, syncnets) = build_subnet_masks(&self.our_topics); - self.required_attnets = attnets; - self.required_syncnets = syncnets; - } - - pub fn fan_out_subscriptions(&mut self, emit: &mut impl FnMut(PeerControl)) { - for (&conn, peer) in &self.peers { - for &topic in &self.our_topics { - emit(PeerControl::P2pGossipSubscribe { - p2p: peer.peer_id, - p2p_connection: conn, - topic, - }); - } - } - } - pub fn status(&self) -> Option<&[u8; STATUS_V2_SIZE]> { self.status.as_ref() } @@ -541,12 +470,6 @@ impl PeerManager { }) } - /// Mesh size for a topic (for tests/introspection). - #[allow(dead_code)] - pub(crate) fn mesh_size(&self, topic: GossipTopic) -> usize { - self.mesh.get(&topic).map(|m| m.len()).unwrap_or(0) - } - /// Size of the archive set. #[allow(dead_code)] pub(crate) fn archived_count(&self) -> usize { @@ -562,7 +485,6 @@ impl PeerManager { ) { match event { PeerEvent::P2pNewConnection { p2p_peer_id, peer_id_full, ip, port, local_dial } => { - tracing::info!("New p2p peer: {ip:?}:{port}, local dial? {local_dial}"); self.on_connected(p2p_peer_id, peer_id_full, ip, port, now, emit, local_dial); } PeerEvent::P2pDisconnect { p2p_peer, peer_id } => { @@ -575,21 +497,32 @@ impl PeerManager { self.database.dial_failed(&peer_id, now + DIAL_FAILURE_BACKOFF); } } - PeerEvent::P2pCannotCreateStream { p2p_peer, protocol, rpc_request } => { + PeerEvent::P2pCannotCreateStream { p2p_peer, protocol, rpc_request, stream_gone } => { if rpc_request { self.release_outbound_in_flight(p2p_peer, protocol); } - self.add_behaviour_penalty(p2p_peer, 1.0, "cannot create stream"); + let offence = if stream_gone { + crate::PeerCounters::ResponseStreamGone.inc(); + "response stream gone" + } else { + crate::PeerCounters::StreamCreditExhausted.inc(); + "stream credit exhausted" + }; + self.add_behaviour_penalty(p2p_peer, 1.0, offence); } PeerEvent::P2pOutboundMessageDropped { p2p_peer, protocol, rpc_request } => { + // Local outbound-ring overflow — a backpressure signal, often + // ours (blocked socket), not peer misbehaviour. No P7: a + // stalled connection drops in bursts and the squared penalty + // would graylist the whole mesh on a local uplink stall. if rpc_request { self.release_outbound_in_flight(p2p_peer, protocol); } - self.add_behaviour_penalty(p2p_peer, 1.0, "outbound message dropped"); + tracing::debug!(p2p_peer, ?protocol, rpc_request, "outbound message dropped"); } PeerEvent::P2pStreamClosed { stream_id } => { - // Premature close on a request-response stream — the - // peer FIN'd or RST'd before the response terminator + // Premature close on an outgoing request-response stream — + // the peer FIN'd or RST'd before the response terminator // (`Complete`/`Error`) was observed. `MidTolerance` // accumulates a signal without fast-banning over a // single flaky session. Gossip / identity streams don't @@ -597,26 +530,32 @@ impl PeerManager { // means multistream-select hadn't negotiated yet, also // skipped (the close there is a protocol-negotiation // failure, distinct from a premature RPC termination). + // Incoming closes are exempt: that's the requester's + // read-timeout giving up on our slow response — penalising + // it blames the wrong side. let protocol = stream_id.protocol(); if protocol.is_request_response() && protocol != StreamProtocol::Unset { - tracing::warn!(?protocol, "stream close misbehaviour"); - self.on_rpc_misbehaviour( - stream_id.peer(), - RpcSeverity::MidTolerance, - "premature rpc stream close", + tracing::warn!( + ?protocol, + incoming = stream_id.is_incoming(), + "stream close misbehaviour" ); - // No terminal response will arrive for this stream — - // release the outbound in-flight slot, else each - // abnormal close permanently burns one of the peer's - // `MAX_RPC_PROTOCOL_IN_FLIGHT` slots and the protocol - // goes dark for the connection's lifetime. - if !stream_id.is_incoming() && - let Some(peer) = self.peers.get_mut(&stream_id.peer()) - { - peer.outbound_in_flight[protocol.ordinal() as usize] = - peer.outbound_in_flight[protocol.ordinal() as usize].saturating_sub(1); - } if !stream_id.is_incoming() { + self.on_rpc_misbehaviour( + stream_id.peer(), + RpcSeverity::MidTolerance, + "premature rpc stream close", + ); + // No terminal response will arrive for this stream — + // release the outbound in-flight slot, else each + // abnormal close permanently burns one of the peer's + // `MAX_RPC_PROTOCOL_IN_FLIGHT` slots and the protocol + // goes dark for the connection's lifetime. + if let Some(peer) = self.peers.get_mut(&stream_id.peer()) { + peer.outbound_in_flight[protocol.ordinal() as usize] = peer + .outbound_in_flight[protocol.ordinal() as usize] + .saturating_sub(1); + } self.on_range_stream_closed(stream_id.peer(), protocol, emit); } } @@ -686,6 +625,29 @@ impl PeerManager { // TODO recv_ts elapsed metric self.on_send_gossip(originator_stream_id.peer(), msg_hash, topic, protobuf, emit); } + PeerEvent::RpcServeOutcome { + p2p_peer, + protocol, + units_total, + units_sent, + missing, + first_chunk_ms, + elapsed_ms, + } => { + let user_agent = + self.peers.get(&p2p_peer).map(|p| p.user_agent).unwrap_or_default(); + tracing::info!( + p2p_peer, + ?protocol, + units_total, + units_sent, + missing, + first_chunk_ms, + elapsed_ms, + user_agent = user_agent.as_str(), + "rpc serve outcome" + ); + } PeerEvent::RpcMisbehaviour { p2p_peer, severity } => { self.on_rpc_misbehaviour(p2p_peer, severity, "rpc chunk/framing violation"); } @@ -890,7 +852,6 @@ impl PeerManager { None }; - tracing::info!("adding peer with p2p connection: {conn}"); self.peers.insert(conn, state); self.peers_by_id.insert(peer_id, conn); self.database.add_peer_id(peer_id, conn); @@ -989,370 +950,6 @@ impl PeerManager { self.database.peer_disconnected(conn) } - // ── Gossip event handlers ─────────────────────────────────────────── - - #[timed] - fn on_subscribe( - &mut self, - conn: usize, - topic: GossipTopic, - now: Instant, - emit: &mut impl FnMut(PeerControl), - ) { - let (peer_id, score) = { - let Some(peer) = self.peers.get_mut(&conn) else { - return; - }; - peer.topics.insert(topic); - (peer.peer_id, peer.cached_score) - }; - - let we_want = self.our_topics.contains(&topic); - let mesh_size = self.mesh.get(&topic).map(|m| m.len()).unwrap_or(0); - tracing::debug!(p2p_peer = conn, ?topic, we_want, mesh_size, "PM peer subscribed"); - - // Opportunistic graft: if this is a topic we care about and our mesh - // is below d_low, pull the peer in. - if we_want && - mesh_size < self.params.d_low as usize && - score >= 0.0 && - !self.is_backed_off(conn, topic, now) - { - self.do_graft(conn, peer_id, topic, now, emit); - } - } - - #[timed] - fn on_unsubscribe( - &mut self, - conn: usize, - topic: GossipTopic, - _now: Instant, - emit: &mut impl FnMut(PeerControl), - ) { - let peer_id = match self.peers.get_mut(&conn) { - Some(p) => { - p.topics.remove(&topic); - p.peer_id - } - None => return, - }; - tracing::debug!(p2p_peer = conn, ?topic, "PM peer unsubscribed"); - // If peer was in our mesh, remove them. - if self.leave_mesh(conn, topic) { - emit(PeerControl::P2pGossipPrune { p2p: peer_id, p2p_connection: conn, topic }); - } - } - - #[timed] - fn on_remote_graft( - &mut self, - conn: usize, - topic: GossipTopic, - now: Instant, - emit: &mut impl FnMut(PeerControl), - ) { - let Some(peer) = self.peers.get(&conn) else { - if let Some(record) = self.database.by_p2p_id(conn) && - let Some(id) = record.peer_id - { - emit(PeerControl::P2pDisconnect { p2p: id, p2p_connection: conn }) - } - return; - }; - let peer_id = peer.peer_id; - let score = peer.cached_score; - let mesh_size = self.mesh.get(&topic).map(|m| m.len()).unwrap_or(0); - let accept = self.our_topics.contains(&topic) && - mesh_size < self.params.d_high as usize && - score >= 0.0 && - !self.is_backed_off(conn, topic, now); - if accept { - self.do_graft(conn, peer_id, topic, now, emit); - tracing::debug!(p2p_peer = conn, ?topic, mesh_size, "PM peer GRAFTed us: accepted"); - } else { - self.do_prune(conn, peer_id, topic, now, emit); - tracing::debug!(p2p_peer = conn, ?topic, mesh_size, "PM peer GRAFTed us: refused"); - } - } - - #[timed] - fn on_remote_prune( - &mut self, - conn: usize, - topic: GossipTopic, - now: Instant, - backoff_seconds: Option, - emit: &mut impl FnMut(PeerControl), - ) { - self.leave_mesh(conn, topic); - let mesh_size = self.mesh.get(&topic).map(|peers| peers.len()).unwrap_or(0); - if !self.peers.contains_key(&conn) { - if let Some(record) = self.database.by_p2p_id(conn) && - let Some(id) = record.peer_id - { - emit(PeerControl::P2pDisconnect { p2p: id, p2p_connection: conn }) - } - return; - } - let backoff = backoff_seconds.map(Duration::from_secs).unwrap_or(self.params.prune_backoff); - self.set_backoff(conn, topic, now, backoff); - tracing::debug!(p2p_peer = conn, ?topic, mesh_size, "PM peer PRUNEd us"); - } - - fn on_ihave(&mut self, conn: usize, hash: MessageId, already_seen: bool, now: Instant) { - // Always count, regardless of dedup state — flood detection treats - // a peer IHAVEing thousands of ids we already have just as badly as - // ids we don't. - let (over_cap, should_iwant) = { - let Some(peer) = self.peers.get_mut(&conn) else { - return; - }; - peer.ihaves_received = peer.ihaves_received.saturating_add(1); - let over_cap = peer.ihaves_received > self.params.max_ihave_length; - // Only send an IWANT (and thus track a promise) if: - // - we don't already have the message, - // - we haven't exceeded the per-heartbeat IWANT budget, - // - the peer hasn't saturated the IHAVE rate cap, - // - the peer clears the gossip threshold (`on_outbound_iwant` drops the frame - // below it — a promise without a sent IWANT can only ever expire). - let should_iwant = !already_seen && - !over_cap && - peer.cached_score >= self.params.gossip_threshold && - peer.iwant_ids_sent < self.params.max_ihave_length; - if should_iwant { - peer.iwant_ids_sent = peer.iwant_ids_sent.saturating_add(1); - } - (over_cap, should_iwant) - }; - if over_cap { - self.add_behaviour_penalty(conn, 1.0, "ihave rate limit exceeded"); - return; - } - if !should_iwant { - return; - } - // Record the promise globally. Dedupe: same peer IHAVEing the same - // id twice is one outstanding promise, not two. - let deadline = now + self.params.iwant_followup; - let entry = self.promises.entry(hash).or_default(); - if !entry.iter().any(|(c, _)| *c == conn) { - entry.push((conn, deadline)); - } - } - - /// Peer sent us an IWANT that hit our mcache. Check retransmission - /// threshold and apply the score gate. - #[timed] - fn on_iwant_received( - &mut self, - conn: usize, - hash: MessageId, - tcache: TCacheRead, - emit: &mut impl FnMut(PeerControl), - ) { - let Some(peer) = self.peers.get_mut(&conn) else { - return; - }; - if peer.msg_cache_insert(hash) > 2 { - // exceeds retransmission threshold - return; - } - if peer.cached_score < self.params.gossip_threshold { - return; - } - emit(PeerControl::P2pSend(P2pSend::Gossip(GossipMsgOut { peer_id: conn, tcache }))); - } - - /// Peer sent us an IDONTWANT - store the message id in the peer message - /// cache. - fn on_idontwant_received(&mut self, conn: usize, hash: MessageId) { - let Some(peer) = self.peers.get_mut(&conn) else { - return; - }; - peer.msg_cache_insert(hash); - } - - fn credit_mesh_delivery(&mut self, conn: usize, topic: GossipTopic) -> Option { - if !self.mesh.get(&topic).is_some_and(|mesh| mesh.contains(&conn)) { - return None; - } - let peer = self.peers.get_mut(&conn)?; - peer.topic_stats.entry(topic).or_default().mesh_deliveries += 1.0; - Some(peer.peer_id) - } - - fn on_gossip_duplicate( - &mut self, - conn: usize, - topic: GossipTopic, - hash: MessageId, - recv_ts: Nanos, - ) { - self.promises.remove(&hash); - if !self.mesh.get(&topic).is_some_and(|mesh| mesh.contains(&conn)) { - return; - } - let Some(peer_id) = self.peers.get(&conn).map(|peer| peer.peer_id) else { - return; - }; - let Some(delivery) = self.recent_deliveries.get_mut(&hash) else { - return; - }; - if delivery.topic != topic || - recv_ts.0.saturating_sub(delivery.received_at.0) > MESH_MESSAGE_DELIVERIES_WINDOW_NS || - !delivery.credit(peer_id) - { - return; - } - if let Some(peer) = self.peers.get_mut(&conn) { - peer.topic_stats.entry(topic).or_default().mesh_deliveries += 1.0; - } - } - - /// A fully-validated inbound gossip message arrived — this is the first - /// (dedup-clean) delivery from any peer. Clear all promises for this id - /// (every peer who IHAVE'd it kept their word, regardless of who - /// actually reached us first), credit P2/P3 on the delivering peer, and - /// fan out the pre-encoded IDONTWANT frame to every mesh peer except - /// the sender so they stop racing this id toward us. - #[timed] - fn on_new_gossip( - &mut self, - sender_conn: usize, - topic: GossipTopic, - msg_hash: MessageId, - recv_ts: Nanos, - idontwant: TCacheRead, - emit: &mut impl FnMut(PeerControl), - ) { - crate::counters::GossipTopicCounters::recv(topic); - - // Any peer who promised this id is released — they did their job; - // we just got another copy from someone else first. - self.promises.remove(&msg_hash); - - if let Some(peer) = self.peers.get_mut(&sender_conn) { - let t = peer.topic_stats.entry(topic).or_default(); - // P2 — first-delivery credit (capped + weighted in `compute_score`). - t.first_deliveries += 1.0; - } - - let credited_peer = self.credit_mesh_delivery(sender_conn, topic); - if scoring::p3_scored(&topic) { - self.recent_deliveries - .insert(msg_hash, RecentDelivery::new(topic, recv_ts, credited_peer)); - } - - // Fan IDONTWANT out to mesh members (except sender) above threshold. - let Some(mesh_peers) = self.mesh.get(&topic) else { - return; - }; - for conn in mesh_peers { - if *conn == sender_conn { - continue; - } - let Some(peer) = self.peers.get(conn) else { - continue; - }; - if peer.cached_score < self.params.gossip_threshold { - continue; - } - emit(PeerControl::P2pSend(P2pSend::Gossip(GossipMsgOut { - peer_id: *conn, - tcache: idontwant, - }))); - } - } - - /// Compression tile has prepared a batched IHAVE frame for `topic`. - /// Fan it out: one `P2pGossipSend` per non-mesh subscriber whose score - /// clears `gossip_threshold`, capped at `d_lazy`. - #[timed] - fn on_outbound_ihave( - &mut self, - topic: GossipTopic, - protobuf: TCacheRead, - emit: &mut impl FnMut(PeerControl), - ) { - let mesh_for_topic = self.mesh.get(&topic); - let cap = self.params.d_lazy as usize; - let mut emitted = 0usize; - for (conn, peer) in &self.peers { - if emitted >= cap { - break; - } - if !peer.topics.contains(&topic) { - continue; - } - if mesh_for_topic.is_some_and(|m| m.contains(conn)) { - continue; // mesh peers get full-body forwards, not IHAVE - } - if peer.cached_score < self.params.gossip_threshold { - continue; - } - emit(PeerControl::P2pSend(P2pSend::Gossip(GossipMsgOut { - peer_id: *conn, - tcache: protobuf, - }))); - emitted += 1; - } - } - - /// Compression tile has prepared an IWANT frame for a peer that just - /// sent us IHAVE. Forward it to the network tile provided the peer is - /// still live and scoring above `gossip_threshold` (mirrors rust-libp2p, - /// which ignores IHAVE — and therefore doesn't send the IWANT reply — - /// for peers below that threshold). - #[timed] - fn on_outbound_iwant( - &mut self, - conn: usize, - tcache: TCacheRead, - emit: &mut impl FnMut(PeerControl), - ) { - let Some(peer) = self.peers.get(&conn) else { - return; - }; - if peer.cached_score < self.params.gossip_threshold { - return; - } - emit(PeerControl::P2pSend(P2pSend::Gossip(GossipMsgOut { peer_id: conn, tcache }))); - } - - #[timed] - fn on_send_gossip( - &mut self, - sender: usize, - msg_hash: MessageId, - topic: GossipTopic, - tcache: TCacheRead, - emit: &mut impl FnMut(PeerControl), - ) { - let Some(meshed_peers) = self.mesh.get(&topic) else { - return; - }; - for peer in meshed_peers { - let Some(peer_state) = self.peers.get_mut(peer) else { - continue; - }; - peer_state.topic_stats.entry(topic).or_default().fanout_total += 1; - if *peer == sender { - continue; - } - if peer_state.cached_score < self.params.gossip_threshold { - continue; - } - if peer_state.msg_cache_contains(&msg_hash) { - // dontwant - continue; - } - peer_state.topic_stats.entry(topic).or_default().fanout_sent += 1; - crate::counters::GossipTopicCounters::sent(topic); - emit(PeerControl::P2pSend(P2pSend::Gossip(GossipMsgOut { peer_id: *peer, tcache }))); - } - } - fn ban_disc_peer(&mut self, peer_id: PeerId, now: Instant, emit: &mut impl FnMut(PeerControl)) { if self.banned_peers.insert(peer_id, now).is_none() { crate::PeerCounters::PeersBanned.inc(); @@ -1510,14 +1107,6 @@ impl PeerManager { self.last_discovery = now; emit(PeerControl::DiscoverNodes); } - - fn add_invalid_delivery(&mut self, conn: usize, topic: GossipTopic) { - if let Some(peer) = self.peers.get_mut(&conn) { - let t = peer.topic_stats.entry(topic).or_default(); - t.invalid_deliveries += 1.0; - } - } - /// A request admitted by `admit_outbound_request` never reached the wire: /// no response terminator or stream close will ever fire for it, so the /// in-flight slot must be released here or it leaks until disconnect @@ -1544,6 +1133,7 @@ impl PeerManager { offence, delta, total = peer.behaviour_penalty, + user_agent = peer.user_agent.as_str(), "P7 behaviour penalty" ); } @@ -1622,21 +1212,17 @@ impl PeerManager { let Some(peer_id) = peer_record.peer_id { let user_agent = peer_record.identify.as_ref().map(|i| i.user_agent()); + let backoff = Self::goodbye_dial_backoff(code); tracing::info!( p2p_peer, code = Self::goodbye_reason(code), ?user_agent, + ?backoff, "received goodbye" ); emit(PeerControl::P2pDisconnect { p2p: peer_id, p2p_connection: p2p_peer }); - if let Some(backoff) = Self::goodbye_dial_backoff(code) { - tracing::info!( - ?peer_id, - code = Self::goodbye_reason(code), - ?backoff, - "goodbye; dial backoff" - ); + if let Some(backoff) = backoff { self.remote_banned_peers.insert(peer_id, now + backoff); } } @@ -1676,150 +1262,17 @@ impl PeerManager { // ── Internal helpers ──────────────────────────────────────────────── - fn is_backed_off(&self, conn: usize, topic: GossipTopic, now: Instant) -> bool { - let Some(deadline) = self.peers.get(&conn).and_then(|p| p.backoffs.get(&topic)) else { - return false; - }; - deadline - .checked_add(self.params.heartbeat_interval) - .map_or(now < *deadline, |deadline_with_slack| now < deadline_with_slack) - } - - fn set_backoff(&mut self, conn: usize, topic: GossipTopic, now: Instant, backoff: Duration) { - let Some(deadline) = now.checked_add(backoff) else { - tracing::warn!(p2p_peer = conn, ?topic, ?backoff, "ignoring oversized prune backoff"); - return; - }; - let Some(peer) = self.peers.get_mut(&conn) else { return }; - peer.backoffs - .entry(topic) - .and_modify(|current| *current = (*current).max(deadline)) - .or_insert(deadline); - } - - fn do_graft( - &mut self, - conn: usize, - peer_id: PeerId, - topic: GossipTopic, - now: Instant, - emit: &mut impl FnMut(PeerControl), - ) { - let mesh = self - .mesh - .entry(topic) - .or_insert_with(|| Vec::with_capacity(self.params.d_high as usize)); - if mesh.contains(&conn) { - return; - } - mesh.push(conn); - // Seed per-topic state so P3 tracking kicks in after grace window. - if let Some(peer) = self.peers.get_mut(&conn) { - let t = peer.topic_stats.entry(topic).or_default(); - t.meshed_since = Some(now); - t.mesh_active = false; - } - tracing::info!(?topic, conn, "GRAFT peer"); - emit(PeerControl::P2pGossipGraft { p2p: peer_id, p2p_connection: conn, topic }); - } - - fn do_prune( - &mut self, - conn: usize, - peer_id: PeerId, - topic: GossipTopic, - now: Instant, - emit: &mut impl FnMut(PeerControl), - ) { - self.leave_mesh(conn, topic); - self.set_backoff(conn, topic, now, self.params.prune_backoff); - emit(PeerControl::P2pGossipPrune { p2p: peer_id, p2p_connection: conn, topic }); - } - - fn leave_mesh(&mut self, conn: usize, topic: GossipTopic) -> bool { - let removed = if let Some(mesh) = self.mesh.get_mut(&topic) && - let Some(index) = mesh.iter().position(|peer| *peer == conn) - { - mesh.swap_remove(index); - true - } else { - false - }; + fn rescore_all(&mut self, now: Instant) { + // Snapshot colocation counts so we don't hold borrows across the + // mutation loop. + let peers_by_prefix: HashMap = + self.ip_colocations.iter().map(|(k, v)| (*k, v.len())).collect(); - if let Some(topic_score) = - self.peers.get_mut(&conn).and_then(|peer| peer.topic_stats.get_mut(&topic)) - { - let threshold = scoring::topic_params(&topic).p3_threshold; - if topic_score.mesh_active && topic_score.mesh_deliveries < threshold { - let deficit = threshold - topic_score.mesh_deliveries; - topic_score.mesh_failure_penalty += deficit * deficit; - } - topic_score.meshed_since = None; - topic_score.mesh_active = false; - } - - removed - } - - fn heartbeat(&mut self, now: Instant) { - // Reset per-heartbeat rate-limit counters on every live peer. - for peer in self.peers.values_mut() { - peer.ihaves_received = 0; - peer.iwant_ids_sent = 0; - } - - let recv_now = Nanos::now(); - self.recent_deliveries.retain(|_, delivery| { - recv_now.0.saturating_sub(delivery.received_at.0) <= MESH_MESSAGE_DELIVERIES_WINDOW_NS - }); - - // Sweep expired promises from the global map. Expired entries - // credit `behaviour_penalty` to the peer who promised but didn't - // come through (nor did anyone else for that id). - let mut penalties: HashMap = HashMap::new(); - self.promises.retain(|_hash, waiters| { - waiters.retain(|(conn, deadline)| { - if now >= *deadline { - *penalties.entry(*conn).or_insert(0) += 1; - false - } else { - true - } - }); - !waiters.is_empty() - }); - for (conn, _count) in penalties { - // TODO seem to be over eagerly banning people here - self.add_behaviour_penalty(conn, 1.0, "broken gossip promises"); - } - } - - fn activate_p3_where_due(&mut self, now: Instant) { - let activation = self.params.mesh_message_deliveries_activation_s; - for peer in self.peers.values_mut() { - for (topic, t) in peer.topic_stats.iter_mut() { - if scoring::p3_scored(topic) && - !t.mesh_active && - let Some(since) = t.meshed_since && - now.saturating_duration_since(since).as_secs_f64() >= activation - { - t.mesh_active = true; - } - } - } - } - - fn rescore_all(&mut self, now: Instant) { - // Snapshot colocation counts so we don't hold borrows across the - // mutation loop. - let peers_by_prefix: HashMap = - self.ip_colocations.iter().map(|(k, v)| (*k, v.len())).collect(); - - for peer in self.peers.values_mut() { - let coloc = *peers_by_prefix.get(&peer.ip_prefix).unwrap_or(&1); - peer.last_breakdown = scoring::score_breakdown(peer, &self.params, coloc, now); - peer.cached_score = peer.last_breakdown.total; - peer.score_valid_at = now; + for peer in self.peers.values_mut() { + let coloc = *peers_by_prefix.get(&peer.ip_prefix).unwrap_or(&1); + peer.last_breakdown = scoring::score_breakdown(peer, &self.params, coloc, now); + peer.cached_score = peer.last_breakdown.total; + peer.score_valid_at = now; } } @@ -1984,223 +1437,6 @@ impl PeerManager { } } - fn manage_mesh(&mut self, now: Instant, emit: &mut impl FnMut(PeerControl)) { - // Self-heal: a mesh entry with no live PeerState means a removal - // path skipped the mesh sweep (see the graylist-evict leak). It - // suppresses grafting via a phantom degree and, once quinn recycles - // the handle, mesh-pushes to a peer that never grafted. - let peers = &self.peers; - for (topic, mesh_peers) in self.mesh.iter_mut() { - mesh_peers.retain(|conn| { - let live = peers.contains_key(conn); - if !live { - tracing::warn!(conn, ?topic, "dropping mesh entry with no peer state"); - } - live - }); - } - - // Iterate over OUR topics (topics we care about). We briefly take - // the topic list so `ensure_mesh_*` can take `&mut self`. - let our_topics = std::mem::take(&mut self.our_topics); - let opportunistic_graft_due = now.saturating_duration_since(self.last_opportunistic_graft) >= - OPPORTUNISTIC_GRAFT_INTERVAL; - for topic in &our_topics { - self.prune_negative_mesh_peers(*topic, now, emit); - self.ensure_mesh_filled(*topic, now, emit); - self.ensure_mesh_capped(*topic, now, emit); - if opportunistic_graft_due { - self.opportunistic_graft(*topic, now, emit); - } - } - self.our_topics = our_topics; - if opportunistic_graft_due { - self.last_opportunistic_graft = now; - } - } - - fn prune_negative_mesh_peers( - &mut self, - topic: GossipTopic, - now: Instant, - emit: &mut impl FnMut(PeerControl), - ) { - let peers: Vec<_> = self - .mesh - .get(&topic) - .into_iter() - .flatten() - .filter_map(|conn| { - self.peers - .get(conn) - .filter(|peer| peer.cached_score < 0.0) - .map(|peer| (*conn, peer.peer_id)) - }) - .collect(); - for (conn, peer_id) in peers { - self.do_prune(conn, peer_id, topic, now, emit); - } - } - - fn ensure_mesh_filled( - &mut self, - topic: GossipTopic, - now: Instant, - emit: &mut impl FnMut(PeerControl), - ) { - let current = self.mesh.get(&topic).map(|m| m.len()).unwrap_or(0); - let d = self.params.d as usize; - if current >= d { - return; - } - let needed = d - current; - // Sort requires a buffer; the emit isn't what forces it. - let mut candidates: Vec = self - .peers - .iter() - .filter_map(|(conn, peer)| { - if !peer.topics.contains(&topic) { - return None; - } - if self.mesh.get(&topic).is_some_and(|m| m.contains(conn)) { - return None; - } - if peer.cached_score < 0.0 { - return None; - } - if self.is_backed_off(*conn, topic, now) { - return None; - } - Some(*conn) - }) - .collect(); - candidates.shuffle(&mut rand::thread_rng()); - for conn in candidates.into_iter().take(needed) { - let Some(peer_id) = self.peers.get(&conn).map(|p| p.peer_id) else { - continue; - }; - self.do_graft(conn, peer_id, topic, now, emit); - } - } - - fn ensure_mesh_capped( - &mut self, - topic: GossipTopic, - now: Instant, - emit: &mut impl FnMut(PeerControl), - ) { - let d_high = self.params.d_high as usize; - let d = self.params.d as usize; - let current = self.mesh.get(&topic).map(|m| m.len()).unwrap_or(0); - // Strictly above d_high (spec heartbeat rule): remote grafts are - // accepted only while below d_high, so a mesh sitting at the cap is - // steady state — firing at the cap turns every graft that fills the - // last slot into a prune wave. - if current <= d_high { - return; - } - let excess = current.saturating_sub(d); - if excess == 0 { - return; - } - // Victims come from settled members only: inside the activation - // window a member's score is noise, and an opportunistic graft would - // otherwise land straight in the eviction pool of the prune wave it - // triggers. Excess beyond the settled pool waits. - let settle = self.params.mesh_message_deliveries_activation_s; - // Sort requires a buffer; the emit isn't what forces it. - let mut ranked: Vec<(usize, f64, PeerId)> = self - .mesh - .get(&topic) - .map(|mesh| { - mesh.iter() - .filter_map(|conn| { - let p = self.peers.get(conn)?; - let since = p.topic_stats.get(&topic)?.meshed_since?; - (now.saturating_duration_since(since).as_secs_f64() >= settle).then_some(( - *conn, - scoring::selection_score(p, &topic, &self.params, now), - p.peer_id, - )) - }) - .collect() - }) - .unwrap_or_default(); - if ranked.is_empty() { - return; - } - let mut rng = rand::thread_rng(); - ranked.shuffle(&mut rng); - ranked.sort_by(|a, b| a.1.partial_cmp(&b.1).unwrap_or(std::cmp::Ordering::Equal)); - let retain = MESH_RETAIN_SCORES.min(d).min(ranked.len()); - let random_end = ranked.len() - retain; - ranked[..random_end].shuffle(&mut rng); - for (conn, _, peer_id) in ranked.into_iter().take(excess) { - self.do_prune(conn, peer_id, topic, now, emit); - } - } - - fn opportunistic_graft( - &mut self, - topic: GossipTopic, - now: Instant, - emit: &mut impl FnMut(PeerControl), - ) { - let Some(mesh) = self.mesh.get(&topic) else { return }; - if mesh.len() <= 1 { - return; - } - // Median over established members only: peers meshed for less than - // the P3 activation window score near zero structurally, so counting - // them reads a freshly-built mesh as underperforming and re-grafts - // (then prunes) before anyone has a chance to establish. - let activation = self.params.mesh_message_deliveries_activation_s; - let mut mesh_scores: Vec<_> = mesh - .iter() - .filter_map(|conn| { - let peer = self.peers.get(conn)?; - let since = peer.topic_stats.get(&topic)?.meshed_since?; - (now.saturating_duration_since(since).as_secs_f64() >= activation) - .then_some(scoring::selection_score(peer, &topic, &self.params, now)) - }) - .collect(); - if mesh_scores.len() <= 1 { - return; - } - mesh_scores.sort_by(|a, b| a.partial_cmp(b).unwrap_or(std::cmp::Ordering::Equal)); - let middle = mesh_scores.len() / 2; - let median = if mesh_scores.len().is_multiple_of(2) { - (mesh_scores[middle - 1] + mesh_scores[middle]) * 0.5 - } else { - mesh_scores[middle] - }; - if median >= self.params.opportunistic_graft_threshold { - return; - } - - let mut candidates: Vec<_> = self - .peers - .iter() - .filter_map(|(conn, peer)| { - if !peer.topics.contains(&topic) || - mesh.contains(conn) || - scoring::selection_score(peer, &topic, &self.params, now) <= median || - self.is_backed_off(*conn, topic, now) - { - return None; - } - Some(*conn) - }) - .collect(); - candidates.shuffle(&mut rand::thread_rng()); - for conn in candidates.into_iter().take(OPPORTUNISTIC_GRAFT_PEERS) { - let Some(peer_id) = self.peers.get(&conn).map(|peer| peer.peer_id) else { - continue; - }; - self.do_graft(conn, peer_id, topic, now, emit); - } - } - fn gc_archived(&mut self, now: Instant) { let ttl = self.params.archived_ttl; self.archived.retain(|_, a| now.saturating_duration_since(a.archived_at) < ttl); @@ -2380,12 +1616,12 @@ type _TopicSet = HashSet; type _TopicScoreAlias = TopicScore; #[cfg(test)] -mod tests { +pub(crate) mod tests { use std::time::Duration; use silver_common::{ BACKFILL_REQUEST_ID, BASE_REQUEST_ID, COLUMN_BACKFILL_REQUEST_ID, Enr, Identify, Keypair, - P2pStreamId, RpcInbound, RpcResponse, RpcResponseInbound, TCacheProducer, + P2pStreamId, RpcInbound, RpcResponse, RpcResponseInbound, ssz_view::BLOCKS_BY_RANGE_REQ_SIZE, }; @@ -2395,9 +1631,9 @@ mod tests { /// `cap.0` via an ad-hoc `|c| cap.0.push(c)` closure passed to /// `handle_event`/`tick`. #[derive(Default)] - struct Captured(Vec); + pub(crate) struct Captured(pub(crate) Vec); - fn fixture( + pub(crate) fn fixture( our_topics: Vec, params: ScoreParams, awaiting_replay: bool, @@ -2417,14 +1653,20 @@ mod tests { ) } - fn peer_id(seed: u8) -> PeerId { + pub(crate) fn peer_id(seed: u8) -> PeerId { let mut bytes = [0u8; 32]; bytes[0] = seed; bytes[31] = 1; Keypair::from_secret(&bytes).unwrap().peer_id() } - fn connect(mgr: &mut PeerManager, cap: &mut Captured, conn: usize, seed: u8, now: Instant) { + pub(crate) fn connect( + mgr: &mut PeerManager, + cap: &mut Captured, + conn: usize, + seed: u8, + now: Instant, + ) { mgr.handle_event( PeerEvent::P2pNewConnection { p2p_peer_id: conn, @@ -2438,1216 +1680,166 @@ mod tests { ); } - /// `on_connected` always emits a `P2pSend::Identify` event; filter - /// it out so subscribe-focused tests can assert on subscribe counts. - fn subscribe_events(cap: &Captured) -> Vec<&PeerControl> { - cap.0.iter().filter(|c| !matches!(c, PeerControl::P2pSend(P2pSend::Identify(_)))).collect() - } - #[test] - fn connect_with_no_topics_emits_nothing() { + fn invalid_frames_increment_behaviour_penalty_and_tick_bans() { let now = Instant::now(); - let (mut mgr, mut cap) = fixture(vec![], ScoreParams::default(), false); + let mut params = ScoreParams::default(); + params.behaviour_penalty_threshold = 0.0; + params.behaviour_penalty_weight = -10.0; // excess^2 * -10 + params.graylist_threshold = -80.0; // behaviour_penalty=3 → score=-90 + params.ip_ban_threshold = 1; // single eviction → BanIp for this test + let (mut mgr, mut cap) = fixture(vec![], params, false); connect(&mut mgr, &mut cap, 1, 1, now); - assert!(subscribe_events(&cap).is_empty()); - } - #[test] - fn connect_emits_subscribe_per_our_topic() { - let now = Instant::now(); - let topics = vec![GossipTopic::BeaconBlock, GossipTopic::VoluntaryExit]; - let (mut mgr, mut cap) = fixture(topics.clone(), ScoreParams::default(), false); - connect(&mut mgr, &mut cap, 1, 1, now); - let subs = subscribe_events(&cap); - assert_eq!(subs.len(), 2); - for e in &subs { - assert!(matches!(e, PeerControl::P2pGossipSubscribe { .. })); + for _ in 0..5 { + mgr.handle_event(PeerEvent::P2pGossipInvalidFrame { p2p_peer: 1 }, now, &mut |c| { + cap.0.push(c) + }); } - } - - #[test] - fn peer_subscribes_and_we_graft_when_mesh_under_d_low() { - let now = Instant::now(); - let topics = vec![GossipTopic::BeaconBlock]; - let (mut mgr, mut cap) = fixture(topics, ScoreParams::default(), false); - connect(&mut mgr, &mut cap, 1, 1, now); - cap.0.clear(); + assert!(mgr.score(1).is_some()); + mgr.tick(now + Duration::from_millis(100), &mut |c| cap.0.push(c)); - mgr.handle_event( - PeerEvent::P2pGossipTopicSubscribe { p2p_peer: 1, topic: GossipTopic::BeaconBlock }, - now, - &mut |c| cap.0.push(c), + assert!( + cap.0.iter().any(|e| matches!(e, PeerControl::Ban { .. })), + "expected Ban, got {:?}", + cap.0 ); - assert!( - cap.0.iter().any(|e| matches!( - e, - PeerControl::P2pGossipGraft { topic, .. } if *topic == GossipTopic::BeaconBlock - )), - "expected a GRAFT, got {:?}", + cap.0.iter().any(|e| matches!(e, PeerControl::BanIp { .. })), + "expected BanIp, got {:?}", cap.0 ); - assert_eq!(mgr.mesh_size(GossipTopic::BeaconBlock), 1); + assert!(mgr.score(1).is_none(), "banned peer should be gone"); } - #[test] - fn negative_score_subscriber_is_not_grafted() { - let now = Instant::now(); - let topic = GossipTopic::BeaconBlock; - let (mut mgr, mut cap) = fixture(vec![topic], ScoreParams::default(), false); - connect(&mut mgr, &mut cap, 1, 1, now); - mgr.peers.get_mut(&1).unwrap().cached_score = -0.1; - cap.0.clear(); - - mgr.handle_event( - PeerEvent::P2pGossipTopicSubscribe { p2p_peer: 1, topic }, - now, - &mut |event| cap.0.push(event), - ); - - assert_eq!(mgr.mesh_size(topic), 0); - assert!(!cap.0.iter().any(|event| matches!(event, PeerControl::P2pGossipGraft { .. }))); - + fn connect_dialler( + mgr: &mut PeerManager, + cap: &mut Captured, + conn: usize, + seed: u8, + local_dial: bool, + now: Instant, + ) { mgr.handle_event( - PeerEvent::P2pGossipTopicGraft { p2p_peer: 1, topic }, + PeerEvent::P2pNewConnection { + p2p_peer_id: conn, + peer_id_full: peer_id(seed), + ip: IpBytes::V4([10, 0, 0, seed]), + port: 4000 + seed as u16, + local_dial, + }, now, - &mut |event| cap.0.push(event), + &mut |c| cap.0.push(c), ); - - assert_eq!(mgr.mesh_size(topic), 0); - assert!(cap.0.iter().any(|event| matches!( - event, - PeerControl::P2pGossipPrune { p2p_connection: 1, topic: pruned, .. } - if *pruned == topic - ))); } #[test] - fn negative_mesh_peer_is_pruned_before_refill() { + fn duplicate_crossing_connections_keep_deterministic_survivor() { let now = Instant::now(); - let topic = GossipTopic::BeaconBlock; - let (mut mgr, mut cap) = fixture(vec![topic], ScoreParams::default(), false); - connect(&mut mgr, &mut cap, 1, 1, now); - mgr.handle_event( - PeerEvent::P2pGossipTopicSubscribe { p2p_peer: 1, topic }, - now, - &mut |event| cap.0.push(event), - ); - assert_eq!(mgr.mesh_size(topic), 1); - mgr.peers.get_mut(&1).unwrap().application_score = -1.0; - cap.0.clear(); + let (mut mgr, mut cap) = fixture(vec![], ScoreParams::default(), false); + connect_dialler(&mut mgr, &mut cap, 1, 1, true, now); + connect_dialler(&mut mgr, &mut cap, 2, 1, false, now); - mgr.tick(now + Duration::from_secs(1), &mut |event| cap.0.push(event)); + let local_lower = peer_id(99).as_bytes() < peer_id(1).as_bytes(); + let survivor = if local_lower { 1 } else { 2 }; + let loser = if local_lower { 2 } else { 1 }; - assert_eq!(mgr.mesh_size(topic), 0); - assert!(cap.0.iter().any(|event| matches!( - event, - PeerControl::P2pGossipPrune { p2p_connection: 1, topic: pruned, .. } - if *pruned == topic + assert_eq!(mgr.peers.len(), 1); + assert!(mgr.peers.contains_key(&survivor)); + assert!(cap.0.iter().any(|c| matches!( + c, + PeerControl::P2pDisconnect { p2p_connection, .. } if *p2p_connection == loser ))); } #[test] - fn randomized_refill_uses_only_eligible_peers() { + fn duplicate_same_direction_keeps_newest() { let now = Instant::now(); - let topic = GossipTopic::BeaconBlock; - let mut params = ScoreParams::default(); - params.d = 4; - params.d_low = 0; - let (mut mgr, mut cap) = fixture(vec![topic], params, false); - for conn in 1..=6 { - connect(&mut mgr, &mut cap, conn, conn as u8, now); - mgr.handle_event( - PeerEvent::P2pGossipTopicSubscribe { p2p_peer: conn, topic }, - now, - &mut |event| cap.0.push(event), - ); - mgr.peers.get_mut(&conn).unwrap().cached_score = conn as f64; - } - mgr.peers.get_mut(&1).unwrap().cached_score = -0.1; - mgr.set_backoff(2, topic, now, Duration::from_secs(60)); - - mgr.ensure_mesh_filled(topic, now, &mut |event| cap.0.push(event)); + let (mut mgr, mut cap) = fixture(vec![], ScoreParams::default(), false); + connect_dialler(&mut mgr, &mut cap, 1, 1, false, now); + connect_dialler(&mut mgr, &mut cap, 2, 1, false, now); - let mesh = &mgr.mesh[&topic]; - assert_eq!(mesh.len(), 4); - assert!(!mesh.contains(&1)); - assert!(!mesh.contains(&2)); - assert!(mesh.iter().all(|conn| (3..=6).contains(conn))); + assert_eq!(mgr.peers.len(), 1); + assert!(mgr.peers.contains_key(&2)); + assert!(cap.0.iter().any(|c| matches!( + c, + PeerControl::P2pDisconnect { p2p_connection, .. } if *p2p_connection == 1 + ))); } #[test] - fn randomized_capping_retains_four_highest_scoring_peers() { + fn ip_colocation_penalty_applies() { let now = Instant::now(); - let topic = GossipTopic::BeaconBlock; let mut params = ScoreParams::default(); - params.d = 8; - params.d_low = 0; - params.d_high = 12; - let (mut mgr, mut cap) = fixture(vec![topic], params, false); - for conn in 1..=12 { - connect(&mut mgr, &mut cap, conn, conn as u8, now); - mgr.do_graft(conn, peer_id(conn as u8), topic, now, &mut |event| cap.0.push(event)); - mgr.peers.get_mut(&conn).unwrap().cached_score = conn as f64; - } - cap.0.clear(); - - mgr.ensure_mesh_capped(topic, now, &mut |event| cap.0.push(event)); - assert_eq!(mgr.mesh[&topic].len(), 12); - assert!(cap.0.is_empty()); - - connect(&mut mgr, &mut cap, 13, 13, now); - mgr.do_graft(13, peer_id(13), topic, now, &mut |event| cap.0.push(event)); - mgr.peers.get_mut(&13).unwrap().cached_score = 13.0; - cap.0.clear(); - - // Over cap, but no member has settled yet: no victims. - mgr.ensure_mesh_capped(topic, now, &mut |event| cap.0.push(event)); - assert_eq!(mgr.mesh[&topic].len(), 13); - assert!(cap.0.is_empty()); - - let settled = - now + Duration::from_secs_f64(mgr.params.mesh_message_deliveries_activation_s); - mgr.ensure_mesh_capped(topic, settled, &mut |event| cap.0.push(event)); - - let mesh = &mgr.mesh[&topic]; - assert_eq!(mesh.len(), 8); - assert!((10..=13).all(|conn| mesh.contains(&conn))); - assert_eq!( - cap.0 - .iter() - .filter(|event| matches!(event, PeerControl::P2pGossipPrune { .. })) - .count(), - 5 - ); - } + params.ip_colocation_threshold = 2; + params.ip_colocation_weight = -5.0; + let (mut mgr, mut cap) = fixture(vec![], params, false); - #[test] - fn opportunistic_graft_runs_on_duration_and_selects_above_median() { - let now = Instant::now(); - let topic = GossipTopic::BeaconBlock; - let mut params = ScoreParams::default(); - params.d = 2; - params.d_low = 0; - params.d_high = 8; - let (mut mgr, mut cap) = fixture(vec![topic], params, false); - for conn in 1..=5 { - connect(&mut mgr, &mut cap, conn, conn as u8, now); + for i in 1..=5u8 { mgr.handle_event( - PeerEvent::P2pGossipTopicSubscribe { p2p_peer: conn, topic }, + PeerEvent::P2pNewConnection { + p2p_peer_id: i as usize, + peer_id_full: peer_id(i), + ip: IpBytes::V4([10, 0, 0, i]), + port: 4000 + i as u16, + local_dial: false, + }, now, - &mut |event| cap.0.push(event), + &mut |c| cap.0.push(c), ); } - mgr.do_graft(1, peer_id(1), topic, now, &mut |event| cap.0.push(event)); - mgr.do_graft(2, peer_id(2), topic, now, &mut |event| cap.0.push(event)); - for (conn, score) in [(1, 1.0), (2, 2.0), (3, 3.0), (4, 4.0), (5, 1.4)] { - mgr.peers.get_mut(&conn).unwrap().cached_score = score; - } - let due = mgr.last_opportunistic_graft + OPPORTUNISTIC_GRAFT_INTERVAL; - - mgr.manage_mesh(due - Duration::from_nanos(1), &mut |event| cap.0.push(event)); - assert_eq!(mgr.mesh_size(topic), 2); - - // Due, but both members are inside the activation window: no median, - // no graft. - mgr.manage_mesh(due, &mut |event| cap.0.push(event)); - assert_eq!(mgr.mesh_size(topic), 2); - assert_eq!(mgr.last_opportunistic_graft, due); - - // Past the activation window the members' scores count. - let established = - due + Duration::from_secs_f64(mgr.params.mesh_message_deliveries_activation_s); - mgr.manage_mesh(established, &mut |event| cap.0.push(event)); - - let mesh = &mgr.mesh[&topic]; - assert_eq!(mesh.len(), 4); - assert!(mesh.contains(&3)); - assert!(mesh.contains(&4)); - assert!(!mesh.contains(&5)); + mgr.tick(now + Duration::from_millis(100), &mut |c| cap.0.push(c)); + + let s = mgr.score(1).unwrap(); + assert!((s - -45.0).abs() < 1e-9, "expected -45, got {s}"); } #[test] - fn received_prune_honors_full_backoff_and_heartbeat_slack() { + fn disconnect_archives_and_reconnect_restores() { let now = Instant::now(); - let topic = GossipTopic::BeaconBlock; let params = ScoreParams::default(); - let heartbeat = params.heartbeat_interval; - let (mut mgr, mut cap) = fixture(vec![topic], params, false); + let (mut mgr, mut cap) = fixture(vec![], params, false); connect(&mut mgr, &mut cap, 1, 1, now); + + mgr.handle_event(PeerEvent::P2pGossipInvalidFrame { p2p_peer: 1 }, now, &mut |c| { + cap.0.push(c) + }); + mgr.handle_event(PeerEvent::P2pGossipInvalidFrame { p2p_peer: 1 }, now, &mut |c| { + cap.0.push(c) + }); mgr.handle_event( - PeerEvent::P2pGossipTopicPrune { p2p_peer: 1, topic, backoff_seconds: Some(7200) }, - now, - &mut |event| cap.0.push(event), - ); - let original_deadline = mgr.peers[&1].backoffs[&topic]; - - mgr.handle_event( - PeerEvent::P2pGossipTopicPrune { p2p_peer: 1, topic, backoff_seconds: Some(60) }, - now + Duration::from_secs(1), - &mut |event| cap.0.push(event), - ); - - assert_eq!(mgr.peers[&1].backoffs[&topic], original_deadline); - let end_with_slack = now + Duration::from_secs(7200) + heartbeat; - assert!(mgr.is_backed_off(1, topic, end_with_slack - Duration::from_nanos(1))); - assert!(!mgr.is_backed_off(1, topic, end_with_slack)); - - connect(&mut mgr, &mut cap, 2, 2, now); - mgr.handle_event( - PeerEvent::P2pGossipTopicPrune { p2p_peer: 2, topic, backoff_seconds: Some(u64::MAX) }, - now, - &mut |event| cap.0.push(event), - ); - assert!(!mgr.peers[&2].backoffs.contains_key(&topic)); - } - - #[test] - fn unsubscribe_preserves_reputation_and_squares_mesh_failure() { - let now = Instant::now(); - let topic = GossipTopic::BeaconBlock; - let (mut mgr, mut cap) = fixture(vec![topic], ScoreParams::default(), false); - connect(&mut mgr, &mut cap, 1, 1, now); - mgr.handle_event( - PeerEvent::P2pGossipTopicSubscribe { p2p_peer: 1, topic }, - now, - &mut |event| cap.0.push(event), - ); - let topic_score = mgr.peers.get_mut(&1).unwrap().topic_stats.get_mut(&topic).unwrap(); - topic_score.first_deliveries = 2.0; - topic_score.mesh_deliveries = 0.2; - topic_score.mesh_active = true; - topic_score.invalid_deliveries = 3.0; - - mgr.handle_event( - PeerEvent::P2pGossipTopicUnsubscribe { p2p_peer: 1, topic }, - now, - &mut |event| cap.0.push(event), - ); - - let topic_score = &mgr.peers[&1].topic_stats[&topic]; - let deficit = scoring::topic_params(&topic).p3_threshold - 0.2; - assert_eq!(mgr.mesh_size(topic), 0); - assert_eq!(topic_score.first_deliveries, 2.0); - assert_eq!(topic_score.invalid_deliveries, 3.0); - assert_eq!(topic_score.mesh_failure_penalty, deficit * deficit); - assert!(topic_score.meshed_since.is_none()); - assert!(!topic_score.mesh_active); - } - - #[test] - fn remote_prune_squares_mesh_failure() { - let now = Instant::now(); - let topic = GossipTopic::BeaconBlock; - let (mut mgr, mut cap) = fixture(vec![topic], ScoreParams::default(), false); - connect(&mut mgr, &mut cap, 1, 1, now); - mgr.handle_event( - PeerEvent::P2pGossipTopicSubscribe { p2p_peer: 1, topic }, - now, - &mut |event| cap.0.push(event), - ); - let topic_score = mgr.peers.get_mut(&1).unwrap().topic_stats.get_mut(&topic).unwrap(); - topic_score.mesh_deliveries = 0.1; - topic_score.mesh_active = true; - - mgr.handle_event( - PeerEvent::P2pGossipTopicPrune { p2p_peer: 1, topic, backoff_seconds: Some(60) }, - now, - &mut |event| cap.0.push(event), - ); - - let topic_score = &mgr.peers[&1].topic_stats[&topic]; - let deficit = scoring::topic_params(&topic).p3_threshold - 0.1; - assert_eq!(topic_score.mesh_failure_penalty, deficit * deficit); - assert!(topic_score.meshed_since.is_none()); - assert!(!topic_score.mesh_active); - } - - #[test] - fn disconnect_squares_mesh_failure_before_archiving() { - let now = Instant::now(); - let topic = GossipTopic::BeaconBlock; - let id = peer_id(1); - let (mut mgr, mut cap) = fixture(vec![topic], ScoreParams::default(), false); - connect(&mut mgr, &mut cap, 1, 1, now); - mgr.handle_event( - PeerEvent::P2pGossipTopicSubscribe { p2p_peer: 1, topic }, - now, - &mut |event| cap.0.push(event), - ); - let topic_score = mgr.peers.get_mut(&1).unwrap().topic_stats.get_mut(&topic).unwrap(); - topic_score.mesh_deliveries = 0.3; - topic_score.mesh_active = true; - - mgr.handle_event( - PeerEvent::P2pDisconnect { p2p_peer: 1, peer_id: id }, - now, - &mut |event| cap.0.push(event), - ); - - let topic_score = &mgr.archived[&id].topic_stats[&topic]; - let deficit = scoring::topic_params(&topic).p3_threshold - 0.3; - assert_eq!(mgr.mesh_size(topic), 0); - assert_eq!(topic_score.mesh_failure_penalty, deficit * deficit); - assert!(topic_score.meshed_since.is_none()); - assert!(!topic_score.mesh_active); - } - - #[test] - fn invalid_frames_increment_behaviour_penalty_and_tick_bans() { - let now = Instant::now(); - let mut params = ScoreParams::default(); - params.behaviour_penalty_threshold = 0.0; - params.behaviour_penalty_weight = -10.0; // excess^2 * -10 - params.graylist_threshold = -80.0; // behaviour_penalty=3 → score=-90 - params.ip_ban_threshold = 1; // single eviction → BanIp for this test - let (mut mgr, mut cap) = fixture(vec![], params, false); - connect(&mut mgr, &mut cap, 1, 1, now); - - for _ in 0..5 { - mgr.handle_event(PeerEvent::P2pGossipInvalidFrame { p2p_peer: 1 }, now, &mut |c| { - cap.0.push(c) - }); - } - assert!(mgr.score(1).is_some()); - mgr.tick(now + Duration::from_millis(100), &mut |c| cap.0.push(c)); - - assert!( - cap.0.iter().any(|e| matches!(e, PeerControl::Ban { .. })), - "expected Ban, got {:?}", - cap.0 - ); - assert!( - cap.0.iter().any(|e| matches!(e, PeerControl::BanIp { .. })), - "expected BanIp, got {:?}", - cap.0 - ); - assert!(mgr.score(1).is_none(), "banned peer should be gone"); - } - - fn connect_dialler( - mgr: &mut PeerManager, - cap: &mut Captured, - conn: usize, - seed: u8, - local_dial: bool, - now: Instant, - ) { - mgr.handle_event( - PeerEvent::P2pNewConnection { - p2p_peer_id: conn, - peer_id_full: peer_id(seed), - ip: IpBytes::V4([10, 0, 0, seed]), - port: 4000 + seed as u16, - local_dial, - }, - now, - &mut |c| cap.0.push(c), - ); - } - - #[test] - fn duplicate_crossing_connections_keep_deterministic_survivor() { - let now = Instant::now(); - let (mut mgr, mut cap) = fixture(vec![], ScoreParams::default(), false); - connect_dialler(&mut mgr, &mut cap, 1, 1, true, now); - connect_dialler(&mut mgr, &mut cap, 2, 1, false, now); - - let local_lower = peer_id(99).as_bytes() < peer_id(1).as_bytes(); - let survivor = if local_lower { 1 } else { 2 }; - let loser = if local_lower { 2 } else { 1 }; - - assert_eq!(mgr.peers.len(), 1); - assert!(mgr.peers.contains_key(&survivor)); - assert!(cap.0.iter().any(|c| matches!( - c, - PeerControl::P2pDisconnect { p2p_connection, .. } if *p2p_connection == loser - ))); - } - - #[test] - fn duplicate_same_direction_keeps_newest() { - let now = Instant::now(); - let (mut mgr, mut cap) = fixture(vec![], ScoreParams::default(), false); - connect_dialler(&mut mgr, &mut cap, 1, 1, false, now); - connect_dialler(&mut mgr, &mut cap, 2, 1, false, now); - - assert_eq!(mgr.peers.len(), 1); - assert!(mgr.peers.contains_key(&2)); - assert!(cap.0.iter().any(|c| matches!( - c, - PeerControl::P2pDisconnect { p2p_connection, .. } if *p2p_connection == 1 - ))); - } - - #[test] - fn ihave_below_gossip_threshold_records_no_promise() { - let now = Instant::now(); - let params = ScoreParams::default(); - let (mut mgr, mut cap) = fixture(vec![], params, false); - connect(&mut mgr, &mut cap, 1, 1, now); - mgr.peers.get_mut(&1).unwrap().cached_score = mgr.params.gossip_threshold - 1.0; - - let hash = silver_common::MessageId { id: [7u8; 20] }; - mgr.handle_event( - PeerEvent::P2pGossipHave { - p2p_peer: 1, - topic: GossipTopic::BeaconBlock, - hash, - already_seen: false, - }, - now, - &mut |c| cap.0.push(c), - ); - assert!(mgr.promises.is_empty()); - } - - #[test] - fn duplicate_delivery_clears_promise() { - let now = Instant::now(); - let params = ScoreParams::default(); - let (mut mgr, mut cap) = fixture(vec![], params, false); - connect(&mut mgr, &mut cap, 1, 1, now); - - let hash = silver_common::MessageId { id: [7u8; 20] }; - mgr.handle_event( - PeerEvent::P2pGossipHave { - p2p_peer: 1, - topic: GossipTopic::BeaconBlock, - hash, - already_seen: false, - }, - now, - &mut |c| cap.0.push(c), - ); - assert_eq!(mgr.promises.len(), 1); - - mgr.handle_event( - PeerEvent::GossipDuplicate { - p2p_peer: 1, - topic: GossipTopic::BeaconBlock, - hash, - recv_ts: Nanos::now(), - }, - now, - &mut |c| cap.0.push(c), - ); - assert!(mgr.promises.is_empty()); - } - - #[test] - fn broken_promise_sweep_adds_penalty() { - let mut now = Instant::now(); - let mut params = ScoreParams::default(); - params.iwant_followup = Duration::from_secs(3); - params.heartbeat_interval = Duration::from_millis(100); - let (mut mgr, mut cap) = fixture(vec![], params, false); - connect(&mut mgr, &mut cap, 1, 1, now); - - let hash = silver_common::MessageId { id: [7u8; 20] }; - mgr.handle_event( - PeerEvent::P2pGossipHave { - p2p_peer: 1, - topic: GossipTopic::BeaconBlock, - hash, - already_seen: false, - }, + PeerEvent::P2pDisconnect { p2p_peer: 1, peer_id: peer_id(1) }, now, &mut |c| cap.0.push(c), ); + assert_eq!(mgr.archived_count(), 1); - now += Duration::from_secs(4); - mgr.tick(now, &mut |c| cap.0.push(c)); - - let s = mgr.score(1).unwrap(); - assert!(s <= 0.0, "expected non-positive score after broken promise, got {s}"); - } - - #[test] - fn ihave_flood_over_heartbeat_cap_penalises() { - let now = Instant::now(); - let mut params = ScoreParams::default(); - params.max_ihave_length = 3; - params.graylist_threshold = -100_000.0; - let (mut mgr, mut cap) = fixture(vec![], params, false); - connect(&mut mgr, &mut cap, 1, 1, now); - - let hash = silver_common::MessageId { id: [9u8; 20] }; - for _ in 0..8 { - mgr.handle_event( - PeerEvent::P2pGossipHave { - p2p_peer: 1, - topic: GossipTopic::BeaconBlock, - hash, - already_seen: false, - }, - now, - &mut |c| cap.0.push(c), - ); - } + connect(&mut mgr, &mut cap, 99, 1, now); mgr.tick(now + Duration::from_millis(100), &mut |c| cap.0.push(c)); - let s = mgr.score(1).unwrap(); - assert!(s < 0.0, "expected negative score after flood, got {s}"); + let s = mgr.score(99).unwrap(); + assert!(s < 0.0, "expected restored penalty score, got {s}"); + assert_eq!(mgr.archived_count(), 0); } #[test] - fn already_seen_ihave_tracks_no_promise() { + fn archived_state_dropped_past_ttl() { let mut now = Instant::now(); let mut params = ScoreParams::default(); - params.iwant_followup = Duration::from_secs(3); - params.heartbeat_interval = Duration::from_millis(100); + params.archived_ttl = Duration::from_secs(10); let (mut mgr, mut cap) = fixture(vec![], params, false); connect(&mut mgr, &mut cap, 1, 1, now); - - let hash = silver_common::MessageId { id: [77u8; 20] }; mgr.handle_event( - PeerEvent::P2pGossipHave { - p2p_peer: 1, - topic: GossipTopic::BeaconBlock, - hash, - already_seen: true, - }, + PeerEvent::P2pDisconnect { p2p_peer: 1, peer_id: peer_id(1) }, now, &mut |c| cap.0.push(c), ); + assert_eq!(mgr.archived_count(), 1); - now += Duration::from_secs(5); - mgr.tick(now, &mut |c| cap.0.push(c)); - let s = mgr.score(1).unwrap(); - assert_eq!(s, 0.0, "no IWANT was issued → no promise → no broken-promise penalty, got {s}"); - } - - #[test] - fn ip_colocation_penalty_applies() { - let now = Instant::now(); - let mut params = ScoreParams::default(); - params.ip_colocation_threshold = 2; - params.ip_colocation_weight = -5.0; - let (mut mgr, mut cap) = fixture(vec![], params, false); - - for i in 1..=5u8 { - mgr.handle_event( - PeerEvent::P2pNewConnection { - p2p_peer_id: i as usize, - peer_id_full: peer_id(i), - ip: IpBytes::V4([10, 0, 0, i]), - port: 4000 + i as u16, - local_dial: false, - }, - now, - &mut |c| cap.0.push(c), - ); - } - mgr.tick(now + Duration::from_millis(100), &mut |c| cap.0.push(c)); - - let s = mgr.score(1).unwrap(); - assert!((s - -45.0).abs() < 1e-9, "expected -45, got {s}"); - } - - #[test] - fn disconnect_archives_and_reconnect_restores() { - let now = Instant::now(); - let params = ScoreParams::default(); - let (mut mgr, mut cap) = fixture(vec![], params, false); - connect(&mut mgr, &mut cap, 1, 1, now); - - mgr.handle_event(PeerEvent::P2pGossipInvalidFrame { p2p_peer: 1 }, now, &mut |c| { - cap.0.push(c) - }); - mgr.handle_event(PeerEvent::P2pGossipInvalidFrame { p2p_peer: 1 }, now, &mut |c| { - cap.0.push(c) - }); - mgr.handle_event( - PeerEvent::P2pDisconnect { p2p_peer: 1, peer_id: peer_id(1) }, - now, - &mut |c| cap.0.push(c), - ); - assert_eq!(mgr.archived_count(), 1); - - connect(&mut mgr, &mut cap, 99, 1, now); - mgr.tick(now + Duration::from_millis(100), &mut |c| cap.0.push(c)); - let s = mgr.score(99).unwrap(); - assert!(s < 0.0, "expected restored penalty score, got {s}"); - assert_eq!(mgr.archived_count(), 0); - } - - #[test] - fn archived_state_dropped_past_ttl() { - let mut now = Instant::now(); - let mut params = ScoreParams::default(); - params.archived_ttl = Duration::from_secs(10); - let (mut mgr, mut cap) = fixture(vec![], params, false); - connect(&mut mgr, &mut cap, 1, 1, now); - mgr.handle_event( - PeerEvent::P2pDisconnect { p2p_peer: 1, peer_id: peer_id(1) }, - now, - &mut |c| cap.0.push(c), - ); - assert_eq!(mgr.archived_count(), 1); - - now += Duration::from_secs(11); + now += Duration::from_secs(11); mgr.tick(now, &mut |c| cap.0.push(c)); assert_eq!(mgr.archived_count(), 0); } - fn mk_tcache_read() -> silver_common::TCacheRead { - let mut producer = silver_common::TCache::producer("test_peer", 1 << 14); - let mut reservation = producer.reserve(64, true).unwrap(); - use std::io::Write as _; - reservation.write_all(&[0u8; 64]).unwrap(); - reservation.read() - } - - #[test] - fn near_first_mesh_deliveries_credit_each_peer_once() { - let now = Instant::now(); - let topic = GossipTopic::BeaconBlock; - let mut params = ScoreParams::default(); - params.d_low = 0; - params.d = 0; - let (mut mgr, mut cap) = fixture(vec![topic], params, false); - for conn in 1..=3 { - connect(&mut mgr, &mut cap, conn, conn as u8, now); - mgr.do_graft(conn, peer_id(conn as u8), topic, now, &mut |event| cap.0.push(event)); - } - let hash = MessageId { id: [11u8; 20] }; - let first_seen = Nanos::from_secs(10); - - mgr.handle_event( - PeerEvent::NewGossip { - p2p_peer: 1, - topic, - msg_hash: hash, - recv_ts: first_seen, - idontwant: mk_tcache_read(), - }, - now, - &mut |event| cap.0.push(event), - ); - mgr.handle_event( - PeerEvent::GossipDuplicate { - p2p_peer: 2, - topic, - hash, - recv_ts: Nanos(first_seen.0 + 1_000_000_000), - }, - now, - &mut |event| cap.0.push(event), - ); - mgr.handle_event( - PeerEvent::GossipDuplicate { - p2p_peer: 2, - topic, - hash, - recv_ts: Nanos(first_seen.0 + 1_500_000_000), - }, - now, - &mut |event| cap.0.push(event), - ); - mgr.handle_event( - PeerEvent::GossipDuplicate { - p2p_peer: 3, - topic, - hash, - recv_ts: Nanos(first_seen.0 + 2_000_000_001), - }, - now, - &mut |event| cap.0.push(event), - ); - - assert_eq!(mgr.peers[&1].topic_stats[&topic].mesh_deliveries, 1.0); - assert_eq!(mgr.peers[&2].topic_stats[&topic].mesh_deliveries, 1.0); - assert_eq!(mgr.peers[&3].topic_stats[&topic].mesh_deliveries, 0.0); - } - - #[test] - fn new_inbound_fulfils_promise_and_credits_p2() { - let mut now = Instant::now(); - let mut params = ScoreParams::default(); - params.iwant_followup = Duration::from_secs(3); - params.heartbeat_interval = Duration::from_millis(100); - let (mut mgr, mut cap) = fixture(vec![], params, false); - connect(&mut mgr, &mut cap, 1, 1, now); - - let hash = silver_common::MessageId { id: [7u8; 20] }; - mgr.handle_event( - PeerEvent::P2pGossipHave { - p2p_peer: 1, - topic: GossipTopic::BeaconBlock, - hash, - already_seen: false, - }, - now, - &mut |c| cap.0.push(c), - ); - - now += Duration::from_secs(1); - mgr.handle_event( - PeerEvent::NewGossip { - p2p_peer: 1, - topic: GossipTopic::BeaconBlock, - msg_hash: hash, - recv_ts: Nanos::now(), - idontwant: mk_tcache_read(), - }, - now, - &mut |c| cap.0.push(c), - ); - - now += Duration::from_secs(5); - mgr.tick(now, &mut |c| cap.0.push(c)); - let score_delivered = mgr.score(1).unwrap(); - - connect(&mut mgr, &mut cap, 2, 2, now); - let other = silver_common::MessageId { id: [8u8; 20] }; - mgr.handle_event( - PeerEvent::P2pGossipHave { - p2p_peer: 2, - topic: GossipTopic::BeaconBlock, - hash: other, - already_seen: false, - }, - now, - &mut |c| cap.0.push(c), - ); - now += Duration::from_secs(5); - mgr.tick(now, &mut |c| cap.0.push(c)); - let score_broken = mgr.score(2).unwrap(); - - assert!( - score_delivered > score_broken, - "delivered peer must score above broken-promise peer: \ - delivered={score_delivered}, broken={score_broken}" - ); - } - - #[test] - fn delivery_from_one_peer_fulfils_all_peers_promises_for_that_id() { - let mut now = Instant::now(); - let mut params = ScoreParams::default(); - params.iwant_followup = Duration::from_secs(3); - params.heartbeat_interval = Duration::from_millis(100); - let (mut mgr, mut cap) = fixture(vec![], params, false); - connect(&mut mgr, &mut cap, 1, 1, now); - connect(&mut mgr, &mut cap, 2, 2, now); - - let hash = silver_common::MessageId { id: [42u8; 20] }; - mgr.handle_event( - PeerEvent::P2pGossipHave { - p2p_peer: 1, - topic: GossipTopic::BeaconBlock, - hash, - already_seen: false, - }, - now, - &mut |c| cap.0.push(c), - ); - mgr.handle_event( - PeerEvent::P2pGossipHave { - p2p_peer: 2, - topic: GossipTopic::BeaconBlock, - hash, - already_seen: false, - }, - now, - &mut |c| cap.0.push(c), - ); - - now += Duration::from_secs(1); - mgr.handle_event( - PeerEvent::NewGossip { - p2p_peer: 1, - topic: GossipTopic::BeaconBlock, - msg_hash: hash, - recv_ts: Nanos::now(), - idontwant: mk_tcache_read(), - }, - now, - &mut |c| cap.0.push(c), - ); - - now += Duration::from_secs(5); - mgr.tick(now, &mut |c| cap.0.push(c)); - - connect(&mut mgr, &mut cap, 3, 3, now); - let other = silver_common::MessageId { id: [99u8; 20] }; - mgr.handle_event( - PeerEvent::P2pGossipHave { - p2p_peer: 3, - topic: GossipTopic::BeaconBlock, - hash: other, - already_seen: false, - }, - now, - &mut |c| cap.0.push(c), - ); - now += Duration::from_secs(5); - mgr.tick(now, &mut |c| cap.0.push(c)); - - let s1 = mgr.score(1).unwrap(); - let s2 = mgr.score(2).unwrap(); - let s3 = mgr.score(3).unwrap(); - assert!(s1 > s3, "delivering peer must out-score broken-promise peer: {s1} vs {s3}"); - assert!( - s2 > s3, - "promise-fulfilled-by-other-peer must out-score broken-promise peer: {s2} vs {s3}" - ); - } - - #[test] - fn new_outbound_ihave_fans_out_to_non_mesh_subscribers() { - let now = Instant::now(); - let mut params = ScoreParams::default(); - params.d_lazy = 3; - params.d_low = 1; // so the first subscriber grafts into mesh, rest stay non-mesh - let (mut mgr, mut cap) = fixture(vec![GossipTopic::BeaconBlock], params, false); - - for i in 1..=4u8 { - connect(&mut mgr, &mut cap, i as usize, i, now); - mgr.handle_event( - PeerEvent::P2pGossipTopicSubscribe { - p2p_peer: i as usize, - topic: GossipTopic::BeaconBlock, - }, - now, - &mut |c| cap.0.push(c), - ); - } - assert_eq!(mgr.mesh_size(GossipTopic::BeaconBlock), 1); - cap.0.clear(); - - mgr.handle_event( - PeerEvent::OutboundIHave { - topic: GossipTopic::BeaconBlock, - msg_count: 2, - protobuf: mk_tcache_read(), - }, - now, - &mut |c| cap.0.push(c), - ); - - let send_ihaves: Vec<_> = cap - .0 - .iter() - .filter_map(|e| { - if let PeerControl::P2pSend(P2pSend::Gossip(GossipMsgOut { peer_id, .. })) = e { - Some(*peer_id) - } else { - None - } - }) - .collect(); - assert_eq!( - send_ihaves.len(), - 3, - "expected 3 IHAVE emissions (d_lazy=3, 3 non-mesh subscribers), got {:?}", - cap.0 - ); - assert!( - send_ihaves.iter().all(|c| !matches!(c, &1)), - "mesh peer (conn=1) must not receive IHAVE, got {send_ihaves:?}" - ); - } - - #[test] - fn new_outbound_ihave_skips_below_threshold_peers() { - let now = Instant::now(); - let mut params = ScoreParams::default(); - params.gossip_threshold = -1.0; - params.graylist_threshold = -1_000_000.0; - let (mut mgr, mut cap) = fixture(vec![GossipTopic::BeaconBlock], params, false); - connect(&mut mgr, &mut cap, 1, 1, now); - mgr.handle_event( - PeerEvent::P2pGossipTopicSubscribe { p2p_peer: 1, topic: GossipTopic::BeaconBlock }, - now, - &mut |c| cap.0.push(c), - ); - for _ in 0..5 { - mgr.handle_event(PeerEvent::P2pGossipInvalidFrame { p2p_peer: 1 }, now, &mut |c| { - cap.0.push(c) - }); - } - mgr.tick(now + Duration::from_millis(10), &mut |c| cap.0.push(c)); - assert!(mgr.score(1).unwrap() < -1.0); - - cap.0.clear(); - mgr.handle_event( - PeerEvent::OutboundIHave { - topic: GossipTopic::BeaconBlock, - msg_count: 1, - protobuf: mk_tcache_read(), - }, - now + Duration::from_millis(20), - &mut |c| cap.0.push(c), - ); - assert!( - !cap.0 - .iter() - .any(|e| matches!(e, PeerControl::P2pSend(P2pSend::Gossip(GossipMsgOut { .. })))), - "below-threshold peer should not receive IHAVE, got {:?}", - cap.0 - ); - } - - #[test] - fn iwant_request_above_threshold_emits_forward() { - use silver_common::{TCache, TCacheRead}; - let now = Instant::now(); - let (mut mgr, mut cap) = fixture(vec![], ScoreParams::default(), false); - connect(&mut mgr, &mut cap, 1, 1, now); - - let mut producer = TCache::producer("test_peer", 1 << 14); - let mut reservation = producer.reserve(64, true).unwrap(); - use std::io::Write as _; - reservation.write_all(&[0u8; 64]).unwrap(); - let tcache: TCacheRead = reservation.read(); - - cap.0.clear(); - let hash = silver_common::MessageId { id: [3u8; 20] }; - mgr.handle_event(PeerEvent::P2pGossipWant { p2p_peer: 1, hash, tcache }, now, &mut |c| { - cap.0.push(c) - }); - - assert!( - cap.0.iter().any(|e| matches!( - e, - PeerControl::P2pSend(P2pSend::Gossip(GossipMsgOut { peer_id: 1, .. })) - )), - "expected ForwardMsg emission, got {:?}", - cap.0 - ); - } - - #[test] - fn iwant_request_below_threshold_drops() { - use silver_common::{TCache, TCacheRead}; - let now = Instant::now(); - let mut params = ScoreParams::default(); - params.gossip_threshold = -1.0; - params.graylist_threshold = -1_000_000.0; - let (mut mgr, mut cap) = fixture(vec![], params, false); - connect(&mut mgr, &mut cap, 1, 1, now); - - for _ in 0..5 { - mgr.handle_event(PeerEvent::P2pGossipInvalidFrame { p2p_peer: 1 }, now, &mut |c| { - cap.0.push(c) - }); - } - mgr.tick(now + Duration::from_millis(10), &mut |c| cap.0.push(c)); - assert!(mgr.score(1).unwrap() < -1.0); - - let mut producer = TCache::producer("test_peer", 1 << 14); - let mut reservation = producer.reserve(64, true).unwrap(); - use std::io::Write as _; - reservation.write_all(&[0u8; 64]).unwrap(); - let tcache: TCacheRead = reservation.read(); - - cap.0.clear(); - let hash = silver_common::MessageId { id: [4u8; 20] }; - mgr.handle_event( - PeerEvent::P2pGossipWant { p2p_peer: 1, hash, tcache }, - now + Duration::from_millis(20), - &mut |c| cap.0.push(c), - ); - - assert!( - !cap.0 - .iter() - .any(|e| matches!(e, PeerControl::P2pSend(P2pSend::Gossip(GossipMsgOut { .. })))), - "expected no ForwardMsg for below-threshold peer, got {:?}", - cap.0 - ); - } - - #[test] - fn new_inbound_fans_out_dontwant_to_mesh_excluding_sender() { - let now = Instant::now(); - let mut params = ScoreParams::default(); - params.d_low = 0; - params.d = 0; - params.d_high = 8; - let (mut mgr, mut cap) = fixture(vec![GossipTopic::BeaconBlock], params, false); - - for i in 1..=4u8 { - connect(&mut mgr, &mut cap, i as usize, i, now); - mgr.handle_event( - PeerEvent::P2pGossipTopicSubscribe { - p2p_peer: i as usize, - topic: GossipTopic::BeaconBlock, - }, - now, - &mut |c| cap.0.push(c), - ); - } - for i in 1..=4usize { - mgr.mesh.entry(GossipTopic::BeaconBlock).or_default().push(i); - } - cap.0.clear(); - - let hash = silver_common::MessageId { id: [55u8; 20] }; - mgr.handle_event( - PeerEvent::NewGossip { - p2p_peer: 2, - topic: GossipTopic::BeaconBlock, - msg_hash: hash, - recv_ts: Nanos::now(), - idontwant: mk_tcache_read(), - }, - now, - &mut |c| cap.0.push(c), - ); - - let dontwants: Vec = cap - .0 - .iter() - .filter_map(|e| match e { - PeerControl::P2pSend(P2pSend::Gossip(GossipMsgOut { peer_id, .. })) => { - Some(*peer_id) - } - _ => None, - }) - .collect(); - - assert_eq!( - dontwants.len(), - 3, - "expected IDONTWANT to 3 non-sender mesh peers, got {:?}", - cap.0 - ); - assert!( - !dontwants.contains(&2), - "sender (conn=2) must not receive IDONTWANT for its own delivery: {dontwants:?}" - ); - for conn in [1usize, 3, 4] { - assert!( - dontwants.contains(&conn), - "expected IDONTWANT to mesh peer {conn}, got {dontwants:?}" - ); - } - } - - #[test] - fn new_inbound_skips_dontwant_for_below_threshold_peer() { - let now = Instant::now(); - let mut params = ScoreParams::default(); - params.gossip_threshold = -1.0; - params.graylist_threshold = -1_000_000.0; - params.d_high = 8; - let (mut mgr, mut cap) = fixture(vec![GossipTopic::BeaconBlock], params, false); - - for i in 1..=2u8 { - connect(&mut mgr, &mut cap, i as usize, i, now); - mgr.handle_event( - PeerEvent::P2pGossipTopicSubscribe { - p2p_peer: i as usize, - topic: GossipTopic::BeaconBlock, - }, - now, - &mut |c| cap.0.push(c), - ); - } - for i in 1..=2usize { - mgr.mesh.entry(GossipTopic::BeaconBlock).or_default().push(i); - } - for _ in 0..5 { - mgr.handle_event(PeerEvent::P2pGossipInvalidFrame { p2p_peer: 2 }, now, &mut |c| { - cap.0.push(c) - }); - } - mgr.tick(now + Duration::from_millis(10), &mut |c| cap.0.push(c)); - assert!(mgr.score(2).unwrap() < -1.0); - cap.0.clear(); - - let hash = silver_common::MessageId { id: [66u8; 20] }; - mgr.handle_event( - PeerEvent::NewGossip { - p2p_peer: 1, - topic: GossipTopic::BeaconBlock, - msg_hash: hash, - recv_ts: Nanos::now(), - idontwant: mk_tcache_read(), - }, - now + Duration::from_millis(20), - &mut |c| cap.0.push(c), - ); - - assert!( - !cap.0 - .iter() - .any(|e| matches!(e, PeerControl::P2pSend(P2pSend::Gossip(GossipMsgOut { .. })))), - "below-threshold mesh peer should not receive IDONTWANT, got {:?}", - cap.0 - ); - } - - #[test] - fn send_gossip_skips_mesh_peer_with_idontwant() { - let now = Instant::now(); - let mut params = ScoreParams::default(); - // d_low=0 disables the auto-graft on subscribe so manual mesh seeding - // is the only thing populating the mesh map. - params.d_low = 0; - params.d = 0; - params.d_high = 8; - let (mut mgr, mut cap) = fixture(vec![GossipTopic::BeaconBlock], params, false); - - for i in 1..=3u8 { - connect(&mut mgr, &mut cap, i as usize, i, now); - mgr.handle_event( - PeerEvent::P2pGossipTopicSubscribe { - p2p_peer: i as usize, - topic: GossipTopic::BeaconBlock, - }, - now, - &mut |c| cap.0.push(c), - ); - } - for i in 1..=3usize { - mgr.mesh.entry(GossipTopic::BeaconBlock).or_default().push(i); - } - - let hash = silver_common::MessageId { id: [0xAB; 20] }; - - // Peer 2 says "don't send me this id". - mgr.handle_event(PeerEvent::P2pGossipDontWant { p2p_peer: 2, hash }, now, &mut |c| { - cap.0.push(c) - }); - - cap.0.clear(); - - // Internal SendGossip with originator stream from peer 1. - let stream_id = - silver_common::P2pStreamId::new(1, 0, silver_common::StreamProtocol::GossipSub, false); - mgr.handle_event( - PeerEvent::SendGossip { - originator_stream_id: stream_id, - topic: GossipTopic::BeaconBlock, - msg_hash: hash, - recv_ts: silver_common::Nanos::now(), - protobuf: mk_tcache_read(), - }, - now, - &mut |c| cap.0.push(c), - ); - - let recipients: Vec = cap - .0 - .iter() - .filter_map(|e| match e { - PeerControl::P2pSend(P2pSend::Gossip(GossipMsgOut { peer_id, .. })) => { - Some(*peer_id) - } - _ => None, - }) - .collect(); - // Peer 1 = sender (skipped), peer 2 = IDONTWANT (skipped), peer 3 = served. - assert_eq!( - recipients, - vec![3], - "expected only peer 3 to receive the broadcast, got {recipients:?}" - ); - } - fn dials(cap: &Captured) -> usize { cap.0.iter().filter(|c| matches!(c, PeerControl::P2pDial { .. })).count() } diff --git a/crates/peer/src/manager/gossip.rs b/crates/peer/src/manager/gossip.rs new file mode 100644 index 00000000..c69dc9c3 --- /dev/null +++ b/crates/peer/src/manager/gossip.rs @@ -0,0 +1,2084 @@ +//! Gossipsub domain: topic subscriptions, mesh membership +//! (graft/prune/backoff, fill/cap/opportunistic maintenance), IHAVE/IWANT +//! promise tracking, delivery crediting, and the heartbeat sweeps. + +use std::{ + collections::HashMap, + time::{Duration, Instant}, +}; + +use flux_profiler::timed; +use rand::seq::SliceRandom; +use silver_common::{ + GossipMsgOut, GossipTopic, MessageId, Nanos, P2pSend, PeerControl, PeerId, TCacheRead, +}; + +use super::{PeerManager, build_subnet_masks}; +use crate::scoring; + +const MESH_MESSAGE_DELIVERIES_WINDOW_NS: u64 = 2_000_000_000; +/// Remote prune this soon after graft = their heartbeat trimming an +/// oversubscribed mesh; re-grafting on the base backoff is futile. +const QUICK_PRUNE_WINDOW: Duration = Duration::from_secs(5); +/// Caps futility escalation at `prune_backoff << 4` (60s → 960s). +const QUICK_PRUNE_MAX_SHIFT: u8 = 4; +const OPPORTUNISTIC_GRAFT_INTERVAL: Duration = Duration::from_secs(60); +const OPPORTUNISTIC_GRAFT_PEERS: usize = 2; + +pub(super) struct RecentDelivery { + pub(super) topic: GossipTopic, + pub(super) received_at: Nanos, + pub(super) first_credited_peer: Option, + pub(super) additional_credited_peers: Vec, +} + +impl RecentDelivery { + fn new(topic: GossipTopic, received_at: Nanos, credited_peer: Option) -> Self { + Self { + topic, + received_at, + first_credited_peer: credited_peer, + additional_credited_peers: Vec::new(), + } + } + + fn credit(&mut self, peer_id: PeerId) -> bool { + if self.first_credited_peer == Some(peer_id) || + self.additional_credited_peers.contains(&peer_id) + { + return false; + } + if self.first_credited_peer.is_none() { + self.first_credited_peer = Some(peer_id); + } else { + self.additional_credited_peers.push(peer_id); + } + true + } +} + +impl PeerManager { + /// Announce our topic subscriptions to every currently-connected peer. + /// Add topics at runtime (deferred long-lived subnets): extends + /// `our_topics` + mesh bookkeeping + subnet masks, and announces + /// SUBSCRIBE to every connected peer. New connections pick the + /// topics up via the normal `on_connected` fan-out. + pub fn activate_topics(&mut self, topics: &[GossipTopic], emit: &mut impl FnMut(PeerControl)) { + for &topic in topics { + if self.our_topics.contains(&topic) { + continue; + } + self.our_topics.push(topic); + self.mesh.insert(topic, Vec::with_capacity(self.params.d_high as usize)); + for (&conn, peer) in &self.peers { + emit(PeerControl::P2pGossipSubscribe { + p2p: peer.peer_id, + p2p_connection: conn, + topic, + }); + } + } + let (attnets, syncnets) = build_subnet_masks(&self.our_topics); + self.required_attnets = attnets; + self.required_syncnets = syncnets; + } + + pub fn fan_out_subscriptions(&mut self, emit: &mut impl FnMut(PeerControl)) { + for (&conn, peer) in &self.peers { + for &topic in &self.our_topics { + emit(PeerControl::P2pGossipSubscribe { + p2p: peer.peer_id, + p2p_connection: conn, + topic, + }); + } + } + } + + /// Mesh size for a topic (for tests/introspection). + #[allow(dead_code)] + pub(crate) fn mesh_size(&self, topic: GossipTopic) -> usize { + self.mesh.get(&topic).map(|m| m.len()).unwrap_or(0) + } + + #[timed] + pub(super) fn on_subscribe( + &mut self, + conn: usize, + topic: GossipTopic, + now: Instant, + emit: &mut impl FnMut(PeerControl), + ) { + let (peer_id, score) = { + let Some(peer) = self.peers.get_mut(&conn) else { + return; + }; + peer.topics.insert(topic); + (peer.peer_id, peer.cached_score) + }; + + let we_want = self.our_topics.contains(&topic); + let mesh_size = self.mesh.get(&topic).map(|m| m.len()).unwrap_or(0); + tracing::debug!(p2p_peer = conn, ?topic, we_want, mesh_size, "PM peer subscribed"); + + // Opportunistic graft: if this is a topic we care about and our mesh + // is below d_low, pull the peer in. + if we_want && + mesh_size < self.params.d_low as usize && + score >= 0.0 && + !self.is_backed_off(conn, topic, now) + { + self.do_graft(conn, peer_id, topic, now, false, emit); + } + } + + #[timed] + pub(super) fn on_unsubscribe( + &mut self, + conn: usize, + topic: GossipTopic, + _now: Instant, + emit: &mut impl FnMut(PeerControl), + ) { + let peer_id = match self.peers.get_mut(&conn) { + Some(p) => { + p.topics.remove(&topic); + p.peer_id + } + None => return, + }; + tracing::debug!(p2p_peer = conn, ?topic, "PM peer unsubscribed"); + // If peer was in our mesh, remove them. + if self.leave_mesh(conn, topic) { + emit(PeerControl::P2pGossipPrune { p2p: peer_id, p2p_connection: conn, topic }); + } + } + + #[timed] + pub(super) fn on_remote_graft( + &mut self, + conn: usize, + topic: GossipTopic, + now: Instant, + emit: &mut impl FnMut(PeerControl), + ) { + let Some(peer) = self.peers.get(&conn) else { + if let Some(record) = self.database.by_p2p_id(conn) && + let Some(id) = record.peer_id + { + emit(PeerControl::P2pDisconnect { p2p: id, p2p_connection: conn }) + } + return; + }; + let peer_id = peer.peer_id; + let score = peer.cached_score; + let mesh_size = self.mesh.get(&topic).map(|m| m.len()).unwrap_or(0); + // No mesh-size gate: refusing at the cap makes well-behaved remotes + // retry on a 60s backoff loop forever and keeps us out of their + // meshes (no first-delivery score → pruned as excess). Reference + // gossipsub accepts and lets the heartbeat trim past d_high. + let accept = self.our_topics.contains(&topic) && + score >= 0.0 && + !self.is_backed_off(conn, topic, now); + if accept { + crate::PeerCounters::MeshGraftAcceptedByUs.inc(); + self.do_graft(conn, peer_id, topic, now, false, emit); + tracing::debug!(p2p_peer = conn, ?topic, mesh_size, "PM peer GRAFTed us: accepted"); + } else { + crate::PeerCounters::MeshGraftRefusedByUs.inc(); + self.do_prune(conn, peer_id, topic, now, "graft refused", emit); + tracing::debug!(p2p_peer = conn, ?topic, mesh_size, "PM peer GRAFTed us: refused"); + } + } + + #[timed] + pub(super) fn on_remote_prune( + &mut self, + conn: usize, + topic: GossipTopic, + now: Instant, + backoff_seconds: Option, + emit: &mut impl FnMut(PeerControl), + ) { + let meshed_since = self + .peers + .get(&conn) + .and_then(|p| p.topic_stats.get(&topic)) + .and_then(|t| t.meshed_since); + let was_in_mesh = self.leave_mesh(conn, topic); + let mesh_size = self.mesh.get(&topic).map(|peers| peers.len()).unwrap_or(0); + if !self.peers.contains_key(&conn) { + if let Some(record) = self.database.by_p2p_id(conn) && + let Some(id) = record.peer_id + { + emit(PeerControl::P2pDisconnect { p2p: id, p2p_connection: conn }) + } + return; + } + let quick = was_in_mesh && + meshed_since.is_some_and(|s| now.saturating_duration_since(s) < QUICK_PRUNE_WINDOW); + let quick_prunes = self + .peers + .get_mut(&conn) + .and_then(|p| p.topic_stats.get_mut(&topic)) + .map(|t| { + if quick { + t.quick_prunes = (t.quick_prunes + 1).min(QUICK_PRUNE_MAX_SHIFT); + } else if was_in_mesh { + t.quick_prunes = 0; + } + t.quick_prunes + }) + .unwrap_or(0); + let backoff = backoff_seconds.map(Duration::from_secs).unwrap_or(self.params.prune_backoff); + // Futility escalation — longer-than-requested backoff is spec-legal + // and stops the 61s graft/trim cycle against saturated meshes. + let backoff = backoff.checked_mul(1 << quick_prunes).unwrap_or(backoff); + self.set_backoff(conn, topic, now, backoff); + if was_in_mesh { + crate::PeerCounters::MeshPrunedByRemote.inc(); + } + let user_agent = self.peers.get(&conn).map(|p| p.user_agent).unwrap_or_default(); + tracing::debug!( + p2p_peer = conn, + ?topic, + mesh_size, + user_agent = user_agent.as_str(), + was_in_mesh, + meshed_for_ms = + ?meshed_since.map(|s| now.saturating_duration_since(s).as_millis() as u64), + backoff_s = backoff.as_secs(), + quick_prunes, + "PM peer PRUNEd us" + ); + } + + pub(super) fn on_ihave( + &mut self, + conn: usize, + hash: MessageId, + already_seen: bool, + now: Instant, + ) { + // Always count, regardless of dedup state — the rate cap treats + // a peer IHAVEing thousands of ids we already have just as badly as + // ids we don't. + let should_iwant = { + let Some(peer) = self.peers.get_mut(&conn) else { + return; + }; + peer.ihaves_received = peer.ihaves_received.saturating_add(1); + // Only send an IWANT (and thus track a promise) if: + // - we don't already have the message, + // - we haven't exceeded the per-heartbeat IWANT budget, + // - the peer hasn't saturated the IHAVE rate cap — excess ids are ignored + // without penalty (matching reference gossipsub; P7 for gossip abuse comes + // from broken promises), + // - the peer clears the gossip threshold (`on_outbound_iwant` drops the frame + // below it — a promise without a sent IWANT can only ever expire). + let should_iwant = !already_seen && + peer.ihaves_received <= self.params.max_ihave_length && + peer.cached_score >= self.params.gossip_threshold && + peer.iwant_ids_sent < self.params.max_ihave_length; + if should_iwant { + peer.iwant_ids_sent = peer.iwant_ids_sent.saturating_add(1); + } + should_iwant + }; + if !should_iwant { + return; + } + // Record the promise globally. Dedupe: same peer IHAVEing the same + // id twice is one outstanding promise, not two. + let deadline = now + self.params.iwant_followup; + let entry = self.promises.entry(hash).or_default(); + if !entry.iter().any(|(c, _)| *c == conn) { + entry.push((conn, deadline)); + } + } + + /// Peer sent us an IWANT that hit our mcache. Check retransmission + /// threshold and apply the score gate. + #[timed] + pub(super) fn on_iwant_received( + &mut self, + conn: usize, + hash: MessageId, + tcache: TCacheRead, + emit: &mut impl FnMut(PeerControl), + ) { + let Some(peer) = self.peers.get_mut(&conn) else { + return; + }; + if peer.msg_cache_insert(hash) > 2 { + // exceeds retransmission threshold + return; + } + if peer.cached_score < self.params.gossip_threshold { + return; + } + emit(PeerControl::P2pSend(P2pSend::Gossip(GossipMsgOut { peer_id: conn, tcache }))); + } + + /// Peer sent us an IDONTWANT - store the message id in the peer message + /// cache. + pub(super) fn on_idontwant_received(&mut self, conn: usize, hash: MessageId) { + let Some(peer) = self.peers.get_mut(&conn) else { + return; + }; + peer.msg_cache_insert(hash); + } + + fn credit_mesh_delivery(&mut self, conn: usize, topic: GossipTopic) -> Option { + if !self.mesh.get(&topic).is_some_and(|mesh| mesh.contains(&conn)) { + return None; + } + let peer = self.peers.get_mut(&conn)?; + peer.topic_stats.entry(topic).or_default().mesh_deliveries += 1.0; + Some(peer.peer_id) + } + + pub(super) fn on_gossip_duplicate( + &mut self, + conn: usize, + topic: GossipTopic, + hash: MessageId, + recv_ts: Nanos, + ) { + self.promises.remove(&hash); + if !self.mesh.get(&topic).is_some_and(|mesh| mesh.contains(&conn)) { + return; + } + let Some(peer_id) = self.peers.get(&conn).map(|peer| peer.peer_id) else { + return; + }; + let Some(delivery) = self.recent_deliveries.get_mut(&hash) else { + return; + }; + if delivery.topic != topic || + recv_ts.0.saturating_sub(delivery.received_at.0) > MESH_MESSAGE_DELIVERIES_WINDOW_NS || + !delivery.credit(peer_id) + { + return; + } + if let Some(peer) = self.peers.get_mut(&conn) { + peer.topic_stats.entry(topic).or_default().mesh_deliveries += 1.0; + } + } + + /// A fully-validated inbound gossip message arrived — this is the first + /// (dedup-clean) delivery from any peer. Clear all promises for this id + /// (every peer who IHAVE'd it kept their word, regardless of who + /// actually reached us first), credit P2/P3 on the delivering peer, and + /// fan out the pre-encoded IDONTWANT frame to every mesh peer except + /// the sender so they stop racing this id toward us. + #[timed] + pub(super) fn on_new_gossip( + &mut self, + sender_conn: usize, + topic: GossipTopic, + msg_hash: MessageId, + recv_ts: Nanos, + idontwant: TCacheRead, + emit: &mut impl FnMut(PeerControl), + ) { + crate::counters::GossipTopicCounters::recv(topic); + + // Any peer who promised this id is released — they did their job; + // we just got another copy from someone else first. + self.promises.remove(&msg_hash); + + if let Some(peer) = self.peers.get_mut(&sender_conn) { + let t = peer.topic_stats.entry(topic).or_default(); + // P2 — first-delivery credit (capped + weighted in `compute_score`). + t.first_deliveries += 1.0; + } + + let credited_peer = self.credit_mesh_delivery(sender_conn, topic); + if scoring::p3_scored(&topic) { + self.recent_deliveries + .insert(msg_hash, RecentDelivery::new(topic, recv_ts, credited_peer)); + } + + // Fan IDONTWANT out to mesh members (except sender) above threshold. + let Some(mesh_peers) = self.mesh.get(&topic) else { + return; + }; + for conn in mesh_peers { + if *conn == sender_conn { + continue; + } + let Some(peer) = self.peers.get(conn) else { + continue; + }; + if peer.cached_score < self.params.gossip_threshold { + continue; + } + emit(PeerControl::P2pSend(P2pSend::Gossip(GossipMsgOut { + peer_id: *conn, + tcache: idontwant, + }))); + } + } + + /// Compression tile has prepared a batched IHAVE frame for `topic`. + /// Fan it out: one `P2pGossipSend` per non-mesh subscriber whose score + /// clears `gossip_threshold`, capped at `d_lazy`. + #[timed] + pub(super) fn on_outbound_ihave( + &mut self, + topic: GossipTopic, + protobuf: TCacheRead, + emit: &mut impl FnMut(PeerControl), + ) { + let mesh_for_topic = self.mesh.get(&topic); + let cap = self.params.d_lazy as usize; + let mut emitted = 0usize; + for (conn, peer) in &self.peers { + if emitted >= cap { + break; + } + if !peer.topics.contains(&topic) { + continue; + } + if mesh_for_topic.is_some_and(|m| m.contains(conn)) { + continue; // mesh peers get full-body forwards, not IHAVE + } + if peer.cached_score < self.params.gossip_threshold { + continue; + } + emit(PeerControl::P2pSend(P2pSend::Gossip(GossipMsgOut { + peer_id: *conn, + tcache: protobuf, + }))); + emitted += 1; + } + } + + /// Compression tile has prepared an IWANT frame for a peer that just + /// sent us IHAVE. Forward it to the network tile provided the peer is + /// still live and scoring above `gossip_threshold` (mirrors rust-libp2p, + /// which ignores IHAVE — and therefore doesn't send the IWANT reply — + /// for peers below that threshold). + #[timed] + pub(super) fn on_outbound_iwant( + &mut self, + conn: usize, + tcache: TCacheRead, + emit: &mut impl FnMut(PeerControl), + ) { + let Some(peer) = self.peers.get(&conn) else { + return; + }; + if peer.cached_score < self.params.gossip_threshold { + return; + } + emit(PeerControl::P2pSend(P2pSend::Gossip(GossipMsgOut { peer_id: conn, tcache }))); + } + + #[timed] + pub(super) fn on_send_gossip( + &mut self, + sender: usize, + msg_hash: MessageId, + topic: GossipTopic, + tcache: TCacheRead, + emit: &mut impl FnMut(PeerControl), + ) { + let Some(meshed_peers) = self.mesh.get(&topic) else { + return; + }; + for peer in meshed_peers { + let Some(peer_state) = self.peers.get_mut(peer) else { + continue; + }; + peer_state.topic_stats.entry(topic).or_default().fanout_total += 1; + if *peer == sender { + continue; + } + if peer_state.cached_score < self.params.gossip_threshold { + continue; + } + if peer_state.msg_cache_contains(&msg_hash) { + // dontwant + continue; + } + peer_state.topic_stats.entry(topic).or_default().fanout_sent += 1; + crate::counters::GossipTopicCounters::sent(topic); + emit(PeerControl::P2pSend(P2pSend::Gossip(GossipMsgOut { peer_id: *peer, tcache }))); + } + } + + pub(super) fn add_invalid_delivery(&mut self, conn: usize, topic: GossipTopic) { + if let Some(peer) = self.peers.get_mut(&conn) { + let t = peer.topic_stats.entry(topic).or_default(); + t.invalid_deliveries += 1.0; + } + } + + /// Unexpired opportunistic graft: still inside the activation window, + /// so its score is structurally ~0 and proves nothing yet. + fn in_opportunistic_grace(&self, conn: usize, topic: GossipTopic, now: Instant) -> bool { + let activation = self.params.mesh_message_deliveries_activation_s; + self.peers.get(&conn).and_then(|p| p.topic_stats.get(&topic)).is_some_and(|t| { + t.opportunistic && + t.meshed_since + .is_some_and(|s| now.saturating_duration_since(s).as_secs_f64() < activation) + }) + } + + fn is_backed_off(&self, conn: usize, topic: GossipTopic, now: Instant) -> bool { + let Some(deadline) = self.peers.get(&conn).and_then(|p| p.backoffs.get(&topic)) else { + return false; + }; + deadline + .checked_add(self.params.heartbeat_interval) + .map_or(now < *deadline, |deadline_with_slack| now < deadline_with_slack) + } + + fn set_backoff(&mut self, conn: usize, topic: GossipTopic, now: Instant, backoff: Duration) { + let Some(deadline) = now.checked_add(backoff) else { + tracing::warn!(p2p_peer = conn, ?topic, ?backoff, "ignoring oversized prune backoff"); + return; + }; + let Some(peer) = self.peers.get_mut(&conn) else { return }; + peer.backoffs + .entry(topic) + .and_modify(|current| *current = (*current).max(deadline)) + .or_insert(deadline); + } + + fn do_graft( + &mut self, + conn: usize, + peer_id: PeerId, + topic: GossipTopic, + now: Instant, + opportunistic: bool, + emit: &mut impl FnMut(PeerControl), + ) { + let mesh = self + .mesh + .entry(topic) + .or_insert_with(|| Vec::with_capacity(self.params.d_high as usize)); + if mesh.contains(&conn) { + return; + } + mesh.push(conn); + // Seed per-topic state so P3 tracking kicks in after grace window. + if let Some(peer) = self.peers.get_mut(&conn) { + let t = peer.topic_stats.entry(topic).or_default(); + t.meshed_since = Some(now); + t.mesh_active = false; + t.opportunistic = opportunistic; + } + tracing::debug!(?topic, conn, "GRAFT peer"); + emit(PeerControl::P2pGossipGraft { p2p: peer_id, p2p_connection: conn, topic }); + } + + fn do_prune( + &mut self, + conn: usize, + peer_id: PeerId, + topic: GossipTopic, + now: Instant, + reason: &'static str, + emit: &mut impl FnMut(PeerControl), + ) { + let meshed_since = self + .peers + .get(&conn) + .and_then(|p| p.topic_stats.get(&topic)) + .and_then(|t| t.meshed_since); + let was_in_mesh = self.leave_mesh(conn, topic); + self.set_backoff(conn, topic, now, self.params.prune_backoff); + if was_in_mesh { + crate::PeerCounters::MeshPrunedByUs.inc(); + } + let user_agent = self.peers.get(&conn).map(|p| p.user_agent).unwrap_or_default(); + tracing::debug!( + p2p_peer = conn, + ?topic, + reason, + user_agent = user_agent.as_str(), + was_in_mesh, + meshed_for_ms = + ?meshed_since.map(|s| now.saturating_duration_since(s).as_millis() as u64), + "PRUNE peer" + ); + emit(PeerControl::P2pGossipPrune { p2p: peer_id, p2p_connection: conn, topic }); + } + + pub(super) fn leave_mesh(&mut self, conn: usize, topic: GossipTopic) -> bool { + let removed = if let Some(mesh) = self.mesh.get_mut(&topic) && + let Some(index) = mesh.iter().position(|peer| *peer == conn) + { + mesh.swap_remove(index); + true + } else { + false + }; + + if let Some(topic_score) = + self.peers.get_mut(&conn).and_then(|peer| peer.topic_stats.get_mut(&topic)) + { + let threshold = scoring::topic_params(&topic).p3_threshold; + if topic_score.mesh_active && topic_score.mesh_deliveries < threshold { + let deficit = threshold - topic_score.mesh_deliveries; + topic_score.mesh_failure_penalty += deficit * deficit; + } + topic_score.meshed_since = None; + topic_score.mesh_active = false; + } + + removed + } + + pub(super) fn heartbeat(&mut self, now: Instant) { + // Reset per-heartbeat rate-limit counters on every live peer. + for peer in self.peers.values_mut() { + peer.ihaves_received = 0; + peer.iwant_ids_sent = 0; + } + + let recv_now = Nanos::now(); + self.recent_deliveries.retain(|_, delivery| { + recv_now.0.saturating_sub(delivery.received_at.0) <= MESH_MESSAGE_DELIVERIES_WINDOW_NS + }); + + // Sweep expired promises from the global map. Expired entries + // credit `behaviour_penalty` to the peer who promised but didn't + // come through (nor did anyone else for that id). + let mut penalties: HashMap = HashMap::new(); + self.promises.retain(|_hash, waiters| { + waiters.retain(|(conn, deadline)| { + if now >= *deadline { + *penalties.entry(*conn).or_insert(0) += 1; + false + } else { + true + } + }); + !waiters.is_empty() + }); + for (conn, _count) in penalties { + // TODO seem to be over eagerly banning people here + self.add_behaviour_penalty(conn, 1.0, "broken gossip promises"); + } + } + + pub(super) fn activate_p3_where_due(&mut self, now: Instant) { + let activation = self.params.mesh_message_deliveries_activation_s; + for peer in self.peers.values_mut() { + for (topic, t) in peer.topic_stats.iter_mut() { + if scoring::p3_scored(topic) && + !t.mesh_active && + let Some(since) = t.meshed_since && + now.saturating_duration_since(since).as_secs_f64() >= activation + { + t.mesh_active = true; + } + } + } + } + + pub(super) fn manage_mesh(&mut self, now: Instant, emit: &mut impl FnMut(PeerControl)) { + // Self-heal: a mesh entry with no live PeerState means a removal + // path skipped the mesh sweep (see the graylist-evict leak). It + // suppresses grafting via a phantom degree and, once quinn recycles + // the handle, mesh-pushes to a peer that never grafted. + let peers = &self.peers; + for (topic, mesh_peers) in self.mesh.iter_mut() { + mesh_peers.retain(|conn| { + let live = peers.contains_key(conn); + if !live { + tracing::warn!(conn, ?topic, "dropping mesh entry with no peer state"); + } + live + }); + } + + // Iterate over OUR topics (topics we care about). We briefly take + // the topic list so `ensure_mesh_*` can take `&mut self`. + let our_topics = std::mem::take(&mut self.our_topics); + let opportunistic_graft_due = now.saturating_duration_since(self.last_opportunistic_graft) >= + OPPORTUNISTIC_GRAFT_INTERVAL; + for topic in &our_topics { + self.prune_negative_mesh_peers(*topic, now, emit); + self.ensure_mesh_filled(*topic, now, emit); + self.ensure_mesh_capped(*topic, now, emit); + if opportunistic_graft_due { + self.opportunistic_graft(*topic, now, emit); + } + } + self.our_topics = our_topics; + if opportunistic_graft_due { + self.last_opportunistic_graft = now; + } + } + + fn prune_negative_mesh_peers( + &mut self, + topic: GossipTopic, + now: Instant, + emit: &mut impl FnMut(PeerControl), + ) { + let peers: Vec<_> = self + .mesh + .get(&topic) + .into_iter() + .flatten() + .filter_map(|conn| { + self.peers + .get(conn) + .filter(|peer| peer.cached_score < 0.0) + .map(|peer| (*conn, peer.peer_id)) + }) + .collect(); + for (conn, peer_id) in peers { + self.do_prune(conn, peer_id, topic, now, "negative score", emit); + } + } + + fn ensure_mesh_filled( + &mut self, + topic: GossipTopic, + now: Instant, + emit: &mut impl FnMut(PeerControl), + ) { + let current = self.mesh.get(&topic).map(|m| m.len()).unwrap_or(0); + let d = self.params.d as usize; + if current >= d { + return; + } + let needed = d - current; + // Sort requires a buffer; the emit isn't what forces it. + let mut candidates: Vec = self + .peers + .iter() + .filter_map(|(conn, peer)| { + if !peer.topics.contains(&topic) { + return None; + } + if self.mesh.get(&topic).is_some_and(|m| m.contains(conn)) { + return None; + } + if peer.cached_score < 0.0 { + return None; + } + if self.is_backed_off(*conn, topic, now) { + return None; + } + Some(*conn) + }) + .collect(); + candidates.shuffle(&mut rand::thread_rng()); + for conn in candidates.into_iter().take(needed) { + let Some(peer_id) = self.peers.get(&conn).map(|p| p.peer_id) else { + continue; + }; + self.do_graft(conn, peer_id, topic, now, false, emit); + } + } + + fn ensure_mesh_capped( + &mut self, + topic: GossipTopic, + now: Instant, + emit: &mut impl FnMut(PeerControl), + ) { + let d_high = self.params.d_high as usize; + let d = self.params.d as usize; + let current = self.mesh.get(&topic).map(|m| m.len()).unwrap_or(0); + // Strictly above d_high (spec heartbeat rule): remote grafts can + // push the mesh past the cap between heartbeats; a mesh sitting at + // exactly d_high is steady state, not a prune trigger. + if current <= d_high { + return; + } + let excess = current.saturating_sub(d); + if excess == 0 { + return; + } + // Lowest selection score evicted first — no random component + // (deliberate spec deviation: proven deliverers are never displaced + // by unproven newcomers). Grace-window members rank ~0 (P3 not yet + // active), below any positive-scoring incumbent a remote-graft + // flood would otherwise displace. Unexpired opportunistic grafts + // are exempt outright — a >median-at-selection score does not + // guarantee surviving a deep trim — and are bounded (≤ + // OPPORTUNISTIC_GRAFT_PEERS per activation window), so the exempt + // set can never dominate the mesh the way blanket grace exemption + // did. + // Sort requires a buffer; the emit isn't what forces it. + let mut ranked: Vec<(usize, f64, PeerId)> = self + .mesh + .get(&topic) + .map(|mesh| { + mesh.iter() + .filter_map(|conn| { + if self.in_opportunistic_grace(*conn, topic, now) { + return None; + } + let p = self.peers.get(conn)?; + Some(( + *conn, + scoring::selection_score(p, &topic, &self.params, now), + p.peer_id, + )) + }) + .collect() + }) + .unwrap_or_default(); + ranked.sort_by(|a, b| a.1.partial_cmp(&b.1).unwrap_or(std::cmp::Ordering::Equal)); + for (conn, _, peer_id) in ranked.into_iter().take(excess) { + self.do_prune(conn, peer_id, topic, now, "mesh capped", emit); + } + } + + fn opportunistic_graft( + &mut self, + topic: GossipTopic, + now: Instant, + emit: &mut impl FnMut(PeerControl), + ) { + let Some(mesh) = self.mesh.get(&topic) else { return }; + if mesh.len() <= 1 { + return; + } + // One round per activation window: stacking exempt grafts while the + // last batch is still unproven would shrink the evictable pool. + if mesh.iter().any(|&conn| self.in_opportunistic_grace(conn, topic, now)) { + return; + } + // Median over established members only: peers meshed for less than + // the P3 activation window score near zero structurally, so counting + // them reads a freshly-built mesh as underperforming and re-grafts + // (then prunes) before anyone has a chance to establish. + let activation = self.params.mesh_message_deliveries_activation_s; + let mut mesh_scores: Vec<_> = mesh + .iter() + .filter_map(|conn| { + let peer = self.peers.get(conn)?; + let since = peer.topic_stats.get(&topic)?.meshed_since?; + (now.saturating_duration_since(since).as_secs_f64() >= activation) + .then_some(scoring::selection_score(peer, &topic, &self.params, now)) + }) + .collect(); + if mesh_scores.len() <= 1 { + return; + } + mesh_scores.sort_by(|a, b| a.partial_cmp(b).unwrap_or(std::cmp::Ordering::Equal)); + let middle = mesh_scores.len() / 2; + let median = if mesh_scores.len().is_multiple_of(2) { + (mesh_scores[middle - 1] + mesh_scores[middle]) * 0.5 + } else { + mesh_scores[middle] + }; + if median >= self.params.opportunistic_graft_threshold { + return; + } + + let mut candidates: Vec<_> = self + .peers + .iter() + .filter_map(|(conn, peer)| { + if !peer.topics.contains(&topic) || + mesh.contains(conn) || + scoring::selection_score(peer, &topic, &self.params, now) <= median || + self.is_backed_off(*conn, topic, now) + { + return None; + } + Some(*conn) + }) + .collect(); + candidates.shuffle(&mut rand::thread_rng()); + for conn in candidates.into_iter().take(OPPORTUNISTIC_GRAFT_PEERS) { + let Some(peer_id) = self.peers.get(&conn).map(|peer| peer.peer_id) else { + continue; + }; + self.do_graft(conn, peer_id, topic, now, true, emit); + } + } +} + +#[cfg(test)] +mod tests { + use silver_common::{PeerEvent, TCacheProducer}; + use silver_config::ScoreParams; + + use super::{ + super::tests::{Captured, connect, fixture, peer_id}, + *, + }; + + /// `on_connected` always emits a `P2pSend::Identify` event; filter + /// it out so subscribe-focused tests can assert on subscribe counts. + fn subscribe_events(cap: &Captured) -> Vec<&PeerControl> { + cap.0.iter().filter(|c| !matches!(c, PeerControl::P2pSend(P2pSend::Identify(_)))).collect() + } + + #[test] + fn connect_with_no_topics_emits_nothing() { + let now = Instant::now(); + let (mut mgr, mut cap) = fixture(vec![], ScoreParams::default(), false); + connect(&mut mgr, &mut cap, 1, 1, now); + assert!(subscribe_events(&cap).is_empty()); + } + + #[test] + fn connect_emits_subscribe_per_our_topic() { + let now = Instant::now(); + let topics = vec![GossipTopic::BeaconBlock, GossipTopic::VoluntaryExit]; + let (mut mgr, mut cap) = fixture(topics.clone(), ScoreParams::default(), false); + connect(&mut mgr, &mut cap, 1, 1, now); + let subs = subscribe_events(&cap); + assert_eq!(subs.len(), 2); + for e in &subs { + assert!(matches!(e, PeerControl::P2pGossipSubscribe { .. })); + } + } + + #[test] + fn peer_subscribes_and_we_graft_when_mesh_under_d_low() { + let now = Instant::now(); + let topics = vec![GossipTopic::BeaconBlock]; + let (mut mgr, mut cap) = fixture(topics, ScoreParams::default(), false); + connect(&mut mgr, &mut cap, 1, 1, now); + cap.0.clear(); + + mgr.handle_event( + PeerEvent::P2pGossipTopicSubscribe { p2p_peer: 1, topic: GossipTopic::BeaconBlock }, + now, + &mut |c| cap.0.push(c), + ); + + assert!( + cap.0.iter().any(|e| matches!( + e, + PeerControl::P2pGossipGraft { topic, .. } if *topic == GossipTopic::BeaconBlock + )), + "expected a GRAFT, got {:?}", + cap.0 + ); + assert_eq!(mgr.mesh_size(GossipTopic::BeaconBlock), 1); + } + + #[test] + fn negative_score_subscriber_is_not_grafted() { + let now = Instant::now(); + let topic = GossipTopic::BeaconBlock; + let (mut mgr, mut cap) = fixture(vec![topic], ScoreParams::default(), false); + connect(&mut mgr, &mut cap, 1, 1, now); + mgr.peers.get_mut(&1).unwrap().cached_score = -0.1; + cap.0.clear(); + + mgr.handle_event( + PeerEvent::P2pGossipTopicSubscribe { p2p_peer: 1, topic }, + now, + &mut |event| cap.0.push(event), + ); + + assert_eq!(mgr.mesh_size(topic), 0); + assert!(!cap.0.iter().any(|event| matches!(event, PeerControl::P2pGossipGraft { .. }))); + + mgr.handle_event( + PeerEvent::P2pGossipTopicGraft { p2p_peer: 1, topic }, + now, + &mut |event| cap.0.push(event), + ); + + assert_eq!(mgr.mesh_size(topic), 0); + assert!(cap.0.iter().any(|event| matches!( + event, + PeerControl::P2pGossipPrune { p2p_connection: 1, topic: pruned, .. } + if *pruned == topic + ))); + } + + #[test] + fn remote_graft_accepted_past_d_high() { + let now = Instant::now(); + let topic = GossipTopic::BeaconBlock; + let mut params = ScoreParams::default(); + params.d_low = 0; + params.d = 0; + params.d_high = 2; + let (mut mgr, mut cap) = fixture(vec![topic], params, false); + for i in 1..=3u8 { + connect(&mut mgr, &mut cap, i as usize, i, now); + mgr.handle_event( + PeerEvent::P2pGossipTopicSubscribe { p2p_peer: i as usize, topic }, + now, + &mut |event| cap.0.push(event), + ); + } + mgr.mesh.entry(topic).or_default().extend([1, 2]); + cap.0.clear(); + + mgr.handle_event( + PeerEvent::P2pGossipTopicGraft { p2p_peer: 3, topic }, + now, + &mut |event| cap.0.push(event), + ); + + assert_eq!(mgr.mesh_size(topic), 3); + assert!( + !cap.0.iter().any(|event| matches!(event, PeerControl::P2pGossipPrune { + p2p_connection: 3, + .. + })) + ); + } + + #[test] + fn negative_mesh_peer_is_pruned_before_refill() { + let now = Instant::now(); + let topic = GossipTopic::BeaconBlock; + let (mut mgr, mut cap) = fixture(vec![topic], ScoreParams::default(), false); + connect(&mut mgr, &mut cap, 1, 1, now); + mgr.handle_event( + PeerEvent::P2pGossipTopicSubscribe { p2p_peer: 1, topic }, + now, + &mut |event| cap.0.push(event), + ); + assert_eq!(mgr.mesh_size(topic), 1); + mgr.peers.get_mut(&1).unwrap().application_score = -1.0; + cap.0.clear(); + + mgr.tick(now + Duration::from_secs(1), &mut |event| cap.0.push(event)); + + assert_eq!(mgr.mesh_size(topic), 0); + assert!(cap.0.iter().any(|event| matches!( + event, + PeerControl::P2pGossipPrune { p2p_connection: 1, topic: pruned, .. } + if *pruned == topic + ))); + } + + #[test] + fn randomized_refill_uses_only_eligible_peers() { + let now = Instant::now(); + let topic = GossipTopic::BeaconBlock; + let mut params = ScoreParams::default(); + params.d = 4; + params.d_low = 0; + let (mut mgr, mut cap) = fixture(vec![topic], params, false); + for conn in 1..=6 { + connect(&mut mgr, &mut cap, conn, conn as u8, now); + mgr.handle_event( + PeerEvent::P2pGossipTopicSubscribe { p2p_peer: conn, topic }, + now, + &mut |event| cap.0.push(event), + ); + mgr.peers.get_mut(&conn).unwrap().cached_score = conn as f64; + } + mgr.peers.get_mut(&1).unwrap().cached_score = -0.1; + mgr.set_backoff(2, topic, now, Duration::from_secs(60)); + + mgr.ensure_mesh_filled(topic, now, &mut |event| cap.0.push(event)); + + let mesh = &mgr.mesh[&topic]; + assert_eq!(mesh.len(), 4); + assert!(!mesh.contains(&1)); + assert!(!mesh.contains(&2)); + assert!(mesh.iter().all(|conn| (3..=6).contains(conn))); + } + + #[test] + fn capping_spares_opportunistic_grafts_within_window() { + let now = Instant::now(); + let topic = GossipTopic::BeaconBlock; + let mut params = ScoreParams::default(); + params.d = 2; + params.d_low = 0; + params.d_high = 3; + let (mut mgr, mut cap) = fixture(vec![topic], params, false); + for conn in 1..=4 { + connect(&mut mgr, &mut cap, conn, conn as u8, now); + mgr.peers.get_mut(&conn).unwrap().cached_score = conn as f64; + } + // 1..=3 are incumbents; 4 is an opportunistic graft with the lowest + // effective score — exempt, so the trim falls on incumbent 1. + for conn in 1..=3 { + mgr.do_graft(conn, peer_id(conn as u8), topic, now, false, &mut |event| { + cap.0.push(event) + }); + } + mgr.do_graft(4, peer_id(4), topic, now, true, &mut |event| cap.0.push(event)); + mgr.peers.get_mut(&4).unwrap().cached_score = 0.0; + cap.0.clear(); + + mgr.ensure_mesh_capped(topic, now, &mut |event| cap.0.push(event)); + let mesh = &mgr.mesh[&topic]; + assert!(mesh.contains(&4)); + assert!(!mesh.contains(&1)); + + // Past the activation window the exemption lapses: lowest score + // (still peer 4) is evicted like anyone else. + mgr.do_graft(1, peer_id(1), topic, now, false, &mut |event| cap.0.push(event)); + mgr.do_graft(2, peer_id(2), topic, now, false, &mut |event| cap.0.push(event)); + let settled = + now + Duration::from_secs_f64(mgr.params.mesh_message_deliveries_activation_s); + mgr.ensure_mesh_capped(topic, settled, &mut |event| cap.0.push(event)); + assert!(!mgr.mesh[&topic].contains(&4)); + } + + #[test] + fn capping_evicts_lowest_scores_first() { + let now = Instant::now(); + let topic = GossipTopic::BeaconBlock; + let mut params = ScoreParams::default(); + params.d = 8; + params.d_low = 0; + params.d_high = 12; + let (mut mgr, mut cap) = fixture(vec![topic], params, false); + for conn in 1..=12 { + connect(&mut mgr, &mut cap, conn, conn as u8, now); + mgr.do_graft(conn, peer_id(conn as u8), topic, now, false, &mut |event| { + cap.0.push(event) + }); + mgr.peers.get_mut(&conn).unwrap().cached_score = conn as f64; + } + cap.0.clear(); + + mgr.ensure_mesh_capped(topic, now, &mut |event| cap.0.push(event)); + assert_eq!(mgr.mesh[&topic].len(), 12); + assert!(cap.0.is_empty()); + + connect(&mut mgr, &mut cap, 13, 13, now); + mgr.do_graft(13, peer_id(13), topic, now, false, &mut |event| cap.0.push(event)); + mgr.peers.get_mut(&13).unwrap().cached_score = 13.0; + cap.0.clear(); + + // Over cap: trims to d immediately — grace-window members are + // eligible victims, top scorers retained. + mgr.ensure_mesh_capped(topic, now, &mut |event| cap.0.push(event)); + + let mesh = &mgr.mesh[&topic]; + assert_eq!(mesh.len(), 8); + assert!((6..=13).all(|conn| mesh.contains(&conn))); + assert_eq!( + cap.0 + .iter() + .filter(|event| matches!(event, PeerControl::P2pGossipPrune { .. })) + .count(), + 5 + ); + } + + #[test] + fn opportunistic_graft_runs_on_duration_and_selects_above_median() { + let now = Instant::now(); + let topic = GossipTopic::BeaconBlock; + let mut params = ScoreParams::default(); + params.d = 2; + params.d_low = 0; + params.d_high = 8; + let (mut mgr, mut cap) = fixture(vec![topic], params, false); + for conn in 1..=5 { + connect(&mut mgr, &mut cap, conn, conn as u8, now); + mgr.handle_event( + PeerEvent::P2pGossipTopicSubscribe { p2p_peer: conn, topic }, + now, + &mut |event| cap.0.push(event), + ); + } + mgr.do_graft(1, peer_id(1), topic, now, false, &mut |event| cap.0.push(event)); + mgr.do_graft(2, peer_id(2), topic, now, false, &mut |event| cap.0.push(event)); + for (conn, score) in [(1, 1.0), (2, 2.0), (3, 3.0), (4, 4.0), (5, 1.4)] { + mgr.peers.get_mut(&conn).unwrap().cached_score = score; + } + let due = mgr.last_opportunistic_graft + OPPORTUNISTIC_GRAFT_INTERVAL; + + mgr.manage_mesh(due - Duration::from_nanos(1), &mut |event| cap.0.push(event)); + assert_eq!(mgr.mesh_size(topic), 2); + + // Due, but both members are inside the activation window: no median, + // no graft. + mgr.manage_mesh(due, &mut |event| cap.0.push(event)); + assert_eq!(mgr.mesh_size(topic), 2); + assert_eq!(mgr.last_opportunistic_graft, due); + + // Past the activation window the members' scores count. + let established = + due + Duration::from_secs_f64(mgr.params.mesh_message_deliveries_activation_s); + mgr.manage_mesh(established, &mut |event| cap.0.push(event)); + + let mesh = &mgr.mesh[&topic]; + assert_eq!(mesh.len(), 4); + assert!(mesh.contains(&3)); + assert!(mesh.contains(&4)); + assert!(!mesh.contains(&5)); + } + + #[test] + fn quick_prune_escalates_backoff_and_long_residency_resets() { + let now = Instant::now(); + let topic = GossipTopic::BeaconBlock; + let (mut mgr, mut cap) = fixture(vec![topic], ScoreParams::default(), false); + connect(&mut mgr, &mut cap, 1, 1, now); + + // Graft, then pruned 1s later (their heartbeat trim): base backoff + // doubles. + mgr.do_graft(1, peer_id(1), topic, now, false, &mut |event| cap.0.push(event)); + let pruned_at = now + Duration::from_secs(1); + mgr.handle_event( + PeerEvent::P2pGossipTopicPrune { p2p_peer: 1, topic, backoff_seconds: Some(60) }, + pruned_at, + &mut |event| cap.0.push(event), + ); + assert_eq!(mgr.peers[&1].topic_stats[&topic].quick_prunes, 1); + assert_eq!(mgr.peers[&1].backoffs[&topic], pruned_at + Duration::from_secs(120)); + + // Second quick trim: ×4. + mgr.peers.get_mut(&1).unwrap().backoffs.clear(); + mgr.do_graft(1, peer_id(1), topic, pruned_at, false, &mut |event| cap.0.push(event)); + let pruned_again = pruned_at + Duration::from_secs(1); + mgr.handle_event( + PeerEvent::P2pGossipTopicPrune { p2p_peer: 1, topic, backoff_seconds: Some(60) }, + pruned_again, + &mut |event| cap.0.push(event), + ); + assert_eq!(mgr.peers[&1].topic_stats[&topic].quick_prunes, 2); + assert_eq!(mgr.peers[&1].backoffs[&topic], pruned_again + Duration::from_secs(240)); + + // A graft that outlives the window resets the escalation. + mgr.peers.get_mut(&1).unwrap().backoffs.clear(); + mgr.do_graft(1, peer_id(1), topic, pruned_again, false, &mut |event| cap.0.push(event)); + let pruned_late = pruned_again + QUICK_PRUNE_WINDOW + Duration::from_secs(1); + mgr.handle_event( + PeerEvent::P2pGossipTopicPrune { p2p_peer: 1, topic, backoff_seconds: Some(60) }, + pruned_late, + &mut |event| cap.0.push(event), + ); + assert_eq!(mgr.peers[&1].topic_stats[&topic].quick_prunes, 0); + assert_eq!(mgr.peers[&1].backoffs[&topic], pruned_late + Duration::from_secs(60)); + } + + #[test] + fn received_prune_honors_full_backoff_and_heartbeat_slack() { + let now = Instant::now(); + let topic = GossipTopic::BeaconBlock; + let params = ScoreParams::default(); + let heartbeat = params.heartbeat_interval; + let (mut mgr, mut cap) = fixture(vec![topic], params, false); + connect(&mut mgr, &mut cap, 1, 1, now); + mgr.handle_event( + PeerEvent::P2pGossipTopicPrune { p2p_peer: 1, topic, backoff_seconds: Some(7200) }, + now, + &mut |event| cap.0.push(event), + ); + let original_deadline = mgr.peers[&1].backoffs[&topic]; + + mgr.handle_event( + PeerEvent::P2pGossipTopicPrune { p2p_peer: 1, topic, backoff_seconds: Some(60) }, + now + Duration::from_secs(1), + &mut |event| cap.0.push(event), + ); + + assert_eq!(mgr.peers[&1].backoffs[&topic], original_deadline); + let end_with_slack = now + Duration::from_secs(7200) + heartbeat; + assert!(mgr.is_backed_off(1, topic, end_with_slack - Duration::from_nanos(1))); + assert!(!mgr.is_backed_off(1, topic, end_with_slack)); + + connect(&mut mgr, &mut cap, 2, 2, now); + mgr.handle_event( + PeerEvent::P2pGossipTopicPrune { p2p_peer: 2, topic, backoff_seconds: Some(u64::MAX) }, + now, + &mut |event| cap.0.push(event), + ); + assert!(!mgr.peers[&2].backoffs.contains_key(&topic)); + } + + #[test] + fn unsubscribe_preserves_reputation_and_squares_mesh_failure() { + let now = Instant::now(); + let topic = GossipTopic::BeaconBlock; + let (mut mgr, mut cap) = fixture(vec![topic], ScoreParams::default(), false); + connect(&mut mgr, &mut cap, 1, 1, now); + mgr.handle_event( + PeerEvent::P2pGossipTopicSubscribe { p2p_peer: 1, topic }, + now, + &mut |event| cap.0.push(event), + ); + let topic_score = mgr.peers.get_mut(&1).unwrap().topic_stats.get_mut(&topic).unwrap(); + topic_score.first_deliveries = 2.0; + topic_score.mesh_deliveries = 0.2; + topic_score.mesh_active = true; + topic_score.invalid_deliveries = 3.0; + + mgr.handle_event( + PeerEvent::P2pGossipTopicUnsubscribe { p2p_peer: 1, topic }, + now, + &mut |event| cap.0.push(event), + ); + + let topic_score = &mgr.peers[&1].topic_stats[&topic]; + let deficit = scoring::topic_params(&topic).p3_threshold - 0.2; + assert_eq!(mgr.mesh_size(topic), 0); + assert_eq!(topic_score.first_deliveries, 2.0); + assert_eq!(topic_score.invalid_deliveries, 3.0); + assert_eq!(topic_score.mesh_failure_penalty, deficit * deficit); + assert!(topic_score.meshed_since.is_none()); + assert!(!topic_score.mesh_active); + } + + #[test] + fn remote_prune_squares_mesh_failure() { + let now = Instant::now(); + let topic = GossipTopic::BeaconBlock; + let (mut mgr, mut cap) = fixture(vec![topic], ScoreParams::default(), false); + connect(&mut mgr, &mut cap, 1, 1, now); + mgr.handle_event( + PeerEvent::P2pGossipTopicSubscribe { p2p_peer: 1, topic }, + now, + &mut |event| cap.0.push(event), + ); + let topic_score = mgr.peers.get_mut(&1).unwrap().topic_stats.get_mut(&topic).unwrap(); + topic_score.mesh_deliveries = 0.1; + topic_score.mesh_active = true; + + mgr.handle_event( + PeerEvent::P2pGossipTopicPrune { p2p_peer: 1, topic, backoff_seconds: Some(60) }, + now, + &mut |event| cap.0.push(event), + ); + + let topic_score = &mgr.peers[&1].topic_stats[&topic]; + let deficit = scoring::topic_params(&topic).p3_threshold - 0.1; + assert_eq!(topic_score.mesh_failure_penalty, deficit * deficit); + assert!(topic_score.meshed_since.is_none()); + assert!(!topic_score.mesh_active); + } + + #[test] + fn disconnect_squares_mesh_failure_before_archiving() { + let now = Instant::now(); + let topic = GossipTopic::BeaconBlock; + let id = peer_id(1); + let (mut mgr, mut cap) = fixture(vec![topic], ScoreParams::default(), false); + connect(&mut mgr, &mut cap, 1, 1, now); + mgr.handle_event( + PeerEvent::P2pGossipTopicSubscribe { p2p_peer: 1, topic }, + now, + &mut |event| cap.0.push(event), + ); + let topic_score = mgr.peers.get_mut(&1).unwrap().topic_stats.get_mut(&topic).unwrap(); + topic_score.mesh_deliveries = 0.3; + topic_score.mesh_active = true; + + mgr.handle_event( + PeerEvent::P2pDisconnect { p2p_peer: 1, peer_id: id }, + now, + &mut |event| cap.0.push(event), + ); + + let topic_score = &mgr.archived[&id].topic_stats[&topic]; + let deficit = scoring::topic_params(&topic).p3_threshold - 0.3; + assert_eq!(mgr.mesh_size(topic), 0); + assert_eq!(topic_score.mesh_failure_penalty, deficit * deficit); + assert!(topic_score.meshed_since.is_none()); + assert!(!topic_score.mesh_active); + } + + #[test] + fn ihave_below_gossip_threshold_records_no_promise() { + let now = Instant::now(); + let params = ScoreParams::default(); + let (mut mgr, mut cap) = fixture(vec![], params, false); + connect(&mut mgr, &mut cap, 1, 1, now); + mgr.peers.get_mut(&1).unwrap().cached_score = mgr.params.gossip_threshold - 1.0; + + let hash = silver_common::MessageId { id: [7u8; 20] }; + mgr.handle_event( + PeerEvent::P2pGossipHave { + p2p_peer: 1, + topic: GossipTopic::BeaconBlock, + hash, + already_seen: false, + }, + now, + &mut |c| cap.0.push(c), + ); + assert!(mgr.promises.is_empty()); + } + + #[test] + fn duplicate_delivery_clears_promise() { + let now = Instant::now(); + let params = ScoreParams::default(); + let (mut mgr, mut cap) = fixture(vec![], params, false); + connect(&mut mgr, &mut cap, 1, 1, now); + + let hash = silver_common::MessageId { id: [7u8; 20] }; + mgr.handle_event( + PeerEvent::P2pGossipHave { + p2p_peer: 1, + topic: GossipTopic::BeaconBlock, + hash, + already_seen: false, + }, + now, + &mut |c| cap.0.push(c), + ); + assert_eq!(mgr.promises.len(), 1); + + mgr.handle_event( + PeerEvent::GossipDuplicate { + p2p_peer: 1, + topic: GossipTopic::BeaconBlock, + hash, + recv_ts: Nanos::now(), + }, + now, + &mut |c| cap.0.push(c), + ); + assert!(mgr.promises.is_empty()); + } + + #[test] + fn broken_promise_sweep_adds_penalty() { + let mut now = Instant::now(); + let mut params = ScoreParams::default(); + params.iwant_followup = Duration::from_secs(3); + params.heartbeat_interval = Duration::from_millis(100); + let (mut mgr, mut cap) = fixture(vec![], params, false); + connect(&mut mgr, &mut cap, 1, 1, now); + + let hash = silver_common::MessageId { id: [7u8; 20] }; + mgr.handle_event( + PeerEvent::P2pGossipHave { + p2p_peer: 1, + topic: GossipTopic::BeaconBlock, + hash, + already_seen: false, + }, + now, + &mut |c| cap.0.push(c), + ); + + now += Duration::from_secs(4); + mgr.tick(now, &mut |c| cap.0.push(c)); + + let s = mgr.score(1).unwrap(); + assert!(s <= 0.0, "expected non-positive score after broken promise, got {s}"); + } + + /// Over-cap IHAVEs are ignored without penalty — no P7, and no promise + /// (an unfulfillable promise would surface later as a broken-promise + /// penalty instead). + #[test] + fn ihave_flood_over_heartbeat_cap_ignored() { + let now = Instant::now(); + let mut params = ScoreParams::default(); + params.max_ihave_length = 3; + params.graylist_threshold = -100_000.0; + let (mut mgr, mut cap) = fixture(vec![], params, false); + connect(&mut mgr, &mut cap, 1, 1, now); + + for i in 0..8u8 { + mgr.handle_event( + PeerEvent::P2pGossipHave { + p2p_peer: 1, + topic: GossipTopic::BeaconBlock, + hash: silver_common::MessageId { id: [i; 20] }, + already_seen: true, + }, + now, + &mut |c| cap.0.push(c), + ); + } + mgr.tick(now + Duration::from_millis(100), &mut |c| cap.0.push(c)); + let s = mgr.score(1).unwrap(); + assert!(s >= 0.0, "expected no penalty for over-cap IHAVEs, got {s}"); + } + + #[test] + fn already_seen_ihave_tracks_no_promise() { + let mut now = Instant::now(); + let mut params = ScoreParams::default(); + params.iwant_followup = Duration::from_secs(3); + params.heartbeat_interval = Duration::from_millis(100); + let (mut mgr, mut cap) = fixture(vec![], params, false); + connect(&mut mgr, &mut cap, 1, 1, now); + + let hash = silver_common::MessageId { id: [77u8; 20] }; + mgr.handle_event( + PeerEvent::P2pGossipHave { + p2p_peer: 1, + topic: GossipTopic::BeaconBlock, + hash, + already_seen: true, + }, + now, + &mut |c| cap.0.push(c), + ); + + now += Duration::from_secs(5); + mgr.tick(now, &mut |c| cap.0.push(c)); + let s = mgr.score(1).unwrap(); + assert_eq!(s, 0.0, "no IWANT was issued → no promise → no broken-promise penalty, got {s}"); + } + + fn mk_tcache_read() -> silver_common::TCacheRead { + let mut producer = silver_common::TCache::producer("test_peer", 1 << 14); + let mut reservation = producer.reserve(64, true).unwrap(); + use std::io::Write as _; + reservation.write_all(&[0u8; 64]).unwrap(); + reservation.read() + } + + #[test] + fn near_first_mesh_deliveries_credit_each_peer_once() { + let now = Instant::now(); + let topic = GossipTopic::BeaconBlock; + let mut params = ScoreParams::default(); + params.d_low = 0; + params.d = 0; + let (mut mgr, mut cap) = fixture(vec![topic], params, false); + for conn in 1..=3 { + connect(&mut mgr, &mut cap, conn, conn as u8, now); + mgr.do_graft(conn, peer_id(conn as u8), topic, now, false, &mut |event| { + cap.0.push(event) + }); + } + let hash = MessageId { id: [11u8; 20] }; + let first_seen = Nanos::from_secs(10); + + mgr.handle_event( + PeerEvent::NewGossip { + p2p_peer: 1, + topic, + msg_hash: hash, + recv_ts: first_seen, + idontwant: mk_tcache_read(), + }, + now, + &mut |event| cap.0.push(event), + ); + mgr.handle_event( + PeerEvent::GossipDuplicate { + p2p_peer: 2, + topic, + hash, + recv_ts: Nanos(first_seen.0 + 1_000_000_000), + }, + now, + &mut |event| cap.0.push(event), + ); + mgr.handle_event( + PeerEvent::GossipDuplicate { + p2p_peer: 2, + topic, + hash, + recv_ts: Nanos(first_seen.0 + 1_500_000_000), + }, + now, + &mut |event| cap.0.push(event), + ); + mgr.handle_event( + PeerEvent::GossipDuplicate { + p2p_peer: 3, + topic, + hash, + recv_ts: Nanos(first_seen.0 + 2_000_000_001), + }, + now, + &mut |event| cap.0.push(event), + ); + + assert_eq!(mgr.peers[&1].topic_stats[&topic].mesh_deliveries, 1.0); + assert_eq!(mgr.peers[&2].topic_stats[&topic].mesh_deliveries, 1.0); + assert_eq!(mgr.peers[&3].topic_stats[&topic].mesh_deliveries, 0.0); + } + + #[test] + fn new_inbound_fulfils_promise_and_credits_p2() { + let mut now = Instant::now(); + let mut params = ScoreParams::default(); + params.iwant_followup = Duration::from_secs(3); + params.heartbeat_interval = Duration::from_millis(100); + let (mut mgr, mut cap) = fixture(vec![], params, false); + connect(&mut mgr, &mut cap, 1, 1, now); + + let hash = silver_common::MessageId { id: [7u8; 20] }; + mgr.handle_event( + PeerEvent::P2pGossipHave { + p2p_peer: 1, + topic: GossipTopic::BeaconBlock, + hash, + already_seen: false, + }, + now, + &mut |c| cap.0.push(c), + ); + + now += Duration::from_secs(1); + mgr.handle_event( + PeerEvent::NewGossip { + p2p_peer: 1, + topic: GossipTopic::BeaconBlock, + msg_hash: hash, + recv_ts: Nanos::now(), + idontwant: mk_tcache_read(), + }, + now, + &mut |c| cap.0.push(c), + ); + + now += Duration::from_secs(5); + mgr.tick(now, &mut |c| cap.0.push(c)); + let score_delivered = mgr.score(1).unwrap(); + + connect(&mut mgr, &mut cap, 2, 2, now); + let other = silver_common::MessageId { id: [8u8; 20] }; + mgr.handle_event( + PeerEvent::P2pGossipHave { + p2p_peer: 2, + topic: GossipTopic::BeaconBlock, + hash: other, + already_seen: false, + }, + now, + &mut |c| cap.0.push(c), + ); + now += Duration::from_secs(5); + mgr.tick(now, &mut |c| cap.0.push(c)); + let score_broken = mgr.score(2).unwrap(); + + assert!( + score_delivered > score_broken, + "delivered peer must score above broken-promise peer: \ + delivered={score_delivered}, broken={score_broken}" + ); + } + + #[test] + fn delivery_from_one_peer_fulfils_all_peers_promises_for_that_id() { + let mut now = Instant::now(); + let mut params = ScoreParams::default(); + params.iwant_followup = Duration::from_secs(3); + params.heartbeat_interval = Duration::from_millis(100); + let (mut mgr, mut cap) = fixture(vec![], params, false); + connect(&mut mgr, &mut cap, 1, 1, now); + connect(&mut mgr, &mut cap, 2, 2, now); + + let hash = silver_common::MessageId { id: [42u8; 20] }; + mgr.handle_event( + PeerEvent::P2pGossipHave { + p2p_peer: 1, + topic: GossipTopic::BeaconBlock, + hash, + already_seen: false, + }, + now, + &mut |c| cap.0.push(c), + ); + mgr.handle_event( + PeerEvent::P2pGossipHave { + p2p_peer: 2, + topic: GossipTopic::BeaconBlock, + hash, + already_seen: false, + }, + now, + &mut |c| cap.0.push(c), + ); + + now += Duration::from_secs(1); + mgr.handle_event( + PeerEvent::NewGossip { + p2p_peer: 1, + topic: GossipTopic::BeaconBlock, + msg_hash: hash, + recv_ts: Nanos::now(), + idontwant: mk_tcache_read(), + }, + now, + &mut |c| cap.0.push(c), + ); + + now += Duration::from_secs(5); + mgr.tick(now, &mut |c| cap.0.push(c)); + + connect(&mut mgr, &mut cap, 3, 3, now); + let other = silver_common::MessageId { id: [99u8; 20] }; + mgr.handle_event( + PeerEvent::P2pGossipHave { + p2p_peer: 3, + topic: GossipTopic::BeaconBlock, + hash: other, + already_seen: false, + }, + now, + &mut |c| cap.0.push(c), + ); + now += Duration::from_secs(5); + mgr.tick(now, &mut |c| cap.0.push(c)); + + let s1 = mgr.score(1).unwrap(); + let s2 = mgr.score(2).unwrap(); + let s3 = mgr.score(3).unwrap(); + assert!(s1 > s3, "delivering peer must out-score broken-promise peer: {s1} vs {s3}"); + assert!( + s2 > s3, + "promise-fulfilled-by-other-peer must out-score broken-promise peer: {s2} vs {s3}" + ); + } + + #[test] + fn new_outbound_ihave_fans_out_to_non_mesh_subscribers() { + let now = Instant::now(); + let mut params = ScoreParams::default(); + params.d_lazy = 3; + params.d_low = 1; // so the first subscriber grafts into mesh, rest stay non-mesh + let (mut mgr, mut cap) = fixture(vec![GossipTopic::BeaconBlock], params, false); + + for i in 1..=4u8 { + connect(&mut mgr, &mut cap, i as usize, i, now); + mgr.handle_event( + PeerEvent::P2pGossipTopicSubscribe { + p2p_peer: i as usize, + topic: GossipTopic::BeaconBlock, + }, + now, + &mut |c| cap.0.push(c), + ); + } + assert_eq!(mgr.mesh_size(GossipTopic::BeaconBlock), 1); + cap.0.clear(); + + mgr.handle_event( + PeerEvent::OutboundIHave { + topic: GossipTopic::BeaconBlock, + msg_count: 2, + protobuf: mk_tcache_read(), + }, + now, + &mut |c| cap.0.push(c), + ); + + let send_ihaves: Vec<_> = cap + .0 + .iter() + .filter_map(|e| { + if let PeerControl::P2pSend(P2pSend::Gossip(GossipMsgOut { peer_id, .. })) = e { + Some(*peer_id) + } else { + None + } + }) + .collect(); + assert_eq!( + send_ihaves.len(), + 3, + "expected 3 IHAVE emissions (d_lazy=3, 3 non-mesh subscribers), got {:?}", + cap.0 + ); + assert!( + send_ihaves.iter().all(|c| !matches!(c, &1)), + "mesh peer (conn=1) must not receive IHAVE, got {send_ihaves:?}" + ); + } + + #[test] + fn new_outbound_ihave_skips_below_threshold_peers() { + let now = Instant::now(); + let mut params = ScoreParams::default(); + params.gossip_threshold = -1.0; + params.graylist_threshold = -1_000_000.0; + let (mut mgr, mut cap) = fixture(vec![GossipTopic::BeaconBlock], params, false); + connect(&mut mgr, &mut cap, 1, 1, now); + mgr.handle_event( + PeerEvent::P2pGossipTopicSubscribe { p2p_peer: 1, topic: GossipTopic::BeaconBlock }, + now, + &mut |c| cap.0.push(c), + ); + for _ in 0..5 { + mgr.handle_event(PeerEvent::P2pGossipInvalidFrame { p2p_peer: 1 }, now, &mut |c| { + cap.0.push(c) + }); + } + mgr.tick(now + Duration::from_millis(10), &mut |c| cap.0.push(c)); + assert!(mgr.score(1).unwrap() < -1.0); + + cap.0.clear(); + mgr.handle_event( + PeerEvent::OutboundIHave { + topic: GossipTopic::BeaconBlock, + msg_count: 1, + protobuf: mk_tcache_read(), + }, + now + Duration::from_millis(20), + &mut |c| cap.0.push(c), + ); + assert!( + !cap.0 + .iter() + .any(|e| matches!(e, PeerControl::P2pSend(P2pSend::Gossip(GossipMsgOut { .. })))), + "below-threshold peer should not receive IHAVE, got {:?}", + cap.0 + ); + } + + #[test] + fn iwant_request_above_threshold_emits_forward() { + use silver_common::{TCache, TCacheRead}; + let now = Instant::now(); + let (mut mgr, mut cap) = fixture(vec![], ScoreParams::default(), false); + connect(&mut mgr, &mut cap, 1, 1, now); + + let mut producer = TCache::producer("test_peer", 1 << 14); + let mut reservation = producer.reserve(64, true).unwrap(); + use std::io::Write as _; + reservation.write_all(&[0u8; 64]).unwrap(); + let tcache: TCacheRead = reservation.read(); + + cap.0.clear(); + let hash = silver_common::MessageId { id: [3u8; 20] }; + mgr.handle_event(PeerEvent::P2pGossipWant { p2p_peer: 1, hash, tcache }, now, &mut |c| { + cap.0.push(c) + }); + + assert!( + cap.0.iter().any(|e| matches!( + e, + PeerControl::P2pSend(P2pSend::Gossip(GossipMsgOut { peer_id: 1, .. })) + )), + "expected ForwardMsg emission, got {:?}", + cap.0 + ); + } + + #[test] + fn iwant_request_below_threshold_drops() { + use silver_common::{TCache, TCacheRead}; + let now = Instant::now(); + let mut params = ScoreParams::default(); + params.gossip_threshold = -1.0; + params.graylist_threshold = -1_000_000.0; + let (mut mgr, mut cap) = fixture(vec![], params, false); + connect(&mut mgr, &mut cap, 1, 1, now); + + for _ in 0..5 { + mgr.handle_event(PeerEvent::P2pGossipInvalidFrame { p2p_peer: 1 }, now, &mut |c| { + cap.0.push(c) + }); + } + mgr.tick(now + Duration::from_millis(10), &mut |c| cap.0.push(c)); + assert!(mgr.score(1).unwrap() < -1.0); + + let mut producer = TCache::producer("test_peer", 1 << 14); + let mut reservation = producer.reserve(64, true).unwrap(); + use std::io::Write as _; + reservation.write_all(&[0u8; 64]).unwrap(); + let tcache: TCacheRead = reservation.read(); + + cap.0.clear(); + let hash = silver_common::MessageId { id: [4u8; 20] }; + mgr.handle_event( + PeerEvent::P2pGossipWant { p2p_peer: 1, hash, tcache }, + now + Duration::from_millis(20), + &mut |c| cap.0.push(c), + ); + + assert!( + !cap.0 + .iter() + .any(|e| matches!(e, PeerControl::P2pSend(P2pSend::Gossip(GossipMsgOut { .. })))), + "expected no ForwardMsg for below-threshold peer, got {:?}", + cap.0 + ); + } + + #[test] + fn new_inbound_fans_out_dontwant_to_mesh_excluding_sender() { + let now = Instant::now(); + let mut params = ScoreParams::default(); + params.d_low = 0; + params.d = 0; + params.d_high = 8; + let (mut mgr, mut cap) = fixture(vec![GossipTopic::BeaconBlock], params, false); + + for i in 1..=4u8 { + connect(&mut mgr, &mut cap, i as usize, i, now); + mgr.handle_event( + PeerEvent::P2pGossipTopicSubscribe { + p2p_peer: i as usize, + topic: GossipTopic::BeaconBlock, + }, + now, + &mut |c| cap.0.push(c), + ); + } + for i in 1..=4usize { + mgr.mesh.entry(GossipTopic::BeaconBlock).or_default().push(i); + } + cap.0.clear(); + + let hash = silver_common::MessageId { id: [55u8; 20] }; + mgr.handle_event( + PeerEvent::NewGossip { + p2p_peer: 2, + topic: GossipTopic::BeaconBlock, + msg_hash: hash, + recv_ts: Nanos::now(), + idontwant: mk_tcache_read(), + }, + now, + &mut |c| cap.0.push(c), + ); + + let dontwants: Vec = cap + .0 + .iter() + .filter_map(|e| match e { + PeerControl::P2pSend(P2pSend::Gossip(GossipMsgOut { peer_id, .. })) => { + Some(*peer_id) + } + _ => None, + }) + .collect(); + + assert_eq!( + dontwants.len(), + 3, + "expected IDONTWANT to 3 non-sender mesh peers, got {:?}", + cap.0 + ); + assert!( + !dontwants.contains(&2), + "sender (conn=2) must not receive IDONTWANT for its own delivery: {dontwants:?}" + ); + for conn in [1usize, 3, 4] { + assert!( + dontwants.contains(&conn), + "expected IDONTWANT to mesh peer {conn}, got {dontwants:?}" + ); + } + } + + #[test] + fn new_inbound_skips_dontwant_for_below_threshold_peer() { + let now = Instant::now(); + let mut params = ScoreParams::default(); + params.gossip_threshold = -1.0; + params.graylist_threshold = -1_000_000.0; + params.d_high = 8; + let (mut mgr, mut cap) = fixture(vec![GossipTopic::BeaconBlock], params, false); + + for i in 1..=2u8 { + connect(&mut mgr, &mut cap, i as usize, i, now); + mgr.handle_event( + PeerEvent::P2pGossipTopicSubscribe { + p2p_peer: i as usize, + topic: GossipTopic::BeaconBlock, + }, + now, + &mut |c| cap.0.push(c), + ); + } + for i in 1..=2usize { + mgr.mesh.entry(GossipTopic::BeaconBlock).or_default().push(i); + } + for _ in 0..5 { + mgr.handle_event(PeerEvent::P2pGossipInvalidFrame { p2p_peer: 2 }, now, &mut |c| { + cap.0.push(c) + }); + } + mgr.tick(now + Duration::from_millis(10), &mut |c| cap.0.push(c)); + assert!(mgr.score(2).unwrap() < -1.0); + cap.0.clear(); + + let hash = silver_common::MessageId { id: [66u8; 20] }; + mgr.handle_event( + PeerEvent::NewGossip { + p2p_peer: 1, + topic: GossipTopic::BeaconBlock, + msg_hash: hash, + recv_ts: Nanos::now(), + idontwant: mk_tcache_read(), + }, + now + Duration::from_millis(20), + &mut |c| cap.0.push(c), + ); + + assert!( + !cap.0 + .iter() + .any(|e| matches!(e, PeerControl::P2pSend(P2pSend::Gossip(GossipMsgOut { .. })))), + "below-threshold mesh peer should not receive IDONTWANT, got {:?}", + cap.0 + ); + } + + #[test] + fn send_gossip_skips_mesh_peer_with_idontwant() { + let now = Instant::now(); + let mut params = ScoreParams::default(); + // d_low=0 disables the auto-graft on subscribe so manual mesh seeding + // is the only thing populating the mesh map. + params.d_low = 0; + params.d = 0; + params.d_high = 8; + let (mut mgr, mut cap) = fixture(vec![GossipTopic::BeaconBlock], params, false); + + for i in 1..=3u8 { + connect(&mut mgr, &mut cap, i as usize, i, now); + mgr.handle_event( + PeerEvent::P2pGossipTopicSubscribe { + p2p_peer: i as usize, + topic: GossipTopic::BeaconBlock, + }, + now, + &mut |c| cap.0.push(c), + ); + } + for i in 1..=3usize { + mgr.mesh.entry(GossipTopic::BeaconBlock).or_default().push(i); + } + + let hash = silver_common::MessageId { id: [0xAB; 20] }; + + // Peer 2 says "don't send me this id". + mgr.handle_event(PeerEvent::P2pGossipDontWant { p2p_peer: 2, hash }, now, &mut |c| { + cap.0.push(c) + }); + + cap.0.clear(); + + // Internal SendGossip with originator stream from peer 1. + let stream_id = + silver_common::P2pStreamId::new(1, 0, silver_common::StreamProtocol::GossipSub, false); + mgr.handle_event( + PeerEvent::SendGossip { + originator_stream_id: stream_id, + topic: GossipTopic::BeaconBlock, + msg_hash: hash, + recv_ts: silver_common::Nanos::now(), + protobuf: mk_tcache_read(), + }, + now, + &mut |c| cap.0.push(c), + ); + + let recipients: Vec = cap + .0 + .iter() + .filter_map(|e| match e { + PeerControl::P2pSend(P2pSend::Gossip(GossipMsgOut { peer_id, .. })) => { + Some(*peer_id) + } + _ => None, + }) + .collect(); + // Peer 1 = sender (skipped), peer 2 = IDONTWANT (skipped), peer 3 = served. + assert_eq!( + recipients, + vec![3], + "expected only peer 3 to receive the broadcast, got {recipients:?}" + ); + } +} diff --git a/crates/peer/src/manager/rpc.rs b/crates/peer/src/manager/rpc.rs index fa857d0f..7e1aa55f 100644 --- a/crates/peer/src/manager/rpc.rs +++ b/crates/peer/src/manager/rpc.rs @@ -359,10 +359,11 @@ impl PeerManager { /// Dispatch an inbound RPC event. For requests this gates on the /// per-peer rate limit, then handles Status/Ping/Goodbye/MetaData - /// inline (response on `emit`); block/data-column requests are - /// admitted but not yet forwarded (TODO). For responses this maps - /// errors to severity, updates peer database via `handle_event`, - /// and releases the outbound in-flight slot on a terminal chunk. + /// inline (response on `emit`); block/column/envelope requests are + /// ignored here — the storage tile consumes the same `RpcInbound` + /// stream and serves them. For responses this maps errors to severity, + /// updates peer database via `handle_event`, and releases the outbound + /// in-flight slot on a terminal chunk. pub fn on_rpc_inbound( &mut self, rpc: RpcInbound, diff --git a/crates/peer/src/state.rs b/crates/peer/src/state.rs index 6dbbd0c6..67e875b7 100644 --- a/crates/peer/src/state.rs +++ b/crates/peer/src/state.rs @@ -60,8 +60,8 @@ pub(crate) struct PeerState { pub behaviour_penalty: f64, // P7, quadratic over threshold // Per-heartbeat rate-limit counters; reset every `heartbeat_interval`. - pub ihaves_received: u16, // gates P7 via max_ihave_messages - pub iwant_ids_sent: u16, // gates P7 via max_ihave_length + pub ihaves_received: u16, // caps IWANT issuance via max_ihave_length + pub iwant_ids_sent: u16, // caps the per-heartbeat IWANT budget // Outbound protocol rate-limit state. pub outbound_rpc_limits: RpcRateLimitSet, @@ -144,6 +144,15 @@ pub(crate) struct TopicScore { /// True once `mesh_message_deliveries_activation_s` has elapsed since /// graft — deficit scoring only applies after this. pub mesh_active: bool, + /// Grafted by `opportunistic_graft` into a sub-median mesh. Exempt from + /// mesh-capped eviction until the activation window elapses, so the + /// trim falls on the poor performers the graft targets. + pub opportunistic: bool, + /// Consecutive remote prunes arriving within `QUICK_PRUNE_WINDOW` of + /// graft — the signature of a saturated remote mesh trimming us at its + /// heartbeat. Scales our re-graft backoff; reset by any prune after a + /// longer residency. + pub quick_prunes: u8, // P3b pub mesh_failure_penalty: f64, // P4 diff --git a/crates/ssz/src/ssz_view.rs b/crates/ssz/src/ssz_view.rs index 53d39074..243241bf 100644 --- a/crates/ssz/src/ssz_view.rs +++ b/crates/ssz/src/ssz_view.rs @@ -1492,6 +1492,66 @@ pub const DC_BY_ROOT_ID_MIN: usize = 36; // columns: List[ColumnIndex, NUMBER_OF_COLUMNS], ColumnIndex = u64. pub const DC_BY_ROOT_ID_MAX: usize = DC_BY_ROOT_ID_MIN + NUMBER_OF_COLUMNS * 8; +// -- DataColumnSidecarsByRootRequest (req/data_column_sidecars_by_root/1) +// +// SSZ List[DataColumnsByRootIdentifier, MAX_REQUEST_BLOCKS]: variable-size +// elements, so a leading offset table (u32 LE per element) precedes the +// element payloads. `offsets[0] / 4` is the element count. + +#[derive(Clone, Copy, Debug)] +#[repr(C)] +pub struct DataColumnsByRootRequestView; + +impl DataColumnsByRootRequestView { + #[inline] + fn offset(buf: &[u8], i: usize) -> usize { + u32::from_le_bytes(buf[i * 4..i * 4 + 4].try_into().unwrap()) as usize + } + + #[inline] + pub fn count(buf: &[u8]) -> usize { + Self::offset(buf, 0) / 4 + } + + /// Element `i`'s bytes — a bare `DataColumnsByRootIdentifier`. Safe for + /// all `i < count(buf)` after `check_size`. + #[inline] + pub fn identifier(buf: &[u8], i: usize) -> &[u8] { + let start = Self::offset(buf, i); + let end = if i + 1 < Self::count(buf) { Self::offset(buf, i + 1) } else { buf.len() }; + &buf[start..end] + } + + /// Offset table well-formed (first offset = 4·count, monotonic, in + /// bounds) and every element a valid identifier. + pub fn check_size(buf: &[u8]) -> bool { + if buf.len() < 4 { + return false; + } + let table_end = Self::offset(buf, 0); + if table_end < 4 || !table_end.is_multiple_of(4) || table_end > buf.len() { + return false; + } + let count = table_end / 4; + if count > MAX_REQUEST_BLOCKS_DENEB { + return false; + } + let mut prev = table_end; + for i in 0..count { + let start = if i == 0 { table_end } else { Self::offset(buf, i) }; + let end = if i + 1 < count { Self::offset(buf, i + 1) } else { buf.len() }; + if start != prev || end < start || end > buf.len() { + return false; + } + if !DataColumnsByRootIdentifierView::check_size(&buf[start..end]) { + return false; + } + prev = end; + } + true + } +} + #[derive(Clone, Copy, Debug)] #[repr(C)] pub struct DataColumnsByRootIdentifierView; diff --git a/crates/storage/src/store.rs b/crates/storage/src/store.rs index eeacea41..9bbc5bc3 100644 --- a/crates/storage/src/store.rs +++ b/crates/storage/src/store.rs @@ -2,6 +2,7 @@ use std::{ collections::{VecDeque, hash_map::Entry}, io::{Error, Read}, path::{Path, PathBuf}, + time::Instant, }; use flux_profiler::timed; @@ -12,7 +13,8 @@ use silver_common::{ ssz_view::{ BeaconBlocksByRangeRequestView, BeaconBlocksByRootRequestView, DataColumnSidecarsByRangeRequestView, DataColumnsByRootIdentifierView, - ExecutionPayloadEnvelopesByRangeRequestView, ExecutionPayloadEnvelopesByRootRequestView, + DataColumnsByRootRequestView, ExecutionPayloadEnvelopesByRangeRequestView, + ExecutionPayloadEnvelopesByRootRequestView, }, }; @@ -217,6 +219,39 @@ impl QueryUnit { struct PendingQuery { stream_id: P2pStreamId, units: VecDeque, + received_at: Instant, + first_chunk_at: Option, + units_total: u32, + units_sent: u32, +} + +impl PendingQuery { + fn new(stream_id: P2pStreamId, units: VecDeque) -> Self { + Self { + stream_id, + received_at: Instant::now(), + first_chunk_at: None, + units_total: units.len() as u32, + units_sent: 0, + units, + } + } + + /// The `RpcServeOutcome` for this query terminating now. + fn outcome(&self, missing: bool) -> PeerEvent { + PeerEvent::RpcServeOutcome { + p2p_peer: self.stream_id.peer(), + protocol: self.stream_id.protocol(), + units_total: self.units_total, + units_sent: self.units_sent, + missing, + first_chunk_ms: self + .first_chunk_at + .map(|t| t.duration_since(self.received_at).as_millis() as u64) + .unwrap_or(0), + elapsed_ms: self.received_at.elapsed().as_millis() as u64, + } + } } /// Unified blocks and data columns disk store. @@ -560,7 +595,7 @@ impl Store { // TODO should not return 'Complete' should return rate limit error if self.query_queue.len() >= MAX_INFLIGHT_QUERIES { tracing::warn!(?stream_id, "queries at capacity"); - self.query_queue.push_back(PendingQuery { stream_id, units: VecDeque::new() }); + self.query_queue.push_back(PendingQuery::new(stream_id, VecDeque::new())); return; } @@ -607,29 +642,33 @@ impl Store { with_root_request( rpc_consumer, read, - DataColumnsByRootIdentifierView::check_size, + DataColumnsByRootRequestView::check_size, |buf| { - let root = DataColumnsByRootIdentifierView::block_root(buf); - let request_columns = DataColumnsByRootIdentifierView::columns(buf) - .chunks_exact(8) - .map(|chunk| u64::from_le_bytes(chunk.try_into().unwrap())); - - tracing::info!(?stream_id, ?root, "storage query"); - - // Serve a specific block's columns regardless of - // canonicity: unfinalized (by block_root) first, else - // the finalized flat store. - if let Some(slot) = self.unfinalized_columns.slot_of(root) { - for column in request_columns { - units.push_back(QueryUnit::UnfinalizedColumn { - slot, - block_root: *root, - column, - }); - } - } else if let Some(&slot) = self.root_index.get(root) { - for column in request_columns { - units.push_back(QueryUnit::Column { slot, column }); + let ids = DataColumnsByRootRequestView::count(buf); + tracing::info!(?stream_id, ids, len = buf.len(), "storage query"); + + for i in 0..ids { + let id = DataColumnsByRootRequestView::identifier(buf, i); + let root = DataColumnsByRootIdentifierView::block_root(id); + let request_columns = DataColumnsByRootIdentifierView::columns(id) + .chunks_exact(8) + .map(|chunk| u64::from_le_bytes(chunk.try_into().unwrap())); + + // Serve a specific block's columns regardless of + // canonicity: unfinalized (by block_root) first, + // else the finalized flat store. + if let Some(slot) = self.unfinalized_columns.slot_of(root) { + for column in request_columns { + units.push_back(QueryUnit::UnfinalizedColumn { + slot, + block_root: *root, + column, + }); + } + } else if let Some(&slot) = self.root_index.get(root) { + for column in request_columns { + units.push_back(QueryUnit::Column { slot, column }); + } } } }, @@ -656,11 +695,14 @@ impl Store { with_root_request( rpc_consumer, read, - BeaconBlocksByRootRequestView::check_size, + |buf| { + tracing::info!("BeaconBlocksByRoot len {}", buf.len()); + BeaconBlocksByRootRequestView::check_size(buf) + }, |buf| { let count = BeaconBlocksByRootRequestView::count(buf); - tracing::info!(?stream_id, count, "storage query"); + tracing::info!(?stream_id, count, len = buf.len(), "storage query"); for i in 0..count { let root = BeaconBlocksByRootRequestView::root(buf, i); @@ -668,13 +710,28 @@ impl Store { // canonicity: unfinalized fork tree first, then the // finalized flat store. if let Some((slot, parent_root)) = self.unfinalized.get(root) { + tracing::info!( + block_root = hex::encode(root), + slot, + "serve unfinalized block" + ); units.push_back(QueryUnit::UnfinalizedBlock { slot, parent_root, block_root: *root, }); } else if let Some(&slot) = self.root_index.get(root) { + tracing::info!( + block_root = hex::encode(root), + slot, + "serve finalized block" + ); units.push_back(QueryUnit::Block { slot }); + } else { + tracing::warn!( + block_root = hex::encode(root), + "BlockByRoot - root not found" + ); } } }, @@ -723,7 +780,7 @@ impl Store { // Unhandled request kind: no response (matches prior behaviour). _ => return, } - self.query_queue.push_back(PendingQuery { stream_id, units }); + self.query_queue.push_back(PendingQuery::new(stream_id, units)); } fn resolve_canonical_range( @@ -782,6 +839,10 @@ fn with_root_request( check_size(buf) { resolve(buf); + } else { + // Fall through with no units: the caller still enqueues the query, + // so the peer gets an immediate bare `Complete`, not a hung stream. + tracing::warn!("root request buffer not resolved!"); } } @@ -1286,13 +1347,19 @@ mod tests { assert_col(&responses[0], &a3); assert_col(&responses[1], &a7); - // DataColumnsByRoot for the fork B's column 3: served despite B being - // non-canonical. SSZ: block_root | offset(=36) | column list. - let mut byroot = [0u8; 44]; - byroot[0..32].copy_from_slice(&root_b); - byroot[32..36].copy_from_slice(&36u32.to_le_bytes()); - byroot[36..44].copy_from_slice(&3u64.to_le_bytes()); - let mut br_res = req_producer.reserve(44, true).unwrap(); + // DataColumnsByRoot spanning both roots in one request: A's column 7 + // and non-canonical B's column 3. Wire format is + // List[DataColumnsByRootIdentifier]: outer offset table (u32 LE per + // element), then per element root | inner offset(=36) | column list. + let mut byroot = Vec::new(); + byroot.extend_from_slice(&8u32.to_le_bytes()); // element 0 at 8 + byroot.extend_from_slice(&52u32.to_le_bytes()); // element 1 at 8 + 44 + for (root, column) in [(&root_a, 7u64), (&root_b, 3u64)] { + byroot.extend_from_slice(root); + byroot.extend_from_slice(&36u32.to_le_bytes()); + byroot.extend_from_slice(&column.to_le_bytes()); + } + let mut br_res = req_producer.reserve(byroot.len(), true).unwrap(); br_res.write_all(&byroot).unwrap(); br_res.flush().unwrap(); let byroot_ssz = br_res.read(); @@ -1307,8 +1374,40 @@ mod tests { _ => {} }) .unwrap(); - assert_eq!(br.len(), 2); // B column 3 + Complete - assert_col(&br[0], &b3); + assert_eq!(br.len(), 3); // A column 7, B column 3, Complete + assert_col(&br[0], &a7); + assert_col(&br[1], &b3); + + // A bare identifier (no outer offset table — the pre-fix encoding) is + // rejected: no units resolve, the peer still gets an immediate bare + // Complete rather than a hung stream. + let mut bare = [0u8; 44]; + bare[0..32].copy_from_slice(&root_b); + bare[32..36].copy_from_slice(&36u32.to_le_bytes()); + bare[36..44].copy_from_slice(&3u64.to_le_bytes()); + let mut bare_res = req_producer.reserve(44, true).unwrap(); + bare_res.write_all(&bare).unwrap(); + bare_res.flush().unwrap(); + let bare_ssz = bare_res.read(); + store.rpc_request(&mut req_consumer, RpcRequestInbound { + stream_id: sid, + request: RpcRequest::DataColumnsByRoot(bare_ssz), + }); + let mut rejected = vec![]; + store + .file_io(|_| fork_digest, 0, &mut producer, &mut |s| match s { + IoEvent::P2pSend(s) => rejected.push(s), + _ => {} + }) + .unwrap(); + assert_eq!(rejected.len(), 1, "malformed by-root request answers a bare Complete"); + assert!(matches!( + &rejected[0], + P2pSend::Rpc(RpcOutbound::Response(RpcResponseOutbound { + response: RpcResponse::Complete, + .. + })) + )); // Finalize on A at slot 42: promote A's columns, prune B's. store.update_head(slot, root_a, slot, root_a); diff --git a/crates/storage/src/store/io.rs b/crates/storage/src/store/io.rs index 2ab94d0e..1c4e0289 100644 --- a/crates/storage/src/store/io.rs +++ b/crates/storage/src/store/io.rs @@ -8,6 +8,7 @@ use std::{ fs::File, io::{Error, ErrorKind, Read, Write}, path::{Path, PathBuf}, + time::Instant, }; use flux_profiler::timed; @@ -266,6 +267,7 @@ impl Store { stream_id: query.stream_id, response: RpcResponse::Complete, })))); + emit(IoEvent::PeerEvent(query.outcome(false))); continue; }; reads += 1; @@ -289,6 +291,8 @@ impl Store { emit(IoEvent::P2pSend(P2pSend::Rpc(RpcOutbound::Response( RpcResponseOutbound { stream_id: query.stream_id, response }, )))); + query.units_sent += 1; + query.first_chunk_at.get_or_insert_with(Instant::now); self.query_queue.push_back(query); } ServeResult::Missing => { @@ -299,6 +303,7 @@ impl Store { emit(IoEvent::P2pSend(P2pSend::Rpc(RpcOutbound::Response( RpcResponseOutbound { stream_id: query.stream_id, response }, )))); + emit(IoEvent::PeerEvent(query.outcome(true))); } ServeResult::ProducerFull => { // Tcache full — un-consume and retry this request first diff --git a/crates/surfer/src/render/gossip_pane.rs b/crates/surfer/src/render/gossip_pane.rs index ea0e8089..d5549e52 100644 --- a/crates/surfer/src/render/gossip_pane.rs +++ b/crates/surfer/src/render/gossip_pane.rs @@ -2,7 +2,7 @@ use std::collections::HashMap; use ratatui::{ Frame, - layout::{Constraint, Rect}, + layout::{Constraint, Direction, Layout, Rect}, style::{Color, Modifier, Style}, widgets::{Block, Borders, Cell, Paragraph, Row, Table}, }; @@ -60,6 +60,13 @@ fn rate(set: &CounterSet, slot: usize) -> String { } pub fn draw(f: &mut Frame, area: Rect, app: &mut App) { + let cols = Layout::default() + .direction(Direction::Horizontal) + .constraints([Constraint::Min(40), Constraint::Length(AGENTS_PANEL_WIDTH)]) + .split(area); + draw_agent_counts(f, cols[1], app); + let area = cols[0]; + let mut meshed: HashMap> = HashMap::new(); for (id, row) in app.peers.rows() { let conn = row.p2p.as_ref().map(|s| s.connection); @@ -186,6 +193,93 @@ pub fn draw(f: &mut Frame, area: Rect, app: &mut App) { f.render_stateful_widget(table, area, &mut app.gossip_table_state); } +/// type (10 chars) + dir + two count columns + borders. +const AGENTS_PANEL_WIDTH: u16 = 29; + +const COMBINED_COLOR: Color = Color::Cyan; + +#[derive(Default)] +struct AgentCounts { + conns_in: usize, + conns_out: usize, + meshes_in: usize, + meshes_out: usize, +} + +impl AgentCounts { + fn add(&mut self, other: &AgentCounts) { + self.conns_in += other.conns_in; + self.conns_out += other.conns_out; + self.meshes_in += other.meshes_in; + self.meshes_out += other.meshes_out; + } +} + +/// Per-client-type counts (the user-agent segment before the first '/', +/// truncated to 10 chars), split by connection direction into in/out pairs: +/// connections and mesh memberships — a peer in 10 meshes contributes 10 to +/// its direction's mesh count. +fn draw_agent_counts(f: &mut Frame, area: Rect, app: &App) { + let mut counts: HashMap = HashMap::new(); + for (_, row) in app.peers.rows() { + let Some(p2p) = &row.p2p else { continue }; + let agent = row.scores.as_ref().map(|s| s.user_agent.as_str()).unwrap_or(""); + let kind = agent.split('/').next().unwrap_or(""); + let kind = if kind.is_empty() { "unknown" } else { kind }; + let entry = counts.entry(kind.chars().take(10).collect()).or_default(); + if p2p.inbound { + entry.conns_in += 1; + entry.meshes_in += row.topics.len(); + } else { + entry.conns_out += 1; + entry.meshes_out += row.topics.len(); + } + } + let mut ranked: Vec<(String, AgentCounts)> = counts.into_iter().collect(); + ranked.sort_by(|a, b| { + (b.1.conns_in + b.1.conns_out) + .cmp(&(a.1.conns_in + a.1.conns_out)) + .then_with(|| a.0.cmp(&b.0)) + }); + let mut total = AgentCounts::default(); + for (_, c) in &ranked { + total.add(c); + } + + let header = Row::new(["agent", "dir", "conn", "mesh"].map(Cell::from)) + .style(Style::default().add_modifier(Modifier::BOLD)); + let mut rows = Vec::with_capacity((ranked.len() + 1) * 3); + let push_triple = |rows: &mut Vec, kind: &str, c: &AgentCounts| { + let dir_row = |name, dir, conn: usize, mesh: usize| { + Row::new([ + Cell::from(String::from(name)), + Cell::from(dir), + Cell::from(format!("{conn}")), + Cell::from(format!("{mesh}")), + ]) + }; + rows.push(dir_row(kind, "in", c.conns_in, c.meshes_in)); + rows.push(dir_row("", "out", c.conns_out, c.meshes_out)); + rows.push( + dir_row("", "all", c.conns_in + c.conns_out, c.meshes_in + c.meshes_out) + .style(Style::default().fg(COMBINED_COLOR)), + ); + }; + for (kind, c) in &ranked { + push_triple(&mut rows, kind, c); + } + push_triple(&mut rows, "total", &total); + let table = Table::new(rows, [ + Constraint::Length(10), + Constraint::Length(3), + Constraint::Length(5), + Constraint::Length(5), + ]) + .header(header) + .block(Block::default().borders(Borders::ALL).title(" peers ")); + f.render_widget(table, area); +} + /// One meshed peer of the selected topic: the peers-tab expansion fields /// plus identity (conn + short id + agent) for log correlation. fn member_row(m: &Member) -> Row<'static> { diff --git a/crates/surfer/src/render/peers_pane.rs b/crates/surfer/src/render/peers_pane.rs index 71b0a1fc..d78798fe 100644 --- a/crates/surfer/src/render/peers_pane.rs +++ b/crates/surfer/src/render/peers_pane.rs @@ -9,10 +9,10 @@ use ratatui::{ use crate::{app::App, sources::peers::PeerRow}; -const NET_COLS: usize = 10; -pub const COLUMNS: [&str; 20] = [ - "conn", "peer", "addr", "age", "rtt", "lost", "rxb", "txb", "rxdg", "txdg", "mesh", "p1", "p2", - "p3", "p3b", "p4", "p5", "p6", "p7", "total", +const NET_COLS: usize = 11; +pub const COLUMNS: [&str; 21] = [ + "conn", "peer", "addr", "age", "rtt", "lost", "rxb", "txb", "rxdg", "txdg", "strm", "mesh", + "p1", "p2", "p3", "p3b", "p4", "p5", "p6", "p7", "total", ]; const SCORE_COLOR: Color = Color::Magenta; @@ -43,6 +43,7 @@ fn sort_key(row: &PeerRow, col: usize) -> Key { 7 => s.tx_blocking as u128, 8 => s.rx_datagrams as u128, 9 => s.tx_datagrams as u128, + 10 => s.streams as u128, _ => 0, }) } else { @@ -50,15 +51,15 @@ fn sort_key(row: &PeerRow, col: usize) -> Key { return if col == NET_COLS { Key::Int(0) } else { Key::Float(0.0) }; }; match col { - 10 => Key::Int(s.mesh_count as u128), - 11 => Key::Float(s.p1_time_in_mesh), - 12 => Key::Float(s.p2_first_deliveries), - 13 => Key::Float(s.p3_mesh_deficit), - 14 => Key::Float(s.p3b_mesh_failure), - 15 => Key::Float(s.p4_invalid), - 16 => Key::Float(s.p5_application), - 17 => Key::Float(s.p6_ip_colocation), - 18 => Key::Float(s.p7_behaviour), + 11 => Key::Int(s.mesh_count as u128), + 12 => Key::Float(s.p1_time_in_mesh), + 13 => Key::Float(s.p2_first_deliveries), + 14 => Key::Float(s.p3_mesh_deficit), + 15 => Key::Float(s.p3b_mesh_failure), + 16 => Key::Float(s.p4_invalid), + 17 => Key::Float(s.p5_application), + 18 => Key::Float(s.p6_ip_colocation), + 19 => Key::Float(s.p7_behaviour), _ => Key::Float(s.total), } } @@ -138,6 +139,7 @@ pub fn draw(f: &mut Frame, area: Rect, app: &mut App) { }; let conn = match &r.p2p { + Some(s) if s.inbound => format!("{} ✓", s.connection), Some(s) => format!("{}", s.connection), None => "·".to_string(), }; @@ -152,6 +154,7 @@ pub fn draw(f: &mut Frame, area: Rect, app: &mut App) { Cell::from(format!("{}", s.tx_blocking)), Cell::from(format!("{}", s.rx_datagrams)), Cell::from(format!("{}", s.tx_datagrams)), + Cell::from(format!("{}", s.streams)), ]), None => cells.extend((2..NET_COLS).map(|_| Cell::from("·"))), } @@ -196,6 +199,7 @@ pub fn draw(f: &mut Frame, area: Rect, app: &mut App) { Constraint::Length(8), Constraint::Length(8), Constraint::Length(5), + Constraint::Length(5), ]; widths.extend(std::iter::repeat_n(Constraint::Length(7), COLUMNS.len() - widths.len())); let table = Table::new(table_rows, widths).header(header).block(block); From 3cd76025be740b2ac3aa473d48a1bf124790490e Mon Sep 17 00:00:00 2001 From: vladimir-ea Date: Fri, 21 Aug 2026 16:59:27 +0100 Subject: [PATCH 02/11] clean up logs --- crates/storage/src/store.rs | 15 +-------------- 1 file changed, 1 insertion(+), 14 deletions(-) diff --git a/crates/storage/src/store.rs b/crates/storage/src/store.rs index 9bbc5bc3..c4ecb841 100644 --- a/crates/storage/src/store.rs +++ b/crates/storage/src/store.rs @@ -695,10 +695,7 @@ impl Store { with_root_request( rpc_consumer, read, - |buf| { - tracing::info!("BeaconBlocksByRoot len {}", buf.len()); - BeaconBlocksByRootRequestView::check_size(buf) - }, + BeaconBlocksByRootRequestView::check_size, |buf| { let count = BeaconBlocksByRootRequestView::count(buf); @@ -710,22 +707,12 @@ impl Store { // canonicity: unfinalized fork tree first, then the // finalized flat store. if let Some((slot, parent_root)) = self.unfinalized.get(root) { - tracing::info!( - block_root = hex::encode(root), - slot, - "serve unfinalized block" - ); units.push_back(QueryUnit::UnfinalizedBlock { slot, parent_root, block_root: *root, }); } else if let Some(&slot) = self.root_index.get(root) { - tracing::info!( - block_root = hex::encode(root), - slot, - "serve finalized block" - ); units.push_back(QueryUnit::Block { slot }); } else { tracing::warn!( From dea9e461a7c6db125d80abba89b3fde2eb2d85ae Mon Sep 17 00:00:00 2001 From: vladimir-ea Date: Fri, 21 Aug 2026 19:46:59 +0100 Subject: [PATCH 03/11] score fix --- crates/config/src/peer_score_params.rs | 8 ++++++-- crates/peer/src/manager.rs | 14 ++++++++------ crates/peer/src/manager/gossip.rs | 20 +++++++++++--------- crates/peer/src/state.rs | 9 +++++++++ 4 files changed, 34 insertions(+), 17 deletions(-) diff --git a/crates/config/src/peer_score_params.rs b/crates/config/src/peer_score_params.rs index cffa75c5..019b036b 100644 --- a/crates/config/src/peer_score_params.rs +++ b/crates/config/src/peer_score_params.rs @@ -161,8 +161,12 @@ impl Default for ScoreParams { // once per slot to accumulate toward the −80 graylist floor. application_score_decay: 0.95, - // P7 — behaviour penalty - behaviour_penalty_threshold: 0.0, + // P7 — behaviour penalty. Squared excess over the threshold: + // with the free budget below, isolated events (a stream race, a + // one-off bad frame) cost nothing; only sustained misbehaviour + // gates. Threshold 0 made a single benign event = -10 = the + // gossip gate, which starved the peer's P3 view of us. + behaviour_penalty_threshold: 5.0, behaviour_penalty_weight: -10.0, behaviour_penalty_decay: 0.999, diff --git a/crates/peer/src/manager.rs b/crates/peer/src/manager.rs index df623bb9..1d1cfb04 100644 --- a/crates/peer/src/manager.rs +++ b/crates/peer/src/manager.rs @@ -501,14 +501,14 @@ impl PeerManager { if rpc_request { self.release_outbound_in_flight(p2p_peer, protocol); } - let offence = if stream_gone { + if stream_gone { + // Their teardown raced our (possibly late) response — + // not peer misbehaviour. Counted, not penalised. crate::PeerCounters::ResponseStreamGone.inc(); - "response stream gone" } else { crate::PeerCounters::StreamCreditExhausted.inc(); - "stream credit exhausted" - }; - self.add_behaviour_penalty(p2p_peer, 1.0, offence); + self.add_behaviour_penalty(p2p_peer, 1.0, "stream credit exhausted"); + } } PeerEvent::P2pOutboundMessageDropped { p2p_peer, protocol, rpc_request } => { // Local outbound-ring overflow — a backpressure signal, often @@ -1797,7 +1797,9 @@ pub(crate) mod tests { #[test] fn disconnect_archives_and_reconnect_restores() { let now = Instant::now(); - let params = ScoreParams::default(); + let mut params = ScoreParams::default(); + // Zero free budget so the archived penalty shows in the score. + params.behaviour_penalty_threshold = 0.0; let (mut mgr, mut cap) = fixture(vec![], params, false); connect(&mut mgr, &mut cap, 1, 1, now); diff --git a/crates/peer/src/manager/gossip.rs b/crates/peer/src/manager/gossip.rs index c69dc9c3..eb0b4074 100644 --- a/crates/peer/src/manager/gossip.rs +++ b/crates/peer/src/manager/gossip.rs @@ -278,7 +278,7 @@ impl PeerManager { // below it — a promise without a sent IWANT can only ever expire). let should_iwant = !already_seen && peer.ihaves_received <= self.params.max_ihave_length && - peer.cached_score >= self.params.gossip_threshold && + peer.gossip_gate_score() >= self.params.gossip_threshold && peer.iwant_ids_sent < self.params.max_ihave_length; if should_iwant { peer.iwant_ids_sent = peer.iwant_ids_sent.saturating_add(1); @@ -314,7 +314,7 @@ impl PeerManager { // exceeds retransmission threshold return; } - if peer.cached_score < self.params.gossip_threshold { + if peer.gossip_gate_score() < self.params.gossip_threshold { return; } emit(PeerControl::P2pSend(P2pSend::Gossip(GossipMsgOut { peer_id: conn, tcache }))); @@ -411,7 +411,7 @@ impl PeerManager { let Some(peer) = self.peers.get(conn) else { continue; }; - if peer.cached_score < self.params.gossip_threshold { + if peer.gossip_gate_score() < self.params.gossip_threshold { continue; } emit(PeerControl::P2pSend(P2pSend::Gossip(GossipMsgOut { @@ -444,7 +444,7 @@ impl PeerManager { if mesh_for_topic.is_some_and(|m| m.contains(conn)) { continue; // mesh peers get full-body forwards, not IHAVE } - if peer.cached_score < self.params.gossip_threshold { + if peer.gossip_gate_score() < self.params.gossip_threshold { continue; } emit(PeerControl::P2pSend(P2pSend::Gossip(GossipMsgOut { @@ -470,7 +470,7 @@ impl PeerManager { let Some(peer) = self.peers.get(&conn) else { return; }; - if peer.cached_score < self.params.gossip_threshold { + if peer.gossip_gate_score() < self.params.gossip_threshold { return; } emit(PeerControl::P2pSend(P2pSend::Gossip(GossipMsgOut { peer_id: conn, tcache }))); @@ -496,7 +496,7 @@ impl PeerManager { if *peer == sender { continue; } - if peer_state.cached_score < self.params.gossip_threshold { + if peer_state.gossip_gate_score() < self.params.gossip_threshold { continue; } if peer_state.msg_cache_contains(&msg_hash) { @@ -1661,6 +1661,8 @@ mod tests { let mut params = ScoreParams::default(); params.iwant_followup = Duration::from_secs(3); params.heartbeat_interval = Duration::from_millis(100); + // Zero free budget so a single broken promise shows in the score. + params.behaviour_penalty_threshold = 0.0; let (mut mgr, mut cap) = fixture(vec![], params, false); connect(&mut mgr, &mut cap, 1, 1, now); connect(&mut mgr, &mut cap, 2, 2, now); @@ -1796,7 +1798,7 @@ mod tests { now, &mut |c| cap.0.push(c), ); - for _ in 0..5 { + for _ in 0..7 { mgr.handle_event(PeerEvent::P2pGossipInvalidFrame { p2p_peer: 1 }, now, &mut |c| { cap.0.push(c) }); @@ -1862,7 +1864,7 @@ mod tests { let (mut mgr, mut cap) = fixture(vec![], params, false); connect(&mut mgr, &mut cap, 1, 1, now); - for _ in 0..5 { + for _ in 0..7 { mgr.handle_event(PeerEvent::P2pGossipInvalidFrame { p2p_peer: 1 }, now, &mut |c| { cap.0.push(c) }); @@ -1983,7 +1985,7 @@ mod tests { for i in 1..=2usize { mgr.mesh.entry(GossipTopic::BeaconBlock).or_default().push(i); } - for _ in 0..5 { + for _ in 0..7 { mgr.handle_event(PeerEvent::P2pGossipInvalidFrame { p2p_peer: 2 }, now, &mut |c| { cap.0.push(c) }); diff --git a/crates/peer/src/state.rs b/crates/peer/src/state.rs index 67e875b7..6a4375f5 100644 --- a/crates/peer/src/state.rs +++ b/crates/peer/src/state.rs @@ -122,6 +122,15 @@ impl PeerState { } /// Inserts or updates msg cache entry, returning previous count + /// Score for gossip-domain gates (`gossip_threshold` comparisons): + /// excludes P5 — an RPC-domain penalty must not silence our gossip + /// toward the peer, which starves their P3 view of us and gets us + /// pruned/disconnected in return. Gossip-domain offences (P4, P7) + /// still count. + pub fn gossip_gate_score(&self) -> f64 { + self.cached_score - self.last_breakdown.p5_application + } + pub fn msg_cache_insert(&mut self, msg_id: MessageId) -> u32 { self.msg_cache.upsert(msg_id) } From f9a9ffa559da849d1caacf7950d12f77257717cb Mon Sep 17 00:00:00 2001 From: vladimir-ea Date: Fri, 21 Aug 2026 19:52:41 +0100 Subject: [PATCH 04/11] fix bad score timeout --- crates/peer/src/manager.rs | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/crates/peer/src/manager.rs b/crates/peer/src/manager.rs index 1d1cfb04..60443287 100644 --- a/crates/peer/src/manager.rs +++ b/crates/peer/src/manager.rs @@ -1491,14 +1491,17 @@ impl PeerManager { } } - /// Dial backoff earned by a Goodbye, `None` = stay dialable. The - /// score-ban family matches lighthouse's 12h `BANNED_BEFORE_DECAY`; - /// TooManyPeers is a routine excess-peer shed that expects us back; - /// wrong-network codes are futile to redial until a fork change. + /// Dial backoff earned by a Goodbye, `None` = stay dialable. Banned + /// matches lighthouse's 12h `BANNED_BEFORE_DECAY`; BadScore is only + /// their *disconnect* threshold — the score keeps decaying and their + /// slowest component (attestation P3b) fades in ~30 min; TooManyPeers + /// is a routine excess-peer shed that expects us back; wrong-network + /// codes are futile to redial until a fork change. fn goodbye_dial_backoff(code: u64) -> Option { match code { GOODBYE_CLIENT_SHUTDOWN => None, 129 => Some(Duration::from_secs(5 * 60)), + 250 => Some(Duration::from_secs(30 * 60)), 3 => Some(Duration::from_secs(3600)), _ => Some(REMOTE_BAN_TTL), } From d9026f146e068a8cd97c2e4467a83bbdf304df88 Mon Sep 17 00:00:00 2001 From: vladimir-ea Date: Fri, 21 Aug 2026 21:17:49 +0100 Subject: [PATCH 05/11] attestation unknown root counter --- crates/beacon_state/tile/src/counters.rs | 4 ++++ crates/beacon_state/tile/src/tile/gossip.rs | 1 + 2 files changed, 5 insertions(+) diff --git a/crates/beacon_state/tile/src/counters.rs b/crates/beacon_state/tile/src/counters.rs index c623ff4d..733d8fe3 100644 --- a/crates/beacon_state/tile/src/counters.rs +++ b/crates/beacon_state/tile/src/counters.rs @@ -9,6 +9,10 @@ silver_common::declare_counters! { // gossip-admission structures at capacity (attestations still // accepted and relayed; aggregation/shedding coverage degrades) AttestationPoolFull, + // Gossip attestation ignored (never relayed): beacon_block_root not + // in fork choice — the reprocess-queue gap vs lighthouse, which + // parks and replays these. + AttestationUnknownRoot, SeenAggregatesFull, AttestationRootMemoFull, // attestation-root memo effectiveness (hit rate is the memo's diff --git a/crates/beacon_state/tile/src/tile/gossip.rs b/crates/beacon_state/tile/src/tile/gossip.rs index 9a4da3bd..b6290072 100644 --- a/crates/beacon_state/tile/src/tile/gossip.rs +++ b/crates/beacon_state/tile/src/tile/gossip.rs @@ -579,6 +579,7 @@ impl BeaconStateTile { return Err(Feedback::Reject(None)); } let Some(idx) = self.fork_choice.find_node_idx(data.beacon_block_root()) else { + BeaconStateCounters::AttestationUnknownRoot.inc(); return Err(Feedback::Ignore); }; match self.fork_choice.checkpoint_block_of(idx, target_epoch * SLOTS_PER_EPOCH) { From db5a425b832b6ed41d1aff9bc1c6c253b9aedf0b Mon Sep 17 00:00:00 2001 From: vladimir-ea Date: Fri, 21 Aug 2026 23:18:28 +0100 Subject: [PATCH 06/11] default idel timeout --- crates/network/src/p2p/quic/mod.rs | 26 +++++++++++++++++++++++--- 1 file changed, 23 insertions(+), 3 deletions(-) diff --git a/crates/network/src/p2p/quic/mod.rs b/crates/network/src/p2p/quic/mod.rs index cf37a6c9..05cf5ccb 100644 --- a/crates/network/src/p2p/quic/mod.rs +++ b/crates/network/src/p2p/quic/mod.rs @@ -21,21 +21,41 @@ pub fn create_endpoint(server_config: Option>) -> Result Arc { + let mut tc = quinn_proto::TransportConfig::default(); + tc.max_idle_timeout(Some(quinn_proto::IdleTimeout::try_from(IDLE_TIMEOUT).unwrap())); + tc.keep_alive_interval(Some(KEEP_ALIVE_INTERVAL)); + Arc::new(tc) +} + +const IDLE_TIMEOUT: std::time::Duration = std::time::Duration::from_secs(30); +const KEEP_ALIVE_INTERVAL: std::time::Duration = std::time::Duration::from_secs(10); + /// QUIC client config with libp2p TLS authentication. pub fn create_client_config( keypair: &Keypair, remote_peer_id: Option, ) -> Result { let rustls_cfg = tls::make_client_config(keypair, remote_peer_id).map_err(Error::other)?; - Ok(ClientConfig::new(Arc::new(QuicClientConfig::try_from(rustls_cfg).map_err(Error::other)?))) + let mut config = + ClientConfig::new(Arc::new(QuicClientConfig::try_from(rustls_cfg).map_err(Error::other)?)); + config.transport_config(transport_config()); + Ok(config) } /// QUIC server config with libp2p TLS authentication. pub fn create_server_config(keypair: &Keypair) -> Result { let rustls_cfg = tls::make_server_config(keypair).map_err(Error::other)?; - Ok(ServerConfig::with_crypto(Arc::new( + let mut config = ServerConfig::with_crypto(Arc::new( QuicServerConfig::try_from(rustls_cfg).map_err(Error::other)?, - ))) + )); + config.transport_config(transport_config()); + Ok(config) } #[derive(Clone, Copy, Debug, Eq, PartialEq)] From fe40d457502deaf1d3f04e78b6aa02fa3ffcc8ac Mon Sep 17 00:00:00 2001 From: vladimir-ea Date: Sat, 22 Aug 2026 00:08:42 +0100 Subject: [PATCH 07/11] gossip getting sped past --- crates/common/src/spine.rs | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/crates/common/src/spine.rs b/crates/common/src/spine.rs index 5781d998..65cdb765 100644 --- a/crates/common/src/spine.rs +++ b/crates/common/src/spine.rs @@ -36,13 +36,13 @@ pub struct SilverSpine { pub tile_info: ShmemData, /// New incoming network gossip messages - #[queue(size(2usize.pow(16)))] + #[queue(size(2usize.pow(24)))] pub gossip_in: SpineQueue, /// New incoming gossip messages - #[queue(size(2usize.pow(16)))] + #[queue(size(2usize.pow(20)))] pub new_gossip: SpineQueue, /// P2p send messages. - #[queue(size(2usize.pow(16)))] + #[queue(size(2usize.pow(24)))] pub p2p_send: SpineQueue, /// RPC recv messages. #[queue(size(2usize.pow(14)))] From 546976d7a998e3dfd8f951376836dc9821cfc8a2 Mon Sep 17 00:00:00 2001 From: vladimir-ea Date: Sat, 22 Aug 2026 00:22:20 +0100 Subject: [PATCH 08/11] gossip getting sped past --- crates/common/src/spine.rs | 6 +++--- crates/network/src/p2p/quic/peer.rs | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/crates/common/src/spine.rs b/crates/common/src/spine.rs index 65cdb765..036511d2 100644 --- a/crates/common/src/spine.rs +++ b/crates/common/src/spine.rs @@ -36,13 +36,13 @@ pub struct SilverSpine { pub tile_info: ShmemData, /// New incoming network gossip messages - #[queue(size(2usize.pow(24)))] + #[queue(size(2usize.pow(19)))] pub gossip_in: SpineQueue, /// New incoming gossip messages - #[queue(size(2usize.pow(20)))] + #[queue(size(2usize.pow(18)))] pub new_gossip: SpineQueue, /// P2p send messages. - #[queue(size(2usize.pow(24)))] + #[queue(size(2usize.pow(19)))] pub p2p_send: SpineQueue, /// RPC recv messages. #[queue(size(2usize.pow(14)))] diff --git a/crates/network/src/p2p/quic/peer.rs b/crates/network/src/p2p/quic/peer.rs index ee3a3ae8..ea515515 100644 --- a/crates/network/src/p2p/quic/peer.rs +++ b/crates/network/src/p2p/quic/peer.rs @@ -634,7 +634,7 @@ fn id_from_connection(conn: &Connection) -> Option { fn out_buffer(id: &P2pStreamId, incoming: bool) -> OutboundBuffer { match id.protocol() { - StreamProtocol::GossipSub => OutboundBuffer::Gossip(OutBuffer::new(4096)), + StreamProtocol::GossipSub => OutboundBuffer::Gossip(OutBuffer::new(32 * 1024)), StreamProtocol::BeaconBlocksByRange | StreamProtocol::BeaconBlocksByRoot | StreamProtocol::DataColumnSidecarsByRange | From 9882f8a0622adb679041738fa060f3552db076a3 Mon Sep 17 00:00:00 2001 From: vladimir-ea Date: Sat, 22 Aug 2026 10:03:37 +0100 Subject: [PATCH 09/11] manage over population --- crates/common/src/spine.rs | 2 +- crates/common/src/spine/messages.rs | 4 ++++ crates/network/src/p2p/quic/peer.rs | 6 +++++- crates/network/src/p2p/streams/state.rs | 9 +++++++-- crates/network/src/tile.rs | 11 +++++++++-- crates/peer/src/manager.rs | 26 +++++++++++++++++++++++++ 6 files changed, 52 insertions(+), 6 deletions(-) diff --git a/crates/common/src/spine.rs b/crates/common/src/spine.rs index 036511d2..ab26d186 100644 --- a/crates/common/src/spine.rs +++ b/crates/common/src/spine.rs @@ -47,7 +47,7 @@ pub struct SilverSpine { /// RPC recv messages. #[queue(size(2usize.pow(14)))] pub rpc_inbound: SpineQueue, - #[queue(size(2usize.pow(14)))] + #[queue(size(2usize.pow(16)))] pub peer_events: SpineQueue, #[queue(size(2usize.pow(14)))] pub peer_control: SpineQueue, diff --git a/crates/common/src/spine/messages.rs b/crates/common/src/spine/messages.rs index 50cada94..d79abf73 100644 --- a/crates/common/src/spine/messages.rs +++ b/crates/common/src/spine/messages.rs @@ -662,6 +662,10 @@ pub enum PeerControl { p2p: PeerId, p2p_connection: usize, }, + P2pPeerGoodbye { + p2p_connection: usize, + code: u32, + }, /// Peer-level ban has timed out — counterpart to `Ban`. Network tile /// removes the peer from any deny-list / discv5 routing-table eviction /// state. Emitted from `tick` when the per-peer ban TTL expires. diff --git a/crates/network/src/p2p/quic/peer.rs b/crates/network/src/p2p/quic/peer.rs index ea515515..2ad1e292 100644 --- a/crates/network/src/p2p/quic/peer.rs +++ b/crates/network/src/p2p/quic/peer.rs @@ -13,7 +13,7 @@ use quinn_proto::{ VarInt, }; use silver_common::{ - P2pConnectionStats, P2pStreamId, PeerId, StreamProtocol, TRead, rpc_rate_limit::RpcRateLimitSet, + rpc_rate_limit::RpcRateLimitSet, P2pConnectionStats, P2pStreamId, PeerId, StreamProtocol, TRead }; use crate::{ @@ -441,7 +441,11 @@ impl Peer { let result = stream.spin(&mut self.connection, context, now, &mut self.inbound_rpc_limits, on_event); if let SpinResult::End = result { + if stream.p2p_id.protocol() == StreamProtocol::Goodbye { + self.shutdown(now); + } self.remove_stream(id); + return; } diff --git a/crates/network/src/p2p/streams/state.rs b/crates/network/src/p2p/streams/state.rs index 0cf78cd5..ca160140 100644 --- a/crates/network/src/p2p/streams/state.rs +++ b/crates/network/src/p2p/streams/state.rs @@ -382,8 +382,13 @@ impl StreamState { RpcWriteRequest::Complete(app_id) => { // close write side io.close_write(id.stream_id())?; - let read = RpcReadResponse::new(app_id, 0, now, &mut codec.dec); - Ok(Self::OutgoingRpc { rpc: RpcOut::ReadResponse(read), codec }) + + if id.protocol() == StreamProtocol::Goodbye { + Ok(Self::Finished) + } else { + let read = RpcReadResponse::new(app_id, 0, now, &mut codec.dec); + Ok(Self::OutgoingRpc { rpc: RpcOut::ReadResponse(read), codec }) + } } other => Ok(Self::OutgoingRpc { rpc: RpcOut::WriteRequest(other), codec }), } diff --git a/crates/network/src/tile.rs b/crates/network/src/tile.rs index 13176ac5..8166b41a 100644 --- a/crates/network/src/tile.rs +++ b/crates/network/src/tile.rs @@ -10,8 +10,7 @@ use mio::{Events, Poll, Token}; use quinn_proto::Transmit; use secp256k1::PublicKey; use silver_common::{ - BeaconStateEvent, GossipMsgIn, GossipMsgOut, P2pSend, PeerControl, PeerEvent, PeerStats, - RpcInbound, RpcOutbound, SilverSpine, + BeaconStateEvent, GossipMsgIn, GossipMsgOut, P2pSend, PeerControl, PeerEvent, PeerStats, RpcInbound, RpcOutbound, RpcRequestOutbound, SilverSpine }; use silver_discovery::{DiscV5, Discovery, DiscoveryEvent}; @@ -95,6 +94,14 @@ impl NetworkTile { PeerControl::P2pDisconnect { p2p: _, p2p_connection } => { self.inner.p2p_endpoint.disconnect(p2p_connection, now); } + PeerControl::P2pPeerGoodbye { p2p_connection, code } => { + let goodbye = RpcOutbound::Request(RpcRequestOutbound { + application_id: 0, + peer: p2p_connection, + request: silver_common::RpcRequest::Goodbye((code as u64).to_le_bytes()), + }); + self.inner.enqueue_rpc_out(goodbye); + } _ => {} // no-ops for this tile } } diff --git a/crates/peer/src/manager.rs b/crates/peer/src/manager.rs index 60443287..6cc24d0f 100644 --- a/crates/peer/src/manager.rs +++ b/crates/peer/src/manager.rs @@ -779,6 +779,9 @@ impl PeerManager { } }); + // 10) manage over population + self.manage_peers(emit); + crate::PeerCounters::PeersConnected.set(self.peers.len() as u64); } @@ -1096,6 +1099,29 @@ impl PeerManager { } } + fn manage_peers(&mut self, emit: &mut impl FnMut(PeerControl)) { + // + 10% for inbound calls. + if self.peers.len() > self.params.max_priority_peers + self.params.max_priority_peers / 10 { + // Remove upto 256 negative or zero scored peers. + let to_remove = (self.params.max_priority_peers - self.peers.len()).min(256); + let mut candidates = [(0usize, f64::MAX); 256]; + let mut offset = 0; + for (id, peer) in &self.peers { + if peer.cached_score <= 0.0 { + candidates[offset] = (*id, peer.cached_score); + offset += 1; + if offset == candidates.len() { + break; + } + } + } + candidates[..to_remove].sort_by(|(_, a), (_, b)| a.total_cmp(b)); + for (id, _) in &candidates[..to_remove] { + emit(PeerControl::P2pPeerGoodbye { p2p_connection: *id, code: 129 }) + } + } + } + fn maybe_request_discovery(&mut self, now: Instant, emit: &mut impl FnMut(PeerControl)) { if self.peers.len() >= self.params.target_peers { return; From 1a25fdd7e65382d1b06641210db39aeb6ac0b2d8 Mon Sep 17 00:00:00 2001 From: vladimir-ea Date: Sat, 22 Aug 2026 10:14:28 +0100 Subject: [PATCH 10/11] manage over population --- crates/network/src/p2p/quic/peer.rs | 27 +++++++--- crates/network/src/tile.rs | 5 +- crates/peer/src/manager.rs | 84 ++++++++++++++++++++++------- crates/peer/src/state.rs | 5 ++ 4 files changed, 93 insertions(+), 28 deletions(-) diff --git a/crates/network/src/p2p/quic/peer.rs b/crates/network/src/p2p/quic/peer.rs index 2ad1e292..bd55f485 100644 --- a/crates/network/src/p2p/quic/peer.rs +++ b/crates/network/src/p2p/quic/peer.rs @@ -13,7 +13,7 @@ use quinn_proto::{ VarInt, }; use silver_common::{ - rpc_rate_limit::RpcRateLimitSet, P2pConnectionStats, P2pStreamId, PeerId, StreamProtocol, TRead + P2pConnectionStats, P2pStreamId, PeerId, StreamProtocol, TRead, rpc_rate_limit::RpcRateLimitSet, }; use crate::{ @@ -400,7 +400,7 @@ impl Peer { on_event, ); for id in to_remove { - self.remove_stream(id); + self.end_stream(id, now); } // Read-response timeouts only fire inside a spin; sweep everything @@ -419,7 +419,7 @@ impl Peer { on_event, ); for id in to_remove { - self.remove_stream(id); + self.end_stream(id, now); } } } @@ -441,11 +441,7 @@ impl Peer { let result = stream.spin(&mut self.connection, context, now, &mut self.inbound_rpc_limits, on_event); if let SpinResult::End = result { - if stream.p2p_id.protocol() == StreamProtocol::Goodbye { - self.shutdown(now); - } - self.remove_stream(id); - + self.end_stream(id, now); return; } @@ -541,6 +537,21 @@ impl Peer { } } + /// Stream state machine reached its end. A flushed outbound Goodbye + /// additionally closes the whole connection — the goodbye contract: + /// send, then hang up. Inbound goodbye streams don't shut down here; + /// the PM owns that disconnect (and a rate-limit-dropped goodbye flood + /// must not hand the flooder a connection close). + fn end_stream(&mut self, id: StreamId, now: Instant) { + if let Some(stream) = self.streams.get(&id) && + stream.p2p_id.protocol() == StreamProtocol::Goodbye && + !stream.p2p_id.is_incoming() + { + self.shutdown(now); + } + self.remove_stream(id); + } + fn remove_stream(&mut self, id: StreamId) { let _ = self.connection.send_stream(id).finish(); let _ = self.connection.recv_stream(id).stop(VarInt::from_u32(0)); diff --git a/crates/network/src/tile.rs b/crates/network/src/tile.rs index 8166b41a..04404973 100644 --- a/crates/network/src/tile.rs +++ b/crates/network/src/tile.rs @@ -10,7 +10,8 @@ use mio::{Events, Poll, Token}; use quinn_proto::Transmit; use secp256k1::PublicKey; use silver_common::{ - BeaconStateEvent, GossipMsgIn, GossipMsgOut, P2pSend, PeerControl, PeerEvent, PeerStats, RpcInbound, RpcOutbound, RpcRequestOutbound, SilverSpine + BeaconStateEvent, GossipMsgIn, GossipMsgOut, P2pSend, PeerControl, PeerEvent, PeerStats, + RpcInbound, RpcOutbound, RpcRequestOutbound, SilverSpine, }; use silver_discovery::{DiscV5, Discovery, DiscoveryEvent}; @@ -95,7 +96,7 @@ impl NetworkTile { self.inner.p2p_endpoint.disconnect(p2p_connection, now); } PeerControl::P2pPeerGoodbye { p2p_connection, code } => { - let goodbye = RpcOutbound::Request(RpcRequestOutbound { + let goodbye = RpcOutbound::Request(RpcRequestOutbound { application_id: 0, peer: p2p_connection, request: silver_common::RpcRequest::Goodbye((code as u64).to_le_bytes()), diff --git a/crates/peer/src/manager.rs b/crates/peer/src/manager.rs index 6cc24d0f..5e0ef95c 100644 --- a/crates/peer/src/manager.rs +++ b/crates/peer/src/manager.rs @@ -1099,28 +1099,44 @@ impl PeerManager { } } + /// Trim over-population: past `max_priority_peers` + 10% inbound + /// headroom, goodbye (TooManyPeers) the worst strictly-negative scorers + /// back toward `max_priority_peers`. Neutral peers — including fresh + /// connections, which start at 0 — are never trimmed: with no negatives + /// we stay over cap until scores differentiate. Per-beat cap keeps + /// removal a trickle rather than a burst. fn manage_peers(&mut self, emit: &mut impl FnMut(PeerControl)) { - // + 10% for inbound calls. - if self.peers.len() > self.params.max_priority_peers + self.params.max_priority_peers / 10 { - // Remove upto 256 negative or zero scored peers. - let to_remove = (self.params.max_priority_peers - self.peers.len()).min(256); - let mut candidates = [(0usize, f64::MAX); 256]; - let mut offset = 0; - for (id, peer) in &self.peers { - if peer.cached_score <= 0.0 { - candidates[offset] = (*id, peer.cached_score); - offset += 1; - if offset == candidates.len() { - break; - } + const MAX_GOODBYES_PER_BEAT: usize = 16; + let cap = self.params.max_priority_peers + self.params.max_priority_peers / 10; + if self.peers.len() <= cap { + return; + } + let excess = self.peers.len() - self.params.max_priority_peers; + + // Bounded scan, no alloc: gather up to array-size negative + // candidates (first found, not globally worst), goodbye the worst + // of those. + let mut candidates = [(0usize, 0.0f64); 256]; + let mut found = 0; + for (id, peer) in &self.peers { + if peer.cached_score < 0.0 && !peer.goodbye_sent { + candidates[found] = (*id, peer.cached_score); + found += 1; + if found == candidates.len() { + break; } } - candidates[..to_remove].sort_by(|(_, a), (_, b)| a.total_cmp(b)); - for (id, _) in &candidates[..to_remove] { - emit(PeerControl::P2pPeerGoodbye { p2p_connection: *id, code: 129 }) - } } - } + let candidates = &mut candidates[..found]; + candidates.sort_unstable_by(|(_, a), (_, b)| a.total_cmp(b)); + + let to_remove = excess.min(MAX_GOODBYES_PER_BEAT).min(found); + for (id, _) in &candidates[..to_remove] { + let Some(peer) = self.peers.get_mut(id) else { continue }; + peer.goodbye_sent = true; + emit(PeerControl::P2pPeerGoodbye { p2p_connection: *id, code: 129 }); + } + } fn maybe_request_discovery(&mut self, now: Instant, emit: &mut impl FnMut(PeerControl)) { if self.peers.len() >= self.params.target_peers { @@ -1823,6 +1839,38 @@ pub(crate) mod tests { assert!((s - -45.0).abs() < 1e-9, "expected -45, got {s}"); } + #[test] + fn manage_peers_trims_worst_negatives_only_once() { + let now = Instant::now(); + let mut params = ScoreParams::default(); + params.max_priority_peers = 4; + let (mut mgr, mut cap) = fixture(vec![], params, false); + for conn in 1..=6usize { + connect(&mut mgr, &mut cap, conn, conn as u8, now); + } + mgr.peers.get_mut(&1).unwrap().cached_score = -5.0; + mgr.peers.get_mut(&2).unwrap().cached_score = -2.0; + cap.0.clear(); + + // 6 peers > cap(4): excess 2 — exactly the two negatives go, the + // four neutral (fresh) peers are never trimmed. + mgr.manage_peers(&mut |event| cap.0.push(event)); + let goodbyes: Vec = cap + .0 + .iter() + .filter_map(|event| match event { + PeerControl::P2pPeerGoodbye { p2p_connection, code: 129 } => Some(*p2p_connection), + _ => None, + }) + .collect(); + assert_eq!(goodbyes, vec![1, 2]); + + // Already-goodbyed peers are not re-selected while they drain. + cap.0.clear(); + mgr.manage_peers(&mut |event| cap.0.push(event)); + assert!(cap.0.is_empty()); + } + #[test] fn disconnect_archives_and_reconnect_restores() { let now = Instant::now(); diff --git a/crates/peer/src/state.rs b/crates/peer/src/state.rs index 6a4375f5..8eb13e97 100644 --- a/crates/peer/src/state.rs +++ b/crates/peer/src/state.rs @@ -78,6 +78,10 @@ pub(crate) struct PeerState { pub cached_score: f64, pub score_valid_at: Instant, pub last_breakdown: ScoreBreakdown, + /// TooManyPeers goodbye emitted; the connection is on its way down — + /// keeps `manage_peers` from re-selecting it while the flush + shutdown + /// completes. + pub goodbye_sent: bool, // Graylisted but kept for data-column coverage; dedups the spare log. pub evict_spared: bool, @@ -105,6 +109,7 @@ impl PeerState { cached_score: 0.0, score_valid_at: now, last_breakdown: ScoreBreakdown::default(), + goodbye_sent: false, evict_spared: false, } } From 10acb1ec2cc9584f3f65ed6b47c96458a98d71a0 Mon Sep 17 00:00:00 2001 From: vladimir-ea Date: Sat, 22 Aug 2026 11:06:51 +0100 Subject: [PATCH 11/11] reject inbound if over capacity --- crates/bin/src/main.rs | 1 + crates/config/src/lib.rs | 8 ++++++++ crates/e2e/src/stack.rs | 4 ++-- crates/network/benches/quic_basic.rs | 4 ++-- crates/network/benches/quic_pingpong.rs | 4 ++-- crates/network/src/lib.rs | 1 + crates/network/src/p2p/mod.rs | 15 ++++++++++++++- 7 files changed, 30 insertions(+), 7 deletions(-) diff --git a/crates/bin/src/main.rs b/crates/bin/src/main.rs index a524c6a0..317a9256 100644 --- a/crates/bin/src/main.rs +++ b/crates/bin/src/main.rs @@ -134,6 +134,7 @@ fn main() -> Result<(), Box> { false, None, ), + config.max_connections(), ); let p2p_context = Context { gossip_producer: incoming_gossip_producer, diff --git a/crates/config/src/lib.rs b/crates/config/src/lib.rs index 2d6fa0a1..9687b9b8 100644 --- a/crates/config/src/lib.rs +++ b/crates/config/src/lib.rs @@ -313,6 +313,14 @@ impl Config { self.discovery_config.clone() } + /// Hard cap on transport connections — inbound accepts are refused at + /// the QUIC layer beyond it. Sits above the peer manager's trim band + /// (`max_priority_peers` + 10%) so score-based trimming has room to + /// work inside it. + pub fn max_connections(&self) -> usize { + self.peer_score_params.max_priority_peers * 12 / 10 + } + pub fn peer_score_params(&self) -> ScoreParams { self.peer_score_params.clone() } diff --git a/crates/e2e/src/stack.rs b/crates/e2e/src/stack.rs index af1a4559..58e8b15f 100644 --- a/crates/e2e/src/stack.rs +++ b/crates/e2e/src/stack.rs @@ -249,7 +249,7 @@ impl PublisherStack { ); let endpoint = quic_endpoint(&keypair, /* is_server= */ true); - let p2p = P2p::new(keypair, endpoint); + let p2p = P2p::new(keypair, endpoint, 1024); let network = NetworkTile::new(disc_addr, discovery, addr, p2p, context) .map_err(std::io::Error::other)?; @@ -363,7 +363,7 @@ impl EchoStack { ); let endpoint = quic_endpoint(&keypair, /* is_server= */ true); - let p2p = P2p::new(keypair, endpoint); + let p2p = P2p::new(keypair, endpoint, 1024); let network = NetworkTile::new(disc_addr, discovery, addr, p2p, context) .map_err(std::io::Error::other)?; diff --git a/crates/network/benches/quic_basic.rs b/crates/network/benches/quic_basic.rs index 73c45ccb..bae507a6 100644 --- a/crates/network/benches/quic_basic.rs +++ b/crates/network/benches/quic_basic.rs @@ -69,7 +69,7 @@ pub fn broadcast(c: &mut Criterion) { identify: None, }; - let p2p = P2p::new(keypair, server_endpoint); + let p2p = P2p::new(keypair, server_endpoint, 1024); ( NetworkTileInner::new( "0.0.0.0:20001".parse().unwrap(), @@ -129,7 +129,7 @@ pub fn broadcast(c: &mut Criterion) { }; let addr = format!("127.0.0.1:{}", 20002 + n); - let mut p2p = P2p::new(keypair, client_endpoint); + let mut p2p = P2p::new(keypair, client_endpoint, 1024); p2p.connect( server_id.clone(), "127.0.0.1:20001".parse().unwrap(), diff --git a/crates/network/benches/quic_pingpong.rs b/crates/network/benches/quic_pingpong.rs index 078784bb..10ed2c15 100644 --- a/crates/network/benches/quic_pingpong.rs +++ b/crates/network/benches/quic_pingpong.rs @@ -59,7 +59,7 @@ pub fn broadcast(c: &mut Criterion) { false, None, ); - let p2p = P2p::new(keypair, server_endpoint); + let p2p = P2p::new(keypair, server_endpoint, 1024); let context = Context { gossip_producer: gi_producer, @@ -136,7 +136,7 @@ pub fn broadcast(c: &mut Criterion) { }; let addr = "127.0.0.1:20002"; - let mut p2p = P2p::new(keypair, client_endpoint); + let mut p2p = P2p::new(keypair, client_endpoint, 1024); p2p.connect( server_id.clone(), "127.0.0.1:20001".parse().unwrap(), diff --git a/crates/network/src/lib.rs b/crates/network/src/lib.rs index c351c052..37bca5fd 100644 --- a/crates/network/src/lib.rs +++ b/crates/network/src/lib.rs @@ -22,6 +22,7 @@ silver_common::declare_counters! { // (the "zombie": peer never responded). DialTimeoutZombie, InboundAccepted, + InboundRefused, InboundHandshakeOk, // Disconnect reason buckets (ConnectionError variants). DisconnectTimedOut, diff --git a/crates/network/src/p2p/mod.rs b/crates/network/src/p2p/mod.rs index ec9971dc..17c9ec63 100644 --- a/crates/network/src/p2p/mod.rs +++ b/crates/network/src/p2p/mod.rs @@ -111,10 +111,13 @@ pub struct P2p { timeout: Option, recv_count: usize, stats_cursor: usize, + /// Hard transport-level connection cap; inbound accepts refused beyond + /// it. Outbound dials are bounded separately by the peer manager. + max_connections: usize, } impl P2p { - pub fn new(keypair: Keypair, endpoint: Endpoint) -> Self { + pub fn new(keypair: Keypair, endpoint: Endpoint, max_connections: usize) -> Self { Self { keypair, endpoint, @@ -123,6 +126,7 @@ impl P2p { timeout: Some(Duration::ZERO), recv_count: 0, stats_cursor: 0, + max_connections, } } @@ -223,6 +227,15 @@ impl P2p { } } DatagramEvent::NewConnection(incoming) => { + // Hard population cap: refuse at the QUIC layer (pre-TLS, + // cheap) — the peer manager's score-based trim only polices + // quality inside the band below this. + if self.peers.len() >= self.max_connections { + crate::NetworkCounters::InboundRefused.inc(); + let rsp = self.endpoint.refuse(incoming, scratch); + let _ = socket.send_to(&scratch[..rsp.size], rsp.destination); + return true; + } match self.endpoint.accept(incoming, now, scratch, None) { Ok((handle, conn)) => { crate::NetworkCounters::InboundAccepted.inc();