From db726f3f50ab0bb3a91b2d51197ac9da8bf4c706 Mon Sep 17 00:00:00 2001 From: Wang Date: Fri, 28 Aug 2026 11:50:47 +0800 Subject: [PATCH 1/8] feat(runtime-host): discover peer mesh routes Sign short-lived route records with each libp2p identity and reconcile them between admitted peers. Feed only verified routes into the shared endpoint while preserving the target PeerId and Runtime Host credential authority. Cover route changes, removal propagation, restart recovery, native identity binding, and the installed CLI composition. Generated-by: Codex --- native/runtime-host-peer/src/bindings.rs | 49 ++ native/runtime-host-peer/src/engine.rs | 68 ++ native/runtime-host-peer/src/lib.rs | 4 +- .../src/__tests__/peer-mesh.test.ts | 85 +- .../src/__tests__/peer-native.test.ts | 35 +- .../runtime-host/src/client/peer-client.ts | 52 ++ packages/runtime-host/src/peer-mesh/index.ts | 8 + packages/runtime-host/src/peer-mesh/model.ts | 76 ++ packages/runtime-host/src/peer-mesh/node.ts | 732 +++++++++++++++++- packages/runtime-host/src/peer-mesh/owner.ts | 82 ++ packages/runtime-host/src/peer-mesh/store.ts | 44 +- .../runtime-host/src/transport/peer-native.ts | 81 ++ scripts/smoke-release-cli-package.mjs | 52 +- 13 files changed, 1278 insertions(+), 90 deletions(-) create mode 100644 packages/runtime-host/src/peer-mesh/owner.ts diff --git a/native/runtime-host-peer/src/bindings.rs b/native/runtime-host-peer/src/bindings.rs index a67ab4aaba..ced2d68479 100644 --- a/native/runtime-host-peer/src/bindings.rs +++ b/native/runtime-host-peer/src/bindings.rs @@ -31,6 +31,7 @@ use tokio::sync::{Mutex as AsyncMutex, mpsc, oneshot, watch}; use crate::engine::{self, EngineCommand, PeerError, StreamCommand}; type IncomingStreamReceiver = mpsc::Receiver, PeerError>>; +const IDENTITY_PAYLOAD_MAX_BYTES: usize = 8 * 1024; #[napi(object)] pub struct StartPeerEndpointOptions { @@ -49,6 +50,12 @@ pub struct ConnectPeerOptions { pub direct_deadline_ms: u32, } +#[napi(object)] +pub struct PeerIdentitySignature { + pub public_key: Buffer, + pub signature: Buffer, +} + #[napi] pub struct PeerEndpoint { peer_id: String, @@ -301,6 +308,38 @@ pub async fn ensure_peer_identity(key_path: String) -> Result { .map_err(peer_error) } +#[napi] +pub async fn sign_peer_identity( + key_path: String, + expected_peer_id: String, + payload: Buffer, +) -> Result { + validate_identity_payload(&payload)?; + let signed = engine::sign_identity( + PathBuf::from(key_path), + parse_peer_id(&expected_peer_id)?, + &payload, + ) + .await + .map_err(peer_error)?; + Ok(PeerIdentitySignature { + public_key: signed.public_key.into(), + signature: signed.signature.into(), + }) +} + +#[napi] +pub fn verify_peer_identity( + peer_id: String, + public_key: Buffer, + payload: Buffer, + signature: Buffer, +) -> Result { + validate_identity_payload(&payload)?; + engine::verify_identity(parse_peer_id(&peer_id)?, &public_key, &payload, &signature) + .map_err(peer_error) +} + fn wrap_stream(stream: engine::PeerStream) -> Result { Ok(PeerStream { peer_id: stream.peer_id.to_string(), @@ -327,6 +366,16 @@ fn parse_addresses(values: Vec, label: &str) -> Result> { .collect() } +fn validate_identity_payload(payload: &[u8]) -> Result<()> { + if payload.is_empty() || payload.len() > IDENTITY_PAYLOAD_MAX_BYTES { + return Err(Error::new( + Status::InvalidArg, + "identity payload must be between 1 and 8192 bytes", + )); + } + Ok(()) +} + fn peer_error(error: PeerError) -> Error { Error::new( Status::GenericFailure, diff --git a/native/runtime-host-peer/src/engine.rs b/native/runtime-host-peer/src/engine.rs index 81a7dcbbaa..7b33d11a9b 100644 --- a/native/runtime-host-peer/src/engine.rs +++ b/native/runtime-host-peer/src/engine.rs @@ -84,6 +84,11 @@ pub struct StartedEndpoint { pub thread: thread::JoinHandle<()>, } +pub struct IdentitySignature { + pub public_key: Vec, + pub signature: Vec, +} + pub struct ConnectOptions { pub request_id: u32, pub peer_id: PeerId, @@ -249,6 +254,37 @@ pub async fn ensure_identity(key_path: PathBuf) -> Result { .to_peer_id()) } +pub async fn sign_identity( + key_path: PathBuf, + expected_peer_id: PeerId, + payload: &[u8], +) -> Result { + let key = identity_store::load_key(&key_path).await?; + if PeerId::from(key.public()) != expected_peer_id { + return Err(PeerError::new( + "peer_identity_mismatch", + "the persisted peer identity does not match the expected PeerId", + )); + } + Ok(IdentitySignature { + public_key: key.public().encode_protobuf(), + signature: key + .sign(payload) + .map_err(|error| PeerError::new("peer_native_failed", error.to_string()))?, + }) +} + +pub fn verify_identity( + peer_id: PeerId, + public_key: &[u8], + payload: &[u8], + signature: &[u8], +) -> Result { + let public_key = identity::PublicKey::try_decode_protobuf(public_key) + .map_err(|error| PeerError::new("peer_native_failed", error.to_string()))?; + Ok(PeerId::from(&public_key) == peer_id && public_key.verify(payload, signature)) +} + pub fn start(options: StartOptions) -> Result { let (ready_tx, ready_rx) = std::sync::mpsc::sync_channel(1); let (command_tx, command_rx) = mpsc::channel(COMMAND_CAPACITY); @@ -1228,6 +1264,38 @@ fn native_error(error: impl std::fmt::Display) -> PeerError { mod tests { use super::*; + #[tokio::test] + async fn identity_signature_is_bound_to_peer_and_payload() { + let root = std::env::temp_dir().join(format!("maka-peer-signature-{}", PeerId::random())); + std::fs::create_dir_all(&root).expect("create test root"); + let key_path = root.join("peer.key"); + let peer_id = ensure_identity(key_path.clone()) + .await + .expect("create identity"); + let proof = sign_identity(key_path, peer_id, b"route") + .await + .expect("sign payload"); + + assert!( + verify_identity(peer_id, &proof.public_key, b"route", &proof.signature) + .expect("verify signature") + ); + assert!( + !verify_identity(peer_id, &proof.public_key, b"other", &proof.signature) + .expect("reject changed payload") + ); + assert!( + !verify_identity( + PeerId::random(), + &proof.public_key, + b"route", + &proof.signature, + ) + .expect("reject changed peer") + ); + std::fs::remove_dir_all(root).expect("remove test root"); + } + #[tokio::test(flavor = "multi_thread")] async fn mesh_control_survives_repeated_application_streams_on_one_endpoint() { let root = std::env::temp_dir().join(format!("maka-peer-test-{}", PeerId::random())); diff --git a/native/runtime-host-peer/src/lib.rs b/native/runtime-host-peer/src/lib.rs index 3726c0c014..8364692451 100644 --- a/native/runtime-host-peer/src/lib.rs +++ b/native/runtime-host-peer/src/lib.rs @@ -21,6 +21,6 @@ mod bindings; mod engine; pub use bindings::{ - ConnectPeerOptions, PeerEndpoint, PeerStream, StartPeerEndpointOptions, ensure_peer_identity, - start_peer_endpoint, + ConnectPeerOptions, PeerEndpoint, PeerIdentitySignature, PeerStream, StartPeerEndpointOptions, + ensure_peer_identity, sign_peer_identity, start_peer_endpoint, verify_peer_identity, }; diff --git a/packages/runtime-host/src/__tests__/peer-mesh.test.ts b/packages/runtime-host/src/__tests__/peer-mesh.test.ts index 81d88fc30b..78f0cfc607 100644 --- a/packages/runtime-host/src/__tests__/peer-mesh.test.ts +++ b/packages/runtime-host/src/__tests__/peer-mesh.test.ts @@ -18,6 +18,7 @@ */ import assert from 'node:assert/strict'; +import { createHash } from 'node:crypto'; import { mkdtemp, rm } from 'node:fs/promises'; import { tmpdir } from 'node:os'; import { join } from 'node:path'; @@ -93,6 +94,51 @@ test('rejects a modified authority-signed roster', () => { ); }); +test('reconciles changed routes, propagates removal, and recovers the verified cache', async () => { + const root = await mkdtemp(join(tmpdir(), 'maka-peer-mesh-routes-')); + const network = new MemoryPeerNetwork(); + const authorityPeer = network.create('peer-a'); + const memberBPeer = network.create('peer-b'); + const memberCPeer = network.create('peer-c'); + const authority = await openPeerMeshNode({ + dataRoot: join(root, 'authority'), + peer: authorityPeer, + }); + const memberB = await openPeerMeshNode({ dataRoot: join(root, 'member-b'), peer: memberBPeer }); + let memberC = await openPeerMeshNode({ dataRoot: join(root, 'member-c'), peer: memberCPeer }); + const serving = [authority.serve(), memberB.serve(), memberC.serve()]; + try { + const mesh = await authority.create(); + await memberB.join(await authority.invite(mesh.roster.roster.meshId)); + await memberC.join(await authority.invite(mesh.roster.roster.meshId)); + + await memberB.reconcile(); + assert.deepEqual(memberB.resolveRoutes('peer-c')?.routeHints, ['/memory/peer-c']); + + memberCPeer.setRouteHints(['/memory/peer-c-moved']); + await memberC.reconcile(); + await memberB.reconcile(); + assert.deepEqual(memberB.resolveRoutes('peer-c')?.routeHints, ['/memory/peer-c-moved']); + + await authority.remove(mesh.roster.roster.meshId, 'peer-b'); + await memberC.reconcile(); + await memberB.reconcile(); + assert.deepEqual(memberB.status(), []); + assert.equal(memberB.resolveRoutes('peer-c'), undefined); + assert.deepEqual(memberC.status()[0]?.roster.roster.members, ['peer-a', 'peer-c']); + + await memberC.close(); + await serving[2]; + memberC = await openPeerMeshNode({ dataRoot: join(root, 'member-c'), peer: memberCPeer }); + assert.deepEqual(memberC.resolveRoutes('peer-a')?.routeHints, ['/memory/peer-a']); + } finally { + await Promise.allSettled([authority.close(), memberB.close(), memberC.close()]); + await Promise.allSettled(serving); + await Promise.allSettled([authorityPeer.close(), memberBPeer.close(), memberCPeer.close()]); + await rm(root, { recursive: true, force: true }); + } +}); + test('closed Mesh records do not permanently consume membership capacity', async () => { const root = await mkdtemp(join(tmpdir(), 'maka-peer-mesh-capacity-')); const peer = new MemoryPeerNetwork().create('peer-a'); @@ -131,7 +177,6 @@ test('retries a committed invitation redemption for the same authenticated peer' authorityPeer.failNextResponse(); await assert.rejects(member.join(invitation)); - await authority.closeMesh(mesh.roster.roster.meshId); await authority.close(); await serving; @@ -148,8 +193,9 @@ test('retries a committed invitation redemption for the same authenticated peer' serving = authority.serve(); const joined = await member.join(invitation); assert.deepEqual(joined.roster.roster.members, ['peer-a', 'peer-b']); - assert.equal(joined.roster.roster.closed, true); - assert.equal(authority.status()[0]?.roster.roster.revision, 3); + assert.equal(joined.roster.roster.closed, false); + await member.reconcile(); + assert.equal(authority.status()[0]?.roster.roster.revision, 2); await authority.close(); await authorityPeer.close(); @@ -213,20 +259,45 @@ class MemoryPeerClient implements PeerMeshTransport { #closed = false; #failNextResponse = false; #stallNextControl = false; + #routeHints: readonly string[]; constructor( private readonly peerId: string, private readonly peers: ReadonlyMap, - ) {} + ) { + this.#routeHints = [`/memory/${peerId}`]; + } identity() { return { peerId: this.peerId, - listenAddresses: [`/memory/${this.peerId}`], + listenAddresses: this.#routeHints, coordinationRelays: [`/memory/relay/${this.peerId}`], } as const; } + setRouteHints(routeHints: readonly string[]): void { + this.#routeHints = [...routeHints]; + } + + signIdentity(payload: Buffer) { + return Promise.resolve({ + publicKey: Buffer.from(this.peerId), + signature: memorySignature(this.peerId, payload), + }); + } + + verifyIdentity( + peerId: string, + payload: Buffer, + proof: { readonly publicKey: Buffer; readonly signature: Buffer }, + ): boolean { + return ( + proof.publicKey.toString() === peerId && + proof.signature.equals(memorySignature(peerId, payload)) + ); + } + async connectMeshControl(input: { readonly peerId: string; }): Promise { @@ -285,6 +356,10 @@ class MemoryPeerClient implements PeerMeshTransport { } } +function memorySignature(peerId: string, payload: Buffer): Buffer { + return createHash('sha256').update(peerId).update(payload).digest(); +} + function memoryStreamPair(localPeerId: string, remotePeerId: string): [MemoryStream, MemoryStream] { const local = new MemoryStream(remotePeerId); const remote = new MemoryStream(localPeerId); diff --git a/packages/runtime-host/src/__tests__/peer-native.test.ts b/packages/runtime-host/src/__tests__/peer-native.test.ts index 97f079daea..7cb8deaf0f 100644 --- a/packages/runtime-host/src/__tests__/peer-native.test.ts +++ b/packages/runtime-host/src/__tests__/peer-native.test.ts @@ -49,18 +49,20 @@ module.exports = { stats, failEndpoint: () => { finishAccept?.(null); finishMeshAccept?.(null); }, ensurePeerIdentity: async () => 'client', + signPeerIdentity: async () => ({ publicKey: Buffer.from('public'), signature: Buffer.from('signature') }), + verifyPeerIdentity: () => true, startPeerEndpoint: () => { stats.starts += 1; return { peerId: 'client', listenAddresses: [], - connect: ({ requestId, peerId }) => { - stats.requests.push(requestId); + connect: ({ requestId, peerId, routeHints, coordinationRelays }) => { + stats.requests.push({ requestId, peerId, routeHints, coordinationRelays }); if (peerId === 'ready') return Promise.resolve(stream); return new Promise((_resolve, reject) => pending.set(requestId, reject)); }, - connectMeshControl: ({ requestId, peerId }) => { - stats.requests.push(requestId); + connectMeshControl: ({ requestId, peerId, routeHints, coordinationRelays }) => { + stats.requests.push({ requestId, peerId, routeHints, coordinationRelays }); if (peerId === 'ready') return Promise.resolve(stream); return new Promise((_resolve, reject) => pending.set(requestId, reject)); }, @@ -85,6 +87,12 @@ module.exports = { const client = createRuntimeHostPeerClient({ nativePath, keyPath: join(directory, 'peer.key'), + routeResolver: { + resolveRoutes: () => ({ + routeHints: ['/memory/discovered'], + coordinationRelays: ['/memory/relay'], + }), + }, }); const abort = new AbortController(); const pending = client.connect(peerConnectInput('pending'), abort.signal); @@ -96,7 +104,20 @@ module.exports = { assert.deepEqual(native.default.stats, { starts: 1, closes: 0, - requests: [1, 2], + requests: [ + { + requestId: 1, + peerId: 'pending', + routeHints: ['/memory/1', '/memory/discovered'], + coordinationRelays: ['/memory/relay'], + }, + { + requestId: 2, + peerId: 'ready', + routeHints: ['/memory/1', '/memory/discovered'], + coordinationRelays: ['/memory/relay'], + }, + ], cancellations: [1, 1], }); @@ -121,7 +142,7 @@ test('rejects an incomplete endpoint API and loads a compatible relative native const incompletePath = join(directory, 'incomplete.cjs'); await writeFile( incompletePath, - 'module.exports = { ensurePeerIdentity: async () => "peer", startPeerEndpoint: () => ({ peerId: "peer", listenAddresses: [] }) };\n', + 'module.exports = { ensurePeerIdentity: async () => "peer", signPeerIdentity: async () => ({ publicKey: Buffer.from("public"), signature: Buffer.from("signature") }), verifyPeerIdentity: () => true, startPeerEndpoint: () => ({ peerId: "peer", listenAddresses: [] }) };\n', ); assert.throws( () => @@ -139,6 +160,8 @@ test('rejects an incomplete endpoint API and loads a compatible relative native `const stream = { read: async () => null, write: async () => {}, close: async () => {}, abort: () => {} }; module.exports = { ensurePeerIdentity: async () => 'peer', + signPeerIdentity: async () => ({ publicKey: Buffer.from('public'), signature: Buffer.from('signature') }), + verifyPeerIdentity: () => true, startPeerEndpoint: () => ({ peerId: 'peer', listenAddresses: [], diff --git a/packages/runtime-host/src/client/peer-client.ts b/packages/runtime-host/src/client/peer-client.ts index d7ca0c6f1a..f160c35198 100644 --- a/packages/runtime-host/src/client/peer-client.ts +++ b/packages/runtime-host/src/client/peer-client.ts @@ -19,7 +19,10 @@ import { RuntimeHostPeerError, + signRuntimeHostPeerIdentity, startRuntimeHostPeerEndpoint, + verifyRuntimeHostPeerIdentity, + type RuntimeHostPeerIdentityProof, type RuntimeHostPeerNativeEndpoint, type RuntimeHostPeerNativeStream, } from '../transport/peer-native.js'; @@ -32,12 +35,23 @@ export interface RuntimeHostPeerConnectInput { readonly directDeadlineMs: number; } +export interface RuntimeHostPeerRouteResolver { + resolveRoutes(peerId: string): + | { + readonly routeHints: readonly string[]; + readonly coordinationRelays: readonly string[]; + } + | undefined; +} + export interface RuntimeHostPeerClient { identity(): Readonly<{ peerId: string; listenAddresses: readonly string[]; coordinationRelays: readonly string[]; }>; + signIdentity(payload: Buffer): Promise; + verifyIdentity(peerId: string, payload: Buffer, proof: RuntimeHostPeerIdentityProof): boolean; connect( input: RuntimeHostPeerConnectInput, signal?: AbortSignal, @@ -58,6 +72,7 @@ export function createRuntimeHostPeerClientFromEnvironment( options: { readonly listenAddresses?: readonly string[]; readonly coordinationRelays?: readonly string[]; + readonly routeResolver?: RuntimeHostPeerRouteResolver; } = {}, ): RuntimeHostPeerClient { const nativePath = environment.MAKA_RUNTIME_HOST_PEER_NATIVE_PATH; @@ -76,6 +91,7 @@ export function createRuntimeHostPeerClient(input: { readonly keyPath: string; readonly listenAddresses?: readonly string[]; readonly coordinationRelays?: readonly string[]; + readonly routeResolver?: RuntimeHostPeerRouteResolver; }): RuntimeHostPeerClient { return new RuntimeHostPeerClientImpl(input); } @@ -85,6 +101,7 @@ class RuntimeHostPeerClientImpl implements RuntimeHostPeerClient { readonly #keyPath: string; readonly #listenAddresses: readonly string[] | undefined; readonly #coordinationRelays: readonly string[] | undefined; + readonly #routeResolver: RuntimeHostPeerRouteResolver | undefined; #endpoint: RuntimeHostPeerNativeEndpoint | undefined; #draining: Promise | undefined; #meshDraining: Promise | undefined; @@ -105,11 +122,13 @@ class RuntimeHostPeerClientImpl implements RuntimeHostPeerClient { readonly keyPath: string; readonly listenAddresses?: readonly string[]; readonly coordinationRelays?: readonly string[]; + readonly routeResolver?: RuntimeHostPeerRouteResolver; }) { this.#nativePath = input.nativePath; this.#keyPath = input.keyPath; this.#listenAddresses = input.listenAddresses; this.#coordinationRelays = input.coordinationRelays; + this.#routeResolver = input.routeResolver; } identity(): Readonly<{ @@ -125,6 +144,26 @@ class RuntimeHostPeerClientImpl implements RuntimeHostPeerClient { }); } + signIdentity(payload: Buffer): Promise { + const peerId = this.#requireEndpoint().peerId; + return signRuntimeHostPeerIdentity({ + nativePath: this.#nativePath, + keyPath: this.#keyPath, + expectedPeerId: peerId, + payload, + }); + } + + verifyIdentity(peerId: string, payload: Buffer, proof: RuntimeHostPeerIdentityProof): boolean { + return verifyRuntimeHostPeerIdentity({ + nativePath: this.#nativePath, + peerId, + payload, + publicKey: proof.publicKey, + signature: proof.signature, + }); + } + async connect( input: RuntimeHostPeerConnectInput, signal?: AbortSignal, @@ -177,8 +216,14 @@ class RuntimeHostPeerClientImpl implements RuntimeHostPeerClient { signal?.throwIfAborted(); const endpoint = this.#requireEndpoint(); const requestId = this.#allocateRequestId(); + const discovered = this.#routeResolver?.resolveRoutes(input.peerId); const connection = endpoint[kind === 'application' ? 'connect' : 'connectMeshControl']({ ...input, + routeHints: mergeAddresses(input.routeHints, discovered?.routeHints), + coordinationRelays: mergeAddresses( + input.coordinationRelays ?? [], + discovered?.coordinationRelays, + ), requestId, }); let settled = false; @@ -303,6 +348,13 @@ class RuntimeHostPeerClientImpl implements RuntimeHostPeerClient { } } +function mergeAddresses( + primary: readonly string[], + secondary: readonly string[] | undefined, +): readonly string[] { + return Object.freeze([...new Set([...primary, ...(secondary ?? [])])].slice(0, 16)); +} + async function cancelPeerConnect( endpoint: RuntimeHostPeerNativeEndpoint, requestId: number, diff --git a/packages/runtime-host/src/peer-mesh/index.ts b/packages/runtime-host/src/peer-mesh/index.ts index 85f3fa825b..63c8e60273 100644 --- a/packages/runtime-host/src/peer-mesh/index.ts +++ b/packages/runtime-host/src/peer-mesh/index.ts @@ -22,11 +22,19 @@ export { type PeerMeshAuthorityTarget, type PeerMeshInvitationV1, type PeerMeshRosterV1, + type PeerMeshRouteRecordV1, type SignedPeerMeshRosterV1, + type SignedPeerMeshRouteRecordV1, } from './model.js'; export { openPeerMeshNode, type PeerMeshNode, + type PeerMeshReconcileResult, + type PeerMeshResolvedRoutes, type PeerMeshStatus, type PeerMeshTransport, } from './node.js'; +export { + openRuntimeHostPeerMeshOwner, + type RuntimeHostPeerMeshOwner, +} from './owner.js'; diff --git a/packages/runtime-host/src/peer-mesh/model.ts b/packages/runtime-host/src/peer-mesh/model.ts index bb618322f5..1ffba804d3 100644 --- a/packages/runtime-host/src/peer-mesh/model.ts +++ b/packages/runtime-host/src/peer-mesh/model.ts @@ -33,6 +33,7 @@ export const PEER_MESH_MAX_MESHES = 16; export const PEER_MESH_MAX_PENDING_INVITATIONS = 32; export const PEER_MESH_MAX_INVITATION_RECORDS = PEER_MESH_MAX_PENDING_INVITATIONS * 3; export const PEER_MESH_MAX_ROUTE_HINTS = 16; +export const PEER_MESH_ROUTE_RECORD_MAX_BYTES = 4 * 1024; export interface PeerMeshRosterV1 { readonly version: 1; @@ -66,6 +67,18 @@ export interface PeerMeshAuthorityKeyPair { readonly privateKey: string; } +export interface PeerMeshRouteRecordV1 extends PeerMeshAuthorityTarget { + readonly version: 1; + readonly sequence: number; + readonly expiresAt: number; +} + +export interface SignedPeerMeshRouteRecordV1 { + readonly route: PeerMeshRouteRecordV1; + readonly publicKey: string; + readonly signature: string; +} + export function generatePeerMeshAuthorityKeyPair(): PeerMeshAuthorityKeyPair { const { publicKey, privateKey } = generateKeyPairSync('ed25519'); return Object.freeze({ @@ -229,6 +242,60 @@ export function decodeAuthorityTarget(value: unknown): PeerMeshAuthorityTarget { }); } +export function canonicalPeerMeshRouteRecord(value: unknown): PeerMeshRouteRecordV1 { + const record = exactObject(value, 'Peer Mesh route record', [ + 'version', + 'peerId', + 'sequence', + 'expiresAt', + 'routeHints', + 'coordinationRelays', + ]); + if (record.version !== 1) throw new Error('Unsupported Peer Mesh route record version'); + const route = Object.freeze({ + version: 1 as const, + peerId: token(record.peerId, 'peerId', 256), + sequence: integer(record.sequence, 'route sequence', 1), + expiresAt: integer(record.expiresAt, 'route expiry', 1), + routeHints: Object.freeze(addressArray(record.routeHints, 'routeHints')), + coordinationRelays: Object.freeze( + addressArray(record.coordinationRelays, 'coordinationRelays'), + ), + }); + if (peerMeshRouteRecordSigningBytes(route).byteLength > PEER_MESH_ROUTE_RECORD_MAX_BYTES) { + throw new Error('Peer Mesh route record is too large'); + } + return route; +} + +export function decodeSignedPeerMeshRouteRecord(value: unknown): SignedPeerMeshRouteRecordV1 { + const record = exactObject(value, 'signed Peer Mesh route record', [ + 'route', + 'publicKey', + 'signature', + ]); + const publicKey = canonicalProof(record.publicKey, 'route public key', 256); + const signature = canonicalProof(record.signature, 'route signature', 256); + return Object.freeze({ + route: canonicalPeerMeshRouteRecord(record.route), + publicKey, + signature, + }); +} + +export function peerMeshRouteRecordSigningBytes(route: PeerMeshRouteRecordV1): Buffer { + return Buffer.from( + `maka.peer-mesh.route.v1\n${JSON.stringify({ + coordinationRelays: route.coordinationRelays, + expiresAt: route.expiresAt, + peerId: route.peerId, + routeHints: route.routeHints, + sequence: route.sequence, + version: route.version, + })}`, + ); +} + function encodeRoster(roster: PeerMeshRosterV1): Buffer { return Buffer.from( `maka.peer-mesh.roster.v1\n${JSON.stringify({ @@ -279,6 +346,15 @@ function decodeCanonicalBase64Url(value: string, label: string): Buffer { return bytes; } +function canonicalProof(value: unknown, label: string, maxBytes: number): string { + const encoded = string(value, label, Math.ceil((maxBytes * 4) / 3)); + const bytes = decodeCanonicalBase64Url(encoded, label); + if (bytes.length === 0 || bytes.length > maxBytes) { + throw new Error(`Invalid Peer Mesh ${label}`); + } + return encoded; +} + function object(value: unknown, label: string): Record { if (!value || typeof value !== 'object' || Array.isArray(value)) { throw new Error(`Invalid ${label}`); diff --git a/packages/runtime-host/src/peer-mesh/node.ts b/packages/runtime-host/src/peer-mesh/node.ts index 76cac380cb..e1cd238625 100644 --- a/packages/runtime-host/src/peer-mesh/node.ts +++ b/packages/runtime-host/src/peer-mesh/node.ts @@ -17,24 +17,33 @@ * under the License. */ -import type { RuntimeHostPeerNativeStream } from '../transport/peer-native.js'; +import type { + RuntimeHostPeerIdentityProof, + RuntimeHostPeerNativeStream, +} from '../transport/peer-native.js'; +import { setTimeout as delay } from 'node:timers/promises'; import { canonicalPeerMeshRoster, + canonicalPeerMeshRouteRecord, createPeerMeshInvitationSecret, decodePeerMeshInvitation, decodeSignedPeerMeshRoster, + decodeSignedPeerMeshRouteRecord, generatePeerMeshAuthorityKeyPair, matchesPeerMeshInvitationSecret, PEER_MESH_MAX_MEMBERS, PEER_MESH_MAX_MESHES, PEER_MESH_MAX_INVITATION_RECORDS, PEER_MESH_MAX_PENDING_INVITATIONS, + peerMeshRouteRecordSigningBytes, peerMeshId, peerMeshInvitationSecretDigest, signPeerMeshRoster, type PeerMeshAuthorityTarget, type PeerMeshInvitationV1, + type PeerMeshRouteRecordV1, type SignedPeerMeshRosterV1, + type SignedPeerMeshRouteRecordV1, } from './model.js'; import { authorityKeys, @@ -50,17 +59,24 @@ const CONNECT_DEADLINE_MS = 30_000; const CONTROL_REQUEST_DEADLINE_MS = 10_000; const MAX_ACTIVE_CONTROL_STREAMS = 32; const MAX_ACTIVE_CONTROL_STREAMS_PER_PEER = 2; +const ROUTE_TTL_MS = 5 * 60 * 1_000; +const ROUTE_REFRESH_LEAD_MS = 60 * 1_000; +const ROUTE_MAX_FUTURE_MS = 10 * 60 * 1_000; +const ROUTE_PAGE_SIZE = 8; +const RECONCILE_INTERVAL_MS = 30 * 1_000; interface RedeemInvitationRequest { readonly kind: 'redeem-invitation'; readonly meshId: string; readonly secret: string; + readonly route: SignedPeerMeshRouteRecordV1; } type RedeemInvitationResponse = | { readonly kind: 'invitation-redeemed'; readonly roster: SignedPeerMeshRosterV1; + readonly routes: readonly SignedPeerMeshRouteRecordV1[]; } | { readonly kind: 'invitation-rejected'; @@ -69,6 +85,30 @@ type RedeemInvitationResponse = type RedeemInvitationRejectionReason = 'invalid' | 'expired' | 'closed' | 'full'; +interface PeerMeshRouteSequence { + readonly peerId: string; + readonly sequence: number; +} + +interface SyncPeerMeshRequest { + readonly kind: 'sync'; + readonly meshId: string; + readonly roster: SignedPeerMeshRosterV1; + readonly route: SignedPeerMeshRouteRecordV1; + readonly knownRoutes: readonly PeerMeshRouteSequence[]; +} + +type SyncPeerMeshResponse = + | { + readonly kind: 'sync-result'; + readonly roster: SignedPeerMeshRosterV1; + readonly routes: readonly SignedPeerMeshRouteRecordV1[]; + readonly more: boolean; + } + | { readonly kind: 'sync-rejected'; readonly reason: 'unknown' }; + +type PeerMeshControlRequest = RedeemInvitationRequest | SyncPeerMeshRequest; + export interface PeerMeshNode { status(): readonly PeerMeshStatus[]; create(): Promise; @@ -76,10 +116,28 @@ export interface PeerMeshNode { join(invitation: PeerMeshInvitationV1, signal?: AbortSignal): Promise; remove(meshId: string, peerId: string): Promise; closeMesh(meshId: string): Promise; + resolveRoutes(peerId: string): PeerMeshResolvedRoutes | undefined; + reconcile(signal?: AbortSignal): Promise; serve(): Promise; close(): Promise; } +export interface PeerMeshResolvedRoutes { + readonly routeHints: readonly string[]; + readonly coordinationRelays: readonly string[]; + readonly expiresAt: number; +} + +export interface PeerMeshReconcileResult { + readonly attempted: number; + readonly synchronized: number; + readonly failures: readonly { + readonly meshId: string; + readonly peerId: string; + readonly message: string; + }[]; +} + export interface PeerMeshStatus { readonly role: 'authority' | 'member'; readonly localPeerId: string; @@ -94,6 +152,8 @@ export interface PeerMeshTransport { listenAddresses: readonly string[]; coordinationRelays: readonly string[]; }>; + signIdentity(payload: Buffer): Promise; + verifyIdentity(peerId: string, payload: Buffer, proof: RuntimeHostPeerIdentityProof): boolean; connectMeshControl( input: { readonly peerId: string; @@ -115,7 +175,14 @@ export async function openPeerMeshNode(input: { readonly now?: () => number; }): Promise { const store = await openPeerMeshStateStore(input.dataRoot, input.peer.identity().peerId); - return new PeerMeshNodeImpl({ ...input, store }); + const node = new PeerMeshNodeImpl({ ...input, store }); + try { + await node.initialize(); + return node; + } catch (error) { + await node.close().catch(() => undefined); + throw error; + } } class PeerMeshNodeImpl implements PeerMeshNode { @@ -125,6 +192,8 @@ class PeerMeshNodeImpl implements PeerMeshNode { readonly #activeControlStreams = new Set(); readonly #lifetime = new AbortController(); #admissionTail = Promise.resolve(); + #reconcileTail = Promise.resolve(); + #routeRefreshTask: Promise | undefined; #serveTask: Promise | undefined; #closeTask: Promise | undefined; @@ -138,6 +207,16 @@ class PeerMeshNodeImpl implements PeerMeshNode { this.#now = input.now ?? Date.now; } + async initialize(): Promise { + for (const state of this.#store.read()) { + for (const route of state.routes) this.#assertRouteSignature(route); + } + await this.#store.mutate((current) => ({ + state: current.map((state) => pruneRoutes(state, this.#now())), + result: undefined, + })); + } + status(): readonly PeerMeshStatus[] { this.#assertOpen(); const identity = this.#peer.identity(); @@ -171,12 +250,14 @@ class PeerMeshNodeImpl implements PeerMeshNode { const state: PeerMeshStateV1 = { role: 'authority', roster, + routes: [], authorityPrivateKey: keys.privateKey, invitations: [], }; return { state: appendMesh(current, state, identity.peerId), result: state }; }); - return peerMeshStatus(state, identity); + await this.#refreshLocalRoute(); + return peerMeshStatus(findMesh(this.#store.read(), state.roster.roster.meshId)!, identity); }); } @@ -253,12 +334,19 @@ class PeerMeshNodeImpl implements PeerMeshNode { operationSignal, ); try { + const localRoute = await this.#signLocalRoute(); const request: RedeemInvitationRequest = { kind: 'redeem-invitation', meshId: invitation.meshId, secret: invitation.secret, + route: localRoute, }; - const response = await exchangeControl(stream, request, operationSignal); + const response = await exchangeControl( + stream, + request, + decodeRedeemResponse, + operationSignal, + ); if (response.kind === 'invitation-rejected') { throw new Error(`Peer Mesh invitation was rejected: ${response.reason}`); } @@ -271,6 +359,8 @@ class PeerMeshNodeImpl implements PeerMeshNode { ) { throw new Error('Peer Mesh authority returned an unrelated roster'); } + const routes = await this.#validateRoutes(response.routes, roster, this.#now()); + const joinedRoutes = mergeRoutes(routes, [localRoute], roster, this.#now()); const state: PeerMeshStateV1 = { role: 'replica', authority: { @@ -279,6 +369,7 @@ class PeerMeshNodeImpl implements PeerMeshNode { coordinationRelays: invitation.coordinationRelays, }, roster, + routes: joinedRoutes, }; const joined = await this.#store.mutate((current) => { const existing = findMesh(current, invitation.meshId); @@ -329,13 +420,41 @@ class PeerMeshNodeImpl implements PeerMeshNode { })); } + resolveRoutes(peerId: string): PeerMeshResolvedRoutes | undefined { + this.#assertOpen(); + const now = this.#now(); + const route = this.#store + .read() + .filter((state) => isActiveMembership(state, this.#peer.identity().peerId)) + .flatMap((state) => state.routes) + .filter(({ route }) => route.peerId === peerId && route.expiresAt > now) + .sort((left, right) => right.route.sequence - left.route.sequence)[0]?.route; + if (!route) return undefined; + return Object.freeze({ + routeHints: route.routeHints, + coordinationRelays: route.coordinationRelays, + expiresAt: route.expiresAt, + }); + } + + reconcile(signal?: AbortSignal): Promise { + if (this.#lifetime.signal.aborted) return Promise.reject(new Error('Peer Mesh node is closed')); + const task = this.#reconcileTail.then(() => this.#reconcile(signal)); + this.#reconcileTail = task.then( + () => undefined, + () => undefined, + ); + return task; + } + async serve(): Promise { if (this.#lifetime.signal.aborted) throw new Error('Peer Mesh node is closed'); if (this.#serveTask) throw new Error('Peer Mesh node is already serving'); - const serving = this.#peer.serveMeshControl( + const inbound = this.#peer.serveMeshControl( (stream) => this.#acceptIncoming(stream), this.#lifetime.signal, ); + const serving = Promise.all([inbound, this.#runReconciliation()]).then(() => undefined); this.#serveTask = serving; try { await serving; @@ -358,10 +477,257 @@ class PeerMeshNodeImpl implements PeerMeshNode { await this.#serveTask?.catch(() => undefined); for (const stream of this.#activeControlStreams) stream.abort(); this.#activeControlStreams.clear(); - await this.#admissionTail; + await Promise.all([this.#admissionTail, this.#reconcileTail]); return this.#store.close(); } + async #runReconciliation(): Promise { + while (!this.#lifetime.signal.aborted) { + await this.reconcile(this.#lifetime.signal).catch(() => undefined); + await delay(RECONCILE_INTERVAL_MS, undefined, { signal: this.#lifetime.signal }).catch( + () => undefined, + ); + } + } + + async #reconcile(signal?: AbortSignal): Promise { + const operationSignal = signal + ? AbortSignal.any([signal, this.#lifetime.signal]) + : this.#lifetime.signal; + operationSignal.throwIfAborted(); + await this.#refreshLocalRoute(); + const identity = this.#peer.identity(); + const targets = new Map< + string, + { readonly meshId: string; readonly target: PeerMeshAuthorityTarget } + >(); + for (const state of this.#store.read()) { + if (!isActiveMembership(state, identity.peerId)) continue; + for (const signed of state.routes) { + const route = signed.route; + if ( + route.peerId !== identity.peerId && + route.expiresAt > this.#now() && + state.roster.roster.members.includes(route.peerId) + ) { + targets.set(`${state.roster.roster.meshId}\0${route.peerId}`, { + meshId: state.roster.roster.meshId, + target: route, + }); + } + } + if (state.role === 'replica' && state.authority.peerId !== identity.peerId) { + const key = `${state.roster.roster.meshId}\0${state.authority.peerId}`; + if (!targets.has(key)) + targets.set(key, { meshId: state.roster.roster.meshId, target: state.authority }); + } + } + const failures: Array<{ meshId: string; peerId: string; message: string }> = []; + let synchronized = 0; + for (const { meshId, target } of targets.values()) { + operationSignal.throwIfAborted(); + try { + await this.#syncPeer(meshId, target, operationSignal); + synchronized += 1; + } catch (error) { + if (operationSignal.aborted) operationSignal.throwIfAborted(); + failures.push({ + meshId, + peerId: target.peerId, + message: boundedErrorMessage(error), + }); + } + } + return Object.freeze({ + attempted: targets.size, + synchronized, + failures: Object.freeze(failures.map((failure) => Object.freeze(failure))), + }); + } + + async #syncPeer( + meshId: string, + target: PeerMeshAuthorityTarget, + signal: AbortSignal, + ): Promise { + for (let page = 0; page <= Math.ceil(PEER_MESH_MAX_MEMBERS / ROUTE_PAGE_SIZE); page += 1) { + const state = findMesh(this.#store.read(), meshId); + const localPeerId = this.#peer.identity().peerId; + if (!state || !isActiveMembership(state, localPeerId)) return; + const route = state.routes.find((candidate) => candidate.route.peerId === localPeerId); + if (!route) throw new Error('Peer Mesh local route is unavailable'); + const stream = await this.#peer.connectMeshControl( + { + peerId: target.peerId, + routeHints: target.routeHints, + coordinationRelays: target.coordinationRelays, + directDeadlineMs: CONNECT_DEADLINE_MS, + }, + signal, + ); + try { + const response = await exchangeControl( + stream, + { + kind: 'sync', + meshId, + roster: state.roster, + route, + knownRoutes: routeSequences(state.routes, this.#now()), + }, + decodeSyncResponse, + signal, + ); + if (response.kind === 'sync-rejected') { + throw new Error(`Peer Mesh synchronization was rejected: ${response.reason}`); + } + await this.#applySync(meshId, response.roster, response.routes); + if (!response.more) return; + } finally { + await stream.close().catch(() => undefined); + } + } + throw new Error('Peer Mesh synchronization exceeded its page bound'); + } + + #refreshLocalRoute(): Promise { + this.#routeRefreshTask ??= this.#refreshLocalRouteOnce().finally(() => { + this.#routeRefreshTask = undefined; + }); + return this.#routeRefreshTask; + } + + async #refreshLocalRouteOnce(): Promise { + const identity = this.#peer.identity(); + const current = this.#store.read(); + const active = current.filter((state) => isActiveMembership(state, identity.peerId)); + if (active.length === 0) return undefined; + const existing = active + .flatMap((state) => state.routes) + .filter(({ route }) => route.peerId === identity.peerId) + .sort((left, right) => right.route.sequence - left.route.sequence)[0]; + const now = this.#now(); + if ( + existing && + existing.route.expiresAt > now + ROUTE_REFRESH_LEAD_MS && + sameAddresses(existing.route.routeHints, identity.listenAddresses) && + sameAddresses(existing.route.coordinationRelays, identity.coordinationRelays) + ) { + return existing; + } + const route = await this.#signLocalRoute((existing?.route.sequence ?? 0) + 1); + await this.#store.mutate((states) => ({ + state: states.map((state) => + isActiveMembership(state, identity.peerId) + ? { ...state, routes: mergeRoutes(state.routes, [route], state.roster, now) } + : state, + ), + result: undefined, + })); + return route; + } + + async #signLocalRoute(sequence?: number): Promise { + const identity = this.#peer.identity(); + const maxSequence = this.#store + .read() + .flatMap((state) => state.routes) + .filter(({ route }) => route.peerId === identity.peerId) + .reduce((maximum, { route }) => Math.max(maximum, route.sequence), 0); + const route = canonicalPeerMeshRouteRecord({ + version: 1, + peerId: identity.peerId, + sequence: sequence ?? maxSequence + 1, + expiresAt: this.#now() + ROUTE_TTL_MS, + routeHints: identity.listenAddresses, + coordinationRelays: identity.coordinationRelays, + }); + const proof = await this.#peer.signIdentity(peerMeshRouteRecordSigningBytes(route)); + const signed = decodeSignedPeerMeshRouteRecord({ + route, + publicKey: proof.publicKey.toString('base64url'), + signature: proof.signature.toString('base64url'), + }); + this.#assertRouteSignature(signed); + return signed; + } + + #validateRoutes( + values: readonly SignedPeerMeshRouteRecordV1[], + roster: SignedPeerMeshRosterV1, + now: number, + ): readonly SignedPeerMeshRouteRecordV1[] { + if (values.length > PEER_MESH_MAX_MEMBERS) throw new Error('Too many Peer Mesh routes'); + const routes = values.map(decodeSignedPeerMeshRouteRecord); + if (new Set(routes.map(({ route }) => route.peerId)).size !== routes.length) { + throw new Error('Duplicate Peer Mesh routes'); + } + for (const signed of routes) { + if (!roster.roster.members.includes(signed.route.peerId)) { + throw new Error('Peer Mesh route is outside the active roster or lifetime'); + } + this.#validateRemoteRoute(signed, signed.route.peerId, now); + } + return Object.freeze(routes); + } + + #validateRemoteRoute( + value: SignedPeerMeshRouteRecordV1, + expectedPeerId: string, + now = this.#now(), + ): SignedPeerMeshRouteRecordV1 { + const signed = decodeSignedPeerMeshRouteRecord(value); + if ( + signed.route.peerId !== expectedPeerId || + signed.route.expiresAt <= now || + signed.route.expiresAt > now + ROUTE_MAX_FUTURE_MS + ) { + throw new Error('Peer Mesh route is outside the authenticated peer or lifetime'); + } + this.#assertRouteSignature(signed); + return signed; + } + + #assertRouteSignature(signedValue: SignedPeerMeshRouteRecordV1): void { + const signed = decodeSignedPeerMeshRouteRecord(signedValue); + const valid = this.#peer.verifyIdentity( + signed.route.peerId, + peerMeshRouteRecordSigningBytes(signed.route), + { + publicKey: Buffer.from(signed.publicKey, 'base64url'), + signature: Buffer.from(signed.signature, 'base64url'), + }, + ); + if (!valid) throw new Error('Peer Mesh route signature is invalid'); + } + + async #applySync( + meshId: string, + rosterValue: SignedPeerMeshRosterV1, + routeValues: readonly SignedPeerMeshRouteRecordV1[], + ): Promise { + const roster = decodeSignedPeerMeshRoster(rosterValue); + if (roster.roster.meshId !== meshId) throw new Error('Peer Mesh synchronization changed Mesh'); + const routes = this.#validateRoutes(routeValues, roster, this.#now()); + const localPeerId = this.#peer.identity().peerId; + await this.#store.mutate((current) => { + const state = findMesh(current, meshId); + if (!state || state.roster.authorityPublicKey !== roster.authorityPublicKey) { + throw new Error('Peer Mesh synchronization has the wrong authority'); + } + const nextRoster = selectRoster(state.roster, roster); + const next = { + ...state, + roster: nextRoster, + routes: + nextRoster.roster.closed || !nextRoster.roster.members.includes(localPeerId) + ? [] + : mergeRoutes(state.routes, routes, nextRoster, this.#now()), + }; + return { state: replaceMesh(current, next), result: undefined }; + }); + } + #assertOpen(): void { if (this.#lifetime.signal.aborted) throw new Error('Peer Mesh node is closed'); } @@ -396,6 +762,9 @@ class PeerMeshNodeImpl implements PeerMeshNode { const updated = { ...state, roster, + routes: next.closed + ? [] + : state.routes.filter(({ route }) => next.members.includes(route.peerId)), invitations: next.closed ? state.invitations.filter(({ status }) => status === 'redeemed') : state.invitations.filter( @@ -445,8 +814,18 @@ class PeerMeshNodeImpl implements PeerMeshNode { async #handleIncoming(stream: RuntimeHostPeerNativeStream): Promise { const deadline = setTimeout(() => stream.abort(), CONTROL_REQUEST_DEADLINE_MS); try { - const request = decodeRedeemRequest(await readFrame(stream)); - const response = await this.#redeem(request, stream.peerId); + const request = decodeControlRequest(await readFrame(stream)); + let response: RedeemInvitationResponse | SyncPeerMeshResponse; + if (request.kind === 'redeem-invitation') { + await this.#refreshLocalRoute(); + response = await this.#redeem( + request, + stream.peerId, + this.#validateRemoteRoute(request.route, stream.peerId), + ); + } else { + response = await this.#sync(request, stream.peerId); + } await writeFrame(stream, response); await stream.close(); } catch { @@ -459,6 +838,7 @@ class PeerMeshNodeImpl implements PeerMeshNode { #redeem( request: RedeemInvitationRequest, remotePeerId: string, + remoteRoute: SignedPeerMeshRouteRecordV1, ): Promise { const now = this.#now(); return this.#store.mutate((current) => { @@ -478,9 +858,21 @@ class PeerMeshNodeImpl implements PeerMeshNode { ) { return { state: current, result: rejected('invalid') }; } + const updated = { + ...state, + routes: mergeAuthenticatedRoute(state.routes, remoteRoute, state.roster, now), + }; return { - state: current, - result: { kind: 'invitation-redeemed', roster: state.roster }, + state: replaceMesh(current, updated), + result: { + kind: 'invitation-redeemed', + roster: updated.roster, + routes: responseRoutes( + updated, + [{ peerId: remotePeerId, sequence: Number.MAX_SAFE_INTEGER }], + now, + ).routes, + }, }; } const remaining = state.invitations.filter( @@ -509,17 +901,27 @@ class PeerMeshNodeImpl implements PeerMeshNode { }; } if (state.roster.roster.members.includes(remotePeerId)) { + const updated = { + ...state, + routes: mergeAuthenticatedRoute(state.routes, remoteRoute, state.roster, now), + invitations: [ + ...remaining.filter( + (record) => record.status === 'pending' || record.peerId !== remotePeerId, + ), + redeemedInvitation(invitation, remotePeerId), + ], + }; return { - state: replaceMesh(current, { - ...state, - invitations: [ - ...remaining.filter( - (record) => record.status === 'pending' || record.peerId !== remotePeerId, - ), - redeemedInvitation(invitation, remotePeerId), - ], - }), - result: { kind: 'invitation-redeemed', roster: state.roster }, + state: replaceMesh(current, updated), + result: { + kind: 'invitation-redeemed', + roster: state.roster, + routes: responseRoutes( + updated, + [{ peerId: remotePeerId, sequence: Number.MAX_SAFE_INTEGER }], + now, + ).routes, + }, }; } const members = [...state.roster.roster.members, remotePeerId].sort(); @@ -531,18 +933,72 @@ class PeerMeshNodeImpl implements PeerMeshNode { }, authorityKeys(state), ); + const updated = { + ...state, + roster, + routes: mergeRoutes(state.routes, [remoteRoute], roster, now), + invitations: [ + ...remaining.filter( + (record) => record.status === 'pending' || record.peerId !== remotePeerId, + ), + redeemedInvitation(invitation, remotePeerId), + ], + }; return { - state: replaceMesh(current, { - ...state, + state: replaceMesh(current, updated), + result: { + kind: 'invitation-redeemed', roster, - invitations: [ - ...remaining.filter( - (record) => record.status === 'pending' || record.peerId !== remotePeerId, - ), - redeemedInvitation(invitation, remotePeerId), - ], - }), - result: { kind: 'invitation-redeemed', roster }, + routes: responseRoutes( + updated, + [{ peerId: remotePeerId, sequence: Number.MAX_SAFE_INTEGER }], + now, + ).routes, + }, + }; + }); + } + + async #sync(request: SyncPeerMeshRequest, remotePeerId: string): Promise { + const remoteRoute = this.#validateRemoteRoute(request.route, remotePeerId); + await this.#refreshLocalRoute(); + const incomingRoster = decodeSignedPeerMeshRoster(request.roster); + return this.#store.mutate((current) => { + const state = findMesh(current, request.meshId); + if (!state || state.roster.authorityPublicKey !== incomingRoster.authorityPublicKey) { + return { state: current, result: { kind: 'sync-rejected', reason: 'unknown' } as const }; + } + const roster = selectRoster(state.roster, incomingRoster); + const localPeerId = this.#peer.identity().peerId; + const bothMembers = + !roster.roster.closed && + roster.roster.members.includes(localPeerId) && + roster.roster.members.includes(remotePeerId); + const updated = { + ...state, + roster, + routes: bothMembers ? mergeRoutes(state.routes, [remoteRoute], roster, this.#now()) : [], + }; + if (!bothMembers) { + return { + state: replaceMesh(current, updated), + result: { + kind: 'sync-result', + roster, + routes: [], + more: false, + } as const, + }; + } + const page = responseRoutes(updated, request.knownRoutes, this.#now()); + return { + state: replaceMesh(current, updated), + result: { + kind: 'sync-result', + roster, + routes: page.routes, + more: page.more, + } as const, }; }); } @@ -632,11 +1088,129 @@ function isActiveMembership(state: PeerMeshStateV1, localPeerId: string): boolea return !state.roster.roster.closed && state.roster.roster.members.includes(localPeerId); } -async function exchangeControl( +function selectRoster( + current: SignedPeerMeshRosterV1, + candidate: SignedPeerMeshRosterV1, +): SignedPeerMeshRosterV1 { + if ( + current.roster.meshId !== candidate.roster.meshId || + current.authorityPublicKey !== candidate.authorityPublicKey + ) { + throw new Error('Peer Mesh roster has the wrong authority'); + } + if (candidate.roster.revision < current.roster.revision) return current; + if (candidate.roster.revision === current.roster.revision) { + if (JSON.stringify(candidate) !== JSON.stringify(current)) { + throw new Error('Peer Mesh roster revision identifies conflicting facts'); + } + return current; + } + return candidate; +} + +function mergeRoutes( + current: readonly SignedPeerMeshRouteRecordV1[], + candidates: readonly SignedPeerMeshRouteRecordV1[], + roster: SignedPeerMeshRosterV1, + now: number, +): readonly SignedPeerMeshRouteRecordV1[] { + const routes = new Map( + current + .filter(({ route }) => route.expiresAt > now && roster.roster.members.includes(route.peerId)) + .map((route) => [route.route.peerId, route] as const), + ); + for (const candidate of candidates) { + if ( + candidate.route.expiresAt <= now || + !roster.roster.members.includes(candidate.route.peerId) + ) { + continue; + } + const existing = routes.get(candidate.route.peerId); + if (!existing || candidate.route.sequence > existing.route.sequence) { + routes.set(candidate.route.peerId, candidate); + continue; + } + if ( + candidate.route.sequence === existing.route.sequence && + JSON.stringify(candidate) !== JSON.stringify(existing) + ) { + throw new Error('Peer Mesh route sequence identifies conflicting facts'); + } + } + return Object.freeze( + [...routes.values()].sort((left, right) => left.route.peerId.localeCompare(right.route.peerId)), + ); +} + +function mergeAuthenticatedRoute( + current: readonly SignedPeerMeshRouteRecordV1[], + candidate: SignedPeerMeshRouteRecordV1, + roster: SignedPeerMeshRosterV1, + now: number, +): readonly SignedPeerMeshRouteRecordV1[] { + const existing = current.find(({ route }) => route.peerId === candidate.route.peerId); + if (existing && existing.route.sequence > candidate.route.sequence) { + return mergeRoutes(current, [], roster, now); + } + return mergeRoutes( + current.filter(({ route }) => route.peerId !== candidate.route.peerId), + [candidate], + roster, + now, + ); +} + +function pruneRoutes(state: State, now: number): State { + const routes = mergeRoutes(state.routes, [], state.roster, now); + return (routes.length === state.routes.length ? state : { ...state, routes }) as State; +} + +function routeSequences( + routes: readonly SignedPeerMeshRouteRecordV1[], + now: number, +): readonly PeerMeshRouteSequence[] { + return Object.freeze( + routes + .filter(({ route }) => route.expiresAt > now) + .map(({ route }) => Object.freeze({ peerId: route.peerId, sequence: route.sequence })) + .sort((left, right) => left.peerId.localeCompare(right.peerId)), + ); +} + +function responseRoutes( + state: PeerMeshStateV1, + knownRoutes: readonly PeerMeshRouteSequence[], + now: number, +): { readonly routes: readonly SignedPeerMeshRouteRecordV1[]; readonly more: boolean } { + const known = new Map(knownRoutes.map(({ peerId, sequence }) => [peerId, sequence])); + const missing = state.routes.filter( + ({ route }) => + route.expiresAt > now && + state.roster.roster.members.includes(route.peerId) && + route.sequence > (known.get(route.peerId) ?? 0), + ); + return Object.freeze({ + routes: Object.freeze(missing.slice(0, ROUTE_PAGE_SIZE)), + more: missing.length > ROUTE_PAGE_SIZE, + }); +} + +function sameAddresses(left: readonly string[], right: readonly string[]): boolean { + return left.length === right.length && left.every((address, index) => address === right[index]); +} + +function boundedErrorMessage(error: unknown): string { + const message = error instanceof Error ? error.message : String(error); + return message.length <= 512 ? message : `${message.slice(0, 509)}...`; +} + +async function exchangeControl( stream: RuntimeHostPeerNativeStream, - request: RedeemInvitationRequest, + request: Request, + decode: (value: unknown) => Response, signal?: AbortSignal, -): Promise { +): Promise { const timeout = AbortSignal.timeout(CONTROL_REQUEST_DEADLINE_MS); const operationSignal = signal ? AbortSignal.any([signal, timeout]) : timeout; const abort = () => stream.abort(); @@ -645,7 +1219,7 @@ async function exchangeControl( try { operationSignal.throwIfAborted(); await writeFrame(stream, request); - const response = decodeRedeemResponse(await readFrame(stream)); + const response = decode(await readFrame(stream)); operationSignal.throwIfAborted(); return response; } catch (error) { @@ -656,24 +1230,41 @@ async function exchangeControl( } } -function decodeRedeemRequest(value: unknown): RedeemInvitationRequest { +function decodeControlRequest(value: unknown): PeerMeshControlRequest { const record = recordValue(value); - if (record.kind !== 'redeem-invitation' || !hasExactKeys(record, ['kind', 'meshId', 'secret'])) { - throw new Error('Unsupported Peer Mesh control request'); + if ( + record.kind === 'redeem-invitation' && + hasExactKeys(record, ['kind', 'meshId', 'secret', 'route']) + ) { + return { + kind: 'redeem-invitation', + meshId: requiredString(record.meshId, 128), + secret: requiredString(record.secret, 64), + route: decodeSignedPeerMeshRouteRecord(record.route), + }; } - return { - kind: 'redeem-invitation', - meshId: requiredString(record.meshId, 128), - secret: requiredString(record.secret, 64), - }; + if ( + record.kind === 'sync' && + hasExactKeys(record, ['kind', 'meshId', 'roster', 'route', 'knownRoutes']) + ) { + return { + kind: 'sync', + meshId: requiredString(record.meshId, 128), + roster: decodeSignedPeerMeshRoster(record.roster), + route: decodeSignedPeerMeshRouteRecord(record.route), + knownRoutes: decodeRouteSequences(record.knownRoutes), + }; + } + throw new Error('Unsupported Peer Mesh control request'); } function decodeRedeemResponse(value: unknown): RedeemInvitationResponse { const record = recordValue(value); - if (record.kind === 'invitation-redeemed' && hasExactKeys(record, ['kind', 'roster'])) { + if (record.kind === 'invitation-redeemed' && hasExactKeys(record, ['kind', 'roster', 'routes'])) { return { kind: 'invitation-redeemed', roster: decodeSignedPeerMeshRoster(record.roster), + routes: decodeRoutePage(record.routes), }; } if ( @@ -689,6 +1280,61 @@ function decodeRedeemResponse(value: unknown): RedeemInvitationResponse { throw new Error('Invalid Peer Mesh control response'); } +function decodeSyncResponse(value: unknown): SyncPeerMeshResponse { + const record = recordValue(value); + if ( + record.kind === 'sync-result' && + hasExactKeys(record, ['kind', 'roster', 'routes', 'more']) && + typeof record.more === 'boolean' + ) { + return { + kind: 'sync-result', + roster: decodeSignedPeerMeshRoster(record.roster), + routes: decodeRoutePage(record.routes), + more: record.more, + }; + } + if ( + record.kind === 'sync-rejected' && + hasExactKeys(record, ['kind', 'reason']) && + record.reason === 'unknown' + ) { + return { kind: 'sync-rejected', reason: record.reason }; + } + throw new Error('Invalid Peer Mesh synchronization response'); +} + +function decodeRoutePage(value: unknown): readonly SignedPeerMeshRouteRecordV1[] { + if (!Array.isArray(value) || value.length > ROUTE_PAGE_SIZE) { + throw new Error('Invalid Peer Mesh route page'); + } + return Object.freeze(value.map(decodeSignedPeerMeshRouteRecord)); +} + +function decodeRouteSequences(value: unknown): readonly PeerMeshRouteSequence[] { + if (!Array.isArray(value) || value.length > PEER_MESH_MAX_MEMBERS) { + throw new Error('Invalid Peer Mesh route sequences'); + } + const sequences = value.map((entry) => { + const record = recordValue(entry); + if (!hasExactKeys(record, ['peerId', 'sequence'])) { + throw new Error('Invalid Peer Mesh route sequence'); + } + const sequence = record.sequence; + if (!Number.isSafeInteger(sequence) || (sequence as number) < 1) { + throw new Error('Invalid Peer Mesh route sequence'); + } + return Object.freeze({ + peerId: requiredString(record.peerId, 256), + sequence: sequence as number, + }); + }); + if (new Set(sequences.map(({ peerId }) => peerId)).size !== sequences.length) { + throw new Error('Duplicate Peer Mesh route sequence'); + } + return Object.freeze(sequences); +} + async function writeFrame(stream: RuntimeHostPeerNativeStream, value: unknown): Promise { const bytes = Buffer.from(`${JSON.stringify(value)}\n`); if (bytes.length > CONTROL_FRAME_MAX_BYTES) diff --git a/packages/runtime-host/src/peer-mesh/owner.ts b/packages/runtime-host/src/peer-mesh/owner.ts new file mode 100644 index 0000000000..8a88763370 --- /dev/null +++ b/packages/runtime-host/src/peer-mesh/owner.ts @@ -0,0 +1,82 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +import { createRuntimeHostPeerClient, type RuntimeHostPeerClient } from '../client/peer-client.js'; +import { openPeerMeshNode, type PeerMeshNode } from './node.js'; + +export interface RuntimeHostPeerMeshOwner { + readonly client: RuntimeHostPeerClient; + readonly mesh: PeerMeshNode; + readonly closed: Promise; + close(): Promise; +} + +export async function openRuntimeHostPeerMeshOwner(input: { + readonly nativePath: string; + readonly keyPath: string; + readonly dataRoot: string; + readonly listenAddresses?: readonly string[]; + readonly coordinationRelays?: readonly string[]; +}): Promise { + let mesh: PeerMeshNode | undefined; + const client = createRuntimeHostPeerClient({ + nativePath: input.nativePath, + keyPath: input.keyPath, + ...(input.listenAddresses ? { listenAddresses: input.listenAddresses } : {}), + ...(input.coordinationRelays ? { coordinationRelays: input.coordinationRelays } : {}), + routeResolver: { resolveRoutes: (peerId) => mesh?.resolveRoutes(peerId) }, + }); + try { + mesh = await openPeerMeshNode({ dataRoot: input.dataRoot, peer: client }); + } catch (error) { + await client.close().catch(() => undefined); + throw error; + } + const serving = mesh.serve(); + void serving.catch(() => undefined); + let closeTask: Promise | undefined; + return Object.freeze({ + client, + mesh, + closed: serving, + close: () => { + closeTask ??= closeOwner(mesh!, client, serving); + return closeTask; + }, + }); +} + +async function closeOwner( + mesh: PeerMeshNode, + client: RuntimeHostPeerClient, + serving: Promise, +): Promise { + const errors: unknown[] = []; + await mesh.close().catch((error: unknown) => { + errors.push(error); + }); + await serving.catch((error: unknown) => { + errors.push(error); + }); + await client.close().catch((error: unknown) => { + errors.push(error); + }); + if (errors.length === 1) throw errors[0]; + if (errors.length > 1) throw new AggregateError(errors, 'Unable to close peer Mesh owner'); +} diff --git a/packages/runtime-host/src/peer-mesh/store.ts b/packages/runtime-host/src/peer-mesh/store.ts index 4e01171259..e5d802dd1c 100644 --- a/packages/runtime-host/src/peer-mesh/store.ts +++ b/packages/runtime-host/src/peer-mesh/store.ts @@ -26,12 +26,15 @@ import { import { decodeAuthorityTarget, decodeSignedPeerMeshRoster, + decodeSignedPeerMeshRouteRecord, PEER_MESH_MAX_INVITATION_RECORDS, + PEER_MESH_MAX_MEMBERS, PEER_MESH_MAX_MESHES, PEER_MESH_MAX_PENDING_INVITATIONS, type PeerMeshAuthorityKeyPair, type PeerMeshAuthorityTarget, type SignedPeerMeshRosterV1, + type SignedPeerMeshRouteRecordV1, validatePeerMeshAuthorityKeyPair, } from './model.js'; @@ -55,6 +58,7 @@ interface RedeemedPeerMeshInvitation { interface PeerMeshStateBase { readonly roster: SignedPeerMeshRosterV1; + readonly routes: readonly SignedPeerMeshRouteRecordV1[]; } export interface PeerMeshAuthorityStateV1 extends PeerMeshStateBase { @@ -176,8 +180,8 @@ export function decodePeerMeshState(value: unknown, localPeerId: string): PeerMe const record = value as Record; const expectedKeys = record.role === 'authority' - ? ['role', 'roster', 'authorityPrivateKey', 'invitations'] - : ['role', 'roster', 'authority']; + ? ['role', 'roster', 'routes', 'authorityPrivateKey', 'invitations'] + : ['role', 'roster', 'routes', 'authority']; if ( Object.keys(record).length !== expectedKeys.length || expectedKeys.some((key) => !Object.hasOwn(record, key)) @@ -188,6 +192,7 @@ export function decodePeerMeshState(value: unknown, localPeerId: string): PeerMe throw new Error('Unsupported Peer Mesh state'); } const roster = decodeSignedPeerMeshRoster(record.roster); + const routes = decodeRoutes(record.routes, roster); if (record.role === 'authority') { if (!roster.roster.members.includes(localPeerId)) { throw new Error('Peer Mesh authority is not present in its roster'); @@ -200,6 +205,7 @@ export function decodePeerMeshState(value: unknown, localPeerId: string): PeerMe return Object.freeze({ role: 'authority', roster, + routes, authorityPrivateKey: privateKey, invitations: Object.freeze(decodeInvitations(record.invitations)), }); @@ -212,6 +218,7 @@ export function decodePeerMeshState(value: unknown, localPeerId: string): PeerMe role: 'replica', authority, roster, + routes, }); } @@ -289,7 +296,7 @@ async function readState( } const record = document as Record; if ( - record.version !== 1 || + (record.version !== 1 && record.version !== 2) || Object.keys(record).length !== 3 || !Object.hasOwn(record, 'localPeerId') || !Object.hasOwn(record, 'meshes') @@ -299,7 +306,8 @@ async function readState( if (boundedString(record.localPeerId, 'localPeerId', 256) !== expectedLocalPeerId) { throw new Error('Peer Mesh state belongs to a different peer identity'); } - return decodePeerMeshStates(record.meshes, expectedLocalPeerId); + const meshes = record.version === 1 ? migratePeerMeshStateV1(record.meshes) : record.meshes; + return decodePeerMeshStates(meshes, expectedLocalPeerId); } catch (error) { if (isNodeError(error, 'ENOENT')) return Object.freeze([]); throw error; @@ -311,7 +319,7 @@ async function writeState( localPeerId: string, state: readonly PeerMeshStateV1[], ): Promise { - const document = `${JSON.stringify({ version: 1, localPeerId, meshes: state }, null, 2)}\n`; + const document = `${JSON.stringify({ version: 2, localPeerId, meshes: state }, null, 2)}\n`; if (Buffer.byteLength(document) > MAX_STATE_BYTES) throw new Error('Peer Mesh state is too large'); const temporary = `${path}.tmp`; @@ -394,6 +402,32 @@ function decodeInvitations(value: unknown): PeerMeshInvitationRecord[] { return invitations; } +function decodeRoutes( + value: unknown, + roster: SignedPeerMeshRosterV1, +): readonly SignedPeerMeshRouteRecordV1[] { + if (!Array.isArray(value) || value.length > PEER_MESH_MAX_MEMBERS) { + throw new Error('Invalid Peer Mesh routes'); + } + const routes = value.map(decodeSignedPeerMeshRouteRecord); + const peerIds = routes.map(({ route }) => route.peerId); + if ( + new Set(peerIds).size !== peerIds.length || + peerIds.some((peerId) => !roster.roster.members.includes(peerId)) + ) { + throw new Error('Invalid Peer Mesh routes'); + } + return Object.freeze(routes); +} + +function migratePeerMeshStateV1(value: unknown): unknown { + if (!Array.isArray(value)) return value; + return value.map((entry) => { + if (!entry || typeof entry !== 'object' || Array.isArray(entry)) return entry; + return { ...(entry as Record), routes: [] }; + }); +} + function boundedString(value: unknown, label: string, max: number): string { if (typeof value !== 'string' || value.length === 0 || value.length > max) { throw new Error(`Invalid Peer Mesh ${label}`); diff --git a/packages/runtime-host/src/transport/peer-native.ts b/packages/runtime-host/src/transport/peer-native.ts index aab2e1abd9..0b1106264a 100644 --- a/packages/runtime-host/src/transport/peer-native.ts +++ b/packages/runtime-host/src/transport/peer-native.ts @@ -54,6 +54,11 @@ export interface RuntimeHostPeerNativeStream { abort(): void; } +export interface RuntimeHostPeerIdentityProof { + readonly publicKey: Buffer; + readonly signature: Buffer; +} + export interface RuntimeHostPeerNativeEndpoint { readonly peerId: string; readonly listenAddresses: readonly string[]; @@ -79,6 +84,17 @@ export interface RuntimeHostPeerNativeEndpoint { interface RuntimeHostPeerNativeModule { ensurePeerIdentity(keyPath: string): Promise; + signPeerIdentity( + keyPath: string, + expectedPeerId: string, + payload: Buffer, + ): Promise; + verifyPeerIdentity( + peerId: string, + publicKey: Buffer, + payload: Buffer, + signature: Buffer, + ): boolean; startPeerEndpoint(options: { readonly keyPath: string; readonly expectedPeerId?: string; @@ -87,6 +103,52 @@ interface RuntimeHostPeerNativeModule { }): unknown; } +export async function signRuntimeHostPeerIdentity(input: { + readonly nativePath: string; + readonly keyPath: string; + readonly expectedPeerId: string; + readonly payload: Buffer; +}): Promise { + try { + const proof = await loadNativeModule(input.nativePath).signPeerIdentity( + input.keyPath, + input.expectedPeerId, + input.payload, + ); + if (!isPeerIdentityProof(proof)) { + throw new RuntimeHostPeerError( + 'peer_native_failed', + 'Native peer identity signature is invalid', + ); + } + return Object.freeze({ + publicKey: Buffer.from(proof.publicKey), + signature: Buffer.from(proof.signature), + }); + } catch (error) { + throw normalizePeerError(error); + } +} + +export function verifyRuntimeHostPeerIdentity(input: { + readonly nativePath: string; + readonly peerId: string; + readonly publicKey: Buffer; + readonly payload: Buffer; + readonly signature: Buffer; +}): boolean { + try { + return loadNativeModule(input.nativePath).verifyPeerIdentity( + input.peerId, + input.publicKey, + input.payload, + input.signature, + ); + } catch (error) { + throw normalizePeerError(error); + } +} + export async function ensureRuntimeHostPeerIdentity(input: { readonly nativePath: string; readonly keyPath: string; @@ -347,11 +409,30 @@ function isPeerNativeModule(value: unknown): value is RuntimeHostPeerNativeModul value !== null && 'ensurePeerIdentity' in value && typeof value.ensurePeerIdentity === 'function' && + 'signPeerIdentity' in value && + typeof value.signPeerIdentity === 'function' && + 'verifyPeerIdentity' in value && + typeof value.verifyPeerIdentity === 'function' && 'startPeerEndpoint' in value && typeof value.startPeerEndpoint === 'function' ); } +function isPeerIdentityProof(value: unknown): value is RuntimeHostPeerIdentityProof { + return ( + typeof value === 'object' && + value !== null && + 'publicKey' in value && + Buffer.isBuffer(value.publicKey) && + value.publicKey.byteLength > 0 && + value.publicKey.byteLength <= 256 && + 'signature' in value && + Buffer.isBuffer(value.signature) && + value.signature.byteLength > 0 && + value.signature.byteLength <= 256 + ); +} + function isPeerNativeEndpoint(value: unknown): value is RuntimeHostPeerNativeEndpoint { return ( typeof value === 'object' && diff --git a/scripts/smoke-release-cli-package.mjs b/scripts/smoke-release-cli-package.mjs index d5801ff8d5..ed7690d381 100644 --- a/scripts/smoke-release-cli-package.mjs +++ b/scripts/smoke-release-cli-package.mjs @@ -229,11 +229,8 @@ async function smokeRuntimeHostPeerProtocol({ packageRoot, cliEntrypoint, root } let host; let connection; let peerClient; - let meshAuthority; - let meshMember; - let meshAuthorityPeer; - let meshMemberPeer; - let meshServing; + let meshAuthorityOwner; + let meshMemberOwner; try { delete process.env.MAKA_RUNTIME_HOST_PEER_NATIVE_PATH; delete process.env.MAKA_RUNTIME_HOST_PEER_KEY_PATH; @@ -305,25 +302,20 @@ async function smokeRuntimeHostPeerProtocol({ packageRoot, cliEntrypoint, root } const meshMemberKeyPath = join(root, 'mesh-member.key'); const meshMemberDataRoot = join(root, 'mesh-member'); - meshAuthorityPeer = client.createRuntimeHostPeerClient({ + meshAuthorityOwner = await mesh.openRuntimeHostPeerMeshOwner({ nativePath, keyPath: join(root, 'mesh-authority.key'), + dataRoot: join(root, 'mesh-authority'), listenAddresses: ['/ip4/127.0.0.1/udp/0/quic-v1'], }); - meshMemberPeer = client.createRuntimeHostPeerClient({ + meshMemberOwner = await mesh.openRuntimeHostPeerMeshOwner({ nativePath, keyPath: meshMemberKeyPath, - listenAddresses: ['/ip4/127.0.0.1/udp/0/quic-v1'], - }); - meshAuthority = await mesh.openPeerMeshNode({ - dataRoot: join(root, 'mesh-authority'), - peer: meshAuthorityPeer, - }); - meshMember = await mesh.openPeerMeshNode({ dataRoot: meshMemberDataRoot, - peer: meshMemberPeer, + listenAddresses: ['/ip4/127.0.0.1/udp/0/quic-v1'], }); - meshServing = meshAuthority.serve(); + const meshAuthority = meshAuthorityOwner.mesh; + let meshMember = meshMemberOwner.mesh; const created = await meshAuthority.create(); const joined = await meshMember.join(await meshAuthority.invite(created.roster.roster.meshId)); if (joined.roster.roster.members.length !== 2) { @@ -331,22 +323,19 @@ async function smokeRuntimeHostPeerProtocol({ packageRoot, cliEntrypoint, root } } const removed = await meshAuthority.remove( created.roster.roster.meshId, - meshMemberPeer.identity().peerId, + meshMemberOwner.client.identity().peerId, ); if (removed.roster.roster.members.length !== 1) { throw new Error('Installed Runtime Host peer Mesh did not remove the invited peer'); } - await meshMember.close(); - await meshMemberPeer.close(); - meshMemberPeer = client.createRuntimeHostPeerClient({ + await meshMemberOwner.close(); + meshMemberOwner = await mesh.openRuntimeHostPeerMeshOwner({ nativePath, keyPath: meshMemberKeyPath, - listenAddresses: ['/ip4/127.0.0.1/udp/0/quic-v1'], - }); - meshMember = await mesh.openPeerMeshNode({ dataRoot: meshMemberDataRoot, - peer: meshMemberPeer, + listenAddresses: ['/ip4/127.0.0.1/udp/0/quic-v1'], }); + meshMember = meshMemberOwner.mesh; const stale = meshMember.status()[0]; if (stale?.roster.roster.revision !== joined.roster.roster.revision) { throw new Error('Installed Runtime Host peer Mesh did not recover the last-known roster'); @@ -360,12 +349,17 @@ async function smokeRuntimeHostPeerProtocol({ packageRoot, cliEntrypoint, root } ) { throw new Error('Installed Runtime Host peer Mesh did not re-admit the removed peer'); } + await meshAuthority.remove( + created.roster.roster.meshId, + meshMemberOwner.client.identity().peerId, + ); + await meshMember.reconcile(); + if (meshMember.status().length !== 0) { + throw new Error('Installed Runtime Host peer Mesh did not propagate member removal'); + } } finally { - await meshAuthority?.close().catch(() => undefined); - await meshMember?.close().catch(() => undefined); - await meshServing?.catch(() => undefined); - await meshAuthorityPeer?.close().catch(() => undefined); - await meshMemberPeer?.close().catch(() => undefined); + await meshMemberOwner?.close().catch(() => undefined); + await meshAuthorityOwner?.close().catch(() => undefined); await connection?.close().catch(() => undefined); await peerClient?.close().catch(() => undefined); await host?.close().catch(() => undefined); From bbd431dd80980a77607a0f2d0c269d7a1fa8ac87 Mon Sep 17 00:00:00 2001 From: Wang Date: Fri, 28 Aug 2026 13:04:11 +0800 Subject: [PATCH 2/8] fix(runtime-host): share peer endpoint with mesh discovery Serve application and Mesh control traffic from one peer identity, preserve verified bootstrap routes, and keep removal convergence scoped to the removed member. Generated-by: OpenAI Codex --- .../src/__tests__/peer-listener.test.ts | 37 +++--- .../src/__tests__/peer-mesh.test.ts | 37 +++++- .../src/__tests__/peer-native.test.ts | 4 +- .../runtime-host/src/client/peer-client.ts | 94 +++++++++---- packages/runtime-host/src/peer-mesh/index.ts | 2 - packages/runtime-host/src/peer-mesh/node.ts | 125 ++++++++---------- packages/runtime-host/src/peer-mesh/owner.ts | 14 +- .../src/server/execution-service.ts | 4 +- .../runtime-host/src/server/listener-set.ts | 7 +- .../runtime-host/src/server/peer-listener.ts | 123 ++++++++--------- scripts/smoke-release-cli-package.mjs | 61 ++++----- 11 files changed, 278 insertions(+), 230 deletions(-) diff --git a/packages/runtime-host/src/__tests__/peer-listener.test.ts b/packages/runtime-host/src/__tests__/peer-listener.test.ts index f1a3a2d62a..d5eb0fc7dc 100644 --- a/packages/runtime-host/src/__tests__/peer-listener.test.ts +++ b/packages/runtime-host/src/__tests__/peer-listener.test.ts @@ -21,14 +21,12 @@ import assert from 'node:assert/strict'; import { setImmediate as waitForImmediate } from 'node:timers/promises'; import { test } from 'node:test'; import { createRuntimeHostPeerListener } from '../server/peer-listener.js'; -import type { - RuntimeHostPeerNativeEndpoint, - RuntimeHostPeerNativeStream, -} from '../transport/peer-native.js'; +import type { RuntimeHostPeerClient } from '../client/peer-client.js'; +import type { RuntimeHostPeerNativeStream } from '../transport/peer-native.js'; test('bounds and aborts pending peer authentication', async () => { const streams = Array.from({ length: 17 }, (_, index) => pendingStream(`remote-peer-${index}`)); - const listener = createRuntimeHostPeerListener(endpointWith([...streams]), {} as never, () => {}); + const listener = createRuntimeHostPeerListener(peerWith([...streams]), {} as never, () => {}); await waitForImmediate(); assert.equal(streams.filter((stream) => stream.aborted).length, 1); @@ -43,7 +41,7 @@ test('bounds and aborts pending peer authentication', async () => { test('expires a peer that does not send its credential', async (context) => { context.mock.timers.enable({ apis: ['setTimeout'] }); const stream = pendingStream(); - const listener = createRuntimeHostPeerListener(endpointWith([stream]), {} as never, () => {}); + const listener = createRuntimeHostPeerListener(peerWith([stream]), {} as never, () => {}); await waitForImmediate(); context.mock.timers.tick(5_000); @@ -55,7 +53,7 @@ test('expires a peer that does not send its credential', async (context) => { test('reports an explicit authentication rejection before closing the stream', async () => { const stream = recordingStream(Buffer.from('{"v":1,"credential":"rejected"}\n')); const listener = createRuntimeHostPeerListener( - endpointWith([stream]), + peerWith([stream]), { authenticate: () => null } as never, () => {}, ); @@ -86,7 +84,7 @@ test('rechecks peer authority at admission after the authentication response is let authentications = 0; let accepted = false; const listener = createRuntimeHostPeerListener( - endpointWith([stream]), + peerWith([stream]), { authenticate: () => (authentications++ === 0 ? { operationGrants: 'all' } : null), } as never, @@ -110,7 +108,7 @@ test('bounds active application streams from one authenticated peer', async () = const streams = Array.from({ length: 5 }, () => authenticatedPendingStream('remote-peer')); let accepted = 0; const listener = createRuntimeHostPeerListener( - endpointWith([...streams]), + peerWith([...streams]), { authenticate: () => ({ operationGrants: 'all' }) } as never, () => { accepted += 1; @@ -124,19 +122,28 @@ test('bounds active application streams from one authenticated peer', async () = await listener.cleanup(); }); -function endpointWith(streams: RuntimeHostPeerNativeStream[]): RuntimeHostPeerNativeEndpoint { +function peerWith(streams: RuntimeHostPeerNativeStream[]): RuntimeHostPeerClient { return { - peerId: 'peer', - listenAddresses: [], + identity: () => ({ peerId: 'peer', listenAddresses: [], coordinationRelays: [] }), + signIdentity: async () => { + throw new Error('not used'); + }, + verifyIdentity: () => false, connect: async () => { throw new Error('not used'); }, connectMeshControl: async () => { throw new Error('not used'); }, - cancelConnect: async () => false, - accept: async () => streams.shift() ?? null, - acceptMeshControl: async () => null, + serveApplication: async (onStream, signal) => { + for (const stream of streams) onStream(stream); + await new Promise((resolve) => + signal.addEventListener('abort', () => resolve(), { once: true }), + ); + }, + serveMeshControl: async () => { + throw new Error('not used'); + }, close: async () => undefined, }; } diff --git a/packages/runtime-host/src/__tests__/peer-mesh.test.ts b/packages/runtime-host/src/__tests__/peer-mesh.test.ts index 78f0cfc607..f7e90f7535 100644 --- a/packages/runtime-host/src/__tests__/peer-mesh.test.ts +++ b/packages/runtime-host/src/__tests__/peer-mesh.test.ts @@ -100,12 +100,22 @@ test('reconciles changed routes, propagates removal, and recovers the verified c const authorityPeer = network.create('peer-a'); const memberBPeer = network.create('peer-b'); const memberCPeer = network.create('peer-c'); + let now = Date.now(); const authority = await openPeerMeshNode({ dataRoot: join(root, 'authority'), peer: authorityPeer, + now: () => now, + }); + const memberB = await openPeerMeshNode({ + dataRoot: join(root, 'member-b'), + peer: memberBPeer, + now: () => now, + }); + let memberC = await openPeerMeshNode({ + dataRoot: join(root, 'member-c'), + peer: memberCPeer, + now: () => now, }); - const memberB = await openPeerMeshNode({ dataRoot: join(root, 'member-b'), peer: memberBPeer }); - let memberC = await openPeerMeshNode({ dataRoot: join(root, 'member-c'), peer: memberCPeer }); const serving = [authority.serve(), memberB.serve(), memberC.serve()]; try { const mesh = await authority.create(); @@ -120,17 +130,29 @@ test('reconciles changed routes, propagates removal, and recovers the verified c await memberB.reconcile(); assert.deepEqual(memberB.resolveRoutes('peer-c')?.routeHints, ['/memory/peer-c-moved']); + now += 6 * 60 * 1_000; + authorityPeer.setRouteHints(['/memory/peer-a-moved']); + await authority.reconcile(); + await memberB.reconcile(); + assert.deepEqual(memberB.resolveRoutes('peer-a')?.routeHints, ['/memory/peer-a-moved']); + await authority.remove(mesh.roster.roster.meshId, 'peer-b'); await memberC.reconcile(); + authorityPeer.setReachable(false); await memberB.reconcile(); assert.deepEqual(memberB.status(), []); assert.equal(memberB.resolveRoutes('peer-c'), undefined); assert.deepEqual(memberC.status()[0]?.roster.roster.members, ['peer-a', 'peer-c']); + assert.deepEqual(memberC.resolveRoutes('peer-a')?.routeHints, ['/memory/peer-a-moved']); await memberC.close(); await serving[2]; - memberC = await openPeerMeshNode({ dataRoot: join(root, 'member-c'), peer: memberCPeer }); - assert.deepEqual(memberC.resolveRoutes('peer-a')?.routeHints, ['/memory/peer-a']); + memberC = await openPeerMeshNode({ + dataRoot: join(root, 'member-c'), + peer: memberCPeer, + now: () => now, + }); + assert.deepEqual(memberC.resolveRoutes('peer-a')?.routeHints, ['/memory/peer-a-moved']); } finally { await Promise.allSettled([authority.close(), memberB.close(), memberC.close()]); await Promise.allSettled(serving); @@ -259,6 +281,7 @@ class MemoryPeerClient implements PeerMeshTransport { #closed = false; #failNextResponse = false; #stallNextControl = false; + #reachable = true; #routeHints: readonly string[]; constructor( @@ -280,6 +303,10 @@ class MemoryPeerClient implements PeerMeshTransport { this.#routeHints = [...routeHints]; } + setReachable(reachable: boolean): void { + this.#reachable = reachable; + } + signIdentity(payload: Buffer) { return Promise.resolve({ publicKey: Buffer.from(this.peerId), @@ -302,7 +329,7 @@ class MemoryPeerClient implements PeerMeshTransport { readonly peerId: string; }): Promise { const remote = this.peers.get(input.peerId); - if (!remote) throw new Error('Peer is unavailable'); + if (!remote || !remote.#reachable) throw new Error('Peer is unavailable'); const [localStream, remoteStream] = memoryStreamPair(this.peerId, input.peerId); if (remote.#failNextResponse) { remote.#failNextResponse = false; diff --git a/packages/runtime-host/src/__tests__/peer-native.test.ts b/packages/runtime-host/src/__tests__/peer-native.test.ts index 7cb8deaf0f..0faf6a04e1 100644 --- a/packages/runtime-host/src/__tests__/peer-native.test.ts +++ b/packages/runtime-host/src/__tests__/peer-native.test.ts @@ -108,13 +108,13 @@ module.exports = { { requestId: 1, peerId: 'pending', - routeHints: ['/memory/1', '/memory/discovered'], + routeHints: ['/memory/discovered', '/memory/1'], coordinationRelays: ['/memory/relay'], }, { requestId: 2, peerId: 'ready', - routeHints: ['/memory/1', '/memory/discovered'], + routeHints: ['/memory/discovered', '/memory/1'], coordinationRelays: ['/memory/relay'], }, ], diff --git a/packages/runtime-host/src/client/peer-client.ts b/packages/runtime-host/src/client/peer-client.ts index f160c35198..80d20278ae 100644 --- a/packages/runtime-host/src/client/peer-client.ts +++ b/packages/runtime-host/src/client/peer-client.ts @@ -60,6 +60,10 @@ export interface RuntimeHostPeerClient { input: RuntimeHostPeerConnectInput, signal?: AbortSignal, ): Promise; + serveApplication( + onStream: (stream: RuntimeHostPeerNativeStream) => void, + signal: AbortSignal, + ): Promise; serveMeshControl( onStream: (stream: RuntimeHostPeerNativeStream) => void, signal: AbortSignal, @@ -89,6 +93,7 @@ export function createRuntimeHostPeerClientFromEnvironment( export function createRuntimeHostPeerClient(input: { readonly nativePath: string; readonly keyPath: string; + readonly expectedPeerId?: string; readonly listenAddresses?: readonly string[]; readonly coordinationRelays?: readonly string[]; readonly routeResolver?: RuntimeHostPeerRouteResolver; @@ -99,19 +104,15 @@ export function createRuntimeHostPeerClient(input: { class RuntimeHostPeerClientImpl implements RuntimeHostPeerClient { readonly #nativePath: string; readonly #keyPath: string; + readonly #expectedPeerId: string | undefined; readonly #listenAddresses: readonly string[] | undefined; readonly #coordinationRelays: readonly string[] | undefined; readonly #routeResolver: RuntimeHostPeerRouteResolver | undefined; #endpoint: RuntimeHostPeerNativeEndpoint | undefined; #draining: Promise | undefined; #meshDraining: Promise | undefined; - #meshConsumer: - | { - readonly onStream: (stream: RuntimeHostPeerNativeStream) => void; - readonly resolve: () => void; - readonly reject: (error: Error) => void; - } - | undefined; + #applicationConsumer: InboundConsumer | undefined; + #meshConsumer: InboundConsumer | undefined; #terminalError: Error | undefined; #nextRequestId = 1; #closed = false; @@ -120,12 +121,14 @@ class RuntimeHostPeerClientImpl implements RuntimeHostPeerClient { constructor(input: { readonly nativePath: string; readonly keyPath: string; + readonly expectedPeerId?: string; readonly listenAddresses?: readonly string[]; readonly coordinationRelays?: readonly string[]; readonly routeResolver?: RuntimeHostPeerRouteResolver; }) { this.#nativePath = input.nativePath; this.#keyPath = input.keyPath; + this.#expectedPeerId = input.expectedPeerId; this.#listenAddresses = input.listenAddresses; this.#coordinationRelays = input.coordinationRelays; this.#routeResolver = input.routeResolver; @@ -178,13 +181,34 @@ class RuntimeHostPeerClientImpl implements RuntimeHostPeerClient { return this.#connect(input, signal, 'mesh-control'); } + serveApplication( + onStream: (stream: RuntimeHostPeerNativeStream) => void, + signal: AbortSignal, + ): Promise { + return this.#serve('application', onStream, signal); + } + serveMeshControl( onStream: (stream: RuntimeHostPeerNativeStream) => void, signal: AbortSignal, + ): Promise { + return this.#serve('mesh', onStream, signal); + } + + #serve( + kind: 'application' | 'mesh', + onStream: (stream: RuntimeHostPeerNativeStream) => void, + signal: AbortSignal, ): Promise { signal.throwIfAborted(); - if (this.#meshConsumer) { - return Promise.reject(new Error('Runtime Host peer Mesh control is already being served')); + if (kind === 'application' ? this.#applicationConsumer : this.#meshConsumer) { + return Promise.reject( + new Error( + kind === 'application' + ? 'Runtime Host peer application traffic is already being served' + : 'Runtime Host peer Mesh control is already being served', + ), + ); } this.#requireEndpoint(); let resolve!: () => void; @@ -194,17 +218,26 @@ class RuntimeHostPeerClientImpl implements RuntimeHostPeerClient { reject = rejectPromise; }); const consumer = { onStream, resolve, reject }; - this.#meshConsumer = consumer; + if (kind === 'application') this.#applicationConsumer = consumer; + else this.#meshConsumer = consumer; const stop = () => { - if (this.#meshConsumer !== consumer) return; - this.#meshConsumer = undefined; + if (kind === 'application') { + if (this.#applicationConsumer !== consumer) return; + this.#applicationConsumer = undefined; + } else { + if (this.#meshConsumer !== consumer) return; + this.#meshConsumer = undefined; + } resolve(); }; signal.addEventListener('abort', stop, { once: true }); if (signal.aborted) stop(); return serving.finally(() => { signal.removeEventListener('abort', stop); - if (this.#meshConsumer === consumer) this.#meshConsumer = undefined; + if (kind === 'application' && this.#applicationConsumer === consumer) { + this.#applicationConsumer = undefined; + } + if (kind === 'mesh' && this.#meshConsumer === consumer) this.#meshConsumer = undefined; }); } @@ -219,10 +252,10 @@ class RuntimeHostPeerClientImpl implements RuntimeHostPeerClient { const discovered = this.#routeResolver?.resolveRoutes(input.peerId); const connection = endpoint[kind === 'application' ? 'connect' : 'connectMeshControl']({ ...input, - routeHints: mergeAddresses(input.routeHints, discovered?.routeHints), + routeHints: mergeAddresses(discovered?.routeHints ?? [], input.routeHints), coordinationRelays: mergeAddresses( - input.coordinationRelays ?? [], - discovered?.coordinationRelays, + discovered?.coordinationRelays ?? [], + input.coordinationRelays, ), requestId, }); @@ -267,6 +300,7 @@ class RuntimeHostPeerClientImpl implements RuntimeHostPeerClient { const endpoint = startRuntimeHostPeerEndpoint({ nativePath: this.#nativePath, keyPath: this.#keyPath, + ...(this.#expectedPeerId ? { expectedPeerId: this.#expectedPeerId } : {}), ...(this.#listenAddresses ? { listenAddresses: this.#listenAddresses } : {}), ...(this.#coordinationRelays ? { coordinationRelays: this.#coordinationRelays } : {}), }); @@ -281,17 +315,20 @@ class RuntimeHostPeerClientImpl implements RuntimeHostPeerClient { while (true) { const stream = await endpoint.accept(); if (!stream) { - if (!this.#closed) { - this.#terminalError = new Error('Runtime Host peer networking stopped unexpectedly'); - } + const error = new Error('Runtime Host peer networking stopped unexpectedly'); + if (!this.#closed) this.#terminalError = error; + this.#finishConsumer('application', this.#closed ? undefined : error); return; } - stream.abort(); + const consumer = this.#applicationConsumer; + if (consumer) consumer.onStream(stream); + else stream.abort(); } } catch (error) { // Connection attempts and streams expose a terminal native failure to // their existing reconnect owners. This owner never replaces its Swarm. this.#terminalError = error instanceof Error ? error : new Error(String(error)); + this.#finishConsumer('application', this.#closed ? undefined : this.#terminalError); } } @@ -302,7 +339,7 @@ class RuntimeHostPeerClientImpl implements RuntimeHostPeerClient { if (!stream) { const error = new Error('Runtime Host peer networking stopped unexpectedly'); if (!this.#closed) this.#terminalError = error; - this.#finishMeshConsumer(this.#closed ? undefined : error); + this.#finishConsumer('mesh', this.#closed ? undefined : error); return; } const consumer = this.#meshConsumer; @@ -312,14 +349,15 @@ class RuntimeHostPeerClientImpl implements RuntimeHostPeerClient { } catch (error) { const failure = error instanceof Error ? error : new Error(String(error)); if (!this.#closed) this.#terminalError = failure; - this.#finishMeshConsumer(this.#closed ? undefined : failure); + this.#finishConsumer('mesh', this.#closed ? undefined : failure); } } - #finishMeshConsumer(error?: Error): void { - const consumer = this.#meshConsumer; + #finishConsumer(kind: 'application' | 'mesh', error?: Error): void { + const consumer = kind === 'application' ? this.#applicationConsumer : this.#meshConsumer; if (!consumer) return; - this.#meshConsumer = undefined; + if (kind === 'application') this.#applicationConsumer = undefined; + else this.#meshConsumer = undefined; if (error) consumer.reject(error); else consumer.resolve(); } @@ -348,6 +386,12 @@ class RuntimeHostPeerClientImpl implements RuntimeHostPeerClient { } } +interface InboundConsumer { + readonly onStream: (stream: RuntimeHostPeerNativeStream) => void; + readonly resolve: () => void; + readonly reject: (error: Error) => void; +} + function mergeAddresses( primary: readonly string[], secondary: readonly string[] | undefined, diff --git a/packages/runtime-host/src/peer-mesh/index.ts b/packages/runtime-host/src/peer-mesh/index.ts index 63c8e60273..bdf41194d1 100644 --- a/packages/runtime-host/src/peer-mesh/index.ts +++ b/packages/runtime-host/src/peer-mesh/index.ts @@ -29,8 +29,6 @@ export { export { openPeerMeshNode, type PeerMeshNode, - type PeerMeshReconcileResult, - type PeerMeshResolvedRoutes, type PeerMeshStatus, type PeerMeshTransport, } from './node.js'; diff --git a/packages/runtime-host/src/peer-mesh/node.ts b/packages/runtime-host/src/peer-mesh/node.ts index e1cd238625..ee3851ae45 100644 --- a/packages/runtime-host/src/peer-mesh/node.ts +++ b/packages/runtime-host/src/peer-mesh/node.ts @@ -116,28 +116,17 @@ export interface PeerMeshNode { join(invitation: PeerMeshInvitationV1, signal?: AbortSignal): Promise; remove(meshId: string, peerId: string): Promise; closeMesh(meshId: string): Promise; - resolveRoutes(peerId: string): PeerMeshResolvedRoutes | undefined; - reconcile(signal?: AbortSignal): Promise; + resolveRoutes(peerId: string): + | { + readonly routeHints: readonly string[]; + readonly coordinationRelays: readonly string[]; + } + | undefined; + reconcile(signal?: AbortSignal): Promise; serve(): Promise; close(): Promise; } -export interface PeerMeshResolvedRoutes { - readonly routeHints: readonly string[]; - readonly coordinationRelays: readonly string[]; - readonly expiresAt: number; -} - -export interface PeerMeshReconcileResult { - readonly attempted: number; - readonly synchronized: number; - readonly failures: readonly { - readonly meshId: string; - readonly peerId: string; - readonly message: string; - }[]; -} - export interface PeerMeshStatus { readonly role: 'authority' | 'member'; readonly localPeerId: string; @@ -211,10 +200,6 @@ class PeerMeshNodeImpl implements PeerMeshNode { for (const state of this.#store.read()) { for (const route of state.routes) this.#assertRouteSignature(route); } - await this.#store.mutate((current) => ({ - state: current.map((state) => pruneRoutes(state, this.#now())), - result: undefined, - })); } status(): readonly PeerMeshStatus[] { @@ -420,7 +405,7 @@ class PeerMeshNodeImpl implements PeerMeshNode { })); } - resolveRoutes(peerId: string): PeerMeshResolvedRoutes | undefined { + resolveRoutes(peerId: string) { this.#assertOpen(); const now = this.#now(); const route = this.#store @@ -433,11 +418,10 @@ class PeerMeshNodeImpl implements PeerMeshNode { return Object.freeze({ routeHints: route.routeHints, coordinationRelays: route.coordinationRelays, - expiresAt: route.expiresAt, }); } - reconcile(signal?: AbortSignal): Promise { + reconcile(signal?: AbortSignal): Promise { if (this.#lifetime.signal.aborted) return Promise.reject(new Error('Peer Mesh node is closed')); const task = this.#reconcileTail.then(() => this.#reconcile(signal)); this.#reconcileTail = task.then( @@ -450,16 +434,22 @@ class PeerMeshNodeImpl implements PeerMeshNode { async serve(): Promise { if (this.#lifetime.signal.aborted) throw new Error('Peer Mesh node is closed'); if (this.#serveTask) throw new Error('Peer Mesh node is already serving'); - const inbound = this.#peer.serveMeshControl( - (stream) => this.#acceptIncoming(stream), - this.#lifetime.signal, - ); - const serving = Promise.all([inbound, this.#runReconciliation()]).then(() => undefined); + const serveLifetime = new AbortController(); + const signal = AbortSignal.any([this.#lifetime.signal, serveLifetime.signal]); + const inbound = this.#peer.serveMeshControl((stream) => this.#acceptIncoming(stream), signal); + const reconciliation = this.#runReconciliation(signal); + const serving = (async () => { + try { + await inbound; + if (!signal.aborted) throw new Error('Peer Mesh control transport stopped unexpectedly'); + } finally { + serveLifetime.abort(); + await reconciliation; + } + })(); this.#serveTask = serving; try { await serving; - if (!this.#lifetime.signal.aborted) - throw new Error('Peer Mesh control transport stopped unexpectedly'); } finally { if (this.#serveTask === serving) { this.#serveTask = undefined; @@ -481,16 +471,14 @@ class PeerMeshNodeImpl implements PeerMeshNode { return this.#store.close(); } - async #runReconciliation(): Promise { - while (!this.#lifetime.signal.aborted) { - await this.reconcile(this.#lifetime.signal).catch(() => undefined); - await delay(RECONCILE_INTERVAL_MS, undefined, { signal: this.#lifetime.signal }).catch( - () => undefined, - ); + async #runReconciliation(signal: AbortSignal): Promise { + while (!signal.aborted) { + await this.reconcile(signal).catch(() => undefined); + await delay(RECONCILE_INTERVAL_MS, undefined, { signal }).catch(() => undefined); } } - async #reconcile(signal?: AbortSignal): Promise { + async #reconcile(signal?: AbortSignal): Promise { const operationSignal = signal ? AbortSignal.any([signal, this.#lifetime.signal]) : this.#lifetime.signal; @@ -507,7 +495,6 @@ class PeerMeshNodeImpl implements PeerMeshNode { const route = signed.route; if ( route.peerId !== identity.peerId && - route.expiresAt > this.#now() && state.roster.roster.members.includes(route.peerId) ) { targets.set(`${state.roster.roster.meshId}\0${route.peerId}`, { @@ -522,27 +509,14 @@ class PeerMeshNodeImpl implements PeerMeshNode { targets.set(key, { meshId: state.roster.roster.meshId, target: state.authority }); } } - const failures: Array<{ meshId: string; peerId: string; message: string }> = []; - let synchronized = 0; for (const { meshId, target } of targets.values()) { operationSignal.throwIfAborted(); try { await this.#syncPeer(meshId, target, operationSignal); - synchronized += 1; - } catch (error) { + } catch { if (operationSignal.aborted) operationSignal.throwIfAborted(); - failures.push({ - meshId, - peerId: target.peerId, - message: boundedErrorMessage(error), - }); } } - return Object.freeze({ - attempted: targets.size, - synchronized, - failures: Object.freeze(failures.map((failure) => Object.freeze(failure))), - }); } async #syncPeer( @@ -613,9 +587,21 @@ class PeerMeshNodeImpl implements PeerMeshNode { sameAddresses(existing.route.routeHints, identity.listenAddresses) && sameAddresses(existing.route.coordinationRelays, identity.coordinationRelays) ) { - return existing; + if ( + active.every((state) => + state.routes.some((candidate) => sameSignedRoute(candidate, existing)), + ) + ) { + return existing; + } } - const route = await this.#signLocalRoute((existing?.route.sequence ?? 0) + 1); + const route = + existing && + existing.route.expiresAt > now + ROUTE_REFRESH_LEAD_MS && + sameAddresses(existing.route.routeHints, identity.listenAddresses) && + sameAddresses(existing.route.coordinationRelays, identity.coordinationRelays) + ? existing + : await this.#signLocalRoute((existing?.route.sequence ?? 0) + 1); await this.#store.mutate((states) => ({ state: states.map((state) => isActiveMembership(state, identity.peerId) @@ -970,16 +956,16 @@ class PeerMeshNodeImpl implements PeerMeshNode { } const roster = selectRoster(state.roster, incomingRoster); const localPeerId = this.#peer.identity().peerId; - const bothMembers = - !roster.roster.closed && - roster.roster.members.includes(localPeerId) && - roster.roster.members.includes(remotePeerId); + const localMember = !roster.roster.closed && roster.roster.members.includes(localPeerId); + const remoteMember = !roster.roster.closed && roster.roster.members.includes(remotePeerId); const updated = { ...state, roster, - routes: bothMembers ? mergeRoutes(state.routes, [remoteRoute], roster, this.#now()) : [], + routes: localMember + ? mergeRoutes(state.routes, remoteMember ? [remoteRoute] : [], roster, this.#now()) + : [], }; - if (!bothMembers) { + if (!localMember || !remoteMember) { return { state: replaceMesh(current, updated), result: { @@ -1116,7 +1102,7 @@ function mergeRoutes( ): readonly SignedPeerMeshRouteRecordV1[] { const routes = new Map( current - .filter(({ route }) => route.expiresAt > now && roster.roster.members.includes(route.peerId)) + .filter(({ route }) => roster.roster.members.includes(route.peerId)) .map((route) => [route.route.peerId, route] as const), ); for (const candidate of candidates) { @@ -1161,11 +1147,6 @@ function mergeAuthenticatedRoute( ); } -function pruneRoutes(state: State, now: number): State { - const routes = mergeRoutes(state.routes, [], state.roster, now); - return (routes.length === state.routes.length ? state : { ...state, routes }) as State; -} - function routeSequences( routes: readonly SignedPeerMeshRouteRecordV1[], now: number, @@ -1200,9 +1181,11 @@ function sameAddresses(left: readonly string[], right: readonly string[]): boole return left.length === right.length && left.every((address, index) => address === right[index]); } -function boundedErrorMessage(error: unknown): string { - const message = error instanceof Error ? error.message : String(error); - return message.length <= 512 ? message : `${message.slice(0, 509)}...`; +function sameSignedRoute( + left: SignedPeerMeshRouteRecordV1, + right: SignedPeerMeshRouteRecordV1, +): boolean { + return JSON.stringify(left) === JSON.stringify(right); } async function exchangeControl( diff --git a/packages/runtime-host/src/peer-mesh/owner.ts b/packages/runtime-host/src/peer-mesh/owner.ts index 8a88763370..b0ac235b91 100644 --- a/packages/runtime-host/src/peer-mesh/owner.ts +++ b/packages/runtime-host/src/peer-mesh/owner.ts @@ -49,16 +49,18 @@ export async function openRuntimeHostPeerMeshOwner(input: { throw error; } const serving = mesh.serve(); - void serving.catch(() => undefined); let closeTask: Promise | undefined; + const close = () => { + closeTask ??= closeOwner(mesh!, client, serving); + return closeTask; + }; + const closed = serving.then(close, close); + void closed.catch(() => undefined); return Object.freeze({ client, mesh, - closed: serving, - close: () => { - closeTask ??= closeOwner(mesh!, client, serving); - return closeTask; - }, + closed, + close, }); } diff --git a/packages/runtime-host/src/server/execution-service.ts b/packages/runtime-host/src/server/execution-service.ts index 23b578977b..9a15c41eeb 100644 --- a/packages/runtime-host/src/server/execution-service.ts +++ b/packages/runtime-host/src/server/execution-service.ts @@ -34,7 +34,7 @@ import { openRuntimeHostAccessAuthority } from './access-authority.js'; import { startRuntimeHostAuthenticatedListenerSet } from './listener-set.js'; import type { StartRuntimeHostWebSocketListenerOptions } from './websocket-listener.js'; import type { PublishedProjectDirectoryRoot } from './project-directory-authority.js'; -import type { StartRuntimeHostPeerListenerOptions } from './peer-listener.js'; +import type { RuntimeHostPeerListenerEndpointOptions } from './peer-listener.js'; export interface ExecutionRuntimeHostServiceOptions { readonly rootPath: string; @@ -46,7 +46,7 @@ export interface ExecutionRuntimeHostServiceOptions { StartRuntimeHostWebSocketListenerOptions, 'accessAuthority' | 'accept' | 'isReady' >; - readonly peer?: Omit; + readonly peer?: RuntimeHostPeerListenerEndpointOptions; } export interface ExecutionRuntimeHostServiceDependencies diff --git a/packages/runtime-host/src/server/listener-set.ts b/packages/runtime-host/src/server/listener-set.ts index d3898ef90b..c26508e581 100644 --- a/packages/runtime-host/src/server/listener-set.ts +++ b/packages/runtime-host/src/server/listener-set.ts @@ -20,9 +20,10 @@ import { startLocalIpcRuntimeHostListener } from './local-ipc-listener.js'; import type { RuntimeHostMessageTransport } from '../transport/message-transport.js'; import type { RuntimeHostConnectionAuthority } from './connection-authority.js'; +import type { RuntimeHostAccessAuthority } from './access-authority.js'; import { startRuntimeHostPeerListener, - type StartRuntimeHostPeerListenerOptions, + type RuntimeHostPeerListenerEndpointOptions, } from './peer-listener.js'; import { startRuntimeHostWebSocketListener, @@ -85,7 +86,9 @@ export async function startRuntimeHostAuthenticatedListenerSet( input: RuntimeHostListenerSetFactoryInput, options: { readonly websocket?: Omit; - readonly peer?: Omit; + readonly peer?: RuntimeHostPeerListenerEndpointOptions & { + readonly accessAuthority: RuntimeHostAccessAuthority; + }; }, ): Promise { const local = await startLocalIpcRuntimeHostListener(input); diff --git a/packages/runtime-host/src/server/peer-listener.ts b/packages/runtime-host/src/server/peer-listener.ts index c509157bd8..5e538bbabf 100644 --- a/packages/runtime-host/src/server/peer-listener.ts +++ b/packages/runtime-host/src/server/peer-listener.ts @@ -22,11 +22,10 @@ import { readRuntimeHostPeerAuthentication, RUNTIME_HOST_PEER_AUTHENTICATION_TIMEOUT_MS, RuntimeHostPeerByteStream, - startRuntimeHostPeerEndpoint, writeRuntimeHostPeerAuthenticationResult, - type RuntimeHostPeerNativeEndpoint, type RuntimeHostPeerNativeStream, } from '../transport/peer-native.js'; +import { createRuntimeHostPeerClient, type RuntimeHostPeerClient } from '../client/peer-client.js'; import type { RuntimeHostAccessAuthority } from './access-authority.js'; import type { RuntimeHostListenerConnection, @@ -37,29 +36,45 @@ const MAX_PENDING_AUTHENTICATIONS = 16; const MAX_ACTIVE_STREAMS = 64; const MAX_ACTIVE_STREAMS_PER_PEER = 4; -export interface StartRuntimeHostPeerListenerOptions { +export interface RuntimeHostPeerListenerConfiguration { readonly nativePath: string; readonly keyPath: string; readonly expectedPeerId?: string; readonly listenAddresses?: readonly string[]; readonly coordinationRelays?: readonly string[]; +} + +export type RuntimeHostPeerListenerEndpointOptions = + | RuntimeHostPeerListenerConfiguration + | { readonly client: RuntimeHostPeerClient }; + +export type StartRuntimeHostPeerListenerOptions = RuntimeHostPeerListenerEndpointOptions & { readonly accessAuthority: RuntimeHostAccessAuthority; readonly accept: (connection: RuntimeHostListenerConnection) => void; -} +}; export function startRuntimeHostPeerListener( options: StartRuntimeHostPeerListenerOptions, ): RuntimeHostPeerListenerContract { - const endpoint = startRuntimeHostPeerEndpoint(options); - return createRuntimeHostPeerListener(endpoint, options.accessAuthority, options.accept); + if ('client' in options) { + return createRuntimeHostPeerListener( + options.client, + options.accessAuthority, + options.accept, + false, + ); + } + const client = createRuntimeHostPeerClient(options); + return createRuntimeHostPeerListener(client, options.accessAuthority, options.accept, true); } export function createRuntimeHostPeerListener( - endpoint: RuntimeHostPeerNativeEndpoint, + client: RuntimeHostPeerClient, accessAuthority: RuntimeHostAccessAuthority, accept: (connection: RuntimeHostListenerConnection) => void, + ownsClient = false, ): RuntimeHostPeerListenerContract { - return new RuntimeHostPeerListener(endpoint, accessAuthority, accept); + return new RuntimeHostPeerListener(client, accessAuthority, accept, ownsClient); } class RuntimeHostPeerListener implements RuntimeHostPeerListenerContract { @@ -67,36 +82,40 @@ class RuntimeHostPeerListener implements RuntimeHostPeerListenerContract { readonly endpoint: string; readonly peerId: string; readonly listenAddresses: readonly string[]; - readonly #endpoint: RuntimeHostPeerNativeEndpoint; + readonly #client: RuntimeHostPeerClient; + readonly #ownsClient: boolean; readonly #accessAuthority: RuntimeHostAccessAuthority; readonly #accept: (connection: RuntimeHostListenerConnection) => void; readonly #transports = new Set(); readonly #streams = new Set(); readonly #authentications = new Map>(); - readonly #acceptTasks: readonly Promise[]; + readonly #serving: Promise; + readonly #serveLifetime = new AbortController(); #acceptFailure: unknown; #admitting = true; #closeAdmissionTask: Promise | undefined; #cleanupTask: Promise | undefined; constructor( - endpoint: RuntimeHostPeerNativeEndpoint, + client: RuntimeHostPeerClient, accessAuthority: RuntimeHostAccessAuthority, accept: (connection: RuntimeHostListenerConnection) => void, + ownsClient: boolean, ) { - this.endpoint = endpoint.peerId; - this.peerId = endpoint.peerId; - this.listenAddresses = Object.freeze([...endpoint.listenAddresses]); - this.#endpoint = endpoint; + const identity = client.identity(); + this.endpoint = identity.peerId; + this.peerId = identity.peerId; + this.listenAddresses = Object.freeze([...identity.listenAddresses]); + this.#client = client; + this.#ownsClient = ownsClient; this.#accessAuthority = accessAuthority; this.#accept = accept; const captureFailure = (error: unknown) => { this.#acceptFailure ??= error; }; - this.#acceptTasks = [ - this.#acceptStreams().catch(captureFailure), - this.#discardMeshStreams().catch(captureFailure), - ]; + this.#serving = client + .serveApplication((stream) => this.#acceptStream(stream), this.#serveLifetime.signal) + .catch(captureFailure); } closeAdmission(): Promise { @@ -112,59 +131,33 @@ class RuntimeHostPeerListener implements RuntimeHostPeerListenerContract { this.#cleanupTask ??= (async () => { await this.closeAdmission(); for (const transport of this.#transports) transport.abort(); - await this.#endpoint.close(); - await Promise.all(this.#acceptTasks); + this.#serveLifetime.abort(); + await this.#serving; + if (this.#ownsClient) await this.#client.close(); if (this.#acceptFailure) throw this.#acceptFailure; })(); return this.#cleanupTask; } - async #acceptStreams(): Promise { - while (true) { - let stream: RuntimeHostPeerNativeStream | null; - try { - stream = await this.#endpoint.accept(); - } catch (error) { - if (this.#cleanupTask) return; - throw error; - } - if (!stream) return; - if (!this.#admitting) { - stream.abort(); - continue; - } - if (this.#authentications.size >= MAX_PENDING_AUTHENTICATIONS) { - stream.abort(); - continue; - } - let peerStreams = 0; - for (const admitted of this.#streams) { - if (admitted.peerId === stream.peerId) peerStreams += 1; - } - if (this.#streams.size >= MAX_ACTIVE_STREAMS || peerStreams >= MAX_ACTIVE_STREAMS_PER_PEER) { - stream.abort(); - continue; - } - this.#streams.add(stream); - const task = this.#authenticateAndAccept(stream).finally(() => { - this.#authentications.delete(stream); - }); - this.#authentications.set(stream, task); - void task; + #acceptStream(stream: RuntimeHostPeerNativeStream): void { + if (!this.#admitting || this.#authentications.size >= MAX_PENDING_AUTHENTICATIONS) { + stream.abort(); + return; } - } - - async #discardMeshStreams(): Promise { - while (true) { - try { - const stream = await this.#endpoint.acceptMeshControl(); - if (!stream) return; - stream.abort(); - } catch (error) { - if (this.#cleanupTask) return; - throw error; - } + let peerStreams = 0; + for (const admitted of this.#streams) { + if (admitted.peerId === stream.peerId) peerStreams += 1; + } + if (this.#streams.size >= MAX_ACTIVE_STREAMS || peerStreams >= MAX_ACTIVE_STREAMS_PER_PEER) { + stream.abort(); + return; } + this.#streams.add(stream); + const task = this.#authenticateAndAccept(stream).finally(() => { + this.#authentications.delete(stream); + }); + this.#authentications.set(stream, task); + void task; } async #authenticateAndAccept(stream: RuntimeHostPeerNativeStream): Promise { diff --git a/scripts/smoke-release-cli-package.mjs b/scripts/smoke-release-cli-package.mjs index ed7690d381..35eb2ee621 100644 --- a/scripts/smoke-release-cli-package.mjs +++ b/scripts/smoke-release-cli-package.mjs @@ -228,7 +228,6 @@ async function smokeRuntimeHostPeerProtocol({ packageRoot, cliEntrypoint, root } const previousKeyPath = process.env.MAKA_RUNTIME_HOST_PEER_KEY_PATH; let host; let connection; - let peerClient; let meshAuthorityOwner; let meshMemberOwner; try { @@ -251,14 +250,15 @@ async function smokeRuntimeHostPeerProtocol({ packageRoot, cliEntrypoint, root } } catch (error) { if (!String(error).includes('peer_identity_mismatch')) throw error; } + meshAuthorityOwner = await mesh.openRuntimeHostPeerMeshOwner({ + nativePath, + keyPath: hostKeyPath, + dataRoot: join(root, 'mesh-authority'), + listenAddresses: ['/ip4/127.0.0.1/udp/0/quic-v1'], + }); host = await server.startExecutionRuntimeHostService({ rootPath: hostRoot, - peer: { - nativePath, - keyPath: hostKeyPath, - expectedPeerId: peerId, - listenAddresses: ['/ip4/127.0.0.1/udp/0/quic-v1'], - }, + peer: { client: meshAuthorityOwner.client }, }); const listener = host.peerListeners[0]; if (!listener || listener.peerId !== peerId || listener.listenAddresses.length === 0) { @@ -274,7 +274,21 @@ async function smokeRuntimeHostPeerProtocol({ packageRoot, cliEntrypoint, root } canUseHostPaths: false, preset: 'terminal-client', }); - peerClient = client.createRuntimeHostPeerClientFromEnvironment(process.env); + const meshMemberKeyPath = join(root, 'mesh-member.key'); + const meshMemberDataRoot = join(root, 'mesh-member'); + meshMemberOwner = await mesh.openRuntimeHostPeerMeshOwner({ + nativePath, + keyPath: meshMemberKeyPath, + dataRoot: meshMemberDataRoot, + listenAddresses: ['/ip4/127.0.0.1/udp/0/quic-v1'], + }); + const meshAuthority = meshAuthorityOwner.mesh; + let meshMember = meshMemberOwner.mesh; + const created = await meshAuthority.create(); + const joined = await meshMember.join(await meshAuthority.invite(created.roster.roster.meshId)); + if (joined.roster.roster.members.length !== 2) { + throw new Error('Installed Runtime Host peer Mesh did not admit the invited peer'); + } connection = await client.connectRemoteRuntimeHostProfile({ profile: { id: 'release-smoke-peer', @@ -284,13 +298,13 @@ async function smokeRuntimeHostPeerProtocol({ packageRoot, cliEntrypoint, root } transport: { kind: 'libp2p-direct', peerId, - routeHints: listener.listenAddresses, + routeHints: ['/ip4/127.0.0.1/udp/1/quic-v1'], coordinationRelays: [], }, }, credential: issued.credential, clientInstanceId: 'release-smoke-peer-client', - peerClient, + peerClient: meshMemberOwner.client, connectTimeoutMs: 10_000, handshakeTimeoutMs: 10_000, readyTimeoutMs: 10_000, @@ -299,28 +313,6 @@ async function smokeRuntimeHostPeerProtocol({ packageRoot, cliEntrypoint, root } if (status.state !== 'ready') { throw new Error(`Installed Runtime Host direct-peer status is ${status.state}`); } - - const meshMemberKeyPath = join(root, 'mesh-member.key'); - const meshMemberDataRoot = join(root, 'mesh-member'); - meshAuthorityOwner = await mesh.openRuntimeHostPeerMeshOwner({ - nativePath, - keyPath: join(root, 'mesh-authority.key'), - dataRoot: join(root, 'mesh-authority'), - listenAddresses: ['/ip4/127.0.0.1/udp/0/quic-v1'], - }); - meshMemberOwner = await mesh.openRuntimeHostPeerMeshOwner({ - nativePath, - keyPath: meshMemberKeyPath, - dataRoot: meshMemberDataRoot, - listenAddresses: ['/ip4/127.0.0.1/udp/0/quic-v1'], - }); - const meshAuthority = meshAuthorityOwner.mesh; - let meshMember = meshMemberOwner.mesh; - const created = await meshAuthority.create(); - const joined = await meshMember.join(await meshAuthority.invite(created.roster.roster.meshId)); - if (joined.roster.roster.members.length !== 2) { - throw new Error('Installed Runtime Host peer Mesh did not admit the invited peer'); - } const removed = await meshAuthority.remove( created.roster.roster.meshId, meshMemberOwner.client.identity().peerId, @@ -358,11 +350,10 @@ async function smokeRuntimeHostPeerProtocol({ packageRoot, cliEntrypoint, root } throw new Error('Installed Runtime Host peer Mesh did not propagate member removal'); } } finally { - await meshMemberOwner?.close().catch(() => undefined); - await meshAuthorityOwner?.close().catch(() => undefined); await connection?.close().catch(() => undefined); - await peerClient?.close().catch(() => undefined); await host?.close().catch(() => undefined); + await meshMemberOwner?.close().catch(() => undefined); + await meshAuthorityOwner?.close().catch(() => undefined); restoreEnvironment('MAKA_RUNTIME_HOST_PEER_NATIVE_PATH', previousNativePath); restoreEnvironment('MAKA_RUNTIME_HOST_PEER_KEY_PATH', previousKeyPath); } From 9dde5e05b9c1166b6784b65e931dcf2d79b7bdb1 Mon Sep 17 00:00:00 2001 From: Wang Date: Fri, 28 Aug 2026 13:22:13 +0800 Subject: [PATCH 3/8] refactor(runtime-host): centralize peer route cache Store each signed peer route once per node, filter it through active Mesh rosters, and allow expired route sequences to restart without weakening fresh-route rollback protection. Generated-by: OpenAI Codex --- .../src/__tests__/peer-mesh.test.ts | 15 ++ .../runtime-host/src/client/peer-client.ts | 2 +- packages/runtime-host/src/peer-mesh/index.ts | 2 - packages/runtime-host/src/peer-mesh/node.ts | 248 ++++++++++-------- packages/runtime-host/src/peer-mesh/owner.ts | 2 + packages/runtime-host/src/peer-mesh/store.ts | 109 +++++--- scripts/smoke-release-cli-package.mjs | 1 + 7 files changed, 224 insertions(+), 155 deletions(-) diff --git a/packages/runtime-host/src/__tests__/peer-mesh.test.ts b/packages/runtime-host/src/__tests__/peer-mesh.test.ts index f7e90f7535..729e7ab16e 100644 --- a/packages/runtime-host/src/__tests__/peer-mesh.test.ts +++ b/packages/runtime-host/src/__tests__/peer-mesh.test.ts @@ -130,12 +130,27 @@ test('reconciles changed routes, propagates removal, and recovers the verified c await memberB.reconcile(); assert.deepEqual(memberB.resolveRoutes('peer-c')?.routeHints, ['/memory/peer-c-moved']); + await memberC.close(); + await serving[2]; + await rm(join(root, 'member-c'), { recursive: true, force: true }); now += 6 * 60 * 1_000; authorityPeer.setRouteHints(['/memory/peer-a-moved']); await authority.reconcile(); await memberB.reconcile(); assert.deepEqual(memberB.resolveRoutes('peer-a')?.routeHints, ['/memory/peer-a-moved']); + memberCPeer.setRouteHints(['/memory/peer-c-rejoined']); + memberC = await openPeerMeshNode({ + dataRoot: join(root, 'member-c'), + peer: memberCPeer, + now: () => now, + }); + serving[2] = memberC.serve(); + await memberC.join(await authority.invite(mesh.roster.roster.meshId)); + assert.deepEqual(authority.resolveRoutes('peer-c')?.routeHints, ['/memory/peer-c-rejoined']); + await memberB.reconcile(); + assert.deepEqual(memberB.resolveRoutes('peer-c')?.routeHints, ['/memory/peer-c-rejoined']); + await authority.remove(mesh.roster.roster.meshId, 'peer-b'); await memberC.reconcile(); authorityPeer.setReachable(false); diff --git a/packages/runtime-host/src/client/peer-client.ts b/packages/runtime-host/src/client/peer-client.ts index 80d20278ae..c5e6f3b3be 100644 --- a/packages/runtime-host/src/client/peer-client.ts +++ b/packages/runtime-host/src/client/peer-client.ts @@ -396,7 +396,7 @@ function mergeAddresses( primary: readonly string[], secondary: readonly string[] | undefined, ): readonly string[] { - return Object.freeze([...new Set([...primary, ...(secondary ?? [])])].slice(0, 16)); + return Object.freeze([...new Set([...primary, ...(secondary ?? [])])].slice(0, 32)); } async function cancelPeerConnect( diff --git a/packages/runtime-host/src/peer-mesh/index.ts b/packages/runtime-host/src/peer-mesh/index.ts index bdf41194d1..24b2590dd9 100644 --- a/packages/runtime-host/src/peer-mesh/index.ts +++ b/packages/runtime-host/src/peer-mesh/index.ts @@ -22,9 +22,7 @@ export { type PeerMeshAuthorityTarget, type PeerMeshInvitationV1, type PeerMeshRosterV1, - type PeerMeshRouteRecordV1, type SignedPeerMeshRosterV1, - type SignedPeerMeshRouteRecordV1, } from './model.js'; export { openPeerMeshNode, diff --git a/packages/runtime-host/src/peer-mesh/node.ts b/packages/runtime-host/src/peer-mesh/node.ts index ee3851ae45..9aaba5289a 100644 --- a/packages/runtime-host/src/peer-mesh/node.ts +++ b/packages/runtime-host/src/peer-mesh/node.ts @@ -197,9 +197,7 @@ class PeerMeshNodeImpl implements PeerMeshNode { } async initialize(): Promise { - for (const state of this.#store.read()) { - for (const route of state.routes) this.#assertRouteSignature(route); - } + for (const route of this.#store.read().routes) this.#assertRouteSignature(route); } status(): readonly PeerMeshStatus[] { @@ -208,7 +206,7 @@ class PeerMeshNodeImpl implements PeerMeshNode { return Object.freeze( this.#store .read() - .filter( + .meshes.filter( (state) => state.role === 'authority' || state.roster.roster.members.includes(identity.peerId), ) @@ -220,7 +218,7 @@ class PeerMeshNodeImpl implements PeerMeshNode { return this.#admitMesh(async () => { const identity = this.#peer.identity(); const state = await this.#store.mutate((current) => { - assertMeshCapacity(current, identity.peerId); + assertMeshCapacity(current.meshes, identity.peerId); const keys = generatePeerMeshAuthorityKeyPair(); const roster = signPeerMeshRoster( canonicalPeerMeshRoster({ @@ -235,14 +233,19 @@ class PeerMeshNodeImpl implements PeerMeshNode { const state: PeerMeshStateV1 = { role: 'authority', roster, - routes: [], authorityPrivateKey: keys.privateKey, invitations: [], }; - return { state: appendMesh(current, state, identity.peerId), result: state }; + return { + state: { ...current, meshes: appendMesh(current.meshes, state, identity.peerId) }, + result: state, + }; }); await this.#refreshLocalRoute(); - return peerMeshStatus(findMesh(this.#store.read(), state.roster.roster.meshId)!, identity); + return peerMeshStatus( + findMesh(this.#store.read().meshes, state.roster.roster.meshId)!, + identity, + ); }); } @@ -257,7 +260,7 @@ class PeerMeshNodeImpl implements PeerMeshNode { ); } return this.#store.mutate((current) => { - const state = requireAuthority(current, meshId); + const state = requireAuthority(current.meshes, meshId); if (state.roster.roster.closed) throw new Error('Peer Mesh is closed'); const invitations = state.invitations.filter( (invitation) => invitation.status === 'redeemed' || invitation.expiresAt > now, @@ -280,17 +283,20 @@ class PeerMeshNodeImpl implements PeerMeshNode { ...target, }; return { - state: replaceMesh(current, { - ...state, - invitations: [ - ...invitations, - { - status: 'pending', - secretDigest: peerMeshInvitationSecretDigest(secret), - expiresAt, - }, - ], - }), + state: { + ...current, + meshes: replaceMesh(current.meshes, { + ...state, + invitations: [ + ...invitations, + { + status: 'pending', + secretDigest: peerMeshInvitationSecretDigest(secret), + expiresAt, + }, + ], + }), + }, result: Object.freeze(invitation), }; }); @@ -300,12 +306,12 @@ class PeerMeshNodeImpl implements PeerMeshNode { return this.#admitMesh(async () => { const invitation = decodePeerMeshInvitation(invitationValue); const current = this.#store.read(); - const existing = findMesh(current, invitation.meshId); + const existing = findMesh(current.meshes, invitation.meshId); const localPeerId = this.#peer.identity().peerId; if (existing?.role === 'authority') { throw new Error('This peer already belongs to that Peer Mesh'); } - if (!existing) assertMeshCapacity(current, localPeerId); + if (!existing) assertMeshCapacity(current.meshes, localPeerId); const operationSignal = signal ? AbortSignal.any([signal, this.#lifetime.signal]) : this.#lifetime.signal; @@ -345,7 +351,6 @@ class PeerMeshNodeImpl implements PeerMeshNode { throw new Error('Peer Mesh authority returned an unrelated roster'); } const routes = await this.#validateRoutes(response.routes, roster, this.#now()); - const joinedRoutes = mergeRoutes(routes, [localRoute], roster, this.#now()); const state: PeerMeshStateV1 = { role: 'replica', authority: { @@ -354,10 +359,9 @@ class PeerMeshNodeImpl implements PeerMeshNode { coordinationRelays: invitation.coordinationRelays, }, roster, - routes: joinedRoutes, }; const joined = await this.#store.mutate((current) => { - const existing = findMesh(current, invitation.meshId); + const existing = findMesh(current.meshes, invitation.meshId); if (existing?.role === 'authority') { throw new Error('This peer already belongs to that Peer Mesh'); } @@ -368,11 +372,15 @@ class PeerMeshNodeImpl implements PeerMeshNode { ) { throw new Error('Peer Mesh invitation did not advance the existing membership'); } - if (!existing) assertMeshCapacity(current, identity.peerId); + if (!existing) assertMeshCapacity(current.meshes, identity.peerId); + const meshes = existing + ? replaceMesh(current.meshes, state) + : appendMesh(current.meshes, state, identity.peerId); return { - state: existing - ? replaceMesh(current, state) - : appendMesh(current, state, identity.peerId), + state: { + meshes, + routes: mergeRoutes(current.routes, [...routes, localRoute], this.#now()), + }, result: state, }; }); @@ -408,10 +416,14 @@ class PeerMeshNodeImpl implements PeerMeshNode { resolveRoutes(peerId: string) { this.#assertOpen(); const now = this.#now(); - const route = this.#store - .read() - .filter((state) => isActiveMembership(state, this.#peer.identity().peerId)) - .flatMap((state) => state.routes) + const stored = this.#store.read(); + const visible = stored.meshes.some( + (state) => + isActiveMembership(state, this.#peer.identity().peerId) && + state.roster.roster.members.includes(peerId), + ); + if (!visible) return undefined; + const route = stored.routes .filter(({ route }) => route.peerId === peerId && route.expiresAt > now) .sort((left, right) => right.route.sequence - left.route.sequence)[0]?.route; if (!route) return undefined; @@ -489,9 +501,10 @@ class PeerMeshNodeImpl implements PeerMeshNode { string, { readonly meshId: string; readonly target: PeerMeshAuthorityTarget } >(); - for (const state of this.#store.read()) { + const stored = this.#store.read(); + for (const state of stored.meshes) { if (!isActiveMembership(state, identity.peerId)) continue; - for (const signed of state.routes) { + for (const signed of stored.routes) { const route = signed.route; if ( route.peerId !== identity.peerId && @@ -505,8 +518,11 @@ class PeerMeshNodeImpl implements PeerMeshNode { } if (state.role === 'replica' && state.authority.peerId !== identity.peerId) { const key = `${state.roster.roster.meshId}\0${state.authority.peerId}`; - if (!targets.has(key)) - targets.set(key, { meshId: state.roster.roster.meshId, target: state.authority }); + const learned = targets.get(key)?.target; + targets.set(key, { + meshId: state.roster.roster.meshId, + target: learned ? mergeTargets(learned, state.authority) : state.authority, + }); } } for (const { meshId, target } of targets.values()) { @@ -525,10 +541,11 @@ class PeerMeshNodeImpl implements PeerMeshNode { signal: AbortSignal, ): Promise { for (let page = 0; page <= Math.ceil(PEER_MESH_MAX_MEMBERS / ROUTE_PAGE_SIZE); page += 1) { - const state = findMesh(this.#store.read(), meshId); + const stored = this.#store.read(); + const state = findMesh(stored.meshes, meshId); const localPeerId = this.#peer.identity().peerId; if (!state || !isActiveMembership(state, localPeerId)) return; - const route = state.routes.find((candidate) => candidate.route.peerId === localPeerId); + const route = stored.routes.find((candidate) => candidate.route.peerId === localPeerId); if (!route) throw new Error('Peer Mesh local route is unavailable'); const stream = await this.#peer.connectMeshControl( { @@ -547,7 +564,7 @@ class PeerMeshNodeImpl implements PeerMeshNode { meshId, roster: state.roster, route, - knownRoutes: routeSequences(state.routes, this.#now()), + knownRoutes: routeSequences(stored.routes, state.roster, this.#now()), }, decodeSyncResponse, signal, @@ -574,10 +591,9 @@ class PeerMeshNodeImpl implements PeerMeshNode { async #refreshLocalRouteOnce(): Promise { const identity = this.#peer.identity(); const current = this.#store.read(); - const active = current.filter((state) => isActiveMembership(state, identity.peerId)); + const active = current.meshes.filter((state) => isActiveMembership(state, identity.peerId)); if (active.length === 0) return undefined; - const existing = active - .flatMap((state) => state.routes) + const existing = current.routes .filter(({ route }) => route.peerId === identity.peerId) .sort((left, right) => right.route.sequence - left.route.sequence)[0]; const now = this.#now(); @@ -587,27 +603,11 @@ class PeerMeshNodeImpl implements PeerMeshNode { sameAddresses(existing.route.routeHints, identity.listenAddresses) && sameAddresses(existing.route.coordinationRelays, identity.coordinationRelays) ) { - if ( - active.every((state) => - state.routes.some((candidate) => sameSignedRoute(candidate, existing)), - ) - ) { - return existing; - } + return existing; } - const route = - existing && - existing.route.expiresAt > now + ROUTE_REFRESH_LEAD_MS && - sameAddresses(existing.route.routeHints, identity.listenAddresses) && - sameAddresses(existing.route.coordinationRelays, identity.coordinationRelays) - ? existing - : await this.#signLocalRoute((existing?.route.sequence ?? 0) + 1); + const route = await this.#signLocalRoute((existing?.route.sequence ?? 0) + 1); await this.#store.mutate((states) => ({ - state: states.map((state) => - isActiveMembership(state, identity.peerId) - ? { ...state, routes: mergeRoutes(state.routes, [route], state.roster, now) } - : state, - ), + state: { ...states, routes: mergeRoutes(states.routes, [route], now) }, result: undefined, })); return route; @@ -617,8 +617,7 @@ class PeerMeshNodeImpl implements PeerMeshNode { const identity = this.#peer.identity(); const maxSequence = this.#store .read() - .flatMap((state) => state.routes) - .filter(({ route }) => route.peerId === identity.peerId) + .routes.filter(({ route }) => route.peerId === identity.peerId) .reduce((maximum, { route }) => Math.max(maximum, route.sequence), 0); const route = canonicalPeerMeshRouteRecord({ version: 1, @@ -697,7 +696,7 @@ class PeerMeshNodeImpl implements PeerMeshNode { const routes = this.#validateRoutes(routeValues, roster, this.#now()); const localPeerId = this.#peer.identity().peerId; await this.#store.mutate((current) => { - const state = findMesh(current, meshId); + const state = findMesh(current.meshes, meshId); if (!state || state.roster.authorityPublicKey !== roster.authorityPublicKey) { throw new Error('Peer Mesh synchronization has the wrong authority'); } @@ -705,12 +704,17 @@ class PeerMeshNodeImpl implements PeerMeshNode { const next = { ...state, roster: nextRoster, - routes: - nextRoster.roster.closed || !nextRoster.roster.members.includes(localPeerId) - ? [] - : mergeRoutes(state.routes, routes, nextRoster, this.#now()), }; - return { state: replaceMesh(current, next), result: undefined }; + return { + state: { + meshes: replaceMesh(current.meshes, next), + routes: + nextRoster.roster.closed || !nextRoster.roster.members.includes(localPeerId) + ? current.routes + : mergeRoutes(current.routes, routes, this.#now()), + }, + result: undefined, + }; }); } @@ -727,7 +731,7 @@ class PeerMeshNodeImpl implements PeerMeshNode { }, ): Promise { return this.#store.mutate((current) => { - const state = requireAuthority(current, meshId); + const state = requireAuthority(current.meshes, meshId); if (state.roster.roster.closed) { if (closedIsSuccess) { return { state: current, result: peerMeshStatus(state, this.#peer.identity()) }; @@ -748,9 +752,6 @@ class PeerMeshNodeImpl implements PeerMeshNode { const updated = { ...state, roster, - routes: next.closed - ? [] - : state.routes.filter(({ route }) => next.members.includes(route.peerId)), invitations: next.closed ? state.invitations.filter(({ status }) => status === 'redeemed') : state.invitations.filter( @@ -759,7 +760,7 @@ class PeerMeshNodeImpl implements PeerMeshNode { ), }; return { - state: replaceMesh(current, updated), + state: { ...current, meshes: replaceMesh(current.meshes, updated) }, result: peerMeshStatus(updated, this.#peer.identity()), }; }); @@ -828,7 +829,7 @@ class PeerMeshNodeImpl implements PeerMeshNode { ): Promise { const now = this.#now(); return this.#store.mutate((current) => { - const state = findMesh(current, request.meshId); + const state = findMesh(current.meshes, request.meshId); if (!state || state.role !== 'authority') return { state: current, result: rejected('invalid') }; const invitation = state.invitations.find(({ secretDigest }) => @@ -846,15 +847,16 @@ class PeerMeshNodeImpl implements PeerMeshNode { } const updated = { ...state, - routes: mergeAuthenticatedRoute(state.routes, remoteRoute, state.roster, now), }; + const routes = mergeAuthenticatedRoute(current.routes, remoteRoute, now); return { - state: replaceMesh(current, updated), + state: { meshes: replaceMesh(current.meshes, updated), routes }, result: { kind: 'invitation-redeemed', roster: updated.roster, routes: responseRoutes( updated, + routes, [{ peerId: remotePeerId, sequence: Number.MAX_SAFE_INTEGER }], now, ).routes, @@ -867,13 +869,19 @@ class PeerMeshNodeImpl implements PeerMeshNode { ); if (invitation.expiresAt <= now) { return { - state: replaceMesh(current, { ...state, invitations: remaining }), + state: { + ...current, + meshes: replaceMesh(current.meshes, { ...state, invitations: remaining }), + }, result: rejected('expired'), }; } if (state.roster.roster.closed) { return { - state: replaceMesh(current, { ...state, invitations: remaining }), + state: { + ...current, + meshes: replaceMesh(current.meshes, { ...state, invitations: remaining }), + }, result: rejected('closed'), }; } @@ -882,14 +890,16 @@ class PeerMeshNodeImpl implements PeerMeshNode { state.roster.roster.members.length >= PEER_MESH_MAX_MEMBERS ) { return { - state: replaceMesh(current, { ...state, invitations: remaining }), + state: { + ...current, + meshes: replaceMesh(current.meshes, { ...state, invitations: remaining }), + }, result: rejected('full'), }; } if (state.roster.roster.members.includes(remotePeerId)) { const updated = { ...state, - routes: mergeAuthenticatedRoute(state.routes, remoteRoute, state.roster, now), invitations: [ ...remaining.filter( (record) => record.status === 'pending' || record.peerId !== remotePeerId, @@ -897,13 +907,15 @@ class PeerMeshNodeImpl implements PeerMeshNode { redeemedInvitation(invitation, remotePeerId), ], }; + const routes = mergeAuthenticatedRoute(current.routes, remoteRoute, now); return { - state: replaceMesh(current, updated), + state: { meshes: replaceMesh(current.meshes, updated), routes }, result: { kind: 'invitation-redeemed', roster: state.roster, routes: responseRoutes( updated, + routes, [{ peerId: remotePeerId, sequence: Number.MAX_SAFE_INTEGER }], now, ).routes, @@ -922,7 +934,6 @@ class PeerMeshNodeImpl implements PeerMeshNode { const updated = { ...state, roster, - routes: mergeRoutes(state.routes, [remoteRoute], roster, now), invitations: [ ...remaining.filter( (record) => record.status === 'pending' || record.peerId !== remotePeerId, @@ -930,13 +941,15 @@ class PeerMeshNodeImpl implements PeerMeshNode { redeemedInvitation(invitation, remotePeerId), ], }; + const routes = mergeRoutes(current.routes, [remoteRoute], now); return { - state: replaceMesh(current, updated), + state: { meshes: replaceMesh(current.meshes, updated), routes }, result: { kind: 'invitation-redeemed', roster, routes: responseRoutes( updated, + routes, [{ peerId: remotePeerId, sequence: Number.MAX_SAFE_INTEGER }], now, ).routes, @@ -950,7 +963,7 @@ class PeerMeshNodeImpl implements PeerMeshNode { await this.#refreshLocalRoute(); const incomingRoster = decodeSignedPeerMeshRoster(request.roster); return this.#store.mutate((current) => { - const state = findMesh(current, request.meshId); + const state = findMesh(current.meshes, request.meshId); if (!state || state.roster.authorityPublicKey !== incomingRoster.authorityPublicKey) { return { state: current, result: { kind: 'sync-rejected', reason: 'unknown' } as const }; } @@ -961,13 +974,14 @@ class PeerMeshNodeImpl implements PeerMeshNode { const updated = { ...state, roster, - routes: localMember - ? mergeRoutes(state.routes, remoteMember ? [remoteRoute] : [], roster, this.#now()) - : [], }; + const routes = + localMember && remoteMember + ? mergeRoutes(current.routes, [remoteRoute], this.#now()) + : current.routes; if (!localMember || !remoteMember) { return { - state: replaceMesh(current, updated), + state: { meshes: replaceMesh(current.meshes, updated), routes }, result: { kind: 'sync-result', roster, @@ -976,9 +990,9 @@ class PeerMeshNodeImpl implements PeerMeshNode { } as const, }; } - const page = responseRoutes(updated, request.knownRoutes, this.#now()); + const page = responseRoutes(updated, routes, request.knownRoutes, this.#now()); return { - state: replaceMesh(current, updated), + state: { meshes: replaceMesh(current.meshes, updated), routes }, result: { kind: 'sync-result', roster, @@ -1097,23 +1111,17 @@ function selectRoster( function mergeRoutes( current: readonly SignedPeerMeshRouteRecordV1[], candidates: readonly SignedPeerMeshRouteRecordV1[], - roster: SignedPeerMeshRosterV1, now: number, ): readonly SignedPeerMeshRouteRecordV1[] { - const routes = new Map( - current - .filter(({ route }) => roster.roster.members.includes(route.peerId)) - .map((route) => [route.route.peerId, route] as const), - ); + const routes = new Map(current.map((route) => [route.route.peerId, route] as const)); for (const candidate of candidates) { + if (candidate.route.expiresAt <= now) continue; + const existing = routes.get(candidate.route.peerId); if ( - candidate.route.expiresAt <= now || - !roster.roster.members.includes(candidate.route.peerId) + !existing || + existing.route.expiresAt <= now || + candidate.route.sequence > existing.route.sequence ) { - continue; - } - const existing = routes.get(candidate.route.peerId); - if (!existing || candidate.route.sequence > existing.route.sequence) { routes.set(candidate.route.peerId, candidate); continue; } @@ -1132,28 +1140,31 @@ function mergeRoutes( function mergeAuthenticatedRoute( current: readonly SignedPeerMeshRouteRecordV1[], candidate: SignedPeerMeshRouteRecordV1, - roster: SignedPeerMeshRosterV1, now: number, ): readonly SignedPeerMeshRouteRecordV1[] { const existing = current.find(({ route }) => route.peerId === candidate.route.peerId); - if (existing && existing.route.sequence > candidate.route.sequence) { - return mergeRoutes(current, [], roster, now); + if ( + existing && + existing.route.expiresAt > now && + existing.route.sequence > candidate.route.sequence + ) { + return current; } return mergeRoutes( current.filter(({ route }) => route.peerId !== candidate.route.peerId), [candidate], - roster, now, ); } function routeSequences( routes: readonly SignedPeerMeshRouteRecordV1[], + roster: SignedPeerMeshRosterV1, now: number, ): readonly PeerMeshRouteSequence[] { return Object.freeze( routes - .filter(({ route }) => route.expiresAt > now) + .filter(({ route }) => route.expiresAt > now && roster.roster.members.includes(route.peerId)) .map(({ route }) => Object.freeze({ peerId: route.peerId, sequence: route.sequence })) .sort((left, right) => left.peerId.localeCompare(right.peerId)), ); @@ -1161,11 +1172,12 @@ function routeSequences( function responseRoutes( state: PeerMeshStateV1, + routes: readonly SignedPeerMeshRouteRecordV1[], knownRoutes: readonly PeerMeshRouteSequence[], now: number, ): { readonly routes: readonly SignedPeerMeshRouteRecordV1[]; readonly more: boolean } { const known = new Map(knownRoutes.map(({ peerId, sequence }) => [peerId, sequence])); - const missing = state.routes.filter( + const missing = routes.filter( ({ route }) => route.expiresAt > now && state.roster.roster.members.includes(route.peerId) && @@ -1181,11 +1193,17 @@ function sameAddresses(left: readonly string[], right: readonly string[]): boole return left.length === right.length && left.every((address, index) => address === right[index]); } -function sameSignedRoute( - left: SignedPeerMeshRouteRecordV1, - right: SignedPeerMeshRouteRecordV1, -): boolean { - return JSON.stringify(left) === JSON.stringify(right); +function mergeTargets( + primary: PeerMeshAuthorityTarget, + fallback: PeerMeshAuthorityTarget, +): PeerMeshAuthorityTarget { + return Object.freeze({ + peerId: primary.peerId, + routeHints: Object.freeze([...new Set([...primary.routeHints, ...fallback.routeHints])]), + coordinationRelays: Object.freeze([ + ...new Set([...primary.coordinationRelays, ...fallback.coordinationRelays]), + ]), + }); } async function exchangeControl( diff --git a/packages/runtime-host/src/peer-mesh/owner.ts b/packages/runtime-host/src/peer-mesh/owner.ts index b0ac235b91..dc6df49494 100644 --- a/packages/runtime-host/src/peer-mesh/owner.ts +++ b/packages/runtime-host/src/peer-mesh/owner.ts @@ -30,6 +30,7 @@ export interface RuntimeHostPeerMeshOwner { export async function openRuntimeHostPeerMeshOwner(input: { readonly nativePath: string; readonly keyPath: string; + readonly expectedPeerId?: string; readonly dataRoot: string; readonly listenAddresses?: readonly string[]; readonly coordinationRelays?: readonly string[]; @@ -38,6 +39,7 @@ export async function openRuntimeHostPeerMeshOwner(input: { const client = createRuntimeHostPeerClient({ nativePath: input.nativePath, keyPath: input.keyPath, + ...(input.expectedPeerId ? { expectedPeerId: input.expectedPeerId } : {}), ...(input.listenAddresses ? { listenAddresses: input.listenAddresses } : {}), ...(input.coordinationRelays ? { coordinationRelays: input.coordinationRelays } : {}), routeResolver: { resolveRoutes: (peerId) => mesh?.resolveRoutes(peerId) }, diff --git a/packages/runtime-host/src/peer-mesh/store.ts b/packages/runtime-host/src/peer-mesh/store.ts index e5d802dd1c..3a7594a770 100644 --- a/packages/runtime-host/src/peer-mesh/store.ts +++ b/packages/runtime-host/src/peer-mesh/store.ts @@ -58,7 +58,6 @@ interface RedeemedPeerMeshInvitation { interface PeerMeshStateBase { readonly roster: SignedPeerMeshRosterV1; - readonly routes: readonly SignedPeerMeshRouteRecordV1[]; } export interface PeerMeshAuthorityStateV1 extends PeerMeshStateBase { @@ -74,11 +73,16 @@ export interface PeerMeshReplicaStateV1 extends PeerMeshStateBase { export type PeerMeshStateV1 = PeerMeshAuthorityStateV1 | PeerMeshReplicaStateV1; +export interface PeerMeshStoredStateV1 { + readonly meshes: readonly PeerMeshStateV1[]; + readonly routes: readonly SignedPeerMeshRouteRecordV1[]; +} + export interface PeerMeshStateStore { - read(): readonly PeerMeshStateV1[]; + read(): PeerMeshStoredStateV1; mutate( - operation: (state: readonly PeerMeshStateV1[]) => { - readonly state: readonly PeerMeshStateV1[]; + operation: (state: PeerMeshStoredStateV1) => { + readonly state: PeerMeshStoredStateV1; readonly result: T; }, ): Promise; @@ -103,7 +107,7 @@ export async function openPeerMeshStateStore( class PeerMeshStateStoreImpl implements PeerMeshStateStore { readonly #path: string; - #state: readonly PeerMeshStateV1[]; + #state: PeerMeshStoredStateV1; #tail = Promise.resolve(); #failure: Error | undefined; #closeTask: Promise | undefined; @@ -113,20 +117,20 @@ class PeerMeshStateStoreImpl implements PeerMeshStateStore { dataRoot: string, private readonly localPeerId: string, private readonly owner: FileLifetimeOwner, - state: readonly PeerMeshStateV1[], + state: PeerMeshStoredStateV1, ) { this.#path = join(dataRoot, STATE_FILE); this.#state = state; } - read(): readonly PeerMeshStateV1[] { + read(): PeerMeshStoredStateV1 { this.#assertOpen(); return this.#state; } mutate( - operation: (state: readonly PeerMeshStateV1[]) => { - readonly state: readonly PeerMeshStateV1[]; + operation: (state: PeerMeshStoredStateV1) => { + readonly state: PeerMeshStoredStateV1; readonly result: T; }, ): Promise { @@ -135,8 +139,9 @@ class PeerMeshStateStoreImpl implements PeerMeshStateStore { if (this.#failure) throw this.#failure; const updated = operation(this.#state); if (updated.state === this.#state) return updated.result; - const canonical = decodePeerMeshStates(updated.state, this.localPeerId); - assertStateAdvance(this.#state, canonical, this.localPeerId); + const candidate = pruneUnreferencedRoutes(updated.state); + const canonical = decodePeerMeshStoredState(candidate, this.localPeerId); + assertStateAdvance(this.#state.meshes, canonical.meshes, this.localPeerId); try { await writeState(this.#path, this.localPeerId, canonical); this.#state = canonical; @@ -180,8 +185,8 @@ export function decodePeerMeshState(value: unknown, localPeerId: string): PeerMe const record = value as Record; const expectedKeys = record.role === 'authority' - ? ['role', 'roster', 'routes', 'authorityPrivateKey', 'invitations'] - : ['role', 'roster', 'routes', 'authority']; + ? ['role', 'roster', 'authorityPrivateKey', 'invitations'] + : ['role', 'roster', 'authority']; if ( Object.keys(record).length !== expectedKeys.length || expectedKeys.some((key) => !Object.hasOwn(record, key)) @@ -192,7 +197,6 @@ export function decodePeerMeshState(value: unknown, localPeerId: string): PeerMe throw new Error('Unsupported Peer Mesh state'); } const roster = decodeSignedPeerMeshRoster(record.roster); - const routes = decodeRoutes(record.routes, roster); if (record.role === 'authority') { if (!roster.roster.members.includes(localPeerId)) { throw new Error('Peer Mesh authority is not present in its roster'); @@ -205,7 +209,6 @@ export function decodePeerMeshState(value: unknown, localPeerId: string): PeerMe return Object.freeze({ role: 'authority', roster, - routes, authorityPrivateKey: privateKey, invitations: Object.freeze(decodeInvitations(record.invitations)), }); @@ -218,7 +221,6 @@ export function decodePeerMeshState(value: unknown, localPeerId: string): PeerMe role: 'replica', authority, roster, - routes, }); } @@ -285,7 +287,7 @@ export function authorityKeys(state: PeerMeshStateV1): PeerMeshAuthorityKeyPair async function readState( path: string, expectedLocalPeerId: string, -): Promise { +): Promise { try { const info = await lstat(path); if (!info.isFile() || info.size > MAX_STATE_BYTES) @@ -295,21 +297,31 @@ async function readState( throw new Error('Invalid Peer Mesh state document'); } const record = document as Record; - if ( - (record.version !== 1 && record.version !== 2) || - Object.keys(record).length !== 3 || - !Object.hasOwn(record, 'localPeerId') || - !Object.hasOwn(record, 'meshes') - ) { + const versionOne = + record.version === 1 && + Object.keys(record).length === 3 && + Object.hasOwn(record, 'localPeerId') && + Object.hasOwn(record, 'meshes'); + const versionTwo = + record.version === 2 && + Object.keys(record).length === 4 && + Object.hasOwn(record, 'localPeerId') && + Object.hasOwn(record, 'meshes') && + Object.hasOwn(record, 'routes'); + if (!versionOne && !versionTwo) { throw new Error('Unsupported Peer Mesh state document'); } if (boundedString(record.localPeerId, 'localPeerId', 256) !== expectedLocalPeerId) { throw new Error('Peer Mesh state belongs to a different peer identity'); } - const meshes = record.version === 1 ? migratePeerMeshStateV1(record.meshes) : record.meshes; - return decodePeerMeshStates(meshes, expectedLocalPeerId); + return decodePeerMeshStoredState( + { meshes: record.meshes, routes: versionOne ? [] : record.routes }, + expectedLocalPeerId, + ); } catch (error) { - if (isNodeError(error, 'ENOENT')) return Object.freeze([]); + if (isNodeError(error, 'ENOENT')) { + return Object.freeze({ meshes: Object.freeze([]), routes: Object.freeze([]) }); + } throw error; } } @@ -317,9 +329,9 @@ async function readState( async function writeState( path: string, localPeerId: string, - state: readonly PeerMeshStateV1[], + state: PeerMeshStoredStateV1, ): Promise { - const document = `${JSON.stringify({ version: 2, localPeerId, meshes: state }, null, 2)}\n`; + const document = `${JSON.stringify({ version: 2, localPeerId, ...state }, null, 2)}\n`; if (Buffer.byteLength(document) > MAX_STATE_BYTES) throw new Error('Peer Mesh state is too large'); const temporary = `${path}.tmp`; @@ -404,28 +416,51 @@ function decodeInvitations(value: unknown): PeerMeshInvitationRecord[] { function decodeRoutes( value: unknown, - roster: SignedPeerMeshRosterV1, + meshes: readonly PeerMeshStateV1[], ): readonly SignedPeerMeshRouteRecordV1[] { - if (!Array.isArray(value) || value.length > PEER_MESH_MAX_MEMBERS) { + if (!Array.isArray(value) || value.length > PEER_MESH_MAX_MESHES * PEER_MESH_MAX_MEMBERS) { throw new Error('Invalid Peer Mesh routes'); } const routes = value.map(decodeSignedPeerMeshRouteRecord); const peerIds = routes.map(({ route }) => route.peerId); + const knownPeers = new Set( + meshes + .filter(({ roster }) => !roster.roster.closed) + .flatMap(({ roster }) => roster.roster.members), + ); if ( new Set(peerIds).size !== peerIds.length || - peerIds.some((peerId) => !roster.roster.members.includes(peerId)) + peerIds.some((peerId) => !knownPeers.has(peerId)) ) { throw new Error('Invalid Peer Mesh routes'); } return Object.freeze(routes); } -function migratePeerMeshStateV1(value: unknown): unknown { - if (!Array.isArray(value)) return value; - return value.map((entry) => { - if (!entry || typeof entry !== 'object' || Array.isArray(entry)) return entry; - return { ...(entry as Record), routes: [] }; - }); +function decodePeerMeshStoredState(value: unknown, localPeerId: string): PeerMeshStoredStateV1 { + if (!value || typeof value !== 'object' || Array.isArray(value)) { + throw new Error('Invalid Peer Mesh state document'); + } + const record = value as Record; + if ( + Object.keys(record).length !== 2 || + !Object.hasOwn(record, 'meshes') || + !Object.hasOwn(record, 'routes') + ) { + throw new Error('Invalid Peer Mesh state document'); + } + const meshes = decodePeerMeshStates(record.meshes, localPeerId); + return Object.freeze({ meshes, routes: decodeRoutes(record.routes, meshes) }); +} + +function pruneUnreferencedRoutes(state: PeerMeshStoredStateV1): PeerMeshStoredStateV1 { + const knownPeers = new Set( + state.meshes + .filter(({ roster }) => !roster.roster.closed) + .flatMap(({ roster }) => roster.roster.members), + ); + const routes = state.routes.filter(({ route }) => knownPeers.has(route.peerId)); + return routes.length === state.routes.length ? state : { ...state, routes }; } function boundedString(value: unknown, label: string, max: number): string { diff --git a/scripts/smoke-release-cli-package.mjs b/scripts/smoke-release-cli-package.mjs index 35eb2ee621..f645bbffb1 100644 --- a/scripts/smoke-release-cli-package.mjs +++ b/scripts/smoke-release-cli-package.mjs @@ -253,6 +253,7 @@ async function smokeRuntimeHostPeerProtocol({ packageRoot, cliEntrypoint, root } meshAuthorityOwner = await mesh.openRuntimeHostPeerMeshOwner({ nativePath, keyPath: hostKeyPath, + expectedPeerId: peerId, dataRoot: join(root, 'mesh-authority'), listenAddresses: ['/ip4/127.0.0.1/udp/0/quic-v1'], }); From c80cc98b408b69780a43b5833b57d861c69b8a01 Mon Sep 17 00:00:00 2001 From: Wang Date: Fri, 28 Aug 2026 13:26:50 +0800 Subject: [PATCH 4/8] refactor(runtime-host): keep mesh routing in mesh node Use the route resolver only for application connections so Mesh control has one target-composition authority. Generated-by: OpenAI Codex --- packages/runtime-host/src/client/peer-client.ts | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/packages/runtime-host/src/client/peer-client.ts b/packages/runtime-host/src/client/peer-client.ts index c5e6f3b3be..1a0bd892b3 100644 --- a/packages/runtime-host/src/client/peer-client.ts +++ b/packages/runtime-host/src/client/peer-client.ts @@ -249,7 +249,8 @@ class RuntimeHostPeerClientImpl implements RuntimeHostPeerClient { signal?.throwIfAborted(); const endpoint = this.#requireEndpoint(); const requestId = this.#allocateRequestId(); - const discovered = this.#routeResolver?.resolveRoutes(input.peerId); + const discovered = + kind === 'application' ? this.#routeResolver?.resolveRoutes(input.peerId) : undefined; const connection = endpoint[kind === 'application' ? 'connect' : 'connectMeshControl']({ ...input, routeHints: mergeAddresses(discovered?.routeHints ?? [], input.routeHints), From 6ca99eccea564fc65cf72d58c3ea65e2a95f4601 Mon Sep 17 00:00:00 2001 From: Wang Date: Fri, 28 Aug 2026 13:33:45 +0800 Subject: [PATCH 5/8] fix(runtime-host): serialize mesh route refresh on join Reuse the node's local route refresh while joining so background reconciliation cannot publish a different fact at the same sequence. Generated-by: OpenAI Codex --- packages/runtime-host/src/peer-mesh/node.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/runtime-host/src/peer-mesh/node.ts b/packages/runtime-host/src/peer-mesh/node.ts index 9aaba5289a..25ed7fd4c4 100644 --- a/packages/runtime-host/src/peer-mesh/node.ts +++ b/packages/runtime-host/src/peer-mesh/node.ts @@ -325,7 +325,7 @@ class PeerMeshNodeImpl implements PeerMeshNode { operationSignal, ); try { - const localRoute = await this.#signLocalRoute(); + const localRoute = (await this.#refreshLocalRoute()) ?? (await this.#signLocalRoute()); const request: RedeemInvitationRequest = { kind: 'redeem-invitation', meshId: invitation.meshId, From 16dd68d1cfaa0d862d92335c39020b5839ab669e Mon Sep 17 00:00:00 2001 From: Wang Date: Fri, 28 Aug 2026 13:36:53 +0800 Subject: [PATCH 6/8] refactor(runtime-host): centralize mesh route sequencing Allocate every local route sequence from the node-level cache instead of accepting caller-supplied values. Generated-by: OpenAI Codex --- packages/runtime-host/src/peer-mesh/node.ts | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/packages/runtime-host/src/peer-mesh/node.ts b/packages/runtime-host/src/peer-mesh/node.ts index 25ed7fd4c4..bef92d5a71 100644 --- a/packages/runtime-host/src/peer-mesh/node.ts +++ b/packages/runtime-host/src/peer-mesh/node.ts @@ -605,7 +605,7 @@ class PeerMeshNodeImpl implements PeerMeshNode { ) { return existing; } - const route = await this.#signLocalRoute((existing?.route.sequence ?? 0) + 1); + const route = await this.#signLocalRoute(); await this.#store.mutate((states) => ({ state: { ...states, routes: mergeRoutes(states.routes, [route], now) }, result: undefined, @@ -613,7 +613,7 @@ class PeerMeshNodeImpl implements PeerMeshNode { return route; } - async #signLocalRoute(sequence?: number): Promise { + async #signLocalRoute(): Promise { const identity = this.#peer.identity(); const maxSequence = this.#store .read() @@ -622,7 +622,7 @@ class PeerMeshNodeImpl implements PeerMeshNode { const route = canonicalPeerMeshRouteRecord({ version: 1, peerId: identity.peerId, - sequence: sequence ?? maxSequence + 1, + sequence: maxSequence + 1, expiresAt: this.#now() + ROUTE_TTL_MS, routeHints: identity.listenAddresses, coordinationRelays: identity.coordinationRelays, From 1ee1f08594a85d92b7daea5d51ccb385c1eeac4d Mon Sep 17 00:00:00 2001 From: Wang Date: Fri, 28 Aug 2026 14:27:07 +0800 Subject: [PATCH 7/8] fix(runtime-host): serialize connects per peer Queue application and Mesh-control dials targeting the same PeerId so the shared native endpoint never rejects a valid concurrent caller as already in progress. Generated-by: OpenAI Codex --- .../src/__tests__/peer-native.test.ts | 53 +++++++++++++++++-- .../runtime-host/src/client/peer-client.ts | 40 ++++++++++++++ 2 files changed, 88 insertions(+), 5 deletions(-) diff --git a/packages/runtime-host/src/__tests__/peer-native.test.ts b/packages/runtime-host/src/__tests__/peer-native.test.ts index 0faf6a04e1..1568f677bc 100644 --- a/packages/runtime-host/src/__tests__/peer-native.test.ts +++ b/packages/runtime-host/src/__tests__/peer-native.test.ts @@ -33,7 +33,7 @@ import { type RuntimeHostPeerNativeStream, } from '../transport/peer-native.js'; -test('shares one peer endpoint while cancelling connection attempts independently', async () => { +test('shares one peer endpoint, serializes same-peer connects, and cancels independently', async () => { const directory = await mkdtemp(join(tmpdir(), 'maka-peer-abort-')); const nativePath = join(directory, 'peer.cjs'); try { @@ -47,6 +47,10 @@ let missFirstCancellation = true; const stream = { read: async () => null, write: async () => {}, close: async () => {}, abort: () => {} }; module.exports = { stats, + resolveConnect: (requestId) => { + pending.get(requestId)?.resolve(stream); + pending.delete(requestId); + }, failEndpoint: () => { finishAccept?.(null); finishMeshAccept?.(null); }, ensurePeerIdentity: async () => 'client', signPeerIdentity: async () => ({ publicKey: Buffer.from('public'), signature: Buffer.from('signature') }), @@ -59,12 +63,12 @@ module.exports = { connect: ({ requestId, peerId, routeHints, coordinationRelays }) => { stats.requests.push({ requestId, peerId, routeHints, coordinationRelays }); if (peerId === 'ready') return Promise.resolve(stream); - return new Promise((_resolve, reject) => pending.set(requestId, reject)); + return new Promise((resolve, reject) => pending.set(requestId, { resolve, reject })); }, connectMeshControl: ({ requestId, peerId, routeHints, coordinationRelays }) => { stats.requests.push({ requestId, peerId, routeHints, coordinationRelays }); if (peerId === 'ready') return Promise.resolve(stream); - return new Promise((_resolve, reject) => pending.set(requestId, reject)); + return new Promise((resolve, reject) => pending.set(requestId, { resolve, reject })); }, cancelConnect: async (requestId) => { stats.cancellations.push(requestId); @@ -72,7 +76,7 @@ module.exports = { missFirstCancellation = false; return false; } - pending.get(requestId)?.(new Error('peer_connect_cancelled: cancelled')); + pending.get(requestId)?.reject(new Error('peer_connect_cancelled: cancelled')); pending.delete(requestId); return true; }, @@ -94,13 +98,30 @@ module.exports = { }), }, }); + const native = await import(nativePath); const abort = new AbortController(); const pending = client.connect(peerConnectInput('pending'), abort.signal); + await waitForRequestCount(native.default.stats, 1); abort.abort(); await assert.rejects(pending, /aborted/u); + const application = client.connect(peerConnectInput('shared')); + await waitForRequestCount(native.default.stats, 2); + const queuedAbort = new AbortController(); + const cancelled = client.connectMeshControl(peerConnectInput('shared'), queuedAbort.signal); + queuedAbort.abort(); + await assert.rejects(cancelled, /aborted/u); + const control = client.connectMeshControl(peerConnectInput('shared')); + await waitForImmediate(); + assert.equal(native.default.stats.requests.length, 2); + native.default.resolveConnect(2); + await application; + await waitForRequestCount(native.default.stats, 3); + assert.equal(native.default.stats.requests.length, 3); + native.default.resolveConnect(3); + await control; + await client.connect(peerConnectInput('ready')); - const native = await import(nativePath); assert.deepEqual(native.default.stats, { starts: 1, closes: 0, @@ -113,6 +134,18 @@ module.exports = { }, { requestId: 2, + peerId: 'shared', + routeHints: ['/memory/discovered', '/memory/1'], + coordinationRelays: ['/memory/relay'], + }, + { + requestId: 3, + peerId: 'shared', + routeHints: ['/memory/1'], + coordinationRelays: [], + }, + { + requestId: 4, peerId: 'ready', routeHints: ['/memory/discovered', '/memory/1'], coordinationRelays: ['/memory/relay'], @@ -212,6 +245,16 @@ test('bounds and separates the peer credential preface from Runtime Host frames' assert.deepEqual(result.remainder, frame); }); +async function waitForRequestCount( + stats: { readonly requests: readonly unknown[] }, + expected: number, +): Promise { + for (let attempt = 0; attempt < 10 && stats.requests.length < expected; attempt += 1) { + await waitForImmediate(); + } + assert.equal(stats.requests.length, expected); +} + function streamWith(chunk: Buffer): RuntimeHostPeerNativeStream { let pending: Buffer | null = chunk; return { diff --git a/packages/runtime-host/src/client/peer-client.ts b/packages/runtime-host/src/client/peer-client.ts index 1a0bd892b3..6464bfdfba 100644 --- a/packages/runtime-host/src/client/peer-client.ts +++ b/packages/runtime-host/src/client/peer-client.ts @@ -114,6 +114,7 @@ class RuntimeHostPeerClientImpl implements RuntimeHostPeerClient { #applicationConsumer: InboundConsumer | undefined; #meshConsumer: InboundConsumer | undefined; #terminalError: Error | undefined; + readonly #connectTails = new Map>(); #nextRequestId = 1; #closed = false; #closeTask: Promise | undefined; @@ -245,6 +246,29 @@ class RuntimeHostPeerClientImpl implements RuntimeHostPeerClient { input: RuntimeHostPeerConnectInput, signal: AbortSignal | undefined, kind: 'application' | 'mesh-control', + ): Promise { + const previous = this.#connectTails.get(input.peerId) ?? Promise.resolve(); + let release!: () => void; + const turn = new Promise((resolve) => { + release = resolve; + }); + const tail = previous.then(() => turn); + this.#connectTails.set(input.peerId, tail); + try { + await waitForPeerConnectTurn(previous, signal); + return await this.#startConnect(input, signal, kind); + } finally { + release(); + void tail.then(() => { + if (this.#connectTails.get(input.peerId) === tail) this.#connectTails.delete(input.peerId); + }); + } + } + + async #startConnect( + input: RuntimeHostPeerConnectInput, + signal: AbortSignal | undefined, + kind: 'application' | 'mesh-control', ): Promise { signal?.throwIfAborted(); const endpoint = this.#requireEndpoint(); @@ -393,6 +417,22 @@ interface InboundConsumer { readonly reject: (error: Error) => void; } +function waitForPeerConnectTurn(previous: Promise, signal?: AbortSignal): Promise { + if (!signal) return previous; + if (signal.aborted) return Promise.reject(signal.reason); + return new Promise((resolve, reject) => { + const onAbort = () => { + signal.removeEventListener('abort', onAbort); + reject(signal.reason); + }; + signal.addEventListener('abort', onAbort, { once: true }); + void previous.then(() => { + signal.removeEventListener('abort', onAbort); + resolve(); + }); + }); +} + function mergeAddresses( primary: readonly string[], secondary: readonly string[] | undefined, From 8bc4f7bb31babc27326970871bf898f2084ea929 Mon Sep 17 00:00:00 2001 From: Wang Date: Fri, 28 Aug 2026 17:03:41 +0800 Subject: [PATCH 8/8] fix(runtime-host): refresh mesh proofs during reconciliation Refresh the local signed route before each synchronization page so earlier unreachable members cannot starve later healthy peers with an expired proof. Add a deterministic long-round regression covering the real per-peer dial budget. Generated-by: OpenAI Codex --- .../src/__tests__/peer-mesh.test.ts | 51 ++++++++++++++++++- packages/runtime-host/src/peer-mesh/node.ts | 1 + 2 files changed, 50 insertions(+), 2 deletions(-) diff --git a/packages/runtime-host/src/__tests__/peer-mesh.test.ts b/packages/runtime-host/src/__tests__/peer-mesh.test.ts index 729e7ab16e..8ba334ed55 100644 --- a/packages/runtime-host/src/__tests__/peer-mesh.test.ts +++ b/packages/runtime-host/src/__tests__/peer-mesh.test.ts @@ -176,6 +176,47 @@ test('reconciles changed routes, propagates removal, and recovers the verified c } }); +test('refreshes its route after earlier peers consume the proof lifetime', async () => { + const root = await mkdtemp(join(tmpdir(), 'maka-peer-mesh-route-refresh-')); + let now = Date.now(); + const network = new MemoryPeerNetwork(() => { + now += 31_000; + }); + const peers = Array.from({ length: 14 }, (_, index) => + network.create(`peer-${String.fromCharCode('a'.charCodeAt(0) + index)}`), + ); + const nodes: PeerMeshNode[] = []; + const serving: Promise[] = []; + try { + for (const [index, peer] of peers.entries()) { + const node = await openPeerMeshNode({ + dataRoot: join(root, String(index)), + peer, + now: () => now, + }); + nodes.push(node); + serving.push(node.serve()); + } + const authority = nodes[0]!; + const healthy = nodes[12]!; + const mesh = await authority.create(); + for (const node of nodes.slice(1)) { + await node.join(await authority.invite(mesh.roster.roster.meshId)); + } + assert.equal(healthy.status()[0]?.roster.roster.revision, 13); + + for (const peer of peers.slice(1, 12)) peer.setReachable(false); + await authority.reconcile(); + + assert.equal(healthy.status()[0]?.roster.roster.revision, 14); + } finally { + await Promise.allSettled(nodes.map((node) => node.close())); + await Promise.allSettled(serving); + await Promise.allSettled(peers.map((peer) => peer.close())); + await rm(root, { recursive: true, force: true }); + } +}); + test('closed Mesh records do not permanently consume membership capacity', async () => { const root = await mkdtemp(join(tmpdir(), 'maka-peer-mesh-capacity-')); const peer = new MemoryPeerNetwork().create('peer-a'); @@ -279,8 +320,10 @@ test('cancels a redemption stalled after the control connection opens', async () class MemoryPeerNetwork { readonly #peers = new Map(); + constructor(private readonly onUnreachable?: () => void) {} + create(peerId: string): MemoryPeerClient { - const peer = new MemoryPeerClient(peerId, this.#peers); + const peer = new MemoryPeerClient(peerId, this.#peers, this.onUnreachable); this.#peers.set(peerId, peer); return peer; } @@ -302,6 +345,7 @@ class MemoryPeerClient implements PeerMeshTransport { constructor( private readonly peerId: string, private readonly peers: ReadonlyMap, + private readonly onUnreachable?: () => void, ) { this.#routeHints = [`/memory/${peerId}`]; } @@ -344,7 +388,10 @@ class MemoryPeerClient implements PeerMeshTransport { readonly peerId: string; }): Promise { const remote = this.peers.get(input.peerId); - if (!remote || !remote.#reachable) throw new Error('Peer is unavailable'); + if (!remote || !remote.#reachable) { + this.onUnreachable?.(); + throw new Error('Peer is unavailable'); + } const [localStream, remoteStream] = memoryStreamPair(this.peerId, input.peerId); if (remote.#failNextResponse) { remote.#failNextResponse = false; diff --git a/packages/runtime-host/src/peer-mesh/node.ts b/packages/runtime-host/src/peer-mesh/node.ts index bef92d5a71..42f87ca0b7 100644 --- a/packages/runtime-host/src/peer-mesh/node.ts +++ b/packages/runtime-host/src/peer-mesh/node.ts @@ -541,6 +541,7 @@ class PeerMeshNodeImpl implements PeerMeshNode { signal: AbortSignal, ): Promise { for (let page = 0; page <= Math.ceil(PEER_MESH_MAX_MEMBERS / ROUTE_PAGE_SIZE); page += 1) { + await this.#refreshLocalRoute(); const stored = this.#store.read(); const state = findMesh(stored.meshes, meshId); const localPeerId = this.#peer.identity().peerId;