From e7a96eb7a5b0eacb2ba26d54e018ab7a45bca242 Mon Sep 17 00:00:00 2001 From: Emilio Jesus Gallego Arias Date: Sun, 23 Aug 2026 14:30:48 -0400 Subject: [PATCH 01/28] fix: fence daemon lifetime to active wrapper requests --- Beam/Broker/Pending.lean | 82 +- Beam/Broker/Protocol.lean | 43 +- Beam/Broker/Server.lean | 142 +++- Beam/Cli/Broker.lean | 125 +-- Beam/Cli/Commands.lean | 153 ++-- Beam/Cli/DaemonManager.lean | 733 +++++++++++++----- Beam/Cli/Feedback.lean | 11 +- Beam/Cli/Info.lean | 10 +- Beam/Cli/Lock.lean | 69 +- Beam/Cli/Project.lean | 7 +- Beam/Cli/RuntimeBundle/Fingerprint.lean | 10 +- Beam/Cli/RuntimeBundle/Metadata.lean | 3 +- Beam/Daemon/Debug.lean | 43 +- Beam/Daemon/Ownership.lean | 103 +++ Beam/Daemon/Paths.lean | 37 + Beam/Daemon/Protocol.lean | 2 +- Beam/System.lean | 94 ++- CHANGELOG.md | 4 + docs/DEVELOPMENT.md | 98 ++- docs/SETUP.md | 5 +- docs/STATUS.md | 28 +- docs/SYNC_AND_DIAGNOSTICS.md | 6 + docs/TESTING.md | 12 +- scripts/install-beam.sh | 18 +- skills/lean-beam/SKILL.md | 13 +- tests/lean/BeamTest/Broker/CliDaemonTest.lean | 249 +++++- tests/lean/BeamTest/Broker/PendingTest.lean | 46 +- tests/lean/BeamTest/Broker/ProtocolTest.lean | 147 ++++ .../BeamTest/Broker/RequestHandleTest.lean | 13 + .../lean/BeamTest/Broker/StreamDedupTest.lean | 1 + tests/test-beam-prune.sh | 27 +- tests/test-beam-wrapper-daemon.sh | 230 +++++- tests/test-beam-wrapper-sandbox.sh | 376 ++++++++- 33 files changed, 2400 insertions(+), 540 deletions(-) create mode 100644 Beam/Daemon/Ownership.lean create mode 100644 Beam/Daemon/Paths.lean diff --git a/Beam/Broker/Pending.lean b/Beam/Broker/Pending.lean index c665646c..48919c04 100644 --- a/Beam/Broker/Pending.lean +++ b/Beam/Broker/Pending.lean @@ -319,13 +319,14 @@ def propagateCancellation end PendingRequestStore structure ActiveRequest where - clientRequestId : String + clientRequestId? : Option String token : Nat cancelRef : IO.Ref Bool private structure ActiveRequestRegistryState where nextToken : Nat := 1 requests : Std.TreeMap String ActiveRequest := {} + anonymousRequests : Std.TreeMap Nat ActiveRequest := {} structure ActiveRequestRegistry where private mutex : Std.Mutex ActiveRequestRegistryState @@ -337,26 +338,31 @@ def create : BaseIO ActiveRequestRegistry := do def register (registry : ActiveRequestRegistry) - (clientRequestId? : Option String) : IO (Except BrokerFailure (Option ActiveRequest)) := do - match clientRequestId? with - | none => - pure (.ok none) - | some clientRequestId => - let cancelRef ← IO.mkRef false - registry.mutex.atomically do - let state ← get + (clientRequestId? : Option String) : IO (Except BrokerFailure ActiveRequest) := do + let cancelRef ← IO.mkRef false + registry.mutex.atomically do + let state ← get + match clientRequestId? with + | none => + let active : ActiveRequest := { clientRequestId?, token := state.nextToken, cancelRef } + set { state with + nextToken := state.nextToken + 1 + anonymousRequests := state.anonymousRequests.insert active.token active + } + pure <| .ok active + | some clientRequestId => if state.requests.contains clientRequestId then pure <| .error { code := .invalidParams message := s!"clientRequestId '{clientRequestId}' is already active" } else - let active : ActiveRequest := { clientRequestId, token := state.nextToken, cancelRef } - set ({ + let active : ActiveRequest := { clientRequestId?, token := state.nextToken, cancelRef } + set { state with nextToken := state.nextToken + 1 requests := state.requests.insert clientRequestId active - } : ActiveRequestRegistryState) - pure <| .ok <| some active + } + pure <| .ok active def unregister (registry : ActiveRequestRegistry) @@ -366,12 +372,42 @@ def unregister | some active => registry.mutex.atomically do let state ← get - match state.requests.get? active.clientRequestId with - | some current => - if current.token == active.token then - set { state with requests := state.requests.erase active.clientRequestId } + match active.clientRequestId? with + | some clientRequestId => + match state.requests.get? clientRequestId with + | some current => + if current.token == active.token then + set { state with requests := state.requests.erase clientRequestId } + | none => pure () | none => - pure () + match state.anonymousRequests.get? active.token with + | some current => + if current.token == active.token then + set { state with anonymousRequests := state.anonymousRequests.erase active.token } + | none => pure () + +def count (registry : ActiveRequestRegistry) : IO Nat := do + registry.mutex.atomically do + let state ← get + pure (state.requests.size + state.anonymousRequests.size) + +def countExcluding + (registry : ActiveRequestRegistry) + (excluded : ActiveRequest) : IO Nat := do + registry.mutex.atomically do + let state ← get + let total := state.requests.size + state.anonymousRequests.size + let exactAdmissionPresent : Bool := + match excluded.clientRequestId? with + | some clientRequestId => + match state.requests.get? clientRequestId with + | some current => current.token == excluded.token + | none => false + | none => + match state.anonymousRequests.get? excluded.token with + | some current => current.token == excluded.token + | none => false + pure <| if exactAdmissionPresent then total - 1 else total def markCancelled (registry : ActiveRequestRegistry) @@ -389,9 +425,13 @@ def markCancelledActive (registry : ActiveRequestRegistry) (active : ActiveRequest) : IO (Option ActiveRequest) := do registry.mutex.atomically do - match (← get).requests.get? active.clientRequestId with - | none => - pure none + let state ← get + let current? := + match active.clientRequestId? with + | some clientRequestId => state.requests.get? clientRequestId + | none => state.anonymousRequests.get? active.token + match current? with + | none => pure none | some current => if current.token == active.token then current.cancelRef.set true diff --git a/Beam/Broker/Protocol.lean b/Beam/Broker/Protocol.lean index 8a0cb70a..1572fff8 100644 --- a/Beam/Broker/Protocol.lean +++ b/Beam/Broker/Protocol.lean @@ -182,6 +182,30 @@ structure Handle where raw : Json deriving Inhabited, FromJson, ToJson +/-- +Internal wrapper-to-daemon fencing identity. It is attached by `lean-beam`, not supplied by normal +broker or MCP clients. +-/ +structure WrapperLeaseContext where + daemonId : String + leaseFile : String + deriving Inhabited, ToJson, BEq, Repr + +instance : FromJson WrapperLeaseContext where + fromJson? json := do + match json with + | .obj fields => + let allowed := #["daemonId", "leaseFile"] + let unexpected := fields.foldl (init := #[]) fun unexpected field _ => + if allowed.contains field then unexpected else unexpected.push field + unless unexpected.isEmpty do + throw s!"wrapper lease accepts no undeclared fields: {String.intercalate ", " unexpected.toList}" + | other => throw s!"wrapper lease must be an object, got {other.compress}" + pure { + daemonId := ← json.getObjValAs? String "daemonId" + leaseFile := ← json.getObjValAs? String "leaseFile" + } + /-- Select which user-facing Lean diagnostic severities a request may display. -/ inductive DiagnosticScope where | errors @@ -207,6 +231,7 @@ structure Request where workspaceId? : Option WorkspaceId := none workspaceMode? : Option Beam.Workspace.InitMode := none clientRequestId? : Option String := none + wrapperLease? : Option WrapperLeaseContext := none cancelRequestId? : Option String := none root? : Option String := none path? : Option String := none @@ -250,8 +275,16 @@ def Op.workspaceScope : Op → WorkspaceScope | .codeActionResolve | .saveOlean | .goals | .todo | .runWith | .release | .initWorkspace | .dropWorkspace => .required +/-- Whether an operation participates in wrapper lease fencing and active-request tracking. -/ +def Op.acceptsWrapperLease : Op → Bool + | .cancel | .shutdown => false + | .ensure | .openDocs | .updateFile | .syncFile | .refreshFile | .close | .runAt | .hover + | .signatureHelp | .definition | .references | .documentSymbols | .workspaceSymbols + | .codeActionResolve | .saveOlean | .goals | .todo | .runWith | .release | .initWorkspace + | .listWorkspaces | .dropWorkspace | .stats | .resetStats => true + private def Op.optionalRequestFields (op : Op) : Array String := - #["clientRequestId"] ++ + #["clientRequestId", "wrapperLease"] ++ (match op.workspaceScope with | .none => #[] | .optional | .required => #["workspaceId"]) ++ @@ -306,6 +339,7 @@ private def Request.optionalJsonFields (req : Request) : List (String × Json) : optionalJsonField "workspaceId" req.workspaceId? ++ optionalJsonField "workspaceMode" req.workspaceMode? ++ optionalJsonField "clientRequestId" req.clientRequestId? ++ + optionalJsonField "wrapperLease" req.wrapperLease? ++ optionalJsonField "cancelRequestId" req.cancelRequestId? ++ optionalJsonField "root" req.root? ++ optionalJsonField "path" req.path? ++ @@ -360,6 +394,10 @@ def Request.validateFields (req : Request) : Except String Unit := do throw s!"broker op '{req.op.key}' accepts no unrelated fields: {String.intercalate ", " unexpected.toList}" if !req.op.usesBackend && req.backend != .lean then throw s!"broker op '{req.op.key}' does not select a backend" + if req.wrapperLease?.isSome && req.clientRequestId?.isNone then + throw "broker requests carrying 'wrapperLease' require 'clientRequestId'" + if req.wrapperLease?.isSome && !req.op.acceptsWrapperLease then + throw s!"broker op '{req.op.key}' does not accept 'wrapperLease'" if (req.op == .stats || req.op == .openDocs) && req.root?.isSome && req.workspaceId?.isNone then throw s!"broker op '{req.op.key}' requires 'workspaceId' when 'root' is present" @@ -383,6 +421,7 @@ instance : FromJson Request where let workspaceId? ← optionalField? (α := WorkspaceId) j "workspaceId" let workspaceMode? ← optionalField? (α := Beam.Workspace.InitMode) j "workspaceMode" let clientRequestId? ← optionalField? (α := String) j "clientRequestId" + let wrapperLease? ← optionalField? (α := WrapperLeaseContext) j "wrapperLease" let cancelRequestId? ← optionalField? (α := String) j "cancelRequestId" let root? ← optionalField? (α := String) j "root" let path? ← optionalField? (α := String) j "path" @@ -410,7 +449,7 @@ instance : FromJson Request where let handle? ← optionalField? (α := Handle) j "handle" let codeAction? ← optionalField? (α := Lsp.CodeAction) j "codeAction" let request : Request := { - op, backend, workspaceId?, workspaceMode?, clientRequestId?, cancelRequestId?, + op, backend, workspaceId?, workspaceMode?, clientRequestId?, wrapperLease?, cancelRequestId?, root?, path?, version?, line?, character?, endLine?, endCharacter?, text?, query?, includeDeclaration?, kinds?, suggest?, storeHandle?, linear?, mode?, compact?, ppFormat?, diagnosticScope?, diagnosticsInResult?, diff --git a/Beam/Broker/Server.lean b/Beam/Broker/Server.lean index 150276e7..e5a7f1b3 100644 --- a/Beam/Broker/Server.lean +++ b/Beam/Broker/Server.lean @@ -24,6 +24,8 @@ import Beam.Broker.Lean import Beam.Broker.LakeSave import Beam.Broker.Readiness import Beam.Broker.SyncResult +import Beam.Daemon.Ownership +import Beam.Daemon.Paths import Beam.LSP.Save import Beam.Path import Std.Sync.Mutex @@ -35,6 +37,8 @@ open IO.FS.Stream namespace Beam.Broker +open Beam.Daemon.Ownership + abbrev brokerStdio : IO.Process.StdioConfig where stdin := .piped stdout := .piped @@ -888,6 +892,8 @@ structure ServerRuntime where endpoint : Transport.Endpoint stop : IO.Ref Bool activeRequests : ActiveRequestRegistry + root : System.FilePath + daemonId? : Option String := none /-- A cancellation capability bound to one active broker request admission. @@ -906,10 +912,20 @@ def ServerRuntime.withState (server : ServerRuntime) (act : M α) : IO α := do set state pure a +private def ServerRuntime.statsResponse + (server : ServerRuntime) + (currentRequest : ActiveRequest) + (workspaceId? : Option WorkspaceId := none) : IO Response := do + let activeRequestCount ← + ActiveRequestRegistry.countExcluding server.activeRequests currentRequest + let payload ← server.withState <| statsPayload workspaceId? + pure <| Response.success <| payload.setObjVal! "activeRequestCount" (toJson activeRequestCount) + def ServerRuntime.create (config : BrokerConfig) (workspaceId : WorkspaceId) - (endpoint : Transport.Endpoint := .tcp 0) : IO ServerRuntime := do + (endpoint : Transport.Endpoint := .tcp 0) + (daemonId? : Option String := none) : IO ServerRuntime := do unless validWorkspaceId workspaceId do throw <| IO.userError "workspace id must be non-empty" let startMonoNanos ← IO.monoNanosNow @@ -919,6 +935,8 @@ def ServerRuntime.create endpoint := endpoint stop := ← IO.mkRef false activeRequests := ← ActiveRequestRegistry.create + root := config.root + daemonId? } private def brokerConfigSame (left right : BrokerConfig) : Bool := @@ -1031,7 +1049,7 @@ def ServerRuntime.dropWorkspace invalidatedHandles := true } : Beam.Workspace.DropResult) -private def requestTracksActiveRequest : Op → Bool +private def requestRecordsMetrics : Op → Bool | .cancel | .stats | .resetStats | .shutdown | .openDocs | .listWorkspaces => false | _ => true @@ -1040,7 +1058,7 @@ private def recordDispatchMetrics (req : Request) (resp : Response) (startedAt : Nat) : IO Unit := do - if requestTracksActiveRequest req.op then + if requestRecordsMetrics req.op then let finishedAt ← IO.monoNanosNow let latencyMs := (finishedAt - startedAt) / 1000000 if let some workspaceId := req.resolvedWorkspaceId? then @@ -1081,6 +1099,72 @@ def RequestHandle.cancel (handle : RequestHandle) : IO Bool := do cancelRegisteredRequest handle.runtime <| ActiveRequestRegistry.markCancelledActive handle.runtime.activeRequests active +private inductive WrapperLeaseInactiveReason where + | generationMismatch + | invalidFileName + | revoked + | missing + | malformed + | invalidPid + | unreadable + +private def WrapperLeaseInactiveReason.key : WrapperLeaseInactiveReason → String + | .generationMismatch => "generationMismatch" + | .invalidFileName => "invalidFileName" + | .revoked => "revoked" + | .missing => "missing" + | .malformed => "malformed" + | .invalidPid => "invalidPid" + | .unreadable => "unreadable" + +private inductive WrapperLeaseValidation where + | active + | inactive (reason : WrapperLeaseInactiveReason) + +/-- +Validate the wrapper's filesystem fence after active-request registration. Retirement first writes +the revocation tombstone and only then observes the active-request count, so a fenced request that +passes this check is necessarily visible to the retiring owner before broker work begins. + +Unfenced broker clients remain valid; their lifecycle must be owned independently or protected by +a foreground wrapper owner such as `lean-beam ensure --hold`. +-/ +private def ServerRuntime.validateWrapperLease + (server : ServerRuntime) + (req : Request) : IO WrapperLeaseValidation := do + if !req.op.acceptsWrapperLease then + return .active + let some lease := req.wrapperLease? + | return .active + try + unless server.daemonId? == some lease.daemonId do + return .inactive .generationMismatch + unless validWrapperLeaseFileName lease.leaseFile do + return .inactive .invalidFileName + let leasePath := (← Beam.Daemon.wrapperLeaseDir server.root) / lease.leaseFile + let revocationPath := wrapperLeaseRevocationPath leasePath + if ← revocationPath.pathExists then + return .inactive .revoked + unless ← leasePath.pathExists do + return .inactive .missing + let text ← IO.FS.readFile leasePath + let json ← + match Json.parse text with + | .ok json => pure json + | .error _ => return .inactive .malformed + let metadata : WrapperLeaseMetadata ← + match fromJson? (α := WrapperLeaseMetadata) json with + | .ok metadata => pure metadata + | .error _ => return .inactive .malformed + if metadata.pid == 0 then + return .inactive .invalidPid + if ← revocationPath.pathExists then + pure (.inactive .revoked) + else + pure .active + catch _ => + pure (.inactive .unreadable) + private def propagatePendingCancellation (session : Session) (cancelRef? : Option (IO.Ref Bool)) : IO Unit := do @@ -2114,9 +2198,10 @@ private def initWorkspaceConfigFromRequest private def handleRequestIO (server : ServerRuntime) (req : Request) - (cancelRef? : Option (IO.Ref Bool) := none) + (activeRequest? : Option ActiveRequest := none) (emitProgress? : Option (SyncFileProgress → IO Unit) := none) (emitDiagnostic? : Option (StreamDiagnostic → IO Unit) := none) : IO (Response × Bool) := do + let cancelRef? := activeRequest?.map (·.cancelRef) match req.op with | .shutdown => let resp ← server.withState do @@ -2126,14 +2211,15 @@ private def handleRequestIO pure <| Response.success (Json.mkObj [("shutdown", toJson true)]) pure (resp, true) | .stats => + let some currentRequest := activeRequest? + | unreachable! match req.workspaceId? with - | none => pure (Response.success (← server.withState statsPayload), false) + | none => pure (← server.statsResponse currentRequest, false) | some _ => match ← validateRequestWorkspace server req with | .error failure => pure (failure.toResponse, false) | .ok workspaceReq => - pure (Response.success - (← server.withState <| statsPayload (some workspaceReq.workspaceId)), false) + pure (← server.statsResponse currentRequest (some workspaceReq.workspaceId), false) | .listWorkspaces => let payload ← server.withState do pure <| workspaceListPayload (← get) @@ -2244,9 +2330,9 @@ private def ServerRuntime.withRequestAdmission | .ok () => pure () try let active? ← - if requestTracksActiveRequest req.op then + if req.op.acceptsWrapperLease then match ← ActiveRequestRegistry.register server.activeRequests req.clientRequestId? with - | .ok active? => pure active? + | .ok active => pure (some active) | .error failure => let resp := BrokerFailure.toResponse failure recordDispatchMetrics server req resp startedAt @@ -2254,6 +2340,19 @@ private def ServerRuntime.withRequestAdmission else pure none try + match ← server.validateWrapperLease req with + | .inactive reason => + let resp := BrokerFailure.toResponse { + code := .requestCancelled + message := "wrapper daemon-lifetime lease is no longer active" + data? := some <| Json.mkObj [ + ("reason", toJson "wrapperLeaseInactive"), + ("leaseState", toJson reason.key) + ] + } + recordDispatchMetrics server req resp startedAt + return (resp, false) + | .active => pure () let handle : RequestHandle := { runtime := server, active? } let (resp, shouldStop) ← act handle traceBroker @@ -2293,7 +2392,7 @@ def ServerRuntime.dispatchRequestWithHandle }, false ) - handleRequestIO server req (handle.active?.map (·.cancelRef)) emitProgress? emitDiagnostic? + handleRequestIO server req handle.active? emitProgress? emitDiagnostic? def ServerRuntime.dispatchRequest (server : ServerRuntime) @@ -2327,6 +2426,17 @@ private partial def watchRoot (server : ServerRuntime) (root : System.FilePath) IO.sleep rootWatchPollMs watchRoot server root +private def watchClientDisconnect + (client : Transport.Connection) + (handle : RequestHandle) : IO Unit := do + try + -- The daemon transport accepts one request per connection. A second receive therefore blocks + -- until the client closes or dies; either outcome should cancel an unfinished admission. + discard <| Transport.recvMsg client + catch _ => + pure () + discard <| handle.cancel + private def handleClient (server : ServerRuntime) (client : Transport.Connection) : IO Unit := do let clientRequestIdRef ← IO.mkRef (none : Option String) let terminalSentRef ← IO.mkRef false @@ -2360,7 +2470,9 @@ private def handleClient (server : ServerRuntime) (client : Transport.Connection let emitDiagnostic : StreamDiagnostic → IO Unit := fun diagnostic => Transport.sendMsg client (toJson (StreamMessage.diagnostic req.clientRequestId? diagnostic)).compress - let (resp, shouldStop) ← server.dispatchRequest req (some emitProgress) (some emitDiagnostic) + let (resp, shouldStop) ← server.dispatchRequestWithHandle req (fun handle => do + let _ ← IO.asTask (prio := Task.Priority.dedicated) <| watchClientDisconnect client handle + pure true) (some emitProgress) (some emitDiagnostic) sendResponse req.clientRequestId? resp if shouldStop then requestStop server @@ -2394,6 +2506,7 @@ private structure CliOptions where endpoint : Transport.Endpoint := .tcp 8765 root? : Option String := none workspaceId? : Option WorkspaceId := none + daemonId? : Option String := none leanCmd? : Option String := none leanPlugin? : Option String := none rocqCmd? : Option String := none @@ -2419,6 +2532,8 @@ private partial def parseCliOptions (opts : CliOptions) : List String → Except parseCliOptions { opts with root? := some root } rest | "--workspace-id" :: workspaceId :: rest => parseCliOptions { opts with workspaceId? := some workspaceId } rest + | "--daemon-id" :: daemonId :: rest => + parseCliOptions { opts with daemonId? := some daemonId } rest | "--lean-cmd" :: leanCmd :: rest => parseCliOptions { opts with leanCmd? := some leanCmd } rest | "--lean-plugin" :: leanPlugin :: rest => @@ -2436,6 +2551,9 @@ def main (args : List String) : IO Unit := do | throw <| IO.userError "missing Beam daemon --workspace-id ID" unless validWorkspaceId workspaceId do throw <| IO.userError "workspace id must be non-empty" + if let some daemonId := opts.daemonId? then + if daemonId.isEmpty then + throw <| IO.userError "daemon id must be non-empty" let root ← Beam.resolveExistingPath <| System.FilePath.mk root let leanPlugin? ← opts.leanPlugin?.mapM (fun path => Beam.resolveExistingPath <| System.FilePath.mk path) let config : BrokerConfig := { @@ -2451,6 +2569,8 @@ def main (args : List String) : IO Unit := do endpoint := opts.endpoint stop := ← IO.mkRef false activeRequests := ← ActiveRequestRegistry.create + root + daemonId? := opts.daemonId? } let rootWatcher ← IO.asTask (prio := Task.Priority.dedicated) <| watchRoot runtime root try diff --git a/Beam/Cli/Broker.lean b/Beam/Cli/Broker.lean index cd5bde41..68a52a5d 100644 --- a/Beam/Cli/Broker.lean +++ b/Beam/Cli/Broker.lean @@ -33,19 +33,6 @@ def withBrokerErrorContext {α} (root : System.FilePath) (action : IO α) : IO catch e => throw <| IO.userError (← daemonFailureMessage root e.toString) -def callBroker (root : System.FilePath) (endpoint : Transport.Endpoint) (req : Request) : IO Unit := - withBrokerErrorContext root do - let req ← withEnvClientRequestId (inProjectDaemonWorkspace req) - let resp ← sendRequest endpoint req - printResponse resp req.clientRequestId? - failOnError resp - -def callBrokerQuiet (root : System.FilePath) (endpoint : Transport.Endpoint) (req : Request) : IO Unit := - withBrokerErrorContext root do - let req ← withEnvClientRequestId (inProjectDaemonWorkspace req) - let resp ← sendRequest endpoint req - failOnError resp - structure BrokerWaitSpec where action : String startMsg : String @@ -76,6 +63,14 @@ private structure WrapperBrokerRequest where request : Request visibleClientRequestId? : Option String +private def attachProjectDaemonLease + (client : ProjectDaemonClient) + (req : Request) : Request := + if req.op.acceptsWrapperLease then + { req with wrapperLease? := some client.wrapperLease } + else + req + private def mkWrapperClientRequestId (req : Request) : IO String := do let pid ← IO.Process.getPID let stamp ← IO.monoNanosNow @@ -96,6 +91,11 @@ private def withWrapperClientRequestId (req : Request) : IO WrapperBrokerRequest visibleClientRequestId? := none } +private def prepareWrapperBrokerRequest + (client : ProjectDaemonClient) + (req : Request) : IO WrapperBrokerRequest := + withWrapperClientRequestId <| attachProjectDaemonLease client (inProjectDaemonWorkspace req) + private def mkInterruptWatcher? (clientRequestId? : Option String) : IO (Option InterruptWatcher) := do match clientRequestId? with | none => pure none @@ -130,44 +130,42 @@ private def awaitBrokerResponse (endpoint : Transport.Endpoint) (req : Request) (visibleClientRequestId? : Option String) - (spec : BrokerWaitSpec) - (interruptWatcher? : Option InterruptWatcher) - (showProgress : Bool) : IO Response := do + (progressSpec? : Option BrokerWaitSpec) + (interruptWatcher? : Option InterruptWatcher) : IO Response := do let mut interruptObserved := false let mut cancelAcknowledged := false let emit := fun msg => IO.eprintln <| annotateRunatMessage visibleClientRequestId? msg - if showProgress then + if let some spec := progressSpec? then emit spec.startMsg let mut waitedMs := 0 try while !(← IO.hasFinished task) do - match interruptWatcher? with - | some watcher => - if (← watcher.interrupted) then - if !interruptObserved then - interruptObserved := true - emit "beam: requesting broker cancellation" - if !cancelAcknowledged then - -- SIGINT can arrive after the wrapper starts the request task but before the broker - -- has registered the client request id as active. Retry until the broker acknowledges - -- cancellation or the original request finishes. - match ← sendBrokerCancellation endpoint req with - | some true => cancelAcknowledged := true - | some false | none => pure () - else - pure () - | none => - pure () + let signalInterrupted ← + match interruptWatcher? with + | some watcher => watcher.interrupted + | none => pure false + if signalInterrupted || (← IO.checkCanceled) then + if !interruptObserved then + interruptObserved := true + emit "beam: requesting broker cancellation" + if !cancelAcknowledged then + -- SIGINT can arrive after the wrapper starts the request task but before the broker + -- has registered the client request id as active. Retry until the broker acknowledges + -- cancellation or the original request finishes. + match ← sendBrokerCancellation endpoint req with + | some true => cancelAcknowledged := true + | some false | none => pure () IO.sleep 500 if !(← IO.hasFinished task) then waitedMs := waitedMs + 500 - if showProgress && waitedMs % 1000 == 0 then - emit <| spec.stillWaitingMsg (waitedMs / 1000) + if waitedMs % 1000 == 0 then + if let some spec := progressSpec? then + emit <| spec.stillWaitingMsg (waitedMs / 1000) let resp ← match (← IO.wait task) with | .ok resp => pure resp | .error err => throw err - if showProgress then + if let some spec := progressSpec? then emit <| spec.completeMsg resp pure resp finally @@ -179,8 +177,7 @@ private def awaitBrokerResponseWithInterrupts (endpoint : Transport.Endpoint) (req : Request) (visibleClientRequestId? : Option String) - (spec : BrokerWaitSpec) - (showProgress : Bool) + (progressSpec? : Option BrokerWaitSpec) (action : IO Response) : IO Response := do -- Wrapper calls synthesize a broker clientRequestId when the user did not provide one. That id -- gives SIGINT cancellation a stable broker key but is kept out of the CLI's public output. @@ -193,7 +190,45 @@ private def awaitBrokerResponseWithInterrupts | some watcher => watcher.stop | none => pure () throw e - awaitBrokerResponse task endpoint req visibleClientRequestId? spec interruptWatcher? showProgress + awaitBrokerResponse task endpoint req visibleClientRequestId? progressSpec? interruptWatcher? + +private structure WrapperBrokerResponse where + response : Response + visibleClientRequestId? : Option String + +private def requestBrokerResponse + (root : System.FilePath) + (client : ProjectDaemonClient) + (req : Request) : IO WrapperBrokerResponse := + withBrokerErrorContext root do + let wrapperReq ← prepareWrapperBrokerRequest client req + let req := wrapperReq.request + let response ← awaitBrokerResponseWithInterrupts client.endpoint req + wrapperReq.visibleClientRequestId? none <| + sendRequest client.endpoint req + pure { response, visibleClientRequestId? := wrapperReq.visibleClientRequestId? } + +/-- Send one lease-fenced wrapper request without printing or interpreting its response. -/ +def requestBroker + (root : System.FilePath) + (client : ProjectDaemonClient) + (req : Request) : IO Response := do + pure (← requestBrokerResponse root client req).response + +def callBroker + (root : System.FilePath) + (client : ProjectDaemonClient) + (req : Request) : IO Unit := do + let result ← requestBrokerResponse root client req + printResponse result.response result.visibleClientRequestId? + failOnError result.response + +def callBrokerQuiet + (root : System.FilePath) + (client : ProjectDaemonClient) + (req : Request) : IO Unit := do + let resp ← requestBroker root client req + failOnError resp private def syncReadinessSuffix (result : SyncFileResult) : String := let readiness := result.readiness @@ -392,11 +427,11 @@ def leanSaveWaitSpec def callBrokerWithProgress (root : System.FilePath) - (endpoint : Transport.Endpoint) + (client : ProjectDaemonClient) (req : Request) (spec : BrokerWaitSpec) : IO Unit := withBrokerErrorContext root do - let wrapperReq ← withWrapperClientRequestId (inProjectDaemonWorkspace req) + let wrapperReq ← prepareWrapperBrokerRequest client req let req := wrapperReq.request let visibleClientRequestId? := wrapperReq.visibleClientRequestId? let showProgress ← progressEnabled @@ -407,8 +442,10 @@ def callBrokerWithProgress onDiagnostic := fun _ diagnostic => IO.eprintln <| annotateRunatMessage visibleClientRequestId? (formatStreamDiagnostic diagnostic) } - let resp ← awaitBrokerResponseWithInterrupts endpoint req visibleClientRequestId? spec showProgress <| - sendRequestWithCallbacks endpoint req callbacks + let progressSpec? := if showProgress then some spec else none + let resp ← awaitBrokerResponseWithInterrupts client.endpoint req visibleClientRequestId? + progressSpec? <| + sendRequestWithCallbacks client.endpoint req callbacks match responseErrorSummary? spec.action spec.failureBoundary resp with | some note => IO.eprintln <| annotateRunatMessage visibleClientRequestId? note diff --git a/Beam/Cli/Commands.lean b/Beam/Cli/Commands.lean index a73c6743..29ffaece 100644 --- a/Beam/Cli/Commands.lean +++ b/Beam/Cli/Commands.lean @@ -12,7 +12,6 @@ import Beam.Cli.Feedback import Beam.Cli.Info import Beam.Cli.InstallPrune import Beam.Cli.LeanOperation -import Beam.Cli.Lock import Beam.Cli.Project import Beam.Cli.RuntimeBundle import Beam.Cli.Usage @@ -31,9 +30,9 @@ private def wrapperDisplayAction (fallback : String) : IO String := do private def updateVersionForRocqGoals (root : System.FilePath) - (endpoint : Transport.Endpoint) + (client : ProjectDaemonClient) (path : String) : IO Nat := do - let resp ← sendRequest endpoint { + let resp ← requestBroker root client { op := .updateFile backend := .rocq workspaceId? := some projectDaemonWorkspaceId @@ -52,16 +51,15 @@ private def runLeanRunAt (textArgs : List String) (storeHandle : Bool := false) : IO Unit := do let root ← projectRoot opts .lean - let daemon ← ensureProjectDaemon home root .lean opts let version ← parseNatArg "version" versionText let line ← parseNatArg "line" lineText let character ← parseNatArg "character" characterText let parsedText ← parseTextArg s!"{action} " textArgs - withWrapperLease root daemon.startedNew do + withProjectDaemon home root .lean opts fun client => do let req ← withEnvClientRequestId <| leanRunAtRequest root path version line character parsedText.text? (storeHandle := storeHandle) maybeEmitTextDebug req.clientRequestId? action parsedText.source parsedText.text? - callBrokerWithProgress root daemon.endpoint req (leanRunAtWaitSpec action path line character) + callBrokerWithProgress root client req (leanRunAtWaitSpec action path line character) private def runLeanRunWith (home : System.FilePath) @@ -80,14 +78,13 @@ private def runLeanRunWith "cannot read both handle json and continuation text from stdin; pass the handle inline, use --handle-file, or use --text-file for the text" ] let root ← projectRoot opts .lean - let daemon ← ensureProjectDaemon home root .lean opts let (handle, textArgs) ← parseHandleInput s!"{action} " args let parsedText ← parseTextArg s!"{action} >" textArgs let req ← withEnvClientRequestId <| leanRunWithRequest root path handle parsedText.text? (linear := linear) maybeEmitTextDebug req.clientRequestId? action parsedText.source parsedText.text? - withWrapperLease root daemon.startedNew do - callBrokerWithProgress root daemon.endpoint req (leanRunWithWaitSpec path (linear := linear)) + withProjectDaemon home root .lean opts fun client => + callBrokerWithProgress root client req (leanRunWithWaitSpec path (linear := linear)) private def runLeanRelease (home : System.FilePath) @@ -96,12 +93,11 @@ private def runLeanRelease (path : String) (args : List String) : IO Unit := do let root ← projectRoot opts .lean - let daemon ← ensureProjectDaemon home root .lean opts let (handle, extra) ← parseHandleInput s!"{action} " args unless extra.isEmpty do throw <| IO.userError (handleArgUsage s!"{action} ") - withWrapperLease root daemon.startedNew do - callBroker root daemon.endpoint <| leanReleaseRequest root path handle + withProjectDaemon home root .lean opts fun client => + callBroker root client <| leanReleaseRequest root path handle private def shutdownProjectDaemon (opts : CliOptions) : IO Unit := do let root ← projectRootAny opts @@ -111,10 +107,7 @@ private def shutdownProjectDaemon (opts : CliOptions) : IO Unit := do if let some endpoint := Beam.Daemon.registryEndpoint? entry then let resp ← sendRequest endpoint { op := .shutdown } printResponse resp - waitForPidGone entry.pid - if ← pidAlive entry.pid then - killPid entry.pid - waitForPidGone entry.pid + finishRegistryDaemonShutdown entry removeRegistry root else stopRegisteredDaemon root @@ -139,9 +132,12 @@ private def runThenHoldUntilInterrupted (act : IO Unit) : IO Unit := do pure () try act - match ← IO.wait task with - | .ok () => pure () - | .error err => throw err + while !(← IO.hasFinished task) && !(← IO.checkCanceled) do + IO.sleep 50 + if ← IO.hasFinished task then + match ← IO.wait task with + | .ok () => pure () + | .error err => throw err finally Std.Internal.UV.Signal.stop signal @@ -151,17 +147,16 @@ private def ensureBackend (backend : Backend) (hold : Bool := false) : IO Unit := do let root ← projectRoot opts backend - let daemon ← ensureProjectDaemon home root backend opts - withWrapperLease root daemon.startedNew do + withProjectDaemon home root backend opts fun client => if hold then runThenHoldUntilInterrupted do - callBroker root daemon.endpoint { + callBroker root client { op := .ensure, backend := backend, root? := some root.toString } (← IO.getStdout).flush IO.eprintln "beam: holding ensured daemon; interrupt this wrapper process when finished" else - callBroker root daemon.endpoint { op := .ensure, backend := backend, root? := some root.toString } + callBroker root client { op := .ensure, backend := backend, root? := some root.toString } def runCommand (home : System.FilePath) (opts : CliOptions) : IO Unit := do match opts.args with @@ -209,85 +204,77 @@ def runCommand (home : System.FilePath) (opts : CliOptions) : IO Unit := do (storeHandle := true) | "lean-hover" :: path :: versionText :: line :: character :: [] => let root ← projectRoot opts .lean - let daemon ← ensureProjectDaemon home root .lean opts let version ← parseNatArg "version" versionText let line ← parseNatArg "line" line let character ← parseNatArg "character" character let action ← wrapperDisplayAction "lean-hover" - withWrapperLease root daemon.startedNew do - callBrokerWithProgress root daemon.endpoint + withProjectDaemon home root .lean opts fun client => + callBrokerWithProgress root client (leanHoverRequest root path version line character) (leanHoverWaitSpec path line character action) | "lean-signature-help" :: path :: versionText :: line :: character :: [] => let root ← projectRoot opts .lean - let daemon ← ensureProjectDaemon home root .lean opts let version ← parseNatArg "version" versionText let line ← parseNatArg "line" line let character ← parseNatArg "character" character let action ← wrapperDisplayAction "lean-signature-help" - withWrapperLease root daemon.startedNew do - callBrokerWithProgress root daemon.endpoint + withProjectDaemon home root .lean opts fun client => + callBrokerWithProgress root client (leanSignatureHelpRequest root path version line character) (leanSignatureHelpWaitSpec path line character action) | "lean-definition" :: path :: versionText :: line :: character :: [] => let root ← projectRoot opts .lean - let daemon ← ensureProjectDaemon home root .lean opts let version ← parseNatArg "version" versionText let line ← parseNatArg "line" line let character ← parseNatArg "character" character let action ← wrapperDisplayAction "lean-definition" - withWrapperLease root daemon.startedNew do - callBrokerWithProgress root daemon.endpoint + withProjectDaemon home root .lean opts fun client => + callBrokerWithProgress root client (leanDefinitionRequest root path version line character) (leanDefinitionWaitSpec path line character action) | "lean-references" :: path :: versionText :: line :: character :: extra => let root ← projectRoot opts .lean - let daemon ← ensureProjectDaemon home root .lean opts let version ← parseNatArg "version" versionText let line ← parseNatArg "line" line let character ← parseNatArg "character" character let includeDeclaration ← parseLeanReferencesArgs extra let action ← wrapperDisplayAction "lean-references" - withWrapperLease root daemon.startedNew do - callBrokerWithProgress root daemon.endpoint + withProjectDaemon home root .lean opts fun client => + callBrokerWithProgress root client (leanReferencesRequest root path version line character includeDeclaration) (leanReferencesWaitSpec path line character action) | "lean-document-symbols" :: path :: versionText :: [] => let root ← projectRoot opts .lean - let daemon ← ensureProjectDaemon home root .lean opts let version ← parseNatArg "version" versionText let action ← wrapperDisplayAction "lean-document-symbols" - withWrapperLease root daemon.startedNew do - callBrokerWithProgress root daemon.endpoint + withProjectDaemon home root .lean opts fun client => + callBrokerWithProgress root client (leanDocumentSymbolsRequest root path version) (leanDocumentSymbolsWaitSpec path action) | "lean-workspace-symbols" :: queryParts => let root ← projectRoot opts .lean - let daemon ← ensureProjectDaemon home root .lean opts let query ← match joinTextArgs queryParts with | some query => pure query | none => throw <| IO.userError "usage: beam [--root PATH] [--port N] lean-workspace-symbols " let action ← wrapperDisplayAction "lean-workspace-symbols" - withWrapperLease root daemon.startedNew do - callBrokerWithProgress root daemon.endpoint + withProjectDaemon home root .lean opts fun client => + callBrokerWithProgress root client (leanWorkspaceSymbolsRequest root query) (leanWorkspaceSymbolsWaitSpec query action) | "lean-goals" :: modeText :: path :: versionText :: line :: character :: [] => let root ← projectRoot opts .lean - let daemon ← ensureProjectDaemon home root .lean opts let mode ← parseLeanGoalsModeArg modeText let version ← parseNatArg "version" versionText let line ← parseNatArg "line" line let character ← parseNatArg "character" character let action ← wrapperDisplayAction "lean-goals" - withWrapperLease root daemon.startedNew do - callBrokerWithProgress root daemon.endpoint + withProjectDaemon home root .lean opts fun client => + callBrokerWithProgress root client (leanGoalsRequest root path version line character mode) (leanGoalsWaitSpec path line character mode (some action)) | "lean-todo" :: path :: versionText :: startLine :: startCharacter :: endLine :: endCharacter :: extra => do let root ← projectRoot opts .lean - let daemon ← ensureProjectDaemon home root .lean opts let version ← parseNatArg "version" versionText let startLine ← parseNatArg "startLine" startLine let startCharacter ← parseNatArg "startCharacter" startCharacter @@ -295,8 +282,8 @@ def runCommand (home : System.FilePath) (opts : CliOptions) : IO Unit := do let endCharacter ← parseNatArg "endCharacter" endCharacter let (kinds?, suggest?) ← parseLeanTodoArgs extra let action ← wrapperDisplayAction "lean-todo" - withWrapperLease root daemon.startedNew do - callBrokerWithProgress root daemon.endpoint + withProjectDaemon home root .lean opts fun client => + callBrokerWithProgress root client (leanTodoRequest root path version startLine startCharacter endLine endCharacter kinds? suggest?) (leanTodoWaitSpec path startLine startCharacter endLine endCharacter action) | "lean-run-with" :: path :: args => @@ -308,56 +295,49 @@ def runCommand (home : System.FilePath) (opts : CliOptions) : IO Unit := do runLeanRelease home opts (← wrapperDisplayAction "lean-release") path args | "lean-save" :: path :: extra => do let root ← projectRoot opts .lean - let daemon ← ensureProjectDaemon home root .lean opts let diagnosticScope ← parseLeanSaveArgs extra let action ← wrapperDisplayAction "lean-save" - withWrapperLease root daemon.startedNew do - callBrokerWithProgress root daemon.endpoint + withProjectDaemon home root .lean opts fun client => + callBrokerWithProgress root client (leanSaveRequest root path diagnosticScope) (leanSaveWaitSpec path (action? := some action)) | "lean-update" :: path :: [] => let root ← projectRoot opts .lean - let daemon ← ensureProjectDaemon home root .lean opts - withWrapperLease root daemon.startedNew do - callBroker root daemon.endpoint <| leanUpdateRequest root path + withProjectDaemon home root .lean opts fun client => + callBroker root client <| leanUpdateRequest root path | "lean-sync" :: path :: extra => do let root ← projectRoot opts .lean - let daemon ← ensureProjectDaemon home root .lean opts let diagnosticScope ← parseLeanSyncArgs extra let action ← wrapperDisplayAction "lean-sync" - withWrapperLease root daemon.startedNew do - callBrokerWithProgress root daemon.endpoint + withProjectDaemon home root .lean opts fun client => + callBrokerWithProgress root client (leanSyncRequest root path diagnosticScope) (syncWaitSpec path action) | "lean-refresh" :: path :: extra => do let root ← projectRoot opts .lean - let daemon ← ensureProjectDaemon home root .lean opts let diagnosticScope ← parseLeanRefreshArgs extra let action ← wrapperDisplayAction "lean-refresh" - withWrapperLease root daemon.startedNew do - callBrokerWithProgress root daemon.endpoint + withProjectDaemon home root .lean opts fun client => + callBrokerWithProgress root client (leanRefreshRequest root path diagnosticScope) (refreshWaitSpec path action) | "lean-close" :: path :: [] => let root ← projectRoot opts .lean - let daemon ← ensureProjectDaemon home root .lean opts - withWrapperLease root daemon.startedNew do - callBroker root daemon.endpoint <| leanCloseRequest root path + withProjectDaemon home root .lean opts fun client => + callBroker root client <| leanCloseRequest root path | "lean-close-save" :: path :: extra => let root ← projectRoot opts .lean - let daemon ← ensureProjectDaemon home root .lean opts let diagnosticScope ← parseLeanCloseSaveArgs extra let action ← wrapperDisplayAction "lean-close-save" - withWrapperLease root daemon.startedNew do - callBrokerWithProgress root daemon.endpoint + withProjectDaemon home root .lean opts fun client => + callBrokerWithProgress root client (leanCloseSaveRequest root path diagnosticScope) (leanSaveWaitSpec path (closeAfter := true) (action? := some action)) | "rocq-goals-after" :: path :: line :: character :: text => let root ← projectRoot opts .rocq - let daemon ← ensureProjectDaemon home root .rocq opts - withWrapperLease root daemon.startedNew do - let version ← updateVersionForRocqGoals root daemon.endpoint path - callBroker root daemon.endpoint { + withProjectDaemon home root .rocq opts fun client => do + let version ← updateVersionForRocqGoals root client path + callBroker root client { op := .goals backend := .rocq root? := some root.toString @@ -372,10 +352,9 @@ def runCommand (home : System.FilePath) (opts : CliOptions) : IO Unit := do } | "rocq-goals-prev" :: path :: line :: character :: text => let root ← projectRoot opts .rocq - let daemon ← ensureProjectDaemon home root .rocq opts - withWrapperLease root daemon.startedNew do - let version ← updateVersionForRocqGoals root daemon.endpoint path - callBroker root daemon.endpoint { + withProjectDaemon home root .rocq opts fun client => do + let version ← updateVersionForRocqGoals root client path + callBroker root client { op := .goals backend := .rocq root? := some root.toString @@ -392,38 +371,26 @@ def runCommand (home : System.FilePath) (opts : CliOptions) : IO Unit := do doctor home opts (if backend == "rocq" then .rocq else .lean) | "open-files" :: [] => let root ← projectRootAny opts - let entry ← lookupProjectDaemon root - if let some endpoint := Beam.Daemon.registryEndpoint? entry then - callBroker root endpoint { + withExistingProjectDaemon root fun client => + callBroker root client { op := .openDocs root? := some root.toString } - else - throw <| IO.userError s!"invalid Beam daemon endpoint registry for {entry.root}" | "cancel" :: requestId :: [] => let root ← projectRootAny opts - let entry ← lookupProjectDaemon root - if let some endpoint := Beam.Daemon.registryEndpoint? entry then - callBroker root endpoint { + withExistingProjectDaemon root fun client => + callBroker root client { op := .cancel cancelRequestId? := some requestId } - else - throw <| IO.userError s!"invalid Beam daemon endpoint registry for {entry.root}" | "stats" :: [] => let root ← projectRootAny opts - let entry ← lookupProjectDaemon root - if let some endpoint := Beam.Daemon.registryEndpoint? entry then - callBroker root endpoint { op := .stats } - else - throw <| IO.userError s!"invalid Beam daemon endpoint registry for {entry.root}" + withExistingProjectDaemon root fun client => + callBroker root client { op := .stats } | "reset-stats" :: [] => let root ← projectRootAny opts - let entry ← lookupProjectDaemon root - if let some endpoint := Beam.Daemon.registryEndpoint? entry then - callBroker root endpoint { op := .resetStats } - else - throw <| IO.userError s!"invalid Beam daemon endpoint registry for {entry.root}" + withExistingProjectDaemon root fun client => + callBroker root client { op := .resetStats } | "shutdown" :: [] => shutdownProjectDaemon opts | _ => diff --git a/Beam/Cli/DaemonManager.lean b/Beam/Cli/DaemonManager.lean index a955d167..e9bb6055 100644 --- a/Beam/Cli/DaemonManager.lean +++ b/Beam/Cli/DaemonManager.lean @@ -11,11 +11,16 @@ import Beam.Cli.Args import Beam.Cli.Lock import Beam.Cli.Project import Beam.Daemon.Debug +import Beam.Daemon.Ownership +import Beam.Daemon.Paths +import Std.Internal.UV.Timer open Lean namespace Beam.Cli +open Beam.Daemon.Ownership + open Beam.Broker open Beam.Daemon @@ -23,9 +28,6 @@ open Beam.Daemon def projectDaemonWorkspaceId : WorkspaceId := "beam-cli-project" -def controlDir (root : System.FilePath) : IO System.FilePath := do - Beam.Daemon.controlDir root - private def defaultProjectControlLockTimeoutMs : Nat := 60000 @@ -55,12 +57,6 @@ unbounded lock helper. def withProjectControlLock (root : System.FilePath) (act : IO α) : IO α := do withLockTimeout (← projectControlLockDir root) (← projectControlLockTimeoutMs) act -def registryPath (root : System.FilePath) : IO System.FilePath := do - Beam.Daemon.registryPath root - -private def readRegistry? (root : System.FilePath) : IO (Option RegistryEntry) := - Beam.Daemon.readRegistry? root - private def computeConfigHash (root : System.FilePath) (leanCmd? : Option String) @@ -91,21 +87,31 @@ def removeRegistry (root : System.FilePath) : IO Unit := do if ← path.pathExists then IO.FS.removeFile path -def killPid (pid : Nat) : IO Unit := do - try - let _ ← IO.Process.output { cmd := (← killCommand), args := #[toString pid] } - pure () - catch _ => - pure () - -partial def waitForPidGone (pid : Nat) (tries : Nat := 20) : IO Unit := do +private partial def waitForRecordedPidGone + (recorded : Beam.RecordedPid) + (tries : Nat := 20) : IO Unit := do if tries == 0 then - pure () - else if ← pidAlive pid then - IO.sleep 100 - waitForPidGone pid (tries - 1) - else - pure () + return + match ← recorded.observe with + | .local true => + IO.sleep 100 + waitForRecordedPidGone recorded (tries - 1) + | .invalid | .local false | .differentDomain | .unknownDomain => + pure () + +/-- +Finish a graceful daemon shutdown with a PID fallback only when the registry PID belongs to the +current process domain. A PID from another or unknown domain must never be probed or killed. +-/ +def finishRegistryDaemonShutdown (entry : RegistryEntry) : IO Unit := do + let recorded : Beam.RecordedPid := { pid := entry.pid, domain? := entry.pidDomain? } + waitForRecordedPidGone recorded + match ← recorded.observe with + | .local true => + if ← recorded.terminateIfLocal then + waitForRecordedPidGone recorded + | .invalid | .local false | .differentDomain | .unknownDomain => + pure () private def stopDaemonEntry (entry : RegistryEntry) : IO Unit := do let mayKillPid ← @@ -126,9 +132,8 @@ private def stopDaemonEntry (entry : RegistryEntry) : IO Unit := do pure true | none => pure true - if mayKillPid && entry.pid > 0 && (← pidAlive entry.pid) then - killPid entry.pid - waitForPidGone entry.pid + if mayKillPid then + finishRegistryDaemonShutdown entry def stopRegisteredDaemon (root : System.FilePath) : IO Unit := do match ← readRegistry? root with @@ -179,19 +184,9 @@ private partial def selectUnoccupiedEndpoint else pure endpoint -private def daemonStartupLogPath (root : System.FilePath) : IO System.FilePath := do - Beam.Daemon.daemonStartupLogPath root - -private def daemonFailureIncidentDir (root : System.FilePath) : IO System.FilePath := do - Beam.Daemon.daemonFailureIncidentDir root - private def daemonFailureIncidentRetainCount : Nat := 50 -def recentDaemonFailureIncidentPaths (root : System.FilePath) (limit : Nat := 5) : - IO (Array System.FilePath) := - Beam.Daemon.recentDaemonFailureIncidentPaths root limit - private def pruneDaemonFailureIncidents (root : System.FilePath) : IO Unit := do let entries ← Beam.Daemon.daemonFailureIncidentEntries root let keep := min daemonFailureIncidentRetainCount entries.size @@ -206,12 +201,6 @@ private def appendMaybeSection (msg : String) : Option String → String | none => msg | some context => msg ++ "\n" ++ context -private def registryEndpointSummary (entry : RegistryEntry) : String := - Beam.Daemon.registryEndpointSummary entry - -private def registryPidStatus (entry : RegistryEntry) : IO String := - Beam.Daemon.registryPidStatus entry - private structure DaemonFailureIncident where schemaVersion : Nat kind : String @@ -238,9 +227,6 @@ private def daemonFailureKind? (detail : String) : Option String := else none -private def startupLogTail? (root : System.FilePath) : IO (Option (System.FilePath × String)) := - Beam.Daemon.startupLogTail? root - private def daemonFailureIncidentTimestampLabel (timestamp : String) : String := (timestamp.replace "-" "").replace ":" "" @@ -268,7 +254,7 @@ private def writeDaemonFailureIncident? | some entry => some <$> registryPidStatus entry let endpoint := registry.map registryEndpointSummary let control ← controlDir root - let observedAt ← utcTimestamp + let observedAt ← Beam.utcTimestamp let incident : DaemonFailureIncident := { schemaVersion := daemonFailureIncidentSchemaVersion kind @@ -295,12 +281,6 @@ private def writeDaemonFailureIncident? catch _ => pure none -private def daemonRegistryContext? (root : System.FilePath) : IO (Option String) := - Beam.Daemon.daemonRegistryContext? root - -def daemonDebugContextJson (root : System.FilePath) : IO Json := - Beam.Daemon.daemonDebugContextJson root - def daemonFailureMessage (root : System.FilePath) (detail : String) : IO String := do match daemonFailureKind? detail with | none => @@ -323,7 +303,7 @@ private def startupFailureMessage (endpoint : Transport.Endpoint) (logPath : Sys else s!"failed to start Beam daemon on {endpointSummary endpoint}\n{detail}" if ← logPath.pathExists then - let logText := trimLine (← IO.FS.readFile logPath) + let logText := Beam.trimLine (← IO.FS.readFile logPath) if logText.isEmpty then pure msg else @@ -331,11 +311,36 @@ private def startupFailureMessage (endpoint : Transport.Endpoint) (logPath : Sys else pure msg -private def startDaemon (desired : DesiredConfig) (endpoint : Transport.Endpoint) (logPath : System.FilePath) : - IO Nat := do +private abbrev daemonStdio : IO.Process.StdioConfig where + stdin := .null + stdout := .null + stderr := .null + +private partial def waitForDaemonChildExit + (child : IO.Process.Child daemonStdio) + (tries : Nat := 20) : IO Unit := do + if tries == 0 || (← child.tryWait).isSome then + return + IO.sleep 100 + waitForDaemonChildExit child (tries - 1) + +private def terminateDaemonChild (child : IO.Process.Child daemonStdio) : IO Unit := do + try + if (← child.tryWait).isNone then + child.kill + waitForDaemonChildExit child + catch _ => + pure () + +private def startDaemon + (desired : DesiredConfig) + (endpoint : Transport.Endpoint) + (logPath : System.FilePath) + (daemonId : String) : IO (IO.Process.Child daemonStdio) := do let mut args : List String := [ "--root", desired.root.toString, - "--workspace-id", projectDaemonWorkspaceId + "--workspace-id", projectDaemonWorkspaceId, + "--daemon-id", daemonId ] match endpoint with | .tcp port => @@ -352,18 +357,15 @@ private def startDaemon (desired : DesiredConfig) (endpoint : Transport.Endpoint let cmd := String.intercalate " " ((desired.daemonBin.toString :: args).map shellQuote) let shell := s!"exec {cmd} >{shellQuote logPath.toString} 2>&1 < /dev/null" let child ← IO.Process.spawn { + toStdioConfig := daemonStdio cmd := "sh" args := #["-c", shell] cwd := some desired.root - stdin := .null - stdout := .null - stderr := .null } - let pid := child.pid.toNat - pure pid + pure child private partial def waitForDaemon - (pid : Nat) + (child : IO.Process.Child daemonStdio) (endpoint : Transport.Endpoint) (logPath : System.FilePath) (root : System.FilePath) @@ -375,23 +377,33 @@ private partial def waitForDaemon else throw <| IO.userError (endpointOccupancyError endpoint (System.FilePath.mk daemonRoot) root) | none => - if !(← pidAlive pid) then + if (← child.tryWait).isSome then throw <| IO.userError (← startupFailureMessage endpoint logPath "Beam daemon process exited before responding") else if tries == 0 then throw <| IO.userError (← startupFailureMessage endpoint logPath "Beam daemon did not become ready before timeout") else IO.sleep 100 - waitForDaemon pid endpoint logPath root (tries - 1) + waitForDaemon child endpoint logPath root (tries - 1) -private def registryEntryFor (desired : DesiredConfig) (pid : Nat) (endpoint : Transport.Endpoint) (opts : CliOptions) : - IO RegistryEntry := do +private def newDaemonGenerationId (configHash : String) : IO String := do + let startedMonoNanos ← IO.monoNanosNow + let nonce := ByteArray.toUInt64LE! (← IO.getRandomBytes 8) + pure s!"{configHash.take 12}-{startedMonoNanos}-{nonce}" + +private def registryEntryFor + (desired : DesiredConfig) + (daemonId : String) + (pid : Nat) + (endpoint : Transport.Endpoint) + (opts : CliOptions) : IO RegistryEntry := do let port? := match endpoint with | .tcp port => some port.toNat + let pidDomain? ← Beam.currentPidDomain? pure { - daemonId := s!"{desired.configHash.take 12}-{pid}" + daemonId pid - pidNamespace? := ← currentPidNamespace? + pidDomain? port? root := desired.root.toString configHash := desired.configHash @@ -402,7 +414,7 @@ private def registryEntryFor (desired : DesiredConfig) (pid : Nat) (endpoint : T clientBin? := some desired.clientBin.toString daemonBin? := some desired.daemonBin.toString bundleId? := some desired.bundleId - startedAt := ← utcTimestamp + startedAt := ← Beam.utcTimestamp requestedPort? := requestedPortNat? opts } @@ -412,19 +424,19 @@ private partial def startDaemonEntry (tries : Nat := 10) : IO (Transport.Endpoint × RegistryEntry) := do let endpoint ← selectUnoccupiedEndpoint desired opts let logPath ← daemonStartupLogPath desired.root - let pid ← startDaemon desired endpoint logPath + let daemonId ← newDaemonGenerationId desired.configHash + let child ← startDaemon desired endpoint logPath daemonId try - waitForDaemon pid endpoint logPath desired.root + waitForDaemon child endpoint logPath desired.root catch err => - if pid > 0 && (← pidAlive pid) then - killPid pid - waitForPidGone pid + terminateDaemonChild child let endpointOccupied ← endpointAcceptsConnection endpoint let startupAddressInUse := startupFailureSuggestsEndpointInUse (toString err) if shouldRetryAutomaticStartup (usesAutomaticTcpEndpoint opts) tries endpointOccupied startupAddressInUse then return ← startDaemonEntry desired opts (tries - 1) throw err - let entry ← registryEntryFor desired pid endpoint opts + let pid := child.pid.toNat + let entry ← registryEntryFor desired daemonId pid endpoint opts pure (endpoint, entry) def desiredConfig (home root : System.FilePath) (required : Backend) : IO DesiredConfig := do @@ -487,151 +499,524 @@ def registryLiveFor (root : System.FilePath) (expectedHash? : Option String := n if !rootOk || !hashOk then pure none else if let some endpoint := registryEndpoint? entry then - -- In PID-isolated sandboxes, the recorded daemon pid can be meaningless outside - -- the namespace that started it. Prefer a root-matching endpoint over pid probes. + -- PID observations are not a liveness fallback across isolated sandboxes; only a + -- root-matching endpoint proves that this registry entry is live. if ← daemonServesRoot endpoint projectDaemonWorkspaceId root then pure (some entry) - else if entry.pid == 0 || !(← pidAlive entry.pid) then - pure none else pure none - else if entry.pid == 0 || !(← pidAlive entry.pid) then - pure none else pure none -structure EnsuredProjectDaemon where - endpoint : Transport.Endpoint - startedNew : Bool := false +private def wrapperLeaseHeartbeatIntervalMs : UInt32 := + 250 -def ensureProjectDaemon (home root : System.FilePath) (backend : Backend) (opts : CliOptions) : - IO EnsuredProjectDaemon := do - let desired ← desiredConfig home root backend - withProjectControlLock root do - if let some live ← registryLiveFor root desired.configHash then - if let some endpoint := registryEndpoint? live then - return { endpoint, startedNew := false } - removeRegistry root - let live? ← registryLiveFor root - if live?.isNone then - removeRegistry root - let (endpoint, entry) ← startDaemonEntry desired opts - writeRegistry root entry - if let some live := live? then - unless live.pid == entry.pid && - live.port? == entry.port? do - stopDaemonEntry live - pure { endpoint, startedNew := true } +private def wrapperLeaseHeartbeatTimeoutNanos : Nat := + 5000000000 + +private def wrapperLeaseHeartbeatWriteRetryMs : UInt32 := + 50 + +private def wrapperLeaseHeartbeatWriteRetries : Nat := + 3 + +private def wrapperLifecyclePollMs : UInt32 := + 50 + +private def daemonRequestProbeTimeoutMs : Nat := + 1000 + +private def wrapperLeaseActionCancelWaitMs : Nat := + 5000 private structure WrapperLease where root : System.FilePath path : System.FilePath + stopHeartbeat : IO.Ref Bool + heartbeatTimer : Std.Internal.UV.Timer + heartbeatTask : Task (Except IO.Error Unit) -private structure WrapperLeaseMetadata where - pid : Nat - pidNamespace? : Option String := none - createdAt : String +/-- Internal typed target for one wrapper request to a daemon generation. -/ +structure ProjectDaemonClient where + endpoint : Transport.Endpoint + wrapperLease : WrapperLeaseContext + +private structure DaemonRetirement where + daemonId : String + ownerLeaseFile : String deriving FromJson, ToJson -private def wrapperLeaseDir (root : System.FilePath) : IO System.FilePath := do - pure ((← controlDir root) / "wrapper-leases") +private inductive EnsuredProjectDaemon where + | reused (client : ProjectDaemonClient) (lease : WrapperLease) + | started (client : ProjectDaemonClient) (lease : WrapperLease) + +private def projectDaemonClient + (endpoint : Transport.Endpoint) + (daemonId : String) + (lease : WrapperLease) : ProjectDaemonClient := + { + endpoint + wrapperLease := { + daemonId + leaseFile := lease.path.fileName.getD lease.path.toString + } + } + +private def removeFileIfExists (path : System.FilePath) : IO Unit := do + if ← path.pathExists then + IO.FS.removeFile path -private def removeWrapperLeasePath (path : System.FilePath) : IO Unit := do +private def removeFileIfExistsBestEffort (path : System.FilePath) : IO Unit := do try - if ← path.pathExists then - IO.FS.removeFile path + removeFileIfExists path catch _ => pure () +private def ensureWrapperLeaseNotRevoked (path : System.FilePath) : IO Unit := do + if ← (wrapperLeaseRevocationPath path).pathExists then + throw <| IO.userError s!"wrapper daemon-lifetime lease was revoked: {path}" + +private def writeWrapperLeaseMetadata + (path : System.FilePath) + (metadata : WrapperLeaseMetadata) : IO Unit := do + ensureWrapperLeaseNotRevoked path + let tmp := path.withExtension "tmp" + IO.FS.writeFile tmp ((toJson metadata).pretty ++ "\n") + try + ensureWrapperLeaseNotRevoked path + catch err => + removeFileIfExistsBestEffort tmp + throw err + IO.FS.rename tmp path + try + ensureWrapperLeaseNotRevoked path + catch err => + removeFileIfExistsBestEffort path + throw err + +private partial def writeWrapperLeaseHeartbeat + (path : System.FilePath) + (metadata : WrapperLeaseMetadata) + (retries : Nat := wrapperLeaseHeartbeatWriteRetries) : IO Unit := do + try + writeWrapperLeaseMetadata path metadata + catch err => + if retries == 0 then + throw err + IO.sleep wrapperLeaseHeartbeatWriteRetryMs + writeWrapperLeaseHeartbeat path metadata (retries - 1) + +private partial def wrapperLeaseHeartbeatLoop + (path : System.FilePath) + (metadata : WrapperLeaseMetadata) + (stop : IO.Ref Bool) + (timer : Std.Internal.UV.Timer) + (tick : IO.Promise Unit) : IO Unit := do + let tickResult ← IO.wait tick.result? + if ← stop.get then + return + let some _ := tickResult + | return + let heartbeatMonoNanos ← IO.monoNanosNow + let metadata := { metadata with heartbeatMonoNanos } + writeWrapperLeaseHeartbeat path metadata + wrapperLeaseHeartbeatLoop path metadata stop timer (← timer.next) + private def acquireWrapperLease (root : System.FilePath) : IO WrapperLease := do let dir ← wrapperLeaseDir root IO.FS.createDirAll dir let pid ← IO.Process.getPID let stamp ← IO.monoNanosNow - let path := dir / s!"{stamp}-{pid}.lease" - let tmp := dir / s!"{stamp}-{pid}.lease.tmp" + let nonce := ByteArray.toUInt64LE! (← IO.getRandomBytes 8) + let path := dir / s!"{stamp}-{pid}-{nonce}.lease" let metadata : WrapperLeaseMetadata := { pid := pid.toNat - pidNamespace? := ← currentPidNamespace? - createdAt := ← utcTimestamp + pidDomain? := ← Beam.currentPidDomain? + heartbeatMonoNanos := stamp } - IO.FS.writeFile tmp ((toJson metadata).pretty ++ "\n") - IO.FS.rename tmp path - pure { root, path } + writeWrapperLeaseMetadata path metadata + let stopHeartbeat ← IO.mkRef false + let heartbeatTimer ← Std.Internal.UV.Timer.mk wrapperLeaseHeartbeatIntervalMs.toUInt64 true + let firstTick ← heartbeatTimer.next + let heartbeatTask ← IO.asTask (prio := Task.Priority.dedicated) <| + wrapperLeaseHeartbeatLoop path metadata stopHeartbeat heartbeatTimer firstTick + pure { root, path, stopHeartbeat, heartbeatTimer, heartbeatTask } + +private def stopWrapperLeaseHeartbeat (lease : WrapperLease) : IO Unit := do + lease.stopHeartbeat.set true + Std.Internal.UV.Timer.stop lease.heartbeatTimer + match ← IO.wait lease.heartbeatTask with + | .ok () => pure () + | .error err => throw err private def releaseWrapperLease (lease : WrapperLease) : IO Unit := do - removeWrapperLeasePath lease.path + try + stopWrapperLeaseHeartbeat lease + finally + removeFileIfExistsBestEffort lease.path + removeFileIfExistsBestEffort (wrapperLeaseRevocationPath lease.path) private def readWrapperLeaseMetadata? (path : System.FilePath) : IO (Option WrapperLeaseMetadata) := do + unless ← path.pathExists do + return none + let text ← + try + IO.FS.readFile path + catch err => + if ← path.pathExists then + throw err + else + return none + match Json.parse text with + | .error _ => pure none + | .ok json => + match fromJson? json with + | .error _ => pure none + | .ok metadata => pure (some metadata) + +private def staleWrapperLease? (path : System.FilePath) : IO Bool := do + if ← (wrapperLeaseRevocationPath path).pathExists then + return true + match ← readWrapperLeaseMetadata? path with + | none => pure true + | some metadata => + let now ← IO.monoNanosNow + let pidObservation ← + (Beam.RecordedPid.mk metadata.pid metadata.pidDomain?).observe + pure <| wrapperLeaseStaleFromObservation pidObservation now + wrapperLeaseHeartbeatTimeoutNanos metadata + +private def revokeWrapperLease (path : System.FilePath) : IO Unit := do + let revocationPath := wrapperLeaseRevocationPath path + unless ← revocationPath.pathExists do + let tmp := revocationPath.withExtension "revoking" + let revocation : WrapperLeaseRevocation := { revokedMonoNanos := ← IO.monoNanosNow } + IO.FS.writeFile tmp ((toJson revocation).pretty ++ "\n") + IO.FS.rename tmp revocationPath + removeFileIfExists path + +private def reapAndObserveOtherWrapperLeases (lease : WrapperLease) : + IO OtherWrapperLeasesObservation := do try - let text ← IO.FS.readFile path - let json ← IO.ofExcept <| Json.parse text - let metadata ← IO.ofExcept <| fromJson? json - pure (some metadata) + let dir ← wrapperLeaseDir lease.root + unless ← dir.pathExists do + return .drained + let entries ← dir.readDir + for entry in entries do + if entry.path != lease.path && entry.fileName.endsWith ".lease" then + let stale? ← + try + pure <| some (← staleWrapperLease? entry.path) + catch _ => + pure none + match stale? with + | none => return .activeOrUnreadable + | some true => + try + revokeWrapperLease entry.path + catch _ => + return .activeOrUnreadable + | some false => return .activeOrUnreadable + pure .drained catch _ => + pure .activeOrUnreadable + +private def removeDaemonRetirement (root : System.FilePath) : IO Unit := do + removeFileIfExists (← daemonRetirementPath root) + +private inductive OwnershipRegistryRead where + | missing + | invalid + | unreadable (error : IO.Error) + | present (entry : RegistryEntry) + +private def readOwnershipRegistry (root : System.FilePath) : IO OwnershipRegistryRead := do + let path ← registryPath root + try + unless ← path.pathExists do + return .missing + let text ← IO.FS.readFile path + match Json.parse text with + | .error _ => pure .invalid + | .ok json => + match fromJson? json with + | .error _ => pure .invalid + | .ok entry => pure (.present entry) + catch err => + pure (.unreadable err) + +private def readOrDiscardInvalidDaemonRetirement? + (root : System.FilePath) : IO (Option DaemonRetirement) := do + let path ← daemonRetirementPath root + unless ← path.pathExists do + return none + let text ← IO.FS.readFile path + let retirement? := do + let json ← Json.parse text + fromJson? json + match retirement? with + | .ok retirement => pure (some retirement) + | .error _ => + removeDaemonRetirement root + pure none + +private def writeDaemonRetirement + (root : System.FilePath) + (retirement : DaemonRetirement) : IO Unit := do + let path ← daemonRetirementPath root + if let some parent := path.parent then + IO.FS.createDirAll parent + let tmp := path.withExtension "tmp" + IO.FS.writeFile tmp ((toJson retirement).pretty ++ "\n") + IO.FS.rename tmp path + +/-- Reconcile stale or invalid retirement state and report whether it still blocks admission. -/ +private def reconcileRetirementAdmission (lease : WrapperLease) : IO Bool := do + let some retirement ← readOrDiscardInvalidDaemonRetirement? lease.root + | return false + let registry ← + match ← readOwnershipRegistry lease.root with + | .missing | .invalid => + removeDaemonRetirement lease.root + return false + | .unreadable err => throw err + | .present registry => pure registry + if registry.daemonId != retirement.daemonId then + removeDaemonRetirement lease.root + return false + unless validWrapperLeaseFileName retirement.ownerLeaseFile do + removeDaemonRetirement lease.root + return false + let ownerPath := (← wrapperLeaseDir lease.root) / retirement.ownerLeaseFile + if ← staleWrapperLease? ownerPath then + revokeWrapperLease ownerPath + removeDaemonRetirement lease.root + pure false + else + pure true + +private partial def ensureProjectDaemonUnderLease + (desired : DesiredConfig) + (opts : CliOptions) + (lease : WrapperLease) : IO EnsuredProjectDaemon := do + let admitted? ← withProjectControlLock desired.root do + if ← reconcileRetirementAdmission lease then + pure none + else + if let some live ← registryLiveFor desired.root desired.configHash then + if let some endpoint := registryEndpoint? live then + return some <| EnsuredProjectDaemon.reused + (projectDaemonClient endpoint live.daemonId lease) lease + removeRegistry desired.root + let live? ← registryLiveFor desired.root + if live?.isNone then + removeRegistry desired.root + let (endpoint, entry) ← startDaemonEntry desired opts + writeRegistry desired.root entry + if let some live := live? then + unless live.pid == entry.pid && live.port? == entry.port? do + stopDaemonEntry live + pure <| some <| EnsuredProjectDaemon.started + (projectDaemonClient endpoint entry.daemonId lease) lease + match admitted? with + | some daemon => pure daemon + | none => + IO.sleep wrapperLifecyclePollMs + ensureProjectDaemonUnderLease desired opts lease + +private def acquireWrapperLeaseForAdmission (root : System.FilePath) : IO WrapperLease := + withProjectControlLock root do + acquireWrapperLease root + +private def admitProjectDaemon + (home root : System.FilePath) + (backend : Backend) + (opts : CliOptions) : IO EnsuredProjectDaemon := do + let lease ← acquireWrapperLeaseForAdmission root + try + let desired ← desiredConfig home root backend + ensureProjectDaemonUnderLease desired opts lease + catch err => + releaseWrapperLease lease + throw err + +private def requestDaemonStatsWithin + (endpoint : Transport.Endpoint) : IO (Option Response) := do + let task ← IO.asTask (prio := Task.Priority.dedicated) <| + sendRequest endpoint { op := .stats } + let mut remainingMs := daemonRequestProbeTimeoutMs + while !(← IO.hasFinished task) && remainingMs > 0 do + IO.sleep wrapperLifecyclePollMs + remainingMs := remainingMs - min remainingMs wrapperLifecyclePollMs.toNat + if ← IO.hasFinished task then + match ← IO.wait task with + | .ok response => pure (some response) + | .error _ => pure none + else + IO.cancel task pure none -private def staleWrapperLease? (currentNamespace? : Option String) (path : System.FilePath) : - IO Bool := do - match ← readWrapperLeaseMetadata? path with - | none => pure false - | some metadata => - if metadata.pid == 0 then - pure true - else if metadata.pidNamespace? == currentNamespace? then - pure (!(← pidAlive metadata.pid)) - else - -- The PID may be meaningful only inside a different sandbox namespace. - pure false - -private def activeOtherWrapperLeases (lease : WrapperLease) : IO (Array IO.FS.DirEntry) := do - let dir ← wrapperLeaseDir lease.root - unless ← dir.pathExists do - return #[] - let currentNamespace? ← currentPidNamespace? - let entries ← dir.readDir - let mut active := #[] - for entry in entries do - if entry.path != lease.path && entry.fileName.endsWith ".lease" then - if ← staleWrapperLease? currentNamespace? entry.path then - removeWrapperLeasePath entry.path - else - active := active.push entry - pure active +private def registryDaemonProvenGone (registry : RegistryEntry) : IO Bool := do + let recorded : Beam.RecordedPid := { pid := registry.pid, domain? := registry.pidDomain? } + match ← recorded.observe with + | .local false => pure true + | .local true => pure <| (← recorded.zombieIfLocal?).getD false + | .invalid | .differentDomain | .unknownDomain => pure false -private partial def waitForOtherWrapperLeases (lease : WrapperLease) (tries : Nat := 600) : IO Unit := do - let others ← activeOtherWrapperLeases lease - if others.isEmpty then - pure () - else if tries == 0 then - pure () +private def observeDaemonRequests + (registry : RegistryEntry) + (endpoint : Transport.Endpoint) : IO DaemonRequestsObservation := do + try + let some resp ← requestDaemonStatsWithin endpoint + | return if ← registryDaemonProvenGone registry then .provenGone else .activeOrUnreadable + unless resp.ok do + return .activeOrUnreadable + let some result := resp.result? + | return .activeOrUnreadable + let activeRequestCount ← IO.ofExcept <| result.getObjValAs? Nat "activeRequestCount" + pure <| if activeRequestCount == 0 then .drained else .activeOrUnreadable + catch _ => + pure .activeOrUnreadable + +private def tryCommitDaemonRetirement + (client : ProjectDaemonClient) + (lease : WrapperLease) : IO RetirementDecision := do + withProjectControlLock lease.root do + let observation ← + match ← readOwnershipRegistry lease.root with + | .present registry => + if registry.daemonId != client.wrapperLease.daemonId then + pure RetirementObservation.replacement + else + let otherLeases ← reapAndObserveOtherWrapperLeases lease + let daemonRequests ← + match otherLeases with + | .drained => observeDaemonRequests registry client.endpoint + | .activeOrUnreadable => pure .activeOrUnreadable + pure <| RetirementObservation.current otherLeases daemonRequests + | .missing | .invalid | .unreadable _ => + pure <| RetirementObservation.unavailable (← reapAndObserveOtherWrapperLeases lease) + match retirementDecision observation with + | .wait => pure .wait + | .obsolete => pure .obsolete + | .commit => + let ownerLeaseFile := lease.path.fileName.getD lease.path.toString + writeDaemonRetirement lease.root { + daemonId := client.wrapperLease.daemonId + ownerLeaseFile + } + pure .commit + +private partial def retireStartedProjectDaemon + (client : ProjectDaemonClient) + (lease : WrapperLease) : IO Bool := do + match ← tryCommitDaemonRetirement client lease with + | .commit => pure true + | .obsolete => pure false + | .wait => + IO.sleep wrapperLifecyclePollMs + retireStartedProjectDaemon client lease + +private def finishStartedProjectDaemonAdmission + (client : ProjectDaemonClient) + (lease : WrapperLease) : IO Unit := do + if ← retireStartedProjectDaemon client lease then + -- Leave the final heartbeat on disk. A successor with matching PID-domain identity can + -- prove this process exited by PID; an unknown or different domain waits for the heartbeat + -- to expire before it clears the retirement fence and observes the daemon endpoint. + stopWrapperLeaseHeartbeat lease else - IO.sleep 50 - waitForOtherWrapperLeases lease (tries - 1) + releaseWrapperLease lease + +private partial def awaitWrapperLeaseAction + (lease : WrapperLease) + (actionTask : Task (Except IO.Error α)) : IO α := do + if ← IO.hasFinished actionTask then + match ← IO.wait actionTask with + | .ok value => pure value + | .error err => throw err + else if ← IO.hasFinished lease.heartbeatTask then + let heartbeatResult ← IO.wait lease.heartbeatTask + IO.cancel actionTask + let mut remainingMs := wrapperLeaseActionCancelWaitMs + while !(← IO.hasFinished actionTask) && remainingMs > 0 do + IO.sleep wrapperLifecyclePollMs + remainingMs := remainingMs - min remainingMs wrapperLifecyclePollMs.toNat + match heartbeatResult with + | .ok () => throw <| IO.userError "wrapper lease heartbeat stopped unexpectedly" + | .error err => throw err + else + IO.sleep wrapperLifecyclePollMs + awaitWrapperLeaseAction lease actionTask + +private def runWhileWrapperLeaseHealthy + (lease : WrapperLease) + (act : IO α) : IO α := do + let actionTask ← IO.asTask (prio := Task.Priority.dedicated) act + awaitWrapperLeaseAction lease actionTask + +def withProjectDaemon + (home root : System.FilePath) + (backend : Backend) + (opts : CliOptions) + (act : ProjectDaemonClient → IO α) : IO α := do + match ← admitProjectDaemon home root backend opts with + | .reused client lease => + let result ← + try + pure <| Except.ok (← runWhileWrapperLeaseHealthy lease (act client)) + catch err => + pure <| Except.error err + releaseWrapperLease lease + match result with + | .ok value => pure value + | .error err => throw err + | .started client lease => + -- The starter owns this daemon generation for its process lifetime. Unlike a reuser, it + -- must finish already-admitted work even if its filesystem heartbeat becomes unhealthy, + -- then retain ownership through broker draining and retirement. + let result ← + try + pure <| Except.ok (← act client) + catch err => + pure <| Except.error err + finishStartedProjectDaemonAdmission client lease + match result with + | .ok value => pure value + | .error err => throw err + +private partial def lookupProjectDaemonUnderLease (lease : WrapperLease) : IO ProjectDaemonClient := do + let endpoint? ← withProjectControlLock lease.root do + if ← reconcileRetirementAdmission lease then + pure none + else + match ← registryLiveFor lease.root with + | some entry => + let endpoint ← Beam.Daemon.endpointFromEntry entry + pure <| some <| projectDaemonClient endpoint entry.daemonId lease + | none => + let msg ← daemonFailureMessage lease.root s!"no live Beam daemon registered for {lease.root}" + stopRegisteredDaemon lease.root + throw <| IO.userError msg + match endpoint? with + | some endpoint => pure endpoint + | none => + IO.sleep wrapperLifecyclePollMs + lookupProjectDaemonUnderLease lease -def withWrapperLease (root : System.FilePath) (startedNew : Bool) (act : IO α) : IO α := do - let lease ← acquireWrapperLease root +def withExistingProjectDaemon + (root : System.FilePath) + (act : ProjectDaemonClient → IO α) : IO α := do + let lease ← acquireWrapperLeaseForAdmission root let result ← try - pure <| Except.ok (← act) + let client ← lookupProjectDaemonUnderLease lease + pure <| Except.ok (← runWhileWrapperLeaseHealthy lease (act client)) catch err => pure <| Except.error err - if startedNew then - -- The wrapper invocation that started the daemon must not exit early while sibling - -- wrapper requests for the same root are still in flight, or its sandbox can kill the daemon. - waitForOtherWrapperLeases lease releaseWrapperLease lease match result with | .ok value => pure value | .error err => throw err -def lookupProjectDaemon (root : System.FilePath) : IO RegistryEntry := do - withProjectControlLock root do - match ← registryLiveFor root with - | some entry => pure entry - | none => - let msg ← daemonFailureMessage root s!"no live Beam daemon registered for {root}" - stopRegisteredDaemon root - throw <| IO.userError msg - end Beam.Cli diff --git a/Beam/Cli/Feedback.lean b/Beam/Cli/Feedback.lean index 8c5544a9..bad6f316 100644 --- a/Beam/Cli/Feedback.lean +++ b/Beam/Cli/Feedback.lean @@ -8,12 +8,13 @@ import Lean import Beam.Broker.Client import Beam.Cli.Args import Beam.Cli.DaemonManager -import Beam.Cli.Lock import Beam.Cli.Output import Beam.Cli.Project import Beam.Daemon.Debug +import Beam.Daemon.Paths import Beam.Feedback import Beam.Feedback.Broker +import Beam.System import Beam.Version open Lean @@ -119,7 +120,7 @@ private def collectNonConfidential (home : System.FilePath) (root? : Option System.FilePath) (warnings : Array String) : IO Beam.Feedback.Collection := do - let generatedAt ← utcTimestamp + let generatedAt ← Beam.utcTimestamp let identity ← versionIdentityJson home let (stats, openDocs, daemon, warnings) ← match root? with @@ -127,7 +128,7 @@ private def collectNonConfidential pure (Json.null, Json.null, Json.null, warnings.push "could not infer project root; daemon debug context was not collected") | some root => do - let daemon ← daemonDebugContextJson root + let daemon ← Beam.Daemon.daemonDebugContextJson root let warnings := warnings ++ Beam.Daemon.daemonDebugWarnings daemon let (stats, openDocs, warnings) ← collectDaemonPayload root warnings pure (stats, openDocs, daemon, warnings) @@ -145,7 +146,7 @@ private def collectNonConfidential private def collectConfidential : IO Beam.Feedback.Collection := do pure { - generatedAt := ← utcTimestamp + generatedAt := ← Beam.utcTimestamp data := Json.mkObj [("identity", confidentialIdentityJson)] } @@ -201,7 +202,7 @@ def run (home : System.FilePath) (cliOpts : CliOptions) (args : List String) : I if Beam.Feedback.Internal.needsEvidenceRoots input then match root? with | some root => do - let control ← controlDir root + let control ← Beam.Daemon.controlDir root pure #[root, control] | none => pure #[] else diff --git a/Beam/Cli/Info.lean b/Beam/Cli/Info.lean index eb3bd855..e993380b 100644 --- a/Beam/Cli/Info.lean +++ b/Beam/Cli/Info.lean @@ -10,6 +10,8 @@ import Beam.Cli.InstallLayout import Beam.Cli.Output import Beam.Cli.Project import Beam.Cli.RuntimeBundle +import Beam.Daemon.Debug +import Beam.Daemon.Paths import Beam.Version open Lean @@ -144,7 +146,7 @@ private def printRocqDoctorInfo (home root : System.FilePath) : IO Unit := do IO.println s!"client binary: {paths.client}" def daemonFailureIncidentDoctorLines (root : System.FilePath) : IO (List String) := do - let incidents ← recentDaemonFailureIncidentPaths root + let incidents ← Beam.Daemon.recentDaemonFailureIncidentPaths root if incidents.isEmpty then pure ["daemon incidents: none"] else @@ -163,14 +165,14 @@ def doctor (home : System.FilePath) (opts : CliOptions) (backend : Backend) : IO match backend with | .lean => printLeanDoctorInfo home root | .rocq => printRocqDoctorInfo home root - let registry ← registryPath root + let registry ← Beam.Daemon.registryPath root IO.println s!"registry: {registry}" match ← registryLiveFor root with | some entry => IO.println "daemon status: live" IO.println s!"daemon pid: {entry.pid}" - if let some pidNamespace := entry.pidNamespace? then - IO.println s!"daemon pid namespace: {pidNamespace}" + if let some pidDomain := entry.pidDomain? then + IO.println s!"daemon pid domain: {pidDomain}" if let some endpoint := Beam.Daemon.registryEndpoint? entry then IO.println s!"daemon endpoint: {Beam.Daemon.endpointSummary endpoint}" else diff --git a/Beam/Cli/Lock.lean b/Beam/Cli/Lock.lean index c672ff6a..5592cc91 100644 --- a/Beam/Cli/Lock.lean +++ b/Beam/Cli/Lock.lean @@ -12,55 +12,50 @@ open Lean namespace Beam.Cli -def trimLine (text : String) : String := - Beam.trimLine text - -def readCmdTrim (cmd : String) (args : Array String := #[]) (cwd? : Option System.FilePath := none) : IO String := do - Beam.readCmdTrim cmd args cwd? - -def commandAvailable (cmd : String) (args : Array String := #["--help"]) : IO Bool := do - Beam.commandAvailable cmd args - -def killCommand : IO String := do - Beam.killCommand - -def pidAlive (pid : Nat) : IO Bool := do - Beam.pidAlive pid - private def lockPollMs : Nat := 100 -private def readLockPid? (lockDir : System.FilePath) : IO (Option Nat) := do +private structure LockOwner where + pid : Nat + pidDomain? : Option String + +private def readRegularFile? (path : System.FilePath) : IO (Option String) := do try - let pidPath := lockDir / "pid" - if ← Beam.regularNonSymlinkFile pidPath then - let text ← IO.FS.readFile pidPath - pure <| trimLine text |>.toNat? + if ← Beam.regularNonSymlinkFile path then + pure <| some (Beam.trimLine (← IO.FS.readFile path)) else pure none catch _ => pure none -private def lockOwnerDescription : Option Nat → String - | some pid => s!"pid {pid}" +private def readLockOwner? (lockDir : System.FilePath) : IO (Option LockOwner) := do + let some pidText ← readRegularFile? (lockDir / "pid") + | return none + let some pid := pidText.toNat? + | return none + let pidDomain? ← readRegularFile? (lockDir / "pid-domain") + pure <| some { pid, pidDomain? := pidDomain?.filter (fun domain => !domain.isEmpty) } + +private def lockOwnerDescription : Option LockOwner → String + | some owner => s!"pid {owner.pid}" | none => "unknown owner" private def lockTimeoutMessage (lockDir : System.FilePath) - (ownerPid? : Option Nat) + (owner? : Option LockOwner) (waitedMs timeoutMs : Nat) : String := s!"timed out after {waitedMs} ms waiting for Beam lock {lockDir}; " ++ - s!"lock owner: {lockOwnerDescription ownerPid?}; timeout: {timeoutMs} ms" + s!"lock owner: {lockOwnerDescription owner?}; timeout: {timeoutMs} ms" -private def removeStaleLock? (lockDir : System.FilePath) (ownerPid? : Option Nat) : IO Bool := do - match ownerPid? with - | some ownerPid => - if !(← pidAlive ownerPid) then +private def removeStaleLock? (lockDir : System.FilePath) (owner? : Option LockOwner) : IO Bool := do + match owner? with + | some owner => + match ← (Beam.RecordedPid.mk owner.pid owner.pidDomain?).observe with + | .local false => if ← lockDir.pathExists then IO.FS.removeDirAll lockDir pure true - else - pure false + | .invalid | .local true | .differentDomain | .unknownDomain => pure false | none => pure false @@ -83,6 +78,8 @@ private partial def acquireLockCore if acquired then try IO.FS.writeFile (lockDir / "pid") s!"{selfPid}\n" + if let some pidDomain := ← Beam.currentPidDomain? then + IO.FS.writeFile (lockDir / "pid-domain") s!"{pidDomain}\n" return catch error => try @@ -94,14 +91,14 @@ private partial def acquireLockCore s!"also failed to remove the acquired lock: {cleanupError}" throw error else - let ownerPid? ← readLockPid? lockDir - if ← removeStaleLock? lockDir ownerPid? then + let owner? ← readLockOwner? lockDir + if ← removeStaleLock? lockDir owner? then acquireLockCore lockDir timeoutMs? waitedMs else match timeoutMs? with | some timeoutMs => if waitedMs >= timeoutMs then - throw <| IO.userError (lockTimeoutMessage lockDir ownerPid? waitedMs timeoutMs) + throw <| IO.userError (lockTimeoutMessage lockDir owner? waitedMs timeoutMs) | none => pure () IO.sleep lockPollMs.toUInt32 @@ -139,10 +136,4 @@ def withLockTimeout (lockDir : System.FilePath) (timeoutMs : Nat) (act : IO α) finally releaseLock lockDir -def currentPidNamespace? : IO (Option String) := do - Beam.currentPidNamespace? - -def utcTimestamp : IO String := do - Beam.utcTimestamp - end Beam.Cli diff --git a/Beam/Cli/Project.lean b/Beam/Cli/Project.lean index aa6bd5d3..4a496c2f 100644 --- a/Beam/Cli/Project.lean +++ b/Beam/Cli/Project.lean @@ -11,6 +11,7 @@ import Beam.Cli.RuntimeBundle import Beam.Lean.Workspace import Beam.Path import Beam.Project +import Beam.System open Lean @@ -86,10 +87,10 @@ def leanToolchain (root : System.FilePath) : IO String := do let path := root / "lean-toolchain" unless ← path.pathExists do throw <| IO.userError s!"missing lean-toolchain in {root}" - pure <| trimLine (← IO.FS.readFile path) + pure <| Beam.trimLine (← IO.FS.readFile path) def leanBin (root : System.FilePath) : IO String := - readCmdTrim "elan" #["which", "lean"] (some root) + Beam.readCmdTrim "elan" #["which", "lean"] (some root) def rocqCandidates (root : System.FilePath) : List System.FilePath := [root / "_opam" / "bin" / "coq-lsp", root / "_opam" / "_opam" / "bin" / "coq-lsp"] @@ -101,7 +102,7 @@ def maybeRocqCmd (root : System.FilePath) : IO (Option String) := do match ← IO.getEnv "BEAM_ROCQ_CMD" with | some cmd => pure (some cmd) | none => - if ← commandAvailable "coq-lsp" then + if ← Beam.commandAvailable "coq-lsp" then pure (some "coq-lsp") else pure none diff --git a/Beam/Cli/RuntimeBundle/Fingerprint.lean b/Beam/Cli/RuntimeBundle/Fingerprint.lean index b0e7e91a..1236684e 100644 --- a/Beam/Cli/RuntimeBundle/Fingerprint.lean +++ b/Beam/Cli/RuntimeBundle/Fingerprint.lean @@ -5,9 +5,9 @@ Author: Emilio J. Gallego Arias -/ import Lean -import Beam.Cli.Lock import Beam.Cli.RuntimeBundle.Source import Beam.Cli.RuntimeBundle.ToolchainPolicy +import Beam.System open Lean @@ -21,12 +21,12 @@ structure ToolchainFingerprint where deriving BEq, Repr, FromJson, ToJson def bundlePlatform : IO String := do - let system := ← readCmdTrim "uname" #["-s"] - let machine := ← readCmdTrim "uname" #["-m"] + let system := ← Beam.readCmdTrim "uname" #["-s"] + let machine := ← Beam.readCmdTrim "uname" #["-m"] pure s!"{system.toLower}-{machine.toLower}" def ensureElan : IO Unit := do - unless ← commandAvailable "elan" do + unless ← Beam.commandAvailable "elan" do throw <| IO.userError "missing elan on PATH" private def readRequiredToolchainCmdTrim (toolchain exe : String) (args : Array String := #[]) : @@ -46,7 +46,7 @@ private def readRequiredToolchainCmdTrim (toolchain exe : String) (args : Array "stderr:", if out.stderr.trimAscii.isEmpty then "(empty)" else out.stderr ] - let text := trimLine out.stdout + let text := Beam.trimLine out.stdout if text.isEmpty then throw <| IO.userError s!"failed to fingerprint Lean toolchain {toolchain}: `elan run {toolchain} {exe}` returned empty stdout" diff --git a/Beam/Cli/RuntimeBundle/Metadata.lean b/Beam/Cli/RuntimeBundle/Metadata.lean index 883cb972..465daba8 100644 --- a/Beam/Cli/RuntimeBundle/Metadata.lean +++ b/Beam/Cli/RuntimeBundle/Metadata.lean @@ -8,6 +8,7 @@ import Lean import Beam.Cli.RuntimeBundle.Fingerprint import Beam.Cli.RuntimeBundle.Paths import Beam.Path +import Beam.System open Lean @@ -121,6 +122,6 @@ def writeBundleMetadata (bundleDir : System.FilePath) (toolchain srcHash : Strin if let some parent := path.parent then IO.FS.createDirAll parent IO.FS.writeFile path - ((bundleMetadataJson toolchain srcHash fingerprint workspace (← utcTimestamp)).pretty ++ "\n") + ((bundleMetadataJson toolchain srcHash fingerprint workspace (← Beam.utcTimestamp)).pretty ++ "\n") end Beam.Cli diff --git a/Beam/Daemon/Debug.lean b/Beam/Daemon/Debug.lean index c1735fb6..8ec84e20 100644 --- a/Beam/Daemon/Debug.lean +++ b/Beam/Daemon/Debug.lean @@ -5,6 +5,7 @@ Author: Emilio J. Gallego Arias -/ import Lean +import Beam.Daemon.Paths import Beam.Daemon.Protocol import Beam.System @@ -12,20 +13,6 @@ open Lean namespace Beam.Daemon -private def beamStateDir (root : System.FilePath) : System.FilePath := - root / ".beam" - -def controlDir (root : System.FilePath) : IO System.FilePath := do - match ← IO.getEnv "BEAM_CONTROL_DIR" with - | some dir => - let tag := toString (hash root.toString) - pure (System.FilePath.mk dir / tag) - | none => - pure (beamStateDir root) - -def registryPath (root : System.FilePath) : IO System.FilePath := do - pure ((← controlDir root) / "beam-daemon.json") - def readRegistry? (root : System.FilePath) : IO (Option RegistryEntry) := do let path ← registryPath root unless ← path.pathExists do @@ -38,12 +25,6 @@ def readRegistry? (root : System.FilePath) : IO (Option RegistryEntry) := do catch _ => pure none -def daemonStartupLogPath (root : System.FilePath) : IO System.FilePath := do - pure ((← controlDir root) / "beam-daemon-startup.log") - -def daemonFailureIncidentDir (root : System.FilePath) : IO System.FilePath := do - pure ((← controlDir root) / "daemon-failures") - def daemonFailureIncidentEntries (root : System.FilePath) : IO (Array IO.FS.DirEntry) := do try let dir ← daemonFailureIncidentDir root @@ -96,16 +77,16 @@ def registryEndpointSummary (entry : RegistryEntry) : String := | none => "invalid" def registryPidStatus (entry : RegistryEntry) : IO String := do - if entry.pid == 0 then - pure "unknown" - else - try - if ← Beam.pidAlive entry.pid then - pure "alive" - else - pure "not alive" - catch _ => - pure "unavailable" + let recorded : Beam.RecordedPid := { pid := entry.pid, domain? := entry.pidDomain? } + try + match ← recorded.observe with + | .invalid => pure "unknown" + | .local true => pure "alive" + | .local false => pure "not alive" + | .differentDomain => pure "different PID domain" + | .unknownDomain => pure "unavailable" + catch _ => + pure "unavailable" def startupLogTail? (root : System.FilePath) : IO (Option (System.FilePath × String)) := do try @@ -174,7 +155,7 @@ def daemonRegistryContext? (root : System.FilePath) : IO (Option String) := do ] ++ (optionLine "toolchain" entry.toolchain?).toList ++ (optionLine "bundleId" entry.bundleId?).toList ++ - (optionLine "pidNamespace" entry.pidNamespace?).toList) + (optionLine "pidDomain" entry.pidDomain?).toList) pure <| some <| String.intercalate "\n" lines catch _ => pure none diff --git a/Beam/Daemon/Ownership.lean b/Beam/Daemon/Ownership.lean new file mode 100644 index 00000000..c0db3ca5 --- /dev/null +++ b/Beam/Daemon/Ownership.lean @@ -0,0 +1,103 @@ +/- +Copyright (c) 2026 Lean FRO LLC. All rights reserved. +Released under Apache 2.0 license as described in the file LICENSE. +Author: Emilio J. Gallego Arias +-/ + +import Lean +import Beam.System + +open Lean + +namespace Beam.Daemon.Ownership + +/-- Internal on-disk metadata for one wrapper's daemon-lifetime lease. -/ +structure WrapperLeaseMetadata where + pid : Nat + pidDomain? : Option String := none + heartbeatMonoNanos : Nat + deriving FromJson, ToJson + +/-- Persistent evidence that an expired lease basename may no longer be renewed or used. -/ +structure WrapperLeaseRevocation where + revokedMonoNanos : Nat + deriving ToJson + +private def wrapperLeaseHeartbeatExpired + (now heartbeat timeout : Nat) : Bool := + now < heartbeat || now - heartbeat > timeout + +/-- Pure lease-staleness policy, separated from PID and clock observation. -/ +def wrapperLeaseStaleFromObservation + (pidObservation : Beam.RecordedPidObservation) + (now timeout : Nat) + (metadata : WrapperLeaseMetadata) : Bool := + metadata.pid == 0 || + wrapperLeaseHeartbeatExpired now metadata.heartbeatMonoNanos timeout || + match pidObservation with + | .invalid => true + | .local alive => !alive + | .differentDomain | .unknownDomain => false + +/-- Retirement markers may refer only to one lease basename inside `wrapper-leases`. -/ +def validWrapperLeaseFileName (name : String) : Bool := + !name.isEmpty && + name.endsWith ".lease" && + !(name.contains '/') && + !(name.contains '\\') + +/-- The tombstone paired with one wrapper lease path. -/ +def wrapperLeaseRevocationPath (path : System.FilePath) : System.FilePath := + path.withExtension "revoked" + +/-- Conservative summary of every lease other than the starter's own lease. -/ +inductive OtherWrapperLeasesObservation where + | drained + | activeOrUnreadable + deriving BEq, Repr + +/-- Whether the daemon still owns broker requests admitted before retirement fencing. -/ +inductive DaemonRequestsObservation where + | drained + | activeOrUnreadable + | provenGone + deriving BEq, Repr + +/-- Typed input to the retirement policy. Replacement generations need no sibling inspection. -/ +inductive RetirementObservation where + | current + (otherLeases : OtherWrapperLeasesObservation) + (daemonRequests : DaemonRequestsObservation) + | replacement + | unavailable (otherLeases : OtherWrapperLeasesObservation) + deriving BEq, Repr + +inductive RetirementDecision where + | wait + | commit + | obsolete + deriving BEq, Repr + +/-- +Decide the starter's next step without performing filesystem mutations. + +A proven replacement or a provably dead current daemon makes the starter obsolete immediately, +avoiding a generation-to-generation lease deadlock. A live current generation commits retirement +only after both sibling leases and admitted broker requests drain. Missing, malformed, or unreadable +registry state can release the starter only after all sibling leases are provably drained. +-/ +def retirementDecision + (observation : RetirementObservation) : RetirementDecision := + match observation with + | .replacement => .obsolete + | .current others requests => + match others, requests with + | .drained, .drained => .commit + | .drained, .provenGone => .obsolete + | _, _ => .wait + | .unavailable others => + match others with + | .drained => .obsolete + | .activeOrUnreadable => .wait + +end Beam.Daemon.Ownership diff --git a/Beam/Daemon/Paths.lean b/Beam/Daemon/Paths.lean new file mode 100644 index 00000000..4f2b54f8 --- /dev/null +++ b/Beam/Daemon/Paths.lean @@ -0,0 +1,37 @@ +/- +Copyright (c) 2026 Lean FRO LLC. All rights reserved. +Released under Apache 2.0 license as described in the file LICENSE. +Author: Emilio J. Gallego Arias +-/ + +import Lean + +namespace Beam.Daemon + +private def beamStateDir (root : System.FilePath) : System.FilePath := + root / ".beam" + +def controlDir (root : System.FilePath) : IO System.FilePath := do + match ← IO.getEnv "BEAM_CONTROL_DIR" with + | some dir => + let tag := toString (hash root.toString) + pure (System.FilePath.mk dir / tag) + | none => + pure (beamStateDir root) + +def registryPath (root : System.FilePath) : IO System.FilePath := do + pure ((← controlDir root) / "beam-daemon.json") + +def daemonStartupLogPath (root : System.FilePath) : IO System.FilePath := do + pure ((← controlDir root) / "beam-daemon-startup.log") + +def wrapperLeaseDir (root : System.FilePath) : IO System.FilePath := do + pure ((← controlDir root) / "wrapper-leases") + +def daemonRetirementPath (root : System.FilePath) : IO System.FilePath := do + pure ((← controlDir root) / "daemon-retirement.json") + +def daemonFailureIncidentDir (root : System.FilePath) : IO System.FilePath := do + pure ((← controlDir root) / "daemon-failures") + +end Beam.Daemon diff --git a/Beam/Daemon/Protocol.lean b/Beam/Daemon/Protocol.lean index a9d6c2b6..8867973b 100644 --- a/Beam/Daemon/Protocol.lean +++ b/Beam/Daemon/Protocol.lean @@ -18,7 +18,7 @@ open Beam.Broker structure RegistryEntry where daemonId : String pid : Nat - pidNamespace? : Option String := none + pidDomain? : Option String := none port? : Option Nat := none root : String configHash : String diff --git a/Beam/System.lean b/Beam/System.lean index 6b0b3b8f..bb23cd97 100644 --- a/Beam/System.lean +++ b/Beam/System.lean @@ -46,7 +46,7 @@ def commandAvailable (cmd : String) (args : Array String := #["--help"]) : IO Bo catch _ => pure false -def killCommand : IO String := do +private def killCommand : IO String := do let candidates := [System.FilePath.mk "/bin/kill", System.FilePath.mk "/usr/bin/kill"] for candidate in candidates do if ← candidate.pathExists then @@ -56,15 +56,99 @@ def killCommand : IO String := do else throw <| IO.userError "could not find kill command" -def pidAlive (pid : Nat) : IO Bool := do +/-- +Test a PID already known to belong to the caller's process domain. + +For PIDs loaded from registries, locks, or other persisted metadata, use `RecordedPid.observe` +instead so a numeric PID from another domain is never probed locally. +-/ +private def localPidAlive (pid : Nat) : IO Bool := do let out ← IO.Process.output { cmd := (← killCommand), args := #["-0", toString pid] } pure (out.exitCode == 0) -def currentPidNamespace? : IO (Option String) := do +/-- Inspect zombie state for a PID already known to belong to the caller's process domain. -/ +private def localPidZombie (pid : Nat) : IO Bool := do + try + let out ← IO.Process.output { + cmd := "ps" + args := #["-o", "stat=", "-p", toString pid] + stdin := .null + stderr := .null + } + pure <| out.exitCode == 0 && (trimLine out.stdout).startsWith "Z" + catch _ => + pure false + +def currentPidDomain? : IO (Option String) := do try - pure <| some (← readCmdTrim "readlink" #["/proc/self/ns/pid"]) + let domain ← readCmdTrim "readlink" #["/proc/self/ns/pid"] + pure <| if domain.isEmpty then none else some domain catch _ => - pure none + try + let system ← readCmdTrim "uname" #["-s"] + -- Darwin has no PID namespaces. A stable host-domain marker lets two processes on the same + -- supported platform compare PID observations without weakening Linux's fail-closed fallback + -- when `/proc` namespace identity is unexpectedly unavailable. + pure <| if system == "Darwin" then some "host:Darwin" else none + catch _ => + pure none + +/-- A PID loaded together with the process-domain identity recorded by its owner. -/ +structure RecordedPid where + pid : Nat + domain? : Option String + deriving BEq, Repr + +/-- The only safe outcomes of observing a PID loaded from persisted metadata. -/ +inductive RecordedPidObservation where + | invalid + | local (alive : Bool) + | differentDomain + | unknownDomain + deriving BEq, Repr + +private inductive RecordedPidDomainRelation where + | invalid + | local + | different + | unknown + +private def RecordedPid.domainRelation (recorded : RecordedPid) : IO RecordedPidDomainRelation := do + if recorded.pid == 0 then + return .invalid + match ← currentPidDomain?, recorded.domain? with + | some current, some owner => + pure <| if current == owner then .local else .different + | _, _ => + pure .unknown + +/-- +Observe a persisted PID only when its recorded process domain matches the caller's current domain. +Different and unknown domains never reach the local PID probe. +-/ +def RecordedPid.observe (recorded : RecordedPid) : IO RecordedPidObservation := do + match ← recorded.domainRelation with + | .invalid => pure .invalid + | .local => pure <| .local (← localPidAlive recorded.pid) + | .different => pure .differentDomain + | .unknown => pure .unknownDomain + +/-- Return zombie state only when a persisted PID belongs to the caller's current process domain. -/ +def RecordedPid.zombieIfLocal? (recorded : RecordedPid) : IO (Option Bool) := do + match ← recorded.domainRelation with + | .local => some <$> localPidZombie recorded.pid + | .invalid | .different | .unknown => pure none + +/-- Send the default termination signal only to a persisted PID in the caller's current domain. -/ +def RecordedPid.terminateIfLocal (recorded : RecordedPid) : IO Bool := do + match ← recorded.domainRelation with + | .local => + try + let out ← IO.Process.output { cmd := (← killCommand), args := #[toString recorded.pid] } + pure (out.exitCode == 0) + catch _ => + pure false + | .invalid | .different | .unknown => pure false def utcTimestamp : IO String := do readCmdTrim "date" #["-u", "+%Y-%m-%dT%H:%M:%SZ"] diff --git a/CHANGELOG.md b/CHANGELOG.md index 329dbc71..6bc83563 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -93,6 +93,10 @@ This project keeps a lightweight, reverse-chronological changelog. Dates use `YY - Install and prune control-file reads now reject non-regular or symlinked paths, and a failed lock owner-PID write removes the lock directory acquired by that process. - `lean-beam ensure --hold` now exits cleanly and promptly after `SIGINT`. +- Wrapper-managed daemons now remain alive for overlapping requests without a fixed 30-second + cutoff, fence retired leases against resurrection, and recover safely from killed or replacement + owners in PID-isolated runners + ([#241](https://github.com/leanprover/lean-beam/pull/241), @ejgallego). - `lean-save` and `lean-close-save` now stage and commit complete artifact sets, preserving prior outputs on reported failure or cancellation and preventing same-worker saves from mixing files ([#217](https://github.com/leanprover/lean-beam/pull/217), @ejgallego). diff --git a/docs/DEVELOPMENT.md b/docs/DEVELOPMENT.md index ca91ef1e..17ad1780 100644 --- a/docs/DEVELOPMENT.md +++ b/docs/DEVELOPMENT.md @@ -148,10 +148,16 @@ carries a continuation handle that names one, and daemon startup receives its in explicitly through `--workspace-id`. The public CLI still manages one daemon per project, but that policy stays in `Beam.Cli`: its request adapter supplies a CLI-owned private identifier. That value is an implementation detail, not part of the broker protocol. Broker stats and open-document -requests without an id remain process-wide and return only the `workspaces` map; the CLI scopes -those requests before sending them. `Beam.Broker.Op.workspaceScope` is the shared operation -classification; CLI and test adapters should use it instead of maintaining their own operation -lists. +requests without an id remain process-wide; stats also report `uptimeMs` and the internal +`activeRequestCount` used by daemon retirement, while both return the `workspaces` map. The count is +the number of other currently admitted requests: the stats request itself is excluded. Every +broker request except `cancel` and `shutdown` is tracked even when it has no client request id; +anonymous requests use an internal admission token so disconnect cancellation cannot affect a later +request. Wrapper requests carry both a generated request id and an internal daemon-lifetime lease +context, so every lease-fenced request participates in that count and can be cancelled by its exact +admission handle. The CLI scopes those requests before sending them. +`Beam.Broker.Op.workspaceScope` is the shared operation classification; CLI and test adapters should +use it instead of maintaining their own operation lists. ## MCP Projection Changes @@ -289,10 +295,10 @@ internally it coordinates several responsibilities around the LSP process: `ServerRuntime.dispatchRequestWithHandle` is the asynchronous admission boundary for in-process consumers such as MCP. It validates operation field ownership, registers the request's active -identity, exposes one opaque `RequestHandle`, and owns unregistering that handle on success, -rejection, or exception. A handle uses a per-admission token and must become inert after that -lexical scope, including when a later request reuses the same client request ID. Keep ordinary -daemon and CLI dispatch on +identity, validates any wrapper lease, exposes one opaque `RequestHandle`, and owns unregistering +that handle on success, rejection, or exception. A handle uses a per-admission token and must become +inert after that lexical scope, including when a later request reuses the same client request ID. +Keep ordinary daemon and CLI dispatch on `ServerRuntime.dispatchRequest`; transport layers must not mutate the active-request registry directly. Pending LSP requests must retain the same per-admission cancellation identity; after a handle has been validated, never fall back to matching a reusable client request ID. @@ -350,6 +356,18 @@ broker-derived decision. This wrapper path is easy to break accidentally, so keep the mental model simple. +A daemon generation is one concrete daemon start identified by the `daemonId` in +`beam-daemon.json`. Every wrapper call holds a heartbeat lease while it may use that generation. +When the wrapper that started a generation sees all sibling leases and daemon-admitted requests +drain, it writes a retirement fence: a short-lived marker that closes wrapper admission until that +starter exits and its owner lease is provably stale. + +The retirement fence belongs to the wrapper-managed lifecycle. An unfenced `beam-client` request +that targets the same daemon is counted after broker admission, but the filesystem marker cannot +prevent a new raw request after the retiring owner samples that count. In a PID-reaping command +runner, keep `lean-beam ensure --hold` active while directing `beam-client` at a wrapper-managed +daemon. A separately launched standalone daemon instead has its own explicit process owner. + What was broken: - Codex-style wrapper calls run in separate PID-isolated sandboxes. @@ -362,21 +380,56 @@ What the fix does: - if the registry endpoint still answers, treat the daemon as live even if the recorded pid looks wrong in the current sandbox +- acquire a wrapper lease under the daemon control lock before inspecting or starting a daemon, so + an owner cannot decide that the generation drained while a new wrapper is joining it - if a wrapper call started the daemon, keep that wrapper call alive until overlapping sibling - wrapper calls for the same project root drain + wrapper calls for the same project root drain, without a fixed request-duration cutoff +- close wrapper admission to a drained generation before its owner exits; a later wrapper waits for + that retirement fence to clear before it observes the endpoint +- assign every daemon start a distinct generation id and let an obsolete starter release its lease + before waiting on wrappers admitted to the replacement generation +- bound the retiring owner's broker stats probe; if that exact registry generation has a dead PID + or zombie process in the observer's PID domain, release its starter without writing a + retirement fence, while timeout and transport failures remain fail-closed when PID identity + cannot prove disappearance +- wait for or kill a registry PID during shutdown only when its recorded PID domain matches the + current wrapper; cross-domain shutdown relies on the validated daemon endpoint and never treats + the same numeric PID as local process identity - `lean-beam ensure --hold` gives agents an explicit foreground owner when they need daemon reuse across separate PID-isolated shell invocations -- wrapper leases include PID namespace metadata, so same-namespace stale leases left by killed - wrappers are pruned without treating different sandbox namespaces as safe to probe by pid -- the regression for this path is +- wrapper leases include a PID-domain identity and a monotonic heartbeat; Linux records the PID + namespace, while macOS records its host process domain. Known same-domain stale leases can be + pruned from PID liveness, while unknown or cross-namespace killed wrappers become stale only after + their heartbeat expires +- expiring a lease first writes a persistent `.revoked` tombstone and only then removes the heartbeat + file; a resumed wrapper cannot recreate or reuse that basename, and a killed wrapper leaves the + small tombstone in place as its fencing record +- each wrapper request carries an internal typed fence containing the daemon generation id and its + lease basename; the daemon registers the request, validates the fence before dispatch, and reports + the active-request count to the retiring owner +- a request admitted before lease revocation keeps the generation owner alive until the broker + finishes or cancels it; a request resumed after revocation is rejected before broker work begins +- daemon transport watches client disconnects and cancels the exact registered request, so killing a + cross-namespace wrapper drains both its heartbeat lease and its broker admission +- a wrapper reusing an existing daemon stops or cancels its request if it can no longer renew its + heartbeat; cancellation uses synthesized request ids for both progress and normally short calls, + and heartbeat cleanup does not wait without a bound for an uncooperative task +- the regressions for this path are + [tests/test-beam-wrapper-daemon.sh](../tests/test-beam-wrapper-daemon.sh) and [tests/test-beam-wrapper-sandbox.sh](../tests/test-beam-wrapper-sandbox.sh) -The generic lock/process helpers live in [Beam/Cli/Lock.lean](../Beam/Cli/Lock.lean). Project -daemon control locks use a bounded wait so a live but stuck wrapper process produces owner -diagnostics instead of making later clients wait silently; `BEAM_CONTROL_LOCK_TIMEOUT_MS` can shorten -or lengthen that wait for local debugging. Bundle build locks intentionally keep the lower-level -unbounded helper because another process may legitimately be compiling a helper bundle. Reusable CLI -argument parsing lives in [Beam/Cli/Args.lean](../Beam/Cli/Args.lean). Project-root inference, +Generic process helpers and the typed `RecordedPid.observe` boundary live in +[Beam/System.lean](../Beam/System.lean). Persisted registry, lease, and lock-owner PIDs must pass +through that boundary; only a matching recorded/current PID-domain pair permits a local liveness, +zombie, or termination operation. Generic directory locks live in +[Beam/Cli/Lock.lean](../Beam/Cli/Lock.lean). Their owner metadata records both PID and PID domain; +only a proven dead same-domain owner is reaped, while missing, malformed, unknown-domain, and +different-domain owners fail closed. Project daemon control locks use a bounded wait so a live but +stuck wrapper process produces owner diagnostics instead of making later clients wait silently; +`BEAM_CONTROL_LOCK_TIMEOUT_MS` can shorten or lengthen that wait for local debugging. Bundle build +locks intentionally keep the lower-level unbounded helper because another process may legitimately +be compiling a helper bundle. Reusable CLI argument parsing lives in +[Beam/Cli/Args.lean](../Beam/Cli/Args.lean). Project-root inference, Lean toolchain lookup, and Rocq command discovery live in [Beam/Cli/Project.lean](../Beam/Cli/Project.lean). Shared filesystem path helpers live in [Beam/Path.lean](../Beam/Path.lean). Use them instead of copying string-prefix checks or raw `IO.FS.realPath` wrappers: @@ -400,9 +453,12 @@ and shared with Lake/elaboration work, so a tiny task that blocks in an OS read normal-priority work on low-core runners. The cheap regression guard is [scripts/check-task-priority.sh](../scripts/check-task-priority.sh). -Daemon registry management, daemon startup/reuse, endpoint selection, and wrapper leases live in -[Beam/Cli/DaemonManager.lean](../Beam/Cli/DaemonManager.lean). Broker request plumbing, progress -messages, cancellation-on-interrupt, and response failure notes live in +Pure wrapper lease metadata, staleness, filename, and retirement decisions live in +[Beam/Daemon/Ownership.lean](../Beam/Daemon/Ownership.lean). Shared registry, lease, retirement, +startup-log, and incident paths live in [Beam/Daemon/Paths.lean](../Beam/Daemon/Paths.lean). Daemon +registry management, daemon startup/reuse, endpoint selection, and effectful wrapper lease handling +live in [Beam/Cli/DaemonManager.lean](../Beam/Cli/DaemonManager.lean). Broker request plumbing, +progress messages, cancellation-on-interrupt, and response failure notes live in [Beam/Cli/Broker.lean](../Beam/Cli/Broker.lean). User-facing stdout/stderr formatting helpers live in [Beam/Cli/Output.lean](../Beam/Cli/Output.lean). Doctor, validated/compatible toolchain registry, install layout/manifest, and MCP config reporting live in diff --git a/docs/SETUP.md b/docs/SETUP.md index 9397dadb..797b821b 100644 --- a/docs/SETUP.md +++ b/docs/SETUP.md @@ -164,8 +164,9 @@ not currently in use. Restart active agent and MCP client sessions before any `prune --apply`; otherwise a process may still be running from a runtime selected for removal. A later request rebuilds any needed bundle that was pruned. Pruning uses the same install lock as the installer and each selected bundle's -build lock, and refuses symlinked installed bundle-cache roots or symlinked and unmarked runtime -directories. +build lock. Lock owner metadata includes a PID-domain identity, so cleanup never interprets a +same-numbered PID from an isolated process domain as the local owner. Pruning also refuses symlinked +installed bundle-cache roots or symlinked and unmarked runtime directories. Apply is incremental rather than transactional: Beam validates and removes one displayed path at a time and reports each successful removal immediately. If a later path fails validation or its lock diff --git a/docs/STATUS.md b/docs/STATUS.md index 1d669ce7..3633fafb 100644 --- a/docs/STATUS.md +++ b/docs/STATUS.md @@ -132,13 +132,17 @@ reuse matching speculative execution rather than replaying it from scratch. Beam apply the source edit. For programmatic local consumers, the preferred machine-readable surface is the JSON stream exposed -by `beam-client request-stream`; wrapper stderr should be treated as human-facing. Broker responses -require an explicit top-level `ok` boolean, giving projection layers an unambiguous success/error -discriminator. A successful response always includes `result`; response and stream envelopes reject -undeclared fields, and typed save/close-save results reject incomplete or extended artifact shapes. -All raw stream variants use the same `kind`, `payload`, and optional outer `clientRequestId` fields; -the terminal response payload does not duplicate transport correlation. Exact event ordering and -examples live in [SYNC_AND_DIAGNOSTICS.md](SYNC_AND_DIAGNOSTICS.md#raw-broker-stream). +by `beam-client request-stream`; wrapper stderr should be treated as human-facing. When that client +targets a wrapper-managed daemon in a PID-reaping command runner, keep `lean-beam ensure --hold` +active for the duration; the wrapper retirement fence does not close admission for unfenced raw +broker clients. A separately launched standalone daemon has its own explicit process owner. Broker +responses require an explicit top-level `ok` boolean, giving projection layers an unambiguous +success/error discriminator. A successful response always includes `result`; response and stream +envelopes reject undeclared fields, and typed save/close-save results reject incomplete or extended +artifact shapes. All raw stream variants use the same `kind`, `payload`, and optional outer +`clientRequestId` fields; the terminal response payload does not duplicate transport correlation. +Exact event ordering and examples live in +[SYNC_AND_DIAGNOSTICS.md](SYNC_AND_DIAGNOSTICS.md#raw-broker-stream). `lean-beam-mcp` is the experimental stdio MCP entry point. User setup lives in [SETUP.md](SETUP.md#mcp-setup); implementation, protocol, tool-list, and conformance notes live in @@ -177,6 +181,16 @@ examples live in [SYNC_AND_DIAGNOSTICS.md](SYNC_AND_DIAGNOSTICS.md#raw-broker-st - In sandboxed agent environments, Beam daemon startup itself may require elevated permissions even when the installed bundle and project-local `.beam` paths resolve correctly. +- PID-isolated wrapper calls acquire heartbeat leases before observing a project daemon. A wrapper + that starts a daemon remains alive until overlapping calls drain, while a killed cross-namespace + wrapper lease normally becomes recoverable after about five seconds. Use `lean-beam ensure --hold` + when separate sandbox commands need one explicit foreground daemon owner. A wrapper reusing an + existing daemon fails or cancels its request if it cannot continue renewing that lease. Lease + expiry writes a persistent revocation fence: already-admitted broker work keeps the owner alive + until it drains, while a suspended wrapper resumed after revocation cannot re-admit work through + the old lease. Retirement stats probes are bounded; an owner releases a dead current generation + only when same-PID-domain liveness or zombie state proves that the exact registered daemon process + is gone. - A startup failure that reports `operation not permitted` through `.beam/beam-daemon-startup.log` is usually an environment restriction, not a bundle-resolution mismatch. - Beam daemon disappearance errors include registry/log context and write a JSON incident record under diff --git a/docs/SYNC_AND_DIAGNOSTICS.md b/docs/SYNC_AND_DIAGNOSTICS.md index c319fd4c..03717576 100644 --- a/docs/SYNC_AND_DIAGNOSTICS.md +++ b/docs/SYNC_AND_DIAGNOSTICS.md @@ -127,6 +127,12 @@ observed it. A request may produce any number of `fileProgress` and `diagnostic` by exactly one terminal `response`; the response is last and no later message belongs to that request. +When `beam-client` targets the per-project daemon managed by `lean-beam` in a PID-reaping command +runner, keep one `lean-beam ensure --hold` process active for the request lifetime. Raw broker +requests do not carry wrapper leases, so the wrapper retirement fence does not own their admission. +A separately launched standalone daemon has its own explicit process owner and does not need the +wrapper hold. + Every stream variant uses the same `kind`, `payload`, and optional correlation envelope. When the request supplies `clientRequestId`, each message repeats it on that outer stream envelope: diff --git a/docs/TESTING.md b/docs/TESTING.md index 0559af10..a7db9e97 100644 --- a/docs/TESTING.md +++ b/docs/TESTING.md @@ -110,8 +110,16 @@ Current Beam coverage includes: [tests/test-beam-fast.sh](../tests/test-beam-fast.sh) - wrapper coverage through [tests/test-beam-wrapper.sh](../tests/test-beam-wrapper.sh), which aggregates focused probe, runtime, sync/save, handle, and diagnostic slices - focused daemon lifecycle coverage in [tests/test-beam-wrapper-daemon.sh](../tests/test-beam-wrapper-daemon.sh), - including self-termination after the project worktree disappears -- Linux-only PID-isolated sandbox wrapper coverage in [tests/test-beam-wrapper-sandbox.sh](../tests/test-beam-wrapper-sandbox.sh) + including valid same-domain and malformed stale-lease cleanup, heartbeat-writer failure, + retirement-fence recovery, bounded recovery when the started daemon dies before retirement, + cross-domain shutdown PID non-interference, fail-closed unreadable registry, lease, and + retirement-fence observations, and + self-termination after the project worktree disappears +- Linux-only PID-isolated sandbox wrapper coverage in [tests/test-beam-wrapper-sandbox.sh](../tests/test-beam-wrapper-sandbox.sh), + including followers active beyond 30 seconds, stop/resume across heartbeat revocation, + daemon-admitted request draining, killed-client disconnect cancellation, killed-follower heartbeat + expiry, retirement recovery, obsolete-owner release across daemon replacement, distinct + generation identity, and concurrent cold-start wrapper admission - zero-build save replay, structured-setup support, batch-only-argument rejection, and stale-save race coverage in [tests/test-beam-save-olean.sh](../tests/test-beam-save-olean.sh) diff --git a/scripts/install-beam.sh b/scripts/install-beam.sh index 3b36726b..66f154df 100755 --- a/scripts/install-beam.sh +++ b/scripts/install-beam.sh @@ -453,19 +453,35 @@ ensure_install_root_ready() { release_install_lock() { if [ "$install_lock_owned" -eq 1 ]; then if [ -d "$install_lock_dir" ]; then - rm -f -- "$install_lock_dir/pid" + rm -f -- "$install_lock_dir/pid" "$install_lock_dir/pid-domain" rmdir "$install_lock_dir" 2>/dev/null || true fi install_lock_owned=0 fi } +current_pid_domain() { + case "$(uname -s)" in + Linux) + readlink /proc/self/ns/pid 2>/dev/null || true + ;; + Darwin) + printf '%s\n' 'host:Darwin' + ;; + esac +} + acquire_install_lock() { + local pid_domain="" require_path_within "$install_lock_dir" "$install_root" "install lock" if mkdir "$install_lock_dir"; then install_lock_owned=1 trap 'release_install_lock' EXIT printf '%s\n' "$$" >"$install_lock_dir/pid" + pid_domain="$(current_pid_domain)" + if [ -n "$pid_domain" ]; then + printf '%s\n' "$pid_domain" >"$install_lock_dir/pid-domain" + fi else die "another Beam install appears to be running: $install_lock_dir" fi diff --git a/skills/lean-beam/SKILL.md b/skills/lean-beam/SKILL.md index 0ca239f1..c1fe6758 100644 --- a/skills/lean-beam/SKILL.md +++ b/skills/lean-beam/SKILL.md @@ -126,6 +126,9 @@ Core workflow contract: before the next command that uses the Lean server; `lean-beam refresh` does not restart it - treat wrapper `stderr` as human-facing only; use stdout JSON or `beam-client request-stream` for machine-readable automation +- when `beam-client` targets a wrapper-managed daemon in a PID-reaping command runner, keep one + `lean-beam ensure --hold` process active for the raw request lifetime; raw broker requests do not + carry wrapper leases, while a separately launched standalone daemon has its own process owner - `lean-beam feedback-report` and `beam_feedback_report` return a report to the caller; Beam does not upload or submit it; before posting non-confidential output, review caller-authored narrative, request/response payloads, local paths, Beam stats, open-file data, daemon logs/incidents, and @@ -308,8 +311,14 @@ Use `lean-beam`, not raw JSON and not raw LSP. - wrapper commands talk to the per-project Beam daemon over localhost TCP; they are not direct in-process Lean calls - `lean-beam ensure --hold` prints the usual JSON ensure response on stdout, keeps the wrapper process alive until interrupted, and is only for environments that reap background daemons when - each command exits; later wrappers recover from same-namespace stale lease files left by killed - wrapper processes + each command exits; later wrappers with matching PID-domain identity recover killed leases from + PID liveness, while unknown or cross-namespace killed leases recover after their heartbeat + expires, normally after about five seconds; expiry revokes that lease, already-admitted broker work + keeps the daemon owner alive until it drains, and a resumed or heartbeat-failed wrapper fails or + cancels instead of reusing the revoked lease; if the exact daemon started by the foreground owner + dies, owner retirement releases after same-domain PID liveness or zombie state proves it is gone +- the wrapper retirement fence closes wrapper admission; keep `lean-beam ensure --hold` active when + an unfenced `beam-client` targets that managed daemon in a PID-reaping command runner `lean-beam` is more than a one-shot probe: diff --git a/tests/lean/BeamTest/Broker/CliDaemonTest.lean b/tests/lean/BeamTest/Broker/CliDaemonTest.lean index db769c66..c75ab8f2 100644 --- a/tests/lean/BeamTest/Broker/CliDaemonTest.lean +++ b/tests/lean/BeamTest/Broker/CliDaemonTest.lean @@ -7,15 +7,18 @@ Author: Emilio J. Gallego Arias import Beam.Broker.Errors import Beam.Cli.Args import Beam.Cli.Broker +import Beam.Daemon.Ownership import Beam.Cli.Info import Beam.Cli.LeanOperation import Beam.Cli.Lock import Beam.Cli.RuntimeBundle import Beam.Daemon.Debug +import Beam.Daemon.Paths import Beam.Path import BeamTest.Broker.JsonAssert open Lean +open Beam.Daemon.Ownership open BeamTest.Broker.JsonAssert (requireJsonNull requireJsonString) namespace BeamTest.Broker.CliDaemonTest @@ -24,6 +27,15 @@ private def require (label : String) (cond : Bool) : IO Unit := do unless cond do throw <| IO.userError label +private def projectDaemonClientForTest + (endpoint : Beam.Broker.Transport.Endpoint) : Beam.Cli.ProjectDaemonClient := { + endpoint + wrapperLease := { + daemonId := "test-daemon" + leaseFile := "test-wrapper.lease" + } +} + private def checkDaemonDebugWarnings : IO Unit := do let debug := Json.mkObj [ ("registry", Json.mkObj [ @@ -68,11 +80,8 @@ private def requireJsonStringContains (label field needle : String) (json : Json let actual ← IO.ofExcept <| json.getObjValAs? String field require s!"{label}: expected {field} to contain {needle}, got {actual}" (actual.contains needle) -private def daemonFailureIncidentDir (root : System.FilePath) : IO System.FilePath := do - pure ((← Beam.Cli.controlDir root) / "daemon-failures") - private def sortedIncidentEntries (root : System.FilePath) : IO (Array IO.FS.DirEntry) := do - let dir ← daemonFailureIncidentDir root + let dir ← Beam.Daemon.daemonFailureIncidentDir root unless ← dir.pathExists do return #[] let entries ← dir.readDir @@ -80,7 +89,7 @@ private def sortedIncidentEntries (root : System.FilePath) : IO (Array IO.FS.Dir (fun a b => a.fileName < b.fileName) private def readSingleDaemonFailureIncidentJson (root : System.FilePath) : IO Json := do - let incidentDir ← daemonFailureIncidentDir root + let incidentDir ← Beam.Daemon.daemonFailureIncidentDir root require "daemon failure should write incident directory" (← incidentDir.pathExists) let incidentEntries ← sortedIncidentEntries root require s!"expected one daemon failure incident, got {incidentEntries.size}" (incidentEntries.size == 1) @@ -122,6 +131,74 @@ private partial def withClosingBrokerEndpoint | .ok value => pure value | .error err => throw err +private partial def withBrokerListener + (act : Beam.Broker.Transport.Listener → Beam.Broker.Transport.Endpoint → IO α) + (tries : Nat := 20) : IO α := do + let stamp ← IO.monoNanosNow + let portNat := 30000 + ((stamp + tries) % 20000) + let endpoint := Beam.Broker.Transport.Endpoint.tcp portNat.toUInt16 + try + let listener ← Beam.Broker.Transport.bindAndListen endpoint 2 + act listener endpoint + catch err => + if tries == 0 then + throw err + else + withBrokerListener act (tries - 1) + +private def serveCancelablePlainRequest + (listener : Beam.Broker.Transport.Listener) + (requestObserved : IO.Promise Unit) : IO Unit := do + let requestConn ← Beam.Broker.Transport.accept listener + try + let requestText ← Beam.Broker.Transport.recvMsg requestConn + let requestJson ← IO.ofExcept <| Json.parse requestText + let request : Beam.Broker.Request ← IO.ofExcept <| fromJson? requestJson + let some requestId := request.clientRequestId? + | throw <| IO.userError "plain wrapper request omitted its synthesized clientRequestId" + requestObserved.resolve () + let cancelConn ← Beam.Broker.Transport.accept listener + try + let cancelText ← Beam.Broker.Transport.recvMsg cancelConn + let cancelJson ← IO.ofExcept <| Json.parse cancelText + let cancelRequest : Beam.Broker.Request ← IO.ofExcept <| fromJson? cancelJson + unless cancelRequest.op == .cancel && cancelRequest.cancelRequestId? == some requestId do + throw <| IO.userError "plain wrapper cancellation did not target the admitted request" + let cancelResponse := Beam.Broker.Response.success <| + Json.mkObj [("cancelled", toJson true)] + Beam.Broker.Transport.sendMsg cancelConn + (toJson (Beam.Broker.StreamMessage.response + cancelRequest.clientRequestId? cancelResponse)).compress + finally + Beam.Broker.Transport.closeConnection cancelConn + let response := Beam.Broker.errorResponseFor + .requestCancelled "cancelled by wrapper lease loss" + Beam.Broker.Transport.sendMsg requestConn + (toJson (Beam.Broker.StreamMessage.response (some requestId) response)).compress + finally + Beam.Broker.Transport.closeConnection requestConn + +private def checkPlainBrokerTaskCancellation : IO Unit := do + withBrokerListener fun listener endpoint => do + let requestObserved ← IO.Promise.new + let serverTask ← IO.asTask (prio := Task.Priority.dedicated) <| + serveCancelablePlainRequest listener requestObserved + let requestTask ← IO.asTask (prio := Task.Priority.dedicated) <| + Beam.Cli.requestBroker (System.FilePath.mk "/tmp") + (projectDaemonClientForTest endpoint) { op := .stats } + let some _ ← IO.wait requestObserved.result? + | throw <| IO.userError "plain wrapper request observation promise dropped" + IO.cancel requestTask + let response ← + match ← IO.wait requestTask with + | .ok response => pure response + | .error err => throw err + require "a cancelled plain wrapper request should receive broker requestCancelled" + (response.error?.any fun err => err.code == "requestCancelled") + match ← IO.wait serverTask with + | .ok () => pure () + | .error err => throw err + private def requireRequestJson (label : String) (actual expected : Beam.Broker.Request) : IO Unit := do @@ -442,12 +519,14 @@ private def checkDaemonFailureContext : IO Unit := do let root := System.FilePath.mk s!"/tmp/beam-daemon-failure-context-{← IO.monoNanosNow}" try IO.FS.createDirAll root - let registryPath ← Beam.Cli.registryPath root + let registryPath ← Beam.Daemon.registryPath root if let some parent := registryPath.parent then IO.FS.createDirAll parent + let pidDomain? ← Beam.currentPidDomain? let entry : Beam.Daemon.RegistryEntry := { daemonId := "daemon-test" pid := 999999999 + pidDomain? port? := some 42424 root := root.toString configHash := "config-test" @@ -456,7 +535,7 @@ private def checkDaemonFailureContext : IO Unit := do startedAt := "2026-07-02T00:00:00Z" } IO.FS.writeFile registryPath ((toJson entry).pretty ++ "\n") - let startupLog := (← Beam.Cli.controlDir root) / "beam-daemon-startup.log" + let startupLog ← Beam.Daemon.daemonStartupLogPath root IO.FS.writeFile startupLog "line 1\nline 2\n" let msg ← Beam.Cli.daemonFailureMessage root "Beam daemon connection closed" requireSubstring "daemon failure context should include registry path" "Beam daemon registry" msg @@ -493,7 +572,7 @@ private def checkDaemonFailureContext : IO Unit := do "startupLogTail" "line 1\nline 2" incidentJson finally try - let control ← Beam.Cli.controlDir root + let control ← Beam.Daemon.controlDir root if ← control.pathExists then IO.FS.removeDirAll control catch _ => @@ -522,7 +601,7 @@ private def checkNoLiveDaemonFailureIncident : IO Unit := do "root" root.toString incidentJson finally try - let control ← Beam.Cli.controlDir root + let control ← Beam.Daemon.controlDir root if ← control.pathExists then IO.FS.removeDirAll control catch _ => @@ -537,7 +616,7 @@ private def checkDaemonFailureUnreadableStartupLog : IO Unit := do let root := System.FilePath.mk s!"/tmp/beam-daemon-unreadable-startup-log-{← IO.monoNanosNow}" try IO.FS.createDirAll root - let startupLog := (← Beam.Cli.controlDir root) / "beam-daemon-startup.log" + let startupLog ← Beam.Daemon.daemonStartupLogPath root IO.FS.createDirAll startupLog let msg ← Beam.Cli.daemonFailureMessage root "Beam daemon connection closed" requireSubstring "unreadable startup log should preserve original daemon failure" @@ -556,7 +635,7 @@ private def checkDaemonFailureUnreadableStartupLog : IO Unit := do "startupLogTail" incidentJson finally try - let control ← Beam.Cli.controlDir root + let control ← Beam.Daemon.controlDir root if ← control.pathExists then IO.FS.removeDirAll control catch _ => @@ -570,7 +649,7 @@ private def checkDaemonFailureUnreadableStartupLog : IO Unit := do private def writeTestRegistryEntry (root : System.FilePath) (port? : Option Nat := none) : IO Unit := do - let registryPath ← Beam.Cli.registryPath root + let registryPath ← Beam.Daemon.registryPath root if let some parent := registryPath.parent then IO.FS.createDirAll parent let entry : Beam.Daemon.RegistryEntry := { @@ -595,7 +674,7 @@ private def checkBrokerConnectionClosedIncident : IO Unit := do | .tcp port => some port.toNat writeTestRegistryEntry root port? let msg ← expectIoErrorMessage "broker connection close should surface daemon failure" <| - Beam.Cli.callBrokerQuiet root endpoint { op := .stats } + Beam.Cli.callBrokerQuiet root (projectDaemonClientForTest endpoint) { op := .stats } requireSubstring "broker connection close should preserve transport failure" "Beam daemon connection closed" msg requireSubstring "broker connection close should include incident path" @@ -610,7 +689,7 @@ private def checkBrokerConnectionClosedIncident : IO Unit := do "registryEndpoint" (Beam.Daemon.endpointSummary endpoint) incidentJson finally try - let control ← Beam.Cli.controlDir root + let control ← Beam.Daemon.controlDir root if ← control.pathExists then IO.FS.removeDirAll control catch _ => @@ -625,7 +704,7 @@ private def checkDaemonFailureIncidentRetention : IO Unit := do let root := System.FilePath.mk s!"/tmp/beam-daemon-incident-retention-{← IO.monoNanosNow}" try IO.FS.createDirAll root - let incidentDir ← daemonFailureIncidentDir root + let incidentDir ← Beam.Daemon.daemonFailureIncidentDir root IO.FS.createDirAll incidentDir for i in [0:55] do IO.FS.writeFile (incidentDir / s!"000000000000000000{i}.json") "{}\n" @@ -642,7 +721,7 @@ private def checkDaemonFailureIncidentRetention : IO Unit := do (newIncident.fileName.startsWith "incident-") finally try - let control ← Beam.Cli.controlDir root + let control ← Beam.Daemon.controlDir root if ← control.pathExists then IO.FS.removeDirAll control catch _ => @@ -673,7 +752,7 @@ private def checkDoctorDaemonFailureIncidentLines : IO Unit := do "daemon-failures" incidentLine finally try - let control ← Beam.Cli.controlDir root + let control ← Beam.Daemon.controlDir root if ← control.pathExists then IO.FS.removeDirAll control catch _ => @@ -756,6 +835,116 @@ private def createSymlink if out.exitCode != 0 then throw <| IO.userError s!"failed to create {label} symlink\n{out.stderr}" +private def checkWrapperLeaseStaleness : IO Unit := do + let fresh : WrapperLeaseMetadata := { + pid := 42 + pidDomain? := some "pid:[owner]" + heartbeatMonoNanos := 1000 + } + let timeout := 500 + require "a fresh cross-domain lease should not depend on an unrelated PID observation" + (!(wrapperLeaseStaleFromObservation .differentDomain 1200 timeout fresh)) + require "a dead same-domain PID should make a fresh lease stale" + (wrapperLeaseStaleFromObservation (.local false) 1200 timeout fresh) + require "a live same-domain PID should preserve a fresh lease" + (!(wrapperLeaseStaleFromObservation (.local true) 1200 timeout fresh)) + require "unknown domain identity should fall back to a fresh heartbeat" + (!(wrapperLeaseStaleFromObservation .unknownDomain 1200 timeout fresh)) + require "heartbeat expiry should revoke even a same-domain live process lease" + (wrapperLeaseStaleFromObservation (.local true) 1501 timeout fresh) + require "heartbeat expiry should make a cross-domain lease stale" + (wrapperLeaseStaleFromObservation .differentDomain 1501 timeout fresh) + require "a heartbeat from a previous monotonic clock epoch should be stale" + (wrapperLeaseStaleFromObservation .differentDomain 999 timeout fresh) + require "pid zero should never hold daemon ownership" + (wrapperLeaseStaleFromObservation .differentDomain 1000 timeout { fresh with pid := 0 }) + +private def checkCurrentPidDomain : IO Unit := do + let selfPid := (← IO.Process.getPID).toNat + let domain? ← Beam.currentPidDomain? + match domain? with + | some domain => + require "a known PID domain should not be empty" (!domain.isEmpty) + let recordedLocal : Beam.RecordedPid := { pid := selfPid, domain? := some domain } + require "a matching PID domain should permit a local liveness observation" + ((← recordedLocal.observe) == .local true) + let different : Beam.RecordedPid := { + pid := selfPid + domain? := some (domain ++ "-other") + } + require "a different PID domain should prevent a local liveness observation" + ((← different.observe) == .differentDomain) + | none => + pure () + let unknown : Beam.RecordedPid := { pid := selfPid, domain? := none } + require "an unknown recorded PID domain must fail closed" + ((← unknown.observe) == .unknownDomain) + let invalid : Beam.RecordedPid := { pid := 0, domain? } + require "PID zero should be classified before domain observation" + ((← invalid.observe) == .invalid) + let system ← Beam.readCmdTrim "uname" #["-s"] + if system == "Darwin" then + require "Darwin processes should share the explicit host PID domain" + (domain? == some "host:Darwin") + +private def checkCrossDomainRegistryPidGuard : IO Unit := do + let child ← IO.Process.spawn { + cmd := "sleep" + args := #["30"] + stdin := .null + stdout := .null + stderr := .null + } + try + let entry : Beam.Daemon.RegistryEntry := { + daemonId := "cross-domain-pid-guard" + pid := child.pid.toNat + pidDomain? := some "beam-test-other-pid-domain" + root := "/tmp/beam-cross-domain-pid-guard" + configHash := "cross-domain-pid-guard" + startedAt := "2026-08-25T00:00:00Z" + } + Beam.Cli.finishRegistryDaemonShutdown entry + require "a cross-domain registry PID must not be waited on or killed" + (← child.tryWait).isNone + finally + try + child.kill + catch _ => + pure () + try + discard <| child.wait + catch _ => + pure () + +private def checkDaemonRetirementPolicy : IO Unit := do + require "a drained current generation should commit retirement" + (retirementDecision (.current .drained .drained) == .commit) + require "an active current generation sibling should keep the owner alive" + (retirementDecision (.current .activeOrUnreadable .drained) == .wait) + require "an admitted broker request should keep the current generation owner alive" + (retirementDecision (.current .drained .activeOrUnreadable) == .wait) + require "a provably dead current daemon should release its starter without fencing" + (retirementDecision (.current .drained .provenGone) == .obsolete) + require "a proven replacement generation should make the old owner obsolete immediately" + (retirementDecision .replacement == .obsolete) + require "unavailable registry state with active or unreadable siblings should fail closed" + (retirementDecision (.unavailable .activeOrUnreadable) == .wait) + require "unavailable registry state may release an owner only after siblings drain" + (retirementDecision (.unavailable .drained) == .obsolete) + +private def checkWrapperLeaseFileNames : IO Unit := do + require "a generated-style lease basename should be accepted" + (validWrapperLeaseFileName "123-42-99.lease") + require "a parent-relative retirement lease path should be rejected" + (!(validWrapperLeaseFileName "../outside.lease")) + require "an absolute retirement lease path should be rejected" + (!(validWrapperLeaseFileName "/tmp/outside.lease")) + require "a nested retirement lease path should be rejected" + (!(validWrapperLeaseFileName "nested/outside.lease")) + require "a non-lease retirement basename should be rejected" + (!(validWrapperLeaseFileName "owner.json")) + private def checkPathCanonicalization : IO Unit := do let stamp ← IO.monoNanosNow let root := System.FilePath.mk s!"/tmp/beam-path-canonical-root-{stamp}" @@ -782,26 +971,42 @@ private def checkPathCanonicalization : IO Unit := do private def checkLockLifecycle : IO Unit := do let root := System.FilePath.mk s!"/tmp/beam-cli-lock-test-{← IO.monoNanosNow}" let lockDir := root / "lock" + let some pidDomain := ← Beam.currentPidDomain? + | throw <| IO.userError "lock lifecycle test requires a known PID domain" + let writeOwner := fun (pid : Nat) (domain : String) => do + IO.FS.writeFile (lockDir / "pid") s!"{pid}\n" + IO.FS.writeFile (lockDir / "pid-domain") s!"{domain}\n" try Beam.Cli.withLock lockDir do require "lock directory should exist while lock is held" (← lockDir.pathExists) require "lock pid file should exist while lock is held" (← (lockDir / "pid").pathExists) + require "lock PID domain file should exist while lock is held" + (← (lockDir / "pid-domain").pathExists) require "lock directory should be removed after release" (!(← lockDir.pathExists)) IO.FS.createDirAll lockDir - IO.FS.writeFile (lockDir / "pid") "999999999\n" + writeOwner 999999999 pidDomain Beam.Cli.withLock lockDir do let pidText := (← IO.FS.readFile (lockDir / "pid")).trimAscii.toString require "stale lock should be replaced with this process lock" (pidText != "999999999") IO.FS.createDirAll lockDir let selfPid ← IO.Process.getPID - IO.FS.writeFile (lockDir / "pid") s!"{selfPid}\n" + writeOwner selfPid.toNat pidDomain expectIoErrorContains "live lock timeout" s!"lock owner: pid {selfPid}" <| Beam.Cli.withLockTimeout lockDir 100 do pure () IO.FS.removeDirAll lockDir + IO.FS.createDirAll lockDir + writeOwner 999999999 (pidDomain ++ "-other") + expectIoErrorContains "cross-domain dead lock timeout" "lock owner: pid 999999999" <| + Beam.Cli.withLockTimeout lockDir 100 do + pure () + require "a dead PID from another domain should not make a lock stale" + (← lockDir.pathExists) + IO.FS.removeDirAll lockDir + let deadPidTarget := root / "dead-pid" IO.FS.writeFile deadPidTarget "999999999\n" IO.FS.createDirAll lockDir @@ -1105,11 +1310,17 @@ def main : IO Unit := do checkDaemonFailureContext checkNoLiveDaemonFailureIncident checkDaemonFailureUnreadableStartupLog + checkPlainBrokerTaskCancellation checkBrokerConnectionClosedIncident checkDaemonFailureIncidentRetention checkDoctorDaemonFailureIncidentLines checkPathRelativeToRoot checkLeanModuleNamePathHelpers + checkWrapperLeaseStaleness + checkCurrentPidDomain + checkCrossDomainRegistryPidGuard + checkDaemonRetirementPolicy + checkWrapperLeaseFileNames checkPathCanonicalization checkLockLifecycle checkLeanToolchainPolicyParsing diff --git a/tests/lean/BeamTest/Broker/PendingTest.lean b/tests/lean/BeamTest/Broker/PendingTest.lean index e29defee..dec60d8a 100644 --- a/tests/lean/BeamTest/Broker/PendingTest.lean +++ b/tests/lean/BeamTest/Broker/PendingTest.lean @@ -51,24 +51,35 @@ private def mkPending private def expectRegistered (label : String) - (result : Except BrokerFailure (Option ActiveRequest)) : IO (Option ActiveRequest) := do + (result : Except BrokerFailure ActiveRequest) : IO ActiveRequest := do match result with - | .ok active? => pure active? + | .ok active => pure active | .error failure => throw <| IO.userError s!"{label}: {failure.message}" private def checkActiveRegistry : IO Unit := do let registry ← ActiveRequestRegistry.create let noneResult ← ActiveRequestRegistry.register registry none - let noneActive : Option ActiveRequest ← - expectRegistered "register without clientRequestId" noneResult - require "register without clientRequestId returns none" (Option.isNone noneActive) + let anonymous ← expectRegistered "register without clientRequestId" noneResult + require "anonymous admission participates in active request count" + ((← ActiveRequestRegistry.count registry) == 1) + require "anonymous admission can be cancelled by its exact handle" + (Option.isSome (← ActiveRequestRegistry.markCancelledActive registry anonymous)) + match ← ensureRequestNotCancelled (some anonymous.cancelRef) with + | .ok _ => throw <| IO.userError "anonymous admission did not observe cancellation" + | .error failure => + discard <| requireFailureCode + "anonymous admission reports broker cancellation" + "requestCancelled" + failure + ActiveRequestRegistry.unregister registry (some anonymous) + require "unregistered anonymous admission leaves no active request" + ((← ActiveRequestRegistry.count registry) == 0) let firstResult ← ActiveRequestRegistry.register registry (some "req-1") - let first? : Option ActiveRequest ← - expectRegistered "register active request" firstResult - let some first := first? - | throw <| IO.userError "register active request returned none" + let first ← expectRegistered "register active request" firstResult + require "count excluding the current admission reports only other requests" + ((← ActiveRequestRegistry.countExcluding registry first) == 0) match ← ActiveRequestRegistry.register registry (some "req-1") with | .ok _ => throw <| IO.userError "duplicate clientRequestId registered successfully" @@ -87,16 +98,13 @@ private def checkActiveRegistry : IO Unit := do "requestCancelled" failure - ActiveRequestRegistry.unregister registry first? + ActiveRequestRegistry.unregister registry (some first) require "unregistered active request is no longer cancellable" (Option.isNone (← ActiveRequestRegistry.markCancelled registry "req-1")) let replacementResult ← ActiveRequestRegistry.register registry (some "req-1") - let replacement? : Option ActiveRequest ← - expectRegistered "register replacement active request" replacementResult - let some replacement := replacement? - | throw <| IO.userError "register replacement active request returned none" - ActiveRequestRegistry.unregister registry first? + let replacement ← expectRegistered "register replacement active request" replacementResult + ActiveRequestRegistry.unregister registry (some first) require "stale active handle cannot cancel replacement" (Option.isNone (← ActiveRequestRegistry.markCancelledActive registry first)) match ← ensureRequestNotCancelled (some replacement.cancelRef) with @@ -114,17 +122,15 @@ private def checkActiveRegistry : IO Unit := do "replacement active request reports broker cancellation" "requestCancelled" failure - ActiveRequestRegistry.unregister registry replacement? + ActiveRequestRegistry.unregister registry (some replacement) private def checkPendingCancellationIdentity : IO Unit := do let registry ← ActiveRequestRegistry.create let firstResult ← ActiveRequestRegistry.register registry (some "reused-id") - let some first ← expectRegistered "register first cancellation identity" firstResult - | throw <| IO.userError "register first cancellation identity returned none" + let first ← expectRegistered "register first cancellation identity" firstResult ActiveRequestRegistry.unregister registry (some first) let replacementResult ← ActiveRequestRegistry.register registry (some "reused-id") - let some replacement ← expectRegistered "register replacement cancellation identity" replacementResult - | throw <| IO.userError "register replacement cancellation identity returned none" + let replacement ← expectRegistered "register replacement cancellation identity" replacementResult let (firstPending, _) ← mkPending (cancelRef? := some first.cancelRef) let (replacementPending, _) ← mkPending (cancelRef? := some replacement.cancelRef) require "first admission matches its pending request" diff --git a/tests/lean/BeamTest/Broker/ProtocolTest.lean b/tests/lean/BeamTest/Broker/ProtocolTest.lean index b2672b8e..839aec01 100644 --- a/tests/lean/BeamTest/Broker/ProtocolTest.lean +++ b/tests/lean/BeamTest/Broker/ProtocolTest.lean @@ -661,6 +661,9 @@ private def checkWorkspaceRoutingFields : IO Unit := do .required require s!"{op.key} has the wrong workspace scope" (op.workspaceScope == expectedScope) + let expectedWrapperLease := op != .cancel && op != .shutdown + require s!"{op.key} has the wrong wrapper-lease admission policy" + (op.acceptsWrapperLease == expectedWrapperLease) let request : Request := { op } let decoded ← expectOk s!"minimal {op.key} request round trip" <| fromJson? (α := Request) (toJson request) @@ -673,6 +676,43 @@ private def checkWorkspaceRoutingFields : IO Unit := do let leanReq : Request := { op := .ensure } requireJsonString "backend-scoped request serialization" "backend" "lean" (toJson leanReq) + let wrapperLease : WrapperLeaseContext := { + daemonId := "daemon-generation" + leaseFile := "123-42-99.lease" + } + let fencedReq : Request := { + op := .stats + clientRequestId? := some "wrapper-lease-round-trip" + wrapperLease? := some wrapperLease + } + let decodedFenced ← expectOk "wrapper lease request round trip" <| + fromJson? (α := Request) (toJson fencedReq) + require "wrapper lease request preserves its typed fence" + (decodedFenced.wrapperLease? == some wrapperLease) + for op in #[Op.cancel, .shutdown] do + let fencedControl : Request := { + op + clientRequestId? := some s!"wrapper-lease-{op.key}" + wrapperLease? := some wrapperLease + } + match fromJson? (α := Request) (toJson fencedControl) with + | .ok _ => throw <| IO.userError s!"broker accepted a wrapper lease for {op.key}" + | .error err => + require s!"{op.key} should reject its wrapper lease explicitly" + (err.contains "does not accept 'wrapperLease'") + match fromJson? (α := Request) <| Json.mkObj [ + ("op", toJson "stats"), + ("clientRequestId", toJson "wrapper-lease-undeclared-field"), + ("wrapperLease", Json.mkObj [ + ("daemonId", toJson "daemon-generation"), + ("leaseFile", toJson "123-42-99.lease"), + ("obsolete", toJson true) + ]) + ] with + | .ok _ => throw <| IO.userError "broker accepted an undeclared wrapper lease field" + | .error err => + require "wrapper lease decoder should identify its undeclared field" (err.contains "obsolete") + let explicitReq : Request := { op := .stats workspaceId? := some "fixture" @@ -800,6 +840,7 @@ private def checkWorkspaceLifecycleProtocol : IO Unit := do requireFieldAbsent "process-wide stats" "root" processStats requireFieldAbsent "process-wide stats" "sessions" processStats requireFieldAbsent "process-wide stats" "byBackend" processStats + requireJsonInt "process-wide stats" "activeRequestCount" 0 processStats discard <| IO.ofExcept <| processStats.getObjVal? "workspaces" let resetResult : Beam.Workspace.InitResult := { @@ -859,6 +900,111 @@ private def checkWorkspaceLifecycleProtocol : IO Unit := do require "typed workspace drop preserves lifecycle state" (decodedDrop.workspaceId == "fixture" && decodedDrop.dropped && decodedDrop.invalidatedHandles) +private def wrapperLeaseInactiveResponse (leaseState : String) (resp : Response) : Bool := + resp.error?.any fun err => + err.code == "requestCancelled" && + err.data?.any fun data => + (data.getObjValAs? String "reason").toOption == some "wrapperLeaseInactive" && + (data.getObjValAs? String "leaseState").toOption == some leaseState + +private def checkWrapperLeaseFence : IO Unit := do + let root := System.FilePath.mk s!"/tmp/beam-wrapper-lease-fence-{← IO.monoNanosNow}" + let daemonId := "daemon-generation" + let leaseFile := "123-42-99.lease" + try + IO.FS.createDirAll root + let leaseDir ← Beam.Daemon.wrapperLeaseDir root + IO.FS.createDirAll leaseDir + let leasePath := leaseDir / leaseFile + let metadata : Beam.Daemon.Ownership.WrapperLeaseMetadata := { + pid := 42 + heartbeatMonoNanos := ← IO.monoNanosNow + } + IO.FS.writeFile leasePath ((toJson metadata).pretty ++ "\n") + let runtime ← Beam.Broker.ServerRuntime.create + ({ root } : Beam.Broker.BrokerConfig) "fixture" (.tcp 0) (some daemonId) + let request : Request := { + op := .stats + clientRequestId? := some "wrapper-lease-valid" + wrapperLease? := some { daemonId, leaseFile } + } + let anonymousLeaseJson := Json.mkObj [ + ("op", toJson "stats"), + ("wrapperLease", toJson ({ daemonId, leaseFile } : WrapperLeaseContext)) + ] + match fromJson? (α := Request) anonymousLeaseJson with + | .ok _ => + throw <| IO.userError "broker accepted an untracked wrapper lease request" + | .error err => + require "wrapper lease validation should require request correlation" + (err.contains "clientRequestId") + match request.validateFields with + | .error err => + throw <| IO.userError s!"valid wrapper lease request failed validation: {err}" + | .ok () => pure () + let (accepted, _) ← runtime.dispatchRequest request + require "a matching live wrapper lease should pass daemon dispatch fencing" accepted.ok + let some acceptedResult := accepted.result? + | throw <| IO.userError "accepted wrapper lease stats response omitted its result" + requireJsonInt "lease-fenced stats" "activeRequestCount" 0 acceptedResult + + let revocationPath := Beam.Daemon.Ownership.wrapperLeaseRevocationPath leasePath + IO.FS.writeFile revocationPath "{}\n" + let (revoked, _) ← runtime.dispatchRequest { + request with clientRequestId? := some "wrapper-lease-revoked" + } + require "a revoked wrapper lease should fail before broker dispatch" + (wrapperLeaseInactiveResponse "revoked" revoked) + + let (wrongGeneration, _) ← runtime.dispatchRequest { + request with + clientRequestId? := some "wrapper-lease-wrong-generation" + wrapperLease? := some { daemonId := "replacement", leaseFile } + } + require "a wrapper lease from another daemon generation should fail before dispatch" + (wrapperLeaseInactiveResponse "generationMismatch" wrongGeneration) + + IO.FS.removeFile revocationPath + IO.FS.removeFile leasePath + let (missing, _) ← runtime.dispatchRequest { + request with clientRequestId? := some "wrapper-lease-missing" + } + require "a missing wrapper lease should fail before broker dispatch" + (wrapperLeaseInactiveResponse "missing" missing) + + IO.FS.writeFile leasePath "{\n" + let (malformed, _) ← runtime.dispatchRequest { + request with clientRequestId? := some "wrapper-lease-malformed" + } + require "a malformed wrapper lease should fail before broker dispatch" + (wrapperLeaseInactiveResponse "malformed" malformed) + IO.FS.removeFile leasePath + IO.FS.createDir leasePath + let (unreadable, _) ← runtime.dispatchRequest { + request with clientRequestId? := some "wrapper-lease-unreadable" + } + require "an unreadable wrapper lease should fail before broker dispatch" + (wrapperLeaseInactiveResponse "unreadable" unreadable) + IO.FS.removeDir leasePath + IO.FS.writeFile leasePath ((toJson { metadata with pid := 0 }).pretty ++ "\n") + let (invalidPid, _) ← runtime.dispatchRequest { + request with clientRequestId? := some "wrapper-lease-invalid-pid" + } + require "a zero-pid wrapper lease should fail before broker dispatch" + (wrapperLeaseInactiveResponse "invalidPid" invalidPid) + let (invalidFileName, _) ← runtime.dispatchRequest { + request with + clientRequestId? := some "wrapper-lease-invalid-file-name" + wrapperLease? := some { daemonId, leaseFile := "../outside.lease" } + } + require "an invalid wrapper lease basename should fail before broker dispatch" + (wrapperLeaseInactiveResponse "invalidFileName" invalidFileName) + require "rejected wrapper leases should leave no active admission" + ((← ActiveRequestRegistry.count runtime.activeRequests) == 0) + finally + if ← root.pathExists then + IO.FS.removeDirAll root + def main : IO Unit := do checkResponseJsonShape checkStreamMessageDecode @@ -873,6 +1019,7 @@ def main : IO Unit := do checkRequestArgsBoundary checkWorkspaceRoutingFields checkWorkspaceLifecycleProtocol + checkWrapperLeaseFence end BeamTest.Broker.ProtocolTest diff --git a/tests/lean/BeamTest/Broker/RequestHandleTest.lean b/tests/lean/BeamTest/Broker/RequestHandleTest.lean index e251d19d..086062a9 100644 --- a/tests/lean/BeamTest/Broker/RequestHandleTest.lean +++ b/tests/lean/BeamTest/Broker/RequestHandleTest.lean @@ -84,6 +84,19 @@ def checkCancellationAndLifetime : IO Unit := do runOnce checkStaleHandleIsolation server req + let anonymousHandleRef ← IO.mkRef (none : Option Beam.Broker.RequestHandle) + let (anonymousResp, _) ← server.dispatchRequestWithHandle + { req with clientRequestId? := none } (fun handle => do + anonymousHandleRef.set (some handle) + unless ← handle.cancel do + throw <| IO.userError "anonymous broker request handle was not cancellable" + pure true) + checkCancelledResponse anonymousResp + let some anonymousHandle ← anonymousHandleRef.get + | throw <| IO.userError "anonymous broker request handle was not captured" + if ← anonymousHandle.cancel then + throw <| IO.userError "anonymous broker request handle remained active after dispatch" + let rejectedHandleRef ← IO.mkRef (none : Option Beam.Broker.RequestHandle) let (rejectedResp, _) ← server.dispatchRequestWithHandle req (fun handle => do rejectedHandleRef.set (some handle) diff --git a/tests/lean/BeamTest/Broker/StreamDedupTest.lean b/tests/lean/BeamTest/Broker/StreamDedupTest.lean index cf495276..81015005 100644 --- a/tests/lean/BeamTest/Broker/StreamDedupTest.lean +++ b/tests/lean/BeamTest/Broker/StreamDedupTest.lean @@ -159,6 +159,7 @@ private def fakeServerWithLeanSession lean := { nextEpoch := 1, session? := some session } } pure { + root state := ← Std.Mutex.new { bootstrapConfig := config workspaces := Std.TreeMap.empty.insert fixtureWorkspaceId workspace diff --git a/tests/test-beam-prune.sh b/tests/test-beam-prune.sh index 8c6faa38..fe9eaf17 100644 --- a/tests/test-beam-prune.sh +++ b/tests/test-beam-prune.sh @@ -56,6 +56,23 @@ write_runtime_manifest() { "$beam_cli" install-manifest "$payload" - fixture-toolchain >"$path" } +case "$(uname -s)" in + Linux) test_pid_domain="$(readlink /proc/self/ns/pid 2>/dev/null || true)" ;; + Darwin) test_pid_domain="host:Darwin" ;; + *) test_pid_domain="" ;; +esac +if [ -z "$test_pid_domain" ]; then + echo "prune lock tests require a known PID domain" >&2 + exit 1 +fi + +write_lock_owner() { + local lock_dir="$1" + local pid="$2" + printf '%s\n' "$pid" >"$lock_dir/pid" + printf '%s\n' "$test_pid_domain" >"$lock_dir/pid-domain" +} + mkdir -p \ "$current_runtime/bin" \ "$current_runtime/libexec" \ @@ -220,7 +237,7 @@ race_err="$tmp_root/race.err" lock_writer_pid="$!" wait_for_file "$race_lock_held" "prune install-lock holder" 10 mkdir "$race_lock" -printf '%s\n' "$lock_writer_pid" >"$race_lock/pid" +write_lock_owner "$race_lock" "$lock_writer_pid" BEAM_HOME="$current_runtime" "$beam_cli" install-prune --apply > /dev/null 2>"$race_err" & race_pid="$!" sleep 0.3 @@ -244,14 +261,14 @@ rm -f "$install_root/current" ln -s "$current_runtime" "$install_root/current" mkdir "$install_root/.install-lock" -printf '%s\n' "$$" >"$install_root/.install-lock/pid" +write_lock_owner "$install_root/.install-lock" "$$" install_lock_err="$tmp_root/install-lock.err" if "$install_root/current/bin/lean-beam" prune --apply > /dev/null 2>"$install_lock_err"; then echo "expected prune to respect the active install lock" >&2 exit 1 fi assert_contains_literal "$install_lock_err" 'timed out after 1000 ms waiting for Beam lock' -rm -f "$install_root/.install-lock/pid" +rm -f "$install_root/.install-lock/pid" "$install_root/.install-lock/pid-domain" rmdir "$install_root/.install-lock" assert_file "$old_runtime/manifest.json" @@ -287,7 +304,7 @@ resolved_partial_runtime="$(beam_test_realpath "$partial_runtime")" stale_bundle_lock="$bundle_root/.locks/200" mkdir -p "$stale_bundle_lock" -printf '%s\n' "$$" >"$stale_bundle_lock/pid" +write_lock_owner "$stale_bundle_lock" "$$" bundle_lock_out="$tmp_root/bundle-lock.out" bundle_lock_err="$tmp_root/bundle-lock.err" if "$install_root/current/bin/lean-beam" prune --apply --bundles \ @@ -303,7 +320,7 @@ assert_contains_literal "$bundle_lock_err" \ assert_contains_literal "$bundle_lock_err" \ 'rerun `lean-beam prune --bundles` to preview the remaining paths' assert_not_exists "$partial_runtime" -rm -f "$stale_bundle_lock/pid" +rm -f "$stale_bundle_lock/pid" "$stale_bundle_lock/pid-domain" rmdir "$stale_bundle_lock" assert_file "$stale_bundle/metadata.json" diff --git a/tests/test-beam-wrapper-daemon.sh b/tests/test-beam-wrapper-daemon.sh index 14550ffb..7b385b0c 100644 --- a/tests/test-beam-wrapper-daemon.sh +++ b/tests/test-beam-wrapper-daemon.sh @@ -61,6 +61,7 @@ if [ -z "${BEAM_INSTALL_BUNDLE_DIR:-}" ]; then fi busy_pid="" hold_pid="" +heartbeat_follower_pid="" removed_root_pid="" removed_root_err="" @@ -70,6 +71,10 @@ cleanup() { kill "$busy_pid" > /dev/null 2>&1 || true wait "$busy_pid" 2>/dev/null || true fi + if [ -n "$heartbeat_follower_pid" ]; then + kill "$heartbeat_follower_pid" > /dev/null 2>&1 || true + wait "$heartbeat_follower_pid" 2>/dev/null || true + fi if [ -n "$removed_root_pid" ]; then kill "$removed_root_pid" > /dev/null 2>&1 || true wait "$removed_root_pid" 2>/dev/null || true @@ -133,20 +138,140 @@ if ! kill -0 "$hold_pid" 2>/dev/null; then fi hold_json="$(cat "$tmp9/hold.out")" assert_json_field_equals "ensure --hold response" "$hold_json" ok true "$tmp9/hold.err" -stop_hold_process true + +lease_dir="$tmp9/.beam/wrapper-leases" +retirement_path="$tmp9/.beam/daemon-retirement.json" + +# A retirement marker is local control state, but its lease field must still be constrained to the +# wrapper-leases directory before cleanup code joins or removes that path. +outside_lease="$tmp9/.beam/outside.lease" +printf 'preserve\n' > "$outside_lease" +retirement_daemon_id="$(read_json_field "$hold_registry" daemonId)" +RETIREMENT_PATH="$retirement_path" DAEMON_ID="$retirement_daemon_id" python3 - <<'PY' +import json, os + +with open(os.environ["RETIREMENT_PATH"], "w") as f: + json.dump({"daemonId": os.environ["DAEMON_ID"], "ownerLeaseFile": "../outside.lease"}, f) + f.write("\n") +PY +"$beam_script" --root "$tmp9" ensure lean > /dev/null +if [ ! -f "$outside_lease" ]; then + echo "expected an invalid retirement owner path not to remove a file outside wrapper-leases" >&2 + exit 1 +fi +if [ -e "$retirement_path" ]; then + echo "expected an invalid retirement owner path to be discarded" >&2 + cat "$retirement_path" >&2 + exit 1 +fi +rm -f "$outside_lease" + +# A reused-daemon request must stop if its heartbeat writer fails; otherwise the owner can prune its +# expired lease while the request is still using the daemon. Block only the follower's atomic tmp +# path so the starter heartbeat remains healthy. +owner_lease="$(find "$lease_dir" -maxdepth 1 -type f -name '*.lease' -print | sed -n '1p')" +if [ -z "$owner_lease" ]; then + echo "expected the foreground owner to hold a wrapper lease" >&2 + exit 1 +fi +"$beam_script" --root "$tmp9" ensure --hold \ + > "$tmp9/heartbeat-follower.out" 2> "$tmp9/heartbeat-follower.err" & +heartbeat_follower_pid="$!" +heartbeat_follower_lease="" +for _ in $(seq 1 100); do + heartbeat_follower_lease="$(find "$lease_dir" -maxdepth 1 -type f -name '*.lease' \ + ! -path "$owner_lease" -print | sed -n '1p')" + if [ -s "$tmp9/heartbeat-follower.out" ] && [ -n "$heartbeat_follower_lease" ]; then + break + fi + sleep 0.05 +done +if [ ! -s "$tmp9/heartbeat-follower.out" ] || [ -z "$heartbeat_follower_lease" ]; then + echo "expected the heartbeat-failure follower to acquire a lease and print ensure output" >&2 + cat "$tmp9/heartbeat-follower.err" >&2 + exit 1 +fi +heartbeat_tmp="${heartbeat_follower_lease%.lease}.tmp" +for _ in $(seq 1 100); do + if mkdir "$heartbeat_tmp" 2>/dev/null; then + break + fi + sleep 0.01 +done +if [ ! -d "$heartbeat_tmp" ]; then + echo "could not block the follower heartbeat tmp path" >&2 + exit 1 +fi +if ! wait_for_exit "$heartbeat_follower_pid" "wrapper with failed heartbeat writer" 100 0.05; then + cat "$tmp9/heartbeat-follower.err" >&2 + exit 1 +fi +set +e +wait "$heartbeat_follower_pid" +heartbeat_follower_status="$?" +set -e +heartbeat_follower_pid="" +rmdir "$heartbeat_tmp" +if [ "$heartbeat_follower_status" -eq 0 ]; then + echo "expected a reused-daemon wrapper with a failed heartbeat writer to fail" >&2 + cat "$tmp9/heartbeat-follower.err" >&2 + exit 1 +fi + +# Neither an unreadable registry nor an unreadable sibling lease may make the starter leave its +# retirement loop. Restore each observation independently and require the owner to remain alive +# until the lease directory is provably drained. +unreadable_lease="$lease_dir/unreadable-sibling.lease" +mkdir "$unreadable_lease" +mv "$hold_registry" "$hold_registry.saved" +mkdir "$hold_registry" +kill -INT "$hold_pid" +sleep 0.5 +if ! kill -0 "$hold_pid" 2>/dev/null; then + echo "expected an owner to stay alive while registry state is unreadable" >&2 + cat "$tmp9/hold.err" >&2 + exit 1 +fi +rmdir "$hold_registry" +mv "$hold_registry.saved" "$hold_registry" +sleep 0.5 +if ! kill -0 "$hold_pid" 2>/dev/null; then + echo "expected an owner to stay alive while a sibling lease is unreadable" >&2 + cat "$tmp9/hold.err" >&2 + exit 1 +fi +rmdir "$unreadable_lease" +if ! wait_for_exit "$hold_pid" "owner after registry and lease recovery" 100 0.05; then + cat "$tmp9/hold.err" >&2 + exit 1 +fi +set +e +wait "$hold_pid" +hold_status="$?" +set -e +hold_pid="" +if [ "$hold_status" -ne 0 ]; then + echo "expected the owner to exit cleanly after registry and lease recovery, got $hold_status" >&2 + cat "$tmp9/hold.err" >&2 + exit 1 +fi "$beam_script" --root "$tmp9" shutdown > /dev/null stale_lease_dir="$tmp9/.beam/wrapper-leases" stale_lease="$stale_lease_dir/stale-dead-wrapper.lease" mkdir -p "$stale_lease_dir" -pid_namespace="$(readlink /proc/self/ns/pid 2>/dev/null || true)" -LEASE_PATH="$stale_lease" PID_NAMESPACE="$pid_namespace" python3 - <<'PY' -import json, os +case "$(uname -s)" in + Linux) pid_domain="$(readlink /proc/self/ns/pid 2>/dev/null || true)" ;; + Darwin) pid_domain="host:Darwin" ;; + *) pid_domain="" ;; +esac +LEASE_PATH="$stale_lease" PID_DOMAIN="$pid_domain" python3 - <<'PY' +import json, os, time metadata = { "pid": 999999999, - "pidNamespace": os.environ["PID_NAMESPACE"] or None, - "createdAt": "test", + "pidDomain": os.environ["PID_DOMAIN"] or None, + "heartbeatMonoNanos": time.monotonic_ns(), } with open(os.environ["LEASE_PATH"], "w") as f: json.dump(metadata, f) @@ -155,12 +280,103 @@ PY "$beam_script" --root "$tmp9" ensure lean > /dev/null if [ -e "$stale_lease" ]; then - echo "expected wrapper ensure to remove a stale same-namespace wrapper lease" >&2 + echo "expected wrapper ensure to remove a stale same-domain wrapper lease" >&2 cat "$stale_lease" >&2 exit 1 fi "$beam_script" --root "$tmp9" shutdown > /dev/null +malformed_lease="$stale_lease_dir/malformed-wrapper.lease" +printf '{\n' > "$malformed_lease" +"$beam_script" --root "$tmp9" ensure lean > /dev/null +if [ -e "$malformed_lease" ]; then + echo "expected wrapper ensure to prune a malformed wrapper lease" >&2 + cat "$malformed_lease" >&2 + exit 1 +fi +"$beam_script" --root "$tmp9" shutdown > /dev/null + +rm -f "$retirement_path" +mkdir "$retirement_path" +set +e +"$beam_script" --root "$tmp9" ensure lean > "$tmp9/retirement-read.out" 2> "$tmp9/retirement-read.err" +retirement_read_status="$?" +set -e +if [ "$retirement_read_status" -eq 0 ]; then + echo "expected an unreadable retirement fence to fail closed" >&2 + cat "$tmp9/retirement-read.out" >&2 + cat "$tmp9/retirement-read.err" >&2 + exit 1 +fi +if [ ! -d "$retirement_path" ]; then + echo "expected an unreadable retirement fence to remain in place" >&2 + exit 1 +fi +rmdir "$retirement_path" +"$beam_script" --root "$tmp9" ensure lean > /dev/null +"$beam_script" --root "$tmp9" shutdown > /dev/null + +# If the daemon started by a foreground owner dies before retirement, that wrapper must not poll +# forever waiting for stats from a process that is provably gone. A later wrapper should replace +# the dead registry generation normally. +rm -f "$tmp9/dead-daemon-hold.out" "$tmp9/dead-daemon-hold.err" +"$beam_script" --root "$tmp9" ensure --hold \ + > "$tmp9/dead-daemon-hold.out" 2> "$tmp9/dead-daemon-hold.err" & +hold_pid="$!" +for _ in $(seq 1 200); do + if [ -s "$tmp9/dead-daemon-hold.out" ] && [ -f "$hold_registry" ]; then + break + fi + sleep 0.05 +done +if [ ! -s "$tmp9/dead-daemon-hold.out" ] || [ ! -f "$hold_registry" ]; then + echo "expected the crash-retirement owner to start a daemon" >&2 + cat "$tmp9/dead-daemon-hold.err" >&2 + exit 1 +fi +dead_daemon_pid="$(read_json_field "$hold_registry" pid)" +dead_daemon_id="$(read_json_field "$hold_registry" daemonId)" +kill -KILL "$dead_daemon_pid" +dead_daemon_stopped="false" +for _ in $(seq 1 100); do + if ! kill -0 "$dead_daemon_pid" 2>/dev/null; then + dead_daemon_stopped="true" + break + fi + if ps -o stat= -p "$dead_daemon_pid" 2>/dev/null | grep -Eq '^[[:space:]]*Z'; then + dead_daemon_stopped="true" + break + fi + sleep 0.05 +done +if [ "$dead_daemon_stopped" != "true" ]; then + echo "expected daemon $dead_daemon_pid to stop before owner retirement" >&2 + exit 1 +fi +kill -INT "$hold_pid" +if ! wait_for_exit "$hold_pid" "owner of a provably dead daemon" 100 0.05; then + cat "$tmp9/dead-daemon-hold.err" >&2 + exit 1 +fi +set +e +wait "$hold_pid" +dead_daemon_owner_status="$?" +set -e +hold_pid="" +if [ "$dead_daemon_owner_status" -ne 0 ]; then + echo "expected the owner of a provably dead daemon to exit cleanly, got $dead_daemon_owner_status" >&2 + cat "$tmp9/dead-daemon-hold.err" >&2 + exit 1 +fi +"$beam_script" --root "$tmp9" ensure lean > /dev/null +replacement_daemon_id="$(read_json_field "$hold_registry" daemonId)" +if [ "$replacement_daemon_id" = "$dead_daemon_id" ]; then + echo "expected the next wrapper to replace the dead daemon generation" >&2 + cat "$hold_registry" >&2 + exit 1 +fi +"$beam_script" --root "$tmp9" shutdown > /dev/null + ( cd "$tmp1" "$beam_script" ensure lean > /dev/null diff --git a/tests/test-beam-wrapper-sandbox.sh b/tests/test-beam-wrapper-sandbox.sh index 8a475ae1..c351521e 100755 --- a/tests/test-beam-wrapper-sandbox.sh +++ b/tests/test-beam-wrapper-sandbox.sh @@ -47,8 +47,12 @@ hold_out="$tmp_root/hold.out" hold_err="$tmp_root/hold.err" owner_out="$tmp_root/owner.out" owner_err="$tmp_root/owner.err" +owner_stop="$tmp_root/owner.stop" follower_out="$tmp_root/follower.out" follower_err="$tmp_root/follower.err" +follower_request_id="" +replacement_owner_a_pid="" +replacement_owner_b_pid="" cleanup() { if [ -n "${hold_pid:-}" ]; then @@ -59,8 +63,19 @@ cleanup() { kill "$owner_pid" > /dev/null 2>&1 || true wait "$owner_pid" 2>/dev/null || true fi + if [ -n "${replacement_owner_a_pid:-}" ]; then + kill "$replacement_owner_a_pid" > /dev/null 2>&1 || true + wait "$replacement_owner_a_pid" 2>/dev/null || true + fi + if [ -n "${replacement_owner_b_pid:-}" ]; then + kill "$replacement_owner_b_pid" > /dev/null 2>&1 || true + wait "$replacement_owner_b_pid" 2>/dev/null || true + fi if [ -n "${follower_pid:-}" ]; then - sandbox_beam cancel wrapper-sandbox-follower > /dev/null 2>&1 || true + if [ -n "$follower_request_id" ]; then + sandbox_beam cancel "$follower_request_id" > /dev/null 2>&1 || true + fi + kill "$follower_pid" > /dev/null 2>&1 || true wait "$follower_pid" 2>/dev/null || true fi remove_owned_tmp_tree "$tmp_root" @@ -83,6 +98,15 @@ sandbox_beam() { -- /usr/bin/env BEAM_CONTROL_DIR="$control_root" "$beam_script" --root "$project_root" "$@" } +assert_no_connection_closed_incidents() { + local label="$1" + if find "$control_root" -path '*/daemon-failures/*connectionClosed*.json' -print -quit | grep -q .; then + echo "expected $label to produce no connectionClosed incident" >&2 + find "$control_root" -path '*/daemon-failures/*.json' -print -exec cat {} \; >&2 + exit 1 + fi +} + sandbox_shell_hold() { local hold_secs="$1" bwrap --new-session --die-with-parent \ @@ -95,10 +119,59 @@ sandbox_shell_hold() { -- /bin/bash -lc "export BEAM_CONTROL_DIR='$control_root'; '$beam_script' --root '$project_root' ensure lean >'$hold_out' 2>'$hold_err'; sleep $hold_secs" } +sandbox_owner_hold() { + local output_path="$1" + local error_path="$2" + local stop_path="$3" + bwrap --new-session --die-with-parent \ + --ro-bind / / \ + --dev-bind /dev /dev \ + --bind /tmp /tmp \ + --proc /proc \ + --unshare-pid \ + --chdir "$project_root" \ + -- /bin/bash -lc \ + "export BEAM_CONTROL_DIR='$control_root'; \ + '$beam_script' --root '$project_root' ensure lean --hold >'$output_path' 2>'$error_path' & \ + wrapper_pid=\$!; \ + while [ ! -f '$stop_path' ]; do sleep 0.05; done; \ + kill -INT \"\$wrapper_pid\"; \ + wait \"\$wrapper_pid\"" +} + +sandbox_paused_follower() { + local version="$1" + local request_id="$2" + local output_path="$3" + local error_path="$4" + local pause_path="$5" + local paused_path="$6" + local resume_path="$7" + bwrap --new-session --die-with-parent \ + --ro-bind / / \ + --dev-bind /dev /dev \ + --bind /tmp /tmp \ + --proc /proc \ + --unshare-pid \ + --chdir "$project_root" \ + -- /bin/bash -lc \ + "export BEAM_CONTROL_DIR='$control_root' BEAM_PROGRESS=1 BEAM_REQUEST_ID='$request_id'; \ + '$beam_script' --root '$project_root' run-at tests/scenario/docs/SlowPoll.lean '$version' 25 2 poll_sleep_cmd >'$output_path' 2>'$error_path' & \ + wrapper_pid=\$!; \ + while [ ! -f '$pause_path' ]; do sleep 0.05; done; \ + kill -STOP \"\$wrapper_pid\"; \ + touch '$paused_path'; \ + while [ ! -f '$resume_path' ]; do sleep 0.05; done; \ + kill -CONT \"\$wrapper_pid\"; \ + wait \"\$wrapper_pid\"" +} + wait_for_registry() { local remaining=300 while [ "$remaining" -gt 0 ]; do - registry="$(find "$control_root" -name beam-daemon.json -print | sed -n '1p')" + # The control lock is intentionally short-lived and may disappear while `find` walks the + # per-root directory. Ignore that observational traversal race and keep probing for the file. + registry="$(find "$control_root" -name beam-daemon.json -print 2>/dev/null | sed -n '1p' || true)" if [ -n "$registry" ] && [ -f "$registry" ]; then return 0 fi @@ -119,10 +192,10 @@ fi daemon_id_1="$(read_json_field "$registry" daemonId)" port_1="$(read_json_field "$registry" port)" -pid_ns_1="$(read_json_field "$registry" pidNamespace 2>/dev/null || true)" +pid_domain_1="$(read_json_field "$registry" pidDomain 2>/dev/null || true)" -if [ -z "$pid_ns_1" ]; then - echo "expected sandboxed wrapper registry to record the daemon pid namespace for debugging" >&2 +if [ -z "$pid_domain_1" ]; then + echo "expected sandboxed wrapper registry to record the daemon PID domain for debugging" >&2 cat "$registry" >&2 exit 1 fi @@ -133,8 +206,8 @@ if ! printf '%s\n' "$doctor_out" | grep -q 'daemon status: live'; then printf '%s\n' "$doctor_out" >&2 exit 1 fi -if ! printf '%s\n' "$doctor_out" | grep -q 'daemon pid namespace: '; then - echo "expected doctor output to surface the daemon pid namespace for debugging" >&2 +if ! printf '%s\n' "$doctor_out" | grep -q 'daemon pid domain: '; then + echo "expected doctor output to surface the daemon pid domain for debugging" >&2 printf '%s\n' "$doctor_out" >&2 exit 1 fi @@ -143,7 +216,7 @@ sandbox_beam ensure lean > /dev/null daemon_id_2="$(read_json_field "$registry" daemonId)" port_2="$(read_json_field "$registry" port)" -pid_ns_2="$(read_json_field "$registry" pidNamespace 2>/dev/null || true)" +pid_domain_2="$(read_json_field "$registry" pidDomain 2>/dev/null || true)" if [ "$daemon_id_1" != "$daemon_id_2" ]; then echo "expected PID-isolated wrapper ensure to reuse the existing daemon instead of starting a new one" >&2 @@ -159,10 +232,10 @@ if [ "$port_1" != "$port_2" ]; then exit 1 fi -if [ "$pid_ns_1" != "$pid_ns_2" ]; then - echo "expected PID-isolated wrapper ensure to preserve the recorded daemon pid namespace" >&2 - printf 'before pid namespace: %s\n' "$pid_ns_1" >&2 - printf 'after pid namespace: %s\n' "$pid_ns_2" >&2 +if [ "$pid_domain_1" != "$pid_domain_2" ]; then + echo "expected PID-isolated wrapper ensure to preserve the recorded daemon PID domain" >&2 + printf 'before PID domain: %s\n' "$pid_domain_1" >&2 + printf 'after PID domain: %s\n' "$pid_domain_2" >&2 exit 1 fi @@ -172,7 +245,7 @@ hold_pid="" find "$control_root" -mindepth 1 -maxdepth 1 -exec rm -rf -- {} + -sandbox_beam ensure lean >"$owner_out" 2>"$owner_err" & +sandbox_owner_hold "$owner_out" "$owner_err" "$owner_stop" & owner_pid="$!" if ! wait_for_registry; then echo "expected owner sandbox wrapper request to create a control-dir registry" >&2 @@ -180,7 +253,8 @@ if ! wait_for_registry; then exit 1 fi follower_version="$(beam_wrapper_update_version "sandbox SlowPoll" sandbox_beam update tests/scenario/docs/SlowPoll.lean)" -BEAM_PROGRESS=1 BEAM_REQUEST_ID=wrapper-sandbox-follower \ +follower_request_id="wrapper-sandbox-follower" +BEAM_PROGRESS=1 BEAM_REQUEST_ID="$follower_request_id" \ sandbox_beam run-at tests/scenario/docs/SlowPoll.lean "$follower_version" 25 2 poll_sleep_cmd \ >"$follower_out" 2>"$follower_err" & follower_pid="$!" @@ -199,6 +273,22 @@ if ! wait_for_nonempty_file "$owner_out" "owner sandbox ensure response"; then exit 1 fi +touch "$owner_stop" + +# The original owner-lifetime implementation stopped tracking siblings after a fixed +# 30-second polling window. Keep the follower active beyond that boundary so this test +# proves ownership follows the request lifetime rather than an elapsed-duration guess. +sleep 31 + +if ! kill -0 "$owner_pid" 2>/dev/null; then + echo "expected the owner sandbox wrapper to outlive a follower active for more than 30 seconds" >&2 + cat "$owner_out" >&2 + cat "$owner_err" >&2 + cat "$follower_out" >&2 + cat "$follower_err" >&2 + exit 1 +fi + if ! kill -0 "$follower_pid" 2>/dev/null; then echo "expected the follower sandbox request to stay alive while the owner request finishes" >&2 cat "$owner_out" >&2 @@ -218,6 +308,7 @@ wait "$follower_pid" follower_status=$? set -e follower_pid="" +follower_request_id="" if [ "$follower_status" = "0" ]; then echo "expected follower sandbox wrapper request to exit non-zero after cancellation" >&2 @@ -229,7 +320,7 @@ fi follower_json="$(cat "$follower_out")" if ! python3 -c 'import json,sys; payload=json.load(sys.stdin); raise SystemExit(0 if payload.get("error", {}).get("code") == "requestCancelled" else 1)' <<<"$follower_json" then - echo "expected follower sandbox wrapper request to report requestCancelled after SIGINT" >&2 + echo "expected follower sandbox wrapper request to report requestCancelled after cancellation" >&2 printf '%s\n' "$follower_json" >&2 cat "$follower_err" >&2 exit 1 @@ -237,3 +328,258 @@ fi wait "$owner_pid" owner_pid="" + +assert_no_connection_closed_incidents "the long-lived follower regression" + +# Heartbeat expiry is a revocation decision, not permission to kill broker work that was already +# admitted. Suspend the entire follower wrapper beyond the timeout, let the owner revoke its +# filesystem lease, then resume it. The daemon-side active-request fence must keep the owner alive; +# the resumed wrapper must cancel cleanly instead of reconnecting through the removed lease. +find "$control_root" -mindepth 1 -maxdepth 1 -exec rm -rf -- {} + +owner_out="$tmp_root/paused-owner.out" +owner_err="$tmp_root/paused-owner.err" +owner_stop="$tmp_root/paused-owner.stop" +follower_out="$tmp_root/paused-follower.out" +follower_err="$tmp_root/paused-follower.err" +pause_follower="$tmp_root/pause-follower" +follower_paused="$tmp_root/follower-paused" +resume_follower="$tmp_root/resume-follower" + +sandbox_owner_hold "$owner_out" "$owner_err" "$owner_stop" & +owner_pid="$!" +if ! wait_for_registry; then + echo "expected paused-follower owner to create a control-dir registry" >&2 + cat "$owner_err" >&2 + exit 1 +fi +paused_version="$(beam_wrapper_update_version "paused sandbox SlowPoll" sandbox_beam update tests/scenario/docs/SlowPoll.lean)" +follower_request_id="wrapper-sandbox-paused-follower" +sandbox_paused_follower "$paused_version" "$follower_request_id" \ + "$follower_out" "$follower_err" "$pause_follower" "$follower_paused" "$resume_follower" & +follower_pid="$!" + +if ! wait_for_file_text "$follower_err" "snapshot progress" "paused follower request progress"; then + cat "$owner_out" >&2 + cat "$owner_err" >&2 + cat "$follower_out" >&2 + cat "$follower_err" >&2 + exit 1 +fi +touch "$pause_follower" +if ! wait_for_file "$follower_paused" "paused follower acknowledgement" 12; then + cat "$follower_err" >&2 + exit 1 +fi +touch "$owner_stop" + +revoked_lease="" +for _ in $(seq 1 120); do + revoked_lease="$(find "$control_root" -name '*.revoked' -print -quit)" + if [ -n "$revoked_lease" ]; then + break + fi + sleep 0.1 +done +if [ -z "$revoked_lease" ]; then + echo "expected the suspended follower lease to receive a persistent revocation tombstone" >&2 + find "$control_root" -name '*.lease' -print -exec cat {} \; >&2 + exit 1 +fi +if ! kill -0 "$owner_pid" 2>/dev/null; then + echo "expected the daemon owner to stay alive for a revoked but still-admitted request" >&2 + cat "$owner_out" >&2 + cat "$owner_err" >&2 + cat "$follower_out" >&2 + cat "$follower_err" >&2 + exit 1 +fi + +touch "$resume_follower" +set +e +wait "$follower_pid" +paused_follower_status="$?" +set -e +follower_pid="" +follower_request_id="" +if [ "$paused_follower_status" -eq 0 ]; then + echo "expected the resumed wrapper to fail after observing lease revocation" >&2 + cat "$follower_out" >&2 + cat "$follower_err" >&2 + exit 1 +fi +if ! python3 -c 'import json,sys; payload=json.load(sys.stdin); raise SystemExit(0 if payload.get("error", {}).get("code") == "requestCancelled" else 1)' < "$follower_out" +then + echo "expected the resumed wrapper request to report requestCancelled" >&2 + cat "$follower_out" >&2 + cat "$follower_err" >&2 + exit 1 +fi +if ! wait_for_exit "$owner_pid" "owner after suspended follower cancellation" 120 0.1; then + cat "$owner_out" >&2 + cat "$owner_err" >&2 + cat "$follower_out" >&2 + cat "$follower_err" >&2 + exit 1 +fi +wait "$owner_pid" +owner_pid="" + +assert_no_connection_closed_incidents "suspended-follower revocation" + +# A follower killed from outside its PID namespace cannot remove its own lease. Its heartbeat +# must expire so the owner can drain, retire the generation, and permit a later clean ensure. +find "$control_root" -mindepth 1 -maxdepth 1 -exec rm -rf -- {} + +owner_out="$tmp_root/killed-owner.out" +owner_err="$tmp_root/killed-owner.err" +owner_stop="$tmp_root/killed-owner.stop" +follower_out="$tmp_root/killed-follower.out" +follower_err="$tmp_root/killed-follower.err" + +sandbox_owner_hold "$owner_out" "$owner_err" "$owner_stop" & +owner_pid="$!" +if ! wait_for_registry; then + echo "expected killed-follower owner to create a control-dir registry" >&2 + cat "$owner_err" >&2 + exit 1 +fi +killed_version="$(beam_wrapper_update_version "killed sandbox SlowPoll" sandbox_beam update tests/scenario/docs/SlowPoll.lean)" +follower_request_id="wrapper-sandbox-killed-follower" +BEAM_PROGRESS=1 BEAM_REQUEST_ID="$follower_request_id" \ + sandbox_beam run-at tests/scenario/docs/SlowPoll.lean "$killed_version" 25 2 poll_sleep_cmd \ + >"$follower_out" 2>"$follower_err" & +follower_pid="$!" + +if ! wait_for_file_text "$follower_err" "snapshot progress" "killed follower request progress"; then + cat "$owner_out" >&2 + cat "$owner_err" >&2 + cat "$follower_out" >&2 + cat "$follower_err" >&2 + exit 1 +fi +touch "$owner_stop" +kill -KILL "$follower_pid" > /dev/null 2>&1 || true +set +e +wait "$follower_pid" 2>/dev/null +set -e +follower_pid="" +follower_request_id="" + +if ! wait_for_exit "$owner_pid" "owner waiting on killed cross-namespace follower" 120 0.1; then + cat "$owner_out" >&2 + cat "$owner_err" >&2 + find "$control_root" -name '*.lease' -print -exec cat {} \; >&2 + exit 1 +fi +wait "$owner_pid" +owner_pid="" + +recovery_json="$(sandbox_beam ensure lean)" +if [ "$(json_text_field "$recovery_json" ok)" != "true" ]; then + echo "expected ensure to recover after pruning a killed follower lease" >&2 + printf '%s\n' "$recovery_json" >&2 + exit 1 +fi + +# Replacing a daemon while its original starter is still active creates two starter leases. The +# obsolete starter must notice the registry generation change before waiting on the replacement +# owner's lease, or both owners keep each other's heartbeats alive forever. +find "$control_root" -mindepth 1 -maxdepth 1 -exec rm -rf -- {} + +replacement_a_out="$tmp_root/replacement-a.out" +replacement_a_err="$tmp_root/replacement-a.err" +replacement_a_stop="$tmp_root/replacement-a.stop" +replacement_b_out="$tmp_root/replacement-b.out" +replacement_b_err="$tmp_root/replacement-b.err" +replacement_b_stop="$tmp_root/replacement-b.stop" + +sandbox_owner_hold "$replacement_a_out" "$replacement_a_err" "$replacement_a_stop" & +replacement_owner_a_pid="$!" +if ! wait_for_registry || ! wait_for_nonempty_file "$replacement_a_out" "original replacement owner response"; then + cat "$replacement_a_out" >&2 + cat "$replacement_a_err" >&2 + exit 1 +fi +replacement_daemon_a="$(read_json_field "$registry" daemonId)" + +replacement_shutdown_json="$(sandbox_beam shutdown)" +if ! python3 -c 'import json,sys; payload=json.load(sys.stdin); raise SystemExit(0 if payload.get("result", {}).get("shutdown") is True else 1)' <<<"$replacement_shutdown_json" +then + echo "expected replacement setup to shut down the original daemon" >&2 + printf '%s\n' "$replacement_shutdown_json" >&2 + exit 1 +fi + +sandbox_owner_hold "$replacement_b_out" "$replacement_b_err" "$replacement_b_stop" & +replacement_owner_b_pid="$!" +if ! wait_for_registry || ! wait_for_nonempty_file "$replacement_b_out" "replacement owner response"; then + cat "$replacement_a_out" >&2 + cat "$replacement_a_err" >&2 + cat "$replacement_b_out" >&2 + cat "$replacement_b_err" >&2 + exit 1 +fi +replacement_daemon_b="$(read_json_field "$registry" daemonId)" +if [ "$replacement_daemon_a" = "$replacement_daemon_b" ]; then + echo "expected successive PID-isolated daemon starts to receive distinct generation ids" >&2 + printf 'original daemonId: %s\nreplacement daemonId: %s\n' \ + "$replacement_daemon_a" "$replacement_daemon_b" >&2 + exit 1 +fi + +touch "$replacement_a_stop" +if ! wait_for_exit "$replacement_owner_a_pid" "obsolete daemon generation owner" 100 0.1; then + cat "$replacement_a_out" >&2 + cat "$replacement_a_err" >&2 + cat "$replacement_b_out" >&2 + cat "$replacement_b_err" >&2 + find "$control_root" -name '*.lease' -print -exec cat {} \; >&2 + exit 1 +fi +wait "$replacement_owner_a_pid" +replacement_owner_a_pid="" + +if ! kill -0 "$replacement_owner_b_pid" 2>/dev/null; then + echo "expected the replacement daemon owner to remain active after the obsolete owner exited" >&2 + cat "$replacement_b_out" >&2 + cat "$replacement_b_err" >&2 + exit 1 +fi +touch "$replacement_b_stop" +if ! wait_for_exit "$replacement_owner_b_pid" "replacement daemon generation owner" 120 0.1; then + cat "$replacement_b_out" >&2 + cat "$replacement_b_err" >&2 + find "$control_root" -name '*.lease' -print -exec cat {} \; >&2 + exit 1 +fi +wait "$replacement_owner_b_pid" +replacement_owner_b_pid="" + +assert_no_connection_closed_incidents "replacement generation ownership" + +# Supplement the deterministic lifetime cases with a cold-start fanout. Every wrapper should +# complete successfully through serialized admission without connection loss. +find "$control_root" -mindepth 1 -maxdepth 1 -exec rm -rf -- {} + +fanout_count=8 +fanout_pids=() +for i in $(seq 1 "$fanout_count"); do + sandbox_beam ensure lean >"$tmp_root/fanout-$i.out" 2>"$tmp_root/fanout-$i.err" & + fanout_pids+=("$!") +done +fanout_failed=false +for pid in ${fanout_pids[@]+"${fanout_pids[@]}"}; do + if ! wait "$pid"; then + fanout_failed=true + fi +done +for i in $(seq 1 "$fanout_count"); do + if [ "$(json_file_text_field "$tmp_root/fanout-$i.out" ok)" != "true" ]; then + echo "expected cold-start fanout wrapper $i to succeed" >&2 + cat "$tmp_root/fanout-$i.out" >&2 + cat "$tmp_root/fanout-$i.err" >&2 + fanout_failed=true + fi +done +if [ "$fanout_failed" = "true" ]; then + exit 1 +fi + +assert_no_connection_closed_incidents "the sandbox lifecycle regressions" From 184ed185b731409d46aadc33845ddb2a56bdf49f Mon Sep 17 00:00:00 2001 From: Emilio Jesus Gallego Arias Date: Tue, 25 Aug 2026 22:29:46 +0200 Subject: [PATCH 02/28] refactor: make wrapper daemon ownership explicit --- Beam/Broker/Pending.lean | 40 +- Beam/Broker/Protocol.lean | 39 +- Beam/Broker/Server.lean | 167 ++--- Beam/Cli/Broker.lean | 17 +- Beam/Cli/Commands.lean | 27 +- Beam/Cli/DaemonManager.lean | 680 +++++------------- Beam/Cli/Usage.lean | 5 +- Beam/Daemon/Debug.lean | 2 +- Beam/Daemon/Ownership.lean | 103 --- Beam/Daemon/Paths.lean | 6 - Beam/Daemon/Protocol.lean | 2 + CHANGELOG.md | 8 +- README.md | 3 +- docs/CUSTOM_TOOLCHAINS.md | 2 +- docs/DEVELOPMENT.md | 122 ++-- docs/ROCQ.md | 9 +- docs/SETUP.md | 20 +- docs/STATUS.md | 30 +- docs/SYNC_AND_DIAGNOSTICS.md | 9 +- docs/TESTING.md | 16 +- scripts/lean-beam | 3 +- skills/lean-beam/SKILL.md | 47 +- skills/lean-beam/references/mcts-search.md | 8 +- .../lean-beam/references/workflow-details.md | 4 +- skills/rocq-beam/SKILL.md | 28 +- tests/lean/BeamTest/Broker/CliDaemonTest.lean | 70 +- tests/lean/BeamTest/Broker/PendingTest.lean | 2 - tests/lean/BeamTest/Broker/ProtocolTest.lean | 168 +---- .../lean/BeamTest/Broker/StreamDedupTest.lean | 1 - tests/lib/beam-wrapper-common.sh | 27 +- tests/test-beam-fast.sh | 32 +- tests/test-beam-install.sh | 48 +- tests/test-beam-save-olean.sh | 60 +- tests/test-beam-toolchain-compat.sh | 47 ++ tests/test-beam-wrapper-daemon.sh | 621 +++++----------- tests/test-beam-wrapper-diagnostics.sh | 7 + tests/test-beam-wrapper-handle.sh | 1 + tests/test-beam-wrapper-probe.sh | 1 + tests/test-beam-wrapper-rocq.sh | 15 +- tests/test-beam-wrapper-runtime.sh | 18 +- tests/test-beam-wrapper-sandbox.sh | 614 +++++----------- tests/test-beam-wrapper-sync-save.sh | 2 + tests/test-stage0-toolchain.sh | 12 +- 43 files changed, 1078 insertions(+), 2065 deletions(-) delete mode 100644 Beam/Daemon/Ownership.lean diff --git a/Beam/Broker/Pending.lean b/Beam/Broker/Pending.lean index 48919c04..3b86cc7b 100644 --- a/Beam/Broker/Pending.lean +++ b/Beam/Broker/Pending.lean @@ -325,6 +325,7 @@ structure ActiveRequest where private structure ActiveRequestRegistryState where nextToken : Nat := 1 + accepting : Bool := true requests : Std.TreeMap String ActiveRequest := {} anonymousRequests : Std.TreeMap Nat ActiveRequest := {} @@ -342,6 +343,11 @@ def register let cancelRef ← IO.mkRef false registry.mutex.atomically do let state ← get + unless state.accepting do + return .error { + code := .requestCancelled + message := "Beam session owner is closing" + } match clientRequestId? with | none => let active : ActiveRequest := { clientRequestId?, token := state.nextToken, cancelRef } @@ -391,23 +397,23 @@ def count (registry : ActiveRequestRegistry) : IO Nat := do let state ← get pure (state.requests.size + state.anonymousRequests.size) -def countExcluding - (registry : ActiveRequestRegistry) - (excluded : ActiveRequest) : IO Nat := do - registry.mutex.atomically do - let state ← get - let total := state.requests.size + state.anonymousRequests.size - let exactAdmissionPresent : Bool := - match excluded.clientRequestId? with - | some clientRequestId => - match state.requests.get? clientRequestId with - | some current => current.token == excluded.token - | none => false - | none => - match state.anonymousRequests.get? excluded.token with - | some current => current.token == excluded.token - | none => false - pure <| if exactAdmissionPresent then total - 1 else total +/-- +Atomically close request admission and mark every admitted request for cancellation. Return `true` +only to the caller that changed the registry from accepting to closed. +-/ +def closeAdmission (registry : ActiveRequestRegistry) : IO Bool := do + let (firstClose, active) ← registry.mutex.atomically do + let state : ActiveRequestRegistryState ← get + if !state.accepting then + pure (false, #[]) + else + set { state with accepting := false } + let named := state.requests.toList.map Prod.snd |>.toArray + let anonymous := state.anonymousRequests.toList.map Prod.snd |>.toArray + pure (true, named ++ anonymous) + for request in active do + request.cancelRef.set true + pure firstClose def markCancelled (registry : ActiveRequestRegistry) diff --git a/Beam/Broker/Protocol.lean b/Beam/Broker/Protocol.lean index 1572fff8..63da0b78 100644 --- a/Beam/Broker/Protocol.lean +++ b/Beam/Broker/Protocol.lean @@ -182,30 +182,6 @@ structure Handle where raw : Json deriving Inhabited, FromJson, ToJson -/-- -Internal wrapper-to-daemon fencing identity. It is attached by `lean-beam`, not supplied by normal -broker or MCP clients. --/ -structure WrapperLeaseContext where - daemonId : String - leaseFile : String - deriving Inhabited, ToJson, BEq, Repr - -instance : FromJson WrapperLeaseContext where - fromJson? json := do - match json with - | .obj fields => - let allowed := #["daemonId", "leaseFile"] - let unexpected := fields.foldl (init := #[]) fun unexpected field _ => - if allowed.contains field then unexpected else unexpected.push field - unless unexpected.isEmpty do - throw s!"wrapper lease accepts no undeclared fields: {String.intercalate ", " unexpected.toList}" - | other => throw s!"wrapper lease must be an object, got {other.compress}" - pure { - daemonId := ← json.getObjValAs? String "daemonId" - leaseFile := ← json.getObjValAs? String "leaseFile" - } - /-- Select which user-facing Lean diagnostic severities a request may display. -/ inductive DiagnosticScope where | errors @@ -231,7 +207,6 @@ structure Request where workspaceId? : Option WorkspaceId := none workspaceMode? : Option Beam.Workspace.InitMode := none clientRequestId? : Option String := none - wrapperLease? : Option WrapperLeaseContext := none cancelRequestId? : Option String := none root? : Option String := none path? : Option String := none @@ -275,8 +250,8 @@ def Op.workspaceScope : Op → WorkspaceScope | .codeActionResolve | .saveOlean | .goals | .todo | .runWith | .release | .initWorkspace | .dropWorkspace => .required -/-- Whether an operation participates in wrapper lease fencing and active-request tracking. -/ -def Op.acceptsWrapperLease : Op → Bool +/-- Whether an operation participates in active-request tracking and exact cancellation. -/ +def Op.tracksActiveRequest : Op → Bool | .cancel | .shutdown => false | .ensure | .openDocs | .updateFile | .syncFile | .refreshFile | .close | .runAt | .hover | .signatureHelp | .definition | .references | .documentSymbols | .workspaceSymbols @@ -284,7 +259,7 @@ def Op.acceptsWrapperLease : Op → Bool | .listWorkspaces | .dropWorkspace | .stats | .resetStats => true private def Op.optionalRequestFields (op : Op) : Array String := - #["clientRequestId", "wrapperLease"] ++ + #["clientRequestId"] ++ (match op.workspaceScope with | .none => #[] | .optional | .required => #["workspaceId"]) ++ @@ -339,7 +314,6 @@ private def Request.optionalJsonFields (req : Request) : List (String × Json) : optionalJsonField "workspaceId" req.workspaceId? ++ optionalJsonField "workspaceMode" req.workspaceMode? ++ optionalJsonField "clientRequestId" req.clientRequestId? ++ - optionalJsonField "wrapperLease" req.wrapperLease? ++ optionalJsonField "cancelRequestId" req.cancelRequestId? ++ optionalJsonField "root" req.root? ++ optionalJsonField "path" req.path? ++ @@ -394,10 +368,6 @@ def Request.validateFields (req : Request) : Except String Unit := do throw s!"broker op '{req.op.key}' accepts no unrelated fields: {String.intercalate ", " unexpected.toList}" if !req.op.usesBackend && req.backend != .lean then throw s!"broker op '{req.op.key}' does not select a backend" - if req.wrapperLease?.isSome && req.clientRequestId?.isNone then - throw "broker requests carrying 'wrapperLease' require 'clientRequestId'" - if req.wrapperLease?.isSome && !req.op.acceptsWrapperLease then - throw s!"broker op '{req.op.key}' does not accept 'wrapperLease'" if (req.op == .stats || req.op == .openDocs) && req.root?.isSome && req.workspaceId?.isNone then throw s!"broker op '{req.op.key}' requires 'workspaceId' when 'root' is present" @@ -421,7 +391,6 @@ instance : FromJson Request where let workspaceId? ← optionalField? (α := WorkspaceId) j "workspaceId" let workspaceMode? ← optionalField? (α := Beam.Workspace.InitMode) j "workspaceMode" let clientRequestId? ← optionalField? (α := String) j "clientRequestId" - let wrapperLease? ← optionalField? (α := WrapperLeaseContext) j "wrapperLease" let cancelRequestId? ← optionalField? (α := String) j "cancelRequestId" let root? ← optionalField? (α := String) j "root" let path? ← optionalField? (α := String) j "path" @@ -449,7 +418,7 @@ instance : FromJson Request where let handle? ← optionalField? (α := Handle) j "handle" let codeAction? ← optionalField? (α := Lsp.CodeAction) j "codeAction" let request : Request := { - op, backend, workspaceId?, workspaceMode?, clientRequestId?, wrapperLease?, cancelRequestId?, + op, backend, workspaceId?, workspaceMode?, clientRequestId?, cancelRequestId?, root?, path?, version?, line?, character?, endLine?, endCharacter?, text?, query?, includeDeclaration?, kinds?, suggest?, storeHandle?, linear?, mode?, compact?, ppFormat?, diagnosticScope?, diagnosticsInResult?, diff --git a/Beam/Broker/Server.lean b/Beam/Broker/Server.lean index e5a7f1b3..ec6fa274 100644 --- a/Beam/Broker/Server.lean +++ b/Beam/Broker/Server.lean @@ -24,8 +24,6 @@ import Beam.Broker.Lean import Beam.Broker.LakeSave import Beam.Broker.Readiness import Beam.Broker.SyncResult -import Beam.Daemon.Ownership -import Beam.Daemon.Paths import Beam.LSP.Save import Beam.Path import Std.Sync.Mutex @@ -37,8 +35,6 @@ open IO.FS.Stream namespace Beam.Broker -open Beam.Daemon.Ownership - abbrev brokerStdio : IO.Process.StdioConfig where stdin := .piped stdout := .piped @@ -892,8 +888,6 @@ structure ServerRuntime where endpoint : Transport.Endpoint stop : IO.Ref Bool activeRequests : ActiveRequestRegistry - root : System.FilePath - daemonId? : Option String := none /-- A cancellation capability bound to one active broker request admission. @@ -914,18 +908,14 @@ def ServerRuntime.withState (server : ServerRuntime) (act : M α) : IO α := do private def ServerRuntime.statsResponse (server : ServerRuntime) - (currentRequest : ActiveRequest) (workspaceId? : Option WorkspaceId := none) : IO Response := do - let activeRequestCount ← - ActiveRequestRegistry.countExcluding server.activeRequests currentRequest let payload ← server.withState <| statsPayload workspaceId? - pure <| Response.success <| payload.setObjVal! "activeRequestCount" (toJson activeRequestCount) + pure <| Response.success payload def ServerRuntime.create (config : BrokerConfig) (workspaceId : WorkspaceId) - (endpoint : Transport.Endpoint := .tcp 0) - (daemonId? : Option String := none) : IO ServerRuntime := do + (endpoint : Transport.Endpoint := .tcp 0) : IO ServerRuntime := do unless validWorkspaceId workspaceId do throw <| IO.userError "workspace id must be non-empty" let startMonoNanos ← IO.monoNanosNow @@ -935,8 +925,6 @@ def ServerRuntime.create endpoint := endpoint stop := ← IO.mkRef false activeRequests := ← ActiveRequestRegistry.create - root := config.root - daemonId? } private def brokerConfigSame (left right : BrokerConfig) : Bool := @@ -1099,72 +1087,6 @@ def RequestHandle.cancel (handle : RequestHandle) : IO Bool := do cancelRegisteredRequest handle.runtime <| ActiveRequestRegistry.markCancelledActive handle.runtime.activeRequests active -private inductive WrapperLeaseInactiveReason where - | generationMismatch - | invalidFileName - | revoked - | missing - | malformed - | invalidPid - | unreadable - -private def WrapperLeaseInactiveReason.key : WrapperLeaseInactiveReason → String - | .generationMismatch => "generationMismatch" - | .invalidFileName => "invalidFileName" - | .revoked => "revoked" - | .missing => "missing" - | .malformed => "malformed" - | .invalidPid => "invalidPid" - | .unreadable => "unreadable" - -private inductive WrapperLeaseValidation where - | active - | inactive (reason : WrapperLeaseInactiveReason) - -/-- -Validate the wrapper's filesystem fence after active-request registration. Retirement first writes -the revocation tombstone and only then observes the active-request count, so a fenced request that -passes this check is necessarily visible to the retiring owner before broker work begins. - -Unfenced broker clients remain valid; their lifecycle must be owned independently or protected by -a foreground wrapper owner such as `lean-beam ensure --hold`. --/ -private def ServerRuntime.validateWrapperLease - (server : ServerRuntime) - (req : Request) : IO WrapperLeaseValidation := do - if !req.op.acceptsWrapperLease then - return .active - let some lease := req.wrapperLease? - | return .active - try - unless server.daemonId? == some lease.daemonId do - return .inactive .generationMismatch - unless validWrapperLeaseFileName lease.leaseFile do - return .inactive .invalidFileName - let leasePath := (← Beam.Daemon.wrapperLeaseDir server.root) / lease.leaseFile - let revocationPath := wrapperLeaseRevocationPath leasePath - if ← revocationPath.pathExists then - return .inactive .revoked - unless ← leasePath.pathExists do - return .inactive .missing - let text ← IO.FS.readFile leasePath - let json ← - match Json.parse text with - | .ok json => pure json - | .error _ => return .inactive .malformed - let metadata : WrapperLeaseMetadata ← - match fromJson? (α := WrapperLeaseMetadata) json with - | .ok metadata => pure metadata - | .error _ => return .inactive .malformed - if metadata.pid == 0 then - return .inactive .invalidPid - if ← revocationPath.pathExists then - pure (.inactive .revoked) - else - pure .active - catch _ => - pure (.inactive .unreadable) - private def propagatePendingCancellation (session : Session) (cancelRef? : Option (IO.Ref Bool)) : IO Unit := do @@ -1173,8 +1095,9 @@ private def propagatePendingCancellation private def requestStop (server : ServerRuntime) : IO Unit := do server.stop.set true try - let conn ← Transport.connect server.endpoint - Transport.closeConnection conn + -- Wake the blocking accept. Both ends are intentionally left to scope cleanup: performing a + -- graceful TCP shutdown on the wake-up pair can wait for its peer and deadlock daemon exit. + discard <| Transport.connect server.endpoint catch _ => pure () @@ -2204,22 +2127,24 @@ private def handleRequestIO let cancelRef? := activeRequest?.map (·.cancelRef) match req.op with | .shutdown => - let resp ← server.withState do - let state ← get - for (_, workspace) in state.workspaces.toList do - shutdownWorkspaceSessions workspace - pure <| Response.success (Json.mkObj [("shutdown", toJson true)]) - pure (resp, true) + let firstClose ← ActiveRequestRegistry.closeAdmission server.activeRequests + if firstClose then + let resp ← server.withState do + let state ← get + for (_, workspace) in state.workspaces.toList do + shutdownWorkspaceSessions workspace + pure <| Response.success (Json.mkObj [("shutdown", toJson true)]) + pure (resp, true) + else + pure (Response.success (Json.mkObj [("shutdown", toJson true)]), false) | .stats => - let some currentRequest := activeRequest? - | unreachable! match req.workspaceId? with - | none => pure (← server.statsResponse currentRequest, false) + | none => pure (← server.statsResponse, false) | some _ => match ← validateRequestWorkspace server req with | .error failure => pure (failure.toResponse, false) | .ok workspaceReq => - pure (← server.statsResponse currentRequest (some workspaceReq.workspaceId), false) + pure (← server.statsResponse (some workspaceReq.workspaceId), false) | .listWorkspaces => let payload ← server.withState do pure <| workspaceListPayload (← get) @@ -2330,7 +2255,7 @@ private def ServerRuntime.withRequestAdmission | .ok () => pure () try let active? ← - if req.op.acceptsWrapperLease then + if req.op.tracksActiveRequest then match ← ActiveRequestRegistry.register server.activeRequests req.clientRequestId? with | .ok active => pure (some active) | .error failure => @@ -2340,19 +2265,6 @@ private def ServerRuntime.withRequestAdmission else pure none try - match ← server.validateWrapperLease req with - | .inactive reason => - let resp := BrokerFailure.toResponse { - code := .requestCancelled - message := "wrapper daemon-lifetime lease is no longer active" - data? := some <| Json.mkObj [ - ("reason", toJson "wrapperLeaseInactive"), - ("leaseState", toJson reason.key) - ] - } - recordDispatchMetrics server req resp startedAt - return (resp, false) - | .active => pure () let handle : RequestHandle := { runtime := server, active? } let (resp, shouldStop) ← act handle traceBroker @@ -2420,12 +2332,23 @@ private partial def watchRoot (server : ServerRuntime) (root : System.FilePath) pure false if !rootAvailable then IO.eprintln s!"Beam daemon root is no longer available; shutting down: {root}" - discard <| server.dispatchRequest { op := .shutdown } - requestStop server + let (_, shouldStop) ← server.dispatchRequest { op := .shutdown } + if shouldStop then + requestStop server else IO.sleep rootWatchPollMs watchRoot server root +private def watchSessionOwnerStdin (server : ServerRuntime) : IO Unit := do + try + discard <| (← IO.getStdin).readToEnd + catch _ => + pure () + unless ← server.stop.get do + let (_, shouldStop) ← server.dispatchRequest { op := .shutdown } + if shouldStop then + requestStop server + private def watchClientDisconnect (client : Transport.Connection) (handle : RequestHandle) : IO Unit := do @@ -2493,7 +2416,7 @@ private partial def acceptLoop (server : ServerRuntime) (listener : Transport.Li else let client ← Transport.accept listener if ← server.stop.get then - Transport.closeConnection client + pure () else let _ ← IO.asTask (prio := Task.Priority.dedicated) do try @@ -2506,7 +2429,7 @@ private structure CliOptions where endpoint : Transport.Endpoint := .tcp 8765 root? : Option String := none workspaceId? : Option WorkspaceId := none - daemonId? : Option String := none + sessionOwnerStdin : Bool := false leanCmd? : Option String := none leanPlugin? : Option String := none rocqCmd? : Option String := none @@ -2532,8 +2455,8 @@ private partial def parseCliOptions (opts : CliOptions) : List String → Except parseCliOptions { opts with root? := some root } rest | "--workspace-id" :: workspaceId :: rest => parseCliOptions { opts with workspaceId? := some workspaceId } rest - | "--daemon-id" :: daemonId :: rest => - parseCliOptions { opts with daemonId? := some daemonId } rest + | "--session-owner-stdin" :: rest => + parseCliOptions { opts with sessionOwnerStdin := true } rest | "--lean-cmd" :: leanCmd :: rest => parseCliOptions { opts with leanCmd? := some leanCmd } rest | "--lean-plugin" :: leanPlugin :: rest => @@ -2551,9 +2474,6 @@ def main (args : List String) : IO Unit := do | throw <| IO.userError "missing Beam daemon --workspace-id ID" unless validWorkspaceId workspaceId do throw <| IO.userError "workspace id must be non-empty" - if let some daemonId := opts.daemonId? then - if daemonId.isEmpty then - throw <| IO.userError "daemon id must be non-empty" let root ← Beam.resolveExistingPath <| System.FilePath.mk root let leanPlugin? ← opts.leanPlugin?.mapM (fun path => Beam.resolveExistingPath <| System.FilePath.mk path) let config : BrokerConfig := { @@ -2563,21 +2483,20 @@ def main (args : List String) : IO Unit := do rocqCmd? := opts.rocqCmd? } let listener ← Transport.bindAndListen opts.endpoint 16 - let startMonoNanos ← IO.monoNanosNow - let runtime : ServerRuntime := { - state := ← Std.Mutex.new (mkInitialState config workspaceId startMonoNanos) - endpoint := opts.endpoint - stop := ← IO.mkRef false - activeRequests := ← ActiveRequestRegistry.create - root - daemonId? := opts.daemonId? - } + let runtime ← ServerRuntime.create config workspaceId opts.endpoint let rootWatcher ← IO.asTask (prio := Task.Priority.dedicated) <| watchRoot runtime root + let ownerWatcher? ← + if opts.sessionOwnerStdin then + some <$> IO.asTask (prio := Task.Priority.dedicated) (watchSessionOwnerStdin runtime) + else + pure none try acceptLoop runtime listener finally runtime.stop.set true Transport.closeListener listener + if let some ownerWatcher := ownerWatcher? then + IO.cancel ownerWatcher discard <| IO.wait rootWatcher end Beam.Broker diff --git a/Beam/Cli/Broker.lean b/Beam/Cli/Broker.lean index 68a52a5d..ed7529fd 100644 --- a/Beam/Cli/Broker.lean +++ b/Beam/Cli/Broker.lean @@ -63,14 +63,6 @@ private structure WrapperBrokerRequest where request : Request visibleClientRequestId? : Option String -private def attachProjectDaemonLease - (client : ProjectDaemonClient) - (req : Request) : Request := - if req.op.acceptsWrapperLease then - { req with wrapperLease? := some client.wrapperLease } - else - req - private def mkWrapperClientRequestId (req : Request) : IO String := do let pid ← IO.Process.getPID let stamp ← IO.monoNanosNow @@ -92,9 +84,8 @@ private def withWrapperClientRequestId (req : Request) : IO WrapperBrokerRequest } private def prepareWrapperBrokerRequest - (client : ProjectDaemonClient) (req : Request) : IO WrapperBrokerRequest := - withWrapperClientRequestId <| attachProjectDaemonLease client (inProjectDaemonWorkspace req) + withWrapperClientRequestId <| inProjectDaemonWorkspace req private def mkInterruptWatcher? (clientRequestId? : Option String) : IO (Option InterruptWatcher) := do match clientRequestId? with @@ -201,14 +192,14 @@ private def requestBrokerResponse (client : ProjectDaemonClient) (req : Request) : IO WrapperBrokerResponse := withBrokerErrorContext root do - let wrapperReq ← prepareWrapperBrokerRequest client req + let wrapperReq ← prepareWrapperBrokerRequest req let req := wrapperReq.request let response ← awaitBrokerResponseWithInterrupts client.endpoint req wrapperReq.visibleClientRequestId? none <| sendRequest client.endpoint req pure { response, visibleClientRequestId? := wrapperReq.visibleClientRequestId? } -/-- Send one lease-fenced wrapper request without printing or interpreting its response. -/ +/-- Send one wrapper request without printing or interpreting its response. -/ def requestBroker (root : System.FilePath) (client : ProjectDaemonClient) @@ -431,7 +422,7 @@ def callBrokerWithProgress (req : Request) (spec : BrokerWaitSpec) : IO Unit := withBrokerErrorContext root do - let wrapperReq ← prepareWrapperBrokerRequest client req + let wrapperReq ← prepareWrapperBrokerRequest req let req := wrapperReq.request let visibleClientRequestId? := wrapperReq.visibleClientRequestId? let showProgress ← progressEnabled diff --git a/Beam/Cli/Commands.lean b/Beam/Cli/Commands.lean index 29ffaece..0cff0e4d 100644 --- a/Beam/Cli/Commands.lean +++ b/Beam/Cli/Commands.lean @@ -107,8 +107,10 @@ private def shutdownProjectDaemon (opts : CliOptions) : IO Unit := do if let some endpoint := Beam.Daemon.registryEndpoint? entry then let resp ← sendRequest endpoint { op := .shutdown } printResponse resp - finishRegistryDaemonShutdown entry + -- Revoking the published generation tells its wrapper owner to close the inherited + -- owner pipe. That unblocks the daemon's stdin watcher during an explicit shutdown. removeRegistry root + finishRegistryDaemonShutdown entry else stopRegisteredDaemon root printJsonLine <| Json.mkObj [ @@ -123,7 +125,9 @@ private def shutdownProjectDaemon (opts : CliOptions) : IO Unit := do private def backendOfName (name : String) : Backend := if name == "rocq" then .rocq else .lean -private def runThenHoldUntilInterrupted (act : IO Unit) : IO Unit := do +private def runThenHoldUntilInterrupted + (owner : ProjectDaemonOwner) + (act : IO Unit) : IO Unit := do let signal ← Std.Internal.UV.Signal.mk 2 false let promise ← Std.Internal.UV.Signal.next signal let task ← IO.asTask (prio := Task.Priority.dedicated) do @@ -132,12 +136,16 @@ private def runThenHoldUntilInterrupted (act : IO Unit) : IO Unit := do pure () try act - while !(← IO.hasFinished task) && !(← IO.checkCanceled) do + while !(← IO.hasFinished task) && !(← owner.exited) && + (← owner.registered) && !(← IO.checkCanceled) do IO.sleep 50 if ← IO.hasFinished task then match ← IO.wait task with | .ok () => pure () | .error err => throw err + else if let some exitCode ← owner.exitCode? then + unless exitCode == 0 do + throw <| IO.userError s!"owned Beam daemon exited with status {exitCode}" finally Std.Internal.UV.Signal.stop signal @@ -147,15 +155,16 @@ private def ensureBackend (backend : Backend) (hold : Bool := false) : IO Unit := do let root ← projectRoot opts backend - withProjectDaemon home root backend opts fun client => - if hold then - runThenHoldUntilInterrupted do - callBroker root client { + if hold then + withProjectDaemonOwner home root backend opts fun owner => + runThenHoldUntilInterrupted owner do + callBroker root owner.client { op := .ensure, backend := backend, root? := some root.toString } (← IO.getStdout).flush - IO.eprintln "beam: holding ensured daemon; interrupt this wrapper process when finished" - else + IO.eprintln "beam: owning Beam session; interrupt this wrapper process when finished" + else + withProjectDaemon home root backend opts fun client => callBroker root client { op := .ensure, backend := backend, root? := some root.toString } def runCommand (home : System.FilePath) (opts : CliOptions) : IO Unit := do diff --git a/Beam/Cli/DaemonManager.lean b/Beam/Cli/DaemonManager.lean index e9bb6055..8242a971 100644 --- a/Beam/Cli/DaemonManager.lean +++ b/Beam/Cli/DaemonManager.lean @@ -11,16 +11,12 @@ import Beam.Cli.Args import Beam.Cli.Lock import Beam.Cli.Project import Beam.Daemon.Debug -import Beam.Daemon.Ownership import Beam.Daemon.Paths -import Std.Internal.UV.Timer open Lean namespace Beam.Cli -open Beam.Daemon.Ownership - open Beam.Broker open Beam.Daemon @@ -99,13 +95,18 @@ private partial def waitForRecordedPidGone | .invalid | .local false | .differentDomain | .unknownDomain => pure () +private def gracefulDaemonShutdownWaitTries : Nat := + 50 + /-- Finish a graceful daemon shutdown with a PID fallback only when the registry PID belongs to the current process domain. A PID from another or unknown domain must never be probed or killed. -/ def finishRegistryDaemonShutdown (entry : RegistryEntry) : IO Unit := do let recorded : Beam.RecordedPid := { pid := entry.pid, domain? := entry.pidDomain? } - waitForRecordedPidGone recorded + -- A broker may spend up to three seconds completing the bounded LSP shutdown path before its + -- accept loop exits. Do not turn that orderly session close into SIGTERM just before it finishes. + waitForRecordedPidGone recorded gracefulDaemonShutdownWaitTries match ← recorded.observe with | .local true => if ← recorded.terminateIfLocal then @@ -312,19 +313,22 @@ private def startupFailureMessage (endpoint : Transport.Endpoint) (logPath : Sys pure msg private abbrev daemonStdio : IO.Process.StdioConfig where - stdin := .null + stdin := .piped stdout := .null stderr := .null private partial def waitForDaemonChildExit - (child : IO.Process.Child daemonStdio) + {cfg : IO.Process.StdioConfig} + (child : IO.Process.Child cfg) (tries : Nat := 20) : IO Unit := do if tries == 0 || (← child.tryWait).isSome then return IO.sleep 100 waitForDaemonChildExit child (tries - 1) -private def terminateDaemonChild (child : IO.Process.Child daemonStdio) : IO Unit := do +private def terminateDaemonChild + {cfg : IO.Process.StdioConfig} + (child : IO.Process.Child cfg) : IO Unit := do try if (← child.tryWait).isNone then child.kill @@ -335,12 +339,11 @@ private def terminateDaemonChild (child : IO.Process.Child daemonStdio) : IO Uni private def startDaemon (desired : DesiredConfig) (endpoint : Transport.Endpoint) - (logPath : System.FilePath) - (daemonId : String) : IO (IO.Process.Child daemonStdio) := do + (logPath : System.FilePath) : IO (IO.Process.Child daemonStdio) := do let mut args : List String := [ "--root", desired.root.toString, "--workspace-id", projectDaemonWorkspaceId, - "--daemon-id", daemonId + "--session-owner-stdin" ] match endpoint with | .tcp port => @@ -355,7 +358,7 @@ private def startDaemon IO.FS.createDirAll parent IO.FS.writeFile logPath "" let cmd := String.intercalate " " ((desired.daemonBin.toString :: args).map shellQuote) - let shell := s!"exec {cmd} >{shellQuote logPath.toString} 2>&1 < /dev/null" + let shell := s!"exec {cmd} >{shellQuote logPath.toString} 2>&1" let child ← IO.Process.spawn { toStdioConfig := daemonStdio cmd := "sh" @@ -400,10 +403,13 @@ private def registryEntryFor match endpoint with | .tcp port => some port.toNat let pidDomain? ← Beam.currentPidDomain? + let ownerPid ← IO.Process.getPID pure { daemonId pid pidDomain? + ownerPid := ownerPid.toNat + ownerPidDomain? := pidDomain? port? root := desired.root.toString configHash := desired.configHash @@ -421,11 +427,11 @@ private def registryEntryFor private partial def startDaemonEntry (desired : DesiredConfig) (opts : CliOptions) - (tries : Nat := 10) : IO (Transport.Endpoint × RegistryEntry) := do + (tries : Nat := 10) : IO (Transport.Endpoint × RegistryEntry × IO.Process.Child daemonStdio) := do let endpoint ← selectUnoccupiedEndpoint desired opts let logPath ← daemonStartupLogPath desired.root let daemonId ← newDaemonGenerationId desired.configHash - let child ← startDaemon desired endpoint logPath daemonId + let child ← startDaemon desired endpoint logPath try waitForDaemon child endpoint logPath desired.root catch err => @@ -437,7 +443,7 @@ private partial def startDaemonEntry throw err let pid := child.pid.toNat let entry ← registryEntryFor desired daemonId pid endpoint opts - pure (endpoint, entry) + pure (endpoint, entry, child) def desiredConfig (home root : System.FilePath) (required : Backend) : IO DesiredConfig := do let defaultPaths ← defaultBundlePaths home @@ -490,533 +496,191 @@ def desiredConfig (home root : System.FilePath) (required : Backend) : IO Desire configHash } -def registryLiveFor (root : System.FilePath) (expectedHash? : Option String := none) : IO (Option RegistryEntry) := do +structure ProjectDaemonClient where + endpoint : Transport.Endpoint + +private def projectDaemonClient (entry : RegistryEntry) : IO ProjectDaemonClient := do + pure { + endpoint := ← Beam.Daemon.endpointFromEntry entry + } + +private def registryOwnerObservable (entry : RegistryEntry) : IO Bool := do + if entry.ownerPid == 0 then + return false + let recorded : Beam.RecordedPid := { + pid := entry.ownerPid + domain? := entry.ownerPidDomain? + } + match ← recorded.observe with + | .invalid | .local false => pure false + | .local true | .differentDomain | .unknownDomain => pure true + +def registryLiveFor + (root : System.FilePath) + (expectedHash? : Option String := none) : IO (Option RegistryEntry) := do match ← readRegistry? root with | none => pure none | some entry => let rootOk ← Beam.sameFilePath (System.FilePath.mk entry.root) root let hashOk := expectedHash?.map (· == entry.configHash) |>.getD true - if !rootOk || !hashOk then + if !rootOk || !hashOk || !(← registryOwnerObservable entry) then pure none - else if let some endpoint := registryEndpoint? entry then - -- PID observations are not a liveness fallback across isolated sandboxes; only a - -- root-matching endpoint proves that this registry entry is live. - if ← daemonServesRoot endpoint projectDaemonWorkspaceId root then - pure (some entry) - else - pure none else - pure none - -private def wrapperLeaseHeartbeatIntervalMs : UInt32 := - 250 - -private def wrapperLeaseHeartbeatTimeoutNanos : Nat := - 5000000000 - -private def wrapperLeaseHeartbeatWriteRetryMs : UInt32 := - 50 - -private def wrapperLeaseHeartbeatWriteRetries : Nat := - 3 - -private def wrapperLifecyclePollMs : UInt32 := - 50 - -private def daemonRequestProbeTimeoutMs : Nat := - 1000 - -private def wrapperLeaseActionCancelWaitMs : Nat := - 5000 - -private structure WrapperLease where - root : System.FilePath - path : System.FilePath - stopHeartbeat : IO.Ref Bool - heartbeatTimer : Std.Internal.UV.Timer - heartbeatTask : Task (Except IO.Error Unit) - -/-- Internal typed target for one wrapper request to a daemon generation. -/ -structure ProjectDaemonClient where - endpoint : Transport.Endpoint - wrapperLease : WrapperLeaseContext + match registryEndpoint? entry with + | none => pure none + | some endpoint => + -- The owner pipe makes endpoint liveness authoritative across PID domains. A + -- same-domain dead owner is rejected immediately; another domain is never probed. + if ← daemonServesRoot endpoint projectDaemonWorkspaceId root then + pure (some entry) + else + pure none -private structure DaemonRetirement where - daemonId : String - ownerLeaseFile : String - deriving FromJson, ToJson +private abbrev detachedDaemonStdio : IO.Process.StdioConfig where + stdin := .null + stdout := .null + stderr := .null -private inductive EnsuredProjectDaemon where - | reused (client : ProjectDaemonClient) (lease : WrapperLease) - | started (client : ProjectDaemonClient) (lease : WrapperLease) +private structure OwnedProjectDaemon where + client : ProjectDaemonClient + entry : RegistryEntry + child : IO.Process.Child daemonStdio + +structure ProjectDaemonOwner where + client : ProjectDaemonClient + private root : System.FilePath + private daemonId : String + private child : IO.Process.Child daemonStdio + private exitCodeRef : IO.Ref (Option UInt32) + +def ProjectDaemonOwner.exitCode? (owner : ProjectDaemonOwner) : IO (Option UInt32) := do + match ← owner.exitCodeRef.get with + | some exitCode => pure (some exitCode) + | none => + let exitCode? ← owner.child.tryWait + if let some exitCode := exitCode? then + owner.exitCodeRef.set (some exitCode) + pure exitCode? -private def projectDaemonClient - (endpoint : Transport.Endpoint) - (daemonId : String) - (lease : WrapperLease) : ProjectDaemonClient := - { - endpoint - wrapperLease := { - daemonId - leaseFile := lease.path.fileName.getD lease.path.toString - } - } +def ProjectDaemonOwner.exited (owner : ProjectDaemonOwner) : IO Bool := + return (← owner.exitCode?).isSome -private def removeFileIfExists (path : System.FilePath) : IO Unit := do - if ← path.pathExists then - IO.FS.removeFile path +/-- Whether this owner generation is still the one published for its project. -/ +def ProjectDaemonOwner.registered (owner : ProjectDaemonOwner) : IO Bool := do + match ← readRegistry? owner.root with + | some current => pure (current.daemonId == owner.daemonId) + | none => pure false -private def removeFileIfExistsBestEffort (path : System.FilePath) : IO Unit := do - try - removeFileIfExists path - catch _ => - pure () +private def activeOwnerMessage (root : System.FilePath) (entry : RegistryEntry) : String := + s!"Beam session for {root} is already owned by wrapper pid {entry.ownerPid}; " ++ + "interrupt that 'lean-beam ensure --hold' process before starting another owner" -private def ensureWrapperLeaseNotRevoked (path : System.FilePath) : IO Unit := do - if ← (wrapperLeaseRevocationPath path).pathExists then - throw <| IO.userError s!"wrapper daemon-lifetime lease was revoked: {path}" +private def missingOwnerMessage (root : System.FilePath) : String := + s!"no live Beam session owner is registered for {root}; " ++ + "start 'lean-beam ensure --hold' for this project and keep it running while using wrapper commands" -private def writeWrapperLeaseMetadata - (path : System.FilePath) - (metadata : WrapperLeaseMetadata) : IO Unit := do - ensureWrapperLeaseNotRevoked path - let tmp := path.withExtension "tmp" - IO.FS.writeFile tmp ((toJson metadata).pretty ++ "\n") - try - ensureWrapperLeaseNotRevoked path - catch err => - removeFileIfExistsBestEffort tmp - throw err - IO.FS.rename tmp path +private def startOwnedProjectDaemon + (desired : DesiredConfig) + (opts : CliOptions) : IO OwnedProjectDaemon := do + if let some live ← registryLiveFor desired.root then + throw <| IO.userError (activeOwnerMessage desired.root live) + -- A non-live registry may refer to a daemon still winding down after owner loss. Ask that exact + -- root-matching endpoint to stop, and use PID fallback only through the typed domain boundary. + stopRegisteredDaemon desired.root + let (endpoint, entry, child) ← startDaemonEntry desired opts try - ensureWrapperLeaseNotRevoked path + writeRegistry desired.root entry catch err => - removeFileIfExistsBestEffort path + terminateDaemonChild child throw err - -private partial def writeWrapperLeaseHeartbeat - (path : System.FilePath) - (metadata : WrapperLeaseMetadata) - (retries : Nat := wrapperLeaseHeartbeatWriteRetries) : IO Unit := do - try - writeWrapperLeaseMetadata path metadata - catch err => - if retries == 0 then - throw err - IO.sleep wrapperLeaseHeartbeatWriteRetryMs - writeWrapperLeaseHeartbeat path metadata (retries - 1) - -private partial def wrapperLeaseHeartbeatLoop - (path : System.FilePath) - (metadata : WrapperLeaseMetadata) - (stop : IO.Ref Bool) - (timer : Std.Internal.UV.Timer) - (tick : IO.Promise Unit) : IO Unit := do - let tickResult ← IO.wait tick.result? - if ← stop.get then - return - let some _ := tickResult - | return - let heartbeatMonoNanos ← IO.monoNanosNow - let metadata := { metadata with heartbeatMonoNanos } - writeWrapperLeaseHeartbeat path metadata - wrapperLeaseHeartbeatLoop path metadata stop timer (← timer.next) - -private def acquireWrapperLease (root : System.FilePath) : IO WrapperLease := do - let dir ← wrapperLeaseDir root - IO.FS.createDirAll dir - let pid ← IO.Process.getPID - let stamp ← IO.monoNanosNow - let nonce := ByteArray.toUInt64LE! (← IO.getRandomBytes 8) - let path := dir / s!"{stamp}-{pid}-{nonce}.lease" - let metadata : WrapperLeaseMetadata := { - pid := pid.toNat - pidDomain? := ← Beam.currentPidDomain? - heartbeatMonoNanos := stamp + pure { + client := { endpoint } + entry + child } - writeWrapperLeaseMetadata path metadata - let stopHeartbeat ← IO.mkRef false - let heartbeatTimer ← Std.Internal.UV.Timer.mk wrapperLeaseHeartbeatIntervalMs.toUInt64 true - let firstTick ← heartbeatTimer.next - let heartbeatTask ← IO.asTask (prio := Task.Priority.dedicated) <| - wrapperLeaseHeartbeatLoop path metadata stopHeartbeat heartbeatTimer firstTick - pure { root, path, stopHeartbeat, heartbeatTimer, heartbeatTask } - -private def stopWrapperLeaseHeartbeat (lease : WrapperLease) : IO Unit := do - lease.stopHeartbeat.set true - Std.Internal.UV.Timer.stop lease.heartbeatTimer - match ← IO.wait lease.heartbeatTask with - | .ok () => pure () - | .error err => throw err - -private def releaseWrapperLease (lease : WrapperLease) : IO Unit := do - try - stopWrapperLeaseHeartbeat lease - finally - removeFileIfExistsBestEffort lease.path - removeFileIfExistsBestEffort (wrapperLeaseRevocationPath lease.path) - -private def readWrapperLeaseMetadata? (path : System.FilePath) : IO (Option WrapperLeaseMetadata) := do - unless ← path.pathExists do - return none - let text ← - try - IO.FS.readFile path - catch err => - if ← path.pathExists then - throw err - else - return none - match Json.parse text with - | .error _ => pure none - | .ok json => - match fromJson? json with - | .error _ => pure none - | .ok metadata => pure (some metadata) - -private def staleWrapperLease? (path : System.FilePath) : IO Bool := do - if ← (wrapperLeaseRevocationPath path).pathExists then - return true - match ← readWrapperLeaseMetadata? path with - | none => pure true - | some metadata => - let now ← IO.monoNanosNow - let pidObservation ← - (Beam.RecordedPid.mk metadata.pid metadata.pidDomain?).observe - pure <| wrapperLeaseStaleFromObservation pidObservation now - wrapperLeaseHeartbeatTimeoutNanos metadata - -private def revokeWrapperLease (path : System.FilePath) : IO Unit := do - let revocationPath := wrapperLeaseRevocationPath path - unless ← revocationPath.pathExists do - let tmp := revocationPath.withExtension "revoking" - let revocation : WrapperLeaseRevocation := { revokedMonoNanos := ← IO.monoNanosNow } - IO.FS.writeFile tmp ((toJson revocation).pretty ++ "\n") - IO.FS.rename tmp revocationPath - removeFileIfExists path - -private def reapAndObserveOtherWrapperLeases (lease : WrapperLease) : - IO OtherWrapperLeasesObservation := do - try - let dir ← wrapperLeaseDir lease.root - unless ← dir.pathExists do - return .drained - let entries ← dir.readDir - for entry in entries do - if entry.path != lease.path && entry.fileName.endsWith ".lease" then - let stale? ← - try - pure <| some (← staleWrapperLease? entry.path) - catch _ => - pure none - match stale? with - | none => return .activeOrUnreadable - | some true => - try - revokeWrapperLease entry.path - catch _ => - return .activeOrUnreadable - | some false => return .activeOrUnreadable - pure .drained - catch _ => - pure .activeOrUnreadable - -private def removeDaemonRetirement (root : System.FilePath) : IO Unit := do - removeFileIfExists (← daemonRetirementPath root) - -private inductive OwnershipRegistryRead where - | missing - | invalid - | unreadable (error : IO.Error) - | present (entry : RegistryEntry) - -private def readOwnershipRegistry (root : System.FilePath) : IO OwnershipRegistryRead := do - let path ← registryPath root - try - unless ← path.pathExists do - return .missing - let text ← IO.FS.readFile path - match Json.parse text with - | .error _ => pure .invalid - | .ok json => - match fromJson? json with - | .error _ => pure .invalid - | .ok entry => pure (.present entry) - catch err => - pure (.unreadable err) - -private def readOrDiscardInvalidDaemonRetirement? - (root : System.FilePath) : IO (Option DaemonRetirement) := do - let path ← daemonRetirementPath root - unless ← path.pathExists do - return none - let text ← IO.FS.readFile path - let retirement? := do - let json ← Json.parse text - fromJson? json - match retirement? with - | .ok retirement => pure (some retirement) - | .error _ => - removeDaemonRetirement root - pure none - -private def writeDaemonRetirement - (root : System.FilePath) - (retirement : DaemonRetirement) : IO Unit := do - let path ← daemonRetirementPath root - if let some parent := path.parent then - IO.FS.createDirAll parent - let tmp := path.withExtension "tmp" - IO.FS.writeFile tmp ((toJson retirement).pretty ++ "\n") - IO.FS.rename tmp path - -/-- Reconcile stale or invalid retirement state and report whether it still blocks admission. -/ -private def reconcileRetirementAdmission (lease : WrapperLease) : IO Bool := do - let some retirement ← readOrDiscardInvalidDaemonRetirement? lease.root - | return false - let registry ← - match ← readOwnershipRegistry lease.root with - | .missing | .invalid => - removeDaemonRetirement lease.root - return false - | .unreadable err => throw err - | .present registry => pure registry - if registry.daemonId != retirement.daemonId then - removeDaemonRetirement lease.root - return false - unless validWrapperLeaseFileName retirement.ownerLeaseFile do - removeDaemonRetirement lease.root - return false - let ownerPath := (← wrapperLeaseDir lease.root) / retirement.ownerLeaseFile - if ← staleWrapperLease? ownerPath then - revokeWrapperLease ownerPath - removeDaemonRetirement lease.root - pure false - else - pure true - -private partial def ensureProjectDaemonUnderLease - (desired : DesiredConfig) - (opts : CliOptions) - (lease : WrapperLease) : IO EnsuredProjectDaemon := do - let admitted? ← withProjectControlLock desired.root do - if ← reconcileRetirementAdmission lease then - pure none - else - if let some live ← registryLiveFor desired.root desired.configHash then - if let some endpoint := registryEndpoint? live then - return some <| EnsuredProjectDaemon.reused - (projectDaemonClient endpoint live.daemonId lease) lease - removeRegistry desired.root - let live? ← registryLiveFor desired.root - if live?.isNone then - removeRegistry desired.root - let (endpoint, entry) ← startDaemonEntry desired opts - writeRegistry desired.root entry - if let some live := live? then - unless live.pid == entry.pid && live.port? == entry.port? do - stopDaemonEntry live - pure <| some <| EnsuredProjectDaemon.started - (projectDaemonClient endpoint entry.daemonId lease) lease - match admitted? with - | some daemon => pure daemon - | none => - IO.sleep wrapperLifecyclePollMs - ensureProjectDaemonUnderLease desired opts lease -private def acquireWrapperLeaseForAdmission (root : System.FilePath) : IO WrapperLease := - withProjectControlLock root do - acquireWrapperLease root - -private def admitProjectDaemon - (home root : System.FilePath) - (backend : Backend) - (opts : CliOptions) : IO EnsuredProjectDaemon := do - let lease ← acquireWrapperLeaseForAdmission root - try - let desired ← desiredConfig home root backend - ensureProjectDaemonUnderLease desired opts lease - catch err => - releaseWrapperLease lease - throw err +private def closeDaemonOwnerPipe + (child : IO.Process.Child daemonStdio) : + IO (IO.Process.Child detachedDaemonStdio) := do + let (_ownerPipe, child) ← child.takeStdin + pure child -private def requestDaemonStatsWithin - (endpoint : Transport.Endpoint) : IO (Option Response) := do - let task ← IO.asTask (prio := Task.Priority.dedicated) <| - sendRequest endpoint { op := .stats } - let mut remainingMs := daemonRequestProbeTimeoutMs - while !(← IO.hasFinished task) && remainingMs > 0 do - IO.sleep wrapperLifecyclePollMs - remainingMs := remainingMs - min remainingMs wrapperLifecyclePollMs.toNat - if ← IO.hasFinished task then - match ← IO.wait task with - | .ok response => pure (some response) - | .error _ => pure none +private partial def waitForOwnedDaemonExit + (child : IO.Process.Child detachedDaemonStdio) + (exitCodeRef : IO.Ref (Option UInt32)) + (tries : Nat) : IO Unit := do + if (← exitCodeRef.get).isSome || tries == 0 then + return + if let some exitCode ← child.tryWait then + exitCodeRef.set (some exitCode) else - IO.cancel task - pure none - -private def registryDaemonProvenGone (registry : RegistryEntry) : IO Bool := do - let recorded : Beam.RecordedPid := { pid := registry.pid, domain? := registry.pidDomain? } - match ← recorded.observe with - | .local false => pure true - | .local true => pure <| (← recorded.zombieIfLocal?).getD false - | .invalid | .differentDomain | .unknownDomain => pure false + IO.sleep 100 + waitForOwnedDaemonExit child exitCodeRef (tries - 1) -private def observeDaemonRequests - (registry : RegistryEntry) - (endpoint : Transport.Endpoint) : IO DaemonRequestsObservation := do +private def removeOwnedRegistry (root : System.FilePath) (daemonId : String) : IO Unit := do try - let some resp ← requestDaemonStatsWithin endpoint - | return if ← registryDaemonProvenGone registry then .provenGone else .activeOrUnreadable - unless resp.ok do - return .activeOrUnreadable - let some result := resp.result? - | return .activeOrUnreadable - let activeRequestCount ← IO.ofExcept <| result.getObjValAs? Nat "activeRequestCount" - pure <| if activeRequestCount == 0 then .drained else .activeOrUnreadable + withProjectControlLock root do + match ← readRegistry? root with + | some current => + if current.daemonId == daemonId then + removeRegistry root + | none => pure () catch _ => - pure .activeOrUnreadable - -private def tryCommitDaemonRetirement - (client : ProjectDaemonClient) - (lease : WrapperLease) : IO RetirementDecision := do - withProjectControlLock lease.root do - let observation ← - match ← readOwnershipRegistry lease.root with - | .present registry => - if registry.daemonId != client.wrapperLease.daemonId then - pure RetirementObservation.replacement - else - let otherLeases ← reapAndObserveOtherWrapperLeases lease - let daemonRequests ← - match otherLeases with - | .drained => observeDaemonRequests registry client.endpoint - | .activeOrUnreadable => pure .activeOrUnreadable - pure <| RetirementObservation.current otherLeases daemonRequests - | .missing | .invalid | .unreadable _ => - pure <| RetirementObservation.unavailable (← reapAndObserveOtherWrapperLeases lease) - match retirementDecision observation with - | .wait => pure .wait - | .obsolete => pure .obsolete - | .commit => - let ownerLeaseFile := lease.path.fileName.getD lease.path.toString - writeDaemonRetirement lease.root { - daemonId := client.wrapperLease.daemonId - ownerLeaseFile - } - pure .commit - -private partial def retireStartedProjectDaemon - (client : ProjectDaemonClient) - (lease : WrapperLease) : IO Bool := do - match ← tryCommitDaemonRetirement client lease with - | .commit => pure true - | .obsolete => pure false - | .wait => - IO.sleep wrapperLifecyclePollMs - retireStartedProjectDaemon client lease - -private def finishStartedProjectDaemonAdmission - (client : ProjectDaemonClient) - (lease : WrapperLease) : IO Unit := do - if ← retireStartedProjectDaemon client lease then - -- Leave the final heartbeat on disk. A successor with matching PID-domain identity can - -- prove this process exited by PID; an unknown or different domain waits for the heartbeat - -- to expire before it clears the retirement fence and observes the daemon endpoint. - stopWrapperLeaseHeartbeat lease - else - releaseWrapperLease lease - -private partial def awaitWrapperLeaseAction - (lease : WrapperLease) - (actionTask : Task (Except IO.Error α)) : IO α := do - if ← IO.hasFinished actionTask then - match ← IO.wait actionTask with - | .ok value => pure value - | .error err => throw err - else if ← IO.hasFinished lease.heartbeatTask then - let heartbeatResult ← IO.wait lease.heartbeatTask - IO.cancel actionTask - let mut remainingMs := wrapperLeaseActionCancelWaitMs - while !(← IO.hasFinished actionTask) && remainingMs > 0 do - IO.sleep wrapperLifecyclePollMs - remainingMs := remainingMs - min remainingMs wrapperLifecyclePollMs.toNat - match heartbeatResult with - | .ok () => throw <| IO.userError "wrapper lease heartbeat stopped unexpectedly" - | .error err => throw err - else - IO.sleep wrapperLifecyclePollMs - awaitWrapperLeaseAction lease actionTask - -private def runWhileWrapperLeaseHealthy - (lease : WrapperLease) - (act : IO α) : IO α := do - let actionTask ← IO.asTask (prio := Task.Priority.dedicated) act - awaitWrapperLeaseAction lease actionTask + pure () -def withProjectDaemon +def withProjectDaemonOwner (home root : System.FilePath) (backend : Backend) (opts : CliOptions) - (act : ProjectDaemonClient → IO α) : IO α := do - match ← admitProjectDaemon home root backend opts with - | .reused client lease => - let result ← - try - pure <| Except.ok (← runWhileWrapperLeaseHealthy lease (act client)) - catch err => - pure <| Except.error err - releaseWrapperLease lease - match result with - | .ok value => pure value - | .error err => throw err - | .started client lease => - -- The starter owns this daemon generation for its process lifetime. Unlike a reuser, it - -- must finish already-admitted work even if its filesystem heartbeat becomes unhealthy, - -- then retain ownership through broker draining and retirement. - let result ← - try - pure <| Except.ok (← act client) - catch err => - pure <| Except.error err - finishStartedProjectDaemonAdmission client lease - match result with - | .ok value => pure value - | .error err => throw err - -private partial def lookupProjectDaemonUnderLease (lease : WrapperLease) : IO ProjectDaemonClient := do - let endpoint? ← withProjectControlLock lease.root do - if ← reconcileRetirementAdmission lease then - pure none - else - match ← registryLiveFor lease.root with - | some entry => - let endpoint ← Beam.Daemon.endpointFromEntry entry - pure <| some <| projectDaemonClient endpoint entry.daemonId lease - | none => - let msg ← daemonFailureMessage lease.root s!"no live Beam daemon registered for {lease.root}" - stopRegisteredDaemon lease.root - throw <| IO.userError msg - match endpoint? with - | some endpoint => pure endpoint - | none => - IO.sleep wrapperLifecyclePollMs - lookupProjectDaemonUnderLease lease - -def withExistingProjectDaemon - (root : System.FilePath) - (act : ProjectDaemonClient → IO α) : IO α := do - let lease ← acquireWrapperLeaseForAdmission root + (act : ProjectDaemonOwner → IO α) : IO α := do + let desired ← desiredConfig home root backend + let owned ← withProjectControlLock root do + startOwnedProjectDaemon desired opts + let exitCodeRef ← IO.mkRef (none : Option UInt32) let result ← try - let client ← lookupProjectDaemonUnderLease lease - pure <| Except.ok (← runWhileWrapperLeaseHealthy lease (act client)) + pure <| Except.ok (← act { + client := owned.client + root + daemonId := owned.entry.daemonId + child := owned.child + exitCodeRef + }) catch err => pure <| Except.error err - releaseWrapperLease lease + let child ← closeDaemonOwnerPipe owned.child + waitForOwnedDaemonExit child exitCodeRef 100 + if (← exitCodeRef.get).isNone then + try + child.kill + catch _ => + pure () + waitForOwnedDaemonExit child exitCodeRef 20 + removeOwnedRegistry root owned.entry.daemonId match result with | .ok value => pure value | .error err => throw err +private def lookupProjectDaemon + (root : System.FilePath) + (expectedHash? : Option String := none) : IO ProjectDaemonClient := do + withProjectControlLock root do + match ← registryLiveFor root expectedHash? with + | some entry => projectDaemonClient entry + | none => + stopRegisteredDaemon root + throw <| IO.userError (missingOwnerMessage root) + +def withProjectDaemon + (home root : System.FilePath) + (backend : Backend) + (_opts : CliOptions) + (act : ProjectDaemonClient → IO α) : IO α := do + let desired ← desiredConfig home root backend + act (← lookupProjectDaemon root (some desired.configHash)) + +def withExistingProjectDaemon + (root : System.FilePath) + (act : ProjectDaemonClient → IO α) : IO α := do + act (← lookupProjectDaemon root) end Beam.Cli diff --git a/Beam/Cli/Usage.lean b/Beam/Cli/Usage.lean index e8f13872..9c3b54e9 100644 --- a/Beam/Cli/Usage.lean +++ b/Beam/Cli/Usage.lean @@ -59,8 +59,9 @@ def usage : String := "For multiline text-carrying Lean probes, prefer --stdin or --text-file ; use -- before", "text that itself starts with --.", "For handle-based commands, use --handle-file when you do not want to inline handle json.", - "Use ensure --hold when a PID-isolated command runner needs one foreground wrapper process", - "to keep a newly-started daemon alive across separate wrapper invocations.", + "Wrapper commands require one live session owner. Start ensure --hold in a foreground process", + "and keep it running across wrapper invocations; interrupt it or run shutdown when finished.", + "Plain ensure checks and warms that owned session but never starts one implicitly.", "Beam does not upload or submit feedback. Use feedback-report to print a pasteable report", "with cheap version, stats, open-files, daemon registry, and daemon incident context.", "Review non-confidential reports before posting because they may contain project context and", diff --git a/Beam/Daemon/Debug.lean b/Beam/Daemon/Debug.lean index 8ec84e20..47f1704b 100644 --- a/Beam/Daemon/Debug.lean +++ b/Beam/Daemon/Debug.lean @@ -115,7 +115,7 @@ private def jsonNonNullField (json : Json) (field : String) : Bool := def daemonDebugWarnings (debug : Json) : Array String := Id.run do let mut warnings := #[] - let recoveryHint := "Run `lean-beam shutdown`, then `lean-beam ensure` from the project root to refresh daemon registry state." + let recoveryHint := "Run `lean-beam shutdown`, then start `lean-beam ensure --hold` from the project root to refresh the owned session." if jsonNonNullField debug "registry" then match jsonStringField? debug "registryPidStatus" with | some "not alive" => diff --git a/Beam/Daemon/Ownership.lean b/Beam/Daemon/Ownership.lean deleted file mode 100644 index c0db3ca5..00000000 --- a/Beam/Daemon/Ownership.lean +++ /dev/null @@ -1,103 +0,0 @@ -/- -Copyright (c) 2026 Lean FRO LLC. All rights reserved. -Released under Apache 2.0 license as described in the file LICENSE. -Author: Emilio J. Gallego Arias --/ - -import Lean -import Beam.System - -open Lean - -namespace Beam.Daemon.Ownership - -/-- Internal on-disk metadata for one wrapper's daemon-lifetime lease. -/ -structure WrapperLeaseMetadata where - pid : Nat - pidDomain? : Option String := none - heartbeatMonoNanos : Nat - deriving FromJson, ToJson - -/-- Persistent evidence that an expired lease basename may no longer be renewed or used. -/ -structure WrapperLeaseRevocation where - revokedMonoNanos : Nat - deriving ToJson - -private def wrapperLeaseHeartbeatExpired - (now heartbeat timeout : Nat) : Bool := - now < heartbeat || now - heartbeat > timeout - -/-- Pure lease-staleness policy, separated from PID and clock observation. -/ -def wrapperLeaseStaleFromObservation - (pidObservation : Beam.RecordedPidObservation) - (now timeout : Nat) - (metadata : WrapperLeaseMetadata) : Bool := - metadata.pid == 0 || - wrapperLeaseHeartbeatExpired now metadata.heartbeatMonoNanos timeout || - match pidObservation with - | .invalid => true - | .local alive => !alive - | .differentDomain | .unknownDomain => false - -/-- Retirement markers may refer only to one lease basename inside `wrapper-leases`. -/ -def validWrapperLeaseFileName (name : String) : Bool := - !name.isEmpty && - name.endsWith ".lease" && - !(name.contains '/') && - !(name.contains '\\') - -/-- The tombstone paired with one wrapper lease path. -/ -def wrapperLeaseRevocationPath (path : System.FilePath) : System.FilePath := - path.withExtension "revoked" - -/-- Conservative summary of every lease other than the starter's own lease. -/ -inductive OtherWrapperLeasesObservation where - | drained - | activeOrUnreadable - deriving BEq, Repr - -/-- Whether the daemon still owns broker requests admitted before retirement fencing. -/ -inductive DaemonRequestsObservation where - | drained - | activeOrUnreadable - | provenGone - deriving BEq, Repr - -/-- Typed input to the retirement policy. Replacement generations need no sibling inspection. -/ -inductive RetirementObservation where - | current - (otherLeases : OtherWrapperLeasesObservation) - (daemonRequests : DaemonRequestsObservation) - | replacement - | unavailable (otherLeases : OtherWrapperLeasesObservation) - deriving BEq, Repr - -inductive RetirementDecision where - | wait - | commit - | obsolete - deriving BEq, Repr - -/-- -Decide the starter's next step without performing filesystem mutations. - -A proven replacement or a provably dead current daemon makes the starter obsolete immediately, -avoiding a generation-to-generation lease deadlock. A live current generation commits retirement -only after both sibling leases and admitted broker requests drain. Missing, malformed, or unreadable -registry state can release the starter only after all sibling leases are provably drained. --/ -def retirementDecision - (observation : RetirementObservation) : RetirementDecision := - match observation with - | .replacement => .obsolete - | .current others requests => - match others, requests with - | .drained, .drained => .commit - | .drained, .provenGone => .obsolete - | _, _ => .wait - | .unavailable others => - match others with - | .drained => .obsolete - | .activeOrUnreadable => .wait - -end Beam.Daemon.Ownership diff --git a/Beam/Daemon/Paths.lean b/Beam/Daemon/Paths.lean index 4f2b54f8..1fd5e23e 100644 --- a/Beam/Daemon/Paths.lean +++ b/Beam/Daemon/Paths.lean @@ -25,12 +25,6 @@ def registryPath (root : System.FilePath) : IO System.FilePath := do def daemonStartupLogPath (root : System.FilePath) : IO System.FilePath := do pure ((← controlDir root) / "beam-daemon-startup.log") -def wrapperLeaseDir (root : System.FilePath) : IO System.FilePath := do - pure ((← controlDir root) / "wrapper-leases") - -def daemonRetirementPath (root : System.FilePath) : IO System.FilePath := do - pure ((← controlDir root) / "daemon-retirement.json") - def daemonFailureIncidentDir (root : System.FilePath) : IO System.FilePath := do pure ((← controlDir root) / "daemon-failures") diff --git a/Beam/Daemon/Protocol.lean b/Beam/Daemon/Protocol.lean index 8867973b..cf4aef0d 100644 --- a/Beam/Daemon/Protocol.lean +++ b/Beam/Daemon/Protocol.lean @@ -19,6 +19,8 @@ structure RegistryEntry where daemonId : String pid : Nat pidDomain? : Option String := none + ownerPid : Nat + ownerPidDomain? : Option String := none port? : Option Nat := none root : String configHash : String diff --git a/CHANGELOG.md b/CHANGELOG.md index 6bc83563..873a534c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -33,6 +33,10 @@ This project keeps a lightweight, reverse-chronological changelog. Dates use `YY ### Changed +- Wrapper daemons now have explicit session ownership: only `lean-beam ensure --hold` starts a + generation, ordinary wrapper commands attach to it, and holder exit closes the daemon through an + inherited pipe without heartbeat leases or retirement fences + ([#241](https://github.com/leanprover/lean-beam/pull/241), @ejgallego). - Long-running Lean operations now separate liveness status, request progress, and diagnostics. Sync and refresh use the discoverable `diagnostic_scope: "errors" | "all"` and `diagnostics_in_result` controls, while save and close-save use `diagnostic_scope`; the obsolete @@ -93,10 +97,6 @@ This project keeps a lightweight, reverse-chronological changelog. Dates use `YY - Install and prune control-file reads now reject non-regular or symlinked paths, and a failed lock owner-PID write removes the lock directory acquired by that process. - `lean-beam ensure --hold` now exits cleanly and promptly after `SIGINT`. -- Wrapper-managed daemons now remain alive for overlapping requests without a fixed 30-second - cutoff, fence retired leases against resurrection, and recover safely from killed or replacement - owners in PID-isolated runners - ([#241](https://github.com/leanprover/lean-beam/pull/241), @ejgallego). - `lean-save` and `lean-close-save` now stage and commit complete artifact sets, preserving prior outputs on reported failure or cancellation and preventing same-worker saves from mixing files ([#217](https://github.com/leanprover/lean-beam/pull/217), @ejgallego). diff --git a/README.md b/README.md index d1354af2..61839176 100644 --- a/README.md +++ b/README.md @@ -104,7 +104,8 @@ including structured Lake options, dynamic libraries, and plugins already applie worker. Modules with batch-only `moreLeanArgs` fail with `saveUnsupportedSetup`: move shared `-D` settings to `leanOptions`, or use `lake build` when the arguments are intentionally batch-only. A running Lean server is not guaranteed to pick up Lake workspace configuration changes; after such a -change, run `lean-beam shutdown` before the next command that uses the Lean server. A successful +change, run `lean-beam shutdown`, then start a new foreground `lean-beam ensure --hold` wrapper +session before the next wrapper command that uses the Lean server. A successful checkpoint is normally sufficient while working; do not add an expensive clean build to every Beam loop. Final project validation should come from CI running `lake build` from clean Lake artifacts. If no successful clean CI result is available, or server-sensitive elaboration is suspected, use the diff --git a/docs/CUSTOM_TOOLCHAINS.md b/docs/CUSTOM_TOOLCHAINS.md index dc50ec6d..946d0187 100644 --- a/docs/CUSTOM_TOOLCHAINS.md +++ b/docs/CUSTOM_TOOLCHAINS.md @@ -93,4 +93,4 @@ BEAM_STAGE0_TOOLCHAIN=lean4-dev bash tests/test-stage0-toolchain.sh The smoke skips when the requested elan toolchain is unavailable. When it runs, it installs Beam with `--custom-toolchain`, checks that `doctor` resolves the installed bundle and reports a -fingerprint, then starts Beam with `ensure`. +fingerprint, then starts and closes an explicit `ensure --hold` wrapper session. diff --git a/docs/DEVELOPMENT.md b/docs/DEVELOPMENT.md index 17ad1780..c686f17f 100644 --- a/docs/DEVELOPMENT.md +++ b/docs/DEVELOPMENT.md @@ -148,13 +148,10 @@ carries a continuation handle that names one, and daemon startup receives its in explicitly through `--workspace-id`. The public CLI still manages one daemon per project, but that policy stays in `Beam.Cli`: its request adapter supplies a CLI-owned private identifier. That value is an implementation detail, not part of the broker protocol. Broker stats and open-document -requests without an id remain process-wide; stats also report `uptimeMs` and the internal -`activeRequestCount` used by daemon retirement, while both return the `workspaces` map. The count is -the number of other currently admitted requests: the stats request itself is excluded. Every +requests without an id remain process-wide; stats report `uptimeMs` and the `workspaces` map. Every broker request except `cancel` and `shutdown` is tracked even when it has no client request id; anonymous requests use an internal admission token so disconnect cancellation cannot affect a later -request. Wrapper requests carry both a generated request id and an internal daemon-lifetime lease -context, so every lease-fenced request participates in that count and can be cancelled by its exact +request. Wrapper requests carry a generated request id so each one can be cancelled by its exact admission handle. The CLI scopes those requests before sending them. `Beam.Broker.Op.workspaceScope` is the shared operation classification; CLI and test adapters should use it instead of maintaining their own operation lists. @@ -285,7 +282,7 @@ The broker is not a raw LSP proxy. Its narrow public job is still to expose smal internally it coordinates several responsibilities around the LSP process: - the CLI owns process identity: project-root detection, bundle selection, registry files, - endpoint/root validation, startup/shutdown, wrapper leases, and control-directory locks + endpoint/root validation, explicit session ownership, startup/shutdown, and control-directory locks - the broker owns request identity: daemon root validation, backend session lifetime, request dispatch, cancellation, active-request bookkeeping, transport errors, and the LSP document mirror - the LSP server and plugin own Lean/Rocq semantic facts: elaboration, diagnostics, progress, @@ -295,8 +292,8 @@ internally it coordinates several responsibilities around the LSP process: `ServerRuntime.dispatchRequestWithHandle` is the asynchronous admission boundary for in-process consumers such as MCP. It validates operation field ownership, registers the request's active -identity, validates any wrapper lease, exposes one opaque `RequestHandle`, and owns unregistering -that handle on success, rejection, or exception. A handle uses a per-admission token and must become +identity, exposes one opaque `RequestHandle`, and owns unregistering that handle on success, +rejection, or exception. A handle uses a per-admission token and must become inert after that lexical scope, including when a later request reuses the same client request ID. Keep ordinary daemon and CLI dispatch on `ServerRuntime.dispatchRequest`; transport layers must not mutate the active-request registry @@ -357,69 +354,53 @@ broker-derived decision. This wrapper path is easy to break accidentally, so keep the mental model simple. A daemon generation is one concrete daemon start identified by the `daemonId` in -`beam-daemon.json`. Every wrapper call holds a heartbeat lease while it may use that generation. -When the wrapper that started a generation sees all sibling leases and daemon-admitted requests -drain, it writes a retirement fence: a short-lived marker that closes wrapper admission until that -starter exits and its owner lease is provably stale. - -The retirement fence belongs to the wrapper-managed lifecycle. An unfenced `beam-client` request -that targets the same daemon is counted after broker admission, but the filesystem marker cannot -prevent a new raw request after the retiring owner samples that count. In a PID-reaping command -runner, keep `lean-beam ensure --hold` active while directing `beam-client` at a wrapper-managed -daemon. A separately launched standalone daemon instead has its own explicit process owner. - -What was broken: - -- Codex-style wrapper calls run in separate PID-isolated sandboxes. -- A later wrapper call could look at the daemon pid in the registry and think the daemon was dead, - even when the daemon was still alive and answering on its TCP endpoint. -- If one wrapper call started the daemon and then exited while sibling wrapper calls were still - using it, that exit could tear the daemon down mid-flight. - -What the fix does: - -- if the registry endpoint still answers, treat the daemon as live even if the recorded pid looks - wrong in the current sandbox -- acquire a wrapper lease under the daemon control lock before inspecting or starting a daemon, so - an owner cannot decide that the generation drained while a new wrapper is joining it -- if a wrapper call started the daemon, keep that wrapper call alive until overlapping sibling - wrapper calls for the same project root drain, without a fixed request-duration cutoff -- close wrapper admission to a drained generation before its owner exits; a later wrapper waits for - that retirement fence to clear before it observes the endpoint -- assign every daemon start a distinct generation id and let an obsolete starter release its lease - before waiting on wrappers admitted to the replacement generation -- bound the retiring owner's broker stats probe; if that exact registry generation has a dead PID - or zombie process in the observer's PID domain, release its starter without writing a - retirement fence, while timeout and transport failures remain fail-closed when PID identity - cannot prove disappearance -- wait for or kill a registry PID during shutdown only when its recorded PID domain matches the - current wrapper; cross-domain shutdown relies on the validated daemon endpoint and never treats - the same numeric PID as local process identity -- `lean-beam ensure --hold` gives agents an explicit foreground owner when they need daemon reuse - across separate PID-isolated shell invocations -- wrapper leases include a PID-domain identity and a monotonic heartbeat; Linux records the PID - namespace, while macOS records its host process domain. Known same-domain stale leases can be - pruned from PID liveness, while unknown or cross-namespace killed wrappers become stale only after - their heartbeat expires -- expiring a lease first writes a persistent `.revoked` tombstone and only then removes the heartbeat - file; a resumed wrapper cannot recreate or reuse that basename, and a killed wrapper leaves the - small tombstone in place as its fencing record -- each wrapper request carries an internal typed fence containing the daemon generation id and its - lease basename; the daemon registers the request, validates the fence before dispatch, and reports - the active-request count to the retiring owner -- a request admitted before lease revocation keeps the generation owner alive until the broker - finishes or cancels it; a request resumed after revocation is rejected before broker work begins -- daemon transport watches client disconnects and cancels the exact registered request, so killing a - cross-namespace wrapper drains both its heartbeat lease and its broker admission -- a wrapper reusing an existing daemon stops or cancels its request if it can no longer renew its - heartbeat; cancellation uses synthesized request ids for both progress and normally short calls, - and heartbeat cleanup does not wait without a bound for an uncooperative task +`beam-daemon.json`. Exactly one foreground `lean-beam ensure --hold` process owns that generation. +It starts the daemon with piped stdin and retains the pipe's write end. The daemon watches the read +end; EOF atomically closes broker admission, marks admitted requests for cancellation, shuts down +backend sessions, and stops the listener. There is no wrapper heartbeat, lease file, revocation +tombstone, or retirement fence. + +Ordinary wrapper commands never start a daemon. Under the per-project control lock they require a +registry whose root and effective configuration match, whose owner is not known dead in the current +PID domain, and whose endpoint answers for the CLI's private workspace and canonical project root. +Endpoint/root validation is authoritative across PID namespaces because numeric PID observations +from another domain are not safe process identity. A same-domain dead owner or a dead endpoint makes +the registry stale; cleanup remains generation-scoped and PID fallback is permitted only through the +typed PID-domain boundary. + +The owner also watches its exact registry generation and daemon child. `lean-beam shutdown` removes +that generation after the typed shutdown response, which makes the holder close its pipe and lets +the daemon's stdin watcher finish. An unexpected nonzero daemon exit is reported by the holder. +Interrupting or killing the holder closes the pipe by process lifetime. A paused holder keeps the +pipe open, so the session remains valid without time-based expiry. If the project root disappears, +the daemon's root watcher and the holder both converge on the same shutdown path. + +This model prevents PID-isolated commands from making contradictory ownership decisions: later +commands may attach to a validated endpoint, but none can silently become a replacement owner. +Starting a new session is always an explicit `lean-beam ensure --hold` action. Raw `beam-client` +requests may attach while that owner remains live; they participate in typed broker admission and +disconnect cancellation but do not own the process. A separately launched standalone daemon has +its own explicit process owner. + +Keep these invariants covered: + +- only `ensure --hold` may create and publish a wrapper daemon generation +- a second owner is rejected while the current endpoint/root generation is live +- ordinary wrapper commands preserve the owner's generation and fail with the exact recovery command + when no owner is live +- owner EOF, explicit shutdown, and project-root disappearance all close admission before backend + teardown and complete with bounded child cleanup +- PID-domain checks gate every PID probe or signal; cross-domain decisions use the validated endpoint +- request IDs and per-admission tokens retain exact disconnect and explicit cancellation semantics +- the regressions for this path are + [tests/test-beam-wrapper-daemon.sh](../tests/test-beam-wrapper-daemon.sh) and + [tests/test-beam-wrapper-sandbox.sh](../tests/test-beam-wrapper-sandbox.sh) - the regressions for this path are [tests/test-beam-wrapper-daemon.sh](../tests/test-beam-wrapper-daemon.sh) and [tests/test-beam-wrapper-sandbox.sh](../tests/test-beam-wrapper-sandbox.sh) Generic process helpers and the typed `RecordedPid.observe` boundary live in -[Beam/System.lean](../Beam/System.lean). Persisted registry, lease, and lock-owner PIDs must pass +[Beam/System.lean](../Beam/System.lean). Persisted registry and lock-owner PIDs must pass through that boundary; only a matching recorded/current PID-domain pair permits a local liveness, zombie, or termination operation. Generic directory locks live in [Beam/Cli/Lock.lean](../Beam/Cli/Lock.lean). Their owner metadata records both PID and PID domain; @@ -453,11 +434,10 @@ and shared with Lake/elaboration work, so a tiny task that blocks in an OS read normal-priority work on low-core runners. The cheap regression guard is [scripts/check-task-priority.sh](../scripts/check-task-priority.sh). -Pure wrapper lease metadata, staleness, filename, and retirement decisions live in -[Beam/Daemon/Ownership.lean](../Beam/Daemon/Ownership.lean). Shared registry, lease, retirement, -startup-log, and incident paths live in [Beam/Daemon/Paths.lean](../Beam/Daemon/Paths.lean). Daemon -registry management, daemon startup/reuse, endpoint selection, and effectful wrapper lease handling -live in [Beam/Cli/DaemonManager.lean](../Beam/Cli/DaemonManager.lean). Broker request plumbing, +Shared registry, startup-log, and incident paths live in +[Beam/Daemon/Paths.lean](../Beam/Daemon/Paths.lean). Daemon registry management, explicit owner +lifetime, endpoint selection, and typed PID-domain cleanup live in +[Beam/Cli/DaemonManager.lean](../Beam/Cli/DaemonManager.lean). Broker request plumbing, progress messages, cancellation-on-interrupt, and response failure notes live in [Beam/Cli/Broker.lean](../Beam/Cli/Broker.lean). User-facing stdout/stderr formatting helpers live in [Beam/Cli/Output.lean](../Beam/Cli/Output.lean). Doctor, validated/compatible toolchain registry, diff --git a/docs/ROCQ.md b/docs/ROCQ.md index c09bb7e8..eb7b18f6 100644 --- a/docs/ROCQ.md +++ b/docs/ROCQ.md @@ -55,13 +55,20 @@ bash tests/setup-rocq-opam.sh Rocq commands are available through the same installed `lean-beam` wrapper: ```bash -lean-beam ensure rocq +# keep this foreground owner running in one terminal/session +lean-beam ensure rocq --hold + +# issue probes from another terminal/session lean-beam doctor rocq lean-beam rocq-goals-after "Demo.v" 2 8 lean-beam rocq-goals-prev "Demo.v" 2 8 lean-beam rocq-goals-prev "Demo.v" 2 8 "intro x." ``` +The holder is the explicit owner of the wrapper daemon and its `coq-lsp` backend. Interrupt it, or +run `lean-beam shutdown`, when the session is finished. Ordinary wrapper commands attach to that +session and do not start a daemon implicitly. + Use `rocq-goals-after` to inspect goals after an existing sentence. Use `rocq-goals-prev` to inspect goals before a sentence, or with extra text to inspect an intermediate tactic prefix while porting a Rocq proof step to Lean. diff --git a/docs/SETUP.md b/docs/SETUP.md index 797b821b..9f76d331 100644 --- a/docs/SETUP.md +++ b/docs/SETUP.md @@ -201,10 +201,14 @@ lean-beam doctor Command positions use Lean/LSP coordinates: line and character are zero-based, and character counts UTF-16 code units. -Then start the per-project daemon and ask questions against a saved Lean file in that project: +Start one foreground owner for the wrapper session and keep it running. In another terminal or agent +process, ask questions against saved Lean files in that project: ```bash -lean-beam ensure +# terminal/session 1 +lean-beam ensure --hold + +# terminal/session 2 update_json="$(lean-beam update "Foo.lean")" printf '%s\n' "$update_json" version="$(printf '%s\n' "$update_json" | python3 -c 'import json,sys; print(json.load(sys.stdin)["result"]["version"])')" @@ -214,6 +218,12 @@ lean-beam goals before "Foo.lean" "$version" 10 2 lean-beam run-at "Foo.lean" "$version" 10 2 "exact trivial" ``` +`lean-beam ensure --hold` is the only wrapper command that starts the per-project daemon. Its +inherited ownership pipe defines the session lifetime: interrupt the holder, or run +`lean-beam shutdown`, to close the daemon and its backend processes. Plain `lean-beam ensure` and +all other wrapper commands attach to an existing owner and fail with a recovery command when none +is live. MCP clients do not need a separate holder; the stdio MCP process owns its runtime session. + The `python3` line extracts `result.version` for shell examples. You can also copy that version number from the printed `lean-beam update` JSON. @@ -265,9 +275,9 @@ checks. The running Lean server and existing file workers are not guaranteed to pick up Lake workspace configuration changes. After editing a lakefile, manifest, package override, `lean-toolchain`, Lean -options, plugins, or dynamic libraries, run `lean-beam shutdown` before the next command that uses -the Lean server. `lean-beam refresh` reopens a file within the current server and is not sufficient -for this case. +options, plugins, or dynamic libraries, run `lean-beam shutdown`, then start a new +`lean-beam ensure --hold` owner before the next wrapper command that uses the Lean server. +`lean-beam refresh` reopens a file within the current server and is not sufficient for this case. ### Final Batch Validation diff --git a/docs/STATUS.md b/docs/STATUS.md index 3633fafb..c783205c 100644 --- a/docs/STATUS.md +++ b/docs/STATUS.md @@ -132,10 +132,10 @@ reuse matching speculative execution rather than replaying it from scratch. Beam apply the source edit. For programmatic local consumers, the preferred machine-readable surface is the JSON stream exposed -by `beam-client request-stream`; wrapper stderr should be treated as human-facing. When that client -targets a wrapper-managed daemon in a PID-reaping command runner, keep `lean-beam ensure --hold` -active for the duration; the wrapper retirement fence does not close admission for unfenced raw -broker clients. A separately launched standalone daemon has its own explicit process owner. Broker +by `beam-client request-stream`; wrapper stderr should be treated as human-facing. A wrapper-managed +daemon exists only while its foreground `lean-beam ensure --hold` owner is alive. Keep that owner +active for wrapper and raw-client requests; those requests attach to the session but do not acquire +daemon ownership. A separately launched standalone daemon has its own explicit process owner. Broker responses require an explicit top-level `ok` boolean, giving projection layers an unambiguous success/error discriminator. A successful response always includes `result`; response and stream envelopes reject undeclared fields, and typed save/close-save results reject incomplete or extended @@ -181,16 +181,14 @@ Exact event ordering and examples live in - In sandboxed agent environments, Beam daemon startup itself may require elevated permissions even when the installed bundle and project-local `.beam` paths resolve correctly. -- PID-isolated wrapper calls acquire heartbeat leases before observing a project daemon. A wrapper - that starts a daemon remains alive until overlapping calls drain, while a killed cross-namespace - wrapper lease normally becomes recoverable after about five seconds. Use `lean-beam ensure --hold` - when separate sandbox commands need one explicit foreground daemon owner. A wrapper reusing an - existing daemon fails or cancels its request if it cannot continue renewing that lease. Lease - expiry writes a persistent revocation fence: already-admitted broker work keeps the owner alive - until it drains, while a suspended wrapper resumed after revocation cannot re-admit work through - the old lease. Retirement stats probes are bounded; an owner releases a dead current generation - only when same-PID-domain liveness or zombie state proves that the exact registered daemon process - is gone. +- Wrapper sessions use explicit ownership. `lean-beam ensure --hold` is the only wrapper command that + starts a project daemon; it passes the daemon an inherited pipe and remains alive as the owner. + Ordinary wrapper calls, including plain `lean-beam ensure`, only attach to that generation and + fail with the exact owner-start command when none is live. Owner EOF shuts down request admission, + backend sessions, and the daemon without heartbeat timeouts or filesystem leases. This works + across PID namespaces because endpoint/root validation is authoritative when PID identity is not + locally observable. A paused owner retains the session; a killed owner closes the pipe; explicit + `lean-beam shutdown` revokes the registry generation so the holder closes it cleanly. - A startup failure that reports `operation not permitted` through `.beam/beam-daemon-startup.log` is usually an environment restriction, not a bundle-resolution mismatch. - Beam daemon disappearance errors include registry/log context and write a JSON incident record under @@ -252,8 +250,8 @@ Exact event ordering and examples live in worker has already applied them; batch-only `moreLeanArgs` fail with `saveUnsupportedSetup`. - Beam does not detect Lake workspace configuration changes during a running Lean session. After editing a lakefile, manifest, package override, `lean-toolchain`, Lean options, plugins, or dynamic - libraries, run `lean-beam shutdown` before the next command that uses the Lean server; - `lean-beam refresh` does not restart the server. + libraries, run `lean-beam shutdown`, then start a new `lean-beam ensure --hold` owner before the + next wrapper command that uses the Lean server; `lean-beam refresh` does not restart the server. - A Beam checkpoint contains the Lean server's accepted environment. Elaborators can distinguish server execution from batch execution, so exceptional custom elaboration can produce an artifact that differs from a fresh `lake build` artifact. Successful checkpoints are normally diff --git a/docs/SYNC_AND_DIAGNOSTICS.md b/docs/SYNC_AND_DIAGNOSTICS.md index 03717576..89e204a5 100644 --- a/docs/SYNC_AND_DIAGNOSTICS.md +++ b/docs/SYNC_AND_DIAGNOSTICS.md @@ -128,10 +128,11 @@ by exactly one terminal `response`; the response is last and no later message be request. When `beam-client` targets the per-project daemon managed by `lean-beam` in a PID-reaping command -runner, keep one `lean-beam ensure --hold` process active for the request lifetime. Raw broker -requests do not carry wrapper leases, so the wrapper retirement fence does not own their admission. -A separately launched standalone daemon has its own explicit process owner and does not need the -wrapper hold. +runner, keep the session's `lean-beam ensure --hold` owner active for the request lifetime. The same +rule applies to wrapper commands: only the holder starts the daemon, while every other command +attaches to its endpoint. Raw broker requests participate in the daemon's typed request admission +and cancellation, but do not own the daemon process. A separately launched standalone daemon has +its own explicit process owner and does not use the wrapper holder. Every stream variant uses the same `kind`, `payload`, and optional correlation envelope. When the request supplies `clientRequestId`, each message repeats it on that outer stream envelope: diff --git a/docs/TESTING.md b/docs/TESTING.md index a7db9e97..638ca488 100644 --- a/docs/TESTING.md +++ b/docs/TESTING.md @@ -110,16 +110,14 @@ Current Beam coverage includes: [tests/test-beam-fast.sh](../tests/test-beam-fast.sh) - wrapper coverage through [tests/test-beam-wrapper.sh](../tests/test-beam-wrapper.sh), which aggregates focused probe, runtime, sync/save, handle, and diagnostic slices - focused daemon lifecycle coverage in [tests/test-beam-wrapper-daemon.sh](../tests/test-beam-wrapper-daemon.sh), - including valid same-domain and malformed stale-lease cleanup, heartbeat-writer failure, - retirement-fence recovery, bounded recovery when the started daemon dies before retirement, - cross-domain shutdown PID non-interference, fail-closed unreadable registry, lease, and - retirement-fence observations, and - self-termination after the project worktree disappears + including the no-implicit-start contract, duplicate-owner rejection, endpoint collision safety, + explicit shutdown, generation replacement, holder reporting after an unexpected daemon crash, + abrupt owner death through inherited-pipe EOF, stale registry cleanup, and self-termination after + the project worktree disappears - Linux-only PID-isolated sandbox wrapper coverage in [tests/test-beam-wrapper-sandbox.sh](../tests/test-beam-wrapper-sandbox.sh), - including followers active beyond 30 seconds, stop/resume across heartbeat revocation, - daemon-admitted request draining, killed-client disconnect cancellation, killed-follower heartbeat - expiry, retirement recovery, obsolete-owner release across daemon replacement, distinct - generation identity, and concurrent cold-start wrapper admission + including cross-namespace endpoint attachment, duplicate-owner rejection, a paused owner without + time-based expiry, explicit shutdown, killed-owner EOF cleanup, stale-registry recovery, distinct + generation identity, and the absence of legacy lease/retirement artifacts - zero-build save replay, structured-setup support, batch-only-argument rejection, and stale-save race coverage in [tests/test-beam-save-olean.sh](../tests/test-beam-save-olean.sh) diff --git a/scripts/lean-beam b/scripts/lean-beam index d9b04562..34690a4b 100755 --- a/scripts/lean-beam +++ b/scripts/lean-beam @@ -51,7 +51,8 @@ notes: - run `lean-beam sync ` when you also need the diagnostics/readiness barrier - for multiline text-carrying Lean probes, prefer `--stdin` or `--text-file `; use `--` before text that starts with `--` - for handle-based commands, use `--handle-file ` when you do not want to inline handle json - - use `ensure --hold` when a PID-isolated command runner needs one foreground process to keep a newly-started daemon alive + - wrapper commands require one live session owner; start `ensure --hold` in a foreground process and keep it running + - plain `ensure` checks and warms that owned session but never starts one implicitly - set `BEAM_DEBUG_TEXT=1` to print the exact escaped text and UTF-8 bytes sent for text-carrying Lean probes - use `lean-beam --version` for bug reports and installed runtime identity checks - validated-toolchains lists exact CI-validated versions; compatible-release-lines lists canonical RC/patch families qualified locally diff --git a/skills/lean-beam/SKILL.md b/skills/lean-beam/SKILL.md index c1fe6758..c30969b3 100644 --- a/skills/lean-beam/SKILL.md +++ b/skills/lean-beam/SKILL.md @@ -73,8 +73,8 @@ and isolation rules; do not treat MCP as a raw Lean LSP proxy. Supported command families: -- bootstrap the Lean backend: `lean-beam ensure` -- keep a sandboxed daemon owner alive across separate commands: `lean-beam ensure --hold` +- start and own a wrapper session: `lean-beam ensure --hold` +- check and warm an already-owned Lean session: `lean-beam ensure` - inspect existing code, navigation data, or proof state: `lean-beam hover`, `lean-beam signature-help`, `lean-beam definition`, `lean-beam references`, `lean-beam document-symbols`, `lean-beam workspace-symbols`, `lean-beam goals before`, @@ -108,8 +108,10 @@ Core workflow contract: - use `lean-beam`, not raw JSON and not raw LSP - Beam never applies source edits to `.lean` files on disk; the client applies source edits - `lean-beam` only sees the on-disk file, not unsaved editor buffers -- in transient PID-sandboxed command runners, start one foreground `lean-beam ensure --hold` - process when you need daemon reuse across separate shell invocations; interrupt it when finished +- before using wrapper workflow commands, start one foreground `lean-beam ensure --hold` process + and keep it running across shell invocations; interrupt it or run `lean-beam shutdown` when finished +- MCP owns its stdio runtime session automatically; do not start a separate wrapper holder solely + for MCP tool calls - after every real Lean source edit: save the file normally, then run `lean-beam update` before the next version-bound probe; run `lean-beam sync` when you need diagnostics/readiness - use `lean-beam save` only for a synced workspace module path in the current Lake workspace package @@ -126,9 +128,9 @@ Core workflow contract: before the next command that uses the Lean server; `lean-beam refresh` does not restart it - treat wrapper `stderr` as human-facing only; use stdout JSON or `beam-client request-stream` for machine-readable automation -- when `beam-client` targets a wrapper-managed daemon in a PID-reaping command runner, keep one - `lean-beam ensure --hold` process active for the raw request lifetime; raw broker requests do not - carry wrapper leases, while a separately launched standalone daemon has its own process owner +- when `beam-client` targets a wrapper-managed daemon, keep that session's + `lean-beam ensure --hold` owner active for the raw request lifetime; raw broker requests attach to + the session but do not own it, while a separately launched standalone daemon has its own owner - `lean-beam feedback-report` and `beam_feedback_report` return a report to the caller; Beam does not upload or submit it; before posting non-confidential output, review caller-authored narrative, request/response payloads, local paths, Beam stats, open-file data, daemon logs/incidents, and @@ -297,28 +299,28 @@ Use `lean-beam`, not raw JSON and not raw LSP. - fully validates exact Lean toolchains listed in `validated-lean-toolchains`, locally qualifies canonical RC/patch variants from `compatible-lean-release-lines`, and accepts exact custom names recorded by the installer in `custom-lean-toolchains` -- owns Beam daemon startup, shutdown, and registry handling +- gives daemon startup authority only to `lean-beam ensure --hold`; ordinary wrapper commands attach + to its registry generation and never start a daemon implicitly +- owns shutdown and registry handling - resolves Lean with `elan which lean` - builds and plugin-qualifies a local fallback bundle only when no matching installed bundle exists for the exact accepted toolchain fingerprint - fails early on toolchains that are neither validated, canonical members of a compatible release line, nor explicitly custom; use `lean-beam validated-toolchains`, `lean-beam compatible-release-lines`, and `lean-beam doctor` to inspect the decision -- restarts the Beam daemon if the effective Lean startup configuration for that root changes +- after effective Lean startup configuration changes, requires shutting down the old session and + starting a new `lean-beam ensure --hold` owner - `lean-beam shutdown`, `lean-beam stats`, and `lean-beam reset-stats` apply to the current project only - `lean-beam prune` previews old installed runtimes; restart active agents and MCP clients before any `--apply`, and add `--bundles` when stale installed bundle caches should also be removed - wrapper commands talk to the per-project Beam daemon over localhost TCP; they are not direct in-process Lean calls -- `lean-beam ensure --hold` prints the usual JSON ensure response on stdout, keeps the wrapper - process alive until interrupted, and is only for environments that reap background daemons when - each command exits; later wrappers with matching PID-domain identity recover killed leases from - PID liveness, while unknown or cross-namespace killed leases recover after their heartbeat - expires, normally after about five seconds; expiry revokes that lease, already-admitted broker work - keeps the daemon owner alive until it drains, and a resumed or heartbeat-failed wrapper fails or - cancels instead of reusing the revoked lease; if the exact daemon started by the foreground owner - dies, owner retirement releases after same-domain PID liveness or zombie state proves it is gone -- the wrapper retirement fence closes wrapper admission; keep `lean-beam ensure --hold` active when - an unfenced `beam-client` targets that managed daemon in a PID-reaping command runner +- `lean-beam ensure --hold` prints the usual JSON ensure response on stdout and keeps the wrapper + process alive as the session owner; an inherited pipe ties daemon lifetime to that process without + heartbeat files or lease expiry +- interrupting or killing the holder closes the owner pipe and shuts down the daemon; explicit + `lean-beam shutdown` releases the holder cleanly +- plain `lean-beam ensure` checks and warms a live owned session; if there is no owner, follow its + error and start `lean-beam ensure --hold` `lean-beam` is more than a one-shot probe: @@ -337,8 +339,8 @@ Use `lean-beam`, not raw JSON and not raw LSP. Default rules: - use `lean-beam`, not raw JSON and not raw LSP +- start and keep one `lean-beam ensure --hold` owner before issuing wrapper workflow commands - start with `lean-beam run-at` -- use `lean-beam ensure --hold` only when your command runner needs a foreground owner for daemon reuse - after every real source edit: save the file to disk normally, then `lean-beam update` before the next version-bound probe; use `lean-beam sync` for diagnostics/readiness - if exact continuation matters: mint a handle @@ -351,11 +353,10 @@ Default rules: If you only remember one workflow, use this one: ```bash -lean-beam ensure -# in PID-isolated command runners, keep this in one foreground session instead +# terminal or long-lived agent process 1: keep this running lean-beam ensure --hold -# inspect existing code or proof state +# terminal or agent process 2: inspect existing code or proof state update_out="$(lean-beam update "Foo.lean")" printf '%s\n' "$update_out" version="$(printf '%s\n' "$update_out" | python3 -c 'import json,sys; print(json.load(sys.stdin)["result"]["version"])')" diff --git a/skills/lean-beam/references/mcts-search.md b/skills/lean-beam/references/mcts-search.md index 4c83b171..eded285f 100644 --- a/skills/lean-beam/references/mcts-search.md +++ b/skills/lean-beam/references/mcts-search.md @@ -36,7 +36,7 @@ Rules: ## Minimal branching example ```bash -lean-beam ensure +# with `lean-beam ensure --hold` running in another process root="$(lean-beam run-at-handle "Proofs.lean" 42 6 "constructor")" # writing handles to files avoids stdin conflicts in larger shell scripts @@ -57,7 +57,7 @@ Use this when you want to explore multiple children from the same preserved basi ## Minimal linear playout example ```bash -lean-beam ensure +# with `lean-beam ensure --hold` running in another process root="$(lean-beam run-at-handle "Proofs.lean" 42 6 "constructor")" # file-backed handles are often easier in longer shell loops printf '%s\n' "$root" > root.handle.json @@ -87,7 +87,7 @@ Use this when you want one evolving playout path instead of a preserved branch p Concrete shell sketch: ```bash -lean-beam ensure +# with `lean-beam ensure --hold` running in another process root="$(lean-beam run-at-handle "Proofs.lean" 42 6 "constructor")" child_a="$(printf '%s\n' "$root" | lean-beam run-with "Proofs.lean" - "constructor")" @@ -102,7 +102,7 @@ printf '%s\n' "$child_b" | lean-beam release "Proofs.lean" - The same sketch with the helper: ```bash -lean-beam ensure +# with `lean-beam ensure --hold` running in another process root="$(lean-beam-search mint "Proofs.lean" 42 6 "constructor")" child_a="$(printf '%s\n' "$root" | lean-beam-search branch "Proofs.lean" "constructor")" child_b="$(printf '%s\n' "$root" | lean-beam-search branch "Proofs.lean" "aesop")" diff --git a/skills/lean-beam/references/workflow-details.md b/skills/lean-beam/references/workflow-details.md index 7f1fc5bf..17c208f7 100644 --- a/skills/lean-beam/references/workflow-details.md +++ b/skills/lean-beam/references/workflow-details.md @@ -210,7 +210,7 @@ What is not a valid checkpoint target: Treat `fileProgress` as observability, not as proof that every call is a full barrier. ```bash -lean-beam ensure +# with `lean-beam ensure --hold` running in another process sync_out="$(lean-beam sync "Foo.lean")" printf '%s\n' "$sync_out" version="$(printf '%s\n' "$sync_out" | python3 -c 'import json,sys; print(json.load(sys.stdin)["result"]["version"])')" @@ -308,7 +308,7 @@ If you edit `A.lean` and `B.lean` imports `A.lean`, a successful probe in `B.lea itself to prove the dependency cone is fresh. ```bash -lean-beam ensure +# with `lean-beam ensure --hold` running in another process # make a real edit in A.lean and save the source file to disk lean-beam sync "A.lean" b_update="$(lean-beam update "B.lean")" diff --git a/skills/rocq-beam/SKILL.md b/skills/rocq-beam/SKILL.md index bace5acc..22bab765 100644 --- a/skills/rocq-beam/SKILL.md +++ b/skills/rocq-beam/SKILL.md @@ -52,7 +52,8 @@ mutation. Supported command families: -- bootstrap the Rocq backend: `lean-beam ensure rocq` +- start and own a Rocq wrapper session: `lean-beam ensure rocq --hold` +- check and warm an already-owned Rocq session: `lean-beam ensure rocq` - inspect goals after an existing sentence: `lean-beam rocq-goals-after` - inspect goals before a sentence or after speculative sentence text within that basis: `lean-beam rocq-goals-prev` @@ -67,6 +68,8 @@ What to treat as the current agent workflow surface: Core workflow contract: - use `lean-beam`, not raw JSON and not raw LSP +- before issuing wrapper probes, start one foreground `lean-beam ensure rocq --hold` process and + keep it running; interrupt it or run `lean-beam shutdown` when finished - save the `.v` file before every new probe after a real edit - `lean-beam` only sees the on-disk file, not unsaved editor buffers - treat ` ` as LSP-style coordinates for the saved file: line `0` is the first @@ -85,9 +88,12 @@ Use `lean-beam`, not raw JSON and not raw LSP. - infers the target project root from the current directory or `--root` - keeps one Beam daemon per project root and records it in `/.beam/beam-daemon.json` - in sandboxed or read-only project trees, set `BEAM_CONTROL_DIR` to a writable directory -- owns Beam daemon startup, shutdown, and registry handling +- gives daemon startup authority only to `lean-beam ensure rocq --hold`; ordinary commands attach + to its registry generation and never start a daemon implicitly +- owns shutdown and registry handling - resolves `coq-lsp` from the target project's local `_opam` when available -- starts a Rocq-capable Beam daemon with explicit startup args instead of relying on inherited editor state +- the explicit owner starts a Rocq-capable Beam daemon with startup args instead of relying on + inherited editor state - wrapper commands talk to the per-project Beam daemon over localhost TCP; they are not direct in-process Rocq calls - in Codex-style sandboxes, Beam daemon startup may still require elevated permissions even when all paths resolve correctly - in the same environments, localhost TCP bind/connect for the Beam daemon and client may also require elevated permissions @@ -106,10 +112,14 @@ Default rules: ## Workflow -Ensure the Rocq backend: +Start the Rocq wrapper-session owner in one terminal or long-lived agent process, then inspect it +from another: ```bash -lean-beam ensure rocq +# terminal/session 1: keep running +lean-beam ensure rocq --hold + +# terminal/session 2 lean-beam stats ``` @@ -150,7 +160,7 @@ Execution model: Default loop: ```bash -lean-beam ensure rocq +# with `lean-beam ensure rocq --hold` running in another process lean-beam rocq-goals-after "Demo.v" 12 4 # make a real edit, save the file @@ -162,14 +172,14 @@ Use cases: 1. Inspect the current proof state after a sentence ```bash -lean-beam ensure rocq +# with `lean-beam ensure rocq --hold` running in another process lean-beam rocq-goals-after "Demo.v" 12 4 ``` 2. Inspect an intermediate tactic state inside one sentence ```bash -lean-beam ensure rocq +# with `lean-beam ensure rocq --hold` running in another process lean-beam rocq-goals-prev "Demo.v" 12 4 "intro x." lean-beam rocq-goals-prev "Demo.v" 12 4 "split." ``` @@ -179,7 +189,7 @@ lean-beam rocq-goals-prev "Demo.v" 12 4 "split." Save the file first, then probe again from the saved document. ```bash -lean-beam ensure rocq +# with `lean-beam ensure rocq --hold` running in another process lean-beam rocq-goals-after "Demo.v" 12 4 # make a real edit in Demo.v and save it diff --git a/tests/lean/BeamTest/Broker/CliDaemonTest.lean b/tests/lean/BeamTest/Broker/CliDaemonTest.lean index c75ab8f2..36995da6 100644 --- a/tests/lean/BeamTest/Broker/CliDaemonTest.lean +++ b/tests/lean/BeamTest/Broker/CliDaemonTest.lean @@ -7,7 +7,6 @@ Author: Emilio J. Gallego Arias import Beam.Broker.Errors import Beam.Cli.Args import Beam.Cli.Broker -import Beam.Daemon.Ownership import Beam.Cli.Info import Beam.Cli.LeanOperation import Beam.Cli.Lock @@ -18,7 +17,6 @@ import Beam.Path import BeamTest.Broker.JsonAssert open Lean -open Beam.Daemon.Ownership open BeamTest.Broker.JsonAssert (requireJsonNull requireJsonString) namespace BeamTest.Broker.CliDaemonTest @@ -30,10 +28,6 @@ private def require (label : String) (cond : Bool) : IO Unit := do private def projectDaemonClientForTest (endpoint : Beam.Broker.Transport.Endpoint) : Beam.Cli.ProjectDaemonClient := { endpoint - wrapperLease := { - daemonId := "test-daemon" - leaseFile := "test-wrapper.lease" - } } private def checkDaemonDebugWarnings : IO Unit := do @@ -50,7 +44,7 @@ private def checkDaemonDebugWarnings : IO Unit := do (warnings.any (fun warning => warning.contains "registry pid is not alive")) require "dead registry pid warning should include recovery hint" (warnings.any (fun warning => - warning.contains "lean-beam shutdown" && warning.contains "lean-beam ensure")) + warning.contains "lean-beam shutdown" && warning.contains "lean-beam ensure --hold")) private def expectIoErrorMessage (label : String) (act : IO α) : IO String := do let result ← @@ -172,7 +166,7 @@ private def serveCancelablePlainRequest finally Beam.Broker.Transport.closeConnection cancelConn let response := Beam.Broker.errorResponseFor - .requestCancelled "cancelled by wrapper lease loss" + .requestCancelled "cancelled by session close" Beam.Broker.Transport.sendMsg requestConn (toJson (Beam.Broker.StreamMessage.response (some requestId) response)).compress finally @@ -527,6 +521,8 @@ private def checkDaemonFailureContext : IO Unit := do daemonId := "daemon-test" pid := 999999999 pidDomain? + ownerPid := 999999999 + ownerPidDomain? := pidDomain? port? := some 42424 root := root.toString configHash := "config-test" @@ -655,6 +651,7 @@ private def writeTestRegistryEntry let entry : Beam.Daemon.RegistryEntry := { daemonId := "daemon-test" pid := 999999999 + ownerPid := 999999999 port? root := root.toString configHash := "config-test" @@ -835,30 +832,6 @@ private def createSymlink if out.exitCode != 0 then throw <| IO.userError s!"failed to create {label} symlink\n{out.stderr}" -private def checkWrapperLeaseStaleness : IO Unit := do - let fresh : WrapperLeaseMetadata := { - pid := 42 - pidDomain? := some "pid:[owner]" - heartbeatMonoNanos := 1000 - } - let timeout := 500 - require "a fresh cross-domain lease should not depend on an unrelated PID observation" - (!(wrapperLeaseStaleFromObservation .differentDomain 1200 timeout fresh)) - require "a dead same-domain PID should make a fresh lease stale" - (wrapperLeaseStaleFromObservation (.local false) 1200 timeout fresh) - require "a live same-domain PID should preserve a fresh lease" - (!(wrapperLeaseStaleFromObservation (.local true) 1200 timeout fresh)) - require "unknown domain identity should fall back to a fresh heartbeat" - (!(wrapperLeaseStaleFromObservation .unknownDomain 1200 timeout fresh)) - require "heartbeat expiry should revoke even a same-domain live process lease" - (wrapperLeaseStaleFromObservation (.local true) 1501 timeout fresh) - require "heartbeat expiry should make a cross-domain lease stale" - (wrapperLeaseStaleFromObservation .differentDomain 1501 timeout fresh) - require "a heartbeat from a previous monotonic clock epoch should be stale" - (wrapperLeaseStaleFromObservation .differentDomain 999 timeout fresh) - require "pid zero should never hold daemon ownership" - (wrapperLeaseStaleFromObservation .differentDomain 1000 timeout { fresh with pid := 0 }) - private def checkCurrentPidDomain : IO Unit := do let selfPid := (← IO.Process.getPID).toNat let domain? ← Beam.currentPidDomain? @@ -900,6 +873,8 @@ private def checkCrossDomainRegistryPidGuard : IO Unit := do daemonId := "cross-domain-pid-guard" pid := child.pid.toNat pidDomain? := some "beam-test-other-pid-domain" + ownerPid := child.pid.toNat + ownerPidDomain? := some "beam-test-other-pid-domain" root := "/tmp/beam-cross-domain-pid-guard" configHash := "cross-domain-pid-guard" startedAt := "2026-08-25T00:00:00Z" @@ -917,34 +892,6 @@ private def checkCrossDomainRegistryPidGuard : IO Unit := do catch _ => pure () -private def checkDaemonRetirementPolicy : IO Unit := do - require "a drained current generation should commit retirement" - (retirementDecision (.current .drained .drained) == .commit) - require "an active current generation sibling should keep the owner alive" - (retirementDecision (.current .activeOrUnreadable .drained) == .wait) - require "an admitted broker request should keep the current generation owner alive" - (retirementDecision (.current .drained .activeOrUnreadable) == .wait) - require "a provably dead current daemon should release its starter without fencing" - (retirementDecision (.current .drained .provenGone) == .obsolete) - require "a proven replacement generation should make the old owner obsolete immediately" - (retirementDecision .replacement == .obsolete) - require "unavailable registry state with active or unreadable siblings should fail closed" - (retirementDecision (.unavailable .activeOrUnreadable) == .wait) - require "unavailable registry state may release an owner only after siblings drain" - (retirementDecision (.unavailable .drained) == .obsolete) - -private def checkWrapperLeaseFileNames : IO Unit := do - require "a generated-style lease basename should be accepted" - (validWrapperLeaseFileName "123-42-99.lease") - require "a parent-relative retirement lease path should be rejected" - (!(validWrapperLeaseFileName "../outside.lease")) - require "an absolute retirement lease path should be rejected" - (!(validWrapperLeaseFileName "/tmp/outside.lease")) - require "a nested retirement lease path should be rejected" - (!(validWrapperLeaseFileName "nested/outside.lease")) - require "a non-lease retirement basename should be rejected" - (!(validWrapperLeaseFileName "owner.json")) - private def checkPathCanonicalization : IO Unit := do let stamp ← IO.monoNanosNow let root := System.FilePath.mk s!"/tmp/beam-path-canonical-root-{stamp}" @@ -1316,11 +1263,8 @@ def main : IO Unit := do checkDoctorDaemonFailureIncidentLines checkPathRelativeToRoot checkLeanModuleNamePathHelpers - checkWrapperLeaseStaleness checkCurrentPidDomain checkCrossDomainRegistryPidGuard - checkDaemonRetirementPolicy - checkWrapperLeaseFileNames checkPathCanonicalization checkLockLifecycle checkLeanToolchainPolicyParsing diff --git a/tests/lean/BeamTest/Broker/PendingTest.lean b/tests/lean/BeamTest/Broker/PendingTest.lean index dec60d8a..943b4e7b 100644 --- a/tests/lean/BeamTest/Broker/PendingTest.lean +++ b/tests/lean/BeamTest/Broker/PendingTest.lean @@ -78,8 +78,6 @@ private def checkActiveRegistry : IO Unit := do let firstResult ← ActiveRequestRegistry.register registry (some "req-1") let first ← expectRegistered "register active request" firstResult - require "count excluding the current admission reports only other requests" - ((← ActiveRequestRegistry.countExcluding registry first) == 0) match ← ActiveRequestRegistry.register registry (some "req-1") with | .ok _ => throw <| IO.userError "duplicate clientRequestId registered successfully" diff --git a/tests/lean/BeamTest/Broker/ProtocolTest.lean b/tests/lean/BeamTest/Broker/ProtocolTest.lean index 839aec01..f25d2aef 100644 --- a/tests/lean/BeamTest/Broker/ProtocolTest.lean +++ b/tests/lean/BeamTest/Broker/ProtocolTest.lean @@ -661,9 +661,9 @@ private def checkWorkspaceRoutingFields : IO Unit := do .required require s!"{op.key} has the wrong workspace scope" (op.workspaceScope == expectedScope) - let expectedWrapperLease := op != .cancel && op != .shutdown - require s!"{op.key} has the wrong wrapper-lease admission policy" - (op.acceptsWrapperLease == expectedWrapperLease) + let expectedTracking := op != .cancel && op != .shutdown + require s!"{op.key} has the wrong active-request tracking policy" + (op.tracksActiveRequest == expectedTracking) let request : Request := { op } let decoded ← expectOk s!"minimal {op.key} request round trip" <| fromJson? (α := Request) (toJson request) @@ -676,43 +676,6 @@ private def checkWorkspaceRoutingFields : IO Unit := do let leanReq : Request := { op := .ensure } requireJsonString "backend-scoped request serialization" "backend" "lean" (toJson leanReq) - let wrapperLease : WrapperLeaseContext := { - daemonId := "daemon-generation" - leaseFile := "123-42-99.lease" - } - let fencedReq : Request := { - op := .stats - clientRequestId? := some "wrapper-lease-round-trip" - wrapperLease? := some wrapperLease - } - let decodedFenced ← expectOk "wrapper lease request round trip" <| - fromJson? (α := Request) (toJson fencedReq) - require "wrapper lease request preserves its typed fence" - (decodedFenced.wrapperLease? == some wrapperLease) - for op in #[Op.cancel, .shutdown] do - let fencedControl : Request := { - op - clientRequestId? := some s!"wrapper-lease-{op.key}" - wrapperLease? := some wrapperLease - } - match fromJson? (α := Request) (toJson fencedControl) with - | .ok _ => throw <| IO.userError s!"broker accepted a wrapper lease for {op.key}" - | .error err => - require s!"{op.key} should reject its wrapper lease explicitly" - (err.contains "does not accept 'wrapperLease'") - match fromJson? (α := Request) <| Json.mkObj [ - ("op", toJson "stats"), - ("clientRequestId", toJson "wrapper-lease-undeclared-field"), - ("wrapperLease", Json.mkObj [ - ("daemonId", toJson "daemon-generation"), - ("leaseFile", toJson "123-42-99.lease"), - ("obsolete", toJson true) - ]) - ] with - | .ok _ => throw <| IO.userError "broker accepted an undeclared wrapper lease field" - | .error err => - require "wrapper lease decoder should identify its undeclared field" (err.contains "obsolete") - let explicitReq : Request := { op := .stats workspaceId? := some "fixture" @@ -840,7 +803,6 @@ private def checkWorkspaceLifecycleProtocol : IO Unit := do requireFieldAbsent "process-wide stats" "root" processStats requireFieldAbsent "process-wide stats" "sessions" processStats requireFieldAbsent "process-wide stats" "byBackend" processStats - requireJsonInt "process-wide stats" "activeRequestCount" 0 processStats discard <| IO.ofExcept <| processStats.getObjVal? "workspaces" let resetResult : Beam.Workspace.InitResult := { @@ -900,110 +862,24 @@ private def checkWorkspaceLifecycleProtocol : IO Unit := do require "typed workspace drop preserves lifecycle state" (decodedDrop.workspaceId == "fixture" && decodedDrop.dropped && decodedDrop.invalidatedHandles) -private def wrapperLeaseInactiveResponse (leaseState : String) (resp : Response) : Bool := - resp.error?.any fun err => - err.code == "requestCancelled" && - err.data?.any fun data => - (data.getObjValAs? String "reason").toOption == some "wrapperLeaseInactive" && - (data.getObjValAs? String "leaseState").toOption == some leaseState - -private def checkWrapperLeaseFence : IO Unit := do - let root := System.FilePath.mk s!"/tmp/beam-wrapper-lease-fence-{← IO.monoNanosNow}" - let daemonId := "daemon-generation" - let leaseFile := "123-42-99.lease" - try - IO.FS.createDirAll root - let leaseDir ← Beam.Daemon.wrapperLeaseDir root - IO.FS.createDirAll leaseDir - let leasePath := leaseDir / leaseFile - let metadata : Beam.Daemon.Ownership.WrapperLeaseMetadata := { - pid := 42 - heartbeatMonoNanos := ← IO.monoNanosNow - } - IO.FS.writeFile leasePath ((toJson metadata).pretty ++ "\n") - let runtime ← Beam.Broker.ServerRuntime.create - ({ root } : Beam.Broker.BrokerConfig) "fixture" (.tcp 0) (some daemonId) - let request : Request := { - op := .stats - clientRequestId? := some "wrapper-lease-valid" - wrapperLease? := some { daemonId, leaseFile } - } - let anonymousLeaseJson := Json.mkObj [ - ("op", toJson "stats"), - ("wrapperLease", toJson ({ daemonId, leaseFile } : WrapperLeaseContext)) - ] - match fromJson? (α := Request) anonymousLeaseJson with - | .ok _ => - throw <| IO.userError "broker accepted an untracked wrapper lease request" - | .error err => - require "wrapper lease validation should require request correlation" - (err.contains "clientRequestId") - match request.validateFields with - | .error err => - throw <| IO.userError s!"valid wrapper lease request failed validation: {err}" - | .ok () => pure () - let (accepted, _) ← runtime.dispatchRequest request - require "a matching live wrapper lease should pass daemon dispatch fencing" accepted.ok - let some acceptedResult := accepted.result? - | throw <| IO.userError "accepted wrapper lease stats response omitted its result" - requireJsonInt "lease-fenced stats" "activeRequestCount" 0 acceptedResult - - let revocationPath := Beam.Daemon.Ownership.wrapperLeaseRevocationPath leasePath - IO.FS.writeFile revocationPath "{}\n" - let (revoked, _) ← runtime.dispatchRequest { - request with clientRequestId? := some "wrapper-lease-revoked" - } - require "a revoked wrapper lease should fail before broker dispatch" - (wrapperLeaseInactiveResponse "revoked" revoked) - - let (wrongGeneration, _) ← runtime.dispatchRequest { - request with - clientRequestId? := some "wrapper-lease-wrong-generation" - wrapperLease? := some { daemonId := "replacement", leaseFile } - } - require "a wrapper lease from another daemon generation should fail before dispatch" - (wrapperLeaseInactiveResponse "generationMismatch" wrongGeneration) - - IO.FS.removeFile revocationPath - IO.FS.removeFile leasePath - let (missing, _) ← runtime.dispatchRequest { - request with clientRequestId? := some "wrapper-lease-missing" - } - require "a missing wrapper lease should fail before broker dispatch" - (wrapperLeaseInactiveResponse "missing" missing) - - IO.FS.writeFile leasePath "{\n" - let (malformed, _) ← runtime.dispatchRequest { - request with clientRequestId? := some "wrapper-lease-malformed" - } - require "a malformed wrapper lease should fail before broker dispatch" - (wrapperLeaseInactiveResponse "malformed" malformed) - IO.FS.removeFile leasePath - IO.FS.createDir leasePath - let (unreadable, _) ← runtime.dispatchRequest { - request with clientRequestId? := some "wrapper-lease-unreadable" - } - require "an unreadable wrapper lease should fail before broker dispatch" - (wrapperLeaseInactiveResponse "unreadable" unreadable) - IO.FS.removeDir leasePath - IO.FS.writeFile leasePath ((toJson { metadata with pid := 0 }).pretty ++ "\n") - let (invalidPid, _) ← runtime.dispatchRequest { - request with clientRequestId? := some "wrapper-lease-invalid-pid" - } - require "a zero-pid wrapper lease should fail before broker dispatch" - (wrapperLeaseInactiveResponse "invalidPid" invalidPid) - let (invalidFileName, _) ← runtime.dispatchRequest { - request with - clientRequestId? := some "wrapper-lease-invalid-file-name" - wrapperLease? := some { daemonId, leaseFile := "../outside.lease" } - } - require "an invalid wrapper lease basename should fail before broker dispatch" - (wrapperLeaseInactiveResponse "invalidFileName" invalidFileName) - require "rejected wrapper leases should leave no active admission" - ((← ActiveRequestRegistry.count runtime.activeRequests) == 0) - finally - if ← root.pathExists then - IO.FS.removeDirAll root +private def checkSessionCloseAdmission : IO Unit := do + let root := System.FilePath.mk "/tmp/beam-session-close-admission" + let runtime ← Beam.Broker.ServerRuntime.create + ({ root } : Beam.Broker.BrokerConfig) "fixture" (.tcp 0) + let (beforeClose, _) ← runtime.dispatchRequest { op := .stats } + require "stats should be admitted before session close" beforeClose.ok + require "first session close should win admission shutdown" + (← ActiveRequestRegistry.closeAdmission runtime.activeRequests) + require "repeated session close should be idempotent" + (!(← ActiveRequestRegistry.closeAdmission runtime.activeRequests)) + let (afterClose, _) ← runtime.dispatchRequest { op := .stats } + require "ordinary requests should be rejected after session close" + (afterClose.error?.any fun err => err.code == "requestCancelled") + let (shutdown, shouldStop) ← runtime.dispatchRequest { op := .shutdown } + require "shutdown remains idempotent after admission closes" shutdown.ok + require "an idempotent shutdown should not claim process teardown" (!shouldStop) + require "closed admission should leave no active request" + ((← ActiveRequestRegistry.count runtime.activeRequests) == 0) def main : IO Unit := do checkResponseJsonShape @@ -1019,7 +895,7 @@ def main : IO Unit := do checkRequestArgsBoundary checkWorkspaceRoutingFields checkWorkspaceLifecycleProtocol - checkWrapperLeaseFence + checkSessionCloseAdmission end BeamTest.Broker.ProtocolTest diff --git a/tests/lean/BeamTest/Broker/StreamDedupTest.lean b/tests/lean/BeamTest/Broker/StreamDedupTest.lean index 81015005..cf495276 100644 --- a/tests/lean/BeamTest/Broker/StreamDedupTest.lean +++ b/tests/lean/BeamTest/Broker/StreamDedupTest.lean @@ -159,7 +159,6 @@ private def fakeServerWithLeanSession lean := { nextEpoch := 1, session? := some session } } pure { - root state := ← Std.Mutex.new { bootstrapConfig := config workspaces := Std.TreeMap.empty.insert fixtureWorkspaceId workspace diff --git a/tests/lib/beam-wrapper-common.sh b/tests/lib/beam-wrapper-common.sh index 8f2a25a5..98eeb6e2 100644 --- a/tests/lib/beam-wrapper-common.sh +++ b/tests/lib/beam-wrapper-common.sh @@ -10,6 +10,7 @@ client="" beam_wrapper_tmp_root="" declare -a beam_wrapper_managed_roots=() declare -a beam_wrapper_managed_pids=() +beam_wrapper_last_owner_pid="" # shellcheck source=tests/lib/wait.sh . tests/lib/wait.sh @@ -438,7 +439,7 @@ beam_wrapper_cleanup() { local pid root for pid in ${beam_wrapper_managed_pids[@]+"${beam_wrapper_managed_pids[@]}"}; do - kill "$pid" > /dev/null 2>&1 || true + kill -INT "$pid" > /dev/null 2>&1 || true wait "$pid" 2>/dev/null || true done @@ -456,6 +457,7 @@ beam_wrapper_init() { beam_wrapper_tmp_root="$(mktemp -d /tmp/beam-wrapper-suite-XXXXXX)" beam_wrapper_managed_roots=() beam_wrapper_managed_pids=() + beam_wrapper_last_owner_pid="" trap beam_wrapper_cleanup EXIT } @@ -467,6 +469,29 @@ beam_wrapper_register_pid() { beam_wrapper_managed_pids+=("$1") } +beam_wrapper_start_owner() { + local root="$1" + local backend="${2:-lean}" + local out="$root/.beam/test-owner.out" + local err="$root/.beam/test-owner.err" + local registry="$root/.beam/beam-daemon.json" + + "$beam_script" --root "$root" ensure "$backend" --hold >"$out" 2>"$err" & + beam_wrapper_last_owner_pid="$!" + beam_wrapper_register_pid "$beam_wrapper_last_owner_pid" + if ! wait_for_file "$registry" "Beam session owner registry" 60 || + ! wait_for_file_text "$err" "owning Beam session" "Beam session owner readiness" 600 0.1; then + echo "expected explicit Beam session owner to become ready for $root" >&2 + if [ -f "$out" ]; then + cat "$out" >&2 + fi + if [ -f "$err" ]; then + cat "$err" >&2 + fi + exit 1 + fi +} + beam_wrapper_prepare_project_root() { local name="$1" local root="$beam_wrapper_tmp_root/$name" diff --git a/tests/test-beam-fast.sh b/tests/test-beam-fast.sh index 9b7c9870..e25b62f0 100644 --- a/tests/test-beam-fast.sh +++ b/tests/test-beam-fast.sh @@ -352,13 +352,43 @@ wrapper_todo_update_out="$(mktemp /tmp/lean-beam-wrapper-todo-update-out-XXXXXX) wrapper_todo_update_err="$(mktemp /tmp/lean-beam-wrapper-todo-update-err-XXXXXX)" wrapper_todo_out="$(mktemp /tmp/lean-beam-wrapper-todo-out-XXXXXX)" wrapper_todo_err="$(mktemp /tmp/lean-beam-wrapper-todo-err-XXXXXX)" +wrapper_todo_owner_out="$(mktemp /tmp/lean-beam-wrapper-todo-owner-out-XXXXXX)" +wrapper_todo_owner_err="$(mktemp /tmp/lean-beam-wrapper-todo-owner-err-XXXXXX)" +wrapper_todo_owner_pid="" wrapper_todo_cleanup() { BEAM_CONTROL_DIR="$wrapper_todo_control_dir" \ scripts/lean-beam --root tests/save_olean_project shutdown > /dev/null 2>&1 || true + if [ -n "$wrapper_todo_owner_pid" ]; then + wait "$wrapper_todo_owner_pid" 2>/dev/null || true + fi rm -rf -- "$wrapper_todo_control_dir" - rm -f "$wrapper_todo_update_out" "$wrapper_todo_update_err" "$wrapper_todo_out" "$wrapper_todo_err" + rm -f "$wrapper_todo_update_out" "$wrapper_todo_update_err" "$wrapper_todo_out" "$wrapper_todo_err" \ + "$wrapper_todo_owner_out" "$wrapper_todo_owner_err" } +BEAM_CONTROL_DIR="$wrapper_todo_control_dir" \ + scripts/lean-beam --root tests/save_olean_project ensure --hold \ + >"$wrapper_todo_owner_out" 2>"$wrapper_todo_owner_err" & +wrapper_todo_owner_pid="$!" +for _ in $(seq 1 600); do + if grep -Fq "owning Beam session" "$wrapper_todo_owner_err"; then + break + fi + if ! kill -0 "$wrapper_todo_owner_pid" 2>/dev/null; then + echo "expected lean-beam todo wrapper owner to remain alive" >&2 + cat "$wrapper_todo_owner_err" >&2 + wrapper_todo_cleanup + exit 1 + fi + sleep 0.1 +done +if ! grep -Fq "owning Beam session" "$wrapper_todo_owner_err"; then + echo "timed out waiting for lean-beam todo wrapper owner" >&2 + cat "$wrapper_todo_owner_err" >&2 + wrapper_todo_cleanup + exit 1 +fi + if ! BEAM_CONTROL_DIR="$wrapper_todo_control_dir" \ scripts/lean-beam --root tests/save_olean_project \ update TodoSmoke.lean \ diff --git a/tests/test-beam-install.sh b/tests/test-beam-install.sh index 5f180af7..3c040e40 100644 --- a/tests/test-beam-install.sh +++ b/tests/test-beam-install.sh @@ -14,17 +14,33 @@ cd "$(dirname "$0")/.." . tests/lib/ci-steps.sh # shellcheck source=tests/lib/install-fixtures.sh . tests/lib/install-fixtures.sh +# shellcheck source=tests/lib/wait.sh +. tests/lib/wait.sh BEAM_TEST_SUITE="${BEAM_TEST_SUITE:-install}" BEAM_INSTALL_TEST_PRESEED_ELAN="${BEAM_INSTALL_TEST_PRESEED_ELAN:-auto}" tmp_root="$(mktemp -d /tmp/beam-install-XXXXXX)" +declare -a installed_owner_pids=() +declare -a installed_owner_roots=() host_elan_home="${ELAN_HOME:-}" if [ -z "$host_elan_home" ] && [ -d "$HOME/.elan" ]; then host_elan_home="$HOME/.elan" fi cleanup() { + local owner_pid owner_root + if [ -x "${installed_lean_beam:-}" ]; then + for owner_root in ${installed_owner_roots[@]+"${installed_owner_roots[@]}"}; do + "$installed_lean_beam" --root "$owner_root" shutdown > /dev/null 2>&1 || true + done + fi + for owner_pid in ${installed_owner_pids[@]+"${installed_owner_pids[@]}"}; do + if kill -0 "$owner_pid" 2>/dev/null; then + kill -INT "$owner_pid" 2>/dev/null || true + fi + wait "$owner_pid" 2>/dev/null || true + done expect_owned_tmp_dir "$tmp_root" rm -rf -- "$tmp_root" } @@ -714,6 +730,24 @@ installed_helper="$HOME/.local/bin/lean-beam-search" installed_mcp="$HOME/.local/bin/lean-beam-mcp" installed_runtime_root="$BEAM_INSTALL_ROOT/current" +start_installed_owner() { + local root="$1" + local label="$2" + local owner_out="$tmp_root/$label-owner.out" + local owner_err="$tmp_root/$label-owner.err" + local owner_pid + "$installed_lean_beam" --root "$root" ensure --hold > "$owner_out" 2> "$owner_err" & + owner_pid="$!" + installed_owner_pids+=("$owner_pid") + installed_owner_roots+=("$root") + if ! wait_for_file_text "$owner_err" "owning Beam session" "$label session owner" 600 0.1; then + echo "expected installed wrapper owner to become ready for $root" >&2 + cat "$owner_out" >&2 + cat "$owner_err" >&2 + return 1 + fi +} + if [ ! -L "$installed_lean_beam" ]; then echo "expected installed lean-beam symlink at $installed_lean_beam" >&2 exit 1 @@ -943,8 +977,16 @@ run_custom_toolchain_install_test() ( assert_doctor_contains "custom toolchain" "$custom_doctor_out" 'project toolchain accepted: true' assert_doctor_contains "custom toolchain" "$custom_doctor_out" 'bundle source: installed' assert_doctor_contains "custom toolchain" "$custom_doctor_out" 'bundle toolchain fingerprint: ' - ELAN_HOME="$custom_elan_home" "$custom_installed_lean_beam" --root "$custom_project_root" ensure > /dev/null + custom_owner_err="$custom_project_root/custom-owner.err" + ELAN_HOME="$custom_elan_home" "$custom_installed_lean_beam" --root "$custom_project_root" ensure --hold \ + > /dev/null 2>"$custom_owner_err" & + custom_owner_pid="$!" + if ! wait_for_file_text "$custom_owner_err" "owning Beam session" "custom-toolchain session owner" 600 0.1; then + exit 1 + fi ELAN_HOME="$custom_elan_home" "$custom_installed_lean_beam" --root "$custom_project_root" shutdown > /dev/null + wait_for_exit "$custom_owner_pid" "custom-toolchain session owner" 120 0.1 + wait "$custom_owner_pid" ) run_step "install custom toolchain runtime" run_custom_toolchain_install_test @@ -1589,6 +1631,8 @@ remove_tmp_file "$unsupported_err" printf 'def bVal : Nat := "broken"\n' > SaveSmoke/B.lean ) +start_installed_owner "$project_root" "installed-stale" + stale_sync_err="$(mktemp "$tmp_root/install-stale-sync-XXXXXX")" if "$installed_lean_beam" --root "$project_root" sync SaveSmoke/A.lean >"$stale_sync_err" 2>&1; then echo "expected installed wrapper sync to fail on a stale imported target" >&2 @@ -1620,6 +1664,8 @@ import SaveSmoke.B #check bVal EOF +start_installed_owner "$project_root_standalone" "installed-standalone" + standalone_sync="$("$installed_lean_beam" --root "$project_root_standalone" sync StandaloneSaveSmoke.lean)" if ! printf '%s\n' "$standalone_sync" | python3 -c 'import json,sys; payload=json.load(sys.stdin); sys.exit(0 if payload.get("error") is None else 1)'; then echo "expected installed wrapper sync to succeed on a standalone file the daemon can open" >&2 diff --git a/tests/test-beam-save-olean.sh b/tests/test-beam-save-olean.sh index 3b26306b..627c028e 100755 --- a/tests/test-beam-save-olean.sh +++ b/tests/test-beam-save-olean.sh @@ -35,6 +35,23 @@ beam() { LAKE_ARTIFACT_CACHE=false "$beam_script" "$@" } +declare -a beam_owner_pids=() +beam_owner_last_pid="" + +beam_start_owner() { + local root="$1" + local owner_out="$root/.beam-test-owner.out" + local owner_err="$root/.beam-test-owner.err" + beam --root "$root" ensure --hold >"$owner_out" 2>"$owner_err" & + beam_owner_last_pid="$!" + beam_owner_pids+=("$beam_owner_last_pid") + if ! wait_for_file_text "$owner_err" "owning Beam session" "save-replay session owner" 600 0.1; then + cat "$owner_out" >&2 + cat "$owner_err" >&2 + exit 1 + fi +} + expect_owned_tmp_path() { case "$1" in /tmp/beam-save-olean-*|/tmp/tmp.*|/tmp/beam-validate-*/tmp/beam-save-olean-*|/tmp/beam-validate-*/tmp/tmp.*) @@ -351,6 +368,16 @@ save_race_broker_trace="${BEAM_SAVE_RACE_BROKER_TRACE:-1}" save_race_watchdog_ms="${BEAM_SAVE_RACE_WAIT_DIAGNOSTICS_WATCHDOG_MS:-10000}" cleanup() { + local root owner_pid + for root in "$tmp2" "$tmp3" "$tmp4" "$tmp5" "$tmp6" "$tmp7" "$tmp8"; do + if [ -d "$root" ]; then + beam --root "$root" shutdown > /dev/null 2>&1 || true + fi + done + for owner_pid in ${beam_owner_pids[@]+"${beam_owner_pids[@]}"}; do + kill -INT "$owner_pid" > /dev/null 2>&1 || true + wait "$owner_pid" 2>/dev/null || true + done remove_owned_tmp_tree "$tmp1" remove_owned_tmp_tree "$tmp2" remove_owned_tmp_tree "$tmp3" @@ -394,9 +421,9 @@ fi (cd "$tmp2" && lake_build > /dev/null) edit_b "$tmp2" +beam_start_owner "$tmp2" ( cd "$tmp2" - beam --root "$tmp2" shutdown > /dev/null 2>&1 || true save_json="$(beam --root "$tmp2" lean-close-save SaveSmoke/B.lean)" if [ "$(BEAM_JSON_PAYLOAD="$save_json" python3 - <<'PY' import json, os @@ -504,9 +531,9 @@ fi (cd "$tmp8" && lake_build SaveSmoke/ModuleB.lean > /dev/null) edit_module_b "$tmp8" +beam_start_owner "$tmp8" ( cd "$tmp8" - beam --root "$tmp8" shutdown > /dev/null 2>&1 || true module_save_json="$(beam --root "$tmp8" lean-close-save SaveSmoke/ModuleB.lean)" for artifact_key in olean oleanServer oleanPrivate ir; do artifact="$(BEAM_JSON_PAYLOAD="$module_save_json" BEAM_ARTIFACT_KEY="$artifact_key" python3 - <<'PY' @@ -539,9 +566,9 @@ fi (cd "$tmp7" && LAKE_ARTIFACT_CACHE=false "$lake_cmd" env lean CheckBatchOnly.lean > /dev/null) (cd "$tmp7" && LAKE_ARTIFACT_CACHE=false "$lake_cmd" env lean CheckStructured.lean > /dev/null) edit_structured_setup "$tmp7" +beam_start_owner "$tmp7" ( cd "$tmp7" - beam --root "$tmp7" shutdown > /dev/null 2>&1 || true structured_save="$(beam --root "$tmp7" lean-save StructuredSetup/UsesOption.lean)" if [ "$(BEAM_JSON_PAYLOAD="$structured_save" python3 - <<'PY' import json, os @@ -596,10 +623,13 @@ PY ) (cd "$tmp3" && lake_build > /dev/null) +remove_owned_tmp_file "$race_sentinel" +LEAN_BEAM_BROKER_TRACE="$save_race_broker_trace" \ + LEAN_BEAM_BROKER_WAIT_DIAGNOSTICS_WATCHDOG_MS="$save_race_watchdog_ms" \ + LEAN_BEAM_SAVE_RACE_SENTINEL="$race_sentinel" \ + beam_start_owner "$tmp3" ( cd "$tmp3" - beam --root "$tmp3" shutdown > /dev/null 2>&1 || true - remove_owned_tmp_file "$race_sentinel" LEAN_BEAM_BROKER_TRACE="$save_race_broker_trace" \ LEAN_BEAM_BROKER_WAIT_DIAGNOSTICS_WATCHDOG_MS="$save_race_watchdog_ms" \ LEAN_BEAM_SAVE_RACE_SENTINEL="$race_sentinel" \ @@ -637,16 +667,15 @@ fi (cd "$tmp4" && lake_build > /dev/null) edit_b_slow "$tmp4" +remove_owned_tmp_file "$cancel_sentinel" +LEAN_BEAM_BROKER_TRACE="$save_race_broker_trace" \ + LEAN_BEAM_BROKER_WAIT_DIAGNOSTICS_WATCHDOG_MS="$save_race_watchdog_ms" \ + LEAN_BEAM_SAVE_RACE_SENTINEL="$cancel_sentinel" \ + beam_start_owner "$tmp4" ( cd "$tmp4" - beam --root "$tmp4" shutdown > /dev/null 2>&1 || true close_out="$(mktemp /tmp/beam-close-save-cancel-out-XXXXXX)" close_err="$(mktemp /tmp/beam-close-save-cancel-err-XXXXXX)" - remove_owned_tmp_file "$cancel_sentinel" - LEAN_BEAM_BROKER_TRACE="$save_race_broker_trace" \ - LEAN_BEAM_BROKER_WAIT_DIAGNOSTICS_WATCHDOG_MS="$save_race_watchdog_ms" \ - LEAN_BEAM_SAVE_RACE_SENTINEL="$cancel_sentinel" \ - beam --root "$tmp4" ensure lean > /dev/null LEAN_BEAM_SAVE_RACE_SENTINEL="$cancel_sentinel" BEAM_REQUEST_ID=cancel-close-save \ beam --root "$tmp4" lean-close-save SaveSmoke/B.lean >"$close_out" 2>"$close_err" & close_pid=$! @@ -682,14 +711,14 @@ edit_b_slow "$tmp4" exit 1 fi beam --root "$tmp4" stats > /dev/null + beam --root "$tmp4" shutdown > /dev/null rm -f "$close_out" "$close_err" ) (cd "$tmp6" && lake_build SaveSmoke/A.lean > /dev/null) +beam_start_owner "$tmp6" ( cd "$tmp6" - beam --root "$tmp6" shutdown > /dev/null 2>&1 || true - beam --root "$tmp6" ensure lean > /dev/null beam --root "$tmp6" lean-sync SaveSmoke/A.lean > /dev/null edit_b "$tmp6" save_out="$(mktemp /tmp/beam-stale-trace-save-out-XXXXXX)" @@ -716,15 +745,15 @@ edit_b_slow "$tmp4" exit 1 fi beam --root "$tmp6" stats > /dev/null + beam --root "$tmp6" shutdown > /dev/null rm -f "$save_out" "$save_err" ) (cd "$tmp5" && lake_build SaveSmoke/A.lean > /dev/null) printf 'def bVal : Nat := "broken"\n' > "$tmp5/SaveSmoke/B.lean" +beam_start_owner "$tmp5" ( cd "$tmp5" - beam --root "$tmp5" shutdown > /dev/null 2>&1 || true - beam --root "$tmp5" ensure lean > /dev/null sync_out="$(mktemp /tmp/beam-stale-sync-out-XXXXXX)" sync_err="$(mktemp /tmp/beam-stale-sync-err-XXXXXX)" save_out="$(mktemp /tmp/beam-stale-save-out-XXXXXX)" @@ -779,5 +808,6 @@ printf 'def bVal : Nat := "broken"\n' > "$tmp5/SaveSmoke/B.lean" exit 1 fi beam --root "$tmp5" stats > /dev/null + beam --root "$tmp5" shutdown > /dev/null rm -f "$sync_out" "$sync_err" "$save_out" "$save_err" ) diff --git a/tests/test-beam-toolchain-compat.sh b/tests/test-beam-toolchain-compat.sh index 47cf0eeb..5be806a8 100644 --- a/tests/test-beam-toolchain-compat.sh +++ b/tests/test-beam-toolchain-compat.sh @@ -11,6 +11,8 @@ cd "$(dirname "$0")/.." . tests/lib/ci-steps.sh # shellcheck source=tests/lib/tmp-guards.sh . tests/lib/tmp-guards.sh +# shellcheck source=tests/lib/wait.sh +. tests/lib/wait.sh BEAM_TEST_SUITE="${BEAM_TEST_SUITE:-beam-toolchain-compat}" bundle_timeout="${BEAM_TOOLCHAIN_COMPAT_TIMEOUT:-600}" @@ -40,6 +42,9 @@ stale_build_stdout="$tmp_env_root/stale-build.stdout" stale_build_stderr="$tmp_env_root/stale-build.stderr" stale_sync_stdout="$tmp_env_root/stale-sync.stdout" stale_sync_stderr="$tmp_env_root/stale-sync.stderr" +stale_owner_stdout="$tmp_env_root/stale-owner.stdout" +stale_owner_stderr="$tmp_env_root/stale-owner.stderr" +stale_owner_pid="" toolchain_failed=false if [ -z "${ELAN_HOME:-}" ] && [ -d "$HOME/.elan" ]; then @@ -57,6 +62,12 @@ cleanup() { ELAN_HOME="$host_elan_home" BEAM_INSTALL_BUNDLE_DIR="$tmp_bundle_dir" \ ./scripts/lean-beam --root "$stale_project_root" shutdown > /dev/null 2>&1 || true fi + if [ -n "${stale_owner_pid:-}" ]; then + if kill -0 "$stale_owner_pid" 2>/dev/null; then + kill -INT "$stale_owner_pid" 2>/dev/null || true + fi + wait "$stale_owner_pid" 2>/dev/null || true + fi if [ "$toolchain_failed" = "true" ]; then case "$keep_tmp_on_failure" in 1|true|True|TRUE|yes|Yes|YES|on|On|ON) @@ -109,6 +120,8 @@ print_toolchain_context() { tail_file "stale build stderr" "$stale_build_stderr" tail_file "stale sync stdout" "$stale_sync_stdout" tail_file "stale sync stderr" "$stale_sync_stderr" + tail_file "stale owner stdout" "$stale_owner_stdout" + tail_file "stale owner stderr" "$stale_owner_stderr" } run_build() { @@ -260,6 +273,35 @@ run_stale_wrapper_checked() { fi } +start_stale_owner() { + run_stale_wrapper ensure --hold > "$stale_owner_stdout" 2> "$stale_owner_stderr" & + stale_owner_pid="$!" + if ! wait_for_file_text "$stale_owner_stderr" "owning Beam session" \ + "toolchain compatibility session owner" 600 0.1; then + print_toolchain_context "explicit session owner failed to start" + return 1 + fi +} + +stop_stale_owner() { + if [ -z "$stale_owner_pid" ]; then + return 0 + fi + if ! run_stale_wrapper shutdown > /dev/null 2> "$stale_sync_stderr"; then + print_toolchain_context "explicit session owner failed to shut down" + return 1 + fi + if ! wait_for_exit "$stale_owner_pid" "toolchain compatibility session owner" 120 0.1; then + print_toolchain_context "explicit session owner did not exit" + return 1 + fi + if ! wait "$stale_owner_pid"; then + print_toolchain_context "explicit session owner exited unsuccessfully" + return 1 + fi + stale_owner_pid="" +} + run_stale_diagnostic_compat() { local rc=0 prepare_stale_diagnostic_project @@ -280,6 +322,10 @@ EOF return "$rc" fi + if ! start_stale_owner; then + return 1 + fi + if ! run_stale_wrapper_checked "initial dependency sync failed" sync SaveSmoke/B.lean; then return 1 fi @@ -320,6 +366,7 @@ EOF print_toolchain_context "stale diagnostic wording changed" return 1 fi + stop_stale_owner } run_step "build beam-cli" run_build diff --git a/tests/test-beam-wrapper-daemon.sh b/tests/test-beam-wrapper-daemon.sh index 7b385b0c..1d5f11a4 100644 --- a/tests/test-beam-wrapper-daemon.sh +++ b/tests/test-beam-wrapper-daemon.sh @@ -23,71 +23,52 @@ if [ ! -x "$beam_cli" ]; then exit 1 fi -stop_hold_process() { - local require_clean_exit="${1:-false}" - if [ -n "$hold_pid" ]; then - kill -INT "$hold_pid" > /dev/null 2>&1 || true - if ! wait_for_exit "$hold_pid" "ensure --hold wrapper" 20 0.1; then - kill "$hold_pid" > /dev/null 2>&1 || true - wait "$hold_pid" 2>/dev/null || true - hold_pid="" - if [ "$require_clean_exit" = "true" ]; then - echo "expected ensure --hold wrapper to exit promptly after SIGINT" >&2 - return 1 - fi - else - local hold_status=0 - set +e - wait "$hold_pid" 2>/dev/null - hold_status="$?" - set -e - hold_pid="" - if [ "$require_clean_exit" = "true" ] && [ "$hold_status" -ne 0 ]; then - echo "expected ensure --hold wrapper to exit cleanly after SIGINT, got $hold_status" >&2 - return 1 - fi - fi - hold_pid="" - fi -} - -tmp1="$(mktemp -d /tmp/beam-wrapper-daemon-a-XXXXXX)" -tmp3="$(mktemp -d /tmp/beam-wrapper-daemon-c-XXXXXX)" -tmp9="$(mktemp -d /tmp/beam-wrapper-daemon-i-XXXXXX)" +tmp1="$(mktemp -d /tmp/beam-wrapper-daemon-owner-a-XXXXXX)" +tmp2="$(mktemp -d /tmp/beam-wrapper-daemon-owner-b-XXXXXX)" owned_bundle_dir="" if [ -z "${BEAM_INSTALL_BUNDLE_DIR:-}" ]; then - owned_bundle_dir="$(mktemp -d /tmp/beam-wrapper-daemon-bundles-XXXXXX)" + owned_bundle_dir="$(mktemp -d /tmp/beam-wrapper-daemon-owner-bundles-XXXXXX)" export BEAM_INSTALL_BUNDLE_DIR="$owned_bundle_dir" fi -busy_pid="" hold_pid="" -heartbeat_follower_pid="" -removed_root_pid="" -removed_root_err="" +root_removed="false" -cleanup() { - stop_hold_process - if [ -n "$busy_pid" ]; then - kill "$busy_pid" > /dev/null 2>&1 || true - wait "$busy_pid" 2>/dev/null || true +stop_hold_process() { + local require_clean_exit="${1:-false}" + if [ -z "$hold_pid" ]; then + return fi - if [ -n "$heartbeat_follower_pid" ]; then - kill "$heartbeat_follower_pid" > /dev/null 2>&1 || true - wait "$heartbeat_follower_pid" 2>/dev/null || true + kill -INT "$hold_pid" > /dev/null 2>&1 || true + if ! wait_for_exit "$hold_pid" "ensure --hold owner" 200 0.05; then + kill "$hold_pid" > /dev/null 2>&1 || true + wait "$hold_pid" 2>/dev/null || true + hold_pid="" + if [ "$require_clean_exit" = "true" ]; then + echo "expected ensure --hold owner to exit promptly after SIGINT" >&2 + return 1 + fi + return fi - if [ -n "$removed_root_pid" ]; then - kill "$removed_root_pid" > /dev/null 2>&1 || true - wait "$removed_root_pid" 2>/dev/null || true + local status=0 + set +e + wait "$hold_pid" + status="$?" + set -e + hold_pid="" + if [ "$require_clean_exit" = "true" ] && [ "$status" -ne 0 ]; then + echo "expected ensure --hold owner to exit cleanly, got $status" >&2 + return 1 fi - if [ -n "$removed_root_err" ]; then - rm -f "$removed_root_err" +} + +cleanup() { + stop_hold_process + if [ "$root_removed" != "true" ]; then + "$beam_script" --root "$tmp1" shutdown > /dev/null 2>&1 || true + remove_owned_tmp_tree "$tmp1" fi - "$beam_script" --root "$tmp1" shutdown > /dev/null 2>&1 || true - "$beam_script" --root "$tmp3" shutdown > /dev/null 2>&1 || true - "$beam_script" --root "$tmp9" shutdown > /dev/null 2>&1 || true - remove_owned_tmp_tree "$tmp1" - remove_owned_tmp_tree "$tmp3" - remove_owned_tmp_tree "$tmp9" + "$beam_script" --root "$tmp2" shutdown > /dev/null 2>&1 || true + remove_owned_tmp_tree "$tmp2" if [ -n "$owned_bundle_dir" ]; then remove_owned_tmp_tree "$owned_bundle_dir" fi @@ -97,449 +78,229 @@ trap cleanup EXIT if [ -n "$owned_bundle_dir" ]; then expect_owned_tmp_dir "$owned_bundle_dir" fi - -fixture_toolchain="$(awk 'NR==1 {print $1}' tests/save_olean_project/lean-toolchain)" -"$beam_cli" bundle-install "$fixture_toolchain" - -for tmp in "$tmp1" "$tmp3" "$tmp9"; do +for tmp in "$tmp1" "$tmp2"; do expect_owned_tmp_dir "$tmp" rsync -a --exclude='.beam/' tests/save_olean_project/ "$tmp"/ remove_tmp_tree_within "$tmp/.beam" "$tmp" mkdir -p "$tmp/.beam" done -"$beam_script" --root "$tmp9" ensure --hold > "$tmp9/hold.out" 2> "$tmp9/hold.err" & -hold_pid="$!" -hold_registry="$tmp9/.beam/beam-daemon.json" -# A cold installed-bundle qualification can build the selected Lean payload before `ensure` -# responds. Keep this bounded, but allow enough time for that legitimate first-use path on CI. -hold_ready_attempts="${BEAM_TEST_HOLD_READY_ATTEMPTS:-1800}" -case "$hold_ready_attempts" in - ''|*[!0-9]*|0) - echo "BEAM_TEST_HOLD_READY_ATTEMPTS must be a positive integer" >&2 - exit 1 - ;; -esac -for _ in $(seq 1 "$hold_ready_attempts"); do - if [ -s "$tmp9/hold.out" ] && [ -f "$hold_registry" ]; then - break - fi - sleep 0.1 -done -if [ ! -s "$tmp9/hold.out" ] || [ ! -f "$hold_registry" ]; then - echo "expected ensure --hold to print an ensure response and create a registry after $hold_ready_attempts readiness probes" >&2 - cat "$tmp9/hold.err" >&2 +fixture_toolchain="$(awk 'NR==1 {print $1}' tests/save_olean_project/lean-toolchain)" +"$beam_cli" bundle-install "$fixture_toolchain" + +missing_owner_out="$tmp1/missing-owner.out" +missing_owner_err="$tmp1/missing-owner.err" +if "$beam_script" --root "$tmp1" ensure > "$missing_owner_out" 2> "$missing_owner_err"; then + echo "expected an ordinary wrapper command to require a session owner" >&2 + cat "$missing_owner_out" >&2 exit 1 fi -if ! kill -0 "$hold_pid" 2>/dev/null; then - echo "expected ensure --hold wrapper process to remain alive" >&2 - cat "$tmp9/hold.err" >&2 +if ! grep -Fq "lean-beam ensure --hold" "$missing_owner_err"; then + echo "expected missing-owner error to name the recovery command" >&2 + cat "$missing_owner_err" >&2 exit 1 fi -hold_json="$(cat "$tmp9/hold.out")" -assert_json_field_equals "ensure --hold response" "$hold_json" ok true "$tmp9/hold.err" -lease_dir="$tmp9/.beam/wrapper-leases" -retirement_path="$tmp9/.beam/daemon-retirement.json" - -# A retirement marker is local control state, but its lease field must still be constrained to the -# wrapper-leases directory before cleanup code joins or removes that path. -outside_lease="$tmp9/.beam/outside.lease" -printf 'preserve\n' > "$outside_lease" -retirement_daemon_id="$(read_json_field "$hold_registry" daemonId)" -RETIREMENT_PATH="$retirement_path" DAEMON_ID="$retirement_daemon_id" python3 - <<'PY' -import json, os +start_owner() { + local root="$1" + local label="$2" + local out="$root/$label.out" + local err="$root/$label.err" + "$beam_script" --root "$root" ensure --hold > "$out" 2> "$err" & + hold_pid="$!" + local registry="$root/.beam/beam-daemon.json" + local attempts="${BEAM_TEST_HOLD_READY_ATTEMPTS:-1800}" + case "$attempts" in + ''|*[!0-9]*|0) + echo "BEAM_TEST_HOLD_READY_ATTEMPTS must be a positive integer" >&2 + exit 1 + ;; + esac + for _ in $(seq 1 "$attempts"); do + if [ -s "$out" ] && [ -f "$registry" ]; then + break + fi + if ! kill -0 "$hold_pid" 2>/dev/null; then + echo "session owner exited before becoming ready" >&2 + cat "$err" >&2 + exit 1 + fi + sleep 0.1 + done + if [ ! -s "$out" ] || [ ! -f "$registry" ]; then + echo "expected ensure --hold to publish its response and registry" >&2 + cat "$err" >&2 + exit 1 + fi + assert_json_field_equals "ensure --hold response" "$(cat "$out")" ok true "$err" +} -with open(os.environ["RETIREMENT_PATH"], "w") as f: - json.dump({"daemonId": os.environ["DAEMON_ID"], "ownerLeaseFile": "../outside.lease"}, f) - f.write("\n") -PY -"$beam_script" --root "$tmp9" ensure lean > /dev/null -if [ ! -f "$outside_lease" ]; then - echo "expected an invalid retirement owner path not to remove a file outside wrapper-leases" >&2 +registry="$tmp1/.beam/beam-daemon.json" +start_owner "$tmp1" "owner-1" +owner1_pid="$hold_pid" +daemon1_pid="$(read_json_field "$registry" pid)" +daemon1_id="$(read_json_field "$registry" daemonId)" +recorded_owner_pid="$(read_json_field "$registry" ownerPid)" +owner_domain="$(read_json_field "$registry" ownerPidDomain)" +case "$recorded_owner_pid" in + ''|*[!0-9]*|0) + echo "expected registry to record a positive session-owner PID" >&2 + cat "$registry" >&2 + exit 1 + ;; +esac +if [ -z "$owner_domain" ]; then + echo "expected registry to record the session owner's PID domain" >&2 + cat "$registry" >&2 exit 1 fi -if [ -e "$retirement_path" ]; then - echo "expected an invalid retirement owner path to be discarded" >&2 - cat "$retirement_path" >&2 +if ! kill -0 "$owner1_pid" 2>/dev/null || ! kill -0 "$daemon1_pid" 2>/dev/null; then + echo "expected both the wrapper owner and daemon to remain alive" >&2 exit 1 fi -rm -f "$outside_lease" -# A reused-daemon request must stop if its heartbeat writer fails; otherwise the owner can prune its -# expired lease while the request is still using the daemon. Block only the follower's atomic tmp -# path so the starter heartbeat remains healthy. -owner_lease="$(find "$lease_dir" -maxdepth 1 -type f -name '*.lease' -print | sed -n '1p')" -if [ -z "$owner_lease" ]; then - echo "expected the foreground owner to hold a wrapper lease" >&2 +ensure_json="$("$beam_script" --root "$tmp1" ensure)" +assert_json_field_equals "owned ensure response" "$ensure_json" ok true +stats_json="$("$beam_script" --root "$tmp1" stats)" +assert_json_field_equals "owned stats response" "$stats_json" ok true + +second_owner_out="$tmp1/second-owner.out" +second_owner_err="$tmp1/second-owner.err" +if "$beam_script" --root "$tmp1" ensure --hold > "$second_owner_out" 2> "$second_owner_err"; then + echo "expected a second foreground owner to be rejected" >&2 + cat "$second_owner_out" >&2 exit 1 fi -"$beam_script" --root "$tmp9" ensure --hold \ - > "$tmp9/heartbeat-follower.out" 2> "$tmp9/heartbeat-follower.err" & -heartbeat_follower_pid="$!" -heartbeat_follower_lease="" -for _ in $(seq 1 100); do - heartbeat_follower_lease="$(find "$lease_dir" -maxdepth 1 -type f -name '*.lease' \ - ! -path "$owner_lease" -print | sed -n '1p')" - if [ -s "$tmp9/heartbeat-follower.out" ] && [ -n "$heartbeat_follower_lease" ]; then - break - fi - sleep 0.05 -done -if [ ! -s "$tmp9/heartbeat-follower.out" ] || [ -z "$heartbeat_follower_lease" ]; then - echo "expected the heartbeat-failure follower to acquire a lease and print ensure output" >&2 - cat "$tmp9/heartbeat-follower.err" >&2 +if ! grep -Fq "already owned" "$second_owner_err"; then + echo "expected duplicate-owner failure to identify the active owner" >&2 + cat "$second_owner_err" >&2 exit 1 fi -heartbeat_tmp="${heartbeat_follower_lease%.lease}.tmp" -for _ in $(seq 1 100); do - if mkdir "$heartbeat_tmp" 2>/dev/null; then - break - fi - sleep 0.01 -done -if [ ! -d "$heartbeat_tmp" ]; then - echo "could not block the follower heartbeat tmp path" >&2 + +port1="$(read_json_field "$registry" port)" +collision_out="$tmp2/collision.out" +collision_err="$tmp2/collision.err" +if "$beam_script" --root "$tmp2" --port "$port1" ensure --hold > "$collision_out" 2> "$collision_err"; then + echo "expected an owner not to claim another project's endpoint" >&2 + cat "$collision_out" >&2 exit 1 fi -if ! wait_for_exit "$heartbeat_follower_pid" "wrapper with failed heartbeat writer" 100 0.05; then - cat "$tmp9/heartbeat-follower.err" >&2 +if ! grep -Fq "already serves Beam root" "$collision_err"; then + echo "expected endpoint collision to identify the served project root" >&2 + cat "$collision_err" >&2 exit 1 fi -set +e -wait "$heartbeat_follower_pid" -heartbeat_follower_status="$?" -set -e -heartbeat_follower_pid="" -rmdir "$heartbeat_tmp" -if [ "$heartbeat_follower_status" -eq 0 ]; then - echo "expected a reused-daemon wrapper with a failed heartbeat writer to fail" >&2 - cat "$tmp9/heartbeat-follower.err" >&2 +if [ -e "$tmp2/.beam/beam-daemon.json" ]; then + echo "expected endpoint collision not to publish a registry" >&2 + cat "$tmp2/.beam/beam-daemon.json" >&2 exit 1 fi -# Neither an unreadable registry nor an unreadable sibling lease may make the starter leave its -# retirement loop. Restore each observation independently and require the owner to remain alive -# until the lease directory is provably drained. -unreadable_lease="$lease_dir/unreadable-sibling.lease" -mkdir "$unreadable_lease" -mv "$hold_registry" "$hold_registry.saved" -mkdir "$hold_registry" -kill -INT "$hold_pid" -sleep 0.5 -if ! kill -0 "$hold_pid" 2>/dev/null; then - echo "expected an owner to stay alive while registry state is unreadable" >&2 - cat "$tmp9/hold.err" >&2 - exit 1 -fi -rmdir "$hold_registry" -mv "$hold_registry.saved" "$hold_registry" -sleep 0.5 -if ! kill -0 "$hold_pid" 2>/dev/null; then - echo "expected an owner to stay alive while a sibling lease is unreadable" >&2 - cat "$tmp9/hold.err" >&2 - exit 1 -fi -rmdir "$unreadable_lease" -if ! wait_for_exit "$hold_pid" "owner after registry and lease recovery" 100 0.05; then - cat "$tmp9/hold.err" >&2 +shutdown_json="$("$beam_script" --root "$tmp1" shutdown)" +assert_json_field_equals "explicit session shutdown" "$shutdown_json" ok true +if ! wait_for_exit "$hold_pid" "owner after explicit shutdown" 200 0.05; then + cat "$tmp1/owner-1.err" >&2 exit 1 fi set +e wait "$hold_pid" -hold_status="$?" +owner_status="$?" set -e hold_pid="" -if [ "$hold_status" -ne 0 ]; then - echo "expected the owner to exit cleanly after registry and lease recovery, got $hold_status" >&2 - cat "$tmp9/hold.err" >&2 +if [ "$owner_status" -ne 0 ]; then + echo "expected owner to exit cleanly after explicit shutdown, got $owner_status" >&2 + cat "$tmp1/owner-1.err" >&2 exit 1 fi -"$beam_script" --root "$tmp9" shutdown > /dev/null - -stale_lease_dir="$tmp9/.beam/wrapper-leases" -stale_lease="$stale_lease_dir/stale-dead-wrapper.lease" -mkdir -p "$stale_lease_dir" -case "$(uname -s)" in - Linux) pid_domain="$(readlink /proc/self/ns/pid 2>/dev/null || true)" ;; - Darwin) pid_domain="host:Darwin" ;; - *) pid_domain="" ;; -esac -LEASE_PATH="$stale_lease" PID_DOMAIN="$pid_domain" python3 - <<'PY' -import json, os, time - -metadata = { - "pid": 999999999, - "pidDomain": os.environ["PID_DOMAIN"] or None, - "heartbeatMonoNanos": time.monotonic_ns(), -} -with open(os.environ["LEASE_PATH"], "w") as f: - json.dump(metadata, f) - f.write("\n") -PY - -"$beam_script" --root "$tmp9" ensure lean > /dev/null -if [ -e "$stale_lease" ]; then - echo "expected wrapper ensure to remove a stale same-domain wrapper lease" >&2 - cat "$stale_lease" >&2 +if ! wait_for_exit "$daemon1_pid" "daemon after explicit session shutdown" 200 0.05; then exit 1 fi -"$beam_script" --root "$tmp9" shutdown > /dev/null - -malformed_lease="$stale_lease_dir/malformed-wrapper.lease" -printf '{\n' > "$malformed_lease" -"$beam_script" --root "$tmp9" ensure lean > /dev/null -if [ -e "$malformed_lease" ]; then - echo "expected wrapper ensure to prune a malformed wrapper lease" >&2 - cat "$malformed_lease" >&2 +if [ -e "$registry" ]; then + echo "expected owner shutdown to remove its registry" >&2 + cat "$registry" >&2 exit 1 fi -"$beam_script" --root "$tmp9" shutdown > /dev/null -rm -f "$retirement_path" -mkdir "$retirement_path" -set +e -"$beam_script" --root "$tmp9" ensure lean > "$tmp9/retirement-read.out" 2> "$tmp9/retirement-read.err" -retirement_read_status="$?" -set -e -if [ "$retirement_read_status" -eq 0 ]; then - echo "expected an unreadable retirement fence to fail closed" >&2 - cat "$tmp9/retirement-read.out" >&2 - cat "$tmp9/retirement-read.err" >&2 +start_owner "$tmp1" "owner-2" +daemon2_pid="$(read_json_field "$registry" pid)" +daemon2_id="$(read_json_field "$registry" daemonId)" +if [ "$daemon2_id" = "$daemon1_id" ]; then + echo "expected a new owner to publish a new daemon generation" >&2 exit 1 fi -if [ ! -d "$retirement_path" ]; then - echo "expected an unreadable retirement fence to remain in place" >&2 - exit 1 -fi -rmdir "$retirement_path" -"$beam_script" --root "$tmp9" ensure lean > /dev/null -"$beam_script" --root "$tmp9" shutdown > /dev/null -# If the daemon started by a foreground owner dies before retirement, that wrapper must not poll -# forever waiting for stats from a process that is provably gone. A later wrapper should replace -# the dead registry generation normally. -rm -f "$tmp9/dead-daemon-hold.out" "$tmp9/dead-daemon-hold.err" -"$beam_script" --root "$tmp9" ensure --hold \ - > "$tmp9/dead-daemon-hold.out" 2> "$tmp9/dead-daemon-hold.err" & -hold_pid="$!" -for _ in $(seq 1 200); do - if [ -s "$tmp9/dead-daemon-hold.out" ] && [ -f "$hold_registry" ]; then - break - fi - sleep 0.05 -done -if [ ! -s "$tmp9/dead-daemon-hold.out" ] || [ ! -f "$hold_registry" ]; then - echo "expected the crash-retirement owner to start a daemon" >&2 - cat "$tmp9/dead-daemon-hold.err" >&2 - exit 1 -fi -dead_daemon_pid="$(read_json_field "$hold_registry" pid)" -dead_daemon_id="$(read_json_field "$hold_registry" daemonId)" -kill -KILL "$dead_daemon_pid" -dead_daemon_stopped="false" -for _ in $(seq 1 100); do - if ! kill -0 "$dead_daemon_pid" 2>/dev/null; then - dead_daemon_stopped="true" - break - fi - if ps -o stat= -p "$dead_daemon_pid" 2>/dev/null | grep -Eq '^[[:space:]]*Z'; then - dead_daemon_stopped="true" - break - fi - sleep 0.05 -done -if [ "$dead_daemon_stopped" != "true" ]; then - echo "expected daemon $dead_daemon_pid to stop before owner retirement" >&2 - exit 1 -fi -kill -INT "$hold_pid" -if ! wait_for_exit "$hold_pid" "owner of a provably dead daemon" 100 0.05; then - cat "$tmp9/dead-daemon-hold.err" >&2 +kill -KILL "$daemon2_pid" +if ! wait_for_exit "$hold_pid" "owner after unexpected daemon crash" 200 0.05; then + cat "$tmp1/owner-2.err" >&2 exit 1 fi set +e wait "$hold_pid" -dead_daemon_owner_status="$?" +crashed_owner_status="$?" set -e hold_pid="" -if [ "$dead_daemon_owner_status" -ne 0 ]; then - echo "expected the owner of a provably dead daemon to exit cleanly, got $dead_daemon_owner_status" >&2 - cat "$tmp9/dead-daemon-hold.err" >&2 +if [ "$crashed_owner_status" -eq 0 ]; then + echo "expected the owner to report an unexpected daemon crash" >&2 exit 1 fi -"$beam_script" --root "$tmp9" ensure lean > /dev/null -replacement_daemon_id="$(read_json_field "$hold_registry" daemonId)" -if [ "$replacement_daemon_id" = "$dead_daemon_id" ]; then - echo "expected the next wrapper to replace the dead daemon generation" >&2 - cat "$hold_registry" >&2 +if ! grep -Fq "owned Beam daemon exited with status" "$tmp1/owner-2.err"; then + echo "expected the owner crash report to include the daemon exit status" >&2 + cat "$tmp1/owner-2.err" >&2 + exit 1 +fi +if [ -e "$registry" ]; then + echo "expected a crashed daemon's owner to remove its exact registry generation" >&2 + cat "$registry" >&2 exit 1 fi -"$beam_script" --root "$tmp9" shutdown > /dev/null - -( - cd "$tmp1" - "$beam_script" ensure lean > /dev/null -) - -reg1="$tmp1/.beam/beam-daemon.json" -expect_file "$reg1" -pid1="$(read_json_field "$reg1" pid)" -port1="$(read_json_field "$reg1" port)" -root1="$(read_json_field "$reg1" root)" -if [ "$root1" != "$(beam_test_realpath "$tmp1")" ]; then - echo "wrapper registry root mismatch: expected $tmp1, got $root1" >&2 +start_owner "$tmp1" "owner-3" +daemon3_pid="$(read_json_field "$registry" pid)" +kill -KILL "$hold_pid" +set +e +wait "$hold_pid" +set -e +hold_pid="" +if ! wait_for_exit "$daemon3_pid" "daemon after owner death" 200 0.05; then + echo "expected owner-pipe EOF to stop the daemon" >&2 exit 1 fi -if ! kill -0 "$pid1" 2>/dev/null; then - echo "expected Beam daemon pid $pid1 to be alive" >&2 +owner_loss_out="$tmp1/owner-loss.out" +owner_loss_err="$tmp1/owner-loss.err" +if "$beam_script" --root "$tmp1" ensure > "$owner_loss_out" 2> "$owner_loss_err"; then + echo "expected a command after owner loss to require a replacement owner" >&2 + cat "$owner_loss_out" >&2 exit 1 fi - -( - cd "$tmp3" - collision_out="$(mktemp /tmp/beam-wrapper-port-collision-out-XXXXXX)" - collision_err="$(mktemp /tmp/beam-wrapper-port-collision-err-XXXXXX)" - if "$beam_script" --port "$port1" ensure lean >"$collision_out" 2>"$collision_err"; then - echo "expected wrapper ensure to reject a port already serving another Beam root" >&2 - cat "$collision_out" >&2 - cat "$collision_err" >&2 - rm -f "$collision_out" "$collision_err" - exit 1 - fi - if ! grep -q 'already serves Beam root' "$collision_err"; then - echo "expected port collision failure to name the existing Beam root" >&2 - cat "$collision_out" >&2 - cat "$collision_err" >&2 - rm -f "$collision_out" "$collision_err" - exit 1 - fi - if [ -f "$tmp3/.beam/beam-daemon.json" ]; then - echo "expected port collision failure not to write a registry for the wrong endpoint" >&2 - cat "$tmp3/.beam/beam-daemon.json" >&2 - rm -f "$collision_out" "$collision_err" - exit 1 - fi - rm -f "$collision_out" "$collision_err" -) - -stale_registry="$tmp3/.beam/beam-daemon.json" -REGISTRY_TEMPLATE="$reg1" STALE_REGISTRY="$stale_registry" STALE_ROOT="$tmp3" python3 - <<'PY' -import json -import os - -with open(os.environ["REGISTRY_TEMPLATE"]) as f: - data = json.load(f) -data["root"] = os.path.realpath(os.environ["STALE_ROOT"]) -data["configHash"] = "stale-registry-test" -with open(os.environ["STALE_REGISTRY"], "w") as f: - json.dump(data, f) - f.write("\n") -PY - -( - cd "$tmp3" - doctor_out="$("$beam_script" doctor lean)" - if ! printf '%s\n' "$doctor_out" | grep -q 'daemon status: stale'; then - echo "expected wrapper doctor to reject a stale registry whose endpoint serves another root" >&2 - printf '%s\n' "$doctor_out" >&2 - exit 1 - fi - "$beam_script" shutdown > /dev/null - if [ -f "$stale_registry" ]; then - echo "expected wrapper shutdown to remove the stale registry" >&2 - cat "$stale_registry" >&2 - exit 1 - fi -) -if ! kill -0 "$pid1" 2>/dev/null; then - echo "expected stale registry shutdown not to kill the real Beam daemon for tmp1" >&2 +if ! grep -Fq "lean-beam ensure --hold" "$owner_loss_err"; then + echo "expected owner-loss recovery to name ensure --hold" >&2 + cat "$owner_loss_err" >&2 exit 1 fi - -busy_port_file="$(mktemp /tmp/beam-wrapper-busy-port-XXXXXX)" -python3 - "$busy_port_file" <<'PY' & -import http.server -import socketserver -import sys - -class Handler(http.server.SimpleHTTPRequestHandler): - def log_message(self, format, *args): - pass - -with socketserver.TCPServer(("127.0.0.1", 0), Handler) as server: - with open(sys.argv[1], "w") as f: - print(server.server_address[1], file=f, flush=True) - server.serve_forever() -PY -busy_pid=$! -for _ in $(seq 1 100); do - if [ -s "$busy_port_file" ]; then - break - fi - if ! kill -0 "$busy_pid" 2>/dev/null; then - echo "expected temporary busy-port server to stay alive" >&2 - exit 1 - fi - sleep 0.05 -done -if [ ! -s "$busy_port_file" ]; then - echo "timed out waiting for temporary busy-port server" >&2 +if [ -e "$registry" ]; then + echo "expected owner-loss recovery to remove the stale registry" >&2 + cat "$registry" >&2 exit 1 fi -busy_port="$(cat "$busy_port_file")" - -( - cd "$tmp3" - busy_out="$(mktemp /tmp/beam-wrapper-busy-port-out-XXXXXX)" - busy_err="$(mktemp /tmp/beam-wrapper-busy-port-err-XXXXXX)" - if "$beam_script" --port "$busy_port" ensure lean >"$busy_out" 2>"$busy_err"; then - echo "expected wrapper ensure to reject a port already used by a non-Beam process" >&2 - cat "$busy_out" >&2 - cat "$busy_err" >&2 - rm -f "$busy_out" "$busy_err" - exit 1 - fi - if ! grep -q 'already in use' "$busy_err"; then - echo "expected non-Beam port collision failure to report the occupied endpoint" >&2 - cat "$busy_out" >&2 - cat "$busy_err" >&2 - rm -f "$busy_out" "$busy_err" - exit 1 - fi - if [ -f "$tmp3/.beam/beam-daemon.json" ]; then - echo "expected non-Beam port collision failure not to write a registry" >&2 - cat "$tmp3/.beam/beam-daemon.json" >&2 - rm -f "$busy_out" "$busy_err" - exit 1 - fi - rm -f "$busy_out" "$busy_err" -) -kill "$busy_pid" > /dev/null 2>&1 || true -wait "$busy_pid" 2>/dev/null || true -busy_pid="" -rm -f "$busy_port_file" -# Git removes the project-local registry together with a worktree. The daemon must observe the -# missing canonical root and stop itself, because no later wrapper invocation can discover it. -removed_root_pid="$pid1" +start_owner "$tmp1" "owner-4" +daemon4_pid="$(read_json_field "$registry" pid)" remove_owned_tmp_tree "$tmp1" -if ! wait_for_exit "$removed_root_pid" "daemon whose worktree was removed" 100 0.1; then - echo "expected Beam daemon $removed_root_pid to exit after its project root disappeared" >&2 +root_removed="true" +if ! wait_for_exit "$daemon4_pid" "daemon whose project root disappeared" 200 0.05; then + echo "expected root disappearance to stop the owned daemon" >&2 exit 1 fi -removed_root_pid="" - -removed_root_err="$(mktemp /tmp/beam-wrapper-removed-root-XXXXXX)" -if "$beam_script" --root "$tmp1" ensure lean > /dev/null 2>"$removed_root_err"; then - echo "expected a wrapper request for the removed worktree to fail" >&2 +if ! wait_for_exit "$hold_pid" "owner whose project root disappeared" 200 0.05; then + echo "expected root disappearance to release the foreground owner" >&2 exit 1 fi -if ! grep -Fq 'workspace root does not resolve' "$removed_root_err"; then - echo "expected a removed-worktree request to report that its workspace root no longer resolves" >&2 - cat "$removed_root_err" >&2 +set +e +wait "$hold_pid" +root_owner_status="$?" +set -e +hold_pid="" +if [ "$root_owner_status" -ne 0 ]; then + echo "expected root-disappearance owner to exit cleanly, got $root_owner_status" >&2 exit 1 fi -rm -f "$removed_root_err" -removed_root_err="" diff --git a/tests/test-beam-wrapper-diagnostics.sh b/tests/test-beam-wrapper-diagnostics.sh index 5c653ae4..6830f96e 100755 --- a/tests/test-beam-wrapper-diagnostics.sh +++ b/tests/test-beam-wrapper-diagnostics.sh @@ -19,6 +19,13 @@ warn_full_root="$(beam_wrapper_prepare_project_root diagnostics-warn-full)" stale_root="$(beam_wrapper_prepare_project_root diagnostics-stale)" renamed_stale_root="$(beam_wrapper_prepare_project_root diagnostics-renamed-stale)" +beam_wrapper_start_owner "$broken_root" +beam_wrapper_start_owner "$guard_msgs_io_stderr_root" +beam_wrapper_start_owner "$warn_root" +beam_wrapper_start_owner "$warn_full_root" +beam_wrapper_start_owner "$stale_root" +beam_wrapper_start_owner "$renamed_stale_root" + fail_json() { local message="$1" local json_payload="$2" diff --git a/tests/test-beam-wrapper-handle.sh b/tests/test-beam-wrapper-handle.sh index 2bfbefae..2d75c87f 100644 --- a/tests/test-beam-wrapper-handle.sh +++ b/tests/test-beam-wrapper-handle.sh @@ -13,6 +13,7 @@ cd "$(dirname "$0")/.." beam_wrapper_init handle_root="$(beam_wrapper_prepare_project_root handle)" +beam_wrapper_start_owner "$handle_root" ( cd "$handle_root" diff --git a/tests/test-beam-wrapper-probe.sh b/tests/test-beam-wrapper-probe.sh index 338e0e61..508bc10d 100644 --- a/tests/test-beam-wrapper-probe.sh +++ b/tests/test-beam-wrapper-probe.sh @@ -13,6 +13,7 @@ cd "$(dirname "$0")/.." beam_wrapper_init project_root="$(beam_wrapper_prepare_project_root probe)" +beam_wrapper_start_owner "$project_root" ( cd "$project_root" diff --git a/tests/test-beam-wrapper-rocq.sh b/tests/test-beam-wrapper-rocq.sh index aad48668..65a6d6de 100755 --- a/tests/test-beam-wrapper-rocq.sh +++ b/tests/test-beam-wrapper-rocq.sh @@ -9,6 +9,8 @@ set -euo pipefail cd "$(dirname "$0")/.." # shellcheck source=tests/lib/tmp-guards.sh . tests/lib/tmp-guards.sh +# shellcheck source=tests/lib/wait.sh +. tests/lib/wait.sh beam_script="$PWD/scripts/lean-beam" rocq_cmd="${BEAM_ROCQ_CMD:-}" @@ -69,10 +71,17 @@ rsync -a \ echo "expected doctor rocq to remain read-only and not build Beam daemon helpers" >&2 exit 1 fi + rocq_owner_err="$tmp_repo/rocq-owner.err" if [ -n "$rocq_cmd" ]; then - BEAM_ROCQ_CMD="$rocq_cmd" "$tmp_repo/scripts/lean-beam" --root "$tmp_repo/tests/rocq/Minimal" ensure rocq > /dev/null + BEAM_ROCQ_CMD="$rocq_cmd" "$tmp_repo/scripts/lean-beam" --root "$tmp_repo/tests/rocq/Minimal" ensure rocq --hold \ + > /dev/null 2>"$rocq_owner_err" & else - "$tmp_repo/scripts/lean-beam" --root "$tmp_repo/tests/rocq/Minimal" ensure rocq > /dev/null + "$tmp_repo/scripts/lean-beam" --root "$tmp_repo/tests/rocq/Minimal" ensure rocq --hold \ + > /dev/null 2>"$rocq_owner_err" & + fi + rocq_owner_pid="$!" + if ! wait_for_file_text "$rocq_owner_err" "owning Beam session" "Rocq session owner" 600 0.1; then + exit 1 fi if [ ! -x ".lake/build/bin/beam-daemon" ] || [ ! -x ".lake/build/bin/beam-client" ]; then echo "expected rocq CLI startup to build missing Beam daemon helpers on demand" >&2 @@ -83,4 +92,6 @@ rsync -a \ else "$tmp_repo/scripts/lean-beam" --root "$tmp_repo/tests/rocq/Minimal" shutdown > /dev/null fi + wait_for_exit "$rocq_owner_pid" "Rocq session owner" 120 0.1 + wait "$rocq_owner_pid" ) diff --git a/tests/test-beam-wrapper-runtime.sh b/tests/test-beam-wrapper-runtime.sh index 332bfef0..7c2c3156 100644 --- a/tests/test-beam-wrapper-runtime.sh +++ b/tests/test-beam-wrapper-runtime.sh @@ -139,6 +139,8 @@ expect_sigint_cancelled() { fi } +beam_wrapper_start_owner "$primary_root" +primary_owner_pid="$beam_wrapper_last_owner_pid" ( cd "$primary_root" "$beam_script" ensure lean > /dev/null @@ -153,10 +155,10 @@ if [ -z "$client1" ]; then client1="$client" fi +beam_wrapper_start_owner "$signal_root" +signal_owner_pid="$beam_wrapper_last_owner_pid" ( cd "$signal_root" - "$beam_script" --root "$signal_root" shutdown > /dev/null 2>&1 || true - "$beam_script" --root "$signal_root" ensure lean > /dev/null slow_version="$(beam_wrapper_update_version "signal SlowPoll" "$beam_script" --root "$signal_root" lean-update tests/scenario/docs/SlowPoll.lean)" command_version="$(beam_wrapper_update_version "signal CommandA" "$beam_script" --root "$signal_root" lean-update tests/scenario/docs/CommandA.lean)" @@ -363,11 +365,13 @@ PY "$beam_script" --root "$signal_root" shutdown > /dev/null 2>&1 || true ) +wait_for_exit "$signal_owner_pid" "first signal-test session owner" 120 0.1 +wait "$signal_owner_pid" +beam_wrapper_start_owner "$signal_root" +signal_owner_pid="$beam_wrapper_last_owner_pid" ( cd "$signal_root" - "$beam_script" --root "$signal_root" shutdown > /dev/null 2>&1 || true - "$beam_script" --root "$signal_root" ensure lean > /dev/null slow_version="$(beam_wrapper_update_version "duplicate SlowPoll" "$beam_script" --root "$signal_root" lean-update tests/scenario/docs/SlowPoll.lean)" command_version="$(beam_wrapper_update_version "duplicate CommandA" "$beam_script" --root "$signal_root" lean-update tests/scenario/docs/CommandA.lean)" @@ -454,7 +458,10 @@ PY "$beam_script" --root "$signal_root" shutdown > /dev/null 2>&1 || true ) +wait_for_exit "$signal_owner_pid" "second signal-test session owner" 120 0.1 +wait "$signal_owner_pid" +beam_wrapper_start_owner "$other_root" ( cd "$other_root" "$beam_script" ensure lean > /dev/null @@ -487,6 +494,7 @@ if ! grep -q "invalidParams" "$cross_err"; then exit 1 fi +beam_wrapper_start_owner "$busy_port_root" ( cd "$busy_port_root" "$beam_script" ensure lean > /dev/null @@ -559,6 +567,8 @@ fi cd "$primary_root" "$beam_script" shutdown > /dev/null ) +wait_for_exit "$primary_owner_pid" "primary session owner" 120 0.1 +wait "$primary_owner_pid" if [ -f "$primary_registry" ]; then echo "expected shutdown to remove the project Beam daemon registry" >&2 diff --git a/tests/test-beam-wrapper-sandbox.sh b/tests/test-beam-wrapper-sandbox.sh index c351521e..026b0a87 100755 --- a/tests/test-beam-wrapper-sandbox.sh +++ b/tests/test-beam-wrapper-sandbox.sh @@ -40,43 +40,26 @@ if ! bwrap --new-session --die-with-parent \ exit 0 fi -tmp_root="$(mktemp -d /tmp/beam-wrapper-sandbox-XXXXXX)" +tmp_root="$(mktemp -d /tmp/beam-wrapper-sandbox-owner-XXXXXX)" project_root="$tmp_root/project" control_root="$tmp_root/control" -hold_out="$tmp_root/hold.out" -hold_err="$tmp_root/hold.err" -owner_out="$tmp_root/owner.out" -owner_err="$tmp_root/owner.err" -owner_stop="$tmp_root/owner.stop" -follower_out="$tmp_root/follower.out" -follower_err="$tmp_root/follower.err" -follower_request_id="" -replacement_owner_a_pid="" -replacement_owner_b_pid="" +owner_pid="" +owner_stop="" +owner_kill="" +owner_resume="" cleanup() { - if [ -n "${hold_pid:-}" ]; then - kill "$hold_pid" > /dev/null 2>&1 || true - wait "$hold_pid" 2>/dev/null || true - fi - if [ -n "${owner_pid:-}" ]; then - kill "$owner_pid" > /dev/null 2>&1 || true - wait "$owner_pid" 2>/dev/null || true - fi - if [ -n "${replacement_owner_a_pid:-}" ]; then - kill "$replacement_owner_a_pid" > /dev/null 2>&1 || true - wait "$replacement_owner_a_pid" 2>/dev/null || true - fi - if [ -n "${replacement_owner_b_pid:-}" ]; then - kill "$replacement_owner_b_pid" > /dev/null 2>&1 || true - wait "$replacement_owner_b_pid" 2>/dev/null || true - fi - if [ -n "${follower_pid:-}" ]; then - if [ -n "$follower_request_id" ]; then - sandbox_beam cancel "$follower_request_id" > /dev/null 2>&1 || true + if [ -n "$owner_pid" ]; then + if [ -n "$owner_resume" ]; then + touch "$owner_resume" + fi + if [ -n "$owner_stop" ]; then + touch "$owner_stop" + fi + if ! wait_for_exit "$owner_pid" "sandbox owner cleanup" 100 0.05; then + kill "$owner_pid" >/dev/null 2>&1 || true fi - kill "$follower_pid" > /dev/null 2>&1 || true - wait "$follower_pid" 2>/dev/null || true + wait "$owner_pid" 2>/dev/null || true fi remove_owned_tmp_tree "$tmp_root" } @@ -84,8 +67,6 @@ trap cleanup EXIT mkdir -p "$project_root" "$control_root" rsync -a --exclude='.beam/' tests/save_olean_project/ "$project_root"/ -mkdir -p "$project_root/tests/scenario/docs" -cp tests/scenario/docs/SlowPoll.lean "$project_root/tests/scenario/docs/SlowPoll.lean" sandbox_beam() { bwrap --new-session --die-with-parent \ @@ -95,58 +76,54 @@ sandbox_beam() { --proc /proc \ --unshare-pid \ --chdir "$project_root" \ - -- /usr/bin/env BEAM_CONTROL_DIR="$control_root" "$beam_script" --root "$project_root" "$@" + -- /usr/bin/env BEAM_CONTROL_DIR="$control_root" \ + "$beam_script" --root "$project_root" "$@" +} + +wait_for_registry() { + local remaining=300 + while [ "$remaining" -gt 0 ]; do + # The control lock is intentionally short-lived and may disappear while `find` walks the + # per-root directory. Ignore that observational traversal race and keep probing for the file. + registry="$(find "$control_root" -name beam-daemon.json -print 2>/dev/null | sed -n '1p' || true)" + if [ -n "$registry" ] && [ -f "$registry" ]; then + return 0 + fi + sleep 0.2 + remaining=$((remaining - 1)) + done + return 1 } assert_no_connection_closed_incidents() { local label="$1" if find "$control_root" -path '*/daemon-failures/*connectionClosed*.json' -print -quit | grep -q .; then echo "expected $label to produce no connectionClosed incident" >&2 - find "$control_root" -path '*/daemon-failures/*.json' -print -exec cat {} \; >&2 + find "$control_root" -path '*/daemon-failures/*.json' -print -exec sed -n '1,160p' {} \; >&2 exit 1 fi } -sandbox_shell_hold() { - local hold_secs="$1" - bwrap --new-session --die-with-parent \ - --ro-bind / / \ - --dev-bind /dev /dev \ - --bind /tmp /tmp \ - --proc /proc \ - --unshare-pid \ - --chdir "$project_root" \ - -- /bin/bash -lc "export BEAM_CONTROL_DIR='$control_root'; '$beam_script' --root '$project_root' ensure lean >'$hold_out' 2>'$hold_err'; sleep $hold_secs" +assert_no_lease_artifacts() { + local artifact + artifact="$(find "$control_root" -type f \ + \( -name '*.lease' -o -name '*.revoked' -o -name '*retir*' \) -print -quit)" + if [ -n "$artifact" ]; then + echo "explicit ownership must not publish lease or retirement artifacts: $artifact" >&2 + exit 1 + fi } -sandbox_owner_hold() { - local output_path="$1" - local error_path="$2" - local stop_path="$3" - bwrap --new-session --die-with-parent \ - --ro-bind / / \ - --dev-bind /dev /dev \ - --bind /tmp /tmp \ - --proc /proc \ - --unshare-pid \ - --chdir "$project_root" \ - -- /bin/bash -lc \ - "export BEAM_CONTROL_DIR='$control_root'; \ - '$beam_script' --root '$project_root' ensure lean --hold >'$output_path' 2>'$error_path' & \ - wrapper_pid=\$!; \ - while [ ! -f '$stop_path' ]; do sleep 0.05; done; \ - kill -INT \"\$wrapper_pid\"; \ - wait \"\$wrapper_pid\"" -} +sandbox_owner() { + local name="$1" + local out="$tmp_root/$name.out" + local err="$tmp_root/$name.err" + local stop="$tmp_root/$name.stop" + local kill_marker="$tmp_root/$name.kill" + local pause="$tmp_root/$name.pause" + local paused="$tmp_root/$name.paused" + local resume="$tmp_root/$name.resume" -sandbox_paused_follower() { - local version="$1" - local request_id="$2" - local output_path="$3" - local error_path="$4" - local pause_path="$5" - local paused_path="$6" - local resume_path="$7" bwrap --new-session --die-with-parent \ --ro-bind / / \ --dev-bind /dev /dev \ @@ -155,431 +132,210 @@ sandbox_paused_follower() { --unshare-pid \ --chdir "$project_root" \ -- /bin/bash -lc \ - "export BEAM_CONTROL_DIR='$control_root' BEAM_PROGRESS=1 BEAM_REQUEST_ID='$request_id'; \ - '$beam_script' --root '$project_root' run-at tests/scenario/docs/SlowPoll.lean '$version' 25 2 poll_sleep_cmd >'$output_path' 2>'$error_path' & \ + "export BEAM_CONTROL_DIR='$control_root'; \ + '$beam_script' --root '$project_root' ensure --hold >'$out' 2>'$err' & \ wrapper_pid=\$!; \ - while [ ! -f '$pause_path' ]; do sleep 0.05; done; \ - kill -STOP \"\$wrapper_pid\"; \ - touch '$paused_path'; \ - while [ ! -f '$resume_path' ]; do sleep 0.05; done; \ - kill -CONT \"\$wrapper_pid\"; \ - wait \"\$wrapper_pid\"" + (paused=false; \ + while kill -0 \"\$wrapper_pid\" 2>/dev/null; do \ + if [ -f '$pause' ] && [ \"\$paused\" = false ]; then \ + kill -STOP \"\$wrapper_pid\"; paused=true; touch '$paused'; \ + fi; \ + if [ -f '$resume' ] && [ \"\$paused\" = true ]; then \ + kill -CONT \"\$wrapper_pid\"; paused=false; \ + fi; \ + if [ -f '$kill_marker' ]; then \ + [ \"\$paused\" = false ] || kill -CONT \"\$wrapper_pid\"; \ + kill -KILL \"\$wrapper_pid\"; exit 0; \ + fi; \ + if [ -f '$stop' ]; then \ + [ \"\$paused\" = false ] || kill -CONT \"\$wrapper_pid\"; \ + kill -INT \"\$wrapper_pid\"; exit 0; \ + fi; \ + sleep 0.05; \ + done) & \ + controller_pid=\$!; \ + set +e; wait \"\$wrapper_pid\" 2>/dev/null; status=\$?; set -e; \ + kill \"\$controller_pid\" 2>/dev/null || true; \ + wait \"\$controller_pid\" 2>/dev/null || true; \ + exit \"\$status\"" & + + owner_pid="$!" + owner_stop="$stop" + owner_kill="$kill_marker" + owner_pause="$pause" + owner_paused="$paused" + owner_resume="$resume" + owner_out="$out" + owner_err="$err" } -wait_for_registry() { - local remaining=300 - while [ "$remaining" -gt 0 ]; do - # The control lock is intentionally short-lived and may disappear while `find` walks the - # per-root directory. Ignore that observational traversal race and keep probing for the file. - registry="$(find "$control_root" -name beam-daemon.json -print 2>/dev/null | sed -n '1p' || true)" - if [ -n "$registry" ] && [ -f "$registry" ]; then - return 0 - fi - sleep 0.2 - remaining=$((remaining - 1)) - done - return 1 -} - -sandbox_shell_hold 10 & -hold_pid="$!" +missing_out="$tmp_root/missing.out" +missing_err="$tmp_root/missing.err" +if sandbox_beam ensure >"$missing_out" 2>"$missing_err"; then + echo "expected an ordinary sandbox command not to start an implicit daemon" >&2 + sed -n '1,160p' "$missing_out" >&2 + exit 1 +fi +if ! grep -Fq "start 'lean-beam ensure --hold'" "$missing_err"; then + echo "expected the missing-owner error to provide the ownership command" >&2 + sed -n '1,160p' "$missing_err" >&2 + exit 1 +fi -if ! wait_for_registry; then - echo "expected sandboxed wrapper ensure to create a control-dir registry" >&2 - cat "$hold_err" >&2 +sandbox_owner owner-1 +if ! wait_for_registry || ! wait_for_nonempty_file "$owner_out" "sandbox owner response"; then + echo "expected ensure --hold to publish an owned daemon" >&2 + sed -n '1,200p' "$owner_err" >&2 exit 1 fi daemon_id_1="$(read_json_field "$registry" daemonId)" port_1="$(read_json_field "$registry" port)" pid_domain_1="$(read_json_field "$registry" pidDomain 2>/dev/null || true)" - -if [ -z "$pid_domain_1" ]; then - echo "expected sandboxed wrapper registry to record the daemon PID domain for debugging" >&2 - cat "$registry" >&2 +owner_pid_domain_1="$(read_json_field "$registry" ownerPidDomain 2>/dev/null || true)" +if [ -z "$pid_domain_1" ] || [ -z "$owner_pid_domain_1" ]; then + echo "expected the registry to record daemon and owner PID domains" >&2 + sed -n '1,160p' "$registry" >&2 exit 1 fi doctor_out="$(sandbox_beam doctor)" if ! printf '%s\n' "$doctor_out" | grep -q 'daemon status: live'; then - echo "expected a PID-isolated wrapper invocation to reuse the live daemon via the registry endpoint" >&2 - printf '%s\n' "$doctor_out" >&2 - exit 1 -fi -if ! printf '%s\n' "$doctor_out" | grep -q 'daemon pid domain: '; then - echo "expected doctor output to surface the daemon pid domain for debugging" >&2 + echo "expected a separate PID namespace to observe the owned daemon endpoint" >&2 printf '%s\n' "$doctor_out" >&2 exit 1 fi -sandbox_beam ensure lean > /dev/null - -daemon_id_2="$(read_json_field "$registry" daemonId)" -port_2="$(read_json_field "$registry" port)" -pid_domain_2="$(read_json_field "$registry" pidDomain 2>/dev/null || true)" - -if [ "$daemon_id_1" != "$daemon_id_2" ]; then - echo "expected PID-isolated wrapper ensure to reuse the existing daemon instead of starting a new one" >&2 - printf 'before daemonId: %s\n' "$daemon_id_1" >&2 - printf 'after daemonId: %s\n' "$daemon_id_2" >&2 - exit 1 -fi - -if [ "$port_1" != "$port_2" ]; then - echo "expected PID-isolated wrapper ensure to preserve the daemon endpoint" >&2 - printf 'before port: %s\n' "$port_1" >&2 - printf 'after port: %s\n' "$port_2" >&2 - exit 1 -fi - -if [ "$pid_domain_1" != "$pid_domain_2" ]; then - echo "expected PID-isolated wrapper ensure to preserve the recorded daemon PID domain" >&2 - printf 'before PID domain: %s\n' "$pid_domain_1" >&2 - printf 'after PID domain: %s\n' "$pid_domain_2" >&2 - exit 1 -fi - -kill "$hold_pid" > /dev/null 2>&1 || true -wait "$hold_pid" 2>/dev/null || true -hold_pid="" - -find "$control_root" -mindepth 1 -maxdepth 1 -exec rm -rf -- {} + - -sandbox_owner_hold "$owner_out" "$owner_err" "$owner_stop" & -owner_pid="$!" -if ! wait_for_registry; then - echo "expected owner sandbox wrapper request to create a control-dir registry" >&2 - cat "$owner_err" >&2 - exit 1 -fi -follower_version="$(beam_wrapper_update_version "sandbox SlowPoll" sandbox_beam update tests/scenario/docs/SlowPoll.lean)" -follower_request_id="wrapper-sandbox-follower" -BEAM_PROGRESS=1 BEAM_REQUEST_ID="$follower_request_id" \ - sandbox_beam run-at tests/scenario/docs/SlowPoll.lean "$follower_version" 25 2 poll_sleep_cmd \ - >"$follower_out" 2>"$follower_err" & -follower_pid="$!" - -if ! wait_for_file_text "$follower_err" "snapshot progress" "follower sandbox wrapper request progress"; then - cat "$owner_out" >&2 - cat "$owner_err" >&2 - cat "$follower_out" >&2 - cat "$follower_err" >&2 - exit 1 -fi -if ! wait_for_nonempty_file "$owner_out" "owner sandbox ensure response"; then - cat "$owner_err" >&2 - cat "$follower_out" >&2 - cat "$follower_err" >&2 +ensure_json="$(sandbox_beam ensure)" +if [ "$(json_text_field "$ensure_json" ok)" != "true" ]; then + echo "expected a separate PID namespace to attach to the owner session" >&2 + printf '%s\n' "$ensure_json" >&2 exit 1 fi - -touch "$owner_stop" - -# The original owner-lifetime implementation stopped tracking siblings after a fixed -# 30-second polling window. Keep the follower active beyond that boundary so this test -# proves ownership follows the request lifetime rather than an elapsed-duration guess. -sleep 31 - -if ! kill -0 "$owner_pid" 2>/dev/null; then - echo "expected the owner sandbox wrapper to outlive a follower active for more than 30 seconds" >&2 - cat "$owner_out" >&2 - cat "$owner_err" >&2 - cat "$follower_out" >&2 - cat "$follower_err" >&2 +if [ "$(read_json_field "$registry" daemonId)" != "$daemon_id_1" ] || \ + [ "$(read_json_field "$registry" port)" != "$port_1" ]; then + echo "ordinary commands must preserve the owner's daemon generation and endpoint" >&2 exit 1 fi -if ! kill -0 "$follower_pid" 2>/dev/null; then - echo "expected the follower sandbox request to stay alive while the owner request finishes" >&2 - cat "$owner_out" >&2 - cat "$owner_err" >&2 - cat "$follower_out" >&2 - cat "$follower_err" >&2 +duplicate_out="$tmp_root/duplicate.out" +duplicate_err="$tmp_root/duplicate.err" +if sandbox_beam ensure --hold >"$duplicate_out" 2>"$duplicate_err"; then + echo "expected a second sandbox owner to be rejected" >&2 exit 1 fi -cancel_json="$(sandbox_beam cancel wrapper-sandbox-follower)" -if ! printf '%s\n' "$cancel_json" | python3 -c 'import json,sys; payload=json.load(sys.stdin); raise SystemExit(0 if payload.get("result", {}).get("cancelled") is True else 1)'; then - echo "expected sandbox wrapper cancel request to acknowledge the follower request id" >&2 - printf '%s\n' "$cancel_json" >&2 +if ! grep -Fq 'already owned' "$duplicate_err"; then + echo "expected duplicate-owner diagnostics to identify the live owner" >&2 + sed -n '1,160p' "$duplicate_err" >&2 exit 1 fi -set +e -wait "$follower_pid" -follower_status=$? -set -e -follower_pid="" -follower_request_id="" -if [ "$follower_status" = "0" ]; then - echo "expected follower sandbox wrapper request to exit non-zero after cancellation" >&2 - cat "$follower_out" >&2 - cat "$follower_err" >&2 +# Ownership is the lifetime of the holder's inherited pipe, not a periodically renewed lease. +# Pausing the holder therefore leaves the session usable by clients in other PID namespaces. +touch "$owner_pause" +if ! wait_for_file "$owner_paused" "paused sandbox owner" 12; then + sed -n '1,160p' "$owner_err" >&2 exit 1 fi - -follower_json="$(cat "$follower_out")" -if ! python3 -c 'import json,sys; payload=json.load(sys.stdin); raise SystemExit(0 if payload.get("error", {}).get("code") == "requestCancelled" else 1)' <<<"$follower_json" -then - echo "expected follower sandbox wrapper request to report requestCancelled after cancellation" >&2 - printf '%s\n' "$follower_json" >&2 - cat "$follower_err" >&2 +sleep 3 +paused_stats="$(sandbox_beam stats)" +if [ "$(json_text_field "$paused_stats" ok)" != "true" ]; then + echo "expected the daemon to remain usable while its explicit owner is paused" >&2 + printf '%s\n' "$paused_stats" >&2 exit 1 fi +assert_no_lease_artifacts +touch "$owner_resume" -wait "$owner_pid" -owner_pid="" - -assert_no_connection_closed_incidents "the long-lived follower regression" - -# Heartbeat expiry is a revocation decision, not permission to kill broker work that was already -# admitted. Suspend the entire follower wrapper beyond the timeout, let the owner revoke its -# filesystem lease, then resume it. The daemon-side active-request fence must keep the owner alive; -# the resumed wrapper must cancel cleanly instead of reconnecting through the removed lease. -find "$control_root" -mindepth 1 -maxdepth 1 -exec rm -rf -- {} + -owner_out="$tmp_root/paused-owner.out" -owner_err="$tmp_root/paused-owner.err" -owner_stop="$tmp_root/paused-owner.stop" -follower_out="$tmp_root/paused-follower.out" -follower_err="$tmp_root/paused-follower.err" -pause_follower="$tmp_root/pause-follower" -follower_paused="$tmp_root/follower-paused" -resume_follower="$tmp_root/resume-follower" - -sandbox_owner_hold "$owner_out" "$owner_err" "$owner_stop" & -owner_pid="$!" -if ! wait_for_registry; then - echo "expected paused-follower owner to create a control-dir registry" >&2 - cat "$owner_err" >&2 +shutdown_json="$(sandbox_beam shutdown)" +if [ "$(json_text_field "$shutdown_json" result.shutdown)" != "true" ]; then + echo "expected explicit shutdown to close the owned sandbox session" >&2 + printf '%s\n' "$shutdown_json" >&2 exit 1 fi -paused_version="$(beam_wrapper_update_version "paused sandbox SlowPoll" sandbox_beam update tests/scenario/docs/SlowPoll.lean)" -follower_request_id="wrapper-sandbox-paused-follower" -sandbox_paused_follower "$paused_version" "$follower_request_id" \ - "$follower_out" "$follower_err" "$pause_follower" "$follower_paused" "$resume_follower" & -follower_pid="$!" - -if ! wait_for_file_text "$follower_err" "snapshot progress" "paused follower request progress"; then - cat "$owner_out" >&2 - cat "$owner_err" >&2 - cat "$follower_out" >&2 - cat "$follower_err" >&2 +if ! wait_for_exit "$owner_pid" "sandbox owner after shutdown" 120 0.1; then + sed -n '1,200p' "$owner_err" >&2 exit 1 fi -touch "$pause_follower" -if ! wait_for_file "$follower_paused" "paused follower acknowledgement" 12; then - cat "$follower_err" >&2 +if ! wait "$owner_pid"; then + echo "expected explicit shutdown to release the sandbox owner cleanly" >&2 + sed -n '1,200p' "$owner_err" >&2 exit 1 fi -touch "$owner_stop" - -revoked_lease="" -for _ in $(seq 1 120); do - revoked_lease="$(find "$control_root" -name '*.revoked' -print -quit)" - if [ -n "$revoked_lease" ]; then - break - fi - sleep 0.1 -done -if [ -z "$revoked_lease" ]; then - echo "expected the suspended follower lease to receive a persistent revocation tombstone" >&2 - find "$control_root" -name '*.lease' -print -exec cat {} \; >&2 - exit 1 -fi -if ! kill -0 "$owner_pid" 2>/dev/null; then - echo "expected the daemon owner to stay alive for a revoked but still-admitted request" >&2 - cat "$owner_out" >&2 - cat "$owner_err" >&2 - cat "$follower_out" >&2 - cat "$follower_err" >&2 +owner_pid="" +if find "$control_root" -name beam-daemon.json -print -quit | grep -q .; then + echo "expected explicit shutdown to remove the owned generation registry" >&2 exit 1 fi -touch "$resume_follower" -set +e -wait "$follower_pid" -paused_follower_status="$?" -set -e -follower_pid="" -follower_request_id="" -if [ "$paused_follower_status" -eq 0 ]; then - echo "expected the resumed wrapper to fail after observing lease revocation" >&2 - cat "$follower_out" >&2 - cat "$follower_err" >&2 +sandbox_owner owner-2 +if ! wait_for_registry || ! wait_for_nonempty_file "$owner_out" "replacement sandbox owner response"; then + sed -n '1,200p' "$owner_err" >&2 exit 1 fi -if ! python3 -c 'import json,sys; payload=json.load(sys.stdin); raise SystemExit(0 if payload.get("error", {}).get("code") == "requestCancelled" else 1)' < "$follower_out" -then - echo "expected the resumed wrapper request to report requestCancelled" >&2 - cat "$follower_out" >&2 - cat "$follower_err" >&2 - exit 1 -fi -if ! wait_for_exit "$owner_pid" "owner after suspended follower cancellation" 120 0.1; then - cat "$owner_out" >&2 - cat "$owner_err" >&2 - cat "$follower_out" >&2 - cat "$follower_err" >&2 +daemon_id_2="$(read_json_field "$registry" daemonId)" +if [ "$daemon_id_1" = "$daemon_id_2" ]; then + echo "expected a replacement owner to publish a new daemon generation" >&2 exit 1 fi -wait "$owner_pid" -owner_pid="" -assert_no_connection_closed_incidents "suspended-follower revocation" - -# A follower killed from outside its PID namespace cannot remove its own lease. Its heartbeat -# must expire so the owner can drain, retire the generation, and permit a later clean ensure. -find "$control_root" -mindepth 1 -maxdepth 1 -exec rm -rf -- {} + -owner_out="$tmp_root/killed-owner.out" -owner_err="$tmp_root/killed-owner.err" -owner_stop="$tmp_root/killed-owner.stop" -follower_out="$tmp_root/killed-follower.out" -follower_err="$tmp_root/killed-follower.err" - -sandbox_owner_hold "$owner_out" "$owner_err" "$owner_stop" & -owner_pid="$!" -if ! wait_for_registry; then - echo "expected killed-follower owner to create a control-dir registry" >&2 - cat "$owner_err" >&2 - exit 1 -fi -killed_version="$(beam_wrapper_update_version "killed sandbox SlowPoll" sandbox_beam update tests/scenario/docs/SlowPoll.lean)" -follower_request_id="wrapper-sandbox-killed-follower" -BEAM_PROGRESS=1 BEAM_REQUEST_ID="$follower_request_id" \ - sandbox_beam run-at tests/scenario/docs/SlowPoll.lean "$killed_version" 25 2 poll_sleep_cmd \ - >"$follower_out" 2>"$follower_err" & -follower_pid="$!" - -if ! wait_for_file_text "$follower_err" "snapshot progress" "killed follower request progress"; then - cat "$owner_out" >&2 - cat "$owner_err" >&2 - cat "$follower_out" >&2 - cat "$follower_err" >&2 +# Killing the holder closes the only write end of the inherited owner pipe. The daemon must stop +# without a heartbeat timeout, and the next ordinary command must clean the stale registry. +touch "$owner_kill" +if ! wait_for_exit "$owner_pid" "killed sandbox owner" 120 0.1; then + sed -n '1,200p' "$owner_err" >&2 exit 1 fi -touch "$owner_stop" -kill -KILL "$follower_pid" > /dev/null 2>&1 || true set +e -wait "$follower_pid" 2>/dev/null -set -e -follower_pid="" -follower_request_id="" - -if ! wait_for_exit "$owner_pid" "owner waiting on killed cross-namespace follower" 120 0.1; then - cat "$owner_out" >&2 - cat "$owner_err" >&2 - find "$control_root" -name '*.lease' -print -exec cat {} \; >&2 - exit 1 -fi wait "$owner_pid" +owner_status="$?" +set -e owner_pid="" - -recovery_json="$(sandbox_beam ensure lean)" -if [ "$(json_text_field "$recovery_json" ok)" != "true" ]; then - echo "expected ensure to recover after pruning a killed follower lease" >&2 - printf '%s\n' "$recovery_json" >&2 +if [ "$owner_status" -eq 0 ]; then + echo "expected the deliberately killed owner wrapper to exit non-zero" >&2 exit 1 fi +sleep 4 -# Replacing a daemon while its original starter is still active creates two starter leases. The -# obsolete starter must notice the registry generation change before waiting on the replacement -# owner's lease, or both owners keep each other's heartbeats alive forever. -find "$control_root" -mindepth 1 -maxdepth 1 -exec rm -rf -- {} + -replacement_a_out="$tmp_root/replacement-a.out" -replacement_a_err="$tmp_root/replacement-a.err" -replacement_a_stop="$tmp_root/replacement-a.stop" -replacement_b_out="$tmp_root/replacement-b.out" -replacement_b_err="$tmp_root/replacement-b.err" -replacement_b_stop="$tmp_root/replacement-b.stop" - -sandbox_owner_hold "$replacement_a_out" "$replacement_a_err" "$replacement_a_stop" & -replacement_owner_a_pid="$!" -if ! wait_for_registry || ! wait_for_nonempty_file "$replacement_a_out" "original replacement owner response"; then - cat "$replacement_a_out" >&2 - cat "$replacement_a_err" >&2 +after_kill_out="$tmp_root/after-kill.out" +after_kill_err="$tmp_root/after-kill.err" +if sandbox_beam ensure >"$after_kill_out" 2>"$after_kill_err"; then + echo "expected an ordinary command not to replace a dead owner implicitly" >&2 + sed -n '1,160p' "$after_kill_out" >&2 exit 1 fi -replacement_daemon_a="$(read_json_field "$registry" daemonId)" - -replacement_shutdown_json="$(sandbox_beam shutdown)" -if ! python3 -c 'import json,sys; payload=json.load(sys.stdin); raise SystemExit(0 if payload.get("result", {}).get("shutdown") is True else 1)' <<<"$replacement_shutdown_json" -then - echo "expected replacement setup to shut down the original daemon" >&2 - printf '%s\n' "$replacement_shutdown_json" >&2 +if ! grep -Fq "start 'lean-beam ensure --hold'" "$after_kill_err"; then + echo "expected dead-owner recovery to require a new explicit owner" >&2 + sed -n '1,160p' "$after_kill_err" >&2 exit 1 fi - -sandbox_owner_hold "$replacement_b_out" "$replacement_b_err" "$replacement_b_stop" & -replacement_owner_b_pid="$!" -if ! wait_for_registry || ! wait_for_nonempty_file "$replacement_b_out" "replacement owner response"; then - cat "$replacement_a_out" >&2 - cat "$replacement_a_err" >&2 - cat "$replacement_b_out" >&2 - cat "$replacement_b_err" >&2 - exit 1 -fi -replacement_daemon_b="$(read_json_field "$registry" daemonId)" -if [ "$replacement_daemon_a" = "$replacement_daemon_b" ]; then - echo "expected successive PID-isolated daemon starts to receive distinct generation ids" >&2 - printf 'original daemonId: %s\nreplacement daemonId: %s\n' \ - "$replacement_daemon_a" "$replacement_daemon_b" >&2 +if find "$control_root" -name beam-daemon.json -print -quit | grep -q .; then + echo "expected dead-owner recovery to remove the stale registry" >&2 exit 1 fi -touch "$replacement_a_stop" -if ! wait_for_exit "$replacement_owner_a_pid" "obsolete daemon generation owner" 100 0.1; then - cat "$replacement_a_out" >&2 - cat "$replacement_a_err" >&2 - cat "$replacement_b_out" >&2 - cat "$replacement_b_err" >&2 - find "$control_root" -name '*.lease' -print -exec cat {} \; >&2 +sandbox_owner owner-3 +if ! wait_for_registry || ! wait_for_nonempty_file "$owner_out" "final sandbox owner response"; then + sed -n '1,200p' "$owner_err" >&2 exit 1 fi -wait "$replacement_owner_a_pid" -replacement_owner_a_pid="" - -if ! kill -0 "$replacement_owner_b_pid" 2>/dev/null; then - echo "expected the replacement daemon owner to remain active after the obsolete owner exited" >&2 - cat "$replacement_b_out" >&2 - cat "$replacement_b_err" >&2 +final_stats="$(sandbox_beam stats)" +if [ "$(json_text_field "$final_stats" ok)" != "true" ]; then + echo "expected a new explicit owner to restore the session" >&2 + printf '%s\n' "$final_stats" >&2 exit 1 fi -touch "$replacement_b_stop" -if ! wait_for_exit "$replacement_owner_b_pid" "replacement daemon generation owner" 120 0.1; then - cat "$replacement_b_out" >&2 - cat "$replacement_b_err" >&2 - find "$control_root" -name '*.lease' -print -exec cat {} \; >&2 +touch "$owner_stop" +if ! wait_for_exit "$owner_pid" "interrupted sandbox owner" 120 0.1; then + sed -n '1,200p' "$owner_err" >&2 exit 1 fi -wait "$replacement_owner_b_pid" -replacement_owner_b_pid="" - -assert_no_connection_closed_incidents "replacement generation ownership" - -# Supplement the deterministic lifetime cases with a cold-start fanout. Every wrapper should -# complete successfully through serialized admission without connection loss. -find "$control_root" -mindepth 1 -maxdepth 1 -exec rm -rf -- {} + -fanout_count=8 -fanout_pids=() -for i in $(seq 1 "$fanout_count"); do - sandbox_beam ensure lean >"$tmp_root/fanout-$i.out" 2>"$tmp_root/fanout-$i.err" & - fanout_pids+=("$!") -done -fanout_failed=false -for pid in ${fanout_pids[@]+"${fanout_pids[@]}"}; do - if ! wait "$pid"; then - fanout_failed=true - fi -done -for i in $(seq 1 "$fanout_count"); do - if [ "$(json_file_text_field "$tmp_root/fanout-$i.out" ok)" != "true" ]; then - echo "expected cold-start fanout wrapper $i to succeed" >&2 - cat "$tmp_root/fanout-$i.out" >&2 - cat "$tmp_root/fanout-$i.err" >&2 - fanout_failed=true - fi -done -if [ "$fanout_failed" = "true" ]; then +if ! wait "$owner_pid"; then + echo "expected SIGINT to release the final sandbox owner cleanly" >&2 + sed -n '1,200p' "$owner_err" >&2 exit 1 fi +owner_pid="" -assert_no_connection_closed_incidents "the sandbox lifecycle regressions" +assert_no_lease_artifacts +assert_no_connection_closed_incidents "the explicit sandbox ownership regressions" diff --git a/tests/test-beam-wrapper-sync-save.sh b/tests/test-beam-wrapper-sync-save.sh index 547894d5..98cef6f6 100755 --- a/tests/test-beam-wrapper-sync-save.sh +++ b/tests/test-beam-wrapper-sync-save.sh @@ -14,6 +14,8 @@ beam_wrapper_init lifecycle_root="$(beam_wrapper_prepare_project_root sync-save)" standalone_root="$(beam_wrapper_prepare_project_root standalone-save)" +beam_wrapper_start_owner "$lifecycle_root" +beam_wrapper_start_owner "$standalone_root" ( cd "$lifecycle_root" diff --git a/tests/test-stage0-toolchain.sh b/tests/test-stage0-toolchain.sh index 020d3699..b7f7a5e6 100755 --- a/tests/test-stage0-toolchain.sh +++ b/tests/test-stage0-toolchain.sh @@ -11,6 +11,8 @@ cd "$(dirname "$0")/.." . tests/lib/assertions.sh # shellcheck source=tests/lib/tmp-guards.sh . tests/lib/tmp-guards.sh +# shellcheck source=tests/lib/wait.sh +. tests/lib/wait.sh toolchain="${BEAM_STAGE0_TOOLCHAIN:-lean4-stage0}" host_home="$HOME" @@ -63,7 +65,15 @@ assert_output_contains "stage0 custom toolchain doctor output" "$doctor_out" 'pr assert_output_contains "stage0 custom toolchain doctor output" "$doctor_out" 'bundle source: installed' assert_output_contains "stage0 custom toolchain doctor output" "$doctor_out" 'bundle toolchain fingerprint: ' +stage0_owner_err="$tmp_root/stage0-owner.err" ELAN_HOME="$host_elan_home" \ - "$install_home/.local/bin/lean-beam" --root "$project_root" ensure >/dev/null + "$install_home/.local/bin/lean-beam" --root "$project_root" ensure --hold \ + >/dev/null 2>"$stage0_owner_err" & +stage0_owner_pid="$!" +if ! wait_for_file_text "$stage0_owner_err" "owning Beam session" "stage0 session owner" 600 0.1; then + exit 1 +fi ELAN_HOME="$host_elan_home" \ "$install_home/.local/bin/lean-beam" --root "$project_root" shutdown >/dev/null +wait_for_exit "$stage0_owner_pid" "stage0 session owner" 120 0.1 +wait "$stage0_owner_pid" From b1e008af97d6c8113000210d20ae36a257a489c0 Mon Sep 17 00:00:00 2001 From: Emilio Jesus Gallego Arias Date: Wed, 26 Aug 2026 00:24:17 +0200 Subject: [PATCH 03/28] fix: harden explicit daemon ownership --- Beam/Broker/Pending.lean | 14 ++- Beam/Broker/Server.lean | 2 +- Beam/Cli/Args.lean | 12 +-- Beam/Cli/Commands.lean | 69 +++++++------ Beam/Cli/DaemonManager.lean | 54 ++++++---- Beam/Cli/Usage.lean | 45 ++++---- CHANGELOG.md | 5 +- docs/DEVELOPMENT.md | 3 - docs/STATUS.md | 8 +- docs/TESTING.md | 3 +- scripts/lean-beam | 45 ++++---- scripts/lean-beam-search | 2 +- skills/lean-beam/SKILL.md | 2 + tests/lean/BeamTest/Broker/PendingTest.lean | 27 +++-- tests/lib/beam-wrapper-common.sh | 12 +++ tests/test-beam-wrapper-daemon.sh | 108 ++++++++++++++++++++ tests/test-beam-wrapper-runtime.sh | 76 ++------------ 17 files changed, 300 insertions(+), 187 deletions(-) diff --git a/Beam/Broker/Pending.lean b/Beam/Broker/Pending.lean index 3b86cc7b..03f46b77 100644 --- a/Beam/Broker/Pending.lean +++ b/Beam/Broker/Pending.lean @@ -269,10 +269,22 @@ end PendingRequest namespace PendingRequestStore -def failAll (store : PendingRequestStore) (failure : ResponseFailure) : IO Unit := do +/-- Fail every pending request, giving an already-marked cancellation token precedence. -/ +def failAllRespectingCancellation + (store : PendingRequestStore) + (fallback : ResponseFailure) : IO Unit := do let pending ← clear store for req in pending do let progress? ← req.progressRef.get + let cancelled ← + match req.cancelRef? with + | some cancelRef => cancelRef.get + | none => pure false + let failure := + if cancelled then + responseFailureFor .requestCancelled "request was cancelled while its backend session closed" + else + fallback let failure := failure.withOptionalFileProgress progress? try req.promise.resolve (.error failure) diff --git a/Beam/Broker/Server.lean b/Beam/Broker/Server.lean index ec6fa274..742eef6e 100644 --- a/Beam/Broker/Server.lean +++ b/Beam/Broker/Server.lean @@ -459,7 +459,7 @@ partial def sessionReaderLoop (session : Session) : IO Unit := do pure () sessionReaderLoop session catch e => - PendingRequestStore.failAll session.pending <| BrokerFailure.toResponseFailure { + PendingRequestStore.failAllRespectingCancellation session.pending <| BrokerFailure.toResponseFailure { code := .workerExited message := e.toString } diff --git a/Beam/Cli/Args.lean b/Beam/Cli/Args.lean index 41db72f7..1c17b781 100644 --- a/Beam/Cli/Args.lean +++ b/Beam/Cli/Args.lean @@ -38,7 +38,7 @@ def hasSubstring (text needle : String) : Bool := | _ => true def textArgUsage (cmdHead : String) : String := - s!"usage: beam [--root PATH] [--port N] {cmdHead} [--stdin | --text-file | -- | ]" + s!"usage: beam [--root PATH] {cmdHead} [--stdin | --text-file | -- | ]" def textArgReadsStdin (args : List String) : Bool := match args with @@ -67,7 +67,7 @@ def parseJsonText (label text : String) : IO Json := do | .error err => throw <| IO.userError s!"invalid {label}: {err}" def handleArgUsage (cmdHead : String) : String := - s!"usage: beam [--root PATH] [--port N] {cmdHead} >" + s!"usage: beam [--root PATH] {cmdHead} >" def handleArgReadsStdin (args : List String) : Bool := match args with @@ -114,7 +114,7 @@ def parseHandleInput (cmdHead : String) (args : List String) : IO (Handle × Lis private def parseLeanDiagnosticScopeArgs (command : String) (args : List String) : IO Beam.Broker.DiagnosticScope := - let usage := s!"usage: beam [--root PATH] [--port N] {command} [+all-diagnostics]" + let usage := s!"usage: beam [--root PATH] {command} [+all-diagnostics]" match args with | [] => pure Beam.Broker.DiagnosticScope.errors | ["+all-diagnostics"] => pure Beam.Broker.DiagnosticScope.all @@ -133,7 +133,7 @@ def parseLeanCloseSaveArgs (args : List String) : IO Beam.Broker.DiagnosticScope parseLeanDiagnosticScopeArgs "lean-close-save" args def leanReferencesUsage : String := - "usage: beam [--root PATH] [--port N] lean-references [--include-declaration|--exclude-declaration]" + "usage: beam [--root PATH] lean-references [--include-declaration|--exclude-declaration]" def parseLeanReferencesArgs (args : List String) : IO Bool := do match args with @@ -143,7 +143,7 @@ def parseLeanReferencesArgs (args : List String) : IO Bool := do | _ => throw <| IO.userError leanReferencesUsage def leanGoalsUsage : String := - "usage: beam [--root PATH] [--port N] lean-goals before|after " + "usage: beam [--root PATH] lean-goals before|after " def parseLeanGoalsModeArg (mode : String) : IO GoalMode := do match mode with @@ -166,7 +166,7 @@ private def parseTodoSuggestArg (value : String) : IO Beam.LSP.Todo.TodoSuggestM throw <| IO.userError s!"invalid todo suggest mode '{value}' (expected one of: {allowed}): {err}" def leanTodoUsage : String := - "usage: beam [--root PATH] [--port N] lean-todo [--kind ...] [--suggest none|basic]" + "usage: beam [--root PATH] lean-todo [--kind ...] [--suggest none|basic]" def parseLeanTodoArgs (args : List String) : IO (Option (Array Beam.LSP.Todo.TodoKind) × Option Beam.LSP.Todo.TodoSuggestMode) := do diff --git a/Beam/Cli/Commands.lean b/Beam/Cli/Commands.lean index 0cff0e4d..6399a17f 100644 --- a/Beam/Cli/Commands.lean +++ b/Beam/Cli/Commands.lean @@ -55,7 +55,7 @@ private def runLeanRunAt let line ← parseNatArg "line" lineText let character ← parseNatArg "character" characterText let parsedText ← parseTextArg s!"{action} " textArgs - withProjectDaemon home root .lean opts fun client => do + withProjectDaemon home root .lean fun client => do let req ← withEnvClientRequestId <| leanRunAtRequest root path version line character parsedText.text? (storeHandle := storeHandle) maybeEmitTextDebug req.clientRequestId? action parsedText.source parsedText.text? @@ -83,7 +83,7 @@ private def runLeanRunWith let req ← withEnvClientRequestId <| leanRunWithRequest root path handle parsedText.text? (linear := linear) maybeEmitTextDebug req.clientRequestId? action parsedText.source parsedText.text? - withProjectDaemon home root .lean opts fun client => + withProjectDaemon home root .lean fun client => callBrokerWithProgress root client req (leanRunWithWaitSpec path (linear := linear)) private def runLeanRelease @@ -96,7 +96,7 @@ private def runLeanRelease let (handle, extra) ← parseHandleInput s!"{action} " args unless extra.isEmpty do throw <| IO.userError (handleArgUsage s!"{action} ") - withProjectDaemon home root .lean opts fun client => + withProjectDaemon home root .lean fun client => callBroker root client <| leanReleaseRequest root path handle private def shutdownProjectDaemon (opts : CliOptions) : IO Unit := do @@ -107,10 +107,9 @@ private def shutdownProjectDaemon (opts : CliOptions) : IO Unit := do if let some endpoint := Beam.Daemon.registryEndpoint? entry then let resp ← sendRequest endpoint { op := .shutdown } printResponse resp - -- Revoking the published generation tells its wrapper owner to close the inherited - -- owner pipe. That unblocks the daemon's stdin watcher during an explicit shutdown. + -- Unpublishing this exact generation releases its wrapper owner. The owner remains the + -- sole process responsible for closing the inherited pipe and reaping its daemon child. removeRegistry root - finishRegistryDaemonShutdown entry else stopRegisteredDaemon root printJsonLine <| Json.mkObj [ @@ -122,8 +121,19 @@ private def shutdownProjectDaemon (opts : CliOptions) : IO Unit := do ("result", Json.mkObj [("shutdown", toJson false), ("reason", toJson ("notFound" : String))]) ] -private def backendOfName (name : String) : Backend := - if name == "rocq" then .rocq else .lean +private def parseBackendName (name : String) : IO Backend := do + match fromJson? (Json.str name) with + | .ok backend => pure backend + | .error err => throw <| IO.userError err + +private def commandMaySelectPort : List String → Bool + | ["ensure", "--hold"] => true + | ["ensure", _, "--hold"] => true + | _ => false + +private def validateRequestedPortScope (opts : CliOptions) : IO Unit := do + if opts.requestedPort?.isSome && !commandMaySelectPort opts.args then + throw <| IO.userError "--port is only valid when starting an owner with 'ensure [lean|rocq] --hold'" private def runThenHoldUntilInterrupted (owner : ProjectDaemonOwner) @@ -164,10 +174,11 @@ private def ensureBackend (← IO.getStdout).flush IO.eprintln "beam: owning Beam session; interrupt this wrapper process when finished" else - withProjectDaemon home root backend opts fun client => + withProjectDaemon home root backend fun client => callBroker root client { op := .ensure, backend := backend, root? := some root.toString } def runCommand (home : System.FilePath) (opts : CliOptions) : IO Unit := do + validateRequestedPortScope opts match opts.args with | [] => throw <| IO.userError usage @@ -203,9 +214,9 @@ def runCommand (home : System.FilePath) (opts : CliOptions) : IO Unit := do | "ensure" :: "--hold" :: [] => ensureBackend home opts .lean (hold := true) | "ensure" :: backend :: [] => - ensureBackend home opts (backendOfName backend) + ensureBackend home opts (← parseBackendName backend) | "ensure" :: backend :: "--hold" :: [] => - ensureBackend home opts (backendOfName backend) (hold := true) + ensureBackend home opts (← parseBackendName backend) (hold := true) | "lean-run-at" :: path :: version :: line :: character :: text => runLeanRunAt home opts (← wrapperDisplayAction "lean-run-at") path version line character text | "lean-run-at-handle" :: path :: version :: line :: character :: text => @@ -217,7 +228,7 @@ def runCommand (home : System.FilePath) (opts : CliOptions) : IO Unit := do let line ← parseNatArg "line" line let character ← parseNatArg "character" character let action ← wrapperDisplayAction "lean-hover" - withProjectDaemon home root .lean opts fun client => + withProjectDaemon home root .lean fun client => callBrokerWithProgress root client (leanHoverRequest root path version line character) (leanHoverWaitSpec path line character action) @@ -227,7 +238,7 @@ def runCommand (home : System.FilePath) (opts : CliOptions) : IO Unit := do let line ← parseNatArg "line" line let character ← parseNatArg "character" character let action ← wrapperDisplayAction "lean-signature-help" - withProjectDaemon home root .lean opts fun client => + withProjectDaemon home root .lean fun client => callBrokerWithProgress root client (leanSignatureHelpRequest root path version line character) (leanSignatureHelpWaitSpec path line character action) @@ -237,7 +248,7 @@ def runCommand (home : System.FilePath) (opts : CliOptions) : IO Unit := do let line ← parseNatArg "line" line let character ← parseNatArg "character" character let action ← wrapperDisplayAction "lean-definition" - withProjectDaemon home root .lean opts fun client => + withProjectDaemon home root .lean fun client => callBrokerWithProgress root client (leanDefinitionRequest root path version line character) (leanDefinitionWaitSpec path line character action) @@ -248,7 +259,7 @@ def runCommand (home : System.FilePath) (opts : CliOptions) : IO Unit := do let character ← parseNatArg "character" character let includeDeclaration ← parseLeanReferencesArgs extra let action ← wrapperDisplayAction "lean-references" - withProjectDaemon home root .lean opts fun client => + withProjectDaemon home root .lean fun client => callBrokerWithProgress root client (leanReferencesRequest root path version line character includeDeclaration) (leanReferencesWaitSpec path line character action) @@ -256,7 +267,7 @@ def runCommand (home : System.FilePath) (opts : CliOptions) : IO Unit := do let root ← projectRoot opts .lean let version ← parseNatArg "version" versionText let action ← wrapperDisplayAction "lean-document-symbols" - withProjectDaemon home root .lean opts fun client => + withProjectDaemon home root .lean fun client => callBrokerWithProgress root client (leanDocumentSymbolsRequest root path version) (leanDocumentSymbolsWaitSpec path action) @@ -265,9 +276,9 @@ def runCommand (home : System.FilePath) (opts : CliOptions) : IO Unit := do let query ← match joinTextArgs queryParts with | some query => pure query - | none => throw <| IO.userError "usage: beam [--root PATH] [--port N] lean-workspace-symbols " + | none => throw <| IO.userError "usage: beam [--root PATH] lean-workspace-symbols " let action ← wrapperDisplayAction "lean-workspace-symbols" - withProjectDaemon home root .lean opts fun client => + withProjectDaemon home root .lean fun client => callBrokerWithProgress root client (leanWorkspaceSymbolsRequest root query) (leanWorkspaceSymbolsWaitSpec query action) @@ -278,7 +289,7 @@ def runCommand (home : System.FilePath) (opts : CliOptions) : IO Unit := do let line ← parseNatArg "line" line let character ← parseNatArg "character" character let action ← wrapperDisplayAction "lean-goals" - withProjectDaemon home root .lean opts fun client => + withProjectDaemon home root .lean fun client => callBrokerWithProgress root client (leanGoalsRequest root path version line character mode) (leanGoalsWaitSpec path line character mode (some action)) @@ -291,7 +302,7 @@ def runCommand (home : System.FilePath) (opts : CliOptions) : IO Unit := do let endCharacter ← parseNatArg "endCharacter" endCharacter let (kinds?, suggest?) ← parseLeanTodoArgs extra let action ← wrapperDisplayAction "lean-todo" - withProjectDaemon home root .lean opts fun client => + withProjectDaemon home root .lean fun client => callBrokerWithProgress root client (leanTodoRequest root path version startLine startCharacter endLine endCharacter kinds? suggest?) (leanTodoWaitSpec path startLine startCharacter endLine endCharacter action) @@ -306,19 +317,19 @@ def runCommand (home : System.FilePath) (opts : CliOptions) : IO Unit := do let root ← projectRoot opts .lean let diagnosticScope ← parseLeanSaveArgs extra let action ← wrapperDisplayAction "lean-save" - withProjectDaemon home root .lean opts fun client => + withProjectDaemon home root .lean fun client => callBrokerWithProgress root client (leanSaveRequest root path diagnosticScope) (leanSaveWaitSpec path (action? := some action)) | "lean-update" :: path :: [] => let root ← projectRoot opts .lean - withProjectDaemon home root .lean opts fun client => + withProjectDaemon home root .lean fun client => callBroker root client <| leanUpdateRequest root path | "lean-sync" :: path :: extra => do let root ← projectRoot opts .lean let diagnosticScope ← parseLeanSyncArgs extra let action ← wrapperDisplayAction "lean-sync" - withProjectDaemon home root .lean opts fun client => + withProjectDaemon home root .lean fun client => callBrokerWithProgress root client (leanSyncRequest root path diagnosticScope) (syncWaitSpec path action) @@ -326,25 +337,25 @@ def runCommand (home : System.FilePath) (opts : CliOptions) : IO Unit := do let root ← projectRoot opts .lean let diagnosticScope ← parseLeanRefreshArgs extra let action ← wrapperDisplayAction "lean-refresh" - withProjectDaemon home root .lean opts fun client => + withProjectDaemon home root .lean fun client => callBrokerWithProgress root client (leanRefreshRequest root path diagnosticScope) (refreshWaitSpec path action) | "lean-close" :: path :: [] => let root ← projectRoot opts .lean - withProjectDaemon home root .lean opts fun client => + withProjectDaemon home root .lean fun client => callBroker root client <| leanCloseRequest root path | "lean-close-save" :: path :: extra => let root ← projectRoot opts .lean let diagnosticScope ← parseLeanCloseSaveArgs extra let action ← wrapperDisplayAction "lean-close-save" - withProjectDaemon home root .lean opts fun client => + withProjectDaemon home root .lean fun client => callBrokerWithProgress root client (leanCloseSaveRequest root path diagnosticScope) (leanSaveWaitSpec path (closeAfter := true) (action? := some action)) | "rocq-goals-after" :: path :: line :: character :: text => let root ← projectRoot opts .rocq - withProjectDaemon home root .rocq opts fun client => do + withProjectDaemon home root .rocq fun client => do let version ← updateVersionForRocqGoals root client path callBroker root client { op := .goals @@ -361,7 +372,7 @@ def runCommand (home : System.FilePath) (opts : CliOptions) : IO Unit := do } | "rocq-goals-prev" :: path :: line :: character :: text => let root ← projectRoot opts .rocq - withProjectDaemon home root .rocq opts fun client => do + withProjectDaemon home root .rocq fun client => do let version ← updateVersionForRocqGoals root client path callBroker root client { op := .goals @@ -377,7 +388,7 @@ def runCommand (home : System.FilePath) (opts : CliOptions) : IO Unit := do text? := joinTextArgs text } | "doctor" :: backend :: [] => - doctor home opts (if backend == "rocq" then .rocq else .lean) + doctor home opts (← parseBackendName backend) | "open-files" :: [] => let root ← projectRootAny opts withExistingProjectDaemon root fun client => diff --git a/Beam/Cli/DaemonManager.lean b/Beam/Cli/DaemonManager.lean index 8242a971..15b72fe7 100644 --- a/Beam/Cli/DaemonManager.lean +++ b/Beam/Cli/DaemonManager.lean @@ -606,7 +606,8 @@ private def closeDaemonOwnerPipe pure child private partial def waitForOwnedDaemonExit - (child : IO.Process.Child detachedDaemonStdio) + {cfg : IO.Process.StdioConfig} + (child : IO.Process.Child cfg) (exitCodeRef : IO.Ref (Option UInt32)) (tries : Nat) : IO Unit := do if (← exitCodeRef.get).isSome || tries == 0 then @@ -628,6 +629,33 @@ private def removeOwnedRegistry (root : System.FilePath) (daemonId : String) : I catch _ => pure () +private def attemptCleanup (act : IO Unit) : IO Unit := do + try + act + catch _ => + pure () + +private def finishOwnedDaemonChild + (owned : OwnedProjectDaemon) + (exitCodeRef : IO.Ref (Option UInt32)) : IO Unit := do + try + let child ← closeDaemonOwnerPipe owned.child + attemptCleanup <| waitForOwnedDaemonExit child exitCodeRef 100 + if (← exitCodeRef.get).isNone then + attemptCleanup child.kill + attemptCleanup <| waitForOwnedDaemonExit child exitCodeRef 20 + pure () + catch _ => + attemptCleanup owned.child.kill + attemptCleanup <| waitForOwnedDaemonExit owned.child exitCodeRef 20 + +private def finishOwnedProjectDaemon + (root : System.FilePath) + (owned : OwnedProjectDaemon) + (exitCodeRef : IO.Ref (Option UInt32)) : IO Unit := do + finishOwnedDaemonChild owned exitCodeRef + removeOwnedRegistry root owned.entry.daemonId + def withProjectDaemonOwner (home root : System.FilePath) (backend : Backend) @@ -637,29 +665,16 @@ def withProjectDaemonOwner let owned ← withProjectControlLock root do startOwnedProjectDaemon desired opts let exitCodeRef ← IO.mkRef (none : Option UInt32) - let result ← - try - pure <| Except.ok (← act { + try + act { client := owned.client root daemonId := owned.entry.daemonId child := owned.child exitCodeRef - }) - catch err => - pure <| Except.error err - let child ← closeDaemonOwnerPipe owned.child - waitForOwnedDaemonExit child exitCodeRef 100 - if (← exitCodeRef.get).isNone then - try - child.kill - catch _ => - pure () - waitForOwnedDaemonExit child exitCodeRef 20 - removeOwnedRegistry root owned.entry.daemonId - match result with - | .ok value => pure value - | .error err => throw err + } + finally + finishOwnedProjectDaemon root owned exitCodeRef private def lookupProjectDaemon (root : System.FilePath) @@ -674,7 +689,6 @@ private def lookupProjectDaemon def withProjectDaemon (home root : System.FilePath) (backend : Backend) - (_opts : CliOptions) (act : ProjectDaemonClient → IO α) : IO α := do let desired ← desiredConfig home root backend act (← lookupProjectDaemon root (some desired.configHash)) diff --git a/Beam/Cli/Usage.lean b/Beam/Cli/Usage.lean index 9c3b54e9..e2bcc3b0 100644 --- a/Beam/Cli/Usage.lean +++ b/Beam/Cli/Usage.lean @@ -13,28 +13,29 @@ def usage : String := "usage:", " beam --version", " beam version", - " beam [--root PATH] [--port N] ensure [lean|rocq] [--hold]", - " beam [--root PATH] [--port N] lean-run-at [--stdin | --text-file | -- | ]", - " beam [--root PATH] [--port N] lean-run-at-handle [--stdin | --text-file | -- | ]", - " beam [--root PATH] [--port N] lean-hover ", - " beam [--root PATH] [--port N] lean-signature-help ", - " beam [--root PATH] [--port N] lean-definition ", - " beam [--root PATH] [--port N] lean-references [--include-declaration|--exclude-declaration]", - " beam [--root PATH] [--port N] lean-document-symbols ", - " beam [--root PATH] [--port N] lean-workspace-symbols ", - " beam [--root PATH] [--port N] lean-goals before|after ", - " beam [--root PATH] [--port N] lean-todo [--kind ...] [--suggest none|basic]", - " beam [--root PATH] [--port N] lean-run-with > [--stdin | --text-file | -- | ]", - " beam [--root PATH] [--port N] lean-run-with-linear > [--stdin | --text-file | -- | ]", - " beam [--root PATH] [--port N] lean-release >", - " beam [--root PATH] [--port N] lean-update ", - " beam [--root PATH] [--port N] lean-sync [+all-diagnostics]", - " beam [--root PATH] [--port N] lean-refresh [+all-diagnostics]", - " beam [--root PATH] [--port N] lean-save [+all-diagnostics]", - " beam [--root PATH] [--port N] lean-close ", - " beam [--root PATH] [--port N] lean-close-save [+all-diagnostics]", - " beam [--root PATH] [--port N] rocq-goals-after [text...]", - " beam [--root PATH] [--port N] rocq-goals-prev [text...]", + " beam [--root PATH] ensure [lean|rocq]", + " beam [--root PATH] [--port N] ensure [lean|rocq] --hold", + " beam [--root PATH] lean-run-at [--stdin | --text-file | -- | ]", + " beam [--root PATH] lean-run-at-handle [--stdin | --text-file | -- | ]", + " beam [--root PATH] lean-hover ", + " beam [--root PATH] lean-signature-help ", + " beam [--root PATH] lean-definition ", + " beam [--root PATH] lean-references [--include-declaration|--exclude-declaration]", + " beam [--root PATH] lean-document-symbols ", + " beam [--root PATH] lean-workspace-symbols ", + " beam [--root PATH] lean-goals before|after ", + " beam [--root PATH] lean-todo [--kind ...] [--suggest none|basic]", + " beam [--root PATH] lean-run-with > [--stdin | --text-file | -- | ]", + " beam [--root PATH] lean-run-with-linear > [--stdin | --text-file | -- | ]", + " beam [--root PATH] lean-release >", + " beam [--root PATH] lean-update ", + " beam [--root PATH] lean-sync [+all-diagnostics]", + " beam [--root PATH] lean-refresh [+all-diagnostics]", + " beam [--root PATH] lean-save [+all-diagnostics]", + " beam [--root PATH] lean-close ", + " beam [--root PATH] lean-close-save [+all-diagnostics]", + " beam [--root PATH] rocq-goals-after [text...]", + " beam [--root PATH] rocq-goals-prev [text...]", " beam [--root PATH] feedback-report --stdin|--input [--bundle none|dir|zip] [--output-dir ] [--no-redact]", " beam bundle-install ", " beam install-prune [--apply] [--bundles]", diff --git a/CHANGELOG.md b/CHANGELOG.md index 873a534c..5f7fc81a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -34,8 +34,9 @@ This project keeps a lightweight, reverse-chronological changelog. Dates use `YY ### Changed - Wrapper daemons now have explicit session ownership: only `lean-beam ensure --hold` starts a - generation, ordinary wrapper commands attach to it, and holder exit closes the daemon through an - inherited pipe without heartbeat leases or retirement fences + generation, ordinary wrapper commands attach to it, `--port` is accepted only by that owner-start + command, and holder exit cancels admitted requests before closing the daemon through an inherited + pipe without heartbeat leases or retirement fences ([#241](https://github.com/leanprover/lean-beam/pull/241), @ejgallego). - Long-running Lean operations now separate liveness status, request progress, and diagnostics. Sync and refresh use the discoverable `diagnostic_scope: "errors" | "all"` and diff --git a/docs/DEVELOPMENT.md b/docs/DEVELOPMENT.md index c686f17f..c2d2171e 100644 --- a/docs/DEVELOPMENT.md +++ b/docs/DEVELOPMENT.md @@ -395,9 +395,6 @@ Keep these invariants covered: - the regressions for this path are [tests/test-beam-wrapper-daemon.sh](../tests/test-beam-wrapper-daemon.sh) and [tests/test-beam-wrapper-sandbox.sh](../tests/test-beam-wrapper-sandbox.sh) -- the regressions for this path are - [tests/test-beam-wrapper-daemon.sh](../tests/test-beam-wrapper-daemon.sh) and - [tests/test-beam-wrapper-sandbox.sh](../tests/test-beam-wrapper-sandbox.sh) Generic process helpers and the typed `RecordedPid.observe` boundary live in [Beam/System.lean](../Beam/System.lean). Persisted registry and lock-owner PIDs must pass diff --git a/docs/STATUS.md b/docs/STATUS.md index c783205c..25acaed8 100644 --- a/docs/STATUS.md +++ b/docs/STATUS.md @@ -184,11 +184,13 @@ Exact event ordering and examples live in - Wrapper sessions use explicit ownership. `lean-beam ensure --hold` is the only wrapper command that starts a project daemon; it passes the daemon an inherited pipe and remains alive as the owner. Ordinary wrapper calls, including plain `lean-beam ensure`, only attach to that generation and - fail with the exact owner-start command when none is live. Owner EOF shuts down request admission, - backend sessions, and the daemon without heartbeat timeouts or filesystem leases. This works + fail with the exact owner-start command when none is live. The optional `--port` override belongs + only to `ensure --hold`; attaching commands reject it. Owner EOF shuts down request admission, + completes admitted requests with `requestCancelled`, and closes backend sessions and the daemon + without heartbeat timeouts or filesystem leases. This works across PID namespaces because endpoint/root validation is authoritative when PID identity is not locally observable. A paused owner retains the session; a killed owner closes the pipe; explicit - `lean-beam shutdown` revokes the registry generation so the holder closes it cleanly. + `lean-beam shutdown` unpublishes the registry generation so the holder closes it cleanly. - A startup failure that reports `operation not permitted` through `.beam/beam-daemon-startup.log` is usually an environment restriction, not a bundle-resolution mismatch. - Beam daemon disappearance errors include registry/log context and write a JSON incident record under diff --git a/docs/TESTING.md b/docs/TESTING.md index 638ca488..e442f38a 100644 --- a/docs/TESTING.md +++ b/docs/TESTING.md @@ -111,7 +111,8 @@ Current Beam coverage includes: - wrapper coverage through [tests/test-beam-wrapper.sh](../tests/test-beam-wrapper.sh), which aggregates focused probe, runtime, sync/save, handle, and diagnostic slices - focused daemon lifecycle coverage in [tests/test-beam-wrapper-daemon.sh](../tests/test-beam-wrapper-daemon.sh), including the no-implicit-start contract, duplicate-owner rejection, endpoint collision safety, - explicit shutdown, generation replacement, holder reporting after an unexpected daemon crash, + explicit shutdown, cancellation of requests active during shutdown or owner loss, exact-generation + cleanup that preserves a replacement registry, holder reporting after an unexpected daemon crash, abrupt owner death through inherited-pipe EOF, stale registry cleanup, and self-termination after the project worktree disappears - Linux-only PID-isolated sandbox wrapper coverage in [tests/test-beam-wrapper-sandbox.sh](../tests/test-beam-wrapper-sandbox.sh), diff --git a/scripts/lean-beam b/scripts/lean-beam index 34690a4b..b30d64a1 100755 --- a/scripts/lean-beam +++ b/scripts/lean-beam @@ -11,28 +11,29 @@ usage() { usage: lean-beam --version lean-beam version - lean-beam [--root PATH] [--port N] ensure [lean|rocq] [--hold] - lean-beam [--root PATH] [--port N] run-at [--stdin | --text-file | -- | ] - lean-beam [--root PATH] [--port N] run-at-handle [--stdin | --text-file | -- | ] - lean-beam [--root PATH] [--port N] hover - lean-beam [--root PATH] [--port N] signature-help - lean-beam [--root PATH] [--port N] definition - lean-beam [--root PATH] [--port N] references [--include-declaration|--exclude-declaration] - lean-beam [--root PATH] [--port N] document-symbols - lean-beam [--root PATH] [--port N] workspace-symbols - lean-beam [--root PATH] [--port N] goals before|after - lean-beam [--root PATH] [--port N] todo [--kind ...] [--suggest none|basic] - lean-beam [--root PATH] [--port N] run-with > [--stdin | --text-file | -- | ] - lean-beam [--root PATH] [--port N] run-with-linear > [--stdin | --text-file | -- | ] - lean-beam [--root PATH] [--port N] release > - lean-beam [--root PATH] [--port N] update - lean-beam [--root PATH] [--port N] sync [+all-diagnostics] - lean-beam [--root PATH] [--port N] refresh [+all-diagnostics] - lean-beam [--root PATH] [--port N] save [+all-diagnostics] - lean-beam [--root PATH] [--port N] close - lean-beam [--root PATH] [--port N] close-save [+all-diagnostics] - lean-beam [--root PATH] [--port N] rocq-goals-after [text...] - lean-beam [--root PATH] [--port N] rocq-goals-prev [text...] + lean-beam [--root PATH] ensure [lean|rocq] + lean-beam [--root PATH] [--port N] ensure [lean|rocq] --hold + lean-beam [--root PATH] run-at [--stdin | --text-file | -- | ] + lean-beam [--root PATH] run-at-handle [--stdin | --text-file | -- | ] + lean-beam [--root PATH] hover + lean-beam [--root PATH] signature-help + lean-beam [--root PATH] definition + lean-beam [--root PATH] references [--include-declaration|--exclude-declaration] + lean-beam [--root PATH] document-symbols + lean-beam [--root PATH] workspace-symbols + lean-beam [--root PATH] goals before|after + lean-beam [--root PATH] todo [--kind ...] [--suggest none|basic] + lean-beam [--root PATH] run-with > [--stdin | --text-file | -- | ] + lean-beam [--root PATH] run-with-linear > [--stdin | --text-file | -- | ] + lean-beam [--root PATH] release > + lean-beam [--root PATH] update + lean-beam [--root PATH] sync [+all-diagnostics] + lean-beam [--root PATH] refresh [+all-diagnostics] + lean-beam [--root PATH] save [+all-diagnostics] + lean-beam [--root PATH] close + lean-beam [--root PATH] close-save [+all-diagnostics] + lean-beam [--root PATH] rocq-goals-after [text...] + lean-beam [--root PATH] rocq-goals-prev [text...] lean-beam [--root PATH] feedback-report --stdin|--input [--bundle none|dir|zip] [--output-dir ] [--no-redact] lean-beam prune [--apply] [--bundles] lean-beam validated-toolchains [lean] diff --git a/scripts/lean-beam-search b/scripts/lean-beam-search index 2c742b82..9df564b0 100755 --- a/scripts/lean-beam-search +++ b/scripts/lean-beam-search @@ -45,7 +45,7 @@ usage: notes: - branch, linear, playout, and release read a prior wrapper response or handle JSON from stdin - mint requires the version returned by `lean-beam update ` or `lean-beam sync ` - - lean-beam opts such as --root and --port may appear before the subcommand + - lean-beam opts such as --root may appear before the subcommand EOF exit 1 } diff --git a/skills/lean-beam/SKILL.md b/skills/lean-beam/SKILL.md index c30969b3..76ef4ede 100644 --- a/skills/lean-beam/SKILL.md +++ b/skills/lean-beam/SKILL.md @@ -317,6 +317,8 @@ Use `lean-beam`, not raw JSON and not raw LSP. - `lean-beam ensure --hold` prints the usual JSON ensure response on stdout and keeps the wrapper process alive as the session owner; an inherited pipe ties daemon lifetime to that process without heartbeat files or lease expiry +- `--port` is an optional owner-start override for `lean-beam ensure --hold`; attaching commands + reject it instead of silently ignoring it - interrupting or killing the holder closes the owner pipe and shuts down the daemon; explicit `lean-beam shutdown` releases the holder cleanly - plain `lean-beam ensure` checks and warms a live owned session; if there is no owner, follow its diff --git a/tests/lean/BeamTest/Broker/PendingTest.lean b/tests/lean/BeamTest/Broker/PendingTest.lean index 943b4e7b..c27d6be8 100644 --- a/tests/lean/BeamTest/Broker/PendingTest.lean +++ b/tests/lean/BeamTest/Broker/PendingTest.lean @@ -166,28 +166,33 @@ private def checkPendingStoreResolve : IO Unit := do require "pending store is empty after remove" ((← PendingRequestStore.snapshot store).isEmpty) -private def checkPendingStoreFailAll : IO Unit := do +private def checkPendingStoreFailAllRespectingCancellation : IO Unit := do let store ← PendingRequestStore.create let firstProgress : SyncFileProgress := { updates := 5, done := false } let secondProgress : SyncFileProgress := { updates := 8, done := true } - let (firstPending, firstPromise) ← mkPending (progress? := some firstProgress) + let cancelRef ← IO.mkRef true + let (firstPending, firstPromise) ← mkPending + (progress? := some firstProgress) (cancelRef? := some cancelRef) let (secondPending, secondPromise) ← mkPending (progress? := some secondProgress) PendingRequestStore.insert store 11 firstPending PendingRequestStore.insert store 12 secondPending - PendingRequestStore.failAll store (responseFailureFor .workerExited "worker exited") - for (label, promise, expectedProgress) in #[ - ("first", firstPromise, firstProgress), - ("second", secondPromise, secondProgress) + PendingRequestStore.failAllRespectingCancellation store + (responseFailureFor .workerExited "worker exited") + for (label, promise, expectedCode, expectedProgress) in #[ + ("cancelled", firstPromise, "requestCancelled", firstProgress), + ("worker-exited", secondPromise, "workerExited", secondProgress) ] do match ← PendingRequest.awaitOutcome promise with | .ok _ => - throw <| IO.userError s!"failAll resolves {label} pending request as an error: expected error" + throw <| IO.userError + s!"failAllRespectingCancellation resolves {label} pending request as an error: expected error" | .error failure => discard <| requireFailureCode - s!"failAll resolves {label} pending request as an error" "workerExited" failure - require s!"failAll preserves {label} pending request progress" + s!"failAllRespectingCancellation resolves {label} pending request as an error" + expectedCode failure + require s!"failAllRespectingCancellation preserves {label} pending request progress" (failure.fileProgress? == some expectedProgress) - require "failAll clears pending store" + require "failAllRespectingCancellation clears pending store" ((← PendingRequestStore.snapshot store).isEmpty) private def checkPendingResolveError : IO Unit := do @@ -411,7 +416,7 @@ def main : IO Unit := do checkActiveRegistry checkPendingCancellationIdentity checkPendingStoreResolve - checkPendingStoreFailAll + checkPendingStoreFailAllRespectingCancellation checkPendingResolveError checkSyncFileProgressDisplay checkSyncFileProgressLines diff --git a/tests/lib/beam-wrapper-common.sh b/tests/lib/beam-wrapper-common.sh index 98eeb6e2..32879688 100644 --- a/tests/lib/beam-wrapper-common.sh +++ b/tests/lib/beam-wrapper-common.sh @@ -469,6 +469,18 @@ beam_wrapper_register_pid() { beam_wrapper_managed_pids+=("$1") } +beam_wrapper_unregister_pid() { + local target="$1" + local pid + local -a kept_pids=() + for pid in ${beam_wrapper_managed_pids[@]+"${beam_wrapper_managed_pids[@]}"}; do + if [ "$pid" != "$target" ]; then + kept_pids+=("$pid") + fi + done + beam_wrapper_managed_pids=(${kept_pids[@]+"${kept_pids[@]}"}) +} + beam_wrapper_start_owner() { local root="$1" local backend="${2:-lean}" diff --git a/tests/test-beam-wrapper-daemon.sh b/tests/test-beam-wrapper-daemon.sh index 1d5f11a4..f8723a84 100644 --- a/tests/test-beam-wrapper-daemon.sh +++ b/tests/test-beam-wrapper-daemon.sh @@ -32,6 +32,53 @@ if [ -z "${BEAM_INSTALL_BUNDLE_DIR:-}" ]; then fi hold_pid="" root_removed="false" +active_request_pid="" + +start_slow_request() { + local root="$1" + local label="$2" + local request_id="$3" + local version + version="$(beam_wrapper_update_version "$label SlowPoll" \ + "$beam_script" --root "$root" lean-update tests/scenario/docs/SlowPoll.lean)" + BEAM_PROGRESS=1 BEAM_REQUEST_ID="$request_id" "$beam_script" --root "$root" \ + lean-run-at tests/scenario/docs/SlowPoll.lean "$version" 25 2 poll_sleep_cmd \ + >"$root/$label.out" 2>"$root/$label.err" & + active_request_pid="$!" + if ! wait_for_file_text "$root/$label.err" "running lean-run-at" \ + "$label request progress" 150 0.1; then + cat "$root/$label.out" >&2 + cat "$root/$label.err" >&2 + exit 1 + fi + if ! kill -0 "$active_request_pid" 2>/dev/null; then + echo "expected $label request to remain active before session close" >&2 + cat "$root/$label.out" >&2 + cat "$root/$label.err" >&2 + exit 1 + fi +} + +expect_slow_request_cancelled() { + local root="$1" + local label="$2" + local request_id="$3" + local status=0 + set +e + wait "$active_request_pid" + status="$?" + set -e + active_request_pid="" + if [ "$status" -eq 0 ]; then + echo "expected $label request to exit non-zero after session close" >&2 + cat "$root/$label.out" >&2 + exit 1 + fi + assert_json_file_field_equals "$label cancellation code" "$root/$label.out" \ + error.code requestCancelled "$root/$label.err" + assert_json_file_field_equals "$label request id" "$root/$label.out" \ + clientRequestId "$request_id" "$root/$label.err" +} stop_hold_process() { local require_clean_exit="${1:-false}" @@ -62,6 +109,11 @@ stop_hold_process() { } cleanup() { + if [ -n "$active_request_pid" ]; then + kill "$active_request_pid" > /dev/null 2>&1 || true + wait "$active_request_pid" 2>/dev/null || true + active_request_pid="" + fi stop_hold_process if [ "$root_removed" != "true" ]; then "$beam_script" --root "$tmp1" shutdown > /dev/null 2>&1 || true @@ -83,11 +135,26 @@ for tmp in "$tmp1" "$tmp2"; do rsync -a --exclude='.beam/' tests/save_olean_project/ "$tmp"/ remove_tmp_tree_within "$tmp/.beam" "$tmp" mkdir -p "$tmp/.beam" + mkdir -p "$tmp/tests/scenario/docs" + cp tests/scenario/docs/SlowPoll.lean "$tmp/tests/scenario/docs/SlowPoll.lean" done fixture_toolchain="$(awk 'NR==1 {print $1}' tests/save_olean_project/lean-toolchain)" "$beam_cli" bundle-install "$fixture_toolchain" +invalid_backend_out="$tmp1/invalid-backend.out" +invalid_backend_err="$tmp1/invalid-backend.err" +if "$beam_script" --root "$tmp1" ensure typo --hold > "$invalid_backend_out" 2> "$invalid_backend_err"; then + echo "expected an unknown owner backend to be rejected" >&2 + cat "$invalid_backend_out" >&2 + exit 1 +fi +if ! grep -Fq "expected backend 'lean' or 'rocq'" "$invalid_backend_err"; then + echo "expected unknown-backend diagnostics to list the valid backend names" >&2 + cat "$invalid_backend_err" >&2 + exit 1 +fi + missing_owner_out="$tmp1/missing-owner.out" missing_owner_err="$tmp1/missing-owner.err" if "$beam_script" --root "$tmp1" ensure > "$missing_owner_out" 2> "$missing_owner_err"; then @@ -164,6 +231,8 @@ assert_json_field_equals "owned ensure response" "$ensure_json" ok true stats_json="$("$beam_script" --root "$tmp1" stats)" assert_json_field_equals "owned stats response" "$stats_json" ok true +start_slow_request "$tmp1" "shutdown-active" "shutdown-active" + second_owner_out="$tmp1/second-owner.out" second_owner_err="$tmp1/second-owner.err" if "$beam_script" --root "$tmp1" ensure --hold > "$second_owner_out" 2> "$second_owner_err"; then @@ -198,6 +267,7 @@ fi shutdown_json="$("$beam_script" --root "$tmp1" shutdown)" assert_json_field_equals "explicit session shutdown" "$shutdown_json" ok true +expect_slow_request_cancelled "$tmp1" "shutdown-active" "shutdown-active" if ! wait_for_exit "$hold_pid" "owner after explicit shutdown" 200 0.05; then cat "$tmp1/owner-1.err" >&2 exit 1 @@ -256,6 +326,7 @@ fi start_owner "$tmp1" "owner-3" daemon3_pid="$(read_json_field "$registry" pid)" +start_slow_request "$tmp1" "owner-loss-active" "owner-loss-active" kill -KILL "$hold_pid" set +e wait "$hold_pid" @@ -265,6 +336,7 @@ if ! wait_for_exit "$daemon3_pid" "daemon after owner death" 200 0.05; then echo "expected owner-pipe EOF to stop the daemon" >&2 exit 1 fi +expect_slow_request_cancelled "$tmp1" "owner-loss-active" "owner-loss-active" owner_loss_out="$tmp1/owner-loss.out" owner_loss_err="$tmp1/owner-loss.err" if "$beam_script" --root "$tmp1" ensure > "$owner_loss_out" 2> "$owner_loss_err"; then @@ -283,6 +355,42 @@ if [ -e "$registry" ]; then exit 1 fi +generation_registry="$tmp2/.beam/beam-daemon.json" +start_owner "$tmp2" "owner-generation" +generation_daemon_pid="$(read_json_field "$generation_registry" pid)" +generation_id="$(read_json_field "$generation_registry" daemonId)" +replacement_generation_id="$generation_id-replacement" +python3 - "$generation_registry" "$replacement_generation_id" <<'PY' +import json +import os +import sys + +path, replacement_id = sys.argv[1:] +with open(path, "r", encoding="utf-8") as stream: + entry = json.load(stream) +entry["daemonId"] = replacement_id +replacement = path + ".replacement" +with open(replacement, "w", encoding="utf-8") as stream: + json.dump(entry, stream, separators=(",", ":")) + stream.write("\n") +os.replace(replacement, path) +PY +if ! wait_for_exit "$hold_pid" "owner after generation replacement" 200 0.05; then + cat "$tmp2/owner-generation.err" >&2 + exit 1 +fi +wait "$hold_pid" +hold_pid="" +if ! wait_for_exit "$generation_daemon_pid" "daemon after generation replacement" 200 0.05; then + exit 1 +fi +if [ "$(read_json_field "$generation_registry" daemonId)" != "$replacement_generation_id" ]; then + echo "expected old-owner cleanup to preserve the replacement registry generation" >&2 + cat "$generation_registry" >&2 + exit 1 +fi +rm -f -- "$generation_registry" + start_owner "$tmp1" "owner-4" daemon4_pid="$(read_json_field "$registry" pid)" remove_owned_tmp_tree "$tmp1" diff --git a/tests/test-beam-wrapper-runtime.sh b/tests/test-beam-wrapper-runtime.sh index 7c2c3156..e523d319 100644 --- a/tests/test-beam-wrapper-runtime.sh +++ b/tests/test-beam-wrapper-runtime.sh @@ -15,7 +15,6 @@ beam_wrapper_init primary_root="$(beam_wrapper_prepare_project_root runtime-primary)" other_root="$(beam_wrapper_prepare_project_root runtime-other)" signal_root="$(beam_wrapper_prepare_project_root_with_scenario_docs runtime-signal)" -busy_port_root="$(beam_wrapper_prepare_project_root runtime-busy-port)" run_sigint_probe() { local project_root="$1" @@ -367,6 +366,7 @@ PY ) wait_for_exit "$signal_owner_pid" "first signal-test session owner" 120 0.1 wait "$signal_owner_pid" +beam_wrapper_unregister_pid "$signal_owner_pid" beam_wrapper_start_owner "$signal_root" signal_owner_pid="$beam_wrapper_last_owner_pid" @@ -460,6 +460,7 @@ signal_owner_pid="$beam_wrapper_last_owner_pid" ) wait_for_exit "$signal_owner_pid" "second signal-test session owner" 120 0.1 wait "$signal_owner_pid" +beam_wrapper_unregister_pid "$signal_owner_pid" beam_wrapper_start_owner "$other_root" ( @@ -494,72 +495,16 @@ if ! grep -q "invalidParams" "$cross_err"; then exit 1 fi -beam_wrapper_start_owner "$busy_port_root" -( - cd "$busy_port_root" - "$beam_script" ensure lean > /dev/null - warm_version="$(beam_wrapper_update_version "busy-port SaveSmoke/B.lean" "$beam_script" lean-update SaveSmoke/B.lean)" - warm_out="$("$beam_script" lean-run-at SaveSmoke/B.lean "$warm_version" 0 2 "#eval bVal")" - if [ "$(BEAM_JSON_PAYLOAD="$warm_out" read_json_text_field ok)" != "true" ]; then - echo "expected busy-port warmup probe to succeed before reuse check" >&2 - printf '%s\n' "$warm_out" >&2 - exit 1 - fi -) - -busy_registry="$(beam_wrapper_registry_path "$busy_port_root")" -beam_wrapper_expect_file "$busy_registry" -pid5="$(read_json_field "$busy_registry" pid)" -port5="$(read_json_field "$busy_registry" port)" -busy_port=43123 -if [ "$busy_port" = "$port5" ]; then - busy_port=43124 -fi - -python3 -m http.server "$busy_port" >/dev/null 2>&1 & -busy_pid=$! -beam_wrapper_register_pid "$busy_pid" -sleep 1 - -( - cd "$busy_port_root" - doctor_out="$("$beam_script" doctor lean)" - if ! printf '%s\n' "$doctor_out" | grep -q 'daemon status: live'; then - echo "expected doctor lean to report a live Beam daemon before requested-port reuse check" >&2 - printf '%s\n' "$doctor_out" >&2 - exit 1 - fi - sed_in_place_portable 's/1/2/' SaveSmoke/B.lean - sync_out="$("$beam_script" --port "$busy_port" lean-sync SaveSmoke/B.lean)" - if [ "$(BEAM_JSON_PAYLOAD="$sync_out" read_json_text_field ok)" != "true" ]; then - echo "expected lean-sync with a busy requested port to reuse the live Beam daemon" >&2 - printf '%s\n' "$sync_out" >&2 - exit 1 - fi - if [ "$(BEAM_JSON_PAYLOAD="$sync_out" read_json_text_field result.version)" != "2" ]; then - echo "expected busy-port lean-sync reuse path to report version 2" >&2 - printf '%s\n' "$sync_out" >&2 - exit 1 - fi - stats_out="$("$beam_script" stats)" - if [ "$(BEAM_JSON_PAYLOAD="$stats_out" read_json_text_field ok)" != "true" ]; then - echo "expected stats to keep working after busy-port lean-sync reuse" >&2 - printf '%s\n' "$stats_out" >&2 - exit 1 - fi -) - -kill "$busy_pid" > /dev/null 2>&1 || true -wait "$busy_pid" 2>/dev/null || true - -pid5_after="$(read_json_field "$busy_registry" pid)" -port5_after="$(read_json_field "$busy_registry" port)" -if [ "$pid5" != "$pid5_after" ] || [ "$port5" != "$port5_after" ]; then - echo "expected requested-port lean-sync reuse to preserve the original registry entry" >&2 +port_scope_out="$(beam_wrapper_mktemp_file port-scope-out)" +port_scope_err="$(beam_wrapper_mktemp_file port-scope-err)" +if "$beam_script" --root "$primary_root" --port 43123 stats >"$port_scope_out" 2>"$port_scope_err"; then + echo "expected --port on an attaching command to be rejected" >&2 + cat "$port_scope_out" >&2 exit 1 fi -if ! kill -0 "$pid5" 2>/dev/null; then - echo "expected original Beam daemon pid $pid5 to remain alive after busy-port lean-sync reuse" >&2 +if ! grep -Fq -- "--port is only valid" "$port_scope_err"; then + echo "expected rejected attaching --port to explain the owner-only scope" >&2 + cat "$port_scope_err" >&2 exit 1 fi @@ -569,6 +514,7 @@ fi ) wait_for_exit "$primary_owner_pid" "primary session owner" 120 0.1 wait "$primary_owner_pid" +beam_wrapper_unregister_pid "$primary_owner_pid" if [ -f "$primary_registry" ]; then echo "expected shutdown to remove the project Beam daemon registry" >&2 From 5371efe4f1cbc355cee83756d00cb89c36559605 Mon Sep 17 00:00:00 2001 From: Emilio Jesus Gallego Arias Date: Wed, 26 Aug 2026 00:31:49 +0200 Subject: [PATCH 04/28] refactor: keep daemon startup failures typed --- Beam/Cli/DaemonManager.lean | 44 ++++++++++++------- Beam/Daemon/Protocol.lean | 6 +-- tests/lean/BeamTest/Broker/CliDaemonTest.lean | 4 +- 3 files changed, 33 insertions(+), 21 deletions(-) diff --git a/Beam/Cli/DaemonManager.lean b/Beam/Cli/DaemonManager.lean index 15b72fe7..695d0f8f 100644 --- a/Beam/Cli/DaemonManager.lean +++ b/Beam/Cli/DaemonManager.lean @@ -297,8 +297,14 @@ def daemonFailureMessage (root : System.FilePath) (detail : String) : IO String pure <| appendMaybeSection msg <| incidentPath?.map fun path => s!"Beam daemon incident: {path}" -private def startupFailureMessage (endpoint : Transport.Endpoint) (logPath : System.FilePath) (detail : String) : - IO String := do +private structure DaemonStartupFailure where + message : String + endpointInUse : Bool := false + +private def daemonStartupFailure + (endpoint : Transport.Endpoint) + (logPath : System.FilePath) + (detail : String) : IO DaemonStartupFailure := do let msg := if detail.isEmpty then s!"failed to start Beam daemon on {endpointSummary endpoint}" else @@ -306,11 +312,14 @@ private def startupFailureMessage (endpoint : Transport.Endpoint) (logPath : Sys if ← logPath.pathExists then let logText := Beam.trimLine (← IO.FS.readFile logPath) if logText.isEmpty then - pure msg + pure { message := msg } else - pure <| msg ++ s!"\nstartup log ({logPath}):\n{logText}" + pure { + message := msg ++ s!"\nstartup log ({logPath}):\n{logText}" + endpointInUse := startupLogSuggestsEndpointInUse logText + } else - pure msg + pure { message := msg } private abbrev daemonStdio : IO.Process.StdioConfig where stdin := .piped @@ -372,18 +381,21 @@ private partial def waitForDaemon (endpoint : Transport.Endpoint) (logPath : System.FilePath) (root : System.FilePath) - (tries : Nat := 300) : IO Unit := do + (tries : Nat := 300) : IO (Except DaemonStartupFailure Unit) := do match ← daemonRoot? endpoint projectDaemonWorkspaceId with | some daemonRoot => if ← Beam.sameFilePath (System.FilePath.mk daemonRoot) root then - pure () + pure (.ok ()) else - throw <| IO.userError (endpointOccupancyError endpoint (System.FilePath.mk daemonRoot) root) + pure <| .error { + message := endpointOccupancyError endpoint (System.FilePath.mk daemonRoot) root + endpointInUse := true + } | none => if (← child.tryWait).isSome then - throw <| IO.userError (← startupFailureMessage endpoint logPath "Beam daemon process exited before responding") + .error <$> daemonStartupFailure endpoint logPath "Beam daemon process exited before responding" else if tries == 0 then - throw <| IO.userError (← startupFailureMessage endpoint logPath "Beam daemon did not become ready before timeout") + .error <$> daemonStartupFailure endpoint logPath "Beam daemon did not become ready before timeout" else IO.sleep 100 waitForDaemon child endpoint logPath root (tries - 1) @@ -432,15 +444,15 @@ private partial def startDaemonEntry let logPath ← daemonStartupLogPath desired.root let daemonId ← newDaemonGenerationId desired.configHash let child ← startDaemon desired endpoint logPath - try - waitForDaemon child endpoint logPath desired.root - catch err => + match ← waitForDaemon child endpoint logPath desired.root with + | .ok () => pure () + | .error failure => terminateDaemonChild child let endpointOccupied ← endpointAcceptsConnection endpoint - let startupAddressInUse := startupFailureSuggestsEndpointInUse (toString err) - if shouldRetryAutomaticStartup (usesAutomaticTcpEndpoint opts) tries endpointOccupied startupAddressInUse then + if shouldRetryAutomaticStartup + (usesAutomaticTcpEndpoint opts) tries endpointOccupied failure.endpointInUse then return ← startDaemonEntry desired opts (tries - 1) - throw err + throw <| IO.userError failure.message let pid := child.pid.toNat let entry ← registryEntryFor desired daemonId pid endpoint opts pure (endpoint, entry, child) diff --git a/Beam/Daemon/Protocol.lean b/Beam/Daemon/Protocol.lean index cf4aef0d..2516834a 100644 --- a/Beam/Daemon/Protocol.lean +++ b/Beam/Daemon/Protocol.lean @@ -85,9 +85,9 @@ def endpointOccupancyError def endpointInUseError (endpoint : Transport.Endpoint) : String := s!"selected endpoint {endpointSummary endpoint} is already in use" -def startupFailureSuggestsEndpointInUse (message : String) : Bool := - message.contains "address already in use" || - message.contains "Address already in use" +def startupLogSuggestsEndpointInUse (logText : String) : Bool := + logText.contains "address already in use" || + logText.contains "Address already in use" def shouldRetryAutomaticStartup (usesAutomaticEndpoint : Bool) diff --git a/tests/lean/BeamTest/Broker/CliDaemonTest.lean b/tests/lean/BeamTest/Broker/CliDaemonTest.lean index 36995da6..ecf537c8 100644 --- a/tests/lean/BeamTest/Broker/CliDaemonTest.lean +++ b/tests/lean/BeamTest/Broker/CliDaemonTest.lean @@ -505,9 +505,9 @@ private def checkStartupRetryPolicy : IO Unit := do require "explicit endpoint should not retry" (!Beam.Daemon.shouldRetryAutomaticStartup false 1 true true) require "Linux bind failure wording should be recognized" - (Beam.Daemon.startupFailureSuggestsEndpointInUse "resource busy (error code: 4294967198, address already in use)") + (Beam.Daemon.startupLogSuggestsEndpointInUse "resource busy (error code: 4294967198, address already in use)") require "macOS bind failure wording should be recognized" - (Beam.Daemon.startupFailureSuggestsEndpointInUse "Address already in use") + (Beam.Daemon.startupLogSuggestsEndpointInUse "Address already in use") private def checkDaemonFailureContext : IO Unit := do let root := System.FilePath.mk s!"/tmp/beam-daemon-failure-context-{← IO.monoNanosNow}" From c8750d16da12d5f80ebb5a2875080c64be316888 Mon Sep 17 00:00:00 2001 From: Emilio Jesus Gallego Arias Date: Wed, 26 Aug 2026 02:28:17 +0200 Subject: [PATCH 05/28] refactor: centralize broker runtime shutdown --- Beam/Broker/Pending.lean | 80 ++++++++++++----- Beam/Broker/Server.lean | 90 ++++++++++++++++--- Beam/Mcp/StdioServer.lean | 2 +- docs/DEVELOPMENT.md | 8 ++ tests/lean/BeamTest/Broker/PendingTest.lean | 33 +++++++ tests/lean/BeamTest/Broker/ProtocolTest.lean | 38 +++++++- .../lean/BeamTest/Broker/StreamDedupTest.lean | 12 ++- 7 files changed, 219 insertions(+), 44 deletions(-) diff --git a/Beam/Broker/Pending.lean b/Beam/Broker/Pending.lean index 03f46b77..702d2400 100644 --- a/Beam/Broker/Pending.lean +++ b/Beam/Broker/Pending.lean @@ -340,14 +340,36 @@ private structure ActiveRequestRegistryState where accepting : Bool := true requests : Std.TreeMap String ActiveRequest := {} anonymousRequests : Std.TreeMap Nat ActiveRequest := {} + drainedSignaled : Bool := false structure ActiveRequestRegistry where private mutex : Std.Mutex ActiveRequestRegistryState + private drained : IO.Promise Unit namespace ActiveRequestRegistry def create : BaseIO ActiveRequestRegistry := do - pure { mutex := ← Std.Mutex.new {} } + pure { + mutex := ← Std.Mutex.new {} + drained := ← IO.Promise.new + } + +private def activeRequestCount + (state : ActiveRequestRegistryState) : Nat := + state.requests.size + state.anonymousRequests.size + +private def markDrainedIfReady + (state : ActiveRequestRegistryState) : ActiveRequestRegistryState × Bool := + if !state.accepting && activeRequestCount state == 0 && !state.drainedSignaled then + ({ state with drainedSignaled := true }, true) + else + (state, false) + +private def resolveDrainedIfNeeded + (registry : ActiveRequestRegistry) + (shouldResolve : Bool) : IO Unit := do + if shouldResolve then + registry.drained.resolve () def register (registry : ActiveRequestRegistry) @@ -388,45 +410,61 @@ def unregister match active? with | none => pure () | some active => - registry.mutex.atomically do + let shouldResolve ← registry.mutex.atomically do let state ← get - match active.clientRequestId? with - | some clientRequestId => - match state.requests.get? clientRequestId with - | some current => - if current.token == active.token then - set { state with requests := state.requests.erase clientRequestId } - | none => pure () - | none => - match state.anonymousRequests.get? active.token with - | some current => - if current.token == active.token then - set { state with anonymousRequests := state.anonymousRequests.erase active.token } - | none => pure () + let state := + match active.clientRequestId? with + | some clientRequestId => + match state.requests.get? clientRequestId with + | some current => + if current.token == active.token then + { state with requests := state.requests.erase clientRequestId } + else + state + | none => state + | none => + match state.anonymousRequests.get? active.token with + | some current => + if current.token == active.token then + { state with anonymousRequests := state.anonymousRequests.erase active.token } + else + state + | none => state + let (state, shouldResolve) := markDrainedIfReady state + set state + pure shouldResolve + resolveDrainedIfNeeded registry shouldResolve def count (registry : ActiveRequestRegistry) : IO Nat := do registry.mutex.atomically do - let state ← get - pure (state.requests.size + state.anonymousRequests.size) + pure (activeRequestCount (← get)) /-- Atomically close request admission and mark every admitted request for cancellation. Return `true` only to the caller that changed the registry from accepting to closed. -/ def closeAdmission (registry : ActiveRequestRegistry) : IO Bool := do - let (firstClose, active) ← registry.mutex.atomically do + let (firstClose, active, shouldResolve) ← registry.mutex.atomically do let state : ActiveRequestRegistryState ← get if !state.accepting then - pure (false, #[]) + pure (false, #[], false) else - set { state with accepting := false } + let (state, shouldResolve) := markDrainedIfReady { state with accepting := false } + set state let named := state.requests.toList.map Prod.snd |>.toArray let anonymous := state.anonymousRequests.toList.map Prod.snd |>.toArray - pure (true, named ++ anonymous) + pure (true, named ++ anonymous, shouldResolve) for request in active do request.cancelRef.set true + resolveDrainedIfNeeded registry shouldResolve pure firstClose +/-- Wait until admission is closed and every request admitted before closure has unregistered. -/ +def awaitDrained (registry : ActiveRequestRegistry) : IO Unit := do + let some _ ← IO.wait registry.drained.result? + | throw <| IO.userError "active request registry drain promise dropped" + pure () + def markCancelled (registry : ActiveRequestRegistry) (clientRequestId : String) : IO (Option ActiveRequest) := do diff --git a/Beam/Broker/Server.lean b/Beam/Broker/Server.lean index 742eef6e..194c0cff 100644 --- a/Beam/Broker/Server.lean +++ b/Beam/Broker/Server.lean @@ -888,6 +888,8 @@ structure ServerRuntime where endpoint : Transport.Endpoint stop : IO.Ref Bool activeRequests : ActiveRequestRegistry + private closeMutex : Std.Mutex Bool + private closeDone : IO.Promise (Except IO.Error Unit) /-- A cancellation capability bound to one active broker request admission. @@ -925,6 +927,8 @@ def ServerRuntime.create endpoint := endpoint stop := ← IO.mkRef false activeRequests := ← ActiveRequestRegistry.create + closeMutex := ← Std.Mutex.new false + closeDone := ← IO.Promise.new } private def brokerConfigSame (left right : BrokerConfig) : Bool := @@ -938,6 +942,72 @@ private def shutdownWorkspaceSessions (workspace : WorkspaceState) : IO Unit := if let some session := session? then shutdownSession session +private def detachBackendSession + (backend : BackendState) : BackendState × Option Session := + match backend.session? with + | none => (backend, none) + | some session => + ({ backend with session? := none, nextEpoch := backend.nextEpoch + 1 }, some session) + +private def detachRuntimeSessions (server : ServerRuntime) : IO (Array Session) := do + server.withState do + let state ← get + let mut sessions := #[] + for (workspaceId, workspace) in state.workspaces.toList do + let (lean, leanSession?) := detachBackendSession workspace.lean + let (rocq, rocqSession?) := detachBackendSession workspace.rocq + if let some session := leanSession? then + sessions := sessions.push session + if let some session := rocqSession? then + sessions := sessions.push session + modify fun state => setWorkspace state workspaceId { workspace with lean, rocq } + pure sessions + +private def shutdownRuntimeSessions (server : ServerRuntime) : IO Unit := do + for session in ← detachRuntimeSessions server do + shutdownSession session + +private def awaitRuntimeClose + (promise : IO.Promise (Except IO.Error Unit)) : IO Unit := do + let some outcome ← IO.wait promise.result? + | throw <| IO.userError "broker runtime close promise dropped" + match outcome with + | .ok () => pure () + | .error err => throw err + +/-- +Close broker admission, cancel admitted requests, shut down every backend session, and wait for +all admitted dispatch scopes to unregister. Concurrent and repeated callers wait for the same +close result; only the caller that started closure receives `true`. +-/ +def ServerRuntime.close (server : ServerRuntime) : IO Bool := do + let leadsClose ← server.closeMutex.atomically do + if ← get then + pure false + else + set true + pure true + if leadsClose then + let outcome ← + try + discard <| ActiveRequestRegistry.closeAdmission server.activeRequests + -- The first sweep unblocks requests already waiting on a backend. An admitted request may + -- have been between admission and session creation when closure began, so repeat the sweep + -- after every dispatch scope has drained to guarantee that no late session survives. + shutdownRuntimeSessions server + ActiveRequestRegistry.awaitDrained server.activeRequests + shutdownRuntimeSessions server + pure (.ok () : Except IO.Error Unit) + catch err => + pure (.error err) + server.closeDone.resolve outcome + match outcome with + | .ok () => pure true + | .error err => throw err + else + awaitRuntimeClose server.closeDone + pure false + def workspaceInitResult (workspaceId : WorkspaceId) (root : System.FilePath) @@ -2127,16 +2197,11 @@ private def handleRequestIO let cancelRef? := activeRequest?.map (·.cancelRef) match req.op with | .shutdown => - let firstClose ← ActiveRequestRegistry.closeAdmission server.activeRequests - if firstClose then - let resp ← server.withState do - let state ← get - for (_, workspace) in state.workspaces.toList do - shutdownWorkspaceSessions workspace - pure <| Response.success (Json.mkObj [("shutdown", toJson true)]) - pure (resp, true) - else - pure (Response.success (Json.mkObj [("shutdown", toJson true)]), false) + let firstClose ← server.close + pure ( + Response.success (Json.mkObj [("shutdown", toJson true)]), + firstClose + ) | .stats => match req.workspaceId? with | none => pure (← server.statsResponse, false) @@ -2332,7 +2397,7 @@ private partial def watchRoot (server : ServerRuntime) (root : System.FilePath) pure false if !rootAvailable then IO.eprintln s!"Beam daemon root is no longer available; shutting down: {root}" - let (_, shouldStop) ← server.dispatchRequest { op := .shutdown } + let shouldStop ← server.close if shouldStop then requestStop server else @@ -2345,7 +2410,7 @@ private def watchSessionOwnerStdin (server : ServerRuntime) : IO Unit := do catch _ => pure () unless ← server.stop.get do - let (_, shouldStop) ← server.dispatchRequest { op := .shutdown } + let shouldStop ← server.close if shouldStop then requestStop server @@ -2498,5 +2563,6 @@ def main (args : List String) : IO Unit := do if let some ownerWatcher := ownerWatcher? then IO.cancel ownerWatcher discard <| IO.wait rootWatcher + discard <| runtime.close end Beam.Broker diff --git a/Beam/Mcp/StdioServer.lean b/Beam/Mcp/StdioServer.lean index b212db80..a83ec724 100644 --- a/Beam/Mcp/StdioServer.lean +++ b/Beam/Mcp/StdioServer.lean @@ -315,7 +315,7 @@ private def Coordinator.closeTransport (coordinator : Coordinator) : IO Unit := match application.runtime? with | none => pure () | some runtime => - discard <| runtime.dispatchRequest { op := .shutdown } + discard <| runtime.close private def Coordinator.admitToolRequest (coordinator : Coordinator) diff --git a/docs/DEVELOPMENT.md b/docs/DEVELOPMENT.md index c2d2171e..7ce37043 100644 --- a/docs/DEVELOPMENT.md +++ b/docs/DEVELOPMENT.md @@ -300,6 +300,14 @@ Keep ordinary daemon and CLI dispatch on directly. Pending LSP requests must retain the same per-admission cancellation identity; after a handle has been validated, never fall back to matching a reusable client request ID. +`ServerRuntime.close` is the shared runtime teardown boundary. It closes admission, marks every +admitted request for cancellation, shuts down backend sessions to unblock pending work, waits for +all admitted dispatch scopes to unregister, and performs a final backend sweep so a request that +was between admission and session creation cannot leave a late process behind. Concurrent and +repeated callers wait for the same result. Transport owners decide what triggers closure and how +their listener or stdio connection stops; they must not duplicate broker draining or backend +teardown. + The thick part of the broker is request orchestration. For `sync`, `runAt`, `goals`, `runWith`, `release`, and `save`, the broker reads the source file, updates the LSP document mirror, waits for the relevant diagnostics/progress barrier when needed, asks the backend for semantic facts, and diff --git a/tests/lean/BeamTest/Broker/PendingTest.lean b/tests/lean/BeamTest/Broker/PendingTest.lean index c27d6be8..3e2ed517 100644 --- a/tests/lean/BeamTest/Broker/PendingTest.lean +++ b/tests/lean/BeamTest/Broker/PendingTest.lean @@ -122,6 +122,38 @@ private def checkActiveRegistry : IO Unit := do failure ActiveRequestRegistry.unregister registry (some replacement) +private def checkActiveRegistryCloseDrain : IO Unit := do + let registry ← ActiveRequestRegistry.create + let named ← expectRegistered "register named request before close" <| + ← ActiveRequestRegistry.register registry (some "closing-request") + let anonymous ← expectRegistered "register anonymous request before close" <| + ← ActiveRequestRegistry.register registry none + require "first admission close should lead closure" + (← ActiveRequestRegistry.closeAdmission registry) + for active in #[named, anonymous] do + match ← ensureRequestNotCancelled (some active.cancelRef) with + | .ok _ => throw <| IO.userError "admission close did not cancel an active request" + | .error failure => + discard <| requireFailureCode + "admission close cancellation" + "requestCancelled" + failure + match ← ActiveRequestRegistry.register registry (some "after-close") with + | .ok _ => throw <| IO.userError "closed admission accepted a new request" + | .error failure => + require "closed admission rejection is typed" (failure.code == .requestCancelled) + let drainTask ← IO.asTask (prio := Task.Priority.dedicated) <| + ActiveRequestRegistry.awaitDrained registry + ActiveRequestRegistry.unregister registry (some named) + IO.sleep 10 + require "drain should wait for every admitted request" (!(← IO.hasFinished drainTask)) + ActiveRequestRegistry.unregister registry (some anonymous) + match ← IO.wait drainTask with + | .ok () => pure () + | .error err => throw err + require "repeated admission close should be idempotent" + (!(← ActiveRequestRegistry.closeAdmission registry)) + private def checkPendingCancellationIdentity : IO Unit := do let registry ← ActiveRequestRegistry.create let firstResult ← ActiveRequestRegistry.register registry (some "reused-id") @@ -414,6 +446,7 @@ private def checkSetupFileProgressStreamsByScope : IO Unit := do def main : IO Unit := do checkActiveRegistry + checkActiveRegistryCloseDrain checkPendingCancellationIdentity checkPendingStoreResolve checkPendingStoreFailAllRespectingCancellation diff --git a/tests/lean/BeamTest/Broker/ProtocolTest.lean b/tests/lean/BeamTest/Broker/ProtocolTest.lean index f25d2aef..0a0b82c0 100644 --- a/tests/lean/BeamTest/Broker/ProtocolTest.lean +++ b/tests/lean/BeamTest/Broker/ProtocolTest.lean @@ -862,16 +862,48 @@ private def checkWorkspaceLifecycleProtocol : IO Unit := do require "typed workspace drop preserves lifecycle state" (decodedDrop.workspaceId == "fixture" && decodedDrop.dropped && decodedDrop.invalidatedHandles) +private partial def waitForCancellation + (cancelRef : IO.Ref Bool) + (tries : Nat := 100) : IO Unit := do + if ← cancelRef.get then + pure () + else if tries == 0 then + throw <| IO.userError "timed out waiting for runtime close cancellation" + else + IO.sleep 10 + waitForCancellation cancelRef (tries - 1) + private def checkSessionCloseAdmission : IO Unit := do let root := System.FilePath.mk "/tmp/beam-session-close-admission" let runtime ← Beam.Broker.ServerRuntime.create ({ root } : Beam.Broker.BrokerConfig) "fixture" (.tcp 0) let (beforeClose, _) ← runtime.dispatchRequest { op := .stats } require "stats should be admitted before session close" beforeClose.ok - require "first session close should win admission shutdown" - (← ActiveRequestRegistry.closeAdmission runtime.activeRequests) + let active ← + match ← ActiveRequestRegistry.register runtime.activeRequests (some "close-drain") with + | .ok active => pure active + | .error failure => throw <| IO.userError failure.message + let closeTask ← IO.asTask (prio := Task.Priority.dedicated) runtime.close + waitForCancellation active.cancelRef + let concurrentCloseTask ← IO.asTask (prio := Task.Priority.dedicated) runtime.close + IO.sleep 10 + require "runtime close should wait for admitted dispatch scopes" + (!(← IO.hasFinished closeTask)) + require "concurrent runtime close should share the same drain" + (!(← IO.hasFinished concurrentCloseTask)) + ActiveRequestRegistry.unregister runtime.activeRequests (some active) + let firstClose ← + match ← IO.wait closeTask with + | .ok firstClose => pure firstClose + | .error err => throw err + require "first runtime close should lead shutdown" firstClose + let concurrentClose ← + match ← IO.wait concurrentCloseTask with + | .ok concurrentClose => pure concurrentClose + | .error err => throw err + require "concurrent runtime close should not lead shutdown" (!concurrentClose) require "repeated session close should be idempotent" - (!(← ActiveRequestRegistry.closeAdmission runtime.activeRequests)) + (!(← runtime.close)) let (afterClose, _) ← runtime.dispatchRequest { op := .stats } require "ordinary requests should be rejected after session close" (afterClose.error?.any fun err => err.code == "requestCancelled") diff --git a/tests/lean/BeamTest/Broker/StreamDedupTest.lean b/tests/lean/BeamTest/Broker/StreamDedupTest.lean index cf495276..7f528731 100644 --- a/tests/lean/BeamTest/Broker/StreamDedupTest.lean +++ b/tests/lean/BeamTest/Broker/StreamDedupTest.lean @@ -158,15 +158,13 @@ private def fakeServerWithLeanSession config lean := { nextEpoch := 1, session? := some session } } - pure { - state := ← Std.Mutex.new { + let server ← Beam.Broker.ServerRuntime.create config fixtureWorkspaceId (.tcp 0) + server.state.atomically do + set ({ bootstrapConfig := config workspaces := Std.TreeMap.empty.insert fixtureWorkspaceId workspace - } - endpoint := .tcp 0 - stop := ← IO.mkRef false - activeRequests := ← Beam.Broker.ActiveRequestRegistry.create - } + } : Beam.Broker.State) + pure server def checkRunAtStreamsSetupDiagnostics : IO Unit := do let rootBase := System.FilePath.mk s!"/tmp/beam-daemon-run-at-stream-{← IO.monoNanosNow}" From 3d3b343b18b0f41d9de78e9b9a403a81ddb6e732 Mon Sep 17 00:00:00 2001 From: Emilio Jesus Gallego Arias Date: Wed, 26 Aug 2026 05:50:53 +0200 Subject: [PATCH 06/28] refactor: make broker authoritative for MCP workspaces --- Beam/Broker/Server.lean | 7 ++ Beam/Mcp/Server.lean | 68 ++++++------------- Beam/Mcp/StdioServer.lean | 3 +- docs/DEVELOPMENT.md | 4 +- .../lean/BeamTest/Broker/McpProtocolTest.lean | 27 ++++---- tests/lean/BeamTest/Broker/ProtocolTest.lean | 8 +++ 6 files changed, 52 insertions(+), 65 deletions(-) diff --git a/Beam/Broker/Server.lean b/Beam/Broker/Server.lean index 194c0cff..383fc317 100644 --- a/Beam/Broker/Server.lean +++ b/Beam/Broker/Server.lean @@ -908,6 +908,13 @@ def ServerRuntime.withState (server : ServerRuntime) (act : M α) : IO α := do set state pure a +/-- Return the canonical root currently owned by `workspaceId`, if that workspace exists. -/ +def ServerRuntime.workspaceRoot? + (server : ServerRuntime) + (workspaceId : WorkspaceId) : IO (Option System.FilePath) := do + server.withState do + pure <| (getWorkspace? (← get) workspaceId).map (·.config.root) + private def ServerRuntime.statsResponse (server : ServerRuntime) (workspaceId? : Option WorkspaceId := none) : IO Response := do diff --git a/Beam/Mcp/Server.lean b/Beam/Mcp/Server.lean index 12b883a0..ca9c39a9 100644 --- a/Beam/Mcp/Server.lean +++ b/Beam/Mcp/Server.lean @@ -44,33 +44,21 @@ inductive ProtocolState where | modern deriving Repr -structure ApplicationState where - workspaces : Std.TreeMap Beam.Broker.WorkspaceId System.FilePath := {} - runtime? : Option Beam.Broker.ServerRuntime := none - structure ServerState where protocol : Std.Mutex ProtocolState - application : IO.Ref ApplicationState + private runtime : IO.Ref (Option Beam.Broker.ServerRuntime) def ServerState.create : IO ServerState := do pure { protocol := ← Std.Mutex.new .undecided - application := ← IO.mkRef {} + runtime := ← IO.mkRef none } def ServerState.protocolState (state : ServerState) : IO ProtocolState := state.protocol.atomically get -def ServerState.applicationState (state : ServerState) : IO ApplicationState := - state.application.get - -private def ApplicationState.trackWorkspace - (state : ApplicationState) - (workspaceId : Beam.Broker.WorkspaceId) - (root : System.FilePath) : ApplicationState := { - state with - workspaces := state.workspaces.insert workspaceId root -} +def ServerState.runtime? (state : ServerState) : IO (Option Beam.Broker.ServerRuntime) := + state.runtime.get structure NotificationSink where send : Json → IO Unit := fun _ => pure () @@ -468,13 +456,11 @@ private def createRuntimeForRoot | .error err => pure <| .error err private def ensureBrokerWorkspace - (state : ServerState) (opts : Options) (runtime : Beam.Broker.ServerRuntime) (workspaceId : Beam.Broker.WorkspaceId) (root : System.FilePath) : IO (Except RpcError System.FilePath) := do - let application ← state.applicationState - match application.workspaces.get? workspaceId with + match ← runtime.workspaceRoot? workspaceId with | some trackedRoot => if trackedRoot == root then pure <| .ok root @@ -487,10 +473,7 @@ private def ensureBrokerWorkspace | .ok config => let brokerResp ← runtime.initWorkspaceWithConfig workspaceId config (some .set) match brokerResp with - | .successResult .. => - state.application.modify fun application => - application.trackWorkspace workspaceId config.root - pure <| .ok config.root + | .successResult .. => pure <| .ok config.root | .errorResult failure => pure <| .error <| RpcError.invalidRequest failure.error.message @@ -501,20 +484,16 @@ private def ensureRuntimeForWorkspace (workspaceId : Beam.Broker.WorkspaceId) (root : System.FilePath) : IO (Except RpcError (Beam.Broker.ServerRuntime × System.FilePath)) := do setupMutex.atomically do - let application ← state.applicationState - match application.runtime? with + match ← state.runtime? with | some runtime => - match ← ensureBrokerWorkspace state opts runtime workspaceId root with + match ← ensureBrokerWorkspace opts runtime workspaceId root with | .ok canonicalRoot => pure <| .ok (runtime, canonicalRoot) | .error err => pure <| .error err | none => match ← createRuntimeForRoot opts workspaceId root with | .error err => pure <| .error err | .ok (runtime, canonicalRoot) => - state.application.modify fun application => { - application.trackWorkspace workspaceId canonicalRoot with - runtime? := some runtime - } + state.runtime.set (some runtime) pure <| .ok (runtime, canonicalRoot) private def workspaceErrorToToolError (err : Beam.Workspace.RootError) : ToolError := @@ -572,7 +551,7 @@ private def decodeBeamStatsResult (json : Json) : Except String BeamStatsResult private def handleBeamStats (state : ServerState) : IO Json := do let runtime ← - match (← state.applicationState).runtime? with + match ← state.runtime? with | some runtime => pure runtime | none => return callToolResult <| Json.mkObj [ @@ -598,12 +577,6 @@ private def handleBeamStats private def handleDropWorkspace (state : ServerState) (workspace : ResolvedWorkspace) : IO Json := do - let application ← state.applicationState - let updateTrackedState : IO Unit := - state.application.modify fun application => { - application with - workspaces := application.workspaces.erase workspace.workspaceId - } let resultJson (dropped invalidatedHandles : Bool) (reason? : Option String := none) : Json := Json.mkObj <| [ ("workspace", toJson workspace.descriptor), @@ -612,7 +585,7 @@ private def handleDropWorkspace ] ++ match reason? with | some reason => [("reason", toJson reason)] | none => [] - match application.runtime? with + match ← state.runtime? with | none => pure <| callToolResult <| resultJson false false (some "notFound") | some runtime => @@ -624,8 +597,6 @@ private def handleDropWorkspace | .successResult payload .. => match fromJson? (α := Beam.Workspace.DropResult) payload with | .ok dropped => - if dropped.dropped then - updateTrackedState pure <| callToolResult <| resultJson dropped.dropped dropped.invalidatedHandles dropped.reason? | .error err => @@ -664,8 +635,7 @@ def Internal.serverVersionText (opts : Options) : IO String := do private def handleBeamVersion (state : ServerState) (opts : Options) : IO Json := do - let application ← state.applicationState - let identity ← serverIdentity opts none (some application.runtime?.isSome) + let identity ← serverIdentity opts none (some (← state.runtime?).isSome) pure <| callToolResult identity.asJson private def collectFeedbackRuntimePayload @@ -850,12 +820,14 @@ def Internal.handleToolCall if params.name == .beamFeedbackReport then let reporter ← CallReporter.create notifier req.id params progress? try - let application ← state.applicationState - let selectedRuntime? := - if application.workspaces.get? workspace.workspaceId == some workspace.root then - application.runtime? - else - none + let selectedRuntime? ← + match ← state.runtime? with + | none => pure none + | some runtime => + if (← runtime.workspaceRoot? workspace.workspaceId) == some workspace.root then + pure (some runtime) + else + pure none let result ← handleBeamFeedback opts workspace.descriptor workspace.workspaceId workspace.root selectedRuntime? params.arguments progress? Internal.traceMcp s!"tools/call feedback complete id={req.id.label} tool={params.name.key}" diff --git a/Beam/Mcp/StdioServer.lean b/Beam/Mcp/StdioServer.lean index a83ec724..79315db6 100644 --- a/Beam/Mcp/StdioServer.lean +++ b/Beam/Mcp/StdioServer.lean @@ -311,8 +311,7 @@ private def Coordinator.closeTransport (coordinator : Coordinator) : IO Unit := coordinator.awaitRequests requests unless alreadyClosing do coordinator.setupMutex.atomically do - let application ← coordinator.state.applicationState - match application.runtime? with + match ← coordinator.state.runtime? with | none => pure () | some runtime => discard <| runtime.close diff --git a/docs/DEVELOPMENT.md b/docs/DEVELOPMENT.md index 7ce37043..7202d88c 100644 --- a/docs/DEVELOPMENT.md +++ b/docs/DEVELOPMENT.md @@ -168,7 +168,9 @@ The local descriptor lives in [Beam/Workspace/Protocol.lean](../Beam/Workspace/Protocol.lean). Every workspace-bound request must carry `{"workspace":{"root":"/absolute/project"}}`. Resolve it through [Beam/Lean/Workspace.lean](../Beam/Lean/Workspace.lean), canonicalize it before deriving the private -broker cache key, and never store a current/default workspace in MCP protocol state. +broker cache key, and never store a current/default workspace in MCP protocol state. MCP server +state owns only the optional shared `ServerRuntime`; workspace membership and canonical roots remain +broker-owned and must be observed through typed broker queries rather than a transport-side mirror. The executable path is split into importable modules: diff --git a/tests/lean/BeamTest/Broker/McpProtocolTest.lean b/tests/lean/BeamTest/Broker/McpProtocolTest.lean index aafde6de..903f7cc9 100644 --- a/tests/lean/BeamTest/Broker/McpProtocolTest.lean +++ b/tests/lean/BeamTest/Broker/McpProtocolTest.lean @@ -856,11 +856,8 @@ private def checkModernProtocol : IO Unit := do requireModernResultEnvelope "modern confidential beam_feedback_report result" feedbackResult requireConfidentialFeedbackResult "modern confidential beam_feedback_report" confidentialSecret feedbackResult - let stateAfterFeedback ← state.applicationState require "modern beam_feedback_report should not create a broker runtime" - stateAfterFeedback.runtime?.isNone - require "modern beam_feedback_report should not create a workspace cache" - stateAfterFeedback.workspaces.toList.isEmpty + (← state.runtime?).isNone let preservedMetaResult := Beam.Mcp.modernResult <| Json.mkObj [ ("_meta", Json.mkObj [("example.test/value", toJson "preserved")]) @@ -1166,11 +1163,8 @@ private def checkServerBasics : IO Unit := do requireConfidentialFeedbackResult "beam feedback confidential" confidentialSecret feedbackConfidentialResult - let stateAfterFeedback ← state.applicationState require "beam_feedback_report should not create a broker runtime" - stateAfterFeedback.runtime?.isNone - require "beam_feedback_report should not create a workspace cache" - stateAfterFeedback.workspaces.toList.isEmpty + (← state.runtime?).isNone let uncachedDropResp ← handleRpcRequest state opts "drop uncached workspace" 24 "tools/call" <| some <| toolCallParams "lean_drop_workspace" <| withWorkspace root (Json.mkObj []) @@ -1188,11 +1182,8 @@ private def checkServerBasics : IO Unit := do uncachedDropStructured requireJsonString "drop uncached workspace structured result" "reason" "notFound" uncachedDropStructured - let stateAfterUncachedDrop ← state.applicationState require "dropping an uncached workspace should not create a broker runtime" - stateAfterUncachedDrop.runtime?.isNone - require "dropping an uncached workspace should not create a workspace cache" - stateAfterUncachedDrop.workspaces.toList.isEmpty + (← state.runtime?).isNone let rawToolResp ← handleRpcRequest state opts "raw tool rejection" 3 "tools/call" <| some <| toolCallParams Beam.LSP.RunAt.method @@ -1407,10 +1398,10 @@ private def callLeanSync some <| toolCallParams "lean_sync" arguments private def shutdownMcpRuntime (state : Beam.Mcp.Server.ServerState) : IO Unit := do - match (← state.applicationState).runtime? with + match ← state.runtime? with | none => pure () | some runtime => - discard <| runtime.dispatchRequest { op := .shutdown } + discard <| runtime.close private def checkIdempotentLifecycleTools : IO Unit := do let root ← mkTempProjectRoot "lean-beam-mcp-idempotent-lifecycle" @@ -1426,6 +1417,12 @@ private def checkIdempotentLifecycleTools : IO Unit := do let syncResp ← callLeanSync state opts notifications root 2 "SaveSmoke/B.lean" let syncResult ← requireObjVal "lifecycle lean_sync response" "result" syncResp requireJsonBool "lifecycle lean_sync result" "isError" false syncResult + let canonicalRoot ← Beam.resolveExistingPath root + let workspaceId := (Beam.Workspace.Descriptor.ofRoot canonicalRoot).cacheKey + let some runtime ← state.runtime? + | throw <| IO.userError "lifecycle lean_sync did not create a broker runtime" + require "MCP lifecycle should read workspace ownership from the broker" + ((← runtime.workspaceRoot? workspaceId) == some canonicalRoot) for (id, label) in #[(3, "first"), (4, "repeated")] do let closeResp ← handleRpcRequestWithNotifications state opts notifications @@ -1449,6 +1446,8 @@ private def checkIdempotentLifecycleTools : IO Unit := do requireJsonBool "first lean_drop_workspace structured result" "dropped" true firstDropStructured requireJsonBool "first lean_drop_workspace structured result" "invalidated_handles" true firstDropStructured + require "MCP workspace drop should update broker-owned workspace state" + ((← runtime.workspaceRoot? workspaceId) == none) let repeatedDropResp ← handleRpcRequestWithNotifications state opts notifications "repeated lean drop workspace" 6 "tools/call" <| diff --git a/tests/lean/BeamTest/Broker/ProtocolTest.lean b/tests/lean/BeamTest/Broker/ProtocolTest.lean index 0a0b82c0..3f22ebe8 100644 --- a/tests/lean/BeamTest/Broker/ProtocolTest.lean +++ b/tests/lean/BeamTest/Broker/ProtocolTest.lean @@ -792,6 +792,10 @@ private def checkWorkspaceLifecycleProtocol : IO Unit := do let runtime ← Beam.Broker.ServerRuntime.create ({ root } : Beam.Broker.BrokerConfig) "fixture" + require "broker workspace query should expose the constructor workspace" + ((← runtime.workspaceRoot? "fixture") == some root) + require "broker workspace query should reject an unknown workspace" + ((← runtime.workspaceRoot? "unknown") == none) for op in #[Op.ensure, .initWorkspace, .dropWorkspace] do let (missingWorkspaceResp, _) ← runtime.dispatchRequest { op } require s!"{op.key} should reject omitted workspace identity" @@ -861,6 +865,10 @@ private def checkWorkspaceLifecycleProtocol : IO Unit := do fromJson? (α := Beam.Workspace.DropResult) dropJson require "typed workspace drop preserves lifecycle state" (decodedDrop.workspaceId == "fixture" && decodedDrop.dropped && decodedDrop.invalidatedHandles) + let dropResp ← runtime.dropWorkspace "fixture" + require "broker workspace drop should succeed" dropResp.ok + require "broker workspace query should observe a dropped workspace" + ((← runtime.workspaceRoot? "fixture") == none) private partial def waitForCancellation (cancelRef : IO.Ref Bool) From 8d42ff7e7be869425eb01bb971b4f0c3e9579011 Mon Sep 17 00:00:00 2001 From: Emilio Jesus Gallego Arias Date: Wed, 26 Aug 2026 07:38:29 +0200 Subject: [PATCH 07/28] fix: preserve cancellation during runtime shutdown --- Beam/Broker/Pending.lean | 45 ++++++---- Beam/Broker/Server.lean | 56 ++++++------ docs/DEVELOPMENT.md | 5 +- tests/lean/BeamTest/Broker/PendingTest.lean | 97 +++++++++++++++------ 4 files changed, 129 insertions(+), 74 deletions(-) diff --git a/Beam/Broker/Pending.lean b/Beam/Broker/Pending.lean index 702d2400..089752ac 100644 --- a/Beam/Broker/Pending.lean +++ b/Beam/Broker/Pending.lean @@ -96,11 +96,34 @@ def resolveError catch _ => pure () -def awaitOutcome (promise : IO.Promise (Except ResponseFailure PendingResult)) : - IO (Except ResponseFailure PendingResult) := do - let some result ← IO.wait promise.result? +/-- Give an already-marked broker cancellation precedence over a concurrent backend failure. -/ +private def failureRespectingCancellation + (cancelRef? : Option (IO.Ref Bool)) + (fallback : ResponseFailure) : IO ResponseFailure := do + let cancelled ← + match cancelRef? with + | some cancelRef => cancelRef.get + | none => pure false + if cancelled then + pure <| + (responseFailureFor .requestCancelled + "request was cancelled before its backend failure was observed") + |>.withOptionalFileProgress fallback.fileProgress? + else + pure fallback + +/-- +Await the pending request, giving its already-marked cancellation identity precedence over a +concurrent backend failure while preserving observations attached to that failure. A completed +backend success remains successful. +-/ +def awaitOutcome (pending : PendingRequest) : IO (Except ResponseFailure PendingResult) := do + let some outcome ← IO.wait pending.promise.result? | throw <| IO.userError "pending broker request promise dropped" - pure result + match outcome with + | .ok result => pure (.ok result) + | .error failure => + pure (.error (← failureRespectingCancellation pending.cancelRef? failure)) private def normalizePublishDiagnostics (params : PublishDiagnosticsParams) : PublishDiagnosticsParams := { @@ -269,22 +292,10 @@ end PendingRequest namespace PendingRequestStore -/-- Fail every pending request, giving an already-marked cancellation token precedence. -/ -def failAllRespectingCancellation - (store : PendingRequestStore) - (fallback : ResponseFailure) : IO Unit := do +def failAll (store : PendingRequestStore) (failure : ResponseFailure) : IO Unit := do let pending ← clear store for req in pending do let progress? ← req.progressRef.get - let cancelled ← - match req.cancelRef? with - | some cancelRef => cancelRef.get - | none => pure false - let failure := - if cancelled then - responseFailureFor .requestCancelled "request was cancelled while its backend session closed" - else - fallback let failure := failure.withOptionalFileProgress progress? try req.promise.resolve (.error failure) diff --git a/Beam/Broker/Server.lean b/Beam/Broker/Server.lean index 383fc317..f5db9390 100644 --- a/Beam/Broker/Server.lean +++ b/Beam/Broker/Server.lean @@ -404,18 +404,17 @@ private def startWaitDiagnosticsWatchdog s!"waitForDiagnostics watchdog after {timeoutMs}ms: {label}" pure () -private def awaitPending (promise : IO.Promise (Except ResponseFailure PendingResult)) : - HandlerM PendingResult := do - requestArg (← liftHandlerIO <| PendingRequest.awaitOutcome promise) +private def awaitPending (pending : PendingRequest) : HandlerM PendingResult := do + requestArg (← liftHandlerIO pending.awaitOutcome) private def awaitWaitForDiagnosticsBarrier (label : String) - (promise : IO.Promise (Except ResponseFailure PendingResult)) : HandlerM PendingResult := do + (pending : PendingRequest) : HandlerM PendingResult := do let doneRef ← liftHandlerIO <| IO.mkRef false liftHandlerIO <| startWaitDiagnosticsWatchdog label doneRef let outcome ← liftHandlerIO <| do try - let outcome ← PendingRequest.awaitOutcome promise + let outcome ← pending.awaitOutcome doneRef.set true pure outcome catch e => @@ -459,7 +458,7 @@ partial def sessionReaderLoop (session : Session) : IO Unit := do pure () sessionReaderLoop session catch e => - PendingRequestStore.failAllRespectingCancellation session.pending <| BrokerFailure.toResponseFailure { + PendingRequestStore.failAll session.pending <| BrokerFailure.toResponseFailure { code := .workerExited message := e.toString } @@ -483,14 +482,14 @@ private def startRequestJsonTrackedDetailed (diagnosticScope : DiagnosticScope := .errors) (emitDiagnostic? : Option (StreamDiagnostic → IO Unit) := none) (cancelRef? : Option (IO.Ref Bool) := none) : - IO (Session × IO.Promise (Except ResponseFailure PendingResult)) := do + IO (Session × PendingRequest) := do let (session, id) := nextRequestId session let progressRef ← IO.mkRef (initialProgress? <|> tracked.map (fun _ => {})) let diagnosticsRef ← IO.mkRef #[] let diagnosticsSeenRef ← IO.mkRef false let seenDiagnosticKeysRef ← IO.mkRef ({} : Std.TreeSet String compare) let promise ← IO.Promise.new - PendingRequestStore.insert session.pending id { + let pending : PendingRequest := { cancelRef? := cancelRef? promise := promise tracked? := tracked @@ -503,12 +502,13 @@ private def startRequestJsonTrackedDetailed emitDiagnostic? := emitDiagnostic? : PendingRequest } + PendingRequestStore.insert session.pending id pending traceBroker s!"lsp request inserted id={id} method={method} clientRequestId={optionLabel clientRequestId?} tracked={tracked.isSome}" try writeLspRequest session.stdin ({ id, method, param : Lean.JsonRpc.Request Json }) traceBroker s!"lsp request sent id={id} method={method}" - pure (session, promise) + pure (session, pending) catch e => discard <| PendingRequestStore.remove session.pending id traceBroker s!"lsp request send failed id={id} method={method} error={e.toString}" @@ -529,10 +529,10 @@ def sendRequestJsonTrackedDetailed (diagnosticScope : DiagnosticScope := .errors) (emitDiagnostic? : Option (StreamDiagnostic → IO Unit) := none) : IO (Except ResponseFailure (Session × Json × Option SyncFileProgress × Array Diagnostic)) := do - let (session, promise) ← + let (session, pending) ← startRequestJsonTrackedDetailed session method param clientRequestId? tracked initialProgress? emitProgress? diagnosticScope emitDiagnostic? - match ← PendingRequest.awaitOutcome promise with + match ← pending.awaitOutcome with | .ok pending => pure <| .ok (session, pending.result, pending.progress?, pending.diagnostics) | .error failure => pure <| .error failure @@ -1255,7 +1255,7 @@ private structure StartedSyncedRequest where version : Nat priorProgress? : Option SyncFileProgress := none tracked : Option (DocumentUri × Nat) := none - promise : IO.Promise (Except ResponseFailure PendingResult) + pending : PendingRequest private def trackedDocumentVersion (uri : DocumentUri) (docState : DocState) : Option (DocumentUri × Nat) := @@ -1305,7 +1305,7 @@ private def startSyncedDocumentRequest pure () let tracked := trackedFor uri docState let params := mkParams uri docState - let (session, promise) ← + let (session, pending) ← startRequestJsonTrackedDetailed session method params (clientRequestId? := clientRequestId?) (tracked := tracked) @@ -1321,7 +1321,7 @@ private def startSyncedDocumentRequest version := docState.version priorProgress? := docState.fileProgress? tracked - promise + pending } private def awaitSyncedDocumentRequest @@ -1329,7 +1329,7 @@ private def awaitSyncedDocumentRequest (started : StartedSyncedRequest) (cancelRef? : Option (IO.Ref Bool) := none) : HandlerM PendingResult := do liftHandlerIO <| propagatePendingCancellation started.session cancelRef? - let pending ← awaitPending started.promise + let pending ← awaitPending started.pending if started.tracked.isSome then withFailureProgress pending.progress? <| liftHandlerIO <| mergeFileProgressIfCurrent server started.session started.uri pending.progress? @@ -1358,7 +1358,7 @@ private structure StartedTrackedBarrier where textMTime : Lake.MTime changed : Bool := false priorProgress? : Option SyncFileProgress := none - promise : IO.Promise (Except ResponseFailure PendingResult) + pending : PendingRequest private def startTrackedDiagnosticsBarrierIO (server : ServerRuntime) @@ -1378,7 +1378,7 @@ private def startTrackedDiagnosticsBarrierIO let tracked := trackedDocumentVersion uri docState let params := toJson (WaitForDiagnosticsParams.mk uri docState.version) let method ← IO.ofExcept <| diagnosticsBarrierMethod session.backend - let (session, promise) ← + let (session, pending) ← startRequestJsonTrackedDetailed session method params (clientRequestId? := req.clientRequestId?) (tracked := tracked) @@ -1397,7 +1397,7 @@ private def startTrackedDiagnosticsBarrierIO textMTime := docState.textMTime changed := synced.changed priorProgress? := docState.fileProgress? - promise + pending } private def finalizeSavedDoc @@ -1548,7 +1548,7 @@ private def saveOleanCore liftHandlerIO <| propagatePendingCancellation started.session cancelRef? let barrier ← awaitWaitForDiagnosticsBarrier s!"save_olean sync barrier clientRequestId={optionLabel req.clientRequestId?} uri={started.uri} version={started.version}" - started.promise + started.pending let barrierResult : DiagnosticsBarrierResult ← withFailureProgress barrier.progress? <| liftHandlerIO <| decodeResponseAs barrier.result if barrierResult.version != started.version then @@ -1616,18 +1616,18 @@ private def saveOleanCore -- Once artifact publication can begin, an older trace must not remain visible: it may have the -- same dependency hash while describing a different in-server artifact family. withFailureProgress barrierProgress? <| liftHandlerIO <| invalidateLeanSaveTrace spec - let (session, savePromise) ← withFailureProgress barrierProgress? <| + let (session, saveRequest) ← withFailureProgress barrierProgress? <| withCurrentMatchingSession server started.session fun current => do - let (current, savePromise) ← startRequestJsonTrackedDetailed current method params + let (current, saveRequest) ← startRequestJsonTrackedDetailed current method params (clientRequestId? := req.clientRequestId?) (cancelRef? := cancelRef?) updateSession current - pure (current, savePromise) + pure (current, saveRequest) withFailureProgress barrierProgress? <| liftHandlerIO <| propagatePendingCancellation session cancelRef? let savePending ← match ← withFailureProgress barrierProgress? <| - liftHandlerIO <| PendingRequest.awaitOutcome savePromise with + liftHandlerIO saveRequest.awaitOutcome with | .ok pending => pure pending | .error failure => throw <| ({ @@ -1693,7 +1693,7 @@ private def handleSyncFileOp liftHandlerIO <| propagatePendingCancellation started.session cancelRef? let pending ← awaitWaitForDiagnosticsBarrier s!"sync_file clientRequestId={optionLabel req.clientRequestId?} uri={started.uri} version={started.version}" - started.promise + started.pending liftHandlerIO <| traceBroker s!"sync_file barrier completed clientRequestId={optionLabel req.clientRequestId?} progress={pending.progress?.isSome} diagnostics={pending.diagnostics.size} diagnosticsSeen={pending.diagnosticsSeen}" let barrierResult : DiagnosticsBarrierResult ← @@ -1943,16 +1943,16 @@ private def handleWorkspaceSymbolsOp HandlerM (Response × Bool) := do let args ← requestArg req.workspaceSymbolsArgs liftFailureIO <| ensureRequestNotCancelled cancelRef? - let (session, promise) ← liftHandlerIO <| server.withState do + let (session, request) ← liftHandlerIO <| server.withState do let session ← ensureSession req.workspaceId req.backend let params := toJson ({ query := args.query : WorkspaceSymbolParams }) - let (session, promise) ← startRequestJsonTrackedDetailed session args.method params + let (session, request) ← startRequestJsonTrackedDetailed session args.method params (clientRequestId? := req.clientRequestId?) (cancelRef? := cancelRef?) updateSession session - pure (session, promise) + pure (session, request) liftHandlerIO <| propagatePendingCancellation session cancelRef? - let pending ← awaitPending promise + let pending ← awaitPending request pure (Response.success pending.result, false) private def codeActionResolveSourceUri diff --git a/docs/DEVELOPMENT.md b/docs/DEVELOPMENT.md index 7202d88c..58b9f057 100644 --- a/docs/DEVELOPMENT.md +++ b/docs/DEVELOPMENT.md @@ -300,7 +300,10 @@ inert after that lexical scope, including when a later request reuses the same c Keep ordinary daemon and CLI dispatch on `ServerRuntime.dispatchRequest`; transport layers must not mutate the active-request registry directly. Pending LSP requests must retain the same per-admission cancellation identity; after a -handle has been validated, never fall back to matching a reusable client request ID. +handle has been validated, never fall back to matching a reusable client request ID. Pass and await +the `PendingRequest` as one value instead of separating its promise from its cancellation reference. +Once that reference is marked, cancellation takes precedence over a concurrent backend failure; +an already-completed backend success remains successful. `ServerRuntime.close` is the shared runtime teardown boundary. It closes admission, marks every admitted request for cancellation, shuts down backend sessions to unblock pending work, waits for diff --git a/tests/lean/BeamTest/Broker/PendingTest.lean b/tests/lean/BeamTest/Broker/PendingTest.lean index 3e2ed517..10185a52 100644 --- a/tests/lean/BeamTest/Broker/PendingTest.lean +++ b/tests/lean/BeamTest/Broker/PendingTest.lean @@ -29,14 +29,13 @@ private def mkPending (tracked? : Option (DocumentUri × Nat) := none) (emitProgress? : Option (SyncFileProgress → IO Unit) := none) (diagnosticScope : DiagnosticScope := .errors) - (emitDiagnostic? : Option (StreamDiagnostic → IO Unit) := none) : - IO (PendingRequest × IO.Promise (Except ResponseFailure PendingResult)) := do + (emitDiagnostic? : Option (StreamDiagnostic → IO Unit) := none) : IO PendingRequest := do let promise ← IO.Promise.new let progressRef ← IO.mkRef progress? let diagnosticsRef ← IO.mkRef #[] let diagnosticsSeenRef ← IO.mkRef false let seenDiagnosticKeysRef ← IO.mkRef ({} : Std.TreeSet String compare) - pure ({ + pure { cancelRef? promise tracked? @@ -47,7 +46,7 @@ private def mkPending emitProgress? diagnosticScope emitDiagnostic? - }, promise) + } private def expectRegistered (label : String) @@ -161,8 +160,8 @@ private def checkPendingCancellationIdentity : IO Unit := do ActiveRequestRegistry.unregister registry (some first) let replacementResult ← ActiveRequestRegistry.register registry (some "reused-id") let replacement ← expectRegistered "register replacement cancellation identity" replacementResult - let (firstPending, _) ← mkPending (cancelRef? := some first.cancelRef) - let (replacementPending, _) ← mkPending (cancelRef? := some replacement.cancelRef) + let firstPending ← mkPending (cancelRef? := some first.cancelRef) + let replacementPending ← mkPending (cancelRef? := some replacement.cancelRef) require "first admission matches its pending request" (← PendingRequestStore.matchesCancellation firstPending first.cancelRef) require "first admission does not match replacement pending request" @@ -175,7 +174,7 @@ private def checkPendingCancellationIdentity : IO Unit := do private def checkPendingStoreResolve : IO Unit := do let store ← PendingRequestStore.create - let (pending, promise) ← mkPending + let pending ← mkPending (progress? := some { updates := 3, done := false }) let id : RequestID := 7 PendingRequestStore.insert store id pending @@ -185,7 +184,7 @@ private def checkPendingStoreResolve : IO Unit := do | throw <| IO.userError "pending store remove missed inserted request" PendingRequest.resolveResponse pending (Json.mkObj [("value", toJson true)]) let result ← - match ← PendingRequest.awaitOutcome promise with + match ← pending.awaitOutcome with | .ok result => pure result | .error failure => throw <| IO.userError @@ -198,44 +197,85 @@ private def checkPendingStoreResolve : IO Unit := do require "pending store is empty after remove" ((← PendingRequestStore.snapshot store).isEmpty) -private def checkPendingStoreFailAllRespectingCancellation : IO Unit := do +private def checkPendingStoreFailAll : IO Unit := do let store ← PendingRequestStore.create let firstProgress : SyncFileProgress := { updates := 5, done := false } let secondProgress : SyncFileProgress := { updates := 8, done := true } let cancelRef ← IO.mkRef true - let (firstPending, firstPromise) ← mkPending + let firstPending ← mkPending (progress? := some firstProgress) (cancelRef? := some cancelRef) - let (secondPending, secondPromise) ← mkPending (progress? := some secondProgress) + let secondPending ← mkPending (progress? := some secondProgress) PendingRequestStore.insert store 11 firstPending PendingRequestStore.insert store 12 secondPending - PendingRequestStore.failAllRespectingCancellation store + PendingRequestStore.failAll store (responseFailureFor .workerExited "worker exited") - for (label, promise, expectedCode, expectedProgress) in #[ - ("cancelled", firstPromise, "requestCancelled", firstProgress), - ("worker-exited", secondPromise, "workerExited", secondProgress) + for (label, pending, expectedCode, expectedProgress) in #[ + ("cancelled", firstPending, "requestCancelled", firstProgress), + ("worker-exited", secondPending, "workerExited", secondProgress) ] do - match ← PendingRequest.awaitOutcome promise with + match ← pending.awaitOutcome with | .ok _ => throw <| IO.userError - s!"failAllRespectingCancellation resolves {label} pending request as an error: expected error" + s!"failAll resolves {label} pending request as an error: expected error" | .error failure => discard <| requireFailureCode - s!"failAllRespectingCancellation resolves {label} pending request as an error" + s!"failAll resolves {label} pending request as an error" expectedCode failure - require s!"failAllRespectingCancellation preserves {label} pending request progress" + require s!"failAll preserves {label} pending request progress" (failure.fileProgress? == some expectedProgress) - require "failAllRespectingCancellation clears pending store" + require "failAll clears pending store" ((← PendingRequestStore.snapshot store).isEmpty) +private def checkPendingOutcomeCancellationPrecedence : IO Unit := do + let progress : SyncFileProgress := { updates := 13, done := true } + let backendFailure := + (responseFailureFor .contentModified "backend worker terminated") + |>.withOptionalFileProgress (some progress) + let cancelRef ← IO.mkRef true + let cancelledPending ← mkPending (cancelRef? := some cancelRef) + cancelledPending.promise.resolve (.error backendFailure) + match ← cancelledPending.awaitOutcome with + | .ok _ => + throw <| IO.userError "cancelled pending backend failure resolved as a success" + | .error failure => + discard <| requireFailureCode + "marked cancellation takes precedence over a backend failure" + "requestCancelled" + failure + require "cancellation precedence preserves backend failure progress" + (failure.fileProgress? == some progress) + + cancelRef.set false + let backendFailurePending ← mkPending (cancelRef? := some cancelRef) + backendFailurePending.promise.resolve (.error backendFailure) + match ← backendFailurePending.awaitOutcome with + | .ok _ => + throw <| IO.userError "uncancelled pending backend failure resolved as a success" + | .error failure => + discard <| requireFailureCode + "backend failure remains authoritative without cancellation" + "contentModified" + failure + + cancelRef.set true + let successPending ← mkPending (cancelRef? := some cancelRef) + successPending.promise.resolve (.ok { result := Json.mkObj [("completed", toJson true)] }) + match ← successPending.awaitOutcome with + | .error failure => + throw <| IO.userError + s!"completed backend success lost to later cancellation: {(toJson failure.toResponse).compress}" + | .ok result => + requireJsonBool "completed backend success remains authoritative" "completed" true result.result + private def checkPendingResolveError : IO Unit := do let expectedProgress : SyncFileProgress := { updates := 4, done := false } - let (pending, promise) ← mkPending (progress? := some expectedProgress) + let pending ← mkPending (progress? := some expectedProgress) let data := Json.mkObj [ ("expectedVersion", toJson (4 : Nat)), ("acceptedVersion", toJson (5 : Nat)) ] PendingRequest.resolveError pending .contentModified "document changed" (some data) - match ← PendingRequest.awaitOutcome promise with + match ← pending.awaitOutcome with | .ok _ => throw <| IO.userError "pending typed error resolved as a success" | .error failure => @@ -286,7 +326,7 @@ private def mkPublishDiagnostics (diagnostics : Array Diagnostic) : PublishDiagn private def observeFileProgress (progress : SyncFileProgress) (ranges : Array Range) : IO SyncFileProgress := do - let (pending, _) ← mkPending + let pending ← mkPending (progress? := some progress) (tracked? := some ("file:///workspace/Foo.lean", 1)) PendingRequest.observeProgress pending (mkFileProgress ranges) @@ -354,7 +394,7 @@ private def checkDiagnosticLineCanExceedProgressRange : IO Unit := do }) let farDiagnostic := mkDiagnostic (mkRange 20 2 20 8) "diagnostic beyond progress range" - let (pending, _) ← mkPending + let pending ← mkPending (progress? := some finished) (tracked? := some ("file:///workspace/Foo.lean", 1)) PendingRequest.observeDiagnostics @@ -373,7 +413,7 @@ private def observeStreamedDiagnostics (diagnosticScope : DiagnosticScope) (diagnostics : Array Diagnostic) : IO (Array StreamDiagnostic) := do let streamedRef ← IO.mkRef #[] - let (pending, _) ← mkPending + let pending ← mkPending (tracked? := some ("file:///workspace/Foo.lean", 1)) (diagnosticScope := diagnosticScope) (emitDiagnostic? := some fun diagnostic => @@ -386,7 +426,7 @@ private def observeStreamedDiagnostics private def checkDiagnosticEmitterFailureIsolation : IO Unit := do let diagnostic := mkDiagnostic (mkRange 1 0 1 4) "stream consumer disconnected" - let (pending, _) ← mkPending + let pending ← mkPending (tracked? := some ("file:///workspace/Foo.lean", 1)) (diagnosticScope := .all) (emitDiagnostic? := some fun _ => @@ -401,7 +441,7 @@ private def checkDiagnosticEmitterFailureIsolation : IO Unit := do ((← pending.diagnosticsRef.get).map (·.message) == #[diagnostic.message]) private def checkProgressEmitterFailureIsolation : IO Unit := do - let (pending, _) ← mkPending + let pending ← mkPending (progress? := some {}) (tracked? := some ("file:///workspace/Foo.lean", 1)) (emitProgress? := some fun _ => @@ -449,7 +489,8 @@ def main : IO Unit := do checkActiveRegistryCloseDrain checkPendingCancellationIdentity checkPendingStoreResolve - checkPendingStoreFailAllRespectingCancellation + checkPendingStoreFailAll + checkPendingOutcomeCancellationPrecedence checkPendingResolveError checkSyncFileProgressDisplay checkSyncFileProgressLines From 1a08a27b519eb51920f885039a312c5b796277eb Mon Sep 17 00:00:00 2001 From: Emilio Jesus Gallego Arias Date: Wed, 26 Aug 2026 15:23:39 +0200 Subject: [PATCH 08/28] refactor: detach workspace sessions before shutdown --- Beam/Broker/Server.lean | 181 ++++++++++++------- Beam/Mcp/Server.lean | 26 +-- docs/DEVELOPMENT.md | 9 + tests/lean/BeamTest/Broker/ProtocolTest.lean | 159 +++++++++++++++- 4 files changed, 292 insertions(+), 83 deletions(-) diff --git a/Beam/Broker/Server.lean b/Beam/Broker/Server.lean index f5db9390..faf0cffa 100644 --- a/Beam/Broker/Server.lean +++ b/Beam/Broker/Server.lean @@ -944,11 +944,6 @@ private def brokerConfigSame (left right : BrokerConfig) : Bool := left.leanPlugin? == right.leanPlugin? && left.rocqCmd? == right.rocqCmd? -private def shutdownWorkspaceSessions (workspace : WorkspaceState) : IO Unit := do - for session? in [workspace.lean.session?, workspace.rocq.session?] do - if let some session := session? then - shutdownSession session - private def detachBackendSession (backend : BackendState) : BackendState × Option Session := match backend.session? with @@ -956,18 +951,28 @@ private def detachBackendSession | some session => ({ backend with session? := none, nextEpoch := backend.nextEpoch + 1 }, some session) +private def collectSessions + (left? right? : Option Session) : Array Session := + match left?, right? with + | none, none => #[] + | some left, none => #[left] + | none, some right => #[right] + | some left, some right => #[left, right] + +private def detachWorkspaceSessions + (workspace : WorkspaceState) : WorkspaceState × Array Session := + let (lean, leanSession?) := detachBackendSession workspace.lean + let (rocq, rocqSession?) := detachBackendSession workspace.rocq + ({ workspace with lean, rocq }, collectSessions leanSession? rocqSession?) + private def detachRuntimeSessions (server : ServerRuntime) : IO (Array Session) := do server.withState do let state ← get - let mut sessions := #[] - for (workspaceId, workspace) in state.workspaces.toList do - let (lean, leanSession?) := detachBackendSession workspace.lean - let (rocq, rocqSession?) := detachBackendSession workspace.rocq - if let some session := leanSession? then - sessions := sessions.push session - if let some session := rocqSession? then - sessions := sessions.push session - modify fun state => setWorkspace state workspaceId { workspace with lean, rocq } + let (state, sessions) := state.workspaces.toList.foldl (init := (state, #[])) fun + (state, sessions) (workspaceId, workspace) => + let (workspace, detached) := detachWorkspaceSessions workspace + (setWorkspace state workspaceId workspace, sessions ++ detached) + set state pure sessions private def shutdownRuntimeSessions (server : ServerRuntime) : IO Unit := do @@ -1040,46 +1045,106 @@ private def duplicateRootWorkspace? else none -def ServerRuntime.initWorkspaceWithConfig - (server : ServerRuntime) +private structure WorkspaceTransition (α : Type) where + state : State + result : Except ResponseFailure α + detachedSessions : Array Session := #[] + +private def initWorkspaceTransition + (state : State) (workspaceId : WorkspaceId) (config : BrokerConfig) - (mode? : Option Beam.Workspace.InitMode := none) : IO Response := do + (mode? : Option Beam.Workspace.InitMode) : WorkspaceTransition Beam.Workspace.InitResult := if !validWorkspaceId workspaceId then - return errorResponseFor .invalidParams "workspace id must be non-empty" - let mode := mode?.getD .set - server.withState do - let state ← get + { state, result := .error <| responseFailureFor .invalidParams + "workspace id must be non-empty" } + else + let mode := mode?.getD .set match getWorkspace? state workspaceId with | some current => if mode == .reset then if let some otherId := duplicateRootWorkspace? state workspaceId config then - pure <| errorResponseFor .invalidParams <| - s!"workspace root {config.root} is already owned by workspace '{otherId}'" + { state, result := .error <| responseFailureFor .invalidParams <| + s!"workspace root {config.root} is already owned by workspace '{otherId}'" } else - shutdownWorkspaceSessions current + let (_, detachedSessions) := detachWorkspaceSessions current let replacement := mkWorkspaceState config - modify fun state => setWorkspace state workspaceId replacement - pure <| Response.success <| toJson <| - workspaceInitResult workspaceId config.root mode false true (some current.config.root) + { + state := setWorkspace state workspaceId replacement + result := .ok <| + workspaceInitResult workspaceId config.root mode false true + (some current.config.root) + detachedSessions + } else if brokerConfigSame current.config config then - pure <| Response.success <| toJson <| - workspaceInitResult workspaceId current.config.root mode true false + { state, result := .ok <| + workspaceInitResult workspaceId current.config.root mode true false } else - pure <| errorResponseFor .invalidParams <| - s!"workspace '{workspaceId}' is already initialized for {current.config.root}; " ++ - s!"use workspaceMode=reset to switch it explicitly to {config.root}" + { state, result := .error <| responseFailureFor .invalidParams <| + s!"workspace '{workspaceId}' is already initialized for {current.config.root}; " ++ + s!"use workspaceMode=reset to switch it explicitly to {config.root}" } | none => if mode == .verify then - pure <| errorResponseFor .invalidParams - s!"workspace '{workspaceId}' is not initialized; use workspaceMode=set first" + { state, result := .error <| responseFailureFor .invalidParams <| + s!"workspace '{workspaceId}' is not initialized; use workspaceMode=set first" } else if let some otherId := duplicateRootWorkspace? state workspaceId config then - pure <| errorResponseFor .invalidParams <| - s!"workspace root {config.root} is already owned by workspace '{otherId}'" + { state, result := .error <| responseFailureFor .invalidParams <| + s!"workspace root {config.root} is already owned by workspace '{otherId}'" } else - modify fun state => setWorkspace state workspaceId (mkWorkspaceState config) - pure <| Response.success <| toJson <| - workspaceInitResult workspaceId config.root mode false false + { + state := setWorkspace state workspaceId (mkWorkspaceState config) + result := .ok <| workspaceInitResult workspaceId config.root mode false false + } + +private def dropWorkspaceTransition + (state : State) + (workspaceId : WorkspaceId) : WorkspaceTransition Beam.Workspace.DropResult := + if !validWorkspaceId workspaceId then + { state, result := .error <| responseFailureFor .invalidParams + "workspace id must be non-empty" } + else + match getWorkspace? state workspaceId with + | none => + { state, result := .ok { + workspaceId + dropped := false + reason? := some "notFound" + } } + | some workspace => + let (_, detachedSessions) := detachWorkspaceSessions workspace + { + state := { state with workspaces := state.workspaces.erase workspaceId } + result := .ok { + workspaceId + dropped := true + invalidatedHandles := true + } + detachedSessions + } + +private def ServerRuntime.runWorkspaceTransition + (server : ServerRuntime) + (transition : State → WorkspaceTransition α) : IO (Except ResponseFailure α) := do + let transition ← server.withState do + let transition := transition (← get) + set transition.state + pure transition + for session in transition.detachedSessions do + shutdownSession session + pure transition.result + +/-- +Initialize, verify, or reset a workspace through a typed in-process boundary. Reset commits the new +workspace ownership atomically, then drains any detached backend sessions outside the state mutex. +-/ +def ServerRuntime.initWorkspaceWithConfig + (server : ServerRuntime) + (workspaceId : WorkspaceId) + (config : BrokerConfig) + (mode? : Option Beam.Workspace.InitMode := none) : + IO (Except ResponseFailure Beam.Workspace.InitResult) := + server.runWorkspaceTransition fun state => + initWorkspaceTransition state workspaceId config mode? private def workspaceListPayload (state : State) : Json := toJson ({ @@ -1091,28 +1156,18 @@ private def workspaceListPayload (state : State) : Json := } : Beam.Workspace.ListEntry) } : Beam.Workspace.ListResult) +private def responseOfTypedResult [ToJson α] : Except ResponseFailure α → Response + | .ok result => Response.success (toJson result) + | .error failure => failure.toResponse + +/-- +Remove a workspace through a typed in-process boundary. The workspace is erased atomically before +its detached backend sessions are drained outside the state mutex. +-/ def ServerRuntime.dropWorkspace (server : ServerRuntime) - (workspaceId : WorkspaceId) : IO Response := do - if !validWorkspaceId workspaceId then - return errorResponseFor .invalidParams "workspace id must be non-empty" - server.withState do - let state ← get - match getWorkspace? state workspaceId with - | none => - pure <| Response.success <| toJson ({ - workspaceId - dropped := false - reason? := some "notFound" - } : Beam.Workspace.DropResult) - | some workspace => - shutdownWorkspaceSessions workspace - modify fun state => { state with workspaces := state.workspaces.erase workspaceId } - pure <| Response.success <| toJson ({ - workspaceId - dropped := true - invalidatedHandles := true - } : Beam.Workspace.DropResult) + (workspaceId : WorkspaceId) : IO (Except ResponseFailure Beam.Workspace.DropResult) := + server.runWorkspaceTransition fun state => dropWorkspaceTransition state workspaceId private def requestRecordsMetrics : Op → Bool | .cancel | .stats | .resetStats | .shutdown | .openDocs | .listWorkspaces => false @@ -2243,14 +2298,14 @@ private def handleRequestIO match ← initWorkspaceConfigFromRequest server req with | .error failure => pure (failure.toResponse, false) | .ok config => - let resp ← server.initWorkspaceWithConfig workspaceId config req.workspaceMode? - pure (resp, false) + let result ← server.initWorkspaceWithConfig workspaceId config req.workspaceMode? + pure (responseOfTypedResult result, false) | .dropWorkspace => match req.requireWorkspaceId with | .error err => pure (errorResponseFor .invalidParams err, false) | .ok workspaceId => - let resp ← server.dropWorkspace workspaceId - pure (resp, false) + let result ← server.dropWorkspace workspaceId + pure (responseOfTypedResult result, false) | .cancel => let targetClientRequestId ← match req.cancelRequestIdArg with diff --git a/Beam/Mcp/Server.lean b/Beam/Mcp/Server.lean index ca9c39a9..a73a0513 100644 --- a/Beam/Mcp/Server.lean +++ b/Beam/Mcp/Server.lean @@ -471,10 +471,9 @@ private def ensureBrokerWorkspace match ← Runtime.mkBrokerConfig (runtimeOptions opts) root with | .error err => pure <| .error err | .ok config => - let brokerResp ← runtime.initWorkspaceWithConfig workspaceId config (some .set) - match brokerResp with - | .successResult .. => pure <| .ok config.root - | .errorResult failure => + match ← runtime.initWorkspaceWithConfig workspaceId config (some .set) with + | .ok initialized => pure <| .ok initialized.root + | .error failure => pure <| .error <| RpcError.invalidRequest failure.error.message private def ensureRuntimeForWorkspace @@ -589,20 +588,11 @@ private def handleDropWorkspace | none => pure <| callToolResult <| resultJson false false (some "notFound") | some runtime => - let (brokerResp, _) ← runtime.dispatchRequest { - op := .dropWorkspace - workspaceId? := some workspace.workspaceId - } - match brokerResp with - | .successResult payload .. => - match fromJson? (α := Beam.Workspace.DropResult) payload with - | .ok dropped => - pure <| callToolResult <| - resultJson dropped.dropped dropped.invalidatedHandles dropped.reason? - | .error err => - pure <| callToolErrorResult <| ToolError.invalidResult - s!"invalid workspace drop result: {err}" - | .errorResult failure => + match ← runtime.dropWorkspace workspace.workspaceId with + | .ok dropped => + pure <| callToolResult <| + resultJson dropped.dropped dropped.invalidatedHandles dropped.reason? + | .error failure => pure <| callToolErrorResult <| ToolError.fromBrokerError failure.error private def resolvedBeamHome? : IO (Option System.FilePath) := do diff --git a/docs/DEVELOPMENT.md b/docs/DEVELOPMENT.md index 58b9f057..3b7cc029 100644 --- a/docs/DEVELOPMENT.md +++ b/docs/DEVELOPMENT.md @@ -156,6 +156,12 @@ admission handle. The CLI scopes those requests before sending them. `Beam.Broker.Op.workspaceScope` is the shared operation classification; CLI and test adapters should use it instead of maintaining their own operation lists. +Treat `ServerRuntime.state` transactions as pure ownership transitions. Never perform process I/O +while holding that mutex: reset, drop, and runtime close must detach backend sessions and commit the +new workspace state atomically, then wait for or terminate the detached processes after releasing +the mutex. This keeps teardown of one workspace from blocking state access for every other +workspace. + ## MCP Projection Changes MCP work should go through the shared Lean operation layer in @@ -171,6 +177,9 @@ carry `{"workspace":{"root":"/absolute/project"}}`. Resolve it through broker cache key, and never store a current/default workspace in MCP protocol state. MCP server state owns only the optional shared `ServerRuntime`; workspace membership and canonical roots remain broker-owned and must be observed through typed broker queries rather than a transport-side mirror. +In-process MCP lifecycle calls use the broker's typed `initWorkspaceWithConfig` and `dropWorkspace` +results directly; do not route them through broker JSON dispatch and decode them back into the same +types. The executable path is split into importable modules: diff --git a/tests/lean/BeamTest/Broker/ProtocolTest.lean b/tests/lean/BeamTest/Broker/ProtocolTest.lean index 3f22ebe8..6d19b1bb 100644 --- a/tests/lean/BeamTest/Broker/ProtocolTest.lean +++ b/tests/lean/BeamTest/Broker/ProtocolTest.lean @@ -865,11 +865,165 @@ private def checkWorkspaceLifecycleProtocol : IO Unit := do fromJson? (α := Beam.Workspace.DropResult) dropJson require "typed workspace drop preserves lifecycle state" (decodedDrop.workspaceId == "fixture" && decodedDrop.dropped && decodedDrop.invalidatedHandles) - let dropResp ← runtime.dropWorkspace "fixture" - require "broker workspace drop should succeed" dropResp.ok + let dropResult ← runtime.dropWorkspace "fixture" + match dropResult with + | .error failure => + throw <| IO.userError s!"broker workspace drop failed: {failure.error.message}" + | .ok dropped => + require "broker workspace drop should succeed" dropped.dropped + require "broker workspace drop should invalidate handles" dropped.invalidatedHandles require "broker workspace query should observe a dropped workspace" ((← runtime.workspaceRoot? "fixture") == none) + match ← runtime.initWorkspaceWithConfig "fixture" ({ root } : Beam.Broker.BrokerConfig) with + | .error failure => + throw <| IO.userError s!"typed broker workspace initialization failed: {failure.error.message}" + | .ok initialized => + require "typed broker workspace initialization should return its workspace" + (initialized.workspaceId == "fixture" && initialized.root == root) + require "typed broker workspace initialization should report a new runtime" + (!initialized.runtimeReused && !initialized.invalidatedHandles) + +private inductive LifecycleTeardown where + | reset + | drop + +private def LifecycleTeardown.label : LifecycleTeardown → String + | .reset => "reset" + | .drop => "drop" + +private partial def waitForPath + (path : System.FilePath) + (tries : Nat := 200) : IO Bool := do + if ← path.pathExists then + pure true + else if tries == 0 then + pure false + else + IO.sleep 10 + waitForPath path (tries - 1) + +private partial def waitForTaskBefore + (task : Task α) + (blockedBy : Task β) + (tries : Nat := 300) : IO (Option α) := do + if ← IO.hasFinished blockedBy then + pure none + else if ← IO.hasFinished task then + pure <| some (← IO.wait task) + else if tries == 0 then + pure none + else + IO.sleep 10 + waitForTaskBefore task blockedBy (tries - 1) + +private def stubbornSession + (workspaceId : WorkspaceId) + (root sentinel : System.FilePath) : IO Session := do + let proc ← IO.Process.spawn { + toStdioConfig := brokerStdio + cmd := "python3" + args := #[ + "-c", + "import pathlib, sys, time; sys.stdin.buffer.readline(); pathlib.Path(sys.argv[1]).write_text('shutdown'); time.sleep(30)", + sentinel.toString + ] + } + let pending ← Std.Mutex.new ({} : Std.TreeMap Lean.JsonRpc.RequestID PendingRequest) + pure { + workspaceId + backend := .lean + root + epoch := 1 + sessionToken := s!"stubborn-{workspaceId}" + proc + stdin := IO.FS.Stream.ofHandle proc.stdin + stdout := IO.FS.Stream.ofHandle proc.stdout + pending + } + +private def runLifecycleTeardown + (kind : LifecycleTeardown) + (runtime : ServerRuntime) + (workspaceId : WorkspaceId) + (replacement : BrokerConfig) : IO (Except ResponseFailure Bool) := do + match kind with + | .reset => + match ← runtime.initWorkspaceWithConfig workspaceId replacement (some .reset) with + | .ok result => pure <| .ok result.invalidatedHandles + | .error failure => pure <| .error failure + | .drop => + match ← runtime.dropWorkspace workspaceId with + | .ok result => pure <| .ok result.invalidatedHandles + | .error failure => pure <| .error failure + +private def checkLifecycleTeardownReleasesStateMutex + (kind : LifecycleTeardown) : IO Unit := do + let nonce ← IO.monoNanosNow + let targetId := "teardown-target" + let observerId := "teardown-observer" + let targetRoot := System.FilePath.mk s!"/tmp/beam-{kind.label}-target-{nonce}" + let replacementRoot := System.FilePath.mk s!"/tmp/beam-{kind.label}-replacement-{nonce}" + let observerRoot := System.FilePath.mk s!"/tmp/beam-{kind.label}-observer-{nonce}" + let sentinel := System.FilePath.mk s!"/tmp/beam-{kind.label}-shutdown-{nonce}" + let targetConfig : BrokerConfig := { root := targetRoot } + let replacementConfig : BrokerConfig := { root := replacementRoot } + let observerConfig : BrokerConfig := { root := observerRoot } + let runtime ← ServerRuntime.create targetConfig targetId (.tcp 0) + let session ← stubbornSession targetId targetRoot sentinel + runtime.state.atomically do + let state ← get + let targetWorkspace : WorkspaceState := { + config := targetConfig + lean := { nextEpoch := 2, session? := some session } + } + let observerWorkspace : WorkspaceState := { config := observerConfig } + let workspaces := state.workspaces.insert targetId targetWorkspace + let workspaces := workspaces.insert observerId observerWorkspace + set { state with workspaces } + let teardownTask ← IO.asTask (prio := Task.Priority.dedicated) <| + runLifecycleTeardown kind runtime targetId replacementConfig + try + unless ← waitForPath sentinel do + throw <| IO.userError s!"{kind.label}: backend did not enter shutdown" + require s!"{kind.label}: teardown fixture should still be waiting for the backend" + (!(← IO.hasFinished teardownTask)) + + let queryTask ← IO.asTask (prio := Task.Priority.dedicated) <| + runtime.workspaceRoot? observerId + let some queryOutcome ← waitForTaskBefore queryTask teardownTask + | throw <| IO.userError s!"{kind.label}: unrelated workspace query blocked on teardown" + let queryRoot ← + match queryOutcome with + | .ok queryRoot => pure queryRoot + | .error err => throw err + require s!"{kind.label}: unrelated workspace query should retain its root" + (queryRoot == some observerRoot) + require s!"{kind.label}: unrelated workspace query should finish before teardown" + (!(← IO.hasFinished teardownTask)) + + let teardownOutcome ← IO.wait teardownTask + let teardownResult ← + match teardownOutcome with + | .ok result => pure result + | .error err => throw err + match teardownResult with + | .error failure => + throw <| IO.userError s!"{kind.label}: lifecycle transition failed: {failure.error.message}" + | .ok invalidatedHandles => + require s!"{kind.label}: lifecycle transition should invalidate handles" invalidatedHandles + finally + try + session.proc.kill + catch _ => + pure () + if ← sentinel.pathExists then + IO.FS.removeFile sentinel + +private def checkLifecycleTeardownConcurrency : IO Unit := do + checkLifecycleTeardownReleasesStateMutex .reset + checkLifecycleTeardownReleasesStateMutex .drop + private partial def waitForCancellation (cancelRef : IO.Ref Bool) (tries : Nat := 100) : IO Unit := do @@ -935,6 +1089,7 @@ def main : IO Unit := do checkRequestArgsBoundary checkWorkspaceRoutingFields checkWorkspaceLifecycleProtocol + checkLifecycleTeardownConcurrency checkSessionCloseAdmission end BeamTest.Broker.ProtocolTest From 138e492b6e370a7643a92542df7fea572861dc25 Mon Sep 17 00:00:00 2001 From: Emilio Jesus Gallego Arias Date: Wed, 26 Aug 2026 16:03:10 +0200 Subject: [PATCH 09/28] fix: unpublish owner generations before drain --- Beam/Broker/Server.lean | 2 +- Beam/Cli/DaemonManager.lean | 22 ++++++--- docs/DEVELOPMENT.md | 10 ++-- docs/MCP.md | 6 +-- tests/lean/BeamTest/Broker/CliDaemonTest.lean | 30 ------------ tests/test-beam-wrapper-daemon.sh | 49 +++++++++++++++++++ tests/test-beam-wrapper-rocq.sh | 18 +++++++ 7 files changed, 90 insertions(+), 47 deletions(-) diff --git a/Beam/Broker/Server.lean b/Beam/Broker/Server.lean index faf0cffa..d757d14e 100644 --- a/Beam/Broker/Server.lean +++ b/Beam/Broker/Server.lean @@ -1020,7 +1020,7 @@ def ServerRuntime.close (server : ServerRuntime) : IO Bool := do awaitRuntimeClose server.closeDone pure false -def workspaceInitResult +private def workspaceInitResult (workspaceId : WorkspaceId) (root : System.FilePath) (mode : Beam.Workspace.InitMode) diff --git a/Beam/Cli/DaemonManager.lean b/Beam/Cli/DaemonManager.lean index 695d0f8f..68d947fe 100644 --- a/Beam/Cli/DaemonManager.lean +++ b/Beam/Cli/DaemonManager.lean @@ -40,7 +40,7 @@ private def projectControlLockTimeoutMs : IO Nat := do "invalid BEAM_CONTROL_LOCK_TIMEOUT_MS value '0': expected a positive timeout" pure timeoutMs -def projectControlLockDir (root : System.FilePath) : IO System.FilePath := do +private def projectControlLockDir (root : System.FilePath) : IO System.FilePath := do pure ((← controlDir root) / "lock") /-- @@ -223,8 +223,6 @@ private def daemonFailureIncidentSchemaVersion : Nat := private def daemonFailureKind? (detail : String) : Option String := if detail.contains "Beam daemon connection closed" then some "connectionClosed" - else if detail.contains "no live Beam daemon registered for " then - some "noLiveDaemon" else none @@ -587,9 +585,13 @@ private def activeOwnerMessage (root : System.FilePath) (entry : RegistryEntry) s!"Beam session for {root} is already owned by wrapper pid {entry.ownerPid}; " ++ "interrupt that 'lean-beam ensure --hold' process before starting another owner" -private def missingOwnerMessage (root : System.FilePath) : String := +private def missingOwnerCommand : Option Backend → String + | some .rocq => "lean-beam ensure rocq --hold" + | some .lean | none => "lean-beam ensure --hold" + +private def missingOwnerMessage (root : System.FilePath) (backend? : Option Backend) : String := s!"no live Beam session owner is registered for {root}; " ++ - "start 'lean-beam ensure --hold' for this project and keep it running while using wrapper commands" + s!"start '{missingOwnerCommand backend?}' for this project and keep it running while using wrapper commands" private def startOwnedProjectDaemon (desired : DesiredConfig) @@ -665,6 +667,9 @@ private def finishOwnedProjectDaemon (root : System.FilePath) (owned : OwnedProjectDaemon) (exitCodeRef : IO.Ref (Option UInt32)) : IO Unit := do + -- Stop publishing this generation before waiting for its child to drain. A second removal is an + -- idempotent retry for cleanup failures and cannot remove a replacement generation. + removeOwnedRegistry root owned.entry.daemonId finishOwnedDaemonChild owned exitCodeRef removeOwnedRegistry root owned.entry.daemonId @@ -690,20 +695,21 @@ def withProjectDaemonOwner private def lookupProjectDaemon (root : System.FilePath) - (expectedHash? : Option String := none) : IO ProjectDaemonClient := do + (expectedHash? : Option String := none) + (backend? : Option Backend := none) : IO ProjectDaemonClient := do withProjectControlLock root do match ← registryLiveFor root expectedHash? with | some entry => projectDaemonClient entry | none => stopRegisteredDaemon root - throw <| IO.userError (missingOwnerMessage root) + throw <| IO.userError (missingOwnerMessage root backend?) def withProjectDaemon (home root : System.FilePath) (backend : Backend) (act : ProjectDaemonClient → IO α) : IO α := do let desired ← desiredConfig home root backend - act (← lookupProjectDaemon root (some desired.configHash)) + act (← lookupProjectDaemon root (some desired.configHash) (some backend)) def withExistingProjectDaemon (root : System.FilePath) diff --git a/docs/DEVELOPMENT.md b/docs/DEVELOPMENT.md index 3b7cc029..69527645 100644 --- a/docs/DEVELOPMENT.md +++ b/docs/DEVELOPMENT.md @@ -156,11 +156,11 @@ admission handle. The CLI scopes those requests before sending them. `Beam.Broker.Op.workspaceScope` is the shared operation classification; CLI and test adapters should use it instead of maintaining their own operation lists. -Treat `ServerRuntime.state` transactions as pure ownership transitions. Never perform process I/O -while holding that mutex: reset, drop, and runtime close must detach backend sessions and commit the -new workspace state atomically, then wait for or terminate the detached processes after releasing -the mutex. This keeps teardown of one workspace from blocking state access for every other -workspace. +Workspace teardown must not wait for backend shutdown while holding `ServerRuntime.state`: reset, +drop, and runtime close detach backend sessions and commit the new workspace state atomically, then +wait for or terminate the detached processes after releasing the mutex. This keeps teardown of one +workspace from blocking state access for every other workspace. Other state transactions, including +session startup and restart, may still perform process I/O while holding that mutex. ## MCP Projection Changes diff --git a/docs/MCP.md b/docs/MCP.md index 0b848205..b1629c42 100644 --- a/docs/MCP.md +++ b/docs/MCP.md @@ -146,9 +146,9 @@ inside the old Lean process is not sufficient to reload workspace configuration. - [Beam/Mcp/Protocol.lean](../Beam/Mcp/Protocol.lean) owns the current MCP JSON-RPC helpers. - [Beam/Mcp/Runtime.lean](../Beam/Mcp/Runtime.lean) owns root-to-runtime configuration. - [Beam/Mcp/SelfCheck.lean](../Beam/Mcp/SelfCheck.lean) owns the installed-wrapper self-check. -- [Beam/Mcp/Server.lean](../Beam/Mcp/Server.lean) owns descriptor resolution, lazy cache dispatch, - the typed protocol-family state machine, the physically separate application registry, and the - synchronous protocol-test seam. +- [Beam/Mcp/Server.lean](../Beam/Mcp/Server.lean) owns descriptor resolution, lazy broker-runtime + access, the typed protocol-family state machine, and the synchronous protocol-test seam. The + broker runtime is authoritative for workspace state. - [Beam/Mcp/StdioServer.lean](../Beam/Mcp/StdioServer.lean) owns the permanent stdin reader, concurrent coordination, cancellation, cache-control barriers, and serialized output. diff --git a/tests/lean/BeamTest/Broker/CliDaemonTest.lean b/tests/lean/BeamTest/Broker/CliDaemonTest.lean index ecf537c8..3174739a 100644 --- a/tests/lean/BeamTest/Broker/CliDaemonTest.lean +++ b/tests/lean/BeamTest/Broker/CliDaemonTest.lean @@ -579,35 +579,6 @@ private def checkDaemonFailureContext : IO Unit := do catch _ => pure () -private def checkNoLiveDaemonFailureIncident : IO Unit := do - let root := System.FilePath.mk s!"/tmp/beam-no-live-daemon-incident-{← IO.monoNanosNow}" - try - IO.FS.createDirAll root - let detail := s!"no live Beam daemon registered for {root}" - let msg ← Beam.Cli.daemonFailureMessage root detail - requireSubstring "no-live daemon failure should include incident path" "Beam daemon incident:" msg - - let incidentJson ← readSingleDaemonFailureIncidentJson root - requireJsonNat "no-live daemon incident should use schema version" "schemaVersion" 1 incidentJson - requireJsonString "no-live daemon incident should classify stale lookup" - "kind" "noLiveDaemon" incidentJson - requireJsonString "no-live daemon incident should keep original detail" - "detail" detail incidentJson - requireJsonString "no-live daemon incident should include root" - "root" root.toString incidentJson - finally - try - let control ← Beam.Daemon.controlDir root - if ← control.pathExists then - IO.FS.removeDirAll control - catch _ => - pure () - try - if ← root.pathExists then - IO.FS.removeDirAll root - catch _ => - pure () - private def checkDaemonFailureUnreadableStartupLog : IO Unit := do let root := System.FilePath.mk s!"/tmp/beam-daemon-unreadable-startup-log-{← IO.monoNanosNow}" try @@ -1255,7 +1226,6 @@ def main : IO Unit := do checkStartupRetryPolicy checkDaemonDebugWarnings checkDaemonFailureContext - checkNoLiveDaemonFailureIncident checkDaemonFailureUnreadableStartupLog checkPlainBrokerTaskCancellation checkBrokerConnectionClosedIncident diff --git a/tests/test-beam-wrapper-daemon.sh b/tests/test-beam-wrapper-daemon.sh index f8723a84..006ce9a1 100644 --- a/tests/test-beam-wrapper-daemon.sh +++ b/tests/test-beam-wrapper-daemon.sh @@ -33,6 +33,7 @@ fi hold_pid="" root_removed="false" active_request_pid="" +paused_daemon_pid="" start_slow_request() { local root="$1" @@ -109,6 +110,10 @@ stop_hold_process() { } cleanup() { + if [ -n "$paused_daemon_pid" ]; then + kill -CONT "$paused_daemon_pid" > /dev/null 2>&1 || true + paused_daemon_pid="" + fi if [ -n "$active_request_pid" ]; then kill "$active_request_pid" > /dev/null 2>&1 || true wait "$active_request_pid" 2>/dev/null || true @@ -324,6 +329,50 @@ if [ -e "$registry" ]; then exit 1 fi +start_owner "$tmp1" "owner-unpublish-before-drain" +draining_daemon_pid="$(read_json_field "$registry" pid)" +kill -STOP "$draining_daemon_pid" +paused_daemon_pid="$draining_daemon_pid" +kill -INT "$hold_pid" +for _ in $(seq 1 40); do + if [ ! -e "$registry" ]; then + break + fi + sleep 0.05 +done +if [ -e "$registry" ]; then + echo "expected an interrupted owner to unpublish its generation before daemon drain" >&2 + cat "$registry" >&2 + exit 1 +fi +if ! kill -0 "$hold_pid" 2>/dev/null || ! kill -0 "$draining_daemon_pid" 2>/dev/null; then + echo "expected the paused daemon and its owner to remain alive during the drain check" >&2 + exit 1 +fi +draining_lookup_out="$tmp1/draining-lookup.out" +draining_lookup_err="$tmp1/draining-lookup.err" +if "$beam_script" --root "$tmp1" ensure > "$draining_lookup_out" 2> "$draining_lookup_err"; then + echo "expected an ordinary command not to attach to an unpublished draining generation" >&2 + cat "$draining_lookup_out" >&2 + exit 1 +fi +if ! grep -Fq "lean-beam ensure --hold" "$draining_lookup_err"; then + echo "expected draining-generation recovery to require a new explicit owner" >&2 + cat "$draining_lookup_err" >&2 + exit 1 +fi +kill -CONT "$draining_daemon_pid" +paused_daemon_pid="" +if ! wait_for_exit "$hold_pid" "owner after unpublish-before-drain check" 200 0.05; then + cat "$tmp1/owner-unpublish-before-drain.err" >&2 + exit 1 +fi +wait "$hold_pid" +hold_pid="" +if ! wait_for_exit "$draining_daemon_pid" "daemon after unpublish-before-drain check" 200 0.05; then + exit 1 +fi + start_owner "$tmp1" "owner-3" daemon3_pid="$(read_json_field "$registry" pid)" start_slow_request "$tmp1" "owner-loss-active" "owner-loss-active" diff --git a/tests/test-beam-wrapper-rocq.sh b/tests/test-beam-wrapper-rocq.sh index 65a6d6de..291185cc 100755 --- a/tests/test-beam-wrapper-rocq.sh +++ b/tests/test-beam-wrapper-rocq.sh @@ -94,4 +94,22 @@ rsync -a \ fi wait_for_exit "$rocq_owner_pid" "Rocq session owner" 120 0.1 wait "$rocq_owner_pid" + rocq_missing_out="$tmp_repo/rocq-missing-owner.out" + rocq_missing_err="$tmp_repo/rocq-missing-owner.err" + if [ -n "$rocq_cmd" ]; then + if BEAM_ROCQ_CMD="$rocq_cmd" "$tmp_repo/scripts/lean-beam" --root "$tmp_repo/tests/rocq/Minimal" ensure rocq \ + >"$rocq_missing_out" 2>"$rocq_missing_err"; then + echo "expected an ordinary Rocq command to require a session owner" >&2 + exit 1 + fi + elif "$tmp_repo/scripts/lean-beam" --root "$tmp_repo/tests/rocq/Minimal" ensure rocq \ + >"$rocq_missing_out" 2>"$rocq_missing_err"; then + echo "expected an ordinary Rocq command to require a session owner" >&2 + exit 1 + fi + if ! grep -Fq "lean-beam ensure rocq --hold" "$rocq_missing_err"; then + echo "expected Rocq missing-owner recovery to name the Rocq ownership command" >&2 + cat "$rocq_missing_err" >&2 + exit 1 + fi ) From 6755a4956fd4cc606430b16502e59811dcbb7516 Mon Sep 17 00:00:00 2001 From: Emilio Jesus Gallego Arias Date: Wed, 26 Aug 2026 17:23:16 +0200 Subject: [PATCH 10/28] refactor: type client failures and own MCP runtime control --- Beam/Broker/Client.lean | 84 ++++++++++--- Beam/Cli/Broker.lean | 116 ++++++++++-------- Beam/Cli/DaemonManager.lean | 16 +-- Beam/Mcp/Server.lean | 32 +++-- Beam/Mcp/StdioServer.lean | 13 +- docs/DEVELOPMENT.md | 16 ++- docs/STATUS.md | 8 +- docs/TESTING.md | 7 +- tests/lean/BeamTest/Broker/CliDaemonTest.lean | 71 +++++++++-- .../lean/BeamTest/Broker/McpProtocolTest.lean | 14 +-- tests/test-beam-wrapper-sandbox.sh | 8 +- 11 files changed, 257 insertions(+), 128 deletions(-) diff --git a/Beam/Broker/Client.lean b/Beam/Broker/Client.lean index 983a9777..4ec5148f 100644 --- a/Beam/Broker/Client.lean +++ b/Beam/Broker/Client.lean @@ -19,6 +19,19 @@ structure StreamCallbacks where abbrev Endpoint := Transport.Endpoint +/-- Classify failures produced while a broker client exchanges one streamed request. -/ +inductive BrokerClientFailureKind where + | transport + | invalidResponse + | streamCallback + deriving BEq, Repr + +/-- Keep broker client failures typed until a CLI or transport presentation boundary. -/ +structure BrokerClientFailure where + kind : BrokerClientFailureKind + detail : String + deriving BEq, Repr + def parsePortText (name value : String) : Except String UInt16 := do let some n := value.toNat? | throw s!"invalid {name} '{value}'" @@ -42,6 +55,14 @@ private def decodeStreamMessage (msg : String) : IO StreamMessage := do | .ok stream => pure stream | .error err => throw <| IO.userError s!"invalid Beam daemon response payload: {err}" +private def captureClientFailure + (kind : BrokerClientFailureKind) + (action : IO α) : IO (Except BrokerClientFailure α) := do + try + pure <| .ok (← action) + catch e => + pure <| .error { kind, detail := e.toString } + private def diagnosticSeverityLabel : Option Lsp.DiagnosticSeverity → String | some .error => "error" | some .warning => "warning" @@ -67,34 +88,61 @@ def formatStreamDiagnostic (diagnostic : StreamDiagnostic) : String := "" s!"beam: diagnostic {severity}{blocking} {diagnostic.path}:{line}:{character}: {message}" -partial def sendRequestWithStream +/-- Send one request while preserving transport, response, and callback failures as typed data. -/ +partial def sendRequestWithStreamResult (endpoint : Endpoint) (req : Request) - (onStream : StreamMessage → IO Unit) : IO Response := do - let client ← Transport.connect endpoint + (onStream : StreamMessage → IO Unit) : IO (Except BrokerClientFailure Response) := do + let client ← + match ← captureClientFailure .transport (Transport.connect endpoint) with + | .ok client => pure client + | .error failure => return .error failure try - Transport.sendMsg client (toJson req).compress - let rec loop : IO Response := do - let msg ← Transport.recvMsg client - let stream ← decodeStreamMessage msg + match ← captureClientFailure .transport <| + Transport.sendMsg client (toJson req).compress with + | .ok () => pure () + | .error failure => return .error failure + let rec loop : IO (Except BrokerClientFailure Response) := do + let msg ← + match ← captureClientFailure .transport (Transport.recvMsg client) with + | .ok msg => pure msg + | .error failure => return .error failure + let stream ← + match ← captureClientFailure .invalidResponse (decodeStreamMessage msg) with + | .ok stream => pure stream + | .error failure => return .error failure unless stream.clientRequestId? == req.clientRequestId? do - throw <| IO.userError - s!"Beam daemon stream request id {stream.clientRequestId?} does not match request id {req.clientRequestId?}" - onStream stream + return .error { + kind := .invalidResponse + detail := + s!"Beam daemon stream request id {stream.clientRequestId?} does not match request id {req.clientRequestId?}" + } + match ← captureClientFailure .streamCallback (onStream stream) with + | .ok () => pure () + | .error failure => return .error failure match stream with | .response _ response => - pure response + pure <| .ok response | .fileProgress .. | .diagnostic .. => loop loop finally Transport.closeConnection client -partial def sendRequestWithCallbacks +partial def sendRequestWithStream (endpoint : Endpoint) (req : Request) - (callbacks : StreamCallbacks := {}) : IO Response := do - sendRequestWithStream endpoint req fun stream => do + (onStream : StreamMessage → IO Unit) : IO Response := do + match ← sendRequestWithStreamResult endpoint req onStream with + | .ok response => pure response + | .error failure => throw <| IO.userError failure.detail + +/-- Send one request with typed client failures and structured progress callbacks. -/ +partial def sendRequestWithCallbacksResult + (endpoint : Endpoint) + (req : Request) + (callbacks : StreamCallbacks := {}) : IO (Except BrokerClientFailure Response) := do + sendRequestWithStreamResult endpoint req fun stream => do match stream with | .response .. => pure () @@ -102,6 +150,14 @@ partial def sendRequestWithCallbacks callbacks.onFileProgress clientRequestId? progress | .diagnostic clientRequestId? diagnostic => callbacks.onDiagnostic clientRequestId? diagnostic + +partial def sendRequestWithCallbacks + (endpoint : Endpoint) + (req : Request) + (callbacks : StreamCallbacks := {}) : IO Response := do + match ← sendRequestWithCallbacksResult endpoint req callbacks with + | .ok response => pure response + | .error failure => throw <| IO.userError failure.detail def sendRequest (endpoint : Endpoint) (req : Request) : IO Response := sendRequestWithCallbacks endpoint req diff --git a/Beam/Cli/Broker.lean b/Beam/Cli/Broker.lean index ed7529fd..cd533c32 100644 --- a/Beam/Cli/Broker.lean +++ b/Beam/Cli/Broker.lean @@ -27,11 +27,14 @@ def inProjectDaemonWorkspace (req : Request) : Request := if req.workspaceId?.isSome then req else { req with workspaceId? := some projectDaemonWorkspaceId } -def withBrokerErrorContext {α} (root : System.FilePath) (action : IO α) : IO α := do - try - action - catch e => - throw <| IO.userError (← daemonFailureMessage root e.toString) +def withBrokerErrorContext + {α} + (root : System.FilePath) + (action : IO (Except BrokerClientFailure α)) : IO α := do + match ← action with + | .ok value => pure value + | .error failure => + throw <| IO.userError (← daemonFailureMessage root failure) structure BrokerWaitSpec where action : String @@ -117,12 +120,12 @@ private def sendBrokerCancellation pure none private def awaitBrokerResponse - (task : Task (Except IO.Error Response)) + (task : Task (Except IO.Error (Except BrokerClientFailure Response))) (endpoint : Transport.Endpoint) (req : Request) (visibleClientRequestId? : Option String) (progressSpec? : Option BrokerWaitSpec) - (interruptWatcher? : Option InterruptWatcher) : IO Response := do + (interruptWatcher? : Option InterruptWatcher) : IO (Except BrokerClientFailure Response) := do let mut interruptObserved := false let mut cancelAcknowledged := false let emit := fun msg => IO.eprintln <| annotateRunatMessage visibleClientRequestId? msg @@ -152,13 +155,17 @@ private def awaitBrokerResponse if waitedMs % 1000 == 0 then if let some spec := progressSpec? then emit <| spec.stillWaitingMsg (waitedMs / 1000) - let resp ← + let result ← match (← IO.wait task) with - | .ok resp => pure resp + | .ok result => pure result | .error err => throw err - if let some spec := progressSpec? then - emit <| spec.completeMsg resp - pure resp + match result with + | .ok response => + if let some spec := progressSpec? then + emit <| spec.completeMsg response + pure <| .ok response + | .error failure => + pure <| .error failure finally match interruptWatcher? with | some watcher => watcher.stop @@ -169,7 +176,8 @@ private def awaitBrokerResponseWithInterrupts (req : Request) (visibleClientRequestId? : Option String) (progressSpec? : Option BrokerWaitSpec) - (action : IO Response) : IO Response := do + (action : IO (Except BrokerClientFailure Response)) : + IO (Except BrokerClientFailure Response) := do -- Wrapper calls synthesize a broker clientRequestId when the user did not provide one. That id -- gives SIGINT cancellation a stable broker key but is kept out of the CLI's public output. let interruptWatcher? ← mkInterruptWatcher? req.clientRequestId? @@ -190,14 +198,14 @@ private structure WrapperBrokerResponse where private def requestBrokerResponse (root : System.FilePath) (client : ProjectDaemonClient) - (req : Request) : IO WrapperBrokerResponse := - withBrokerErrorContext root do - let wrapperReq ← prepareWrapperBrokerRequest req - let req := wrapperReq.request - let response ← awaitBrokerResponseWithInterrupts client.endpoint req + (req : Request) : IO WrapperBrokerResponse := do + let wrapperReq ← prepareWrapperBrokerRequest req + let req := wrapperReq.request + let response ← withBrokerErrorContext root do + awaitBrokerResponseWithInterrupts client.endpoint req wrapperReq.visibleClientRequestId? none <| - sendRequest client.endpoint req - pure { response, visibleClientRequestId? := wrapperReq.visibleClientRequestId? } + sendRequestWithCallbacksResult client.endpoint req + pure { response, visibleClientRequestId? := wrapperReq.visibleClientRequestId? } /-- Send one wrapper request without printing or interpreting its response. -/ def requestBroker @@ -420,40 +428,40 @@ def callBrokerWithProgress (root : System.FilePath) (client : ProjectDaemonClient) (req : Request) - (spec : BrokerWaitSpec) : IO Unit := - withBrokerErrorContext root do - let wrapperReq ← prepareWrapperBrokerRequest req - let req := wrapperReq.request - let visibleClientRequestId? := wrapperReq.visibleClientRequestId? - let showProgress ← progressEnabled - let callbacks : StreamCallbacks := { - onFileProgress := fun _ progress => do - if showProgress then - IO.eprintln <| annotateRunatMessage visibleClientRequestId? (spec.progressMsg progress) - onDiagnostic := fun _ diagnostic => - IO.eprintln <| annotateRunatMessage visibleClientRequestId? (formatStreamDiagnostic diagnostic) - } - let progressSpec? := if showProgress then some spec else none - let resp ← awaitBrokerResponseWithInterrupts client.endpoint req visibleClientRequestId? + (spec : BrokerWaitSpec) : IO Unit := do + let wrapperReq ← prepareWrapperBrokerRequest req + let req := wrapperReq.request + let visibleClientRequestId? := wrapperReq.visibleClientRequestId? + let showProgress ← progressEnabled + let callbacks : StreamCallbacks := { + onFileProgress := fun _ progress => do + if showProgress then + IO.eprintln <| annotateRunatMessage visibleClientRequestId? (spec.progressMsg progress) + onDiagnostic := fun _ diagnostic => + IO.eprintln <| annotateRunatMessage visibleClientRequestId? (formatStreamDiagnostic diagnostic) + } + let progressSpec? := if showProgress then some spec else none + let resp ← withBrokerErrorContext root do + awaitBrokerResponseWithInterrupts client.endpoint req visibleClientRequestId? progressSpec? <| - sendRequestWithCallbacks client.endpoint req callbacks - match responseErrorSummary? spec.action spec.failureBoundary resp with - | some note => - IO.eprintln <| annotateRunatMessage visibleClientRequestId? note - | none => - pure () - match responseRecoveryHint? resp with - | some note => - IO.eprintln <| annotateRunatMessage visibleClientRequestId? note - | none => - pure () - match spec.responseNote? resp with - | some note => - IO.eprintln <| annotateRunatMessage visibleClientRequestId? note - | none => - pure () - maybeEmitLiteralBackslashNewlineHint visibleClientRequestId? req resp - printResponse resp visibleClientRequestId? - failOnError resp + sendRequestWithCallbacksResult client.endpoint req callbacks + match responseErrorSummary? spec.action spec.failureBoundary resp with + | some note => + IO.eprintln <| annotateRunatMessage visibleClientRequestId? note + | none => + pure () + match responseRecoveryHint? resp with + | some note => + IO.eprintln <| annotateRunatMessage visibleClientRequestId? note + | none => + pure () + match spec.responseNote? resp with + | some note => + IO.eprintln <| annotateRunatMessage visibleClientRequestId? note + | none => + pure () + maybeEmitLiteralBackslashNewlineHint visibleClientRequestId? req resp + printResponse resp visibleClientRequestId? + failOnError resp end Beam.Cli diff --git a/Beam/Cli/DaemonManager.lean b/Beam/Cli/DaemonManager.lean index 68d947fe..4d27cf68 100644 --- a/Beam/Cli/DaemonManager.lean +++ b/Beam/Cli/DaemonManager.lean @@ -220,11 +220,10 @@ private structure DaemonFailureIncident where private def daemonFailureIncidentSchemaVersion : Nat := 1 -private def daemonFailureKind? (detail : String) : Option String := - if detail.contains "Beam daemon connection closed" then - some "connectionClosed" - else - none +private def daemonFailureIncidentKind? : BrokerClientFailureKind → Option String + | .transport => some "brokerTransportFailure" + | .invalidResponse => some "invalidBrokerResponse" + | .streamCallback => none private def daemonFailureIncidentTimestampLabel (timestamp : String) : String := (timestamp.replace "-" "").replace ":" "" @@ -280,8 +279,11 @@ private def writeDaemonFailureIncident? catch _ => pure none -def daemonFailureMessage (root : System.FilePath) (detail : String) : IO String := do - match daemonFailureKind? detail with +def daemonFailureMessage + (root : System.FilePath) + (failure : BrokerClientFailure) : IO String := do + let detail := failure.detail + match daemonFailureIncidentKind? failure.kind with | none => pure detail | some kind => diff --git a/Beam/Mcp/Server.lean b/Beam/Mcp/Server.lean index a73a0513..695dc81d 100644 --- a/Beam/Mcp/Server.lean +++ b/Beam/Mcp/Server.lean @@ -47,11 +47,13 @@ inductive ProtocolState where structure ServerState where protocol : Std.Mutex ProtocolState private runtime : IO.Ref (Option Beam.Broker.ServerRuntime) + private runtimeControl : Std.Mutex Unit def ServerState.create : IO ServerState := do pure { protocol := ← Std.Mutex.new .undecided runtime := ← IO.mkRef none + runtimeControl := ← Std.Mutex.new () } def ServerState.protocolState (state : ServerState) : IO ProtocolState := @@ -60,6 +62,22 @@ def ServerState.protocolState (state : ServerState) : IO ProtocolState := def ServerState.runtime? (state : ServerState) : IO (Option Beam.Broker.ServerRuntime) := state.runtime.get +private def ServerState.withRuntimeControl + (state : ServerState) + (action : IO α) : IO α := + state.runtimeControl.atomically action + +/-- Close and forget the in-process broker runtime owned by this MCP server state. -/ +def ServerState.closeRuntime (state : ServerState) : IO Unit := + state.withRuntimeControl do + match ← state.runtime.get with + | none => pure () + | some runtime => + -- Transfer ownership out of the state before waiting for broker teardown. A concurrent + -- creator remains excluded by `runtimeControl`, and repeated close calls are idempotent. + state.runtime.set none + discard <| runtime.close + structure NotificationSink where send : Json → IO Unit := fun _ => pure () @@ -479,10 +497,9 @@ private def ensureBrokerWorkspace private def ensureRuntimeForWorkspace (state : ServerState) (opts : Options) - (setupMutex : Std.Mutex Unit) (workspaceId : Beam.Broker.WorkspaceId) (root : System.FilePath) : IO (Except RpcError (Beam.Broker.ServerRuntime × System.FilePath)) := do - setupMutex.atomically do + state.withRuntimeControl do match ← state.runtime? with | some runtime => match ← ensureBrokerWorkspace opts runtime workspaceId root with @@ -760,7 +777,6 @@ private def brokerRequestForTool def Internal.handleToolCall (state : ServerState) (opts : Options) - (setupMutex : Std.Mutex Unit) (brokerClientRequestId : String) (beforeDispatch : Beam.Broker.RequestHandle → IO Bool) (req : Request) @@ -787,7 +803,7 @@ def Internal.handleToolCall | .error err => return .ok <| callToolErrorResult err if initialProgress == 0 then emitProgress? progress? s!"{params.name.key}: preparing workspace eviction" - let result ← setupMutex.atomically do + let result ← state.withRuntimeControl do handleDropWorkspace state workspace Internal.traceMcp s!"tools/call workspace drop complete id={req.id.label} tool={params.name.key}" @@ -834,7 +850,7 @@ def Internal.handleToolCall reporter.emitPreparing try let (runtime, root) ← - match ← ensureRuntimeForWorkspace state opts setupMutex workspace.workspaceId workspace.root with + match ← ensureRuntimeForWorkspace state opts workspace.workspaceId workspace.root with | .ok runtimeAndRoot => Internal.traceMcp s!"tools/call runtime ready id={req.id.label} tool={params.name.key}" pure runtimeAndRoot @@ -867,7 +883,6 @@ def Internal.handleToolCall private def handleReadyOperationRequest (state : ServerState) (opts : Options) - (setupMutex : Std.Mutex Unit) (brokerClientRequestId : String) (req : Request) (admitted : AdmittedRequestContext) @@ -891,7 +906,7 @@ private def handleReadyOperationRequest pure <| successResponseForEra era req.id result | "tools/call" => let parsedParams := parseCallToolParams req.params? - match ← Internal.handleToolCall state opts setupMutex brokerClientRequestId + match ← Internal.handleToolCall state opts brokerClientRequestId (fun _ => pure true) req admitted parsedParams notifications with | .ok result => pure <| successResponseForEra era req.id result | .error err => pure <| errorResponse req.id err @@ -1024,7 +1039,6 @@ def Internal.handleRequestForProtocol (req : Request) (evidence : RequestProtocolEvidence) (notifications : NotificationSink := {}) : IO Json := do - let setupMutex ← Std.Mutex.new () let brokerClientRequestId := s!"mcp:sync:{req.id.label}" let era ← match ← Internal.admitCompatibleRequest state evidence with @@ -1068,7 +1082,7 @@ def Internal.handleRequestForProtocol | .error err => pure <| errorResponse req.id err | .ok admitted => handleReadyOperationRequest - state opts setupMutex brokerClientRequestId req admitted notifications + state opts brokerClientRequestId req admitted notifications | method => match era with | .modern _ => diff --git a/Beam/Mcp/StdioServer.lean b/Beam/Mcp/StdioServer.lean index 79315db6..a9b26044 100644 --- a/Beam/Mcp/StdioServer.lean +++ b/Beam/Mcp/StdioServer.lean @@ -103,22 +103,20 @@ private inductive RequestRegistrationError where /- Nested coordinator locks flow in one direction: -* setup → progress → request → output during workspace cache eviction * progress → request → output for request notifications * request → output for active request messages -Routing is released before setup, request, or output is acquired. Output acquires no coordinator lock. +Routing is released before runtime control, request, or output is acquired. Runtime control is owned +by `ServerState` and does not acquire coordinator locks. Output acquires no coordinator lock. -/ private structure Coordinator where state : ServerState - setupMutex : Std.Mutex Unit routing : Std.Mutex RoutingState output : OutputSink private def Coordinator.create : IO Coordinator := do pure { state := ← ServerState.create - setupMutex := ← Std.Mutex.new () routing := ← Std.Mutex.new {} output := ← OutputSink.create } @@ -310,11 +308,7 @@ private def Coordinator.closeTransport (coordinator : Coordinator) : IO Unit := coordinator.beginClosing coordinator.awaitRequests requests unless alreadyClosing do - coordinator.setupMutex.atomically do - match ← coordinator.state.runtime? with - | none => pure () - | some runtime => - discard <| runtime.close + coordinator.state.closeRuntime private def Coordinator.admitToolRequest (coordinator : Coordinator) @@ -339,7 +333,6 @@ private def Coordinator.executeToolRequest match ← Internal.handleToolCall coordinator.state opts - coordinator.setupMutex request.brokerId request.bindBrokerRequest req diff --git a/docs/DEVELOPMENT.md b/docs/DEVELOPMENT.md index 69527645..4d649e30 100644 --- a/docs/DEVELOPMENT.md +++ b/docs/DEVELOPMENT.md @@ -177,6 +177,9 @@ carry `{"workspace":{"root":"/absolute/project"}}`. Resolve it through broker cache key, and never store a current/default workspace in MCP protocol state. MCP server state owns only the optional shared `ServerRuntime`; workspace membership and canonical roots remain broker-owned and must be observed through typed broker queries rather than a transport-side mirror. +`ServerState` also owns the runtime-control mutex used by every transport and direct request entry +point. Runtime creation, workspace eviction, and close are serialized there; close first transfers +the runtime out of `ServerState`, then drains it while preventing a competing creation. In-process MCP lifecycle calls use the broker's typed `initWorkspaceWithConfig` and `dropWorkspace` results directly; do not route them through broker JSON dispatch and decode them back into the same types. @@ -392,10 +395,14 @@ typed PID-domain boundary. The owner also watches its exact registry generation and daemon child. `lean-beam shutdown` removes that generation after the typed shutdown response, which makes the holder close its pipe and lets -the daemon's stdin watcher finish. An unexpected nonzero daemon exit is reported by the holder. -Interrupting or killing the holder closes the pipe by process lifetime. A paused holder keeps the -pipe open, so the session remains valid without time-based expiry. If the project root disappears, -the daemon's root watcher and the holder both converge on the same shutdown path. +the daemon's stdin watcher finish. On every holder exit path, the holder removes its exact registry +generation before waiting for the daemon child to drain, then retries the same generation-scoped +removal after bounded child cleanup. A draining daemon is therefore never advertised as attachable, +and neither removal can delete a replacement generation. An unexpected nonzero daemon exit is +reported by the holder. Interrupting or killing the holder closes the pipe by process lifetime. A +paused holder keeps the pipe open, so the session remains valid without time-based expiry. If the +project root disappears, the daemon's root watcher and the holder both converge on the same shutdown +path. This model prevents PID-isolated commands from making contradictory ownership decisions: later commands may attach to a validated endpoint, but none can silently become a replacement owner. @@ -410,6 +417,7 @@ Keep these invariants covered: - a second owner is rejected while the current endpoint/root generation is live - ordinary wrapper commands preserve the owner's generation and fail with the exact recovery command when no owner is live +- holder teardown unpublishes its exact generation before child drain and cannot remove a replacement - owner EOF, explicit shutdown, and project-root disappearance all close admission before backend teardown and complete with bounded child cleanup - PID-domain checks gate every PID probe or signal; cross-domain decisions use the validated endpoint diff --git a/docs/STATUS.md b/docs/STATUS.md index 25acaed8..65cd99ee 100644 --- a/docs/STATUS.md +++ b/docs/STATUS.md @@ -193,9 +193,11 @@ Exact event ordering and examples live in `lean-beam shutdown` unpublishes the registry generation so the holder closes it cleanly. - A startup failure that reports `operation not permitted` through `.beam/beam-daemon-startup.log` is usually an environment restriction, not a bundle-resolution mismatch. -- Beam daemon disappearance errors include registry/log context and write a JSON incident record under - `.beam/daemon-failures/` or the per-root subdirectory of `BEAM_CONTROL_DIR`. Beam keeps the latest - 50 incident records and `lean-beam doctor` lists recent incident paths. +- Typed broker transport and invalid-response failures include registry/log context and write a JSON + incident record under `.beam/daemon-failures/` or the per-root subdirectory of + `BEAM_CONTROL_DIR`. Incident kinds are `brokerTransportFailure` and `invalidBrokerResponse`; + callback/display failures do not create daemon incidents. Beam keeps the latest 50 incident records, + and `lean-beam doctor` lists recent incident paths. - A standalone Beam daemon watches its canonical project root. If a git worktree or project directory is removed while the daemon is active, it shuts down its backend sessions and exits instead of remaining undiscoverable after its project-local registry disappears. A later wrapper diff --git a/docs/TESTING.md b/docs/TESTING.md index e442f38a..f96a605c 100644 --- a/docs/TESTING.md +++ b/docs/TESTING.md @@ -112,9 +112,10 @@ Current Beam coverage includes: - focused daemon lifecycle coverage in [tests/test-beam-wrapper-daemon.sh](../tests/test-beam-wrapper-daemon.sh), including the no-implicit-start contract, duplicate-owner rejection, endpoint collision safety, explicit shutdown, cancellation of requests active during shutdown or owner loss, exact-generation - cleanup that preserves a replacement registry, holder reporting after an unexpected daemon crash, - abrupt owner death through inherited-pipe EOF, stale registry cleanup, and self-termination after - the project worktree disappears + cleanup that preserves a replacement registry, registry removal before a paused daemon can finish + draining, rejection of attachment to that unpublished draining generation, holder reporting after + an unexpected daemon crash, abrupt owner death through inherited-pipe EOF, stale registry cleanup, + and self-termination after the project worktree disappears - Linux-only PID-isolated sandbox wrapper coverage in [tests/test-beam-wrapper-sandbox.sh](../tests/test-beam-wrapper-sandbox.sh), including cross-namespace endpoint attachment, duplicate-owner rejection, a paused owner without time-based expiry, explicit shutdown, killed-owner EOF cleanup, stale-registry recovery, distinct diff --git a/tests/lean/BeamTest/Broker/CliDaemonTest.lean b/tests/lean/BeamTest/Broker/CliDaemonTest.lean index 3174739a..b1b86208 100644 --- a/tests/lean/BeamTest/Broker/CliDaemonTest.lean +++ b/tests/lean/BeamTest/Broker/CliDaemonTest.lean @@ -66,6 +66,11 @@ private def expectIoErrorContains (label needle : String) (act : IO α) : IO Uni private def requireSubstring (label needle haystack : String) : IO Unit := do require s!"{label}: expected '{needle}' in '{haystack}'" (Beam.Cli.hasSubstring haystack needle) +private def brokerTransportFailure (detail : String) : Beam.Broker.BrokerClientFailure := { + kind := .transport + detail +} + private def requireJsonNat (label field : String) (expected : Nat) (json : Json) : IO Unit := do let actual ← IO.ofExcept <| json.getObjValAs? Nat field require s!"{label}: expected {field}={expected}, got {actual}" (actual == expected) @@ -533,7 +538,8 @@ private def checkDaemonFailureContext : IO Unit := do IO.FS.writeFile registryPath ((toJson entry).pretty ++ "\n") let startupLog ← Beam.Daemon.daemonStartupLogPath root IO.FS.writeFile startupLog "line 1\nline 2\n" - let msg ← Beam.Cli.daemonFailureMessage root "Beam daemon connection closed" + let detail := "synthetic broker transport failure" + let msg ← Beam.Cli.daemonFailureMessage root (brokerTransportFailure detail) requireSubstring "daemon failure context should include registry path" "Beam daemon registry" msg requireSubstring "daemon failure context should include daemon id" "daemonId: daemon-test" msg requireSubstring "daemon failure context should include dead pid status" "pid: 999999999 (not alive)" msg @@ -546,10 +552,10 @@ private def checkDaemonFailureContext : IO Unit := do let incidentJson ← readSingleDaemonFailureIncidentJson root requireJsonNat "daemon failure incident should use schema version" "schemaVersion" 1 incidentJson - requireJsonString "daemon failure incident should classify connection close" - "kind" "connectionClosed" incidentJson + requireJsonString "daemon failure incident should classify the typed transport failure" + "kind" "brokerTransportFailure" incidentJson requireJsonString "daemon failure incident should keep original detail" - "detail" "Beam daemon connection closed" incidentJson + "detail" detail incidentJson requireJsonString "daemon failure incident should include root" "root" root.toString incidentJson requireJsonString "daemon failure incident should include registry path" @@ -585,7 +591,8 @@ private def checkDaemonFailureUnreadableStartupLog : IO Unit := do IO.FS.createDirAll root let startupLog ← Beam.Daemon.daemonStartupLogPath root IO.FS.createDirAll startupLog - let msg ← Beam.Cli.daemonFailureMessage root "Beam daemon connection closed" + let msg ← Beam.Cli.daemonFailureMessage root <| + brokerTransportFailure "Beam daemon connection closed" requireSubstring "unreadable startup log should preserve original daemon failure" "Beam daemon connection closed" msg requireSubstring "unreadable startup log should still write incident path" @@ -594,8 +601,8 @@ private def checkDaemonFailureUnreadableStartupLog : IO Unit := do (!Beam.Cli.hasSubstring msg "Beam daemon log tail") let incidentJson ← readSingleDaemonFailureIncidentJson root - requireJsonString "unreadable startup log incident should classify connection close" - "kind" "connectionClosed" incidentJson + requireJsonString "unreadable startup log incident should classify transport failure" + "kind" "brokerTransportFailure" incidentJson requireJsonNull "unreadable startup log incident should omit startup log path" "startupLogPath" incidentJson requireJsonNull "unreadable startup log incident should omit startup log tail" @@ -613,6 +620,43 @@ private def checkDaemonFailureUnreadableStartupLog : IO Unit := do catch _ => pure () +private def checkTypedDaemonFailureClassification : IO Unit := do + let root := System.FilePath.mk s!"/tmp/beam-daemon-typed-failure-{← IO.monoNanosNow}" + try + IO.FS.createDirAll root + let callbackDetail := "synthetic stream callback failure" + let callbackMsg ← Beam.Cli.daemonFailureMessage root { + kind := .streamCallback + detail := callbackDetail + } + require "stream callback failure should preserve its detail" (callbackMsg == callbackDetail) + require "stream callback failure should not create a daemon incident" + (← sortedIncidentEntries root).isEmpty + + let invalidDetail := "synthetic invalid response" + let invalidMsg ← Beam.Cli.daemonFailureMessage root { + kind := .invalidResponse + detail := invalidDetail + } + requireSubstring "invalid response should include incident path" "Beam daemon incident:" invalidMsg + let incidentJson ← readSingleDaemonFailureIncidentJson root + requireJsonString "invalid response incident should retain its typed classification" + "kind" "invalidBrokerResponse" incidentJson + requireJsonString "invalid response incident should retain its detail" + "detail" invalidDetail incidentJson + finally + try + let control ← Beam.Daemon.controlDir root + if ← control.pathExists then + IO.FS.removeDirAll control + catch _ => + pure () + try + if ← root.pathExists then + IO.FS.removeDirAll root + catch _ => + pure () + private def writeTestRegistryEntry (root : System.FilePath) (port? : Option Nat := none) : IO Unit := do @@ -649,8 +693,8 @@ private def checkBrokerConnectionClosedIncident : IO Unit := do "Beam daemon incident:" msg let incidentJson ← readSingleDaemonFailureIncidentJson root - requireJsonString "broker close incident should classify connection close" - "kind" "connectionClosed" incidentJson + requireJsonString "broker close incident should classify the typed transport failure" + "kind" "brokerTransportFailure" incidentJson requireJsonStringContains "broker close incident should keep transport detail" "detail" "Beam daemon connection closed" incidentJson requireJsonString "broker close incident should include endpoint summary" @@ -676,11 +720,12 @@ private def checkDaemonFailureIncidentRetention : IO Unit := do IO.FS.createDirAll incidentDir for i in [0:55] do IO.FS.writeFile (incidentDir / s!"000000000000000000{i}.json") "{}\n" - let msg ← Beam.Cli.daemonFailureMessage root "Beam daemon connection closed" + let msg ← Beam.Cli.daemonFailureMessage root <| + brokerTransportFailure "Beam daemon connection closed" requireSubstring "retention failure should include incident path" "Beam daemon incident:" msg let entries ← sortedIncidentEntries root require s!"daemon incident retention should keep 50 files, got {entries.size}" (entries.size == 50) - let newIncidents := entries.filter (fun entry => entry.fileName.contains "connectionClosed") + let newIncidents := entries.filter (fun entry => entry.fileName.contains "brokerTransportFailure") require "daemon incident retention should keep newly written incident" (newIncidents.size == 1) let some newIncident := newIncidents[0]? @@ -708,7 +753,8 @@ private def checkDoctorDaemonFailureIncidentLines : IO Unit := do require "doctor should report no daemon incidents when directory is absent" (absentLines == ["daemon incidents: none"]) - discard <| Beam.Cli.daemonFailureMessage root "Beam daemon connection closed" + discard <| Beam.Cli.daemonFailureMessage root <| + brokerTransportFailure "Beam daemon connection closed" let lines ← Beam.Cli.daemonFailureIncidentDoctorLines root require s!"doctor should report one recent daemon incident, got {lines}" (lines.head? == some "daemon incidents: 1 recent") @@ -1227,6 +1273,7 @@ def main : IO Unit := do checkDaemonDebugWarnings checkDaemonFailureContext checkDaemonFailureUnreadableStartupLog + checkTypedDaemonFailureClassification checkPlainBrokerTaskCancellation checkBrokerConnectionClosedIncident checkDaemonFailureIncidentRetention diff --git a/tests/lean/BeamTest/Broker/McpProtocolTest.lean b/tests/lean/BeamTest/Broker/McpProtocolTest.lean index 903f7cc9..6c98732a 100644 --- a/tests/lean/BeamTest/Broker/McpProtocolTest.lean +++ b/tests/lean/BeamTest/Broker/McpProtocolTest.lean @@ -1397,12 +1397,6 @@ private def callLeanSync handleRpcRequestWithNotifications state opts notifications s!"lean_sync {path}" id "tools/call" <| some <| toolCallParams "lean_sync" arguments -private def shutdownMcpRuntime (state : Beam.Mcp.Server.ServerState) : IO Unit := do - match ← state.runtime? with - | none => pure () - | some runtime => - discard <| runtime.close - private def checkIdempotentLifecycleTools : IO Unit := do let root ← mkTempProjectRoot "lean-beam-mcp-idempotent-lifecycle" let state ← Beam.Mcp.Server.ServerState.create @@ -1463,8 +1457,12 @@ private def checkIdempotentLifecycleTools : IO Unit := do repeatedDropStructured requireJsonString "repeated lean_drop_workspace structured result" "reason" "notFound" repeatedDropStructured + + state.closeRuntime + require "MCP runtime close should clear ServerState ownership" (← state.runtime?).isNone + state.closeRuntime finally - shutdownMcpRuntime state + state.closeRuntime try if ← root.pathExists then IO.FS.removeDirAll root @@ -1605,7 +1603,7 @@ private def checkDiagnosticLogForwarding : IO Unit := do requireJsonBool "error log data" "completion_blocking" false errorData requireFieldAbsent "error log data" "save_blocking" errorData finally - shutdownMcpRuntime state + state.closeRuntime try if ← root.pathExists then IO.FS.removeDirAll root diff --git a/tests/test-beam-wrapper-sandbox.sh b/tests/test-beam-wrapper-sandbox.sh index 026b0a87..1411ca7f 100755 --- a/tests/test-beam-wrapper-sandbox.sh +++ b/tests/test-beam-wrapper-sandbox.sh @@ -95,10 +95,10 @@ wait_for_registry() { return 1 } -assert_no_connection_closed_incidents() { +assert_no_daemon_failure_incidents() { local label="$1" - if find "$control_root" -path '*/daemon-failures/*connectionClosed*.json' -print -quit | grep -q .; then - echo "expected $label to produce no connectionClosed incident" >&2 + if find "$control_root" -path '*/daemon-failures/*.json' -print -quit | grep -q .; then + echo "expected $label to produce no daemon failure incident" >&2 find "$control_root" -path '*/daemon-failures/*.json' -print -exec sed -n '1,160p' {} \; >&2 exit 1 fi @@ -338,4 +338,4 @@ fi owner_pid="" assert_no_lease_artifacts -assert_no_connection_closed_incidents "the explicit sandbox ownership regressions" +assert_no_daemon_failure_incidents "the explicit sandbox ownership regressions" From cbf874dd1260a6f7ea5510f41a717959e6d035ab Mon Sep 17 00:00:00 2001 From: Emilio Jesus Gallego Arias Date: Wed, 26 Aug 2026 19:09:32 +0200 Subject: [PATCH 11/28] fix: harden MCP completion and client failures --- Beam/Broker/Client.lean | 48 ++++---- Beam/Cli/DaemonManager.lean | 10 +- Beam/Mcp/StdioServer.lean | 103 +++++++++++------- docs/DEVELOPMENT.md | 8 ++ docs/MCP.md | 21 ++++ docs/TESTING.md | 14 +-- tests/lean/BeamTest/Broker/CliDaemonTest.lean | 18 +-- tests/test-mcp-stdio.py | 14 +++ 8 files changed, 147 insertions(+), 89 deletions(-) diff --git a/Beam/Broker/Client.lean b/Beam/Broker/Client.lean index 4ec5148f..c3735c1a 100644 --- a/Beam/Broker/Client.lean +++ b/Beam/Broker/Client.lean @@ -19,18 +19,19 @@ structure StreamCallbacks where abbrev Endpoint := Transport.Endpoint -/-- Classify failures produced while a broker client exchanges one streamed request. -/ -inductive BrokerClientFailureKind where - | transport - | invalidResponse - | streamCallback - deriving BEq, Repr - /-- Keep broker client failures typed until a CLI or transport presentation boundary. -/ -structure BrokerClientFailure where - kind : BrokerClientFailureKind - detail : String - deriving BEq, Repr +inductive BrokerClientFailure where + | transport (error : IO.Error) + | invalidResponse (detail : String) + | streamCallback (error : IO.Error) + +def BrokerClientFailure.detail : BrokerClientFailure → String + | .transport error | .streamCallback error => error.toString + | .invalidResponse detail => detail + +def BrokerClientFailure.toIOError : BrokerClientFailure → IO.Error + | .transport error | .streamCallback error => error + | .invalidResponse detail => IO.userError detail def parsePortText (name value : String) : Except String UInt16 := do let some n := value.toNat? @@ -47,21 +48,21 @@ def parseEndpointOption (args : List String) : Except String (Endpoint × List S | _ => pure (.tcp 8765, args) -private def decodeStreamMessage (msg : String) : IO StreamMessage := do +private def decodeStreamMessage (msg : String) : Except String StreamMessage := do match Json.parse msg with - | .error err => throw <| IO.userError s!"invalid Beam daemon response json: {err}" + | .error err => throw s!"invalid Beam daemon response json: {err}" | .ok json => match fromJson? (α := StreamMessage) json with | .ok stream => pure stream - | .error err => throw <| IO.userError s!"invalid Beam daemon response payload: {err}" + | .error err => throw s!"invalid Beam daemon response payload: {err}" private def captureClientFailure - (kind : BrokerClientFailureKind) + (failure : IO.Error → BrokerClientFailure) (action : IO α) : IO (Except BrokerClientFailure α) := do try pure <| .ok (← action) catch e => - pure <| .error { kind, detail := e.toString } + pure <| .error (failure e) private def diagnosticSeverityLabel : Option Lsp.DiagnosticSeverity → String | some .error => "error" @@ -108,15 +109,12 @@ partial def sendRequestWithStreamResult | .ok msg => pure msg | .error failure => return .error failure let stream ← - match ← captureClientFailure .invalidResponse (decodeStreamMessage msg) with + match decodeStreamMessage msg with | .ok stream => pure stream - | .error failure => return .error failure + | .error detail => return .error (.invalidResponse detail) unless stream.clientRequestId? == req.clientRequestId? do - return .error { - kind := .invalidResponse - detail := - s!"Beam daemon stream request id {stream.clientRequestId?} does not match request id {req.clientRequestId?}" - } + return .error <| .invalidResponse <| + s!"Beam daemon stream request id {stream.clientRequestId?} does not match request id {req.clientRequestId?}" match ← captureClientFailure .streamCallback (onStream stream) with | .ok () => pure () | .error failure => return .error failure @@ -135,7 +133,7 @@ partial def sendRequestWithStream (onStream : StreamMessage → IO Unit) : IO Response := do match ← sendRequestWithStreamResult endpoint req onStream with | .ok response => pure response - | .error failure => throw <| IO.userError failure.detail + | .error failure => throw failure.toIOError /-- Send one request with typed client failures and structured progress callbacks. -/ partial def sendRequestWithCallbacksResult @@ -157,7 +155,7 @@ partial def sendRequestWithCallbacks (callbacks : StreamCallbacks := {}) : IO Response := do match ← sendRequestWithCallbacksResult endpoint req callbacks with | .ok response => pure response - | .error failure => throw <| IO.userError failure.detail + | .error failure => throw failure.toIOError def sendRequest (endpoint : Endpoint) (req : Request) : IO Response := sendRequestWithCallbacks endpoint req diff --git a/Beam/Cli/DaemonManager.lean b/Beam/Cli/DaemonManager.lean index 4d27cf68..97eb2bfe 100644 --- a/Beam/Cli/DaemonManager.lean +++ b/Beam/Cli/DaemonManager.lean @@ -220,10 +220,10 @@ private structure DaemonFailureIncident where private def daemonFailureIncidentSchemaVersion : Nat := 1 -private def daemonFailureIncidentKind? : BrokerClientFailureKind → Option String - | .transport => some "brokerTransportFailure" - | .invalidResponse => some "invalidBrokerResponse" - | .streamCallback => none +private def daemonFailureIncidentKind? : BrokerClientFailure → Option String + | .transport _ => some "brokerTransportFailure" + | .invalidResponse _ => some "invalidBrokerResponse" + | .streamCallback _ => none private def daemonFailureIncidentTimestampLabel (timestamp : String) : String := (timestamp.replace "-" "").replace ":" "" @@ -283,7 +283,7 @@ def daemonFailureMessage (root : System.FilePath) (failure : BrokerClientFailure) : IO String := do let detail := failure.detail - match daemonFailureIncidentKind? failure.kind with + match daemonFailureIncidentKind? failure with | none => pure detail | some kind => diff --git a/Beam/Mcp/StdioServer.lean b/Beam/Mcp/StdioServer.lean index a9b26044..3ab5a5ec 100644 --- a/Beam/Mcp/StdioServer.lean +++ b/Beam/Mcp/StdioServer.lean @@ -92,7 +92,10 @@ private structure InFlightRequest where private structure RoutingState where nextBrokerId : Nat := 1 + -- `inFlight` is the client-ID routing surface. `admitted` retains exact generations through + -- terminal output so EOF shutdown and control fences can still await them after ID retirement. inFlight : Std.TreeMap RequestId InFlightRequest := {} + admitted : Std.TreeMap String InFlightRequest := {} controlBarrier? : Option (IO.Promise Unit) := none closing : Bool := false @@ -146,10 +149,11 @@ private def Coordinator.registerRequest routing with nextBrokerId := routing.nextBrokerId + 1 inFlight := routing.inFlight.insert id request + admitted := routing.admitted.insert brokerId request } pure <| .ok request -private def Coordinator.eraseRequest +private def Coordinator.retireRequestId (coordinator : Coordinator) (request : InFlightRequest) : IO Unit := do coordinator.routing.atomically do @@ -162,6 +166,13 @@ private def Coordinator.eraseRequest routing | none => routing +private def Coordinator.completeRequest + (coordinator : Coordinator) + (request : InFlightRequest) : IO Unit := do + coordinator.routing.atomically do + modify fun routing => + { routing with admitted := routing.admitted.erase request.brokerId } + private def InFlightRequest.resolveDone (request : InFlightRequest) : IO Unit := do try request.done.resolve () @@ -225,22 +236,27 @@ private def Coordinator.finishRequest (coordinator : Coordinator) (request : InFlightRequest) (response : Json) : IO Unit := do + let sendResponse ← request.state.atomically do + let current ← get + match current.phase with + | .active => + set { current with phase := .completed } + pure true + | .clientCancelled => + set { current with phase := .completed } + pure false + | .completed => + pure false + -- Retire the exact admission before its terminal response becomes visible. A client may reuse + -- an ID as soon as it observes that response; retaining the routing entry until after the write + -- creates a race in which the new request is mistaken for a duplicate active request. + coordinator.retireRequestId request try - let sendResponse ← request.state.atomically do - let current ← get - match current.phase with - | .active => - set { current with phase := .completed } - pure true - | .clientCancelled => - set { current with phase := .completed } - pure false - | .completed => - pure false if sendResponse then coordinator.output.send response finally - coordinator.eraseRequest request + -- Barriers continue to observe completion only after the terminal write has finished. + coordinator.completeRequest request request.resolveDone private def InFlightRequest.markClientCancelled @@ -280,7 +296,7 @@ private def Coordinator.beginClosing (coordinator : Coordinator) : IO (Bool × Array InFlightRequest) := do let (alreadyClosing, requests) ← coordinator.routing.atomically do let routing ← get - let requests := routing.inFlight.toList.map Prod.snd |>.toArray + let requests := routing.admitted.toList.map Prod.snd |>.toArray set { routing with closing := true } pure (routing.closing, requests) for request in requests do @@ -300,7 +316,7 @@ private def Coordinator.otherInFlightRequests (coordinator : Coordinator) (request : InFlightRequest) : IO (Array InFlightRequest) := do coordinator.routing.atomically do - pure <| (← get).inFlight.toList.filterMap (fun (_, other) => + pure <| (← get).admitted.toList.filterMap (fun (_, other) => if other.brokerId == request.brokerId then none else some other) |>.toArray private def Coordinator.closeTransport (coordinator : Coordinator) : IO Unit := do @@ -345,7 +361,7 @@ private def Coordinator.executeToolRequest catch e => pure <| errorResponse req.id (RpcError.internalError e.toString) -private def Coordinator.runToolRequest +private def Coordinator.toolRequestResponse (coordinator : Coordinator) (opts : Options) (req : Request) @@ -353,23 +369,24 @@ private def Coordinator.runToolRequest (parsedParams : Except String CallToolParams) (request : InFlightRequest) (barrier? : Option (IO.Promise Unit)) - (initialProgress : Nat := 0) - (beforeFinish : IO Unit := pure ()) : IO Unit := do - let response ← - try - awaitControlBarrier barrier? - if ← request.isActive then - coordinator.executeToolRequest opts req admitted parsedParams request initialProgress - else - pure <| errorResponse req.id <| - RpcError.invalidRequest "request was cancelled before execution" - catch e => - pure <| errorResponse req.id (RpcError.internalError e.toString) + (initialProgress : Nat := 0) : IO Json := do try - beforeFinish + awaitControlBarrier barrier? + if ← request.isActive then + coordinator.executeToolRequest opts req admitted parsedParams request initialProgress + else + pure <| errorResponse req.id <| + RpcError.invalidRequest "request was cancelled before execution" + catch e => + pure <| errorResponse req.id (RpcError.internalError e.toString) + +private def finishReporterSafely + (req : Request) + (finishReporter : IO Unit) : IO Unit := do + try + finishReporter catch e => Internal.traceMcp s!"request reporter finish failed id={req.id.label}: {e.toString}" - coordinator.finishRequest request response private def Coordinator.spawnToolRequest (coordinator : Coordinator) @@ -387,7 +404,9 @@ private def Coordinator.spawnToolRequest let barrier? ← coordinator.currentControlBarrier? let _ ← IO.asTask (prio := Task.Priority.dedicated) do try - coordinator.runToolRequest opts req admitted parsedParams request barrier? + let response ← + coordinator.toolRequestResponse opts req admitted parsedParams request barrier? + coordinator.finishRequest request response catch e => if !Beam.Mcp.Stdio.isBrokenPipeError e then Internal.traceMcp s!"request completion failed id={req.id.label}: {e.toString}" @@ -420,20 +439,24 @@ private def Coordinator.handleControlToolRequest let (previous?, done) ← coordinator.pushControlBarrier let priorRequests ← coordinator.otherInFlightRequests request let _ ← IO.asTask (prio := Task.Priority.dedicated) do + let response ← + try + -- A control operation is a full stream-order fence: work admitted before it drains, + -- while work admitted afterward waits on `done`. + coordinator.awaitRequests priorRequests + coordinator.toolRequestResponse opts req admitted parsedParams request previous? + initialProgress + catch e => + if !Beam.Mcp.Stdio.isBrokenPipeError e then + Internal.traceMcp s!"workspace control completion failed id={req.id.label}: {e.toString}" + pure <| errorResponse req.id (RpcError.internalError e.toString) try - -- A control operation is a full stream-order fence: work admitted before it drains, - -- while work admitted afterward waits on `done`. - coordinator.awaitRequests priorRequests - coordinator.runToolRequest opts req admitted parsedParams request previous? - initialProgress finishReporter + finishReporterSafely req finishReporter + coordinator.finishRequest request response catch e => if !Beam.Mcp.Stdio.isBrokenPipeError e then Internal.traceMcp s!"workspace control completion failed id={req.id.label}: {e.toString}" - finishReporter - coordinator.finishRequest request <| - errorResponse req.id (RpcError.internalError e.toString) finally - finishReporter resolvePromise done pure () diff --git a/docs/DEVELOPMENT.md b/docs/DEVELOPMENT.md index 4d649e30..2a50a794 100644 --- a/docs/DEVELOPMENT.md +++ b/docs/DEVELOPMENT.md @@ -184,6 +184,14 @@ In-process MCP lifecycle calls use the broker's typed `initWorkspaceWithConfig` results directly; do not route them through broker JSON dispatch and decode them back into the same types. +CLI and MCP share semantic dispatch, not transport coordination. A daemon accepts one broker request +per socket connection, so the connection itself supplies response routing and disconnect lifetime. +MCP multiplexes requests over one stdio stream, so `Beam.Mcp.StdioServer` must own exact JSON-RPC ID +routing, serialized output, client cancellation, and workspace-control barriers. Both paths converge +on `ServerRuntime.dispatchRequestWithHandle`, whose admission handle is the shared cancellation and +drain boundary. Keep the two ingress coordinators separate unless a future transport has the same +wire-level ownership rules; do not duplicate semantic operation dispatch above that boundary. + The executable path is split into importable modules: - [Beam/Mcp/Protocol.lean](../Beam/Mcp/Protocol.lean): current MCP JSON-RPC helpers diff --git a/docs/MCP.md b/docs/MCP.md index b1629c42..5ed126c8 100644 --- a/docs/MCP.md +++ b/docs/MCP.md @@ -130,6 +130,27 @@ After editing a lakefile, manifest, package override, `lean-toolchain`, Lean opt dynamic libraries, drop that workspace or restart the MCP server before the next request. Re-syncing inside the old Lean process is not sufficient to reload workspace configuration. +## Transport Lifetime + +The `lean-beam-mcp` stdio process owns its optional in-process broker runtime. MCP clients do not +start or attach to a wrapper daemon and do not need a separate `lean-beam ensure --hold` owner. The +first workspace-bound request creates the runtime lazily; later descriptors share that runtime while +the broker remains authoritative for workspace membership. `lean_drop_workspace` evicts one cached +workspace but does not end the stdio session. + +Closing stdin or reaching EOF closes MCP request admission. The server cooperatively cancels active +cancellable requests, waits for every admitted request and non-cancellable workspace eviction to +finish, then closes the broker runtime and all remaining backend sessions. A JSON-RPC request ID is +active only until its request reaches terminal completion. The server retires that exact admission +before publishing its terminal response, so a client may reuse the ID after observing the response; +the completion barrier is resolved only after the response write finishes. + +CLI ingress has a different transport lifetime: a standalone daemon accepts one request per socket +connection, and disconnecting that connection cancels its exact broker admission. MCP carries many +overlapping requests on one stdio stream and therefore owns application-level ID routing, +cancellation, output serialization, and workspace-control fences. Both paths use the same broker +request dispatcher and runtime teardown boundary. + ## Code Ownership - [Beam/Broker/Protocol.lean](../Beam/Broker/Protocol.lean) owns broker request, response, handle, diff --git a/docs/TESTING.md b/docs/TESTING.md index f96a605c..b45b9425 100644 --- a/docs/TESTING.md +++ b/docs/TESTING.md @@ -226,13 +226,13 @@ PYTHONDONTWRITEBYTECODE=1 python3 tests/test-mcp-stdio.py \ ``` This scenario covers out-of-order tool responses, exact string/numeric request-ID separation, -duplicate active-ID suppression, exact modern and legacy broker cancellation, per-request progress -ordering, deterministic overlap between a gated request in one workspace and a fast request in -another, single-flight first use, simultaneous cold first use of distinct roots, stateless -multi-root isolation, non-cancellable cache eviction with ordering on both sides of the global -fence, lazy recreation, and EOF cancellation and teardown. The full stdio suite also checks modern -request-ID reuse after a terminal response and rejects a proof handle carried across an MCP process -restart. The slow Beam suite runs +duplicate active-ID suppression, repeated asynchronous request-ID reuse after terminal responses, +exact modern and legacy broker cancellation, per-request progress ordering, deterministic overlap +between a gated request in one workspace and a fast request in another, single-flight first use, +simultaneous cold first use of distinct roots, stateless multi-root isolation, non-cancellable cache +eviction with ordering on both sides of the global fence, lazy recreation, and EOF cancellation and +teardown. The full stdio suite also checks modern request-ID reuse and rejects a proof handle carried +across an MCP process restart. The slow Beam suite runs `--scenario multi-toolchain-workspaces` after installing both fixture toolchains and verifies that one MCP process keeps both project-specific Lean sessions active. diff --git a/tests/lean/BeamTest/Broker/CliDaemonTest.lean b/tests/lean/BeamTest/Broker/CliDaemonTest.lean index b1b86208..16bb2217 100644 --- a/tests/lean/BeamTest/Broker/CliDaemonTest.lean +++ b/tests/lean/BeamTest/Broker/CliDaemonTest.lean @@ -66,10 +66,8 @@ private def expectIoErrorContains (label needle : String) (act : IO α) : IO Uni private def requireSubstring (label needle haystack : String) : IO Unit := do require s!"{label}: expected '{needle}' in '{haystack}'" (Beam.Cli.hasSubstring haystack needle) -private def brokerTransportFailure (detail : String) : Beam.Broker.BrokerClientFailure := { - kind := .transport - detail -} +private def brokerTransportFailure (detail : String) : Beam.Broker.BrokerClientFailure := + .transport (IO.userError detail) private def requireJsonNat (label field : String) (expected : Nat) (json : Json) : IO Unit := do let actual ← IO.ofExcept <| json.getObjValAs? Nat field @@ -625,19 +623,15 @@ private def checkTypedDaemonFailureClassification : IO Unit := do try IO.FS.createDirAll root let callbackDetail := "synthetic stream callback failure" - let callbackMsg ← Beam.Cli.daemonFailureMessage root { - kind := .streamCallback - detail := callbackDetail - } + let callbackMsg ← Beam.Cli.daemonFailureMessage root <| + .streamCallback (IO.userError callbackDetail) require "stream callback failure should preserve its detail" (callbackMsg == callbackDetail) require "stream callback failure should not create a daemon incident" (← sortedIncidentEntries root).isEmpty let invalidDetail := "synthetic invalid response" - let invalidMsg ← Beam.Cli.daemonFailureMessage root { - kind := .invalidResponse - detail := invalidDetail - } + let invalidMsg ← Beam.Cli.daemonFailureMessage root <| + .invalidResponse invalidDetail requireSubstring "invalid response should include incident path" "Beam daemon incident:" invalidMsg let incidentJson ← readSingleDaemonFailureIncidentJson root requireJsonString "invalid response incident should retain its typed classification" diff --git a/tests/test-mcp-stdio.py b/tests/test-mcp-stdio.py index 07542153..3e7e80d9 100644 --- a/tests/test-mcp-stdio.py +++ b/tests/test-mcp-stdio.py @@ -1866,6 +1866,20 @@ def run_concurrent_dispatch(repo_root, fixture_root, timeout, server_trace=False require(isinstance(slow_structured, dict), f"slow runAt missing structured content: {slow_result}") require_success("slow concurrent runAt", slow_structured) expect_result(client.request("ping", request_id=slow_id)) + async_reuse_id = "async-tool-request-id-reuse" + for reuse_iteration in range(64): + version_result = expect_result( + client.request( + "tools/call", + {"name": "beam_version", "arguments": {}}, + request_id=async_reuse_id, + ) + ) + require( + version_result.get("isError") is not True, + f"async request-ID reuse tool call {reuse_iteration} failed: {version_result}", + ) + expect_result(client.request("ping", request_id=async_reuse_id)) require( len(status_log_notifications(client, slow_id)) == status_count, f"slow no-token runAt emitted duplicate or post-response statuses: {client.notifications}", From a9011198c763981c3e2b258097274146a12458b0 Mon Sep 17 00:00:00 2001 From: Emilio Jesus Gallego Arias Date: Wed, 26 Aug 2026 20:05:17 +0200 Subject: [PATCH 12/28] fix: harden MCP transport teardown --- Beam/Broker/Client.lean | 2 +- Beam/Mcp/Server.lean | 2 +- Beam/Mcp/StdioServer.lean | 32 ++++---- docs/MCP.md | 9 ++- .../lean/BeamTest/Broker/McpProtocolTest.lean | 80 +++++++++++++++---- tests/test-mcp-stdio.py | 37 +++++++++ 6 files changed, 125 insertions(+), 37 deletions(-) diff --git a/Beam/Broker/Client.lean b/Beam/Broker/Client.lean index c3735c1a..00ed387d 100644 --- a/Beam/Broker/Client.lean +++ b/Beam/Broker/Client.lean @@ -29,7 +29,7 @@ def BrokerClientFailure.detail : BrokerClientFailure → String | .transport error | .streamCallback error => error.toString | .invalidResponse detail => detail -def BrokerClientFailure.toIOError : BrokerClientFailure → IO.Error +private def BrokerClientFailure.toIOError : BrokerClientFailure → IO.Error | .transport error | .streamCallback error => error | .invalidResponse detail => IO.userError detail diff --git a/Beam/Mcp/Server.lean b/Beam/Mcp/Server.lean index 695dc81d..9864dcac 100644 --- a/Beam/Mcp/Server.lean +++ b/Beam/Mcp/Server.lean @@ -59,7 +59,7 @@ def ServerState.create : IO ServerState := do def ServerState.protocolState (state : ServerState) : IO ProtocolState := state.protocol.atomically get -def ServerState.runtime? (state : ServerState) : IO (Option Beam.Broker.ServerRuntime) := +private def ServerState.runtime? (state : ServerState) : IO (Option Beam.Broker.ServerRuntime) := state.runtime.get private def ServerState.withRuntimeControl diff --git a/Beam/Mcp/StdioServer.lean b/Beam/Mcp/StdioServer.lean index 3ab5a5ec..74fbff70 100644 --- a/Beam/Mcp/StdioServer.lean +++ b/Beam/Mcp/StdioServer.lean @@ -272,6 +272,19 @@ private def InFlightRequest.markClientCancelled | .clientCancelled | .completed => pure (false, none) +private def InFlightRequest.cancel (request : InFlightRequest) : IO Unit := do + let (cancelled, brokerRequest?) ← request.markClientCancelled + if cancelled then + match brokerRequest? with + | none => pure () + | some brokerRequest => + let _ ← IO.asTask (prio := Task.Priority.dedicated) do + try + discard <| brokerRequest.cancel + catch e => + Internal.traceMcp s!"broker cancellation failed id={request.id.label}: {e.toString}" + pure () + private def Coordinator.cancelRequest (coordinator : Coordinator) (id : RequestId) : IO Unit := do @@ -279,18 +292,7 @@ private def Coordinator.cancelRequest pure <| (← get).inFlight.get? id match request? with | none => pure () - | some request => - let (cancelled, brokerRequest?) ← request.markClientCancelled - if cancelled then - match brokerRequest? with - | none => pure () - | some brokerRequest => - let _ ← IO.asTask (prio := Task.Priority.dedicated) do - try - discard <| brokerRequest.cancel - catch e => - Internal.traceMcp s!"broker cancellation failed id={id.label}: {e.toString}" - pure () + | some request => request.cancel private def Coordinator.beginClosing (coordinator : Coordinator) : IO (Bool × Array InFlightRequest) := do @@ -300,7 +302,7 @@ private def Coordinator.beginClosing set { routing with closing := true } pure (routing.closing, requests) for request in requests do - coordinator.cancelRequest request.id + request.cancel pure (alreadyClosing, requests) private def awaitRequestDone (request : InFlightRequest) : IO Unit := do @@ -312,7 +314,7 @@ private def Coordinator.awaitRequests for request in requests do awaitRequestDone request -private def Coordinator.otherInFlightRequests +private def Coordinator.otherAdmittedRequests (coordinator : Coordinator) (request : InFlightRequest) : IO (Array InFlightRequest) := do coordinator.routing.atomically do @@ -437,7 +439,7 @@ private def Coordinator.handleControlToolRequest | some reporter => reporter.finish | none => pure () let (previous?, done) ← coordinator.pushControlBarrier - let priorRequests ← coordinator.otherInFlightRequests request + let priorRequests ← coordinator.otherAdmittedRequests request let _ ← IO.asTask (prio := Task.Priority.dedicated) do let response ← try diff --git a/docs/MCP.md b/docs/MCP.md index 5ed126c8..5151953d 100644 --- a/docs/MCP.md +++ b/docs/MCP.md @@ -134,9 +134,10 @@ inside the old Lean process is not sufficient to reload workspace configuration. The `lean-beam-mcp` stdio process owns its optional in-process broker runtime. MCP clients do not start or attach to a wrapper daemon and do not need a separate `lean-beam ensure --hold` owner. The -first workspace-bound request creates the runtime lazily; later descriptors share that runtime while -the broker remains authoritative for workspace membership. `lean_drop_workspace` evicts one cached -workspace but does not end the stdio session. +first Lean operation that needs broker execution creates the runtime lazily; later descriptors share +that runtime while the broker remains authoritative for workspace membership. Feedback and cache +eviction can inspect or update an absent runtime without creating one. `lean_drop_workspace` evicts +one cached workspace but does not end the stdio session. Closing stdin or reaching EOF closes MCP request admission. The server cooperatively cancels active cancellable requests, waits for every admitted request and non-cancellable workspace eviction to @@ -145,7 +146,7 @@ active only until its request reaches terminal completion. The server retires th before publishing its terminal response, so a client may reuse the ID after observing the response; the completion barrier is resolved only after the response write finishes. -CLI ingress has a different transport lifetime: a standalone daemon accepts one request per socket +CLI ingress has a different transport lifetime: a broker daemon accepts one request per socket connection, and disconnecting that connection cancels its exact broker admission. MCP carries many overlapping requests on one stdio stream and therefore owns application-level ID routing, cancellation, output serialization, and workspace-control fences. Both paths use the same broker diff --git a/tests/lean/BeamTest/Broker/McpProtocolTest.lean b/tests/lean/BeamTest/Broker/McpProtocolTest.lean index 6c98732a..e3f0c1f6 100644 --- a/tests/lean/BeamTest/Broker/McpProtocolTest.lean +++ b/tests/lean/BeamTest/Broker/McpProtocolTest.lean @@ -757,6 +757,51 @@ private def expectToolErrorCode (label expectedCode : String) (resp : Json) : IO requireJsonString s!"{label} structured error" "code" expectedCode structured pure structured +private def requireRuntimeActiveResult + (label : String) + (expected : Bool) + (response : Json) : IO Unit := do + let result ← requireObjVal s!"{label} response" "result" response + requireJsonBool s!"{label} result" "isError" false result + let structured ← requireObjVal s!"{label} result" "structuredContent" result + requireJsonBool s!"{label} structured" "runtime_active" expected structured + +private def requireLegacyRuntimeActive + (state : Beam.Mcp.Server.ServerState) + (opts : Beam.Mcp.Server.Options) + (label : String) + (id : Nat) + (expected : Bool) + (notifications : Beam.Mcp.Server.NotificationSink := {}) : IO Unit := do + let response ← handleRpcRequestWithNotifications state opts notifications label id "tools/call" <| + some <| toolCallParams "beam_version" + requireRuntimeActiveResult label expected response + +private def requireModernRuntimeActive + (state : Beam.Mcp.Server.ServerState) + (opts : Beam.Mcp.Server.Options) + (label : String) + (id : Nat) + (expected : Bool) : IO Unit := do + let response ← handleRpcRequest state opts label id "tools/call" <| some <| modernParams [ + ("name", toJson "beam_version"), + ("arguments", Json.mkObj []) + ] + requireRuntimeActiveResult label expected response + +private def legacyStatsWorkspaces + (state : Beam.Mcp.Server.ServerState) + (opts : Beam.Mcp.Server.Options) + (notifications : Beam.Mcp.Server.NotificationSink) + (label : String) + (id : Nat) : IO Json := do + let response ← handleRpcRequestWithNotifications state opts notifications label id "tools/call" <| + some <| toolCallParams "beam_stats" + let result ← requireObjVal s!"{label} response" "result" response + requireJsonBool s!"{label} result" "isError" false result + let structured ← requireObjVal s!"{label} result" "structuredContent" result + requireObjVal s!"{label} structured" "workspaces" structured + private def requireModernResultEnvelope (label : String) (result : Json) : IO Unit := do requireJsonString label "resultType" "complete" result let resultMeta ← requireObjVal label "_meta" result @@ -856,8 +901,8 @@ private def checkModernProtocol : IO Unit := do requireModernResultEnvelope "modern confidential beam_feedback_report result" feedbackResult requireConfidentialFeedbackResult "modern confidential beam_feedback_report" confidentialSecret feedbackResult - require "modern beam_feedback_report should not create a broker runtime" - (← state.runtime?).isNone + requireModernRuntimeActive state opts + "modern beam_feedback_report should not create a broker runtime" 1022 false let preservedMetaResult := Beam.Mcp.modernResult <| Json.mkObj [ ("_meta", Json.mkObj [("example.test/value", toJson "preserved")]) @@ -1163,8 +1208,8 @@ private def checkServerBasics : IO Unit := do requireConfidentialFeedbackResult "beam feedback confidential" confidentialSecret feedbackConfidentialResult - require "beam_feedback_report should not create a broker runtime" - (← state.runtime?).isNone + requireLegacyRuntimeActive state opts + "beam_feedback_report should not create a broker runtime" 26 false let uncachedDropResp ← handleRpcRequest state opts "drop uncached workspace" 24 "tools/call" <| some <| toolCallParams "lean_drop_workspace" <| withWorkspace root (Json.mkObj []) @@ -1182,8 +1227,8 @@ private def checkServerBasics : IO Unit := do uncachedDropStructured requireJsonString "drop uncached workspace structured result" "reason" "notFound" uncachedDropStructured - require "dropping an uncached workspace should not create a broker runtime" - (← state.runtime?).isNone + requireLegacyRuntimeActive state opts + "dropping an uncached workspace should not create a broker runtime" 27 false let rawToolResp ← handleRpcRequest state opts "raw tool rejection" 3 "tools/call" <| some <| toolCallParams Beam.LSP.RunAt.method @@ -1413,12 +1458,13 @@ private def checkIdempotentLifecycleTools : IO Unit := do requireJsonBool "lifecycle lean_sync result" "isError" false syncResult let canonicalRoot ← Beam.resolveExistingPath root let workspaceId := (Beam.Workspace.Descriptor.ofRoot canonicalRoot).cacheKey - let some runtime ← state.runtime? - | throw <| IO.userError "lifecycle lean_sync did not create a broker runtime" - require "MCP lifecycle should read workspace ownership from the broker" - ((← runtime.workspaceRoot? workspaceId) == some canonicalRoot) + let workspaces ← legacyStatsWorkspaces state opts notifications + "lifecycle stats after sync" 3 + let workspaceStats ← requireObjVal + "lifecycle stats after sync workspaces" workspaceId workspaces + requireJsonString "lifecycle stats workspace" "root" canonicalRoot.toString workspaceStats - for (id, label) in #[(3, "first"), (4, "repeated")] do + for (id, label) in #[(4, "first"), (5, "repeated")] do let closeResp ← handleRpcRequestWithNotifications state opts notifications s!"{label} lean close" id "tools/call" <| some <| toolCallParams "lean_close" <| withWorkspace root <| Json.mkObj [ @@ -1431,7 +1477,7 @@ private def checkIdempotentLifecycleTools : IO Unit := do requireJsonBool s!"{label} lean_close structured result" "closed" true closeStructured let firstDropResp ← handleRpcRequestWithNotifications state opts notifications - "first lean drop workspace" 5 "tools/call" <| + "first lean drop workspace" 6 "tools/call" <| some <| toolCallParams "lean_drop_workspace" <| withWorkspace root (Json.mkObj []) let firstDropResult ← requireObjVal "first lean_drop_workspace response" "result" firstDropResp requireJsonBool "first lean_drop_workspace result" "isError" false firstDropResult @@ -1440,11 +1486,12 @@ private def checkIdempotentLifecycleTools : IO Unit := do requireJsonBool "first lean_drop_workspace structured result" "dropped" true firstDropStructured requireJsonBool "first lean_drop_workspace structured result" "invalidated_handles" true firstDropStructured - require "MCP workspace drop should update broker-owned workspace state" - ((← runtime.workspaceRoot? workspaceId) == none) + let workspaces ← legacyStatsWorkspaces state opts notifications + "lifecycle stats after drop" 7 + requireFieldAbsent "lifecycle stats after drop workspaces" workspaceId workspaces let repeatedDropResp ← handleRpcRequestWithNotifications state opts notifications - "repeated lean drop workspace" 6 "tools/call" <| + "repeated lean drop workspace" 8 "tools/call" <| some <| toolCallParams "lean_drop_workspace" <| withWorkspace root (Json.mkObj []) let repeatedDropResult ← requireObjVal "repeated lean_drop_workspace response" "result" repeatedDropResp @@ -1459,7 +1506,8 @@ private def checkIdempotentLifecycleTools : IO Unit := do repeatedDropStructured state.closeRuntime - require "MCP runtime close should clear ServerState ownership" (← state.runtime?).isNone + requireLegacyRuntimeActive state opts + "MCP runtime close should clear ServerState ownership" 9 false notifications state.closeRuntime finally state.closeRuntime diff --git a/tests/test-mcp-stdio.py b/tests/test-mcp-stdio.py index 3e7e80d9..95de965a 100644 --- a/tests/test-mcp-stdio.py +++ b/tests/test-mcp-stdio.py @@ -2089,7 +2089,44 @@ def run_concurrent_dispatch(repo_root, fixture_root, timeout, server_trace=False eof_request_id = "eof-inflight" client.send_request("tools/call", slow_params, request_id=eof_request_id) wait_for_file(started_path, timeout, "EOF runAt gate sentinel") + eof_drop_token = "eof-workspace-drop-progress" + eof_drop_id = client.send_request( + "tools/call", + { + "name": "lean_drop_workspace", + "arguments": {"workspace": workspace_descriptor(project_root)}, + "_meta": {"progressToken": eof_drop_token}, + }, + request_id="eof-workspace-drop", + ) + wait_for_progress_notification( + client, + eof_drop_token, + min(timeout, 5.0), + "EOF workspace drop admission", + ) client.close_input() + eof_drop_result = expect_result(client.read_response(eof_drop_id)) + require( + eof_drop_result.get("isError") is not True, + f"workspace drop admitted before EOF failed: {eof_drop_result}", + ) + eof_dropped = eof_drop_result.get("structuredContent") + require( + isinstance(eof_dropped, dict) and eof_dropped.get("dropped") is True, + f"workspace drop admitted before EOF did not evict the runtime: {eof_drop_result}", + ) + require_progress_sequence( + client.progress_notifications(eof_drop_token), + eof_drop_token, + "EOF workspace drop progress", + ) + returncode = client.wait_for_exit_after_eof(timeout) + require(returncode == 0, f"legacy server exited with code {returncode} after active EOF") + require( + not client.response_ready(eof_request_id), + "request cancelled by EOF produced a terminal response", + ) client.forget_request(eof_request_id) finally: if not release_path.exists(): From d79871a13443d1d18d4a91a85fd3fcfbcd501a02 Mon Sep 17 00:00:00 2001 From: Emilio Jesus Gallego Arias Date: Wed, 26 Aug 2026 21:35:13 +0200 Subject: [PATCH 13/28] fix: authenticate and harden daemon shutdown --- Beam/Broker/Protocol.lean | 6 + Beam/Broker/Server.lean | 218 ++++++++++-------- Beam/Cli/DaemonManager.lean | 52 +++-- Beam/Daemon/Protocol.lean | 40 +++- Beam/Mcp/Server.lean | 45 ++-- docs/DEVELOPMENT.md | 19 +- docs/STATUS.md | 7 +- docs/TESTING.md | 5 +- tests/lean/BeamTest/Broker/FixtureUtil.lean | 4 +- tests/lean/BeamTest/Broker/ProcessUtil.lean | 15 +- tests/lean/BeamTest/Broker/ProtocolTest.lean | 30 +-- .../BeamTest/Broker/RequestHandleTest.lean | 14 +- .../Broker/RequestStreamContractTest.lean | 51 +++- .../lean/BeamTest/Broker/StreamDedupTest.lean | 2 +- tests/test-beam-fast.sh | 3 +- tests/test-mcp-http-bridge.py | 6 +- 16 files changed, 315 insertions(+), 202 deletions(-) diff --git a/Beam/Broker/Protocol.lean b/Beam/Broker/Protocol.lean index 63da0b78..40b80bc4 100644 --- a/Beam/Broker/Protocol.lean +++ b/Beam/Broker/Protocol.lean @@ -14,6 +14,12 @@ namespace Beam.Broker abbrev WorkspaceId := Beam.Workspace.WorkspaceId +/-- Identity of one wrapper-owned daemon generation. -/ +structure DaemonIdentity where + daemonId : String + configHash : String + deriving BEq, Repr, FromJson, ToJson + instance : Repr Lsp.DiagnosticSeverity where reprPrec severity _ := match severity with diff --git a/Beam/Broker/Server.lean b/Beam/Broker/Server.lean index d757d14e..40a1a7cf 100644 --- a/Beam/Broker/Server.lean +++ b/Beam/Broker/Server.lean @@ -120,13 +120,13 @@ private def requestMethod (method : Except String String) : HandlerM String := | .ok method => pure method | .error msg => throw <| responseFailureFor .invalidParams msg -private def runHandler (act : HandlerM (Response × Bool)) : IO (Response × Bool) := do +private def runHandler (act : HandlerM Response) : IO Response := do try match ← act.run with | .ok result => pure result - | .error failure => pure (failure.toResponse, false) + | .error failure => pure failure.toResponse catch e => - pure (errorResponseFor .internalError e.toString, false) + pure <| errorResponseFor .internalError e.toString private def mkSessionToken : IO String := do let pid ← IO.Process.getPID @@ -886,6 +886,7 @@ private def modifyCurrentSessionIfMatching structure ServerRuntime where state : Std.Mutex State endpoint : Transport.Endpoint + daemonIdentity? : Option DaemonIdentity stop : IO.Ref Bool activeRequests : ActiveRequestRegistry private closeMutex : Std.Mutex Bool @@ -919,12 +920,17 @@ private def ServerRuntime.statsResponse (server : ServerRuntime) (workspaceId? : Option WorkspaceId := none) : IO Response := do let payload ← server.withState <| statsPayload workspaceId? + let payload := + match server.daemonIdentity? with + | some identity => payload.setObjVal! "daemonIdentity" (toJson identity) + | none => payload pure <| Response.success payload def ServerRuntime.create (config : BrokerConfig) (workspaceId : WorkspaceId) - (endpoint : Transport.Endpoint := .tcp 0) : IO ServerRuntime := do + (endpoint : Transport.Endpoint := .tcp 0) + (daemonIdentity? : Option DaemonIdentity := none) : IO ServerRuntime := do unless validWorkspaceId workspaceId do throw <| IO.userError "workspace id must be non-empty" let startMonoNanos ← IO.monoNanosNow @@ -932,6 +938,7 @@ def ServerRuntime.create pure { state := ← Std.Mutex.new state endpoint := endpoint + daemonIdentity? stop := ← IO.mkRef false activeRequests := ← ActiveRequestRegistry.create closeMutex := ← Std.Mutex.new false @@ -990,9 +997,9 @@ private def awaitRuntimeClose /-- Close broker admission, cancel admitted requests, shut down every backend session, and wait for all admitted dispatch scopes to unregister. Concurrent and repeated callers wait for the same -close result; only the caller that started closure receives `true`. +close result. -/ -def ServerRuntime.close (server : ServerRuntime) : IO Bool := do +def ServerRuntime.close (server : ServerRuntime) : IO Unit := do let leadsClose ← server.closeMutex.atomically do if ← get then pure false @@ -1014,11 +1021,10 @@ def ServerRuntime.close (server : ServerRuntime) : IO Bool := do pure (.error err) server.closeDone.resolve outcome match outcome with - | .ok () => pure true + | .ok () => pure () | .error err => throw err else awaitRuntimeClose server.closeDone - pure false private def workspaceInitResult (workspaceId : WorkspaceId) @@ -1233,6 +1239,12 @@ private def requestStop (server : ServerRuntime) : IO Unit := do catch _ => pure () +private def closeAndRequestStop (server : ServerRuntime) : IO Unit := do + try + server.close + finally + requestStop server + private structure WorkspaceRequest extends Request where workspaceId : WorkspaceId @@ -1735,7 +1747,7 @@ private def handleSyncFileOp (cancelRef? : Option (IO.Ref Bool) := none) (emitProgress? : Option (SyncFileProgress → IO Unit) := none) (emitDiagnostic? : Option (StreamDiagnostic → IO Unit) := none) : - HandlerM (Response × Bool) := do + HandlerM Response := do if req.backend != .lean then throw <| responseFailureFor .invalidParams "sync_file diagnostics barrier is only supported for Lean" @@ -1784,9 +1796,7 @@ private def handleSyncFileOp recordCompletedSync server started.session started.uri started.version liftHandlerIO <| traceBroker s!"sync_file response ready clientRequestId={optionLabel req.clientRequestId?} version={started.version} saveReady={saveReadiness.saveReady}" - pure (syncFileSuccessResponse - syncResult fileProgress?, - false) + pure <| syncFileSuccessResponse syncResult fileProgress? private def closeTrackedFileIfOpen (server : ServerRuntime) @@ -1806,7 +1816,7 @@ private def handleRefreshFileOp (cancelRef? : Option (IO.Ref Bool) := none) (emitProgress? : Option (SyncFileProgress → IO Unit) := none) (emitDiagnostic? : Option (StreamDiagnostic → IO Unit) := none) : - HandlerM (Response × Bool) := do + HandlerM Response := do let path ← requestArg req.pathArg liftFailureIO <| ensureRequestNotCancelled cancelRef? closeTrackedFileIfOpen server req path @@ -1816,7 +1826,7 @@ private def handleUpdateFileOp (server : ServerRuntime) (req : WorkspaceRequest) (cancelRef? : Option (IO.Ref Bool) := none) : - HandlerM (Response × Bool) := do + HandlerM Response := do let path ← requestArg req.pathArg liftFailureIO <| ensureRequestNotCancelled cancelRef? let snapshot ← liftHandlerIO <| readRequestSyncSnapshot server req path @@ -1825,11 +1835,11 @@ private def handleUpdateFileOp let synced ← syncFileSnapshotDetailed session snapshot updateSession synced.session pure synced - pure (Response.success (toJson ({ + pure <| Response.success (toJson ({ version := updated.version changed := updated.changed : UpdateFileResult - })), false) + })) private def handleCloseOp (server : ServerRuntime) @@ -1837,21 +1847,21 @@ private def handleCloseOp (cancelRef? : Option (IO.Ref Bool) := none) (emitProgress? : Option (SyncFileProgress → IO Unit) := none) (emitDiagnostic? : Option (StreamDiagnostic → IO Unit) := none) : - HandlerM (Response × Bool) := do + HandlerM Response := do let path ← requestArg req.pathArg if req.saveArtifacts?.getD false then let saved ← saveOlean server req path cancelRef? emitProgress? emitDiagnostic? finalizeSavedDoc server saved.session saved.uri saved.version true - pure (saveCompletedResponse saved true, false) + pure <| saveCompletedResponse saved true else liftHandlerIO <| server.withState do match ← currentSession? req.workspaceId req.backend with | some session => let session ← closeFile session path updateSession session - pure (Response.success (Json.mkObj [("closed", toJson true)]), false) + pure <| Response.success (Json.mkObj [("closed", toJson true)]) | none => - pure (Response.success (Json.mkObj [("closed", toJson true)]), false) + pure <| Response.success (Json.mkObj [("closed", toJson true)]) private def runAtSetupProgressEmitter? (emitDiagnostic? : Option (StreamDiagnostic → IO Unit)) : @@ -1866,7 +1876,7 @@ private def handleRunAtOp (cancelRef? : Option (IO.Ref Bool) := none) (emitProgress? : Option (SyncFileProgress → IO Unit) := none) (emitDiagnostic? : Option (StreamDiagnostic → IO Unit) := none) : - HandlerM (Response × Bool) := do + HandlerM Response := do let args ← requestArg req.runAtArgs liftFailureIO <| ensureRequestNotCancelled cancelRef? let snapshot ← liftHandlerIO <| readRequestSyncSnapshot server req args.path @@ -1888,11 +1898,10 @@ private def handleRunAtOp (emitDiagnostic? := runAtSetupProgressEmitter? emitDiagnostic?) (cancelRef? := cancelRef?) let pending ← awaitSyncedDocumentRequest server started cancelRef? - pure ( + pure <| Response.withOptionalFileProgress (Response.success (wrapResultHandle started.session pending.result)) - pending.progress?, - false) + pending.progress? private def positionLspParams (args : PositionArgs) @@ -1912,7 +1921,7 @@ private def handlePositionLspOp (extraFields : List (String × Json) := []) (cancelRef? : Option (IO.Ref Bool) := none) (emitProgress? : Option (SyncFileProgress → IO Unit) := none) : - HandlerM (Response × Bool) := do + HandlerM Response := do liftFailureIO <| ensureRequestNotCancelled cancelRef? let snapshot ← liftHandlerIO <| readRequestSyncSnapshot server req args.path let started ← liftFailureIO <| server.withState do @@ -1925,14 +1934,14 @@ private def handlePositionLspOp (emitProgress? := emitProgress?) (cancelRef? := cancelRef?) let pending ← awaitSyncedDocumentRequest server started cancelRef? - pure (Response.withOptionalFileProgress (Response.success pending.result) pending.progress?, false) + pure <| Response.withOptionalFileProgress (Response.success pending.result) pending.progress? private def handleHoverOp (server : ServerRuntime) (req : WorkspaceRequest) (cancelRef? : Option (IO.Ref Bool) := none) (emitProgress? : Option (SyncFileProgress → IO Unit) := none) : - HandlerM (Response × Bool) := do + HandlerM Response := do let args ← requestArg req.hoverArgs handlePositionLspOp server req args.toPositionArgs args.method (cancelRef? := cancelRef?) (emitProgress? := emitProgress?) @@ -1942,7 +1951,7 @@ private def handleSignatureHelpOp (req : WorkspaceRequest) (cancelRef? : Option (IO.Ref Bool) := none) (emitProgress? : Option (SyncFileProgress → IO Unit) := none) : - HandlerM (Response × Bool) := do + HandlerM Response := do let args ← requestArg req.signatureHelpArgs handlePositionLspOp server req args.toPositionArgs args.method (cancelRef? := cancelRef?) (emitProgress? := emitProgress?) @@ -1952,7 +1961,7 @@ private def handleDefinitionOp (req : WorkspaceRequest) (cancelRef? : Option (IO.Ref Bool) := none) (emitProgress? : Option (SyncFileProgress → IO Unit) := none) : - HandlerM (Response × Bool) := do + HandlerM Response := do let args ← requestArg req.definitionArgs handlePositionLspOp server req args.toPositionArgs args.method (cancelRef? := cancelRef?) (emitProgress? := emitProgress?) @@ -1962,7 +1971,7 @@ private def handleReferencesOp (req : WorkspaceRequest) (cancelRef? : Option (IO.Ref Bool) := none) (emitProgress? : Option (SyncFileProgress → IO Unit) := none) : - HandlerM (Response × Bool) := do + HandlerM Response := do let args ← requestArg req.referencesArgs handlePositionLspOp server req args.toPositionArgs args.method [("context", Json.mkObj [("includeDeclaration", toJson args.includeDeclaration)])] @@ -1973,7 +1982,7 @@ private def handleDocumentSymbolsOp (req : WorkspaceRequest) (cancelRef? : Option (IO.Ref Bool) := none) (emitProgress? : Option (SyncFileProgress → IO Unit) := none) : - HandlerM (Response × Bool) := do + HandlerM Response := do let args ← requestArg req.documentSymbolsArgs liftFailureIO <| ensureRequestNotCancelled cancelRef? let snapshot ← liftHandlerIO <| readRequestSyncSnapshot server req args.path @@ -1989,13 +1998,13 @@ private def handleDocumentSymbolsOp (emitProgress? := emitProgress?) (cancelRef? := cancelRef?) let pending ← awaitSyncedDocumentRequest server started cancelRef? - pure (Response.withOptionalFileProgress (Response.success pending.result) pending.progress?, false) + pure <| Response.withOptionalFileProgress (Response.success pending.result) pending.progress? private def handleWorkspaceSymbolsOp (server : ServerRuntime) (req : WorkspaceRequest) (cancelRef? : Option (IO.Ref Bool) := none) : - HandlerM (Response × Bool) := do + HandlerM Response := do let args ← requestArg req.workspaceSymbolsArgs liftFailureIO <| ensureRequestNotCancelled cancelRef? let (session, request) ← liftHandlerIO <| server.withState do @@ -2008,7 +2017,7 @@ private def handleWorkspaceSymbolsOp pure (session, request) liftHandlerIO <| propagatePendingCancellation session cancelRef? let pending ← awaitPending request - pure (Response.success pending.result, false) + pure <| Response.success pending.result private def codeActionResolveSourceUri (action : CodeAction) : Except ResponseFailure DocumentUri := do @@ -2027,7 +2036,7 @@ private def handleCodeActionResolveOp (req : WorkspaceRequest) (cancelRef? : Option (IO.Ref Bool) := none) (emitProgress? : Option (SyncFileProgress → IO Unit) := none) : - HandlerM (Response × Bool) := do + HandlerM Response := do let args ← requestArg req.codeActionResolveArgs liftFailureIO <| ensureRequestNotCancelled cancelRef? let snapshot ← liftHandlerIO <| readRequestSyncSnapshot server req args.path @@ -2050,7 +2059,7 @@ private def handleCodeActionResolveOp version := started.version codeAction := resolved } - pure (Response.withOptionalFileProgress (Response.success (toJson payload)) pending.progress?, false) + pure <| Response.withOptionalFileProgress (Response.success (toJson payload)) pending.progress? private def handleSaveOleanOp (server : ServerRuntime) @@ -2058,18 +2067,18 @@ private def handleSaveOleanOp (cancelRef? : Option (IO.Ref Bool) := none) (emitProgress? : Option (SyncFileProgress → IO Unit) := none) (emitDiagnostic? : Option (StreamDiagnostic → IO Unit) := none) : - HandlerM (Response × Bool) := do + HandlerM Response := do let path ← requestArg req.pathArg let saved ← saveOlean server req path cancelRef? emitProgress? emitDiagnostic? finalizeSavedDoc server saved.session saved.uri saved.version false - pure (saveCompletedResponse saved false, false) + pure <| saveCompletedResponse saved false private def handleGoalsOp (server : ServerRuntime) (req : WorkspaceRequest) (cancelRef? : Option (IO.Ref Bool) := none) (emitProgress? : Option (SyncFileProgress → IO Unit) := none) : - HandlerM (Response × Bool) := do + HandlerM Response := do let args ← requestArg req.goalsArgs if req.backend == .lean && req.text?.isSome then throw <| responseFailureFor .invalidParams @@ -2106,14 +2115,14 @@ private def handleGoalsOp (emitProgress? := emitProgress?) (cancelRef? := cancelRef?) let pending ← awaitSyncedDocumentRequest server started cancelRef? - pure (Response.withOptionalFileProgress (Response.success pending.result) pending.progress?, false) + pure <| Response.withOptionalFileProgress (Response.success pending.result) pending.progress? private def handleTodoOp (server : ServerRuntime) (req : WorkspaceRequest) (cancelRef? : Option (IO.Ref Bool) := none) (emitProgress? : Option (SyncFileProgress → IO Unit) := none) : - HandlerM (Response × Bool) := do + HandlerM Response := do let args ← requestArg req.todoArgs liftFailureIO <| ensureRequestNotCancelled cancelRef? let range : Lsp.Range := { @@ -2140,14 +2149,14 @@ private def handleTodoOp (emitProgress? := emitProgress?) (cancelRef? := cancelRef?) let pending ← awaitSyncedDocumentRequest server started cancelRef? - pure (Response.withOptionalFileProgress (Response.success pending.result) pending.progress?, false) + pure <| Response.withOptionalFileProgress (Response.success pending.result) pending.progress? private def handleRunWithOp (server : ServerRuntime) (req : WorkspaceRequest) (cancelRef? : Option (IO.Ref Bool) := none) (emitProgress? : Option (SyncFileProgress → IO Unit) := none) : - HandlerM (Response × Bool) := do + HandlerM Response := do let args ← requestArg req.runWithArgs liftFailureIO <| ensureRequestNotCancelled cancelRef? let snapshot ← liftHandlerIO <| readRequestSyncSnapshot server req args.path @@ -2180,18 +2189,17 @@ private def handleRunWithOp (cancelRef? := cancelRef?) pure startedResult let pending ← awaitSyncedDocumentRequest server started cancelRef? - pure ( + pure <| Response.withOptionalFileProgress (Response.success (wrapResultHandle started.session pending.result)) - pending.progress?, - false) + pending.progress? private def handleReleaseOp (server : ServerRuntime) (req : WorkspaceRequest) (cancelRef? : Option (IO.Ref Bool) := none) (emitProgress? : Option (SyncFileProgress → IO Unit) := none) : - HandlerM (Response × Bool) := do + HandlerM Response := do let args ← requestArg req.releaseArgs liftFailureIO <| ensureRequestNotCancelled cancelRef? let snapshot ← liftHandlerIO <| readRequestSyncSnapshot server req args.path @@ -2218,7 +2226,7 @@ private def handleReleaseOp (cancelRef? := cancelRef?) pure startedResult let pending ← awaitSyncedDocumentRequest server started cancelRef? - pure (Response.withOptionalFileProgress (Response.success pending.result) pending.progress?, false) + pure <| Response.withOptionalFileProgress (Response.success pending.result) pending.progress? private def initWorkspaceConfigFromRequest (server : ServerRuntime) @@ -2255,67 +2263,64 @@ private def handleRequestIO (req : Request) (activeRequest? : Option ActiveRequest := none) (emitProgress? : Option (SyncFileProgress → IO Unit) := none) - (emitDiagnostic? : Option (StreamDiagnostic → IO Unit) := none) : IO (Response × Bool) := do + (emitDiagnostic? : Option (StreamDiagnostic → IO Unit) := none) : IO Response := do let cancelRef? := activeRequest?.map (·.cancelRef) match req.op with | .shutdown => - let firstClose ← server.close - pure ( - Response.success (Json.mkObj [("shutdown", toJson true)]), - firstClose - ) + server.close + pure <| Response.success (Json.mkObj [("shutdown", toJson true)]) | .stats => match req.workspaceId? with - | none => pure (← server.statsResponse, false) + | none => server.statsResponse | some _ => match ← validateRequestWorkspace server req with - | .error failure => pure (failure.toResponse, false) + | .error failure => pure failure.toResponse | .ok workspaceReq => - pure (← server.statsResponse (some workspaceReq.workspaceId), false) + server.statsResponse (some workspaceReq.workspaceId) | .listWorkspaces => let payload ← server.withState do pure <| workspaceListPayload (← get) - pure (Response.success payload, false) + pure <| Response.success payload | .resetStats => let now ← IO.monoNanosNow let resp ← server.withState do resetMetrics now pure <| Response.success (Json.mkObj [("reset", toJson true)]) - pure (resp, false) + pure resp | .openDocs => match req.workspaceId? with - | none => pure (Response.success (← server.withState openDocsPayload), false) + | none => pure <| Response.success (← server.withState openDocsPayload) | some _ => match ← validateRequestWorkspace server req with - | .error failure => pure (failure.toResponse, false) + | .error failure => pure failure.toResponse | .ok workspaceReq => - pure (Response.success - (← server.withState <| openDocsPayload (some workspaceReq.workspaceId)), false) + pure <| Response.success + (← server.withState <| openDocsPayload (some workspaceReq.workspaceId)) | .initWorkspace => match req.requireWorkspaceId with - | .error err => pure (errorResponseFor .invalidParams err, false) + | .error err => pure <| errorResponseFor .invalidParams err | .ok workspaceId => match ← initWorkspaceConfigFromRequest server req with - | .error failure => pure (failure.toResponse, false) + | .error failure => pure failure.toResponse | .ok config => let result ← server.initWorkspaceWithConfig workspaceId config req.workspaceMode? - pure (responseOfTypedResult result, false) + pure <| responseOfTypedResult result | .dropWorkspace => match req.requireWorkspaceId with - | .error err => pure (errorResponseFor .invalidParams err, false) + | .error err => pure <| errorResponseFor .invalidParams err | .ok workspaceId => let result ← server.dropWorkspace workspaceId - pure (responseOfTypedResult result, false) + pure <| responseOfTypedResult result | .cancel => let targetClientRequestId ← match req.cancelRequestIdArg with | .ok targetClientRequestId => pure targetClientRequestId - | .error failure => return (failure.toResponse, false) + | .error failure => return failure.toResponse let cancelled ← cancelActiveRequest server targetClientRequestId - pure (Response.success (Json.mkObj [("cancelled", toJson cancelled)]), false) + pure <| Response.success (Json.mkObj [("cancelled", toJson cancelled)]) | op => match ← validateRequestWorkspace server req with - | .error failure => pure (failure.toResponse, false) + | .error failure => pure failure.toResponse | .ok workspaceReq => match op with | .ensure => @@ -2332,7 +2337,7 @@ private def handleRequestIO pure <| Response.success payload catch e => pure <| errorResponseFor .internalError e.toString - pure (resp, false) + pure resp | .updateFile => runHandler <| handleUpdateFileOp server workspaceReq cancelRef? | .syncFile => runHandler <| handleSyncFileOp server workspaceReq cancelRef? emitProgress? emitDiagnostic? @@ -2368,7 +2373,7 @@ private def handleRequestIO private def ServerRuntime.withRequestAdmission (server : ServerRuntime) (req : Request) - (act : RequestHandle → IO (Response × Bool)) : IO (Response × Bool) := do + (act : RequestHandle → IO Response) : IO Response := do let startedAt ← IO.monoNanosNow traceBroker s!"dispatch start op={req.op.key} clientRequestId={optionLabel req.clientRequestId?}" @@ -2378,7 +2383,7 @@ private def ServerRuntime.withRequestAdmission traceBroker s!"dispatch rejected op={req.op.key} clientRequestId={optionLabel req.clientRequestId?} error={err}" recordDispatchMetrics server req resp startedAt - return (resp, false) + return resp | .ok () => pure () try let active? ← @@ -2388,16 +2393,16 @@ private def ServerRuntime.withRequestAdmission | .error failure => let resp := BrokerFailure.toResponse failure recordDispatchMetrics server req resp startedAt - return (resp, false) + return resp else pure none try let handle : RequestHandle := { runtime := server, active? } - let (resp, shouldStop) ← act handle + let resp ← act handle traceBroker s!"dispatch complete op={req.op.key} clientRequestId={optionLabel req.clientRequestId?} ok={resp.ok}" recordDispatchMetrics server req resp startedAt - pure (resp, shouldStop) + pure resp finally ActiveRequestRegistry.unregister server.activeRequests active? catch e => @@ -2405,7 +2410,7 @@ private def ServerRuntime.withRequestAdmission traceBroker s!"dispatch exception op={req.op.key} clientRequestId={optionLabel req.clientRequestId?} error={e.toString}" recordDispatchMetrics server req resp startedAt - pure (resp, false) + pure resp /-- Admit `req`, expose its exact cancellation handle to `beforeDispatch`, and @@ -2421,23 +2426,20 @@ def ServerRuntime.dispatchRequestWithHandle (req : Request) (beforeDispatch : RequestHandle → IO Bool) (emitProgress? : Option (SyncFileProgress → IO Unit) := none) - (emitDiagnostic? : Option (StreamDiagnostic → IO Unit) := none) : IO (Response × Bool) := do + (emitDiagnostic? : Option (StreamDiagnostic → IO Unit) := none) : IO Response := do server.withRequestAdmission req fun handle => do unless ← beforeDispatch handle do - return ( - BrokerFailure.toResponse { - code := .requestCancelled - message := "request was cancelled before broker dispatch" - }, - false - ) + return BrokerFailure.toResponse { + code := .requestCancelled + message := "request was cancelled before broker dispatch" + } handleRequestIO server req handle.active? emitProgress? emitDiagnostic? def ServerRuntime.dispatchRequest (server : ServerRuntime) (req : Request) (emitProgress? : Option (SyncFileProgress → IO Unit) := none) - (emitDiagnostic? : Option (StreamDiagnostic → IO Unit) := none) : IO (Response × Bool) := do + (emitDiagnostic? : Option (StreamDiagnostic → IO Unit) := none) : IO Response := do server.dispatchRequestWithHandle req (fun _ => pure true) emitProgress? emitDiagnostic? private def rootWatchPollMs : UInt32 := @@ -2459,9 +2461,7 @@ private partial def watchRoot (server : ServerRuntime) (root : System.FilePath) pure false if !rootAvailable then IO.eprintln s!"Beam daemon root is no longer available; shutting down: {root}" - let shouldStop ← server.close - if shouldStop then - requestStop server + closeAndRequestStop server else IO.sleep rootWatchPollMs watchRoot server root @@ -2472,9 +2472,7 @@ private def watchSessionOwnerStdin (server : ServerRuntime) : IO Unit := do catch _ => pure () unless ← server.stop.get do - let shouldStop ← server.close - if shouldStop then - requestStop server + closeAndRequestStop server private def watchClientDisconnect (client : Transport.Connection) @@ -2514,18 +2512,25 @@ private def handleClient (server : ServerRuntime) (client : Transport.Connection | Except.error failure => sendResponse (← clientRequestIdRef.get) failure.toResponse | Except.ok req => + let stopsTransport := + req.op == .shutdown && match req.validateFields with + | .ok () => true + | .error _ => false let emitProgress : SyncFileProgress → IO Unit := fun progress => Transport.sendMsg client (toJson (StreamMessage.fileProgress req.clientRequestId? progress)).compress let emitDiagnostic : StreamDiagnostic → IO Unit := fun diagnostic => Transport.sendMsg client (toJson (StreamMessage.diagnostic req.clientRequestId? diagnostic)).compress - let (resp, shouldStop) ← server.dispatchRequestWithHandle req (fun handle => do + let resp ← server.dispatchRequestWithHandle req (fun handle => do let _ ← IO.asTask (prio := Task.Priority.dedicated) <| watchClientDisconnect client handle pure true) (some emitProgress) (some emitDiagnostic) - sendResponse req.clientRequestId? resp - if shouldStop then + -- Stopping only wakes the listener; the accepted connection remains available for the + -- terminal response. Do this first so a disconnected shutdown caller cannot strand a + -- closed runtime behind a live listener. + if stopsTransport then requestStop server + sendResponse req.clientRequestId? resp catch e => unless ← terminalSentRef.get do let clientRequestId? ← clientRequestIdRef.get @@ -2556,6 +2561,8 @@ private structure CliOptions where endpoint : Transport.Endpoint := .tcp 8765 root? : Option String := none workspaceId? : Option WorkspaceId := none + daemonId? : Option String := none + configHash? : Option String := none sessionOwnerStdin : Bool := false leanCmd? : Option String := none leanPlugin? : Option String := none @@ -2582,6 +2589,10 @@ private partial def parseCliOptions (opts : CliOptions) : List String → Except parseCliOptions { opts with root? := some root } rest | "--workspace-id" :: workspaceId :: rest => parseCliOptions { opts with workspaceId? := some workspaceId } rest + | "--daemon-id" :: daemonId :: rest => + parseCliOptions { opts with daemonId? := some daemonId } rest + | "--config-hash" :: configHash :: rest => + parseCliOptions { opts with configHash? := some configHash } rest | "--session-owner-stdin" :: rest => parseCliOptions { opts with sessionOwnerStdin := true } rest | "--lean-cmd" :: leanCmd :: rest => @@ -2601,6 +2612,17 @@ def main (args : List String) : IO Unit := do | throw <| IO.userError "missing Beam daemon --workspace-id ID" unless validWorkspaceId workspaceId do throw <| IO.userError "workspace id must be non-empty" + let daemonIdentity? ← + match opts.daemonId?, opts.configHash? with + | none, none => pure none + | some daemonId, some configHash => + if daemonId.isEmpty || configHash.isEmpty then + throw <| IO.userError "daemon identity values must be non-empty" + pure <| some { daemonId, configHash } + | some _, none => + throw <| IO.userError "--daemon-id requires --config-hash" + | none, some _ => + throw <| IO.userError "--config-hash requires --daemon-id" let root ← Beam.resolveExistingPath <| System.FilePath.mk root let leanPlugin? ← opts.leanPlugin?.mapM (fun path => Beam.resolveExistingPath <| System.FilePath.mk path) let config : BrokerConfig := { @@ -2610,7 +2632,7 @@ def main (args : List String) : IO Unit := do rocqCmd? := opts.rocqCmd? } let listener ← Transport.bindAndListen opts.endpoint 16 - let runtime ← ServerRuntime.create config workspaceId opts.endpoint + let runtime ← ServerRuntime.create config workspaceId opts.endpoint daemonIdentity? let rootWatcher ← IO.asTask (prio := Task.Priority.dedicated) <| watchRoot runtime root let ownerWatcher? ← if opts.sessionOwnerStdin then @@ -2625,6 +2647,6 @@ def main (args : List String) : IO Unit := do if let some ownerWatcher := ownerWatcher? then IO.cancel ownerWatcher discard <| IO.wait rootWatcher - discard <| runtime.close + runtime.close end Beam.Broker diff --git a/Beam/Cli/DaemonManager.lean b/Beam/Cli/DaemonManager.lean index 97eb2bfe..e8fe4882 100644 --- a/Beam/Cli/DaemonManager.lean +++ b/Beam/Cli/DaemonManager.lean @@ -118,19 +118,18 @@ private def stopDaemonEntry (entry : RegistryEntry) : IO Unit := do let mayKillPid ← match registryEndpoint? entry with | some endpoint => - match ← daemonRoot? endpoint projectDaemonWorkspaceId with - | some daemonRoot => - if ← Beam.sameFilePath (System.FilePath.mk daemonRoot) (System.FilePath.mk entry.root) then - try - let _ ← sendRequest endpoint { op := .shutdown } - pure () - catch _ => - pure () - pure true - else - pure false - | none => - pure true + if ← daemonServesGeneration endpoint projectDaemonWorkspaceId + (System.FilePath.mk entry.root) entry.identity then + try + let _ ← sendRequest endpoint { op := .shutdown } + pure () + catch _ => + pure () + pure true + else if (← daemonRoot? endpoint projectDaemonWorkspaceId).isNone then + pure true + else + pure false | none => pure true if mayKillPid then @@ -348,10 +347,13 @@ private def terminateDaemonChild private def startDaemon (desired : DesiredConfig) (endpoint : Transport.Endpoint) - (logPath : System.FilePath) : IO (IO.Process.Child daemonStdio) := do + (logPath : System.FilePath) + (identity : DaemonIdentity) : IO (IO.Process.Child daemonStdio) := do let mut args : List String := [ "--root", desired.root.toString, "--workspace-id", projectDaemonWorkspaceId, + "--daemon-id", identity.daemonId, + "--config-hash", identity.configHash, "--session-owner-stdin" ] match endpoint with @@ -381,24 +383,25 @@ private partial def waitForDaemon (endpoint : Transport.Endpoint) (logPath : System.FilePath) (root : System.FilePath) + (identity : DaemonIdentity) (tries : Nat := 300) : IO (Except DaemonStartupFailure Unit) := do - match ← daemonRoot? endpoint projectDaemonWorkspaceId with - | some daemonRoot => - if ← Beam.sameFilePath (System.FilePath.mk daemonRoot) root then - pure (.ok ()) - else + if ← daemonServesGeneration endpoint projectDaemonWorkspaceId root identity then + pure (.ok ()) + else + match ← daemonRoot? endpoint projectDaemonWorkspaceId with + | some daemonRoot => pure <| .error { message := endpointOccupancyError endpoint (System.FilePath.mk daemonRoot) root endpointInUse := true } - | none => + | none => if (← child.tryWait).isSome then .error <$> daemonStartupFailure endpoint logPath "Beam daemon process exited before responding" else if tries == 0 then .error <$> daemonStartupFailure endpoint logPath "Beam daemon did not become ready before timeout" else IO.sleep 100 - waitForDaemon child endpoint logPath root (tries - 1) + waitForDaemon child endpoint logPath root identity (tries - 1) private def newDaemonGenerationId (configHash : String) : IO String := do let startedMonoNanos ← IO.monoNanosNow @@ -443,8 +446,9 @@ private partial def startDaemonEntry let endpoint ← selectUnoccupiedEndpoint desired opts let logPath ← daemonStartupLogPath desired.root let daemonId ← newDaemonGenerationId desired.configHash - let child ← startDaemon desired endpoint logPath - match ← waitForDaemon child endpoint logPath desired.root with + let identity : DaemonIdentity := { daemonId, configHash := desired.configHash } + let child ← startDaemon desired endpoint logPath identity + match ← waitForDaemon child endpoint logPath desired.root identity with | .ok () => pure () | .error failure => terminateDaemonChild child @@ -543,7 +547,7 @@ def registryLiveFor | some endpoint => -- The owner pipe makes endpoint liveness authoritative across PID domains. A -- same-domain dead owner is rejected immediately; another domain is never probed. - if ← daemonServesRoot endpoint projectDaemonWorkspaceId root then + if ← daemonServesGeneration endpoint projectDaemonWorkspaceId root entry.identity then pure (some entry) else pure none diff --git a/Beam/Daemon/Protocol.lean b/Beam/Daemon/Protocol.lean index 2516834a..771dc9e0 100644 --- a/Beam/Daemon/Protocol.lean +++ b/Beam/Daemon/Protocol.lean @@ -35,6 +35,11 @@ structure RegistryEntry where requestedPort? : Option Nat := none deriving FromJson, ToJson +def RegistryEntry.identity (entry : RegistryEntry) : DaemonIdentity := { + daemonId := entry.daemonId + configHash := entry.configHash +} + structure DesiredConfig where root : System.FilePath leanCmd? : Option String := none @@ -61,22 +66,33 @@ def endpointFromEntry (entry : RegistryEntry) : IO Transport.Endpoint := do def endpointSummary (endpoint : Transport.Endpoint) : String := Transport.endpointDescription endpoint -private def statsRoot? (resp : Response) : Option String := do +private structure DaemonProbe where + root : String + identity? : Option DaemonIdentity + +private def daemonProbeOfResponse? (resp : Response) : Option DaemonProbe := do let result ← resp.result? - result.getObjValAs? String "root" |>.toOption + let root ← result.getObjValAs? String "root" |>.toOption + let identity? := result.getObjValAs? DaemonIdentity "daemonIdentity" |>.toOption + pure { root, identity? } -def daemonRoot? +private def daemonProbe? (endpoint : Transport.Endpoint) - (workspaceId : WorkspaceId) : IO (Option String) := do + (workspaceId : WorkspaceId) : IO (Option DaemonProbe) := do try let resp ← sendRequest endpoint { op := .stats, workspaceId? := some workspaceId } if resp.ok then - pure (statsRoot? resp) + pure (daemonProbeOfResponse? resp) else pure none catch _ => pure none +def daemonRoot? + (endpoint : Transport.Endpoint) + (workspaceId : WorkspaceId) : IO (Option String) := do + pure <| (← daemonProbe? endpoint workspaceId).map (·.root) + def endpointOccupancyError (endpoint : Transport.Endpoint) (daemonRoot requestedRoot : System.FilePath) : String := @@ -105,6 +121,20 @@ def daemonServesRoot | some daemonRoot => Beam.sameFilePath (System.FilePath.mk daemonRoot) root | none => pure false +/-- Whether an endpoint serves the expected root and exact wrapper-owned daemon generation. -/ +def daemonServesGeneration + (endpoint : Transport.Endpoint) + (workspaceId : WorkspaceId) + (root : System.FilePath) + (identity : DaemonIdentity) : IO Bool := do + match ← daemonProbe? endpoint workspaceId with + | some probe => + if probe.identity? != some identity then + pure false + else + Beam.sameFilePath (System.FilePath.mk probe.root) root + | none => pure false + def endpointAcceptsConnection (endpoint : Transport.Endpoint) : IO Bool := do try let conn ← Transport.connect endpoint diff --git a/Beam/Mcp/Server.lean b/Beam/Mcp/Server.lean index 9864dcac..1cfad97c 100644 --- a/Beam/Mcp/Server.lean +++ b/Beam/Mcp/Server.lean @@ -46,37 +46,30 @@ inductive ProtocolState where structure ServerState where protocol : Std.Mutex ProtocolState - private runtime : IO.Ref (Option Beam.Broker.ServerRuntime) - private runtimeControl : Std.Mutex Unit + private runtime : Std.Mutex (Option Beam.Broker.ServerRuntime) def ServerState.create : IO ServerState := do pure { protocol := ← Std.Mutex.new .undecided - runtime := ← IO.mkRef none - runtimeControl := ← Std.Mutex.new () + runtime := ← Std.Mutex.new none } def ServerState.protocolState (state : ServerState) : IO ProtocolState := state.protocol.atomically get private def ServerState.runtime? (state : ServerState) : IO (Option Beam.Broker.ServerRuntime) := - state.runtime.get - -private def ServerState.withRuntimeControl - (state : ServerState) - (action : IO α) : IO α := - state.runtimeControl.atomically action + state.runtime.atomically get /-- Close and forget the in-process broker runtime owned by this MCP server state. -/ def ServerState.closeRuntime (state : ServerState) : IO Unit := - state.withRuntimeControl do - match ← state.runtime.get with + state.runtime.atomically do + match ← get with | none => pure () | some runtime => -- Transfer ownership out of the state before waiting for broker teardown. A concurrent - -- creator remains excluded by `runtimeControl`, and repeated close calls are idempotent. - state.runtime.set none - discard <| runtime.close + -- creator remains excluded by the same mutex, and repeated close calls are idempotent. + set (none : Option Beam.Broker.ServerRuntime) + runtime.close structure NotificationSink where send : Json → IO Unit := fun _ => pure () @@ -499,8 +492,8 @@ private def ensureRuntimeForWorkspace (opts : Options) (workspaceId : Beam.Broker.WorkspaceId) (root : System.FilePath) : IO (Except RpcError (Beam.Broker.ServerRuntime × System.FilePath)) := do - state.withRuntimeControl do - match ← state.runtime? with + state.runtime.atomically do + match ← get with | some runtime => match ← ensureBrokerWorkspace opts runtime workspaceId root with | .ok canonicalRoot => pure <| .ok (runtime, canonicalRoot) @@ -509,7 +502,7 @@ private def ensureRuntimeForWorkspace match ← createRuntimeForRoot opts workspaceId root with | .error err => pure <| .error err | .ok (runtime, canonicalRoot) => - state.runtime.set (some runtime) + set (some runtime) pure <| .ok (runtime, canonicalRoot) private def workspaceErrorToToolError (err : Beam.Workspace.RootError) : ToolError := @@ -574,7 +567,7 @@ private def handleBeamStats ("uptimeMs", toJson (0 : Nat)), ("workspaces", Json.mkObj []) ] - let (brokerResp, _) ← runtime.dispatchRequest { op := .stats } + let brokerResp ← runtime.dispatchRequest { op := .stats } match normalizeBrokerResponse .beamStats brokerResp with | .error err => pure <| callToolErrorResult err @@ -591,7 +584,7 @@ private def handleBeamStats pure <| callToolResult result private def handleDropWorkspace - (state : ServerState) + (runtime? : Option Beam.Broker.ServerRuntime) (workspace : ResolvedWorkspace) : IO Json := do let resultJson (dropped invalidatedHandles : Bool) (reason? : Option String := none) : Json := Json.mkObj <| [ @@ -601,7 +594,7 @@ private def handleDropWorkspace ] ++ match reason? with | some reason => [("reason", toJson reason)] | none => [] - match ← state.runtime? with + match runtime? with | none => pure <| callToolResult <| resultJson false false (some "notFound") | some runtime => @@ -654,13 +647,13 @@ private def collectFeedbackRuntimePayload | none => pure (Json.null, Json.null, warnings.push "no active MCP Lean runtime was available for stats/open-files") | some runtime => - let (statsResp, _) ← runtime.dispatchRequest { + let statsResp ← runtime.dispatchRequest { op := .stats workspaceId? := some workspaceId root? := some root.toString } let (stats, warnings) := Beam.Feedback.responsePayloadOrWarning "stats" statsResp warnings - let (openResp, _) ← runtime.dispatchRequest { + let openResp ← runtime.dispatchRequest { op := .openDocs workspaceId? := some workspaceId root? := some root.toString @@ -803,8 +796,8 @@ def Internal.handleToolCall | .error err => return .ok <| callToolErrorResult err if initialProgress == 0 then emitProgress? progress? s!"{params.name.key}: preparing workspace eviction" - let result ← state.withRuntimeControl do - handleDropWorkspace state workspace + let result ← state.runtime.atomically do + handleDropWorkspace (← get) workspace Internal.traceMcp s!"tools/call workspace drop complete id={req.id.label} tool={params.name.key}" return .ok result @@ -864,7 +857,7 @@ def Internal.handleToolCall let emitBrokerProgress? : Option (Beam.Broker.SyncFileProgress → IO Unit) := reporter.progress?.map fun _ => reporter.emitFileProgress Internal.traceMcp s!"tools/call dispatch broker id={req.id.label} tool={params.name.key}" - let (brokerResp, _) ← runtime.dispatchRequestWithHandle brokerReq beforeDispatch + let brokerResp ← runtime.dispatchRequestWithHandle brokerReq beforeDispatch (emitProgress? := emitBrokerProgress?) (emitDiagnostic? := some emitDiagnostic) Internal.traceMcp diff --git a/docs/DEVELOPMENT.md b/docs/DEVELOPMENT.md index 2a50a794..7f1d3487 100644 --- a/docs/DEVELOPMENT.md +++ b/docs/DEVELOPMENT.md @@ -304,7 +304,8 @@ The broker is not a raw LSP proxy. Its narrow public job is still to expose smal internally it coordinates several responsibilities around the LSP process: - the CLI owns process identity: project-root detection, bundle selection, registry files, - endpoint/root validation, explicit session ownership, startup/shutdown, and control-directory locks + endpoint/root/generation validation, explicit session ownership, startup/shutdown, and + control-directory locks - the broker owns request identity: daemon root validation, backend session lifetime, request dispatch, cancellation, active-request bookkeeping, transport errors, and the LSP document mirror - the LSP server and plugin own Lean/Rocq semantic facts: elaboration, diagnostics, progress, @@ -388,14 +389,16 @@ This wrapper path is easy to break accidentally, so keep the mental model simple A daemon generation is one concrete daemon start identified by the `daemonId` in `beam-daemon.json`. Exactly one foreground `lean-beam ensure --hold` process owns that generation. -It starts the daemon with piped stdin and retains the pipe's write end. The daemon watches the read -end; EOF atomically closes broker admission, marks admitted requests for cancellation, shuts down -backend sessions, and stops the listener. There is no wrapper heartbeat, lease file, revocation -tombstone, or retirement fence. +It passes the daemon that identity and the effective configuration hash, starts it with piped stdin, +and retains the pipe's write end. Endpoint attachment requires the root and this exact generation +identity to match. The daemon watches the read end; EOF atomically closes broker admission, marks +admitted requests for cancellation, shuts down backend sessions, and stops the listener. There is +no wrapper heartbeat, lease file, revocation tombstone, or retirement fence. Ordinary wrapper commands never start a daemon. Under the per-project control lock they require a -registry whose root and effective configuration match, whose owner is not known dead in the current -PID domain, and whose endpoint answers for the CLI's private workspace and canonical project root. + registry whose root and effective configuration match, whose owner is not known dead in the current + PID domain, and whose endpoint answers for the CLI's private workspace, canonical project root, and + exact daemon generation identity. Endpoint/root validation is authoritative across PID namespaces because numeric PID observations from another domain are not safe process identity. A same-domain dead owner or a dead endpoint makes the registry stale; cleanup remains generation-scoped and PID fallback is permitted only through the @@ -422,7 +425,7 @@ its own explicit process owner. Keep these invariants covered: - only `ensure --hold` may create and publish a wrapper daemon generation -- a second owner is rejected while the current endpoint/root generation is live +- a second owner is rejected while the current endpoint/root/generation identity is live - ordinary wrapper commands preserve the owner's generation and fail with the exact recovery command when no owner is live - holder teardown unpublishes its exact generation before child drain and cannot remove a replacement diff --git a/docs/STATUS.md b/docs/STATUS.md index 65cd99ee..ebca5425 100644 --- a/docs/STATUS.md +++ b/docs/STATUS.md @@ -186,9 +186,10 @@ Exact event ordering and examples live in Ordinary wrapper calls, including plain `lean-beam ensure`, only attach to that generation and fail with the exact owner-start command when none is live. The optional `--port` override belongs only to `ensure --hold`; attaching commands reject it. Owner EOF shuts down request admission, - completes admitted requests with `requestCancelled`, and closes backend sessions and the daemon - without heartbeat timeouts or filesystem leases. This works - across PID namespaces because endpoint/root validation is authoritative when PID identity is not + marks admitted requests for cancellation, and closes backend sessions and the daemon after those + requests drain; a backend success that completed before cancellation remains successful. This + happens without heartbeat timeouts or filesystem leases and works across PID namespaces because + endpoint, root, and generation-identity validation are authoritative when PID identity is not locally observable. A paused owner retains the session; a killed owner closes the pipe; explicit `lean-beam shutdown` unpublishes the registry generation so the holder closes it cleanly. - A startup failure that reports `operation not permitted` through `.beam/beam-daemon-startup.log` is diff --git a/docs/TESTING.md b/docs/TESTING.md index b45b9425..8977f47f 100644 --- a/docs/TESTING.md +++ b/docs/TESTING.md @@ -105,8 +105,9 @@ Additional Beam lanes: Current Beam coverage includes: - fast Beam daemon smoke, request-stream, save-stream, startup-handshake, tracked-diagnostic dedup, - exact broker request-handle lifetime, protocol tests, and validated-toolchain/release-line CI - policy consistency through + exact broker request-handle lifetime, authenticated daemon-generation probes, shutdown after the + requesting TCP client resets its connection, protocol tests, and validated-toolchain/release-line + CI policy consistency through [tests/test-beam-fast.sh](../tests/test-beam-fast.sh) - wrapper coverage through [tests/test-beam-wrapper.sh](../tests/test-beam-wrapper.sh), which aggregates focused probe, runtime, sync/save, handle, and diagnostic slices - focused daemon lifecycle coverage in [tests/test-beam-wrapper-daemon.sh](../tests/test-beam-wrapper-daemon.sh), diff --git a/tests/lean/BeamTest/Broker/FixtureUtil.lean b/tests/lean/BeamTest/Broker/FixtureUtil.lean index 1419df94..51954318 100644 --- a/tests/lean/BeamTest/Broker/FixtureUtil.lean +++ b/tests/lean/BeamTest/Broker/FixtureUtil.lean @@ -19,7 +19,9 @@ def copySaveProjectFixture (dest : System.FilePath) : IO Unit := do IO.FS.createDirAll dest let out ← IO.Process.output { cmd := "rsync" - args := #["-a", s!"{src.toString}/", s!"{dest.toString}/"] + -- Local wrapper/install tests may leave an ignored runtime cache in this source fixture. It is + -- neither fixture input nor safe to duplicate into every isolated broker test project. + args := #["-a", "--exclude", ".beam/", s!"{src.toString}/", s!"{dest.toString}/"] } if out.exitCode != 0 then throw <| IO.userError s!"failed to copy save_olean_project fixture\n{out.stderr}" diff --git a/tests/lean/BeamTest/Broker/ProcessUtil.lean b/tests/lean/BeamTest/Broker/ProcessUtil.lean index 961a73ea..eeeacccd 100644 --- a/tests/lean/BeamTest/Broker/ProcessUtil.lean +++ b/tests/lean/BeamTest/Broker/ProcessUtil.lean @@ -148,7 +148,8 @@ def killLeanServerForEndpoint def spawnLeanBrokerWithPlugin (endpoint : Beam.Broker.Endpoint) (root leanPlugin : System.FilePath) - (leanCmd : String := "lean") : IO (IO.Process.Child nullBrokerStdio) := do + (leanCmd : String := "lean") + (identity? : Option Beam.Broker.DaemonIdentity := none) : IO (IO.Process.Child nullBrokerStdio) := do let port := match endpoint with | .tcp port => port @@ -161,15 +162,21 @@ def spawnLeanBrokerWithPlugin "--workspace-id", testWorkspaceId, "--lean-cmd", leanCmd, "--lean-plugin", leanPlugin.toString - ] + ] ++ match identity? with + | some identity => #[ + "--daemon-id", identity.daemonId, + "--config-hash", identity.configHash + ] + | none => #[] setsid := true } def spawnLeanBroker (endpoint : Beam.Broker.Endpoint) (root : System.FilePath) - (leanCmd : String := "lean") : IO (IO.Process.Child nullBrokerStdio) := do - spawnLeanBrokerWithPlugin endpoint root (← BeamTest.TestHarness.pluginPath) leanCmd + (leanCmd : String := "lean") + (identity? : Option Beam.Broker.DaemonIdentity := none) : IO (IO.Process.Child nullBrokerStdio) := do + spawnLeanBrokerWithPlugin endpoint root (← BeamTest.TestHarness.pluginPath) leanCmd identity? def spawnRocqBroker (endpoint : Beam.Broker.Endpoint) diff --git a/tests/lean/BeamTest/Broker/ProtocolTest.lean b/tests/lean/BeamTest/Broker/ProtocolTest.lean index 6d19b1bb..8e97c877 100644 --- a/tests/lean/BeamTest/Broker/ProtocolTest.lean +++ b/tests/lean/BeamTest/Broker/ProtocolTest.lean @@ -797,11 +797,11 @@ private def checkWorkspaceLifecycleProtocol : IO Unit := do require "broker workspace query should reject an unknown workspace" ((← runtime.workspaceRoot? "unknown") == none) for op in #[Op.ensure, .initWorkspace, .dropWorkspace] do - let (missingWorkspaceResp, _) ← runtime.dispatchRequest { op } + let missingWorkspaceResp ← runtime.dispatchRequest { op } require s!"{op.key} should reject omitted workspace identity" (missingWorkspaceResp.error?.any fun err => err.code == "invalidParams" && err.message.contains "workspaceId is required") - let (processStatsResp, _) ← runtime.dispatchRequest { op := .stats } + let processStatsResp ← runtime.dispatchRequest { op := .stats } let some processStats := processStatsResp.result? | throw <| IO.userError s!"process-wide stats failed: {(toJson processStatsResp).compress}" requireFieldAbsent "process-wide stats" "root" processStats @@ -1039,7 +1039,7 @@ private def checkSessionCloseAdmission : IO Unit := do let root := System.FilePath.mk "/tmp/beam-session-close-admission" let runtime ← Beam.Broker.ServerRuntime.create ({ root } : Beam.Broker.BrokerConfig) "fixture" (.tcp 0) - let (beforeClose, _) ← runtime.dispatchRequest { op := .stats } + let beforeClose ← runtime.dispatchRequest { op := .stats } require "stats should be admitted before session close" beforeClose.ok let active ← match ← ActiveRequestRegistry.register runtime.activeRequests (some "close-drain") with @@ -1054,24 +1054,18 @@ private def checkSessionCloseAdmission : IO Unit := do require "concurrent runtime close should share the same drain" (!(← IO.hasFinished concurrentCloseTask)) ActiveRequestRegistry.unregister runtime.activeRequests (some active) - let firstClose ← - match ← IO.wait closeTask with - | .ok firstClose => pure firstClose - | .error err => throw err - require "first runtime close should lead shutdown" firstClose - let concurrentClose ← - match ← IO.wait concurrentCloseTask with - | .ok concurrentClose => pure concurrentClose - | .error err => throw err - require "concurrent runtime close should not lead shutdown" (!concurrentClose) - require "repeated session close should be idempotent" - (!(← runtime.close)) - let (afterClose, _) ← runtime.dispatchRequest { op := .stats } + match ← IO.wait closeTask with + | .ok () => pure () + | .error err => throw err + match ← IO.wait concurrentCloseTask with + | .ok () => pure () + | .error err => throw err + runtime.close + let afterClose ← runtime.dispatchRequest { op := .stats } require "ordinary requests should be rejected after session close" (afterClose.error?.any fun err => err.code == "requestCancelled") - let (shutdown, shouldStop) ← runtime.dispatchRequest { op := .shutdown } + let shutdown ← runtime.dispatchRequest { op := .shutdown } require "shutdown remains idempotent after admission closes" shutdown.ok - require "an idempotent shutdown should not claim process teardown" (!shouldStop) require "closed admission should leave no active request" ((← ActiveRequestRegistry.count runtime.activeRequests) == 0) diff --git a/tests/lean/BeamTest/Broker/RequestHandleTest.lean b/tests/lean/BeamTest/Broker/RequestHandleTest.lean index 086062a9..6195c37f 100644 --- a/tests/lean/BeamTest/Broker/RequestHandleTest.lean +++ b/tests/lean/BeamTest/Broker/RequestHandleTest.lean @@ -28,7 +28,7 @@ private def checkStaleHandleIsolation (server : Beam.Broker.ServerRuntime) (req : Beam.Broker.Request) : IO Unit := do let staleHandleRef ← IO.mkRef (none : Option Beam.Broker.RequestHandle) - let (completedResp, _) ← server.dispatchRequestWithHandle req (fun handle => do + let completedResp ← server.dispatchRequestWithHandle req (fun handle => do staleHandleRef.set (some handle) unless ← handle.cancel do throw <| IO.userError "completed broker request handle was not cancellable" @@ -39,7 +39,7 @@ private def checkStaleHandleIsolation let staleCancelledRef ← IO.mkRef false let replacementCancelledRef ← IO.mkRef false - let (replacementResp, _) ← server.dispatchRequestWithHandle req (fun replacement => do + let replacementResp ← server.dispatchRequestWithHandle req (fun replacement => do staleCancelledRef.set (← staleHandle.cancel) replacementCancelledRef.set (← replacement.cancel) pure true) @@ -68,7 +68,7 @@ def checkCancellationAndLifetime : IO Unit := do } let runOnce : IO Unit := do let handleRef ← IO.mkRef (none : Option Beam.Broker.RequestHandle) - let (resp, _) ← server.dispatchRequestWithHandle req (fun handle => do + let resp ← server.dispatchRequestWithHandle req (fun handle => do handleRef.set (some handle) unless ← handle.cancel do throw <| IO.userError "new broker request handle was not cancellable" @@ -85,7 +85,7 @@ def checkCancellationAndLifetime : IO Unit := do checkStaleHandleIsolation server req let anonymousHandleRef ← IO.mkRef (none : Option Beam.Broker.RequestHandle) - let (anonymousResp, _) ← server.dispatchRequestWithHandle + let anonymousResp ← server.dispatchRequestWithHandle { req with clientRequestId? := none } (fun handle => do anonymousHandleRef.set (some handle) unless ← handle.cancel do @@ -98,7 +98,7 @@ def checkCancellationAndLifetime : IO Unit := do throw <| IO.userError "anonymous broker request handle remained active after dispatch" let rejectedHandleRef ← IO.mkRef (none : Option Beam.Broker.RequestHandle) - let (rejectedResp, _) ← server.dispatchRequestWithHandle req (fun handle => do + let rejectedResp ← server.dispatchRequestWithHandle req (fun handle => do rejectedHandleRef.set (some handle) pure false) checkCancelledResponse rejectedResp @@ -109,7 +109,7 @@ def checkCancellationAndLifetime : IO Unit := do runOnce let failedHandleRef ← IO.mkRef (none : Option Beam.Broker.RequestHandle) - let (failedResp, _) ← server.dispatchRequestWithHandle req (fun handle => do + let failedResp ← server.dispatchRequestWithHandle req (fun handle => do failedHandleRef.set (some handle) throw <| IO.userError "before-dispatch test failure") checkErrorCode "failed before-dispatch callback" "internalError" failedResp @@ -121,7 +121,7 @@ def checkCancellationAndLifetime : IO Unit := do let callbackInvoked ← IO.mkRef false let invalidReq := { req with cancelRequestId? := some "unrelated-field" } - let (invalidResp, _) ← server.dispatchRequestWithHandle invalidReq (fun _ => do + let invalidResp ← server.dispatchRequestWithHandle invalidReq (fun _ => do callbackInvoked.set true pure true) checkErrorCode "invalid request before admission" "invalidParams" invalidResp diff --git a/tests/lean/BeamTest/Broker/RequestStreamContractTest.lean b/tests/lean/BeamTest/Broker/RequestStreamContractTest.lean index 2243f8d6..0e0298b3 100644 --- a/tests/lean/BeamTest/Broker/RequestStreamContractTest.lean +++ b/tests/lean/BeamTest/Broker/RequestStreamContractTest.lean @@ -5,6 +5,7 @@ Author: Emilio J. Gallego Arias -/ import Beam.Broker.Protocol +import Beam.Daemon.Protocol import BeamTest.Broker.RequestStreamUtil import BeamTest.Broker.TestUtil import BeamTest.Fixtures.TodoFixture @@ -55,14 +56,54 @@ private def syncVersion let result ← requireSyncFileResult s!"sync version for {path}" (← expectOk resp) pure result.version +private partial def waitForBrokerExit + (broker : IO.Process.Child nullBrokerStdio) + (tries : Nat := 200) : IO Unit := do + if (← broker.tryWait).isSome then + pure () + else if tries == 0 then + throw <| IO.userError "daemon remained alive after its shutdown client disconnected" + else + IO.sleep 25 + waitForBrokerExit broker (tries - 1) + +private def sendShutdownAndResetConnection (port : UInt16) : IO Unit := do + let payload := (toJson ({ op := .shutdown } : Beam.Broker.Request)).compress + let script := String.intercalate ";" [ + "import socket,struct,sys", + "s=socket.create_connection(('127.0.0.1',int(sys.argv[1])))", + "s.setsockopt(socket.SOL_SOCKET,socket.SO_LINGER,struct.pack('ii',1,0))", + "p=sys.argv[2].encode('utf-8')", + "s.sendall(str(len(p)).encode('ascii')+b'\\n'+p)", + "s.close()" + ] + let out ← IO.Process.output { + cmd := "python3" + args := #["-c", script, toString port.toNat, payload] + } + if out.exitCode != 0 then + throw <| IO.userError s!"failed to reset shutdown connection\n{out.stderr}" + def main : IO Unit := do let port ← freshTcpPort let endpoint : Beam.Broker.Endpoint := .tcp port let root ← mkTempProjectRoot "beam-daemon-request-stream" copySaveProjectFixture root - let broker ← spawnLeanBroker endpoint root + let identity : Beam.Broker.DaemonIdentity := { + daemonId := "request-stream-generation" + configHash := "request-stream-config" + } + let broker ← spawnLeanBroker endpoint root (identity? := some identity) try waitForBrokerReadyForRoot endpoint root + if !(← Beam.Daemon.daemonServesGeneration endpoint testWorkspaceId root identity) then + throw <| IO.userError "daemon did not report its expected generation identity" + if ← Beam.Daemon.daemonServesGeneration endpoint testWorkspaceId root + { identity with daemonId := identity.daemonId ++ "-other" } then + throw <| IO.userError "daemon accepted a mismatched generation identity" + if ← Beam.Daemon.daemonServesGeneration endpoint testWorkspaceId root + { identity with configHash := identity.configHash ++ "-other" } then + throw <| IO.userError "daemon accepted a mismatched configuration identity" discard <| expectOk (← runClient endpoint { op := .ensure, root? := some root.toString }) let todoVersion ← syncVersion endpoint root BeamTest.Fixtures.TodoFixture.brokerPath @@ -244,13 +285,17 @@ def main : IO Unit := do expectErrorCode "stale trace save_olean" Beam.Broker.saveTraceStaleCode staleTraceSaveResp discard <| expectOk (← runClient endpoint { op := .stats }) - discard <| expectOk (← runClient endpoint { op := .shutdown }) + sendShutdownAndResetConnection port + waitForBrokerExit broker finally try broker.kill catch _ => pure () - discard <| broker.tryWait + try + discard <| broker.tryWait + catch _ => + pure () try IO.FS.removeDirAll root catch _ => diff --git a/tests/lean/BeamTest/Broker/StreamDedupTest.lean b/tests/lean/BeamTest/Broker/StreamDedupTest.lean index 7f528731..62bcbd6b 100644 --- a/tests/lean/BeamTest/Broker/StreamDedupTest.lean +++ b/tests/lean/BeamTest/Broker/StreamDedupTest.lean @@ -184,7 +184,7 @@ def checkRunAtStreamsSetupDiagnostics : IO Unit := do let server ← fakeServerWithLeanSession root session let streamedRef ← IO.mkRef #[] try - let (resp, _) ← server.dispatchRequest { + let resp ← server.dispatchRequest { op := .runAt workspaceId? := some fixtureWorkspaceId root? := some root.toString diff --git a/tests/test-beam-fast.sh b/tests/test-beam-fast.sh index e25b62f0..37547474 100644 --- a/tests/test-beam-fast.sh +++ b/tests/test-beam-fast.sh @@ -248,7 +248,8 @@ env ${mcp_stdio_env[@]+"${mcp_stdio_env[@]}"} \ --restart-cycles 1 \ --timeout "$mcp_stdio_timeout" \ > /dev/null -python3 tests/test-mcp-http-bridge.py > /dev/null +mcp_http_timeout="${BEAM_MCP_HTTP_TIMEOUT:-60}" +python3 tests/test-mcp-http-bridge.py --timeout "$mcp_http_timeout" > /dev/null mcp_self_check_timeout="${BEAM_MCP_SELF_CHECK_TIMEOUT_MS:-120000}" (cd tests/save_olean_project && \ LEAN_BEAM_MCP_SELF_CHECK_TIMEOUT_MS="$mcp_self_check_timeout" \ diff --git a/tests/test-mcp-http-bridge.py b/tests/test-mcp-http-bridge.py index db62af64..29822ae6 100644 --- a/tests/test-mcp-http-bridge.py +++ b/tests/test-mcp-http-bridge.py @@ -259,7 +259,11 @@ def main(): with tempfile.TemporaryDirectory(prefix="lean-beam-mcp-http-") as tmp: tmp_path = Path(tmp) project_root = tmp_path / "project" - shutil.copytree(repo_root / "tests" / "save_olean_project", project_root) + shutil.copytree( + repo_root / "tests" / "save_olean_project", + project_root, + ignore=shutil.ignore_patterns(".beam"), + ) ready_file = tmp_path / "ready.json" child_stderr_file = tmp_path / "lean-beam-mcp.stderr" workspace = {"root": str(project_root.resolve())} From 5cfe095b9aa01c11a42ff005daaca7ddde4ebb0c Mon Sep 17 00:00:00 2001 From: Emilio Jesus Gallego Arias Date: Wed, 26 Aug 2026 22:10:36 +0200 Subject: [PATCH 14/28] fix: make daemon shutdown observations exact --- Beam/Broker/Pending.lean | 14 ++--- Beam/Broker/Server.lean | 16 +++-- Beam/Cli/DaemonManager.lean | 59 +++++++++---------- Beam/Daemon/Protocol.lean | 37 +++++++----- docs/TESTING.md | 6 +- tests/lean/BeamTest/Broker/PendingTest.lean | 8 +-- .../Broker/RequestStreamContractTest.lean | 52 +++++++++++++--- 7 files changed, 115 insertions(+), 77 deletions(-) diff --git a/Beam/Broker/Pending.lean b/Beam/Broker/Pending.lean index 089752ac..5f6826c0 100644 --- a/Beam/Broker/Pending.lean +++ b/Beam/Broker/Pending.lean @@ -450,25 +450,21 @@ def count (registry : ActiveRequestRegistry) : IO Nat := do registry.mutex.atomically do pure (activeRequestCount (← get)) -/-- -Atomically close request admission and mark every admitted request for cancellation. Return `true` -only to the caller that changed the registry from accepting to closed. --/ -def closeAdmission (registry : ActiveRequestRegistry) : IO Bool := do - let (firstClose, active, shouldResolve) ← registry.mutex.atomically do +/-- Atomically close request admission and mark every admitted request for cancellation. -/ +def closeAdmission (registry : ActiveRequestRegistry) : IO Unit := do + let (active, shouldResolve) ← registry.mutex.atomically do let state : ActiveRequestRegistryState ← get if !state.accepting then - pure (false, #[], false) + pure (#[], false) else let (state, shouldResolve) := markDrainedIfReady { state with accepting := false } set state let named := state.requests.toList.map Prod.snd |>.toArray let anonymous := state.anonymousRequests.toList.map Prod.snd |>.toArray - pure (true, named ++ anonymous, shouldResolve) + pure (named ++ anonymous, shouldResolve) for request in active do request.cancelRef.set true resolveDrainedIfNeeded registry shouldResolve - pure firstClose /-- Wait until admission is closed and every request admitted before closure has unregistered. -/ def awaitDrained (registry : ActiveRequestRegistry) : IO Unit := do diff --git a/Beam/Broker/Server.lean b/Beam/Broker/Server.lean index 40a1a7cf..9d8913ff 100644 --- a/Beam/Broker/Server.lean +++ b/Beam/Broker/Server.lean @@ -1009,7 +1009,7 @@ def ServerRuntime.close (server : ServerRuntime) : IO Unit := do if leadsClose then let outcome ← try - discard <| ActiveRequestRegistry.closeAdmission server.activeRequests + ActiveRequestRegistry.closeAdmission server.activeRequests -- The first sweep unblocks requests already waiting on a backend. An admitted request may -- have been between admission and session creation when closure began, so repeat the sweep -- after every dispatch scope has drained to guarantee that no late session survives. @@ -2525,12 +2525,16 @@ private def handleClient (server : ServerRuntime) (client : Transport.Connection let resp ← server.dispatchRequestWithHandle req (fun handle => do let _ ← IO.asTask (prio := Task.Priority.dedicated) <| watchClientDisconnect client handle pure true) (some emitProgress) (some emitDiagnostic) - -- Stopping only wakes the listener; the accepted connection remains available for the - -- terminal response. Do this first so a disconnected shutdown caller cannot strand a - -- closed runtime behind a live listener. if stopsTransport then - requestStop server - sendResponse req.clientRequestId? resp + -- A successful send is the transport's flush boundary. Wake the listener only after the + -- terminal response has been handed off, but do so even when the caller disconnected so + -- a closed runtime cannot remain behind a live listener. + try + sendResponse req.clientRequestId? resp + finally + requestStop server + else + sendResponse req.clientRequestId? resp catch e => unless ← terminalSentRef.get do let clientRequestId? ← clientRequestIdRef.get diff --git a/Beam/Cli/DaemonManager.lean b/Beam/Cli/DaemonManager.lean index e8fe4882..7b04ba1a 100644 --- a/Beam/Cli/DaemonManager.lean +++ b/Beam/Cli/DaemonManager.lean @@ -115,25 +115,22 @@ def finishRegistryDaemonShutdown (entry : RegistryEntry) : IO Unit := do pure () private def stopDaemonEntry (entry : RegistryEntry) : IO Unit := do - let mayKillPid ← - match registryEndpoint? entry with - | some endpoint => - if ← daemonServesGeneration endpoint projectDaemonWorkspaceId - (System.FilePath.mk entry.root) entry.identity then + match registryEndpoint? entry with + | none => + finishRegistryDaemonShutdown entry + | some endpoint => + match ← daemonGenerationStatus endpoint projectDaemonWorkspaceId + (System.FilePath.mk entry.root) entry.identity with + | .exact => try - let _ ← sendRequest endpoint { op := .shutdown } - pure () + discard <| sendRequest endpoint { op := .shutdown } catch _ => pure () - pure true - else if (← daemonRoot? endpoint projectDaemonWorkspaceId).isNone then - pure true - else - pure false - | none => - pure true - if mayKillPid then - finishRegistryDaemonShutdown entry + finishRegistryDaemonShutdown entry + | .unavailable => + finishRegistryDaemonShutdown entry + | .wrongRoot _ | .wrongGeneration _ => + pure () def stopRegisteredDaemon (root : System.FilePath) : IO Unit := do match ← readRegistry? root with @@ -385,16 +382,19 @@ private partial def waitForDaemon (root : System.FilePath) (identity : DaemonIdentity) (tries : Nat := 300) : IO (Except DaemonStartupFailure Unit) := do - if ← daemonServesGeneration endpoint projectDaemonWorkspaceId root identity then - pure (.ok ()) - else - match ← daemonRoot? endpoint projectDaemonWorkspaceId with - | some daemonRoot => - pure <| .error { - message := endpointOccupancyError endpoint (System.FilePath.mk daemonRoot) root - endpointInUse := true - } - | none => + match ← daemonGenerationStatus endpoint projectDaemonWorkspaceId root identity with + | .exact => pure (.ok ()) + | .wrongRoot daemonRoot => + pure <| .error { + message := endpointOccupancyError endpoint (System.FilePath.mk daemonRoot) root + endpointInUse := true + } + | .wrongGeneration daemonRoot => + pure <| .error { + message := endpointGenerationMismatchError endpoint (System.FilePath.mk daemonRoot) + endpointInUse := true + } + | .unavailable => if (← child.tryWait).isSome then .error <$> daemonStartupFailure endpoint logPath "Beam daemon process exited before responding" else if tries == 0 then @@ -547,10 +547,9 @@ def registryLiveFor | some endpoint => -- The owner pipe makes endpoint liveness authoritative across PID domains. A -- same-domain dead owner is rejected immediately; another domain is never probed. - if ← daemonServesGeneration endpoint projectDaemonWorkspaceId root entry.identity then - pure (some entry) - else - pure none + match ← daemonGenerationStatus endpoint projectDaemonWorkspaceId root entry.identity with + | .exact => pure (some entry) + | .unavailable | .wrongRoot _ | .wrongGeneration _ => pure none private abbrev detachedDaemonStdio : IO.Process.StdioConfig where stdin := .null diff --git a/Beam/Daemon/Protocol.lean b/Beam/Daemon/Protocol.lean index 771dc9e0..dbd83d1b 100644 --- a/Beam/Daemon/Protocol.lean +++ b/Beam/Daemon/Protocol.lean @@ -101,6 +101,12 @@ def endpointOccupancyError def endpointInUseError (endpoint : Transport.Endpoint) : String := s!"selected endpoint {endpointSummary endpoint} is already in use" +def endpointGenerationMismatchError + (endpoint : Transport.Endpoint) + (daemonRoot : System.FilePath) : String := + s!"selected endpoint {endpointSummary endpoint} already serves Beam root {daemonRoot} " ++ + "with another daemon generation" + def startupLogSuggestsEndpointInUse (logText : String) : Bool := logText.contains "address already in use" || logText.contains "Address already in use" @@ -111,29 +117,28 @@ def shouldRetryAutomaticStartup (endpointOccupied startupAddressInUse : Bool) : Bool := usesAutomaticEndpoint && tries > 0 && (endpointOccupied || startupAddressInUse) --- A listening TCP port is not enough evidence that it belongs to this project: --- random auto-port selection can collide with an unrelated Beam daemon. -def daemonServesRoot - (endpoint : Transport.Endpoint) - (workspaceId : WorkspaceId) - (root : System.FilePath) : IO Bool := do - match ← daemonRoot? endpoint workspaceId with - | some daemonRoot => Beam.sameFilePath (System.FilePath.mk daemonRoot) root - | none => pure false +inductive DaemonGenerationStatus where + | unavailable + | wrongRoot (daemonRoot : String) + | wrongGeneration (daemonRoot : String) + | exact + deriving Repr -/-- Whether an endpoint serves the expected root and exact wrapper-owned daemon generation. -/ -def daemonServesGeneration +/-- Classify one endpoint observation against the expected root and wrapper daemon generation. -/ +def daemonGenerationStatus (endpoint : Transport.Endpoint) (workspaceId : WorkspaceId) (root : System.FilePath) - (identity : DaemonIdentity) : IO Bool := do + (identity : DaemonIdentity) : IO DaemonGenerationStatus := do match ← daemonProbe? endpoint workspaceId with + | none => pure .unavailable | some probe => - if probe.identity? != some identity then - pure false + unless ← Beam.sameFilePath (System.FilePath.mk probe.root) root do + return .wrongRoot probe.root + if probe.identity? == some identity then + pure .exact else - Beam.sameFilePath (System.FilePath.mk probe.root) root - | none => pure false + pure <| .wrongGeneration probe.root def endpointAcceptsConnection (endpoint : Transport.Endpoint) : IO Bool := do try diff --git a/docs/TESTING.md b/docs/TESTING.md index 8977f47f..7be70c70 100644 --- a/docs/TESTING.md +++ b/docs/TESTING.md @@ -105,9 +105,9 @@ Additional Beam lanes: Current Beam coverage includes: - fast Beam daemon smoke, request-stream, save-stream, startup-handshake, tracked-diagnostic dedup, - exact broker request-handle lifetime, authenticated daemon-generation probes, shutdown after the - requesting TCP client resets its connection, protocol tests, and validated-toolchain/release-line - CI policy consistency through + exact broker request-handle lifetime, identity-matched daemon-generation probes, terminal shutdown + response delivery, shutdown after the requesting TCP client resets its connection, protocol tests, + and validated-toolchain/release-line CI policy consistency through [tests/test-beam-fast.sh](../tests/test-beam-fast.sh) - wrapper coverage through [tests/test-beam-wrapper.sh](../tests/test-beam-wrapper.sh), which aggregates focused probe, runtime, sync/save, handle, and diagnostic slices - focused daemon lifecycle coverage in [tests/test-beam-wrapper-daemon.sh](../tests/test-beam-wrapper-daemon.sh), diff --git a/tests/lean/BeamTest/Broker/PendingTest.lean b/tests/lean/BeamTest/Broker/PendingTest.lean index 10185a52..f62813ed 100644 --- a/tests/lean/BeamTest/Broker/PendingTest.lean +++ b/tests/lean/BeamTest/Broker/PendingTest.lean @@ -127,8 +127,7 @@ private def checkActiveRegistryCloseDrain : IO Unit := do ← ActiveRequestRegistry.register registry (some "closing-request") let anonymous ← expectRegistered "register anonymous request before close" <| ← ActiveRequestRegistry.register registry none - require "first admission close should lead closure" - (← ActiveRequestRegistry.closeAdmission registry) + ActiveRequestRegistry.closeAdmission registry for active in #[named, anonymous] do match ← ensureRequestNotCancelled (some active.cancelRef) with | .ok _ => throw <| IO.userError "admission close did not cancel an active request" @@ -150,8 +149,9 @@ private def checkActiveRegistryCloseDrain : IO Unit := do match ← IO.wait drainTask with | .ok () => pure () | .error err => throw err - require "repeated admission close should be idempotent" - (!(← ActiveRequestRegistry.closeAdmission registry)) + ActiveRequestRegistry.closeAdmission registry + require "repeated admission close should preserve the drained state" + ((← ActiveRequestRegistry.count registry) == 0) private def checkPendingCancellationIdentity : IO Unit := do let registry ← ActiveRequestRegistry.create diff --git a/tests/lean/BeamTest/Broker/RequestStreamContractTest.lean b/tests/lean/BeamTest/Broker/RequestStreamContractTest.lean index 0e0298b3..d605a3d6 100644 --- a/tests/lean/BeamTest/Broker/RequestStreamContractTest.lean +++ b/tests/lean/BeamTest/Broker/RequestStreamContractTest.lean @@ -62,7 +62,7 @@ private partial def waitForBrokerExit if (← broker.tryWait).isSome then pure () else if tries == 0 then - throw <| IO.userError "daemon remained alive after its shutdown client disconnected" + throw <| IO.userError "daemon remained alive after shutdown" else IO.sleep 25 waitForBrokerExit broker (tries - 1) @@ -84,6 +84,24 @@ private def sendShutdownAndResetConnection (port : UInt16) : IO Unit := do if out.exitCode != 0 then throw <| IO.userError s!"failed to reset shutdown connection\n{out.stderr}" +private def checkShutdownResponseBeforeExit (root : System.FilePath) : IO Unit := do + let port ← freshTcpPort + let endpoint : Beam.Broker.Endpoint := .tcp port + let broker ← spawnLeanBroker endpoint root + try + waitForBrokerReadyForRoot endpoint root + discard <| expectOk (← runClient endpoint { op := .shutdown }) + waitForBrokerExit broker + finally + try + broker.kill + catch _ => + pure () + try + discard <| broker.tryWait + catch _ => + pure () + def main : IO Unit := do let port ← freshTcpPort let endpoint : Beam.Broker.Endpoint := .tcp port @@ -96,14 +114,25 @@ def main : IO Unit := do let broker ← spawnLeanBroker endpoint root (identity? := some identity) try waitForBrokerReadyForRoot endpoint root - if !(← Beam.Daemon.daemonServesGeneration endpoint testWorkspaceId root identity) then - throw <| IO.userError "daemon did not report its expected generation identity" - if ← Beam.Daemon.daemonServesGeneration endpoint testWorkspaceId root - { identity with daemonId := identity.daemonId ++ "-other" } then - throw <| IO.userError "daemon accepted a mismatched generation identity" - if ← Beam.Daemon.daemonServesGeneration endpoint testWorkspaceId root - { identity with configHash := identity.configHash ++ "-other" } then - throw <| IO.userError "daemon accepted a mismatched configuration identity" + match ← Beam.Daemon.daemonGenerationStatus endpoint testWorkspaceId root identity with + | .exact => pure () + | status => + throw <| IO.userError + s!"daemon did not report its expected generation identity: {repr status}" + for mismatched in #[ + { identity with daemonId := identity.daemonId ++ "-other" }, + { identity with configHash := identity.configHash ++ "-other" } + ] do + match ← Beam.Daemon.daemonGenerationStatus endpoint testWorkspaceId root mismatched with + | .wrongGeneration _ => pure () + | status => + throw <| IO.userError s!"daemon generation mismatch was classified as {repr status}" + let otherRoot := root / "other-root" + IO.FS.createDirAll otherRoot + match ← Beam.Daemon.daemonGenerationStatus endpoint testWorkspaceId otherRoot identity with + | .wrongRoot _ => pure () + | status => + throw <| IO.userError s!"daemon root mismatch was classified as {repr status}" discard <| expectOk (← runClient endpoint { op := .ensure, root? := some root.toString }) let todoVersion ← syncVersion endpoint root BeamTest.Fixtures.TodoFixture.brokerPath @@ -287,6 +316,11 @@ def main : IO Unit := do sendShutdownAndResetConnection port waitForBrokerExit broker + match ← Beam.Daemon.daemonGenerationStatus endpoint testWorkspaceId root identity with + | .unavailable => pure () + | status => + throw <| IO.userError s!"stopped daemon was classified as {repr status}" + checkShutdownResponseBeforeExit root finally try broker.kill From 2f781993df3132f33c3842f3c4998afd99f3c12f Mon Sep 17 00:00:00 2001 From: Emilio Jesus Gallego Arias Date: Wed, 26 Aug 2026 22:21:10 +0200 Subject: [PATCH 15/28] test: restore daemon ownership safety coverage --- Beam/Mcp/StdioServer.lean | 14 +++--- docs/TESTING.md | 13 ++--- tests/test-beam-wrapper-daemon.sh | 82 ++++++++++++++++++++++++++++++- 3 files changed, 93 insertions(+), 16 deletions(-) diff --git a/Beam/Mcp/StdioServer.lean b/Beam/Mcp/StdioServer.lean index 74fbff70..353ff344 100644 --- a/Beam/Mcp/StdioServer.lean +++ b/Beam/Mcp/StdioServer.lean @@ -295,15 +295,15 @@ private def Coordinator.cancelRequest | some request => request.cancel private def Coordinator.beginClosing - (coordinator : Coordinator) : IO (Bool × Array InFlightRequest) := do - let (alreadyClosing, requests) ← coordinator.routing.atomically do + (coordinator : Coordinator) : IO (Array InFlightRequest) := do + let requests ← coordinator.routing.atomically do let routing ← get let requests := routing.admitted.toList.map Prod.snd |>.toArray set { routing with closing := true } - pure (routing.closing, requests) + pure requests for request in requests do request.cancel - pure (alreadyClosing, requests) + pure requests private def awaitRequestDone (request : InFlightRequest) : IO Unit := do awaitPromise s!"in-flight request {request.id.label}" request.done @@ -322,11 +322,9 @@ private def Coordinator.otherAdmittedRequests if other.brokerId == request.brokerId then none else some other) |>.toArray private def Coordinator.closeTransport (coordinator : Coordinator) : IO Unit := do - let (alreadyClosing, requests) ← - coordinator.beginClosing + let requests ← coordinator.beginClosing coordinator.awaitRequests requests - unless alreadyClosing do - coordinator.state.closeRuntime + coordinator.state.closeRuntime private def Coordinator.admitToolRequest (coordinator : Coordinator) diff --git a/docs/TESTING.md b/docs/TESTING.md index 7be70c70..2efaf526 100644 --- a/docs/TESTING.md +++ b/docs/TESTING.md @@ -111,12 +111,13 @@ Current Beam coverage includes: [tests/test-beam-fast.sh](../tests/test-beam-fast.sh) - wrapper coverage through [tests/test-beam-wrapper.sh](../tests/test-beam-wrapper.sh), which aggregates focused probe, runtime, sync/save, handle, and diagnostic slices - focused daemon lifecycle coverage in [tests/test-beam-wrapper-daemon.sh](../tests/test-beam-wrapper-daemon.sh), - including the no-implicit-start contract, duplicate-owner rejection, endpoint collision safety, - explicit shutdown, cancellation of requests active during shutdown or owner loss, exact-generation - cleanup that preserves a replacement registry, registry removal before a paused daemon can finish - draining, rejection of attachment to that unpublished draining generation, holder reporting after - an unexpected daemon crash, abrupt owner death through inherited-pipe EOF, stale registry cleanup, - and self-termination after the project worktree disappears + including the no-implicit-start contract, duplicate-owner rejection, Beam and non-Beam endpoint + collision safety, cross-root stale-registry cleanup that preserves the daemon serving the other + root, explicit shutdown, cancellation of requests active during shutdown or owner loss, + exact-generation cleanup that preserves a replacement registry, registry removal before a paused + daemon can finish draining, rejection of attachment to that unpublished draining generation, + holder reporting after an unexpected daemon crash, abrupt owner death through inherited-pipe EOF, + stale registry cleanup, and self-termination after the project worktree disappears - Linux-only PID-isolated sandbox wrapper coverage in [tests/test-beam-wrapper-sandbox.sh](../tests/test-beam-wrapper-sandbox.sh), including cross-namespace endpoint attachment, duplicate-owner rejection, a paused owner without time-based expiry, explicit shutdown, killed-owner EOF cleanup, stale-registry recovery, distinct diff --git a/tests/test-beam-wrapper-daemon.sh b/tests/test-beam-wrapper-daemon.sh index 006ce9a1..f3e34b7a 100644 --- a/tests/test-beam-wrapper-daemon.sh +++ b/tests/test-beam-wrapper-daemon.sh @@ -34,6 +34,8 @@ hold_pid="" root_removed="false" active_request_pid="" paused_daemon_pid="" +busy_pid="" +busy_port_file="" start_slow_request() { local root="$1" @@ -110,6 +112,15 @@ stop_hold_process() { } cleanup() { + if [ -n "$busy_pid" ]; then + kill "$busy_pid" > /dev/null 2>&1 || true + wait "$busy_pid" 2>/dev/null || true + busy_pid="" + fi + if [ -n "$busy_port_file" ]; then + rm -f -- "$busy_port_file" + busy_port_file="" + fi if [ -n "$paused_daemon_pid" ]; then kill -CONT "$paused_daemon_pid" > /dev/null 2>&1 || true paused_daemon_pid="" @@ -236,8 +247,6 @@ assert_json_field_equals "owned ensure response" "$ensure_json" ok true stats_json="$("$beam_script" --root "$tmp1" stats)" assert_json_field_equals "owned stats response" "$stats_json" ok true -start_slow_request "$tmp1" "shutdown-active" "shutdown-active" - second_owner_out="$tmp1/second-owner.out" second_owner_err="$tmp1/second-owner.err" if "$beam_script" --root "$tmp1" ensure --hold > "$second_owner_out" 2> "$second_owner_err"; then @@ -270,6 +279,75 @@ if [ -e "$tmp2/.beam/beam-daemon.json" ]; then exit 1 fi +stale_registry="$tmp2/.beam/beam-daemon.json" +REGISTRY_TEMPLATE="$registry" STALE_REGISTRY="$stale_registry" STALE_ROOT="$tmp2" python3 - <<'PY' +import json +import os + +with open(os.environ["REGISTRY_TEMPLATE"], encoding="utf-8") as stream: + entry = json.load(stream) +entry["root"] = os.path.realpath(os.environ["STALE_ROOT"]) +replacement = os.environ["STALE_REGISTRY"] + ".replacement" +with open(replacement, "w", encoding="utf-8") as stream: + json.dump(entry, stream, separators=(",", ":")) + stream.write("\n") +os.replace(replacement, os.environ["STALE_REGISTRY"]) +PY +"$beam_script" --root "$tmp2" shutdown > /dev/null +if [ -e "$stale_registry" ]; then + echo "expected shutdown to remove a stale cross-root registry" >&2 + cat "$stale_registry" >&2 + exit 1 +fi +if ! kill -0 "$owner1_pid" 2>/dev/null || ! kill -0 "$daemon1_pid" 2>/dev/null; then + echo "stale cross-root registry cleanup must not stop the daemon or owner serving the other root" >&2 + exit 1 +fi + +busy_port_file="$(mktemp "$tmp2/non-beam-port-XXXXXX")" +python3 - "$busy_port_file" <<'PY' & +import socketserver +import sys + +class Handler(socketserver.BaseRequestHandler): + def handle(self): + self.request.recv(4096) + +with socketserver.TCPServer(("127.0.0.1", 0), Handler) as server: + with open(sys.argv[1], "w", encoding="utf-8") as stream: + print(server.server_address[1], file=stream, flush=True) + server.serve_forever() +PY +busy_pid="$!" +if ! wait_for_nonempty_file "$busy_port_file" "non-Beam occupied port"; then + exit 1 +fi +busy_port="$(cat "$busy_port_file")" +busy_out="$tmp2/non-beam-port.out" +busy_err="$tmp2/non-beam-port.err" +if "$beam_script" --root "$tmp2" --port "$busy_port" ensure --hold > "$busy_out" 2> "$busy_err"; then + echo "expected owner startup to reject a port occupied by a non-Beam service" >&2 + cat "$busy_out" >&2 + exit 1 +fi +if ! grep -Fq "already in use" "$busy_err"; then + echo "expected non-Beam port collision to report the occupied endpoint" >&2 + cat "$busy_err" >&2 + exit 1 +fi +if [ -e "$tmp2/.beam/beam-daemon.json" ]; then + echo "expected non-Beam port collision not to publish a registry" >&2 + cat "$tmp2/.beam/beam-daemon.json" >&2 + exit 1 +fi +kill "$busy_pid" > /dev/null 2>&1 || true +wait "$busy_pid" 2>/dev/null || true +busy_pid="" +rm -f -- "$busy_port_file" +busy_port_file="" + +start_slow_request "$tmp1" "shutdown-active" "shutdown-active" + shutdown_json="$("$beam_script" --root "$tmp1" shutdown)" assert_json_field_equals "explicit session shutdown" "$shutdown_json" ok true expect_slow_request_cancelled "$tmp1" "shutdown-active" "shutdown-active" From 50411fa552e4ec2d295cf762d487d5101b8080d8 Mon Sep 17 00:00:00 2001 From: Emilio Jesus Gallego Arias Date: Wed, 26 Aug 2026 22:57:17 +0200 Subject: [PATCH 16/28] fix: bound daemon identity probes --- Beam/Broker/Client.lean | 50 +++++++++-- Beam/Broker/Transport.lean | 77 ++++++++++++++--- Beam/Cli/Commands.lean | 2 +- Beam/Cli/DaemonManager.lean | 60 ++++++++------ Beam/Daemon/Protocol.lean | 83 ++++++++++++------- Beam/Mcp/StdioServer.lean | 8 +- docs/TESTING.md | 3 +- tests/lean/BeamTest/Broker/CliDaemonTest.lean | 35 ++++++++ tests/test-beam-wrapper-daemon.sh | 50 +++++++++++ 9 files changed, 290 insertions(+), 78 deletions(-) diff --git a/Beam/Broker/Client.lean b/Beam/Broker/Client.lean index 00ed387d..9083d725 100644 --- a/Beam/Broker/Client.lean +++ b/Beam/Broker/Client.lean @@ -24,14 +24,19 @@ inductive BrokerClientFailure where | transport (error : IO.Error) | invalidResponse (detail : String) | streamCallback (error : IO.Error) + | responseTimeout (timeoutMs : Nat) def BrokerClientFailure.detail : BrokerClientFailure → String | .transport error | .streamCallback error => error.toString | .invalidResponse detail => detail + | .responseTimeout timeoutMs => + s!"Beam daemon response timed out after {timeoutMs} ms" private def BrokerClientFailure.toIOError : BrokerClientFailure → IO.Error | .transport error | .streamCallback error => error | .invalidResponse detail => IO.userError detail + | .responseTimeout timeoutMs => + IO.userError s!"Beam daemon response timed out after {timeoutMs} ms" def parsePortText (name value : String) : Except String UInt16 := do let some n := value.toNat? @@ -89,11 +94,16 @@ def formatStreamDiagnostic (diagnostic : StreamDiagnostic) : String := "" s!"beam: diagnostic {severity}{blocking} {diagnostic.path}:{line}:{character}: {message}" -/-- Send one request while preserving transport, response, and callback failures as typed data. -/ -partial def sendRequestWithStreamResult +private structure ResponseDeadline where + timeoutMs : Nat + deadlineNanos : Nat + +/-- Send one request while preserving transport, response, callback, and timeout failures. -/ +private partial def sendRequestWithStreamResultCore (endpoint : Endpoint) (req : Request) - (onStream : StreamMessage → IO Unit) : IO (Except BrokerClientFailure Response) := do + (onStream : StreamMessage → IO Unit) + (responseTimeoutMs? : Option Nat) : IO (Except BrokerClientFailure Response) := do let client ← match ← captureClientFailure .transport (Transport.connect endpoint) with | .ok client => pure client @@ -103,11 +113,24 @@ partial def sendRequestWithStreamResult Transport.sendMsg client (toJson req).compress with | .ok () => pure () | .error failure => return .error failure + let deadline? : Option ResponseDeadline ← responseTimeoutMs?.mapM fun timeoutMs => do + pure { + timeoutMs + deadlineNanos := (← IO.monoNanosNow) + timeoutMs * 1000000 + } let rec loop : IO (Except BrokerClientFailure Response) := do let msg ← - match ← captureClientFailure .transport (Transport.recvMsg client) with - | .ok msg => pure msg - | .error failure => return .error failure + match deadline? with + | none => + match ← captureClientFailure .transport (Transport.recvMsg client) with + | .ok msg => pure msg + | .error failure => return .error failure + | some deadline => + match ← captureClientFailure .transport <| + Transport.recvMsgUntil client deadline.deadlineNanos with + | .ok (some msg) => pure msg + | .ok none => return .error (.responseTimeout deadline.timeoutMs) + | .error failure => return .error failure let stream ← match decodeStreamMessage msg with | .ok stream => pure stream @@ -127,6 +150,21 @@ partial def sendRequestWithStreamResult finally Transport.closeConnection client +/-- Send one request while preserving transport, response, and callback failures as typed data. -/ +partial def sendRequestWithStreamResult + (endpoint : Endpoint) + (req : Request) + (onStream : StreamMessage → IO Unit) : IO (Except BrokerClientFailure Response) := do + sendRequestWithStreamResultCore endpoint req onStream none + +/-- Send one request with an absolute timeout for receiving its complete response stream. -/ +partial def sendRequestWithStreamTimeoutResult + (endpoint : Endpoint) + (req : Request) + (timeoutMs : Nat) + (onStream : StreamMessage → IO Unit) : IO (Except BrokerClientFailure Response) := do + sendRequestWithStreamResultCore endpoint req onStream (some timeoutMs) + partial def sendRequestWithStream (endpoint : Endpoint) (req : Request) diff --git a/Beam/Broker/Transport.lean b/Beam/Broker/Transport.lean index 669072cf..646a87dd 100644 --- a/Beam/Broker/Transport.lean +++ b/Beam/Broker/Transport.lean @@ -41,6 +41,34 @@ private def waitTcpPromise (promise : IO.Promise (Except IO.Error α)) (failureM | .ok value => pure value | .error err => throw <| IO.userError s!"{failureMessage}: {err}" +private inductive ReceiveWaitResult (α : Type) where + | completed (value : α) + | timedOut + +private partial def waitTcpReceivePromiseUntil + (client : TCP.Socket) + (promise : IO.Promise (Except IO.Error α)) + (deadlineNanos : Nat) + (failureMessage : String) + (pollMs : Nat := 10) : IO (ReceiveWaitResult α) := do + let resultTask := promise.result? + let rec loop : IO (ReceiveWaitResult α) := do + if ← IO.hasFinished resultTask then + let some result ← IO.wait resultTask + | throw <| IO.userError failureMessage + match result with + | .ok value => pure <| .completed value + | .error err => throw <| IO.userError s!"{failureMessage}: {err}" + else if (← IO.monoNanosNow) >= deadlineNanos then + -- The caller abandons and closes this connection after timeout. Cancel the exact pending UV + -- receive first so no read remains attached to the socket. + TCP.Socket.cancelRecv client + pure .timedOut + else + IO.sleep pollMs.toUInt32 + loop + loop + def connect (endpoint : Endpoint) : IO Connection := do match endpoint with | .tcp port => @@ -84,29 +112,51 @@ private def sendMsgTcp (client : TCP.Socket) (msg : String) : IO Unit := do let promise ← TCP.Socket.send client #[header, bytes] waitTcpPromise promise "Beam daemon connection closed before TCP send completed" -private def recvMsgTcp (client : TCP.Socket) : IO String := do +private def receiveTcp (client : TCP.Socket) (size : UInt64) : IO (ReceiveWaitResult (Option ByteArray)) := do + let promise ← TCP.Socket.recv? client size + .completed <$> waitTcpPromise promise "Beam daemon connection closed during TCP receive" + +private def receiveTcpUntil + (client : TCP.Socket) + (size : UInt64) + (deadlineNanos : Nat) : IO (ReceiveWaitResult (Option ByteArray)) := do + let promise ← TCP.Socket.recv? client size + waitTcpReceivePromiseUntil client promise deadlineNanos + "Beam daemon connection closed during TCP receive" + +private def recvMsgTcpUsing + (receive : UInt64 → IO (ReceiveWaitResult (Option ByteArray))) : IO (Option String) := do let mut header := ByteArray.empty repeat - let promise ← TCP.Socket.recv? client 1 - let some chunk ← waitTcpPromise promise "Beam daemon connection closed during TCP receive" - | throw <| IO.userError "Beam daemon connection closed" - if chunk[0]! == '\n'.toUInt8 then - break - header := header ++ chunk + match ← receive 1 with + | .timedOut => return none + | .completed none => throw <| IO.userError "Beam daemon connection closed" + | .completed (some chunk) => + if chunk[0]! == '\n'.toUInt8 then + break + header := header ++ chunk let some lenStr := String.fromUTF8? header | throw <| IO.userError "invalid Beam daemon header" let some len := lenStr.toNat? | throw <| IO.userError "invalid Beam daemon length" let mut payload := ByteArray.empty while payload.size < len do - let promise ← TCP.Socket.recv? client (len - payload.size).toUInt64 - let some chunk ← waitTcpPromise promise "Beam daemon connection closed during TCP receive" - | throw <| IO.userError "Beam daemon connection closed" - payload := payload ++ chunk + match ← receive (len - payload.size).toUInt64 with + | .timedOut => return none + | .completed none => throw <| IO.userError "Beam daemon connection closed" + | .completed (some chunk) => payload := payload ++ chunk let some msg := String.fromUTF8? payload | throw <| IO.userError "invalid Beam daemon UTF-8" + pure (some msg) + +private def recvMsgTcp (client : TCP.Socket) : IO String := do + let some msg ← recvMsgTcpUsing (receiveTcp client) + | throw <| IO.userError "unbounded Beam daemon receive timed out" pure msg +private def recvMsgTcpUntil (client : TCP.Socket) (deadlineNanos : Nat) : IO (Option String) := do + recvMsgTcpUsing (receiveTcpUntil client · deadlineNanos) + def sendMsg (conn : Connection) (msg : String) : IO Unit := do match conn with | .tcp client => sendMsgTcp client msg @@ -115,4 +165,9 @@ def recvMsg (conn : Connection) : IO String := do match conn with | .tcp client => recvMsgTcp client +/-- Receive one framed message by an absolute monotonic deadline, returning `none` on timeout. -/ +def recvMsgUntil (conn : Connection) (deadlineNanos : Nat) : IO (Option String) := do + match conn with + | .tcp client => recvMsgTcpUntil client deadlineNanos + end Beam.Broker.Transport diff --git a/Beam/Cli/Commands.lean b/Beam/Cli/Commands.lean index 6399a17f..2058d4e8 100644 --- a/Beam/Cli/Commands.lean +++ b/Beam/Cli/Commands.lean @@ -146,7 +146,7 @@ private def runThenHoldUntilInterrupted pure () try act - while !(← IO.hasFinished task) && !(← owner.exited) && + while !(← IO.hasFinished task) && (← owner.exitCode?).isNone && (← owner.registered) && !(← IO.checkCanceled) do IO.sleep 50 if ← IO.hasFinished task then diff --git a/Beam/Cli/DaemonManager.lean b/Beam/Cli/DaemonManager.lean index 7b04ba1a..d8b407b9 100644 --- a/Beam/Cli/DaemonManager.lean +++ b/Beam/Cli/DaemonManager.lean @@ -129,7 +129,7 @@ private def stopDaemonEntry (entry : RegistryEntry) : IO Unit := do finishRegistryDaemonShutdown entry | .unavailable => finishRegistryDaemonShutdown entry - | .wrongRoot _ | .wrongGeneration _ => + | .unrecognized _ | .wrongRoot _ | .wrongGeneration _ => pure () def stopRegisteredDaemon (root : System.FilePath) : IO Unit := do @@ -165,21 +165,29 @@ private partial def selectUnoccupiedEndpoint (opts : CliOptions) (tries : Nat := 10) : IO Transport.Endpoint := do let endpoint ← selectEndpoint opts - match ← daemonRoot? endpoint projectDaemonWorkspaceId with - | none => - pure () - | some daemonRoot => - if usesAutomaticTcpEndpoint opts && tries > 0 then - return ← selectUnoccupiedEndpoint desired opts (tries - 1) - else - throw <| IO.userError (endpointOccupancyError endpoint (System.FilePath.mk daemonRoot) desired.root) - if ← endpointAcceptsConnection endpoint then + let retryOrReject (message : String) : IO Transport.Endpoint := do if usesAutomaticTcpEndpoint opts && tries > 0 then - return ← selectUnoccupiedEndpoint desired opts (tries - 1) + selectUnoccupiedEndpoint desired opts (tries - 1) else - throw <| IO.userError (endpointInUseError endpoint) - else - pure endpoint + throw <| IO.userError message + match ← daemonRootResult endpoint projectDaemonWorkspaceId with + | .ok daemonRoot => + retryOrReject <| endpointOccupancyError endpoint + (System.FilePath.mk daemonRoot) desired.root + | .error failure => + let occupied ← + match failure with + | .transport _ => endpointAcceptsConnection endpoint + | .invalidResponse _ | .streamCallback _ | .responseTimeout _ => pure true + if !occupied then + pure endpoint + else + let message := + match failure with + | .transport _ => endpointInUseError endpoint + | .invalidResponse _ | .streamCallback _ | .responseTimeout _ => + endpointProtocolError endpoint failure.detail + retryOrReject message private def daemonFailureIncidentRetainCount : Nat := 50 @@ -220,6 +228,7 @@ private def daemonFailureIncidentKind? : BrokerClientFailure → Option String | .transport _ => some "brokerTransportFailure" | .invalidResponse _ => some "invalidBrokerResponse" | .streamCallback _ => none + | .responseTimeout _ => some "brokerResponseTimeout" private def daemonFailureIncidentTimestampLabel (timestamp : String) : String := (timestamp.replace "-" "").replace ":" "" @@ -382,6 +391,14 @@ private partial def waitForDaemon (root : System.FilePath) (identity : DaemonIdentity) (tries : Nat := 300) : IO (Except DaemonStartupFailure Unit) := do + let retryOrFail (detail : String) : IO (Except DaemonStartupFailure Unit) := do + if (← child.tryWait).isSome then + .error <$> daemonStartupFailure endpoint logPath "Beam daemon process exited before responding" + else if tries == 0 then + .error <$> daemonStartupFailure endpoint logPath detail + else + IO.sleep 100 + waitForDaemon child endpoint logPath root identity (tries - 1) match ← daemonGenerationStatus endpoint projectDaemonWorkspaceId root identity with | .exact => pure (.ok ()) | .wrongRoot daemonRoot => @@ -394,14 +411,10 @@ private partial def waitForDaemon message := endpointGenerationMismatchError endpoint (System.FilePath.mk daemonRoot) endpointInUse := true } + | .unrecognized detail => + retryOrFail (endpointProtocolError endpoint detail) | .unavailable => - if (← child.tryWait).isSome then - .error <$> daemonStartupFailure endpoint logPath "Beam daemon process exited before responding" - else if tries == 0 then - .error <$> daemonStartupFailure endpoint logPath "Beam daemon did not become ready before timeout" - else - IO.sleep 100 - waitForDaemon child endpoint logPath root identity (tries - 1) + retryOrFail "Beam daemon did not become ready before timeout" private def newDaemonGenerationId (configHash : String) : IO String := do let startedMonoNanos ← IO.monoNanosNow @@ -549,7 +562,7 @@ def registryLiveFor -- same-domain dead owner is rejected immediately; another domain is never probed. match ← daemonGenerationStatus endpoint projectDaemonWorkspaceId root entry.identity with | .exact => pure (some entry) - | .unavailable | .wrongRoot _ | .wrongGeneration _ => pure none + | .unavailable | .unrecognized _ | .wrongRoot _ | .wrongGeneration _ => pure none private abbrev detachedDaemonStdio : IO.Process.StdioConfig where stdin := .null @@ -577,9 +590,6 @@ def ProjectDaemonOwner.exitCode? (owner : ProjectDaemonOwner) : IO (Option UInt3 owner.exitCodeRef.set (some exitCode) pure exitCode? -def ProjectDaemonOwner.exited (owner : ProjectDaemonOwner) : IO Bool := - return (← owner.exitCode?).isSome - /-- Whether this owner generation is still the one published for its project. -/ def ProjectDaemonOwner.registered (owner : ProjectDaemonOwner) : IO Bool := do match ← readRegistry? owner.root with diff --git a/Beam/Daemon/Protocol.lean b/Beam/Daemon/Protocol.lean index dbd83d1b..bf4e6246 100644 --- a/Beam/Daemon/Protocol.lean +++ b/Beam/Daemon/Protocol.lean @@ -70,28 +70,41 @@ private structure DaemonProbe where root : String identity? : Option DaemonIdentity -private def daemonProbeOfResponse? (resp : Response) : Option DaemonProbe := do - let result ← resp.result? - let root ← result.getObjValAs? String "root" |>.toOption - let identity? := result.getObjValAs? DaemonIdentity "daemonIdentity" |>.toOption +private def daemonProbeResponseTimeoutMs : Nat := + 2000 + +private def daemonProbeOfResponse (resp : Response) : Except BrokerClientFailure DaemonProbe := do + unless resp.ok do + throw <| .invalidResponse s!"Beam daemon stats probe failed: {(toJson resp).compress}" + let some result := resp.result? + | throw <| .invalidResponse "Beam daemon stats probe omitted its result" + let root ← + match result.getObjValAs? String "root" with + | .ok root => pure root + | .error err => throw <| .invalidResponse s!"invalid Beam daemon stats root: {err}" + let identity? ← + match result.getObjVal? "daemonIdentity" with + | .error _ => pure none + | .ok identityJson => + match fromJson? identityJson with + | .ok identity => pure (some identity) + | .error err => + throw <| .invalidResponse s!"invalid Beam daemon identity: {err}" pure { root, identity? } -private def daemonProbe? +private def daemonProbe (endpoint : Transport.Endpoint) - (workspaceId : WorkspaceId) : IO (Option DaemonProbe) := do - try - let resp ← sendRequest endpoint { op := .stats, workspaceId? := some workspaceId } - if resp.ok then - pure (daemonProbeOfResponse? resp) - else - pure none - catch _ => - pure none - -def daemonRoot? + (workspaceId : WorkspaceId) : IO (Except BrokerClientFailure DaemonProbe) := do + match ← sendRequestWithStreamTimeoutResult endpoint + { op := .stats, workspaceId? := some workspaceId } + daemonProbeResponseTimeoutMs (fun _ => pure ()) with + | .ok resp => pure <| daemonProbeOfResponse resp + | .error failure => pure <| .error failure + +def daemonRootResult (endpoint : Transport.Endpoint) - (workspaceId : WorkspaceId) : IO (Option String) := do - pure <| (← daemonProbe? endpoint workspaceId).map (·.root) + (workspaceId : WorkspaceId) : IO (Except BrokerClientFailure String) := do + pure <| (← daemonProbe endpoint workspaceId).map (·.root) def endpointOccupancyError (endpoint : Transport.Endpoint) @@ -107,6 +120,9 @@ def endpointGenerationMismatchError s!"selected endpoint {endpointSummary endpoint} already serves Beam root {daemonRoot} " ++ "with another daemon generation" +def endpointProtocolError (endpoint : Transport.Endpoint) (detail : String) : String := + s!"selected endpoint {endpointSummary endpoint} did not return a valid Beam daemon response: {detail}" + def startupLogSuggestsEndpointInUse (logText : String) : Bool := logText.contains "address already in use" || logText.contains "Address already in use" @@ -117,8 +133,17 @@ def shouldRetryAutomaticStartup (endpointOccupied startupAddressInUse : Bool) : Bool := usesAutomaticEndpoint && tries > 0 && (endpointOccupied || startupAddressInUse) +def endpointAcceptsConnection (endpoint : Transport.Endpoint) : IO Bool := do + try + let conn ← Transport.connect endpoint + Transport.closeConnection conn + pure true + catch _ => + pure false + inductive DaemonGenerationStatus where | unavailable + | unrecognized (detail : String) | wrongRoot (daemonRoot : String) | wrongGeneration (daemonRoot : String) | exact @@ -130,9 +155,17 @@ def daemonGenerationStatus (workspaceId : WorkspaceId) (root : System.FilePath) (identity : DaemonIdentity) : IO DaemonGenerationStatus := do - match ← daemonProbe? endpoint workspaceId with - | none => pure .unavailable - | some probe => + match ← daemonProbe endpoint workspaceId with + | .error failure => + match failure with + | .transport _ => + if ← endpointAcceptsConnection endpoint then + pure <| .unrecognized failure.detail + else + pure .unavailable + | .invalidResponse _ | .streamCallback _ | .responseTimeout _ => + pure <| .unrecognized failure.detail + | .ok probe => unless ← Beam.sameFilePath (System.FilePath.mk probe.root) root do return .wrongRoot probe.root if probe.identity? == some identity then @@ -140,12 +173,4 @@ def daemonGenerationStatus else pure <| .wrongGeneration probe.root -def endpointAcceptsConnection (endpoint : Transport.Endpoint) : IO Bool := do - try - let conn ← Transport.connect endpoint - Transport.closeConnection conn - pure true - catch _ => - pure false - end Beam.Daemon diff --git a/Beam/Mcp/StdioServer.lean b/Beam/Mcp/StdioServer.lean index 353ff344..25304858 100644 --- a/Beam/Mcp/StdioServer.lean +++ b/Beam/Mcp/StdioServer.lean @@ -308,9 +308,7 @@ private def Coordinator.beginClosing private def awaitRequestDone (request : InFlightRequest) : IO Unit := do awaitPromise s!"in-flight request {request.id.label}" request.done -private def Coordinator.awaitRequests - (_coordinator : Coordinator) - (requests : Array InFlightRequest) : IO Unit := do +private def awaitRequests (requests : Array InFlightRequest) : IO Unit := do for request in requests do awaitRequestDone request @@ -323,7 +321,7 @@ private def Coordinator.otherAdmittedRequests private def Coordinator.closeTransport (coordinator : Coordinator) : IO Unit := do let requests ← coordinator.beginClosing - coordinator.awaitRequests requests + awaitRequests requests coordinator.state.closeRuntime private def Coordinator.admitToolRequest @@ -443,7 +441,7 @@ private def Coordinator.handleControlToolRequest try -- A control operation is a full stream-order fence: work admitted before it drains, -- while work admitted afterward waits on `done`. - coordinator.awaitRequests priorRequests + awaitRequests priorRequests coordinator.toolRequestResponse opts req admitted parsedParams request previous? initialProgress catch e => diff --git a/docs/TESTING.md b/docs/TESTING.md index 2efaf526..ae1e2e29 100644 --- a/docs/TESTING.md +++ b/docs/TESTING.md @@ -112,7 +112,8 @@ Current Beam coverage includes: - wrapper coverage through [tests/test-beam-wrapper.sh](../tests/test-beam-wrapper.sh), which aggregates focused probe, runtime, sync/save, handle, and diagnostic slices - focused daemon lifecycle coverage in [tests/test-beam-wrapper-daemon.sh](../tests/test-beam-wrapper-daemon.sh), including the no-implicit-start contract, duplicate-owner rejection, Beam and non-Beam endpoint - collision safety, cross-root stale-registry cleanup that preserves the daemon serving the other + collision safety, a bounded identity probe against a silent non-Beam listener, cross-root + stale-registry cleanup that preserves the daemon serving the other root, explicit shutdown, cancellation of requests active during shutdown or owner loss, exact-generation cleanup that preserves a replacement registry, registry removal before a paused daemon can finish draining, rejection of attachment to that unpublished draining generation, diff --git a/tests/lean/BeamTest/Broker/CliDaemonTest.lean b/tests/lean/BeamTest/Broker/CliDaemonTest.lean index 16bb2217..a08695bf 100644 --- a/tests/lean/BeamTest/Broker/CliDaemonTest.lean +++ b/tests/lean/BeamTest/Broker/CliDaemonTest.lean @@ -99,6 +99,17 @@ private def closeAcceptedConnection (listener : Beam.Broker.Transport.Listener) let conn ← Beam.Broker.Transport.accept listener Beam.Broker.Transport.closeConnection conn +private def holdAcceptedConnection + (listener : Beam.Broker.Transport.Listener) + (release : IO.Promise Unit) : IO Unit := do + let conn ← Beam.Broker.Transport.accept listener + try + let some _ ← IO.wait release.result? + | throw <| IO.userError "silent endpoint release promise dropped" + pure () + finally + Beam.Broker.Transport.closeConnection conn + private partial def withClosingBrokerEndpoint (act : Beam.Broker.Transport.Endpoint → IO α) (tries : Nat := 20) : IO α := do @@ -143,6 +154,29 @@ private partial def withBrokerListener else withBrokerListener act (tries - 1) +private def checkSilentEndpointProbeTimeout : IO Unit := do + withBrokerListener fun listener endpoint => do + let release ← IO.Promise.new + let serverTask ← IO.asTask (prio := Task.Priority.dedicated) <| + holdAcceptedConnection listener release + try + let identity : Beam.Broker.DaemonIdentity := { + daemonId := "silent-endpoint" + configHash := "silent-endpoint" + } + match ← Beam.Daemon.daemonGenerationStatus endpoint + Beam.Cli.projectDaemonWorkspaceId (System.FilePath.mk "/tmp") identity with + | .unrecognized detail => + require "silent endpoint should report its bounded response timeout" + (detail.contains "response timed out") + | status => + throw <| IO.userError s!"silent endpoint was classified as {repr status}" + finally + release.resolve () + match ← IO.wait serverTask with + | .ok () => pure () + | .error err => throw err + private def serveCancelablePlainRequest (listener : Beam.Broker.Transport.Listener) (requestObserved : IO.Promise Unit) : IO Unit := do @@ -1268,6 +1302,7 @@ def main : IO Unit := do checkDaemonFailureContext checkDaemonFailureUnreadableStartupLog checkTypedDaemonFailureClassification + checkSilentEndpointProbeTimeout checkPlainBrokerTaskCancellation checkBrokerConnectionClosedIncident checkDaemonFailureIncidentRetention diff --git a/tests/test-beam-wrapper-daemon.sh b/tests/test-beam-wrapper-daemon.sh index f3e34b7a..573be033 100644 --- a/tests/test-beam-wrapper-daemon.sh +++ b/tests/test-beam-wrapper-daemon.sh @@ -346,6 +346,56 @@ busy_pid="" rm -f -- "$busy_port_file" busy_port_file="" +busy_port_file="$(mktemp "$tmp2/silent-non-beam-port-XXXXXX")" +python3 - "$busy_port_file" <<'PY' & +import socketserver +import sys +import time + +class Handler(socketserver.BaseRequestHandler): + def handle(self): + time.sleep(10) + +with socketserver.TCPServer(("127.0.0.1", 0), Handler) as server: + with open(sys.argv[1], "w", encoding="utf-8") as stream: + print(server.server_address[1], file=stream, flush=True) + server.serve_forever() +PY +busy_pid="$!" +if ! wait_for_nonempty_file "$busy_port_file" "silent non-Beam occupied port"; then + exit 1 +fi +busy_port="$(cat "$busy_port_file")" +busy_out="$tmp2/silent-non-beam-port.out" +busy_err="$tmp2/silent-non-beam-port.err" +silent_probe_started="$SECONDS" +if "$beam_script" --root "$tmp2" --port "$busy_port" ensure --hold > "$busy_out" 2> "$busy_err"; then + echo "expected owner startup to reject a silent non-Beam service" >&2 + cat "$busy_out" >&2 + exit 1 +fi +silent_probe_elapsed="$((SECONDS - silent_probe_started))" +if [ "$silent_probe_elapsed" -ge 8 ]; then + echo "expected silent non-Beam probe to honor its response deadline, took ${silent_probe_elapsed}s" >&2 + cat "$busy_err" >&2 + exit 1 +fi +if ! grep -Fq "response timed out" "$busy_err"; then + echo "expected silent non-Beam collision to report the bounded probe timeout" >&2 + cat "$busy_err" >&2 + exit 1 +fi +if [ -e "$tmp2/.beam/beam-daemon.json" ]; then + echo "expected silent non-Beam collision not to publish a registry" >&2 + cat "$tmp2/.beam/beam-daemon.json" >&2 + exit 1 +fi +kill "$busy_pid" > /dev/null 2>&1 || true +wait "$busy_pid" 2>/dev/null || true +busy_pid="" +rm -f -- "$busy_port_file" +busy_port_file="" + start_slow_request "$tmp1" "shutdown-active" "shutdown-active" shutdown_json="$("$beam_script" --root "$tmp1" shutdown)" From 194f9b9a3e7221c55eeeb35e0fa7f3e65176d369 Mon Sep 17 00:00:00 2001 From: Emilio Jesus Gallego Arias Date: Wed, 26 Aug 2026 23:14:07 +0200 Subject: [PATCH 17/28] fix: bound daemon shutdown responses --- Beam/Broker/Client.lean | 8 +++ Beam/Cli/Commands.lean | 15 ++++-- Beam/Cli/DaemonManager.lean | 50 ++++++++++++------- Beam/Daemon/Protocol.lean | 6 +-- docs/DEVELOPMENT.md | 8 +-- tests/lean/BeamTest/Broker/CliDaemonTest.lean | 41 ++++++++++++--- 6 files changed, 93 insertions(+), 35 deletions(-) diff --git a/Beam/Broker/Client.lean b/Beam/Broker/Client.lean index 9083d725..70d7227e 100644 --- a/Beam/Broker/Client.lean +++ b/Beam/Broker/Client.lean @@ -32,6 +32,14 @@ def BrokerClientFailure.detail : BrokerClientFailure → String | .responseTimeout timeoutMs => s!"Beam daemon response timed out after {timeoutMs} ms" +instance : Repr BrokerClientFailure where + reprPrec failure _ := Std.Format.text <| + match failure with + | .transport error => s!"BrokerClientFailure.transport {error}" + | .invalidResponse detail => s!"BrokerClientFailure.invalidResponse {detail}" + | .streamCallback error => s!"BrokerClientFailure.streamCallback {error}" + | .responseTimeout timeoutMs => s!"BrokerClientFailure.responseTimeout {timeoutMs}" + private def BrokerClientFailure.toIOError : BrokerClientFailure → IO.Error | .transport error | .streamCallback error => error | .invalidResponse detail => IO.userError detail diff --git a/Beam/Cli/Commands.lean b/Beam/Cli/Commands.lean index 2058d4e8..963c8d6e 100644 --- a/Beam/Cli/Commands.lean +++ b/Beam/Cli/Commands.lean @@ -105,11 +105,16 @@ private def shutdownProjectDaemon (opts : CliOptions) : IO Unit := do match ← registryLiveFor root with | some entry => if let some endpoint := Beam.Daemon.registryEndpoint? entry then - let resp ← sendRequest endpoint { op := .shutdown } - printResponse resp - -- Unpublishing this exact generation releases its wrapper owner. The owner remains the - -- sole process responsible for closing the inherited pipe and reaping its daemon child. - removeRegistry root + let result ← requestDaemonShutdown endpoint + try + match result with + | .ok resp => printResponse resp + | .error failure => + throw <| IO.userError (← daemonFailureMessage root failure) + finally + -- Unpublishing this exact generation releases its wrapper owner. The owner remains the + -- sole process responsible for closing the inherited pipe and reaping its daemon child. + removeRegistryGeneration root entry.daemonId else stopRegisteredDaemon root printJsonLine <| Json.mkObj [ diff --git a/Beam/Cli/DaemonManager.lean b/Beam/Cli/DaemonManager.lean index d8b407b9..ee00c08c 100644 --- a/Beam/Cli/DaemonManager.lean +++ b/Beam/Cli/DaemonManager.lean @@ -83,6 +83,25 @@ def removeRegistry (root : System.FilePath) : IO Unit := do if ← path.pathExists then IO.FS.removeFile path +/-- Remove a registry entry only when it still names the observed daemon generation. -/ +def removeRegistryGeneration (root : System.FilePath) (daemonId : String) : IO Unit := do + match ← readRegistry? root with + | some current => + if current.daemonId == daemonId then + removeRegistry root + | none => pure () + +private def daemonShutdownResponseTimeoutMs : Nat := + 30000 + +/-- Ask a daemon to shut down without allowing its response stream to hold CLI control forever. -/ +def requestDaemonShutdown + (endpoint : Transport.Endpoint) + (responseTimeoutMs : Nat := daemonShutdownResponseTimeoutMs) : + IO (Except BrokerClientFailure Response) := do + sendRequestWithStreamTimeoutResult endpoint { op := .shutdown } + responseTimeoutMs (fun _ => pure ()) + private partial def waitForRecordedPidGone (recorded : Beam.RecordedPid) (tries : Nat := 20) : IO Unit := do @@ -115,22 +134,24 @@ def finishRegistryDaemonShutdown (entry : RegistryEntry) : IO Unit := do pure () private def stopDaemonEntry (entry : RegistryEntry) : IO Unit := do + let root := System.FilePath.mk entry.root + let releaseGeneration := removeRegistryGeneration root entry.daemonId + let releaseAndFinish := do + releaseGeneration + finishRegistryDaemonShutdown entry match registryEndpoint? entry with | none => - finishRegistryDaemonShutdown entry + releaseAndFinish | some endpoint => match ← daemonGenerationStatus endpoint projectDaemonWorkspaceId - (System.FilePath.mk entry.root) entry.identity with + root entry.identity with | .exact => - try - discard <| sendRequest endpoint { op := .shutdown } - catch _ => - pure () - finishRegistryDaemonShutdown entry + discard <| requestDaemonShutdown endpoint + releaseAndFinish | .unavailable => - finishRegistryDaemonShutdown entry + releaseAndFinish | .unrecognized _ | .wrongRoot _ | .wrongGeneration _ => - pure () + releaseGeneration def stopRegisteredDaemon (root : System.FilePath) : IO Unit := do match ← readRegistry? root with @@ -138,7 +159,6 @@ def stopRegisteredDaemon (root : System.FilePath) : IO Unit := do removeRegistry root | some entry => stopDaemonEntry entry - removeRegistry root private def requestedPortNat? (opts : CliOptions) : Option Nat := opts.requestedPort?.map (·.toNat) @@ -411,8 +431,8 @@ private partial def waitForDaemon message := endpointGenerationMismatchError endpoint (System.FilePath.mk daemonRoot) endpointInUse := true } - | .unrecognized detail => - retryOrFail (endpointProtocolError endpoint detail) + | .unrecognized failure => + retryOrFail (endpointProtocolError endpoint failure.detail) | .unavailable => retryOrFail "Beam daemon did not become ready before timeout" @@ -650,11 +670,7 @@ private partial def waitForOwnedDaemonExit private def removeOwnedRegistry (root : System.FilePath) (daemonId : String) : IO Unit := do try withProjectControlLock root do - match ← readRegistry? root with - | some current => - if current.daemonId == daemonId then - removeRegistry root - | none => pure () + removeRegistryGeneration root daemonId catch _ => pure () diff --git a/Beam/Daemon/Protocol.lean b/Beam/Daemon/Protocol.lean index bf4e6246..4a0a0da8 100644 --- a/Beam/Daemon/Protocol.lean +++ b/Beam/Daemon/Protocol.lean @@ -143,7 +143,7 @@ def endpointAcceptsConnection (endpoint : Transport.Endpoint) : IO Bool := do inductive DaemonGenerationStatus where | unavailable - | unrecognized (detail : String) + | unrecognized (failure : BrokerClientFailure) | wrongRoot (daemonRoot : String) | wrongGeneration (daemonRoot : String) | exact @@ -160,11 +160,11 @@ def daemonGenerationStatus match failure with | .transport _ => if ← endpointAcceptsConnection endpoint then - pure <| .unrecognized failure.detail + pure <| .unrecognized failure else pure .unavailable | .invalidResponse _ | .streamCallback _ | .responseTimeout _ => - pure <| .unrecognized failure.detail + pure <| .unrecognized failure | .ok probe => unless ← Beam.sameFilePath (System.FilePath.mk probe.root) root do return .wrongRoot probe.root diff --git a/docs/DEVELOPMENT.md b/docs/DEVELOPMENT.md index 7f1d3487..d6aa2ec4 100644 --- a/docs/DEVELOPMENT.md +++ b/docs/DEVELOPMENT.md @@ -396,9 +396,11 @@ admitted requests for cancellation, shuts down backend sessions, and stops the l no wrapper heartbeat, lease file, revocation tombstone, or retirement fence. Ordinary wrapper commands never start a daemon. Under the per-project control lock they require a - registry whose root and effective configuration match, whose owner is not known dead in the current - PID domain, and whose endpoint answers for the CLI's private workspace, canonical project root, and - exact daemon generation identity. +registry whose root and effective configuration match, whose owner is not known dead in the current +PID domain, and whose endpoint answers for the CLI's private workspace, canonical project root, and +exact daemon generation identity. Identity probes have a bounded response deadline. An endpoint +that accepts a connection but stays silent or returns malformed data is unrecognized, so validation +fails closed and PID fallback is not permitted. Endpoint/root validation is authoritative across PID namespaces because numeric PID observations from another domain are not safe process identity. A same-domain dead owner or a dead endpoint makes the registry stale; cleanup remains generation-scoped and PID fallback is permitted only through the diff --git a/tests/lean/BeamTest/Broker/CliDaemonTest.lean b/tests/lean/BeamTest/Broker/CliDaemonTest.lean index a08695bf..7d65cee9 100644 --- a/tests/lean/BeamTest/Broker/CliDaemonTest.lean +++ b/tests/lean/BeamTest/Broker/CliDaemonTest.lean @@ -145,14 +145,19 @@ private partial def withBrokerListener let stamp ← IO.monoNanosNow let portNat := 30000 + ((stamp + tries) % 20000) let endpoint := Beam.Broker.Transport.Endpoint.tcp portNat.toUInt16 - try - let listener ← Beam.Broker.Transport.bindAndListen endpoint 2 - act listener endpoint - catch err => + let listenerResult ← + try + pure <| Except.ok (← Beam.Broker.Transport.bindAndListen endpoint 2) + catch err => + pure <| Except.error err + match listenerResult with + | .error err => if tries == 0 then throw err else withBrokerListener act (tries - 1) + | .ok listener => + act listener endpoint private def checkSilentEndpointProbeTimeout : IO Unit := do withBrokerListener fun listener endpoint => do @@ -166,9 +171,11 @@ private def checkSilentEndpointProbeTimeout : IO Unit := do } match ← Beam.Daemon.daemonGenerationStatus endpoint Beam.Cli.projectDaemonWorkspaceId (System.FilePath.mk "/tmp") identity with - | .unrecognized detail => - require "silent endpoint should report its bounded response timeout" - (detail.contains "response timed out") + | .unrecognized (.responseTimeout timeoutMs) => + require "silent endpoint should preserve its typed response timeout" + (timeoutMs == 2000) + | .unrecognized failure => + throw <| IO.userError s!"silent endpoint reported {repr failure}" | status => throw <| IO.userError s!"silent endpoint was classified as {repr status}" finally @@ -177,6 +184,25 @@ private def checkSilentEndpointProbeTimeout : IO Unit := do | .ok () => pure () | .error err => throw err +private def checkSilentShutdownTimeout : IO Unit := do + withBrokerListener fun listener endpoint => do + let release ← IO.Promise.new + let serverTask ← IO.asTask (prio := Task.Priority.dedicated) <| + holdAcceptedConnection listener release + try + match ← Beam.Cli.requestDaemonShutdown endpoint 50 with + | .error (.responseTimeout timeoutMs) => + require "silent shutdown should preserve its typed response timeout" (timeoutMs == 50) + | .error failure => + throw <| IO.userError s!"silent shutdown reported {repr failure}" + | .ok response => + throw <| IO.userError s!"silent shutdown returned {toJson response}" + finally + release.resolve () + match ← IO.wait serverTask with + | .ok () => pure () + | .error err => throw err + private def serveCancelablePlainRequest (listener : Beam.Broker.Transport.Listener) (requestObserved : IO.Promise Unit) : IO Unit := do @@ -1303,6 +1329,7 @@ def main : IO Unit := do checkDaemonFailureUnreadableStartupLog checkTypedDaemonFailureClassification checkSilentEndpointProbeTimeout + checkSilentShutdownTimeout checkPlainBrokerTaskCancellation checkBrokerConnectionClosedIncident checkDaemonFailureIncidentRetention From c0a2d44cecf4a0e08df1c454921aac49b374a6c6 Mon Sep 17 00:00:00 2001 From: Emilio Jesus Gallego Arias Date: Thu, 27 Aug 2026 19:37:25 +0200 Subject: [PATCH 18/28] refactor: tighten daemon and MCP resource ownership --- Beam/Broker/Client.lean | 31 +- Beam/Broker/Server.lean | 335 +++++++++++------- Beam/Broker/Transport.lean | 11 +- Beam/Cli/Broker.lean | 153 ++++---- Beam/Cli/Commands.lean | 53 +-- Beam/Cli/DaemonManager.lean | 145 +++++--- Beam/Cli/Lock.lean | 46 +-- Beam/Daemon/Protocol.lean | 2 +- Beam/Mcp/Server.lean | 43 ++- Beam/Mcp/Stdio.lean | 15 +- Beam/Mcp/StdioServer.lean | 279 +++++++++------ Beam/System.lean | 19 - docs/DEVELOPMENT.md | 39 +- docs/MCP.md | 3 +- docs/STATUS.md | 11 +- docs/TESTING.md | 8 +- skills/lean-beam/agents/openai.yaml | 2 +- skills/rocq-beam/agents/openai.yaml | 2 +- tests/lean/BeamTest/Broker/CliDaemonTest.lean | 9 +- tests/lean/BeamTest/Broker/ProtocolTest.lean | 4 +- .../BeamTest/Broker/StartupHandshakeTest.lean | 18 + .../lean/BeamTest/Broker/StreamDedupTest.lean | 2 +- tests/test-mcp-stdio.py | 124 +++++-- 23 files changed, 830 insertions(+), 524 deletions(-) diff --git a/Beam/Broker/Client.lean b/Beam/Broker/Client.lean index 70d7227e..10b35062 100644 --- a/Beam/Broker/Client.lean +++ b/Beam/Broker/Client.lean @@ -19,15 +19,29 @@ structure StreamCallbacks where abbrev Endpoint := Transport.Endpoint +/-- The transport operation that produced a typed broker client failure. -/ +inductive BrokerTransportOperation where + | connect + | send + | receive + deriving Repr, BEq + +private def BrokerTransportOperation.label : BrokerTransportOperation → String + | .connect => "connect" + | .send => "send" + | .receive => "receive" + /-- Keep broker client failures typed until a CLI or transport presentation boundary. -/ inductive BrokerClientFailure where - | transport (error : IO.Error) + | transport (operation : BrokerTransportOperation) (error : IO.Error) | invalidResponse (detail : String) | streamCallback (error : IO.Error) | responseTimeout (timeoutMs : Nat) def BrokerClientFailure.detail : BrokerClientFailure → String - | .transport error | .streamCallback error => error.toString + | .transport operation error => + s!"Beam daemon {operation.label} failed: {error}" + | .streamCallback error => error.toString | .invalidResponse detail => detail | .responseTimeout timeoutMs => s!"Beam daemon response timed out after {timeoutMs} ms" @@ -35,13 +49,14 @@ def BrokerClientFailure.detail : BrokerClientFailure → String instance : Repr BrokerClientFailure where reprPrec failure _ := Std.Format.text <| match failure with - | .transport error => s!"BrokerClientFailure.transport {error}" + | .transport operation error => s!"BrokerClientFailure.transport {repr operation} {error}" | .invalidResponse detail => s!"BrokerClientFailure.invalidResponse {detail}" | .streamCallback error => s!"BrokerClientFailure.streamCallback {error}" | .responseTimeout timeoutMs => s!"BrokerClientFailure.responseTimeout {timeoutMs}" private def BrokerClientFailure.toIOError : BrokerClientFailure → IO.Error - | .transport error | .streamCallback error => error + | failure@(.transport ..) => IO.userError failure.detail + | .streamCallback error => error | .invalidResponse detail => IO.userError detail | .responseTimeout timeoutMs => IO.userError s!"Beam daemon response timed out after {timeoutMs} ms" @@ -113,11 +128,11 @@ private partial def sendRequestWithStreamResultCore (onStream : StreamMessage → IO Unit) (responseTimeoutMs? : Option Nat) : IO (Except BrokerClientFailure Response) := do let client ← - match ← captureClientFailure .transport (Transport.connect endpoint) with + match ← captureClientFailure (.transport .connect) (Transport.connect endpoint) with | .ok client => pure client | .error failure => return .error failure try - match ← captureClientFailure .transport <| + match ← captureClientFailure (.transport .send) <| Transport.sendMsg client (toJson req).compress with | .ok () => pure () | .error failure => return .error failure @@ -130,11 +145,11 @@ private partial def sendRequestWithStreamResultCore let msg ← match deadline? with | none => - match ← captureClientFailure .transport (Transport.recvMsg client) with + match ← captureClientFailure (.transport .receive) (Transport.recvMsg client) with | .ok msg => pure msg | .error failure => return .error failure | some deadline => - match ← captureClientFailure .transport <| + match ← captureClientFailure (.transport .receive) <| Transport.recvMsgUntil client deadline.deadlineNanos with | .ok (some msg) => pure msg | .ok none => return .error (.responseTimeout deadline.timeoutMs) diff --git a/Beam/Broker/Server.lean b/Beam/Broker/Server.lean index 9d8913ff..be8104f0 100644 --- a/Beam/Broker/Server.lean +++ b/Beam/Broker/Server.lean @@ -182,35 +182,30 @@ private partial def waitForProcessExitWithTimeout loop (remainingMs - min pollMs remainingMs) loop timeoutMs -private def shutdownSession (session : Session) : IO Unit := do - try - writeLspRequest session.stdin ({ id := 0, method := "shutdown", param := Json.null : Lean.JsonRpc.Request Json }) - let task ← IO.asTask (prio := Task.Priority.dedicated) session.stdout.readLspMessage - let _ ← waitForTaskWithTimeout task sessionShutdownReplyTimeoutMs - pure () - catch _ => - pure () - try - writeLspNotification session.stdin ({ method := "exit", param := Json.null : Lean.JsonRpc.Notification Json }) - catch _ => - pure () - unless ← waitForProcessExitWithTimeout session.proc sessionShutdownReplyTimeoutMs do +private def terminateBackendProcess (proc : IO.Process.Child brokerStdio) : IO Unit := do + let running ← try - session.proc.kill + pure (← proc.tryWait).isNone catch _ => - pure () + pure true + if running then try - if let some kill := ← killCommand? then - let _ ← IO.Process.output { - cmd := kill.toString - args := #["-9", toString session.proc.pid.toNat] - } - pure () + proc.kill catch _ => pure () - discard <| waitForProcessExitWithTimeout session.proc sessionShutdownReplyTimeoutMs + unless ← waitForProcessExitWithTimeout proc sessionShutdownReplyTimeoutMs do + try + if let some kill := ← killCommand? then + let _ ← IO.Process.output { + cmd := kill.toString + args := #["-9", toString proc.pid.toNat] + } + pure () + catch _ => + pure () + discard <| waitForProcessExitWithTimeout proc sessionShutdownReplyTimeoutMs try - discard <| session.proc.tryWait + discard <| proc.tryWait catch _ => pure () @@ -462,14 +457,7 @@ partial def sessionReaderLoop (session : Session) : IO Unit := do code := .workerExited message := e.toString } - try - session.proc.kill - catch _ => - pure () - try - discard <| session.proc.tryWait - catch _ => - pure () + terminateBackendProcess session.proc private def startRequestJsonTrackedDetailed (session : Session) @@ -518,6 +506,29 @@ private def startRequestJsonTrackedDetailed pure () throw e +private def shutdownSession (session : Session) : IO Unit := do + let session ← + try + let (session, pending) ← + startRequestJsonTrackedDetailed session "shutdown" Json.null + let task ← IO.asTask (prio := Task.Priority.dedicated) pending.awaitOutcome + if (← waitForTaskWithTimeout task sessionShutdownReplyTimeoutMs).isNone then + PendingRequestStore.failAll session.pending <| BrokerFailure.toResponseFailure { + code := .workerExited + message := "backend session shutdown timed out" + } + discard <| waitForTaskWithTimeout task sessionShutdownReplyTimeoutMs + pure session + catch _ => + pure session + try + writeLspNotification session.stdin + ({ method := "exit", param := Json.null : Lean.JsonRpc.Notification Json }) + catch _ => + pure () + unless ← waitForProcessExitWithTimeout session.proc sessionShutdownReplyTimeoutMs do + terminateBackendProcess session.proc + def sendRequestJsonTrackedDetailed (session : Session) (method : String) @@ -555,6 +566,76 @@ private partial def awaitInitializeResponse (stdout : IO.FS.Stream) : IO Unit := | .request .. => throw <| IO.userError "unexpected server request before initialize completed" +private def backendInitializeTimeoutMs : Nat := + 30000 + +/-- +Acquire a fully initialized backend session or terminate the provisional child before failing. + +The caller adopts the returned session into broker state. No child ownership escapes this function +until the initialization response and `initialized` notification have both completed. +-/ +private def acquireBackendSession + (workspaceId : WorkspaceId) + (backend : Backend) + (config : BrokerConfig) + (epoch : Nat) : IO Session := do + let root := config.root + let (cmd, args, env) ← backendCommand config backend + let proc ← IO.Process.spawn { + toStdioConfig := brokerStdio + cmd := cmd + args := args + env := env + cwd := root.toString + } + let (session, initializeTask) ← + try + let stdin := IO.FS.Stream.ofHandle proc.stdin + let stdout := IO.FS.Stream.ofHandle proc.stdout + let pending ← PendingRequestStore.create + let sessionToken ← mkSessionToken + let session : Session := { + workspaceId + backend + root + epoch + sessionToken + proc + stdin + stdout + pending + } + writeLspRequest stdin + ({ id := 0, method := "initialize", param := initializeParams backend root + : Lean.JsonRpc.Request Json }) + let initializeTask ← IO.asTask (prio := Task.Priority.dedicated) <| + awaitInitializeResponse stdout + pure (session, initializeTask) + catch err => + terminateBackendProcess proc + throw err + try + match ← waitForTaskWithTimeout initializeTask backendInitializeTimeoutMs with + | some (.ok ()) => pure () + | some (.error err) => throw err + | none => + throw <| IO.userError <| + s!"backend initialize timed out after {backendInitializeTimeoutMs} ms" + writeLspNotification session.stdin + ({ method := "initialized", param := Json.mkObj [] : Lean.JsonRpc.Notification Json }) + let _ ← IO.asTask (prio := Task.Priority.dedicated) do + try + sessionReaderLoop session + catch e => + IO.eprintln s!"broker session reader task failed: {e.toString}" + pure session + catch err => + IO.cancel initializeTask + terminateBackendProcess proc + discard <| waitForTaskWithTimeout initializeTask sessionShutdownReplyTimeoutMs + throw err + private def requireWorkspace (workspaceId : WorkspaceId) : M WorkspaceState := do let state ← get match getWorkspace? state workspaceId with @@ -568,7 +649,6 @@ private def ensureSession (workspaceId : WorkspaceId) (backend : Backend) : M Se | some workspace => pure workspace | none => throw <| IO.userError s!"unknown Beam workspace '{workspaceId}'" let config := workspace.config - let root := config.root let backendState := getBackendState workspace backend let (backendState, restart) ← match backendState.session? with | some session => @@ -587,37 +667,8 @@ private def ensureSession (workspaceId : WorkspaceId) (backend : Backend) : M Se | none => st pure session | none => - let (cmd, args, env) ← backendCommand config backend - let proc ← IO.Process.spawn { - toStdioConfig := brokerStdio - cmd := cmd - args := args - env := env - cwd := root.toString - } - let stdin := IO.FS.Stream.ofHandle proc.stdin - let stdout := IO.FS.Stream.ofHandle proc.stdout - let pending ← PendingRequestStore.create - let sessionToken ← mkSessionToken - let mut session : Session := { - workspaceId - backend - root - epoch := backendState.nextEpoch - sessionToken - proc - stdin - stdout - pending - } - writeLspRequest stdin ({ id := 0, method := "initialize", param := initializeParams backend root : Lean.JsonRpc.Request Json }) - awaitInitializeResponse stdout - writeLspNotification stdin ({ method := "initialized", param := Json.mkObj [] : Lean.JsonRpc.Notification Json }) - let _ ← IO.asTask (prio := Task.Priority.dedicated) do - try - sessionReaderLoop session - catch e => - IO.eprintln s!"broker session reader task failed: {e.toString}" + let session ← + acquireBackendSession workspaceId backend config backendState.nextEpoch recordSessionSpawn workspaceId backend restart let backendState := { backendState with session? := some session } modify fun st => @@ -885,9 +936,7 @@ private def modifyCurrentSessionIfMatching structure ServerRuntime where state : Std.Mutex State - endpoint : Transport.Endpoint daemonIdentity? : Option DaemonIdentity - stop : IO.Ref Bool activeRequests : ActiveRequestRegistry private closeMutex : Std.Mutex Bool private closeDone : IO.Promise (Except IO.Error Unit) @@ -929,7 +978,6 @@ private def ServerRuntime.statsResponse def ServerRuntime.create (config : BrokerConfig) (workspaceId : WorkspaceId) - (endpoint : Transport.Endpoint := .tcp 0) (daemonIdentity? : Option DaemonIdentity := none) : IO ServerRuntime := do unless validWorkspaceId workspaceId do throw <| IO.userError "workspace id must be non-empty" @@ -937,9 +985,7 @@ def ServerRuntime.create let state := mkInitialState config workspaceId startMonoNanos pure { state := ← Std.Mutex.new state - endpoint := endpoint daemonIdentity? - stop := ← IO.mkRef false activeRequests := ← ActiveRequestRegistry.create closeMutex := ← Std.Mutex.new false closeDone := ← IO.Promise.new @@ -960,11 +1006,7 @@ private def detachBackendSession private def collectSessions (left? right? : Option Session) : Array Session := - match left?, right? with - | none, none => #[] - | some left, none => #[left] - | none, some right => #[right] - | some left, some right => #[left, right] + #[left?, right?].filterMap id private def detachWorkspaceSessions (workspace : WorkspaceState) : WorkspaceState × Array Session := @@ -975,16 +1017,32 @@ private def detachWorkspaceSessions private def detachRuntimeSessions (server : ServerRuntime) : IO (Array Session) := do server.withState do let state ← get - let (state, sessions) := state.workspaces.toList.foldl (init := (state, #[])) fun + let (state, sessions) := state.workspaces.toList.foldl (init := (state, [])) fun (state, sessions) (workspaceId, workspace) => let (workspace, detached) := detachWorkspaceSessions workspace - (setWorkspace state workspaceId workspace, sessions ++ detached) + (setWorkspace state workspaceId workspace, detached.toList.reverse ++ sessions) set state - pure sessions + pure sessions.reverse.toArray + +private def recordFirstCleanupError + (firstError? : Option IO.Error) + (phase : IO Unit) : IO (Option IO.Error) := do + try + phase + pure firstError? + catch err => + pure (firstError? <|> some err) + +private def shutdownSessionsBestEffort : + List Session → Option IO.Error → IO Unit + | [], none => pure () + | [], some err => throw err + | session :: sessions, firstError? => do + let firstError? ← recordFirstCleanupError firstError? <| shutdownSession session + shutdownSessionsBestEffort sessions firstError? private def shutdownRuntimeSessions (server : ServerRuntime) : IO Unit := do - for session in ← detachRuntimeSessions server do - shutdownSession session + shutdownSessionsBestEffort (← detachRuntimeSessions server).toList none private def awaitRuntimeClose (promise : IO.Promise (Except IO.Error Unit)) : IO Unit := do @@ -1007,18 +1065,18 @@ def ServerRuntime.close (server : ServerRuntime) : IO Unit := do set true pure true if leadsClose then - let outcome ← - try - ActiveRequestRegistry.closeAdmission server.activeRequests - -- The first sweep unblocks requests already waiting on a backend. An admitted request may - -- have been between admission and session creation when closure began, so repeat the sweep - -- after every dispatch scope has drained to guarantee that no late session survives. - shutdownRuntimeSessions server - ActiveRequestRegistry.awaitDrained server.activeRequests - shutdownRuntimeSessions server - pure (.ok () : Except IO.Error Unit) - catch err => - pure (.error err) + -- Retain the first failure but run every teardown phase. In particular, a failed first session + -- sweep must not skip admission drain or the final sweep for sessions created during closure. + let firstError? ← recordFirstCleanupError none <| + ActiveRequestRegistry.closeAdmission server.activeRequests + let firstError? ← recordFirstCleanupError firstError? <| shutdownRuntimeSessions server + let firstError? ← recordFirstCleanupError firstError? <| + ActiveRequestRegistry.awaitDrained server.activeRequests + let firstError? ← recordFirstCleanupError firstError? <| shutdownRuntimeSessions server + let outcome := + match firstError? with + | none => .ok () + | some err => .error err server.closeDone.resolve outcome match outcome with | .ok () => pure () @@ -1230,20 +1288,32 @@ private def propagatePendingCancellation (cancelRef? : Option (IO.Ref Bool)) : IO Unit := do PendingRequestStore.propagateCancellation session.pending session.stdin cancelRef? -private def requestStop (server : ServerRuntime) : IO Unit := do - server.stop.set true +private structure DaemonTransport where + endpoint : Transport.Endpoint + listener : Transport.Listener + stop : IO.Ref Bool + +private def DaemonTransport.create (endpoint : Transport.Endpoint) : IO DaemonTransport := do + let stop ← IO.mkRef false + let listener ← Transport.bindAndListen endpoint 16 + pure { endpoint, listener, stop } + +private def requestStop (transport : DaemonTransport) : IO Unit := do + transport.stop.set true try -- Wake the blocking accept. Both ends are intentionally left to scope cleanup: performing a -- graceful TCP shutdown on the wake-up pair can wait for its peer and deadlock daemon exit. - discard <| Transport.connect server.endpoint + discard <| Transport.connect transport.endpoint catch _ => pure () -private def closeAndRequestStop (server : ServerRuntime) : IO Unit := do +private def closeAndRequestStop + (server : ServerRuntime) + (transport : DaemonTransport) : IO Unit := do try server.close finally - requestStop server + requestStop transport private structure WorkspaceRequest extends Request where workspaceId : WorkspaceId @@ -2450,8 +2520,11 @@ A standalone daemon cannot rely on its registry after the project directory disa default registry lives below that directory and is removed with it. Stop the broker proactively so removing a git worktree does not strand either the daemon or its backend processes. -/ -private partial def watchRoot (server : ServerRuntime) (root : System.FilePath) : IO Unit := do - if ← server.stop.get then +private partial def watchRoot + (server : ServerRuntime) + (transport : DaemonTransport) + (root : System.FilePath) : IO Unit := do + if ← transport.stop.get then pure () else let rootAvailable ← @@ -2461,18 +2534,20 @@ private partial def watchRoot (server : ServerRuntime) (root : System.FilePath) pure false if !rootAvailable then IO.eprintln s!"Beam daemon root is no longer available; shutting down: {root}" - closeAndRequestStop server + closeAndRequestStop server transport else IO.sleep rootWatchPollMs - watchRoot server root + watchRoot server transport root -private def watchSessionOwnerStdin (server : ServerRuntime) : IO Unit := do +private def watchSessionOwnerStdin + (server : ServerRuntime) + (transport : DaemonTransport) : IO Unit := do try discard <| (← IO.getStdin).readToEnd catch _ => pure () - unless ← server.stop.get do - closeAndRequestStop server + unless ← transport.stop.get do + closeAndRequestStop server transport private def watchClientDisconnect (client : Transport.Connection) @@ -2485,7 +2560,10 @@ private def watchClientDisconnect pure () discard <| handle.cancel -private def handleClient (server : ServerRuntime) (client : Transport.Connection) : IO Unit := do +private def handleClient + (server : ServerRuntime) + (transport : DaemonTransport) + (client : Transport.Connection) : IO Unit := do let clientRequestIdRef ← IO.mkRef (none : Option String) let terminalSentRef ← IO.mkRef false let sendResponse (clientRequestId? : Option String) (resp : Response) : IO Unit := do @@ -2532,7 +2610,7 @@ private def handleClient (server : ServerRuntime) (client : Transport.Connection try sendResponse req.clientRequestId? resp finally - requestStop server + requestStop transport else sendResponse req.clientRequestId? resp catch e => @@ -2546,20 +2624,22 @@ private def handleClient (server : ServerRuntime) (client : Transport.Connection finally Transport.closeConnection client -private partial def acceptLoop (server : ServerRuntime) (listener : Transport.Listener) : IO Unit := do - if ← server.stop.get then +private partial def acceptLoop + (server : ServerRuntime) + (transport : DaemonTransport) : IO Unit := do + if ← transport.stop.get then pure () else - let client ← Transport.accept listener - if ← server.stop.get then + let client ← Transport.accept transport.listener + if ← transport.stop.get then pure () else let _ ← IO.asTask (prio := Task.Priority.dedicated) do try - handleClient server client + handleClient server transport client catch e => IO.eprintln s!"broker client task failed: {e.toString}" - acceptLoop server listener + acceptLoop server transport private structure CliOptions where endpoint : Transport.Endpoint := .tcp 8765 @@ -2635,22 +2715,35 @@ def main (args : List String) : IO Unit := do leanPlugin? := leanPlugin? rocqCmd? := opts.rocqCmd? } - let listener ← Transport.bindAndListen opts.endpoint 16 - let runtime ← ServerRuntime.create config workspaceId opts.endpoint daemonIdentity? - let rootWatcher ← IO.asTask (prio := Task.Priority.dedicated) <| watchRoot runtime root + let runtime ← ServerRuntime.create config workspaceId daemonIdentity? + let transport ← DaemonTransport.create opts.endpoint + let rootWatcher ← IO.asTask (prio := Task.Priority.dedicated) <| + watchRoot runtime transport root let ownerWatcher? ← if opts.sessionOwnerStdin then - some <$> IO.asTask (prio := Task.Priority.dedicated) (watchSessionOwnerStdin runtime) + some <$> IO.asTask (prio := Task.Priority.dedicated) + (watchSessionOwnerStdin runtime transport) else pure none try - acceptLoop runtime listener + acceptLoop runtime transport finally - runtime.stop.set true - Transport.closeListener listener - if let some ownerWatcher := ownerWatcher? then - IO.cancel ownerWatcher - discard <| IO.wait rootWatcher - runtime.close + let firstError? ← recordFirstCleanupError none <| transport.stop.set true + let firstError? ← recordFirstCleanupError firstError? <| + Transport.closeListener transport.listener + let firstError? ← + match ownerWatcher? with + | none => pure firstError? + | some ownerWatcher => + recordFirstCleanupError firstError? do + try + IO.cancel ownerWatcher + finally + discard <| IO.wait ownerWatcher + let firstError? ← recordFirstCleanupError firstError? do + discard <| IO.wait rootWatcher + let firstError? ← recordFirstCleanupError firstError? runtime.close + if let some err := firstError? then + throw err end Beam.Broker diff --git a/Beam/Broker/Transport.lean b/Beam/Broker/Transport.lean index 646a87dd..5ddfa3bc 100644 --- a/Beam/Broker/Transport.lean +++ b/Beam/Broker/Transport.lean @@ -39,7 +39,7 @@ private def waitTcpPromise (promise : IO.Promise (Except IO.Error α)) (failureM | throw <| IO.userError failureMessage match result with | .ok value => pure value - | .error err => throw <| IO.userError s!"{failureMessage}: {err}" + | .error err => throw err private inductive ReceiveWaitResult (α : Type) where | completed (value : α) @@ -58,7 +58,7 @@ private partial def waitTcpReceivePromiseUntil | throw <| IO.userError failureMessage match result with | .ok value => pure <| .completed value - | .error err => throw <| IO.userError s!"{failureMessage}: {err}" + | .error err => throw err else if (← IO.monoNanosNow) >= deadlineNanos then -- The caller abandons and closes this connection after timeout. Cancel the exact pending UV -- receive first so no read remains attached to the socket. @@ -103,8 +103,11 @@ def closeConnection (conn : Connection) : IO Unit := do def closeListener (listener : Listener) : IO Unit := do match listener with - | .tcp _ => - pure () + | .tcp server => + try + TCP.Socket.cancelAccept server + catch _ => + pure () private def sendMsgTcp (client : TCP.Socket) (msg : String) : IO Unit := do let bytes := msg.toUTF8 diff --git a/Beam/Cli/Broker.lean b/Beam/Cli/Broker.lean index cd533c32..659a939d 100644 --- a/Beam/Cli/Broker.lean +++ b/Beam/Cli/Broker.lean @@ -27,7 +27,7 @@ def inProjectDaemonWorkspace (req : Request) : Request := if req.workspaceId?.isSome then req else { req with workspaceId? := some projectDaemonWorkspaceId } -def withBrokerErrorContext +private def withBrokerErrorContext {α} (root : System.FilePath) (action : IO (Except BrokerClientFailure α)) : IO α := do @@ -45,17 +45,41 @@ structure BrokerWaitSpec where failureBoundary : String := "before the request completed" responseNote? : Response → Option String := fun _ => none -private structure InterruptWatcher where - signal : Std.Internal.UV.Signal - task : Task (Except IO.Error Unit) +structure InterruptWatcher where + interrupted : IO Bool + awaitInterrupt : IO Unit -private def InterruptWatcher.stop (watcher : InterruptWatcher) : IO Unit := - Std.Internal.UV.Signal.stop watcher.signal +private def closeInterruptSignal (signal : Std.Internal.UV.Signal) : IO Unit := do + -- `Signal.stop` alone leaves an unresolved `next` promise and its waiter leaked. Cancel the + -- pending wait first, then stop the underlying signal resource. + try + Std.Internal.UV.Signal.cancel signal + finally + Std.Internal.UV.Signal.stop signal -private def InterruptWatcher.interrupted (watcher : InterruptWatcher) : IO Bool := - IO.hasFinished watcher.task +/-- Acquire one non-repeating SIGINT watcher and release its pending wait on every exit path. -/ +def withInterruptWatcher (act : InterruptWatcher → IO α) : IO α := do + let signal ← Std.Internal.UV.Signal.mk 2 false + let promise ← + try + Std.Internal.UV.Signal.next signal + catch err => + Std.Internal.UV.Signal.stop signal + throw err + let event := promise.result? + let watcher : InterruptWatcher := { + interrupted := IO.hasFinished event + awaitInterrupt := do + let some _ ← IO.wait event + | throw <| IO.userError "SIGINT watcher promise dropped" + pure () + } + try + act watcher + finally + closeInterruptSignal signal -def progressEnabled : IO Bool := do +private def progressEnabled : IO Bool := do match ← envFlag? "BEAM_PROGRESS" with | some enabled => pure enabled @@ -64,6 +88,7 @@ def progressEnabled : IO Bool := do private structure WrapperBrokerRequest where request : Request + clientRequestId : String visibleClientRequestId? : Option String private def mkWrapperClientRequestId (req : Request) : IO String := do @@ -77,12 +102,14 @@ private def withWrapperClientRequestId (req : Request) : IO WrapperBrokerRequest | some clientRequestId => pure { request := req + clientRequestId visibleClientRequestId? := some clientRequestId } | none => let clientRequestId ← mkWrapperClientRequestId req pure { request := { req with clientRequestId? := some clientRequestId } + clientRequestId visibleClientRequestId? := none } @@ -90,28 +117,16 @@ private def prepareWrapperBrokerRequest (req : Request) : IO WrapperBrokerRequest := withWrapperClientRequestId <| inProjectDaemonWorkspace req -private def mkInterruptWatcher? (clientRequestId? : Option String) : IO (Option InterruptWatcher) := do - match clientRequestId? with - | none => pure none - | some _ => - let signal ← Std.Internal.UV.Signal.mk 2 false - let promise ← Std.Internal.UV.Signal.next signal - let task ← IO.asTask (prio := Task.Priority.dedicated) do - let some _ ← IO.wait promise.result? - | throw <| IO.userError "SIGINT watcher promise dropped" - pure () - pure <| some { signal, task } - def decodeCancelAcknowledged? (resp : Response) : Option Bool := do let result ← resp.result? result.getObjValAs? Bool "cancelled" |>.toOption private def sendBrokerCancellation (endpoint : Transport.Endpoint) - (req : Request) : IO (Option Bool) := do + (clientRequestId : String) : IO (Option Bool) := do let cancelReq : Request := { op := .cancel - cancelRequestId? := req.clientRequestId? + cancelRequestId? := some clientRequestId } try let resp ← sendRequest endpoint (← withEnvClientRequestId cancelReq) @@ -122,74 +137,60 @@ private def sendBrokerCancellation private def awaitBrokerResponse (task : Task (Except IO.Error (Except BrokerClientFailure Response))) (endpoint : Transport.Endpoint) - (req : Request) + (clientRequestId : String) (visibleClientRequestId? : Option String) (progressSpec? : Option BrokerWaitSpec) - (interruptWatcher? : Option InterruptWatcher) : IO (Except BrokerClientFailure Response) := do + (interruptWatcher : InterruptWatcher) : IO (Except BrokerClientFailure Response) := do let mut interruptObserved := false let mut cancelAcknowledged := false let emit := fun msg => IO.eprintln <| annotateRunatMessage visibleClientRequestId? msg if let some spec := progressSpec? then emit spec.startMsg let mut waitedMs := 0 - try - while !(← IO.hasFinished task) do - let signalInterrupted ← - match interruptWatcher? with - | some watcher => watcher.interrupted - | none => pure false - if signalInterrupted || (← IO.checkCanceled) then - if !interruptObserved then - interruptObserved := true - emit "beam: requesting broker cancellation" - if !cancelAcknowledged then - -- SIGINT can arrive after the wrapper starts the request task but before the broker - -- has registered the client request id as active. Retry until the broker acknowledges - -- cancellation or the original request finishes. - match ← sendBrokerCancellation endpoint req with - | some true => cancelAcknowledged := true - | some false | none => pure () - IO.sleep 500 - if !(← IO.hasFinished task) then - waitedMs := waitedMs + 500 - if waitedMs % 1000 == 0 then - if let some spec := progressSpec? then - emit <| spec.stillWaitingMsg (waitedMs / 1000) - let result ← - match (← IO.wait task) with - | .ok result => pure result - | .error err => throw err - match result with - | .ok response => + while !(← IO.hasFinished task) do + let signalInterrupted ← interruptWatcher.interrupted + if signalInterrupted || (← IO.checkCanceled) then + if !interruptObserved then + interruptObserved := true + emit "beam: requesting broker cancellation" + if !cancelAcknowledged then + -- SIGINT can arrive after the wrapper starts the request task but before the broker + -- has registered the client request id as active. Retry until the broker acknowledges + -- cancellation or the original request finishes. + match ← sendBrokerCancellation endpoint clientRequestId with + | some true => cancelAcknowledged := true + | some false | none => pure () + IO.sleep 500 + if !(← IO.hasFinished task) then + waitedMs := waitedMs + 500 + if waitedMs % 1000 == 0 then if let some spec := progressSpec? then - emit <| spec.completeMsg response - pure <| .ok response - | .error failure => - pure <| .error failure - finally - match interruptWatcher? with - | some watcher => watcher.stop - | none => pure () + emit <| spec.stillWaitingMsg (waitedMs / 1000) + let result ← + match (← IO.wait task) with + | .ok result => pure result + | .error err => throw err + match result with + | .ok response => + if let some spec := progressSpec? then + emit <| spec.completeMsg response + pure <| .ok response + | .error failure => + pure <| .error failure private def awaitBrokerResponseWithInterrupts (endpoint : Transport.Endpoint) - (req : Request) + (clientRequestId : String) (visibleClientRequestId? : Option String) (progressSpec? : Option BrokerWaitSpec) (action : IO (Except BrokerClientFailure Response)) : IO (Except BrokerClientFailure Response) := do -- Wrapper calls synthesize a broker clientRequestId when the user did not provide one. That id -- gives SIGINT cancellation a stable broker key but is kept out of the CLI's public output. - let interruptWatcher? ← mkInterruptWatcher? req.clientRequestId? - let task ← - try - IO.asTask (prio := Task.Priority.dedicated) action - catch e => - match interruptWatcher? with - | some watcher => watcher.stop - | none => pure () - throw e - awaitBrokerResponse task endpoint req visibleClientRequestId? progressSpec? interruptWatcher? + withInterruptWatcher fun interruptWatcher => do + let task ← IO.asTask (prio := Task.Priority.dedicated) action + awaitBrokerResponse task endpoint clientRequestId visibleClientRequestId? progressSpec? + interruptWatcher private structure WrapperBrokerResponse where response : Response @@ -202,7 +203,7 @@ private def requestBrokerResponse let wrapperReq ← prepareWrapperBrokerRequest req let req := wrapperReq.request let response ← withBrokerErrorContext root do - awaitBrokerResponseWithInterrupts client.endpoint req + awaitBrokerResponseWithInterrupts client.endpoint wrapperReq.clientRequestId wrapperReq.visibleClientRequestId? none <| sendRequestWithCallbacksResult client.endpoint req pure { response, visibleClientRequestId? := wrapperReq.visibleClientRequestId? } @@ -442,8 +443,8 @@ def callBrokerWithProgress } let progressSpec? := if showProgress then some spec else none let resp ← withBrokerErrorContext root do - awaitBrokerResponseWithInterrupts client.endpoint req visibleClientRequestId? - progressSpec? <| + awaitBrokerResponseWithInterrupts client.endpoint wrapperReq.clientRequestId + visibleClientRequestId? progressSpec? <| sendRequestWithCallbacksResult client.endpoint req callbacks match responseErrorSummary? spec.action spec.failureBoundary resp with | some note => diff --git a/Beam/Cli/Commands.lean b/Beam/Cli/Commands.lean index 963c8d6e..5fa36e8d 100644 --- a/Beam/Cli/Commands.lean +++ b/Beam/Cli/Commands.lean @@ -15,7 +15,6 @@ import Beam.Cli.LeanOperation import Beam.Cli.Project import Beam.Cli.RuntimeBundle import Beam.Cli.Usage -import Std.Internal.UV.Signal open Lean @@ -101,30 +100,14 @@ private def runLeanRelease private def shutdownProjectDaemon (opts : CliOptions) : IO Unit := do let root ← projectRootAny opts - withProjectControlLock root do - match ← registryLiveFor root with - | some entry => - if let some endpoint := Beam.Daemon.registryEndpoint? entry then - let result ← requestDaemonShutdown endpoint - try - match result with - | .ok resp => printResponse resp - | .error failure => - throw <| IO.userError (← daemonFailureMessage root failure) - finally - -- Unpublishing this exact generation releases its wrapper owner. The owner remains the - -- sole process responsible for closing the inherited pipe and reaping its daemon child. - removeRegistryGeneration root entry.daemonId - else - stopRegisteredDaemon root - printJsonLine <| Json.mkObj [ - ("result", Json.mkObj [("shutdown", toJson false), ("reason", toJson ("notFound" : String))]) - ] - | none => - stopRegisteredDaemon root - printJsonLine <| Json.mkObj [ - ("result", Json.mkObj [("shutdown", toJson false), ("reason", toJson ("notFound" : String))]) - ] + match ← shutdownRegisteredProjectDaemon root with + | .ok (some resp) => printResponse resp + | .ok none => + printJsonLine <| Json.mkObj [ + ("result", Json.mkObj [("shutdown", toJson false), ("reason", toJson ("notFound" : String))]) + ] + | .error failure => + throw <| IO.userError (← daemonFailureMessage root failure) private def parseBackendName (name : String) : IO Backend := do match fromJson? (Json.str name) with @@ -142,27 +125,17 @@ private def validateRequestedPortScope (opts : CliOptions) : IO Unit := do private def runThenHoldUntilInterrupted (owner : ProjectDaemonOwner) - (act : IO Unit) : IO Unit := do - let signal ← Std.Internal.UV.Signal.mk 2 false - let promise ← Std.Internal.UV.Signal.next signal - let task ← IO.asTask (prio := Task.Priority.dedicated) do - let some _ ← IO.wait promise.result? - | throw <| IO.userError "SIGINT watcher promise dropped" - pure () - try + (act : IO Unit) : IO Unit := + withInterruptWatcher fun watcher => do act - while !(← IO.hasFinished task) && (← owner.exitCode?).isNone && + while !(← watcher.interrupted) && (← owner.exitCode?).isNone && (← owner.registered) && !(← IO.checkCanceled) do IO.sleep 50 - if ← IO.hasFinished task then - match ← IO.wait task with - | .ok () => pure () - | .error err => throw err + if ← watcher.interrupted then + watcher.awaitInterrupt else if let some exitCode ← owner.exitCode? then unless exitCode == 0 do throw <| IO.userError s!"owned Beam daemon exited with status {exitCode}" - finally - Std.Internal.UV.Signal.stop signal private def ensureBackend (home : System.FilePath) diff --git a/Beam/Cli/DaemonManager.lean b/Beam/Cli/DaemonManager.lean index ee00c08c..f1d7565d 100644 --- a/Beam/Cli/DaemonManager.lean +++ b/Beam/Cli/DaemonManager.lean @@ -50,8 +50,15 @@ Project control operations should fail with owner diagnostics instead of waiting live but stuck wrapper process. Longer bundle build locks intentionally use the lower-level unbounded lock helper. -/ -def withProjectControlLock (root : System.FilePath) (act : IO α) : IO α := do - withLockTimeout (← projectControlLockDir root) (← projectControlLockTimeoutMs) act +private structure ProjectControl where + root : System.FilePath + +/-- Supply project registry mutation only for the dynamic extent of the project control lock. -/ +private def withProjectControl + (root : System.FilePath) + (act : ProjectControl → IO α) : IO α := do + withLockTimeout (← projectControlLockDir root) (← projectControlLockTimeoutMs) do + act { root } private def computeConfigHash (root : System.FilePath) @@ -70,25 +77,25 @@ private def computeConfigHash acc := mixField acc bundleId s!"{acc.toNat}" -private def writeRegistry (root : System.FilePath) (entry : RegistryEntry) : IO Unit := do - let path ← registryPath root +private def writeRegistry (control : ProjectControl) (entry : RegistryEntry) : IO Unit := do + let path ← registryPath control.root if let some parent := path.parent then IO.FS.createDirAll parent let tmp := path.withExtension "tmp" IO.FS.writeFile tmp ((toJson entry).pretty ++ "\n") IO.FS.rename tmp path -def removeRegistry (root : System.FilePath) : IO Unit := do - let path ← registryPath root +private def removeRegistry (control : ProjectControl) : IO Unit := do + let path ← registryPath control.root if ← path.pathExists then IO.FS.removeFile path /-- Remove a registry entry only when it still names the observed daemon generation. -/ -def removeRegistryGeneration (root : System.FilePath) (daemonId : String) : IO Unit := do - match ← readRegistry? root with +private def removeRegistryGeneration (control : ProjectControl) (daemonId : String) : IO Unit := do + match ← readRegistry? control.root with | some current => if current.daemonId == daemonId then - removeRegistry root + removeRegistry control | none => pure () private def daemonShutdownResponseTimeoutMs : Nat := @@ -133,32 +140,35 @@ def finishRegistryDaemonShutdown (entry : RegistryEntry) : IO Unit := do | .invalid | .local false | .differentDomain | .unknownDomain => pure () -private def stopDaemonEntry (entry : RegistryEntry) : IO Unit := do - let root := System.FilePath.mk entry.root - let releaseGeneration := removeRegistryGeneration root entry.daemonId +private def stopDaemonEntry (control : ProjectControl) (entry : RegistryEntry) : IO Unit := do + let releaseGeneration := removeRegistryGeneration control entry.daemonId let releaseAndFinish := do - releaseGeneration - finishRegistryDaemonShutdown entry + try + releaseGeneration + finally + finishRegistryDaemonShutdown entry match registryEndpoint? entry with | none => releaseAndFinish | some endpoint => match ← daemonGenerationStatus endpoint projectDaemonWorkspaceId - root entry.identity with + control.root entry.identity with | .exact => - discard <| requestDaemonShutdown endpoint - releaseAndFinish + try + discard <| requestDaemonShutdown endpoint + finally + releaseAndFinish | .unavailable => releaseAndFinish | .unrecognized _ | .wrongRoot _ | .wrongGeneration _ => releaseGeneration -def stopRegisteredDaemon (root : System.FilePath) : IO Unit := do - match ← readRegistry? root with +private def stopRegisteredDaemon (control : ProjectControl) : IO Unit := do + match ← readRegistry? control.root with | none => - removeRegistry root + removeRegistry control | some entry => - stopDaemonEntry entry + stopDaemonEntry control entry private def requestedPortNat? (opts : CliOptions) : Option Nat := opts.requestedPort?.map (·.toNat) @@ -197,14 +207,14 @@ private partial def selectUnoccupiedEndpoint | .error failure => let occupied ← match failure with - | .transport _ => endpointAcceptsConnection endpoint + | .transport _ _ => endpointAcceptsConnection endpoint | .invalidResponse _ | .streamCallback _ | .responseTimeout _ => pure true if !occupied then pure endpoint else let message := match failure with - | .transport _ => endpointInUseError endpoint + | .transport _ _ => endpointInUseError endpoint | .invalidResponse _ | .streamCallback _ | .responseTimeout _ => endpointProtocolError endpoint failure.detail retryOrReject message @@ -245,7 +255,7 @@ private def daemonFailureIncidentSchemaVersion : Nat := 1 private def daemonFailureIncidentKind? : BrokerClientFailure → Option String - | .transport _ => some "brokerTransportFailure" + | .transport _ _ => some "brokerTransportFailure" | .invalidResponse _ => some "invalidBrokerResponse" | .streamCallback _ => none | .responseTimeout _ => some "brokerResponseTimeout" @@ -404,21 +414,30 @@ private def startDaemon } pure child -private partial def waitForDaemon +private def daemonStartupTimeoutMs : Nat := + 30000 + +private partial def waitForDaemonUntil (child : IO.Process.Child daemonStdio) (endpoint : Transport.Endpoint) (logPath : System.FilePath) (root : System.FilePath) (identity : DaemonIdentity) - (tries : Nat := 300) : IO (Except DaemonStartupFailure Unit) := do + (deadlineNanos : Nat) + (timeoutDetail : String) : IO (Except DaemonStartupFailure Unit) := do + if (← child.tryWait).isSome then + return .error (← daemonStartupFailure endpoint logPath + "Beam daemon process exited before responding") + if (← IO.monoNanosNow) >= deadlineNanos then + return .error (← daemonStartupFailure endpoint logPath timeoutDetail) let retryOrFail (detail : String) : IO (Except DaemonStartupFailure Unit) := do if (← child.tryWait).isSome then .error <$> daemonStartupFailure endpoint logPath "Beam daemon process exited before responding" - else if tries == 0 then + else if (← IO.monoNanosNow) >= deadlineNanos then .error <$> daemonStartupFailure endpoint logPath detail else IO.sleep 100 - waitForDaemon child endpoint logPath root identity (tries - 1) + waitForDaemonUntil child endpoint logPath root identity deadlineNanos detail match ← daemonGenerationStatus endpoint projectDaemonWorkspaceId root identity with | .exact => pure (.ok ()) | .wrongRoot daemonRoot => @@ -436,6 +455,16 @@ private partial def waitForDaemon | .unavailable => retryOrFail "Beam daemon did not become ready before timeout" +private def waitForDaemon + (child : IO.Process.Child daemonStdio) + (endpoint : Transport.Endpoint) + (logPath : System.FilePath) + (root : System.FilePath) + (identity : DaemonIdentity) : IO (Except DaemonStartupFailure Unit) := do + let deadlineNanos := (← IO.monoNanosNow) + daemonStartupTimeoutMs * 1000000 + waitForDaemonUntil child endpoint logPath root identity deadlineNanos + "Beam daemon did not become ready before timeout" + private def newDaemonGenerationId (configHash : String) : IO String := do let startedMonoNanos ← IO.monoNanosNow let nonce := ByteArray.toUInt64LE! (← IO.getRandomBytes 8) @@ -481,8 +510,20 @@ private partial def startDaemonEntry let daemonId ← newDaemonGenerationId desired.configHash let identity : DaemonIdentity := { daemonId, configHash := desired.configHash } let child ← startDaemon desired endpoint logPath identity - match ← waitForDaemon child endpoint logPath desired.root identity with - | .ok () => pure () + let readiness : Except DaemonStartupFailure RegistryEntry ← + try + match ← waitForDaemon child endpoint logPath desired.root identity with + | .ok () => + let entry ← registryEntryFor desired daemonId child.pid.toNat endpoint opts + pure (.ok entry) + | .error failure => + pure (.error failure) + catch err => + terminateDaemonChild child + throw err + match readiness with + | .ok entry => + pure (endpoint, entry, child) | .error failure => terminateDaemonChild child let endpointOccupied ← endpointAcceptsConnection endpoint @@ -490,9 +531,6 @@ private partial def startDaemonEntry (usesAutomaticTcpEndpoint opts) tries endpointOccupied failure.endpointInUse then return ← startDaemonEntry desired opts (tries - 1) throw <| IO.userError failure.message - let pid := child.pid.toNat - let entry ← registryEntryFor desired daemonId pid endpoint opts - pure (endpoint, entry, child) def desiredConfig (home root : System.FilePath) (required : Backend) : IO DesiredConfig := do let defaultPaths ← defaultBundlePaths home @@ -584,6 +622,30 @@ def registryLiveFor | .exact => pure (some entry) | .unavailable | .unrecognized _ | .wrongRoot _ | .wrongGeneration _ => pure none +/-- +Shut down the live wrapper daemon, if any, while holding its project control scope. + +The generation is unpublished even when the bounded response fails, releasing the foreground owner +without exposing registry mutation or its lock precondition to callers. +-/ +def shutdownRegisteredProjectDaemon + (root : System.FilePath) : IO (Except BrokerClientFailure (Option Response)) := do + withProjectControl root fun control => do + match ← registryLiveFor root with + | some entry => + match registryEndpoint? entry with + | some endpoint => + try + pure <| (← requestDaemonShutdown endpoint).map some + finally + removeRegistryGeneration control entry.daemonId + | none => + stopRegisteredDaemon control + pure (.ok none) + | none => + stopRegisteredDaemon control + pure (.ok none) + private abbrev detachedDaemonStdio : IO.Process.StdioConfig where stdin := .null stdout := .null @@ -629,16 +691,17 @@ private def missingOwnerMessage (root : System.FilePath) (backend? : Option Back s!"start '{missingOwnerCommand backend?}' for this project and keep it running while using wrapper commands" private def startOwnedProjectDaemon + (control : ProjectControl) (desired : DesiredConfig) (opts : CliOptions) : IO OwnedProjectDaemon := do if let some live ← registryLiveFor desired.root then throw <| IO.userError (activeOwnerMessage desired.root live) -- A non-live registry may refer to a daemon still winding down after owner loss. Ask that exact -- root-matching endpoint to stop, and use PID fallback only through the typed domain boundary. - stopRegisteredDaemon desired.root + stopRegisteredDaemon control let (endpoint, entry, child) ← startDaemonEntry desired opts try - writeRegistry desired.root entry + writeRegistry control entry catch err => terminateDaemonChild child throw err @@ -669,8 +732,8 @@ private partial def waitForOwnedDaemonExit private def removeOwnedRegistry (root : System.FilePath) (daemonId : String) : IO Unit := do try - withProjectControlLock root do - removeRegistryGeneration root daemonId + withProjectControl root fun control => + removeRegistryGeneration control daemonId catch _ => pure () @@ -710,8 +773,8 @@ def withProjectDaemonOwner (opts : CliOptions) (act : ProjectDaemonOwner → IO α) : IO α := do let desired ← desiredConfig home root backend - let owned ← withProjectControlLock root do - startOwnedProjectDaemon desired opts + let owned ← withProjectControl root fun control => + startOwnedProjectDaemon control desired opts let exitCodeRef ← IO.mkRef (none : Option UInt32) try act { @@ -728,11 +791,11 @@ private def lookupProjectDaemon (root : System.FilePath) (expectedHash? : Option String := none) (backend? : Option Backend := none) : IO ProjectDaemonClient := do - withProjectControlLock root do + withProjectControl root fun control => do match ← registryLiveFor root expectedHash? with | some entry => projectDaemonClient entry | none => - stopRegisteredDaemon root + stopRegisteredDaemon control throw <| IO.userError (missingOwnerMessage root backend?) def withProjectDaemon diff --git a/Beam/Cli/Lock.lean b/Beam/Cli/Lock.lean index 5592cc91..e22becb5 100644 --- a/Beam/Cli/Lock.lean +++ b/Beam/Cli/Lock.lean @@ -15,6 +15,11 @@ namespace Beam.Cli private def lockPollMs : Nat := 100 +private structure LockDeadline where + timeoutMs : Nat + startedNanos : Nat + deadlineNanos : Nat + private structure LockOwner where pid : Nat pidDomain? : Option String @@ -61,8 +66,7 @@ private def removeStaleLock? (lockDir : System.FilePath) (owner? : Option LockOw private partial def acquireLockCore (lockDir : System.FilePath) - (timeoutMs? : Option Nat) - (waitedMs : Nat := 0) : IO Unit := do + (deadline? : Option LockDeadline) : IO Unit := do if let some parent := lockDir.parent then IO.FS.createDirAll parent let selfPid ← IO.Process.getPID @@ -93,34 +97,36 @@ private partial def acquireLockCore else let owner? ← readLockOwner? lockDir if ← removeStaleLock? lockDir owner? then - acquireLockCore lockDir timeoutMs? waitedMs + acquireLockCore lockDir deadline? else - match timeoutMs? with - | some timeoutMs => - if waitedMs >= timeoutMs then - throw <| IO.userError (lockTimeoutMessage lockDir owner? waitedMs timeoutMs) + match deadline? with + | some deadline => + let now ← IO.monoNanosNow + if now >= deadline.deadlineNanos then + let waitedMs := (now - deadline.startedNanos) / 1000000 + throw <| IO.userError <| + lockTimeoutMessage lockDir owner? waitedMs deadline.timeoutMs | none => pure () IO.sleep lockPollMs.toUInt32 - acquireLockCore lockDir timeoutMs? (waitedMs + lockPollMs) + acquireLockCore lockDir deadline? -def acquireLock (lockDir : System.FilePath) : IO Unit := +private def acquireLock (lockDir : System.FilePath) : IO Unit := acquireLockCore lockDir none -/-- -Acquire a directory lock, but fail with lock owner diagnostics after `timeoutMs`. - -The unbounded `acquireLock` remains available for long-running build/install locks. This bounded -variant is for short project-control critical sections where silent infinite waiting hides daemon -or wrapper failures. --/ -def acquireLockTimeout (lockDir : System.FilePath) (timeoutMs : Nat) : IO Unit := - acquireLockCore lockDir (some timeoutMs) +private def acquireLockTimeout (lockDir : System.FilePath) (timeoutMs : Nat) : IO Unit := do + let startedNanos ← IO.monoNanosNow + acquireLockCore lockDir <| some { + timeoutMs + startedNanos + deadlineNanos := startedNanos + timeoutMs * 1000000 + } -def releaseLock (lockDir : System.FilePath) : IO Unit := do +private def releaseLock (lockDir : System.FilePath) : IO Unit := do if ← lockDir.pathExists then IO.FS.removeDirAll lockDir +/-- Run `act` while holding an unbounded directory lock. -/ def withLock (lockDir : System.FilePath) (act : IO α) : IO α := do acquireLock lockDir try @@ -128,7 +134,7 @@ def withLock (lockDir : System.FilePath) (act : IO α) : IO α := do finally releaseLock lockDir -/-- Run `act` while holding a bounded directory lock. -/ +/-- Run `act` while holding a directory lock until an absolute monotonic deadline. -/ def withLockTimeout (lockDir : System.FilePath) (timeoutMs : Nat) (act : IO α) : IO α := do acquireLockTimeout lockDir timeoutMs try diff --git a/Beam/Daemon/Protocol.lean b/Beam/Daemon/Protocol.lean index 4a0a0da8..5ee5f298 100644 --- a/Beam/Daemon/Protocol.lean +++ b/Beam/Daemon/Protocol.lean @@ -158,7 +158,7 @@ def daemonGenerationStatus match ← daemonProbe endpoint workspaceId with | .error failure => match failure with - | .transport _ => + | .transport _ _ => if ← endpointAcceptsConnection endpoint then pure <| .unrecognized failure else diff --git a/Beam/Mcp/Server.lean b/Beam/Mcp/Server.lean index 1cfad97c..edd3d75a 100644 --- a/Beam/Mcp/Server.lean +++ b/Beam/Mcp/Server.lean @@ -317,9 +317,11 @@ private def RequestStatusEmitter.emitOnce Std.Internal.UV.Timer.stop emitter.timer private def RequestStatusEmitter.finish (emitter : RequestStatusEmitter) : IO Unit := do - emitter.state.atomically do - modify fun current => { current with finished := true } - Std.Internal.UV.Timer.stop emitter.timer + try + emitter.state.atomically do + modify fun current => { current with finished := true } + finally + Std.Internal.UV.Timer.stop emitter.timer private def RequestStatusEmitter.create (notifier : Notifier) @@ -328,19 +330,26 @@ private def RequestStatusEmitter.create (path? : Option String) : IO RequestStatusEmitter := do let delayMs ← requestStatusDelayMs let timer ← Std.Internal.UV.Timer.mk delayMs.toUInt64 false - let emitter : RequestStatusEmitter := { - requestId := requestId.json - tool - path? - state := ← Std.Mutex.new {} - timer - emitStatus := emitToolStatusLog notifier - } - let timerResult ← timer.next - IO.chainTask timerResult.result? fun - | some () => emitter.emitOnce .running s!"{toolTarget tool path?} is still working." - | none => pure () - pure emitter + try + let emitter : RequestStatusEmitter := { + requestId := requestId.json + tool + path? + state := ← Std.Mutex.new {} + timer + emitStatus := emitToolStatusLog notifier + } + let timerResult ← timer.next + IO.chainTask timerResult.result? fun + | some () => emitter.emitOnce .running s!"{toolTarget tool path?} is still working." + | none => pure () + pure emitter + catch err => + try + Std.Internal.UV.Timer.stop timer + catch _ => + pure () + throw err private def toolPath? (arguments : Json) : Option String := (arguments.getObjValAs? String "path").toOption @@ -840,8 +849,8 @@ def Internal.handleToolCall Internal.traceMcp s!"tools/call invalid input id={req.id.label} tool={params.name.key} error={err}" return .ok <| callToolErrorResult <| ToolError.invalidInput err let reporter ← CallReporter.create notifier req.id params progress? - reporter.emitPreparing try + reporter.emitPreparing let (runtime, root) ← match ← ensureRuntimeForWorkspace state opts workspace.workspaceId workspace.root with | .ok runtimeAndRoot => diff --git a/Beam/Mcp/Stdio.lean b/Beam/Mcp/Stdio.lean index 7e81a3b2..9b4a3ba4 100644 --- a/Beam/Mcp/Stdio.lean +++ b/Beam/Mcp/Stdio.lean @@ -10,9 +10,10 @@ open Lean namespace Beam.Mcp.Stdio -def isBrokenPipeError (err : IO.Error) : Bool := - let msg := err.toString - msg.contains "broken pipe" || msg.contains "Broken pipe" || msg.contains "EPIPE" +/-- Whether writing failed because the output resource has disappeared, including POSIX `EPIPE`. -/ +def isClosedOutputError : IO.Error → Bool + | .resourceVanished _ _ => true + | _ => false def stripLineEnding (line : String) : String := let line := @@ -25,16 +26,8 @@ def stripLineEnding (line : String) : String := else line -def writeJsonLineToStream (stream : IO.FS.Stream) (json : Json) : IO Unit := do - stream.putStr (json.compress ++ "\n") - stream.flush - def writeJsonLineToHandle (handle : IO.FS.Handle) (json : Json) : IO Unit := do handle.putStr (json.compress ++ "\n") handle.flush -def writeStdoutJsonLine (json : Json) : IO Unit := do - let stdout ← IO.getStdout - writeJsonLineToStream stdout json - end Beam.Mcp.Stdio diff --git a/Beam/Mcp/StdioServer.lean b/Beam/Mcp/StdioServer.lean index 25304858..653b9fac 100644 --- a/Beam/Mcp/StdioServer.lean +++ b/Beam/Mcp/StdioServer.lean @@ -199,14 +199,22 @@ private def Coordinator.currentControlBarrier? coordinator.routing.atomically do pure (← get).controlBarrier? +private structure ControlRequestFence where + previous? : Option (IO.Promise Unit) + done : IO.Promise Unit + priorRequests : Array InFlightRequest + private def Coordinator.pushControlBarrier - (coordinator : Coordinator) : IO (Option (IO.Promise Unit) × IO.Promise Unit) := do + (coordinator : Coordinator) + (request : InFlightRequest) : IO ControlRequestFence := do let done ← IO.Promise.new - let previous? ← coordinator.routing.atomically do + let (previous?, priorRequests) ← coordinator.routing.atomically do let routing ← get + let priorRequests := routing.admitted.toList.filterMap (fun (_, other) => + if other.brokerId == request.brokerId then none else some other) |>.toArray set { routing with controlBarrier? := some done } - pure routing.controlBarrier? - pure (previous?, done) + pure (routing.controlBarrier?, priorRequests) + pure { previous?, done, priorRequests } private def awaitControlBarrier (barrier? : Option (IO.Promise Unit)) : IO Unit := do match barrier? with @@ -236,28 +244,97 @@ private def Coordinator.finishRequest (coordinator : Coordinator) (request : InFlightRequest) (response : Json) : IO Unit := do - let sendResponse ← request.state.atomically do - let current ← get - match current.phase with - | .active => - set { current with phase := .completed } - pure true - | .clientCancelled => - set { current with phase := .completed } - pure false - | .completed => - pure false - -- Retire the exact admission before its terminal response becomes visible. A client may reuse - -- an ID as soon as it observes that response; retaining the routing entry until after the write - -- creates a race in which the new request is mistaken for a duplicate active request. - coordinator.retireRequestId request try + let sendResponse ← request.state.atomically do + let current ← get + match current.phase with + | .active => + set { current with phase := .completed } + pure true + | .clientCancelled => + set { current with phase := .completed } + pure false + | .completed => + pure false + -- Retire the exact admission before its terminal response becomes visible. A client may reuse + -- an ID as soon as it observes that response; retaining the routing entry until after the write + -- creates a race in which the new request is mistaken for a duplicate active request. + coordinator.retireRequestId request if sendResponse then coordinator.output.send response finally - -- Barriers continue to observe completion only after the terminal write has finished. - coordinator.completeRequest request - request.resolveDone + -- Retry retirement when an earlier state/routing operation failed, and make every cleanup step + -- independent. Barriers observe completion only after the terminal write has finished. + try + coordinator.retireRequestId request + finally + try + coordinator.completeRequest request + finally + request.resolveDone + +private def traceMcpSafely (message : String) : IO Unit := do + try + Internal.traceMcp message + catch _ => + pure () + +/-- +Own one registered request through terminal completion. + +`after` belongs to the same ownership scope and runs after the request's terminal write attempt. In +particular, a workspace-control fence is not released before the request is retired and its `done` +promise is resolved. +-/ +private def Coordinator.runOwnedRequest + (coordinator : Coordinator) + (req : Request) + (request : InFlightRequest) + (work : IO Json) + (after : IO Unit := pure ()) : IO Unit := do + try + let response ← + try + work + catch e => + traceMcpSafely s!"request work failed id={req.id.label}: {e.toString}" + pure <| errorResponse req.id (RpcError.internalError e.toString) + try + coordinator.finishRequest request response + catch e => + if !Beam.Mcp.Stdio.isClosedOutputError e then + traceMcpSafely s!"request completion failed id={req.id.label}: {e.toString}" + pure () + finally + try + after + catch e => + traceMcpSafely s!"request finalizer failed id={req.id.label}: {e.toString}" + +/-- Project a worker or setup failure through the registered request's single completion path. -/ +private def Coordinator.failOwnedRequest + (coordinator : Coordinator) + (req : Request) + (request : InFlightRequest) + (error : IO.Error) + (after : IO Unit := pure ()) : IO Unit := do + traceMcpSafely s!"request work failed id={req.id.label}: {error.toString}" + coordinator.runOwnedRequest req request + (pure <| errorResponse req.id (RpcError.internalError error.toString)) after + +/-- Transfer a registered request to a worker, or complete it synchronously if task startup fails. -/ +private def Coordinator.spawnOwnedRequest + (coordinator : Coordinator) + (req : Request) + (request : InFlightRequest) + (work : IO Json) + (after : IO Unit := pure ()) : IO Unit := do + try + let _ ← IO.asTask (prio := Task.Priority.dedicated) do + coordinator.runOwnedRequest req request work after + pure () + catch e => + coordinator.failOwnedRequest req request e after private def InFlightRequest.markClientCancelled (request : InFlightRequest) : IO (Bool × Option Beam.Broker.RequestHandle) := do @@ -312,17 +389,12 @@ private def awaitRequests (requests : Array InFlightRequest) : IO Unit := do for request in requests do awaitRequestDone request -private def Coordinator.otherAdmittedRequests - (coordinator : Coordinator) - (request : InFlightRequest) : IO (Array InFlightRequest) := do - coordinator.routing.atomically do - pure <| (← get).admitted.toList.filterMap (fun (_, other) => - if other.brokerId == request.brokerId then none else some other) |>.toArray - private def Coordinator.closeTransport (coordinator : Coordinator) : IO Unit := do - let requests ← coordinator.beginClosing - awaitRequests requests - coordinator.state.closeRuntime + try + let requests ← coordinator.beginClosing + awaitRequests requests + finally + coordinator.state.closeRuntime private def Coordinator.admitToolRequest (coordinator : Coordinator) @@ -343,21 +415,18 @@ private def Coordinator.executeToolRequest let notifications : NotificationSink := { send := fun json => request.sendIfActive coordinator.output json } - try - match ← Internal.handleToolCall - coordinator.state - opts - request.brokerId - request.bindBrokerRequest - req - admitted - parsedParams - notifications - initialProgress with - | .ok result => pure <| successResponseForEra admitted.era req.id result - | .error err => pure <| errorResponse req.id err - catch e => - pure <| errorResponse req.id (RpcError.internalError e.toString) + match ← Internal.handleToolCall + coordinator.state + opts + request.brokerId + request.bindBrokerRequest + req + admitted + parsedParams + notifications + initialProgress with + | .ok result => pure <| successResponseForEra admitted.era req.id result + | .error err => pure <| errorResponse req.id err private def Coordinator.toolRequestResponse (coordinator : Coordinator) @@ -368,15 +437,12 @@ private def Coordinator.toolRequestResponse (request : InFlightRequest) (barrier? : Option (IO.Promise Unit)) (initialProgress : Nat := 0) : IO Json := do - try - awaitControlBarrier barrier? - if ← request.isActive then - coordinator.executeToolRequest opts req admitted parsedParams request initialProgress - else - pure <| errorResponse req.id <| - RpcError.invalidRequest "request was cancelled before execution" - catch e => - pure <| errorResponse req.id (RpcError.internalError e.toString) + awaitControlBarrier barrier? + if ← request.isActive then + coordinator.executeToolRequest opts req admitted parsedParams request initialProgress + else + pure <| errorResponse req.id <| + RpcError.invalidRequest "request was cancelled before execution" private def finishReporterSafely (req : Request) @@ -384,7 +450,7 @@ private def finishReporterSafely try finishReporter catch e => - Internal.traceMcp s!"request reporter finish failed id={req.id.label}: {e.toString}" + traceMcpSafely s!"request reporter finish failed id={req.id.label}: {e.toString}" private def Coordinator.spawnToolRequest (coordinator : Coordinator) @@ -393,22 +459,26 @@ private def Coordinator.spawnToolRequest (evidence : RequestProtocolEvidence) (parsedParams : Except String CallToolParams) (request : InFlightRequest) : IO Unit := do + let admission ← + try + coordinator.admitToolRequest req evidence + catch e => + coordinator.failOwnedRequest req request e + return let admitted ← - match ← coordinator.admitToolRequest req evidence with + match admission with | .ok admitted => pure admitted | .error response => - coordinator.finishRequest request response + coordinator.runOwnedRequest req request (pure response) return - let barrier? ← coordinator.currentControlBarrier? - let _ ← IO.asTask (prio := Task.Priority.dedicated) do + let barrier? ← try - let response ← - coordinator.toolRequestResponse opts req admitted parsedParams request barrier? - coordinator.finishRequest request response + coordinator.currentControlBarrier? catch e => - if !Beam.Mcp.Stdio.isBrokenPipeError e then - Internal.traceMcp s!"request completion failed id={req.id.label}: {e.toString}" - pure () + coordinator.failOwnedRequest req request e + return + coordinator.spawnOwnedRequest req request <| + coordinator.toolRequestResponse opts req admitted parsedParams request barrier? private def Coordinator.handleControlToolRequest (coordinator : Coordinator) @@ -417,46 +487,45 @@ private def Coordinator.handleControlToolRequest (evidence : RequestProtocolEvidence) (parsedParams : Except String CallToolParams) (request : InFlightRequest) : IO Unit := do - match ← coordinator.admitToolRequest req evidence with - | .error response => coordinator.finishRequest request response + let admission ← + try + coordinator.admitToolRequest req evidence + catch e => + coordinator.failOwnedRequest req request e + return + match admission with + | .error response => coordinator.runOwnedRequest req request (pure response) | .ok admitted => - let notifications : NotificationSink := { - send := fun json => request.sendIfActive coordinator.output json - } - let reporter? ← - match parsedParams with - | .ok params => - Internal.createPreDispatchReporter? - coordinator.state req.id admitted params notifications - | .error _ => pure none - let initialProgress := reporter?.map (·.initialProgress) |>.getD 0 - let finishReporter : IO Unit := - match reporter? with - | some reporter => reporter.finish - | none => pure () - let (previous?, done) ← coordinator.pushControlBarrier - let priorRequests ← coordinator.otherAdmittedRequests request - let _ ← IO.asTask (prio := Task.Priority.dedicated) do - let response ← - try - -- A control operation is a full stream-order fence: work admitted before it drains, - -- while work admitted afterward waits on `done`. - awaitRequests priorRequests - coordinator.toolRequestResponse opts req admitted parsedParams request previous? - initialProgress - catch e => - if !Beam.Mcp.Stdio.isBrokenPipeError e then - Internal.traceMcp s!"workspace control completion failed id={req.id.label}: {e.toString}" - pure <| errorResponse req.id (RpcError.internalError e.toString) + let fence ← try - finishReporterSafely req finishReporter - coordinator.finishRequest request response + coordinator.pushControlBarrier request catch e => - if !Beam.Mcp.Stdio.isBrokenPipeError e then - Internal.traceMcp s!"workspace control completion failed id={req.id.label}: {e.toString}" - finally - resolvePromise done - pure () + coordinator.failOwnedRequest req request e + return + let work : IO Json := do + let notifications : NotificationSink := { + send := fun json => request.sendIfActive coordinator.output json + } + let reporter? ← + match parsedParams with + | .ok params => + Internal.createPreDispatchReporter? + coordinator.state req.id admitted params notifications + | .error _ => pure none + let run (initialProgress : Nat) : IO Json := do + -- A control operation is a full stream-order fence: work admitted before it drains, + -- while work admitted afterward waits on `done`. + awaitRequests fence.priorRequests + coordinator.toolRequestResponse opts req admitted parsedParams request fence.previous? + initialProgress + match reporter? with + | none => run 0 + | some reporter => + try + run reporter.initialProgress + finally + finishReporterSafely req reporter.finish + coordinator.spawnOwnedRequest req request work (resolvePromise fence.done) private def isWorkspaceControl : Except String CallToolParams → Bool | .ok params => params.name == .leanDropWorkspace @@ -567,7 +636,7 @@ partial def runStdio (opts : Options) : IO Unit := do try loop catch e => - if Beam.Mcp.Stdio.isBrokenPipeError e then + if Beam.Mcp.Stdio.isClosedOutputError e then pure () else throw e diff --git a/Beam/System.lean b/Beam/System.lean index bb23cd97..a66cac59 100644 --- a/Beam/System.lean +++ b/Beam/System.lean @@ -66,19 +66,6 @@ private def localPidAlive (pid : Nat) : IO Bool := do let out ← IO.Process.output { cmd := (← killCommand), args := #["-0", toString pid] } pure (out.exitCode == 0) -/-- Inspect zombie state for a PID already known to belong to the caller's process domain. -/ -private def localPidZombie (pid : Nat) : IO Bool := do - try - let out ← IO.Process.output { - cmd := "ps" - args := #["-o", "stat=", "-p", toString pid] - stdin := .null - stderr := .null - } - pure <| out.exitCode == 0 && (trimLine out.stdout).startsWith "Z" - catch _ => - pure false - def currentPidDomain? : IO (Option String) := do try let domain ← readCmdTrim "readlink" #["/proc/self/ns/pid"] @@ -133,12 +120,6 @@ def RecordedPid.observe (recorded : RecordedPid) : IO RecordedPidObservation := | .different => pure .differentDomain | .unknown => pure .unknownDomain -/-- Return zombie state only when a persisted PID belongs to the caller's current process domain. -/ -def RecordedPid.zombieIfLocal? (recorded : RecordedPid) : IO (Option Bool) := do - match ← recorded.domainRelation with - | .local => some <$> localPidZombie recorded.pid - | .invalid | .different | .unknown => pure none - /-- Send the default termination signal only to a persisted PID in the caller's current domain. -/ def RecordedPid.terminateIfLocal (recorded : RecordedPid) : IO Bool := do match ← recorded.domainRelation with diff --git a/docs/DEVELOPMENT.md b/docs/DEVELOPMENT.md index d6aa2ec4..f6f937db 100644 --- a/docs/DEVELOPMENT.md +++ b/docs/DEVELOPMENT.md @@ -189,8 +189,10 @@ per socket connection, so the connection itself supplies response routing and di MCP multiplexes requests over one stdio stream, so `Beam.Mcp.StdioServer` must own exact JSON-RPC ID routing, serialized output, client cancellation, and workspace-control barriers. Both paths converge on `ServerRuntime.dispatchRequestWithHandle`, whose admission handle is the shared cancellation and -drain boundary. Keep the two ingress coordinators separate unless a future transport has the same -wire-level ownership rules; do not duplicate semantic operation dispatch above that boundary. +drain boundary. `ServerRuntime` remains transport-agnostic; the daemon's private transport context +owns its endpoint, listener, and stop state. Keep the two ingress coordinators separate unless a +future transport has the same wire-level ownership rules; do not duplicate semantic operation +dispatch above that boundary. The executable path is split into importable modules: @@ -211,6 +213,9 @@ Keep these stdio invariants explicit: - the server emits no JSON-RPC requests to clients - request IDs preserve their string-versus-integer type and their original JSON spelling - ordinary calls may overlap; cache eviction is a full stream-order fence and shutdown drains work +- once a request is registered, synchronous setup must either transfer it to an owned worker or + complete it terminally; worker finalization always retires the request and releases its control + fence, including after task-start, reporting, or output failures - ordinary tool calls bind cancellation to the exact broker admission handle; do not reintroduce request-ID polling between MCP and the broker - routing/output locks do not acquire setup, progress, or per-request locks @@ -325,6 +330,8 @@ handle has been validated, never fall back to matching a reusable client request the `PendingRequest` as one value instead of separating its promise from its cancellation reference. Once that reference is marked, cancellation takes precedence over a concurrent backend failure; an already-completed backend success remains successful. +After initialization, `sessionReaderLoop` is the backend session's only stdout reader. Shutdown +replies use the same pending-request store and reader loop; do not add a second direct stdout read. `ServerRuntime.close` is the shared runtime teardown boundary. It closes admission, marks every admitted request for cancellation, shuts down backend sessions to unblock pending work, waits for @@ -334,6 +341,12 @@ repeated callers wait for the same result. Transport owners decide what triggers their listener or stdio connection stops; they must not duplicate broker draining or backend teardown. +A newly spawned backend remains a provisional resource until its initialization response arrives +within the 30-second startup deadline and the broker sends `initialized`. Any initialization error, +timeout, or notification-write failure terminates and reaps that child before the acquisition fails; +only a fully initialized session enters workspace state. Keep this acquisition bracket intact so +runtime closure never has to discover an unowned provisional process. + The thick part of the broker is request orchestration. For `sync`, `runAt`, `goals`, `runWith`, `release`, and `save`, the broker reads the source file, updates the LSP document mirror, waits for the relevant diagnostics/progress barrier when needed, asks the backend for semantic facts, and @@ -407,15 +420,15 @@ the registry stale; cleanup remains generation-scoped and PID fallback is permit typed PID-domain boundary. The owner also watches its exact registry generation and daemon child. `lean-beam shutdown` removes -that generation after the typed shutdown response, which makes the holder close its pipe and lets -the daemon's stdin watcher finish. On every holder exit path, the holder removes its exact registry -generation before waiting for the daemon child to drain, then retries the same generation-scoped -removal after bounded child cleanup. A draining daemon is therefore never advertised as attachable, -and neither removal can delete a replacement generation. An unexpected nonzero daemon exit is -reported by the holder. Interrupting or killing the holder closes the pipe by process lifetime. A -paused holder keeps the pipe open, so the session remains valid without time-based expiry. If the -project root disappears, the daemon's root watcher and the holder both converge on the same shutdown -path. +that generation after the typed shutdown response or a bounded response failure, which makes the +holder close its pipe and lets the daemon's stdin watcher finish. On every holder exit path, the +holder removes its exact registry generation before waiting for the daemon child to drain, then +retries the same generation-scoped removal after bounded child cleanup. A draining daemon is +therefore never advertised as attachable, and neither removal can delete a replacement generation. +An unexpected nonzero daemon exit is reported by the holder. Interrupting or killing the holder +closes the pipe by process lifetime. A paused holder keeps the pipe open, so the session remains valid +without time-based expiry. If the project root disappears, the daemon's root watcher and the holder +both converge on the same shutdown path. This model prevents PID-isolated commands from making contradictory ownership decisions: later commands may attach to a validated endpoint, but none can silently become a replacement owner. @@ -441,8 +454,8 @@ Keep these invariants covered: Generic process helpers and the typed `RecordedPid.observe` boundary live in [Beam/System.lean](../Beam/System.lean). Persisted registry and lock-owner PIDs must pass -through that boundary; only a matching recorded/current PID-domain pair permits a local liveness, -zombie, or termination operation. Generic directory locks live in +through that boundary; only a matching recorded/current PID-domain pair permits a local liveness or +termination operation. Generic directory locks live in [Beam/Cli/Lock.lean](../Beam/Cli/Lock.lean). Their owner metadata records both PID and PID domain; only a proven dead same-domain owner is reaped, while missing, malformed, unknown-domain, and different-domain owners fail closed. Project daemon control locks use a bounded wait so a live but diff --git a/docs/MCP.md b/docs/MCP.md index 5151953d..4e7e3749 100644 --- a/docs/MCP.md +++ b/docs/MCP.md @@ -144,7 +144,8 @@ cancellable requests, waits for every admitted request and non-cancellable works finish, then closes the broker runtime and all remaining backend sessions. A JSON-RPC request ID is active only until its request reaches terminal completion. The server retires that exact admission before publishing its terminal response, so a client may reuse the ID after observing the response; -the completion barrier is resolved only after the response write finishes. +the completion barrier is resolved only after the terminal response write attempt finishes, including +when that write fails. CLI ingress has a different transport lifetime: a broker daemon accepts one request per socket connection, and disconnecting that connection cancels its exact broker admission. MCP carries many diff --git a/docs/STATUS.md b/docs/STATUS.md index ebca5425..8e056f1e 100644 --- a/docs/STATUS.md +++ b/docs/STATUS.md @@ -194,11 +194,12 @@ Exact event ordering and examples live in `lean-beam shutdown` unpublishes the registry generation so the holder closes it cleanly. - A startup failure that reports `operation not permitted` through `.beam/beam-daemon-startup.log` is usually an environment restriction, not a bundle-resolution mismatch. -- Typed broker transport and invalid-response failures include registry/log context and write a JSON - incident record under `.beam/daemon-failures/` or the per-root subdirectory of - `BEAM_CONTROL_DIR`. Incident kinds are `brokerTransportFailure` and `invalidBrokerResponse`; - callback/display failures do not create daemon incidents. Beam keeps the latest 50 incident records, - and `lean-beam doctor` lists recent incident paths. +- Typed broker transport, invalid-response, and response-timeout failures include registry/log + context and write a JSON incident record under `.beam/daemon-failures/` or the per-root + subdirectory of `BEAM_CONTROL_DIR`. Incident kinds are `brokerTransportFailure`, + `invalidBrokerResponse`, and `brokerResponseTimeout`; callback/display failures do not create + daemon incidents. Beam keeps the latest 50 incident records, and `lean-beam doctor` lists recent + incident paths. - A standalone Beam daemon watches its canonical project root. If a git worktree or project directory is removed while the daemon is active, it shuts down its backend sessions and exits instead of remaining undiscoverable after its project-local registry disappears. A later wrapper diff --git a/docs/TESTING.md b/docs/TESTING.md index ae1e2e29..05837c1d 100644 --- a/docs/TESTING.md +++ b/docs/TESTING.md @@ -104,7 +104,8 @@ Additional Beam lanes: Current Beam coverage includes: -- fast Beam daemon smoke, request-stream, save-stream, startup-handshake, tracked-diagnostic dedup, +- fast Beam daemon smoke, request-stream, save-stream, startup-handshake with failed provisional + backend cleanup, tracked-diagnostic dedup, exact broker request-handle lifetime, identity-matched daemon-generation probes, terminal shutdown response delivery, shutdown after the requesting TCP client resets its connection, protocol tests, and validated-toolchain/release-line CI policy consistency through @@ -234,8 +235,9 @@ exact modern and legacy broker cancellation, per-request progress ordering, dete between a gated request in one workspace and a fast request in another, single-flight first use, simultaneous cold first use of distinct roots, stateless multi-root isolation, non-cancellable cache eviction with ordering on both sides of the global fence, lazy recreation, and EOF cancellation and -teardown. The full stdio suite also checks modern request-ID reuse and rejects a proof handle carried -across an MCP process restart. The slow Beam suite runs +teardown. The full stdio suite also checks modern request-ID reuse, synchronous and workspace-control +closed-output teardown, and rejects a proof handle carried across an MCP process restart. The slow +Beam suite runs `--scenario multi-toolchain-workspaces` after installing both fixture toolchains and verifies that one MCP process keeps both project-specific Lean sessions active. diff --git a/skills/lean-beam/agents/openai.yaml b/skills/lean-beam/agents/openai.yaml index fc5e080f..9d4c3f6b 100644 --- a/skills/lean-beam/agents/openai.yaml +++ b/skills/lean-beam/agents/openai.yaml @@ -5,7 +5,7 @@ interface: display_name: "Lean Beam" short_description: "Lean probes, sync, and exact chaining" - default_prompt: "Use $lean-beam on an external Lean project for isolated on-disk Lean probes through beam; use lean-beam sync after real edits, use lean-beam ensure --hold only when a sandbox needs daemon reuse across commands, do not assume separate lean-beam run-at calls chain, and use pre-stable handle workflows only when exact speculative continuation matters." + default_prompt: "Use $lean-beam on an external Lean project for isolated on-disk Lean probes through beam; start lean-beam ensure --hold and keep it running while using wrapper commands, use lean-beam sync after real edits, do not assume separate lean-beam run-at calls chain, and use pre-stable handle workflows only when exact speculative continuation matters." policy: allow_implicit_invocation: true diff --git a/skills/rocq-beam/agents/openai.yaml b/skills/rocq-beam/agents/openai.yaml index 0595e2dd..1da4eb9b 100644 --- a/skills/rocq-beam/agents/openai.yaml +++ b/skills/rocq-beam/agents/openai.yaml @@ -5,7 +5,7 @@ interface: display_name: "Rocq Beam Aux" short_description: "Optional Rocq-to-Lean goal probes" - default_prompt: "Use $rocq-beam when an external Rocq project needs the optional Rocq goal-probe surface exposed through lean-beam for Rocq-to-Lean porting; save before probing, use lean-beam rocq-goals-after / rocq-goals-prev, and do not assume hidden cross-request proof state, handles, unsaved-buffer access, or a Rocq run-at command." + default_prompt: "Use $rocq-beam when an external Rocq project needs the optional Rocq goal-probe surface exposed through lean-beam for Rocq-to-Lean porting; start lean-beam ensure rocq --hold and keep it running while using wrapper commands, save before probing, use lean-beam rocq-goals-after / rocq-goals-prev, and do not assume hidden cross-request proof state, handles, unsaved-buffer access, or a Rocq run-at command." policy: allow_implicit_invocation: true diff --git a/tests/lean/BeamTest/Broker/CliDaemonTest.lean b/tests/lean/BeamTest/Broker/CliDaemonTest.lean index 7d65cee9..93b6483f 100644 --- a/tests/lean/BeamTest/Broker/CliDaemonTest.lean +++ b/tests/lean/BeamTest/Broker/CliDaemonTest.lean @@ -67,7 +67,7 @@ private def requireSubstring (label needle haystack : String) : IO Unit := do require s!"{label}: expected '{needle}' in '{haystack}'" (Beam.Cli.hasSubstring haystack needle) private def brokerTransportFailure (detail : String) : Beam.Broker.BrokerClientFailure := - .transport (IO.userError detail) + .transport .receive (IO.userError detail) private def requireJsonNat (label field : String) (expected : Nat) (json : Json) : IO Unit := do let actual ← IO.ofExcept <| json.getObjValAs? Nat field @@ -597,7 +597,8 @@ private def checkDaemonFailureContext : IO Unit := do let startupLog ← Beam.Daemon.daemonStartupLogPath root IO.FS.writeFile startupLog "line 1\nline 2\n" let detail := "synthetic broker transport failure" - let msg ← Beam.Cli.daemonFailureMessage root (brokerTransportFailure detail) + let failure := brokerTransportFailure detail + let msg ← Beam.Cli.daemonFailureMessage root failure requireSubstring "daemon failure context should include registry path" "Beam daemon registry" msg requireSubstring "daemon failure context should include daemon id" "daemonId: daemon-test" msg requireSubstring "daemon failure context should include dead pid status" "pid: 999999999 (not alive)" msg @@ -612,8 +613,8 @@ private def checkDaemonFailureContext : IO Unit := do requireJsonNat "daemon failure incident should use schema version" "schemaVersion" 1 incidentJson requireJsonString "daemon failure incident should classify the typed transport failure" "kind" "brokerTransportFailure" incidentJson - requireJsonString "daemon failure incident should keep original detail" - "detail" detail incidentJson + requireJsonString "daemon failure incident should retain typed operation context" + "detail" failure.detail incidentJson requireJsonString "daemon failure incident should include root" "root" root.toString incidentJson requireJsonString "daemon failure incident should include registry path" diff --git a/tests/lean/BeamTest/Broker/ProtocolTest.lean b/tests/lean/BeamTest/Broker/ProtocolTest.lean index 8e97c877..1ff26db5 100644 --- a/tests/lean/BeamTest/Broker/ProtocolTest.lean +++ b/tests/lean/BeamTest/Broker/ProtocolTest.lean @@ -969,7 +969,7 @@ private def checkLifecycleTeardownReleasesStateMutex let targetConfig : BrokerConfig := { root := targetRoot } let replacementConfig : BrokerConfig := { root := replacementRoot } let observerConfig : BrokerConfig := { root := observerRoot } - let runtime ← ServerRuntime.create targetConfig targetId (.tcp 0) + let runtime ← ServerRuntime.create targetConfig targetId let session ← stubbornSession targetId targetRoot sentinel runtime.state.atomically do let state ← get @@ -1038,7 +1038,7 @@ private partial def waitForCancellation private def checkSessionCloseAdmission : IO Unit := do let root := System.FilePath.mk "/tmp/beam-session-close-admission" let runtime ← Beam.Broker.ServerRuntime.create - ({ root } : Beam.Broker.BrokerConfig) "fixture" (.tcp 0) + ({ root } : Beam.Broker.BrokerConfig) "fixture" let beforeClose ← runtime.dispatchRequest { op := .stats } require "stats should be admitted before session close" beforeClose.ok let active ← diff --git a/tests/lean/BeamTest/Broker/StartupHandshakeTest.lean b/tests/lean/BeamTest/Broker/StartupHandshakeTest.lean index a3cfce45..f459d8f2 100644 --- a/tests/lean/BeamTest/Broker/StartupHandshakeTest.lean +++ b/tests/lean/BeamTest/Broker/StartupHandshakeTest.lean @@ -18,6 +18,7 @@ private def writeFakeServer (root : System.FilePath) : IO System.FilePath := do let body := String.intercalate "\n" [ "#!/usr/bin/env bash", "set -euo pipefail", + "printf '%s\\n' \"$$\" > \"$(dirname \"$0\")/fake-lean.pid\"", "frame() {", " local body=\"$1\"", " printf 'Content-Length: %s\\r\\n\\r\\n%s' \"${#body}\" \"$body\"", @@ -37,6 +38,19 @@ private def writeFakeServer (root : System.FilePath) : IO System.FilePath := do throw <| IO.userError s!"failed to chmod fake startup server\n{out.stderr}" pure script +private partial def waitForProcessGone (pid : Nat) (tries : Nat := 80) : IO Unit := do + let out ← IO.Process.output { + cmd := "/bin/kill" + args := #["-0", toString pid] + } + if out.exitCode != 0 then + pure () + else if tries == 0 then + throw <| IO.userError s!"failed provisional backend process {pid} remained alive" + else + IO.sleep 25 + waitForProcessGone pid (tries - 1) + def main : IO Unit := do let endpoint ← freshTcpEndpoint let root ← mkTempProjectRoot "beam-daemon-startup" @@ -54,6 +68,10 @@ def main : IO Unit := do throw <| IO.userError s!"expected internalError for startup failure, got {(toJson resp).compress}" unless err.message.contains "initialize failed" do throw <| IO.userError s!"expected startup failure to mention initialize failure, got {(toJson resp).compress}" + let pidText ← IO.FS.readFile (root / "fake-lean.pid") + let some pid := pidText.trimAscii.toString.toNat? + | throw <| IO.userError s!"invalid fake backend pid '{pidText}'" + waitForProcessGone pid finally try broker.kill diff --git a/tests/lean/BeamTest/Broker/StreamDedupTest.lean b/tests/lean/BeamTest/Broker/StreamDedupTest.lean index 62bcbd6b..1724da81 100644 --- a/tests/lean/BeamTest/Broker/StreamDedupTest.lean +++ b/tests/lean/BeamTest/Broker/StreamDedupTest.lean @@ -158,7 +158,7 @@ private def fakeServerWithLeanSession config lean := { nextEpoch := 1, session? := some session } } - let server ← Beam.Broker.ServerRuntime.create config fixtureWorkspaceId (.tcp 0) + let server ← Beam.Broker.ServerRuntime.create config fixtureWorkspaceId server.state.atomically do set ({ bootstrapConfig := config diff --git a/tests/test-mcp-stdio.py b/tests/test-mcp-stdio.py index 95de965a..adcde7e4 100644 --- a/tests/test-mcp-stdio.py +++ b/tests/test-mcp-stdio.py @@ -3177,53 +3177,117 @@ def run_legacy_eof_teardown(repo_root, fixture_root, timeout): client.close() +def require_clean_exit_after_closed_stdout(client, message, label): + try: + client.proc.stdout.close() + client.send_message(message) + client.close_input() + try: + client.proc.wait(timeout=5) + except subprocess.TimeoutExpired: + client.proc.kill() + fail(f"lean-beam-mcp did not exit after stdout was closed during {label}") + client.stderr_thread.join(timeout=1) + stderr = "\n".join(client.stderr_lines) + require( + client.proc.returncode == 0, + f"lean-beam-mcp exited with {client.proc.returncode} during {label}\n{stderr}", + ) + if client.server_trace: + unexpected = [ + line for line in stderr.splitlines() + if not line.startswith("lean-beam-mcp trace ") + ] + require( + not unexpected, + f"lean-beam-mcp wrote unexpected non-trace stderr during {label}:\n" + + "\n".join(unexpected), + ) + else: + require( + stderr.strip() == "", + f"lean-beam-mcp wrote unexpected stderr during {label}:\n{stderr}", + ) + finally: + if client.proc.poll() is None: + client.proc.kill() + client.proc.wait(timeout=5) + + def run_closed_stdout_regression(repo_root, fixture_root, timeout): with tempfile.TemporaryDirectory(prefix="lean-beam-mcp-closed-stdout-") as tmp: project_root = Path(tmp) / "project" copy_project_fixture(fixture_root, project_root) - client = McpClient( + initialize_client = McpClient( repo_root, project_root, timeout, - label="closed-stdout-regression", + label="closed-stdout-initialize", + drain_stdout=False, + ) + require_clean_exit_after_closed_stdout( + initialize_client, + { + "jsonrpc": "2.0", + "id": "closed-stdout-initialize", + "method": "initialize", + "params": initialize_params(), + }, + "initialize response", + ) + + control_client = McpClient( + repo_root, + project_root, + timeout, + label="closed-stdout-control", drain_stdout=False, ) - stderr = "" try: - client.proc.stdout.close() - client.send_message( + control_client.send_message( { "jsonrpc": "2.0", - "id": "closed-stdout", + "id": "closed-stdout-control-initialize", "method": "initialize", "params": initialize_params(), } ) - client.proc.stdin.close() - try: - client.proc.wait(timeout=5) - except subprocess.TimeoutExpired: - client.proc.kill() - fail("lean-beam-mcp did not exit after stdout was closed") - client.stderr_thread.join(timeout=1) - stderr = "\n".join(client.stderr_lines) - require(client.proc.returncode == 0, f"lean-beam-mcp exited with {client.proc.returncode}\n{stderr}") - if client.server_trace: - unexpected = [ - line for line in stderr.splitlines() - if not line.startswith("lean-beam-mcp trace ") - ] - require( - not unexpected, - "lean-beam-mcp wrote unexpected non-trace stderr after closed stdout:\n" - + "\n".join(unexpected), - ) - else: - require(stderr.strip() == "", f"lean-beam-mcp wrote unexpected stderr after closed stdout:\n{stderr}") + ready, _, _ = select.select([control_client.proc.stdout], [], [], timeout) + require(ready, "closed-stdout control regression did not receive initialize response") + initialized = json.loads(control_client.proc.stdout.readline()) + require( + initialized.get("id") == "closed-stdout-control-initialize", + f"closed-stdout control regression received wrong initialize response: {initialized}", + ) + expect_result(initialized) + control_client.send_message( + { + "jsonrpc": "2.0", + "method": "notifications/initialized", + } + ) + require_clean_exit_after_closed_stdout( + control_client, + { + "jsonrpc": "2.0", + "id": "closed-stdout-control", + "method": "tools/call", + "params": { + "name": "lean_drop_workspace", + "arguments": { + "workspace": workspace_descriptor(project_root), + }, + "_meta": { + "progressToken": "closed-stdout-control-progress", + }, + }, + }, + "workspace-control response", + ) finally: - if client.proc.poll() is None: - client.proc.kill() - client.proc.wait(timeout=5) + if control_client.proc.poll() is None: + control_client.proc.kill() + control_client.proc.wait(timeout=5) def main(): From 09bdbe51ccd7bfc69a1bb2b725c23097a28e34e5 Mon Sep 17 00:00:00 2001 From: Emilio Jesus Gallego Arias Date: Thu, 27 Aug 2026 20:03:34 +0200 Subject: [PATCH 19/28] fix: close remaining daemon lifetime gaps --- Beam/Broker/Server.lean | 116 +++++++++++++----- Beam/Cli/Broker.lean | 5 +- scripts/lean-beam | 3 +- tests/lean/BeamTest/Broker/CliDaemonTest.lean | 11 +- tests/test-beam-prune.sh | 8 +- 5 files changed, 106 insertions(+), 37 deletions(-) diff --git a/Beam/Broker/Server.lean b/Beam/Broker/Server.lean index be8104f0..52701649 100644 --- a/Beam/Broker/Server.lean +++ b/Beam/Broker/Server.lean @@ -1193,8 +1193,7 @@ private def ServerRuntime.runWorkspaceTransition let transition := transition (← get) set transition.state pure transition - for session in transition.detachedSessions do - shutdownSession session + shutdownSessionsBestEffort transition.detachedSessions.toList none pure transition.result /-- @@ -2688,6 +2687,88 @@ private partial def parseCliOptions (opts : CliOptions) : List String → Except | arg :: _ => throw s!"unexpected Beam daemon argument '{arg}'" +private abbrev DaemonWatcherTask := Task (Except IO.Error Unit) + +private structure DaemonResources where + runtime : ServerRuntime + transport : DaemonTransport + rootWatcher : DaemonWatcherTask + ownerWatcher? : Option DaemonWatcherTask + +private def closeDaemonParts + (runtime : ServerRuntime) + (transport? : Option DaemonTransport) + (rootWatcher? ownerWatcher? : Option DaemonWatcherTask) : IO Unit := do + let firstError? ← + match transport? with + | none => pure none + | some transport => recordFirstCleanupError none <| transport.stop.set true + let firstError? ← + match transport? with + | none => pure firstError? + | some transport => + recordFirstCleanupError firstError? <| Transport.closeListener transport.listener + let firstError? ← + match ownerWatcher? with + | none => pure firstError? + | some ownerWatcher => + recordFirstCleanupError firstError? do + try + IO.cancel ownerWatcher + finally + discard <| IO.wait ownerWatcher + let firstError? ← + match rootWatcher? with + | none => pure firstError? + | some rootWatcher => + recordFirstCleanupError firstError? do + discard <| IO.wait rootWatcher + let firstError? ← recordFirstCleanupError firstError? runtime.close + if let some err := firstError? then + throw err + +private def DaemonResources.close (resources : DaemonResources) : IO Unit := + closeDaemonParts resources.runtime (some resources.transport) (some resources.rootWatcher) + resources.ownerWatcher? + +private def throwAfterBestEffortCleanup + (err : IO.Error) + (cleanup : IO Unit) : IO α := do + try + cleanup + catch _ => + pure () + throw err + +private def acquireDaemonResources + (opts : CliOptions) + (config : BrokerConfig) + (workspaceId : WorkspaceId) + (daemonIdentity? : Option DaemonIdentity) + (root : System.FilePath) : IO DaemonResources := do + let runtime ← ServerRuntime.create config workspaceId daemonIdentity? + let transport ← + try + DaemonTransport.create opts.endpoint + catch err => + throwAfterBestEffortCleanup err runtime.close + let rootWatcher ← + try + IO.asTask (prio := Task.Priority.dedicated) <| watchRoot runtime transport root + catch err => + throwAfterBestEffortCleanup err <| closeDaemonParts runtime (some transport) none none + let ownerWatcher? ← + try + if opts.sessionOwnerStdin then + some <$> IO.asTask (prio := Task.Priority.dedicated) + (watchSessionOwnerStdin runtime transport) + else + pure none + catch err => + throwAfterBestEffortCleanup err <| + closeDaemonParts runtime (some transport) (some rootWatcher) none + pure { runtime, transport, rootWatcher, ownerWatcher? } + def main (args : List String) : IO Unit := do let opts ← IO.ofExcept <| parseCliOptions {} args let some root := opts.root? @@ -2715,35 +2796,10 @@ def main (args : List String) : IO Unit := do leanPlugin? := leanPlugin? rocqCmd? := opts.rocqCmd? } - let runtime ← ServerRuntime.create config workspaceId daemonIdentity? - let transport ← DaemonTransport.create opts.endpoint - let rootWatcher ← IO.asTask (prio := Task.Priority.dedicated) <| - watchRoot runtime transport root - let ownerWatcher? ← - if opts.sessionOwnerStdin then - some <$> IO.asTask (prio := Task.Priority.dedicated) - (watchSessionOwnerStdin runtime transport) - else - pure none + let resources ← acquireDaemonResources opts config workspaceId daemonIdentity? root try - acceptLoop runtime transport + acceptLoop resources.runtime resources.transport finally - let firstError? ← recordFirstCleanupError none <| transport.stop.set true - let firstError? ← recordFirstCleanupError firstError? <| - Transport.closeListener transport.listener - let firstError? ← - match ownerWatcher? with - | none => pure firstError? - | some ownerWatcher => - recordFirstCleanupError firstError? do - try - IO.cancel ownerWatcher - finally - discard <| IO.wait ownerWatcher - let firstError? ← recordFirstCleanupError firstError? do - discard <| IO.wait rootWatcher - let firstError? ← recordFirstCleanupError firstError? runtime.close - if let some err := firstError? then - throw err + resources.close end Beam.Broker diff --git a/Beam/Cli/Broker.lean b/Beam/Cli/Broker.lean index 659a939d..9cadd173 100644 --- a/Beam/Cli/Broker.lean +++ b/Beam/Cli/Broker.lean @@ -64,7 +64,10 @@ def withInterruptWatcher (act : InterruptWatcher → IO α) : IO α := do try Std.Internal.UV.Signal.next signal catch err => - Std.Internal.UV.Signal.stop signal + try + Std.Internal.UV.Signal.stop signal + catch _ => + pure () throw err let event := promise.result? let watcher : InterruptWatcher := { diff --git a/scripts/lean-beam b/scripts/lean-beam index b30d64a1..e3a3aee3 100755 --- a/scripts/lean-beam +++ b/scripts/lean-beam @@ -47,7 +47,8 @@ usage: notes: - lean-beam keeps the Lean wrapper surface primary; optional Rocq goal probes are documented in docs/ROCQ.md - - common Rocq entry points are `lean-beam ensure rocq`, `lean-beam doctor rocq`, `lean-beam rocq-goals-after`, and `lean-beam rocq-goals-prev` + - start a Rocq session with `lean-beam ensure rocq --hold`; plain `ensure rocq` only checks and warms that owned session + - other common Rocq entry points are `lean-beam doctor rocq`, `lean-beam rocq-goals-after`, and `lean-beam rocq-goals-prev` - run `lean-beam update ` first, then pass its returned `version` to Lean position/range probes - run `lean-beam sync ` when you also need the diagnostics/readiness barrier - for multiline text-carrying Lean probes, prefer `--stdin` or `--text-file `; use `--` before text that starts with `--` diff --git a/tests/lean/BeamTest/Broker/CliDaemonTest.lean b/tests/lean/BeamTest/Broker/CliDaemonTest.lean index 93b6483f..b4b8af4f 100644 --- a/tests/lean/BeamTest/Broker/CliDaemonTest.lean +++ b/tests/lean/BeamTest/Broker/CliDaemonTest.lean @@ -97,7 +97,12 @@ private def readSingleDaemonFailureIncidentJson (root : System.FilePath) : IO Js private def closeAcceptedConnection (listener : Beam.Broker.Transport.Listener) : IO Unit := do let conn ← Beam.Broker.Transport.accept listener - Beam.Broker.Transport.closeConnection conn + try + -- Read the complete request before closing so the client failure is deterministically on the + -- receive boundary. The operating system may still describe that close as EOF or ECONNRESET. + discard <| Beam.Broker.Transport.recvMsg conn + finally + Beam.Broker.Transport.closeConnection conn private def holdAcceptedConnection (listener : Beam.Broker.Transport.Listener) @@ -743,7 +748,7 @@ private def checkBrokerConnectionClosedIncident : IO Unit := do let msg ← expectIoErrorMessage "broker connection close should surface daemon failure" <| Beam.Cli.callBrokerQuiet root (projectDaemonClientForTest endpoint) { op := .stats } requireSubstring "broker connection close should preserve transport failure" - "Beam daemon connection closed" msg + "Beam daemon receive failed:" msg requireSubstring "broker connection close should include incident path" "Beam daemon incident:" msg @@ -751,7 +756,7 @@ private def checkBrokerConnectionClosedIncident : IO Unit := do requireJsonString "broker close incident should classify the typed transport failure" "kind" "brokerTransportFailure" incidentJson requireJsonStringContains "broker close incident should keep transport detail" - "detail" "Beam daemon connection closed" incidentJson + "detail" "Beam daemon receive failed:" incidentJson requireJsonString "broker close incident should include endpoint summary" "registryEndpoint" (Beam.Daemon.endpointSummary endpoint) incidentJson finally diff --git a/tests/test-beam-prune.sh b/tests/test-beam-prune.sh index fe9eaf17..3530cd8b 100644 --- a/tests/test-beam-prune.sh +++ b/tests/test-beam-prune.sh @@ -267,7 +267,9 @@ if "$install_root/current/bin/lean-beam" prune --apply > /dev/null 2>"$install_l echo "expected prune to respect the active install lock" >&2 exit 1 fi -assert_contains_literal "$install_lock_err" 'timed out after 1000 ms waiting for Beam lock' +assert_contains_literal "$install_lock_err" 'timed out after ' +assert_contains_literal "$install_lock_err" ' ms waiting for Beam lock' +assert_contains_literal "$install_lock_err" 'timeout: 1000 ms' rm -f "$install_root/.install-lock/pid" "$install_root/.install-lock/pid-domain" rmdir "$install_root/.install-lock" assert_file "$old_runtime/manifest.json" @@ -313,7 +315,9 @@ if "$install_root/current/bin/lean-beam" prune --apply --bundles \ exit 1 fi assert_contains_literal "$bundle_lock_out" "removed runtime: $resolved_partial_runtime" -assert_contains_literal "$bundle_lock_err" 'timed out after 1000 ms waiting for Beam lock' +assert_contains_literal "$bundle_lock_err" 'timed out after ' +assert_contains_literal "$bundle_lock_err" ' ms waiting for Beam lock' +assert_contains_literal "$bundle_lock_err" 'timeout: 1000 ms' assert_contains_literal "$bundle_lock_err" \ 'prune stopped before completing the displayed plan; any removals reported above were applied' # shellcheck disable=SC2016 From 3c135b0239f8160826ea784863ff2e07171cf0a7 Mon Sep 17 00:00:00 2001 From: Emilio Jesus Gallego Arias Date: Thu, 27 Aug 2026 20:19:04 +0200 Subject: [PATCH 20/28] refactor: tighten daemon and MCP resource scopes --- Beam/Broker/Server.lean | 39 ++++++++++++++++++++++----------------- Beam/Mcp/StdioServer.lean | 22 ++++++++++++++-------- tests/test-beam-prune.sh | 15 +++++++++------ 3 files changed, 45 insertions(+), 31 deletions(-) diff --git a/Beam/Broker/Server.lean b/Beam/Broker/Server.lean index 52701649..002d3318 100644 --- a/Beam/Broker/Server.lean +++ b/Beam/Broker/Server.lean @@ -2697,17 +2697,11 @@ private structure DaemonResources where private def closeDaemonParts (runtime : ServerRuntime) - (transport? : Option DaemonTransport) + (transport : DaemonTransport) (rootWatcher? ownerWatcher? : Option DaemonWatcherTask) : IO Unit := do - let firstError? ← - match transport? with - | none => pure none - | some transport => recordFirstCleanupError none <| transport.stop.set true - let firstError? ← - match transport? with - | none => pure firstError? - | some transport => - recordFirstCleanupError firstError? <| Transport.closeListener transport.listener + let firstError? ← recordFirstCleanupError none <| transport.stop.set true + let firstError? ← recordFirstCleanupError firstError? <| + Transport.closeListener transport.listener let firstError? ← match ownerWatcher? with | none => pure firstError? @@ -2728,7 +2722,7 @@ private def closeDaemonParts throw err private def DaemonResources.close (resources : DaemonResources) : IO Unit := - closeDaemonParts resources.runtime (some resources.transport) (some resources.rootWatcher) + closeDaemonParts resources.runtime resources.transport (some resources.rootWatcher) resources.ownerWatcher? private def throwAfterBestEffortCleanup @@ -2756,7 +2750,7 @@ private def acquireDaemonResources try IO.asTask (prio := Task.Priority.dedicated) <| watchRoot runtime transport root catch err => - throwAfterBestEffortCleanup err <| closeDaemonParts runtime (some transport) none none + throwAfterBestEffortCleanup err <| closeDaemonParts runtime transport none none let ownerWatcher? ← try if opts.sessionOwnerStdin then @@ -2766,9 +2760,23 @@ private def acquireDaemonResources pure none catch err => throwAfterBestEffortCleanup err <| - closeDaemonParts runtime (some transport) (some rootWatcher) none + closeDaemonParts runtime transport (some rootWatcher) none pure { runtime, transport, rootWatcher, ownerWatcher? } +/-- Acquire the daemon runtime, listener, and watcher tasks for exactly the dynamic extent of `act`. -/ +private def withDaemonResources + (opts : CliOptions) + (config : BrokerConfig) + (workspaceId : WorkspaceId) + (daemonIdentity? : Option DaemonIdentity) + (root : System.FilePath) + (act : DaemonResources → IO α) : IO α := do + let resources ← acquireDaemonResources opts config workspaceId daemonIdentity? root + try + act resources + finally + resources.close + def main (args : List String) : IO Unit := do let opts ← IO.ofExcept <| parseCliOptions {} args let some root := opts.root? @@ -2796,10 +2804,7 @@ def main (args : List String) : IO Unit := do leanPlugin? := leanPlugin? rocqCmd? := opts.rocqCmd? } - let resources ← acquireDaemonResources opts config workspaceId daemonIdentity? root - try + withDaemonResources opts config workspaceId daemonIdentity? root fun resources => acceptLoop resources.runtime resources.transport - finally - resources.close end Beam.Broker diff --git a/Beam/Mcp/StdioServer.lean b/Beam/Mcp/StdioServer.lean index 653b9fac..e38d5f9c 100644 --- a/Beam/Mcp/StdioServer.lean +++ b/Beam/Mcp/StdioServer.lean @@ -64,9 +64,16 @@ private structure OutputSink where private def OutputSink.create : BaseIO OutputSink := do pure { mutex := ← Std.Mutex.new () } -private def OutputSink.send (sink : OutputSink) (json : Json) : IO Unit := do +private def OutputSink.sendWhen + (sink : OutputSink) + (condition : IO Bool) + (json : Json) : IO Unit := do sink.mutex.atomically do - writeJsonLine json + if ← condition then + writeJsonLine json + +private def OutputSink.send (sink : OutputSink) (json : Json) : IO Unit := do + sink.sendWhen (pure true) json private inductive RequestPhase where | active @@ -106,11 +113,12 @@ private inductive RequestRegistrationError where /- Nested coordinator locks flow in one direction: -* progress → request → output for request notifications -* request → output for active request messages +* progress → output → request for request notifications +* output → request for active request messages Routing is released before runtime control, request, or output is acquired. Runtime control is owned -by `ServerState` and does not acquire coordinator locks. Output acquires no coordinator lock. +by `ServerState` and does not acquire coordinator locks. Request state is inspected only after output +serialization is available, so cancellation never waits for a blocked stdout write. -/ private structure Coordinator where state : ServerState @@ -225,9 +233,7 @@ private def InFlightRequest.sendIfActive (request : InFlightRequest) (output : OutputSink) (json : Json) : IO Unit := do - request.state.atomically do - if (← get).phase == .active then - output.send json + output.sendWhen request.isActive json private def InFlightRequest.bindBrokerRequest (request : InFlightRequest) diff --git a/tests/test-beam-prune.sh b/tests/test-beam-prune.sh index 3530cd8b..28d3eb42 100644 --- a/tests/test-beam-prune.sh +++ b/tests/test-beam-prune.sh @@ -73,6 +73,13 @@ write_lock_owner() { printf '%s\n' "$test_pid_domain" >"$lock_dir/pid-domain" } +assert_lock_timeout() { + local path="$1" + assert_contains_literal "$path" 'timed out after ' + assert_contains_literal "$path" ' ms waiting for Beam lock' + assert_contains_literal "$path" 'timeout: 1000 ms' +} + mkdir -p \ "$current_runtime/bin" \ "$current_runtime/libexec" \ @@ -267,9 +274,7 @@ if "$install_root/current/bin/lean-beam" prune --apply > /dev/null 2>"$install_l echo "expected prune to respect the active install lock" >&2 exit 1 fi -assert_contains_literal "$install_lock_err" 'timed out after ' -assert_contains_literal "$install_lock_err" ' ms waiting for Beam lock' -assert_contains_literal "$install_lock_err" 'timeout: 1000 ms' +assert_lock_timeout "$install_lock_err" rm -f "$install_root/.install-lock/pid" "$install_root/.install-lock/pid-domain" rmdir "$install_root/.install-lock" assert_file "$old_runtime/manifest.json" @@ -315,9 +320,7 @@ if "$install_root/current/bin/lean-beam" prune --apply --bundles \ exit 1 fi assert_contains_literal "$bundle_lock_out" "removed runtime: $resolved_partial_runtime" -assert_contains_literal "$bundle_lock_err" 'timed out after ' -assert_contains_literal "$bundle_lock_err" ' ms waiting for Beam lock' -assert_contains_literal "$bundle_lock_err" 'timeout: 1000 ms' +assert_lock_timeout "$bundle_lock_err" assert_contains_literal "$bundle_lock_err" \ 'prune stopped before completing the displayed plan; any removals reported above were applied' # shellcheck disable=SC2016 From dfe38535b1c6c9503b027f3331ffd59389ceaebf Mon Sep 17 00:00:00 2001 From: Emilio Jesus Gallego Arias Date: Thu, 27 Aug 2026 22:37:51 +0200 Subject: [PATCH 21/28] refactor: enforce explicit wrapper daemon ownership --- Beam/Broker/Protocol.lean | 8 +- Beam/Broker/Server.lean | 84 +++- Beam/Broker/Transport.lean | 19 +- Beam/Cli/Broker.lean | 27 +- Beam/Cli/DaemonManager.lean | 469 ++++++++++++------ Beam/Cli/Feedback.lean | 25 +- Beam/Cli/Info.lean | 28 +- Beam/Cli/InstallPrune.lean | 55 +- Beam/Cli/Lock.lean | 163 +++--- Beam/Daemon/Debug.lean | 80 ++- Beam/Daemon/Protocol.lean | 32 +- Beam/System.lean | 11 - docs/COMPATIBILITY.md | 4 + docs/DEVELOPMENT.md | 84 ++-- docs/SETUP.md | 15 +- docs/STATUS.md | 16 +- docs/SYNC_AND_DIAGNOSTICS.md | 13 +- docs/TESTING.md | 21 +- tests/lean/BeamTest/Broker/CliDaemonTest.lean | 167 ++++--- tests/lean/BeamTest/Broker/ProtocolTest.lean | 39 ++ .../Broker/RequestStreamContractTest.lean | 8 +- tests/test-beam-prune.sh | 29 +- tests/test-beam-wrapper-daemon.sh | 237 ++++++++- tests/test-beam-wrapper-sandbox.sh | 20 +- 24 files changed, 1142 insertions(+), 512 deletions(-) diff --git a/Beam/Broker/Protocol.lean b/Beam/Broker/Protocol.lean index 40b80bc4..ef35a74c 100644 --- a/Beam/Broker/Protocol.lean +++ b/Beam/Broker/Protocol.lean @@ -213,6 +213,7 @@ structure Request where workspaceId? : Option WorkspaceId := none workspaceMode? : Option Beam.Workspace.InitMode := none clientRequestId? : Option String := none + daemonCapability? : Option String := none cancelRequestId? : Option String := none root? : Option String := none path? : Option String := none @@ -265,7 +266,7 @@ def Op.tracksActiveRequest : Op → Bool | .listWorkspaces | .dropWorkspace | .stats | .resetStats => true private def Op.optionalRequestFields (op : Op) : Array String := - #["clientRequestId"] ++ + #["clientRequestId", "daemonCapability"] ++ (match op.workspaceScope with | .none => #[] | .optional | .required => #["workspaceId"]) ++ @@ -320,6 +321,7 @@ private def Request.optionalJsonFields (req : Request) : List (String × Json) : optionalJsonField "workspaceId" req.workspaceId? ++ optionalJsonField "workspaceMode" req.workspaceMode? ++ optionalJsonField "clientRequestId" req.clientRequestId? ++ + optionalJsonField "daemonCapability" req.daemonCapability? ++ optionalJsonField "cancelRequestId" req.cancelRequestId? ++ optionalJsonField "root" req.root? ++ optionalJsonField "path" req.path? ++ @@ -397,6 +399,7 @@ instance : FromJson Request where let workspaceId? ← optionalField? (α := WorkspaceId) j "workspaceId" let workspaceMode? ← optionalField? (α := Beam.Workspace.InitMode) j "workspaceMode" let clientRequestId? ← optionalField? (α := String) j "clientRequestId" + let daemonCapability? ← optionalField? (α := String) j "daemonCapability" let cancelRequestId? ← optionalField? (α := String) j "cancelRequestId" let root? ← optionalField? (α := String) j "root" let path? ← optionalField? (α := String) j "path" @@ -424,7 +427,8 @@ instance : FromJson Request where let handle? ← optionalField? (α := Handle) j "handle" let codeAction? ← optionalField? (α := Lsp.CodeAction) j "codeAction" let request : Request := { - op, backend, workspaceId?, workspaceMode?, clientRequestId?, cancelRequestId?, + op, backend, workspaceId?, workspaceMode?, clientRequestId?, daemonCapability?, + cancelRequestId?, root?, path?, version?, line?, character?, endLine?, endCharacter?, text?, query?, includeDeclaration?, kinds?, suggest?, storeHandle?, linear?, mode?, compact?, ppFormat?, diagnosticScope?, diagnosticsInResult?, diff --git a/Beam/Broker/Server.lean b/Beam/Broker/Server.lean index 002d3318..bb7e3c06 100644 --- a/Beam/Broker/Server.lean +++ b/Beam/Broker/Server.lean @@ -937,6 +937,7 @@ private def modifyCurrentSessionIfMatching structure ServerRuntime where state : Std.Mutex State daemonIdentity? : Option DaemonIdentity + private daemonCapability? : Option String activeRequests : ActiveRequestRegistry private closeMutex : Std.Mutex Bool private closeDone : IO.Promise (Except IO.Error Unit) @@ -978,7 +979,8 @@ private def ServerRuntime.statsResponse def ServerRuntime.create (config : BrokerConfig) (workspaceId : WorkspaceId) - (daemonIdentity? : Option DaemonIdentity := none) : IO ServerRuntime := do + (daemonIdentity? : Option DaemonIdentity := none) + (daemonCapability? : Option String := none) : IO ServerRuntime := do unless validWorkspaceId workspaceId do throw <| IO.userError "workspace id must be non-empty" let startMonoNanos ← IO.monoNanosNow @@ -986,6 +988,7 @@ def ServerRuntime.create pure { state := ← Std.Mutex.new state daemonIdentity? + daemonCapability? activeRequests := ← ActiveRequestRegistry.create closeMutex := ← Std.Mutex.new false closeDone := ← IO.Promise.new @@ -1287,15 +1290,38 @@ private def propagatePendingCancellation (cancelRef? : Option (IO.Ref Bool)) : IO Unit := do PendingRequestStore.propagateCancellation session.pending session.stdin cancelRef? +private structure ClientPermits where + available : Std.Mutex Nat + +private def ClientPermits.create (count : Nat) : BaseIO ClientPermits := do + pure { available := ← Std.Mutex.new count } + +private def ClientPermits.tryAcquire (permits : ClientPermits) : BaseIO Bool := do + permits.available.atomically do + let available ← get + if available == 0 then + pure false + else + set (available - 1) + pure true + +private def ClientPermits.release (permits : ClientPermits) : BaseIO Unit := do + permits.available.atomically do + modify (· + 1) + private structure DaemonTransport where endpoint : Transport.Endpoint listener : Transport.Listener stop : IO.Ref Bool + clientPermits : ClientPermits + +private def maxDaemonClients : Nat := + 64 private def DaemonTransport.create (endpoint : Transport.Endpoint) : IO DaemonTransport := do let stop ← IO.mkRef false let listener ← Transport.bindAndListen endpoint 16 - pure { endpoint, listener, stop } + pure { endpoint, listener, stop, clientPermits := ← ClientPermits.create maxDaemonClients } private def requestStop (transport : DaemonTransport) : IO Unit := do transport.stop.set true @@ -2446,6 +2472,16 @@ private def ServerRuntime.withRequestAdmission let startedAt ← IO.monoNanosNow traceBroker s!"dispatch start op={req.op.key} clientRequestId={optionLabel req.clientRequestId?}" + if let some expected := server.daemonCapability? then + unless req.daemonCapability? == some expected do + let resp := errorResponseFor .invalidParams "invalid Beam daemon capability" + recordDispatchMetrics server req resp startedAt + return resp + if req.op == .initWorkspace || req.op == .dropWorkspace then + let resp := errorResponseFor .invalidParams + s!"broker op '{req.op.key}' is unavailable in wrapper-owned daemon mode" + recordDispatchMetrics server req resp startedAt + return resp match req.validateFields with | .error err => let resp := errorResponseFor .invalidParams err @@ -2570,7 +2606,10 @@ private def handleClient (toJson (StreamMessage.response clientRequestId? resp)).compress terminalSentRef.set true try - let msg ← Transport.recvMsg client + let initialRequestTimeoutMs := 5000 + let deadlineNanos := (← IO.monoNanosNow) + initialRequestTimeoutMs * 1000000 + let some msg ← Transport.recvMsgUntil client deadlineNanos + | throw <| IO.userError s!"Beam daemon initial request timed out after {initialRequestTimeoutMs} ms" let request : Except ResponseFailure Request ← match Json.parse msg with | .error err => @@ -2631,13 +2670,21 @@ private partial def acceptLoop else let client ← Transport.accept transport.listener if ← transport.stop.get then - pure () + Transport.closeConnection client else - let _ ← IO.asTask (prio := Task.Priority.dedicated) do - try - handleClient server transport client - catch e => - IO.eprintln s!"broker client task failed: {e.toString}" + if ← transport.clientPermits.tryAcquire then + let serve := do + try + handleClient server transport client + catch e => + IO.eprintln s!"broker client task failed: {e.toString}" + let _ ← IO.asTask (prio := Task.Priority.dedicated) do + try + serve + finally + transport.clientPermits.release + else + Transport.closeConnection client acceptLoop server transport private structure CliOptions where @@ -2739,8 +2786,9 @@ private def acquireDaemonResources (config : BrokerConfig) (workspaceId : WorkspaceId) (daemonIdentity? : Option DaemonIdentity) + (daemonCapability? : Option String) (root : System.FilePath) : IO DaemonResources := do - let runtime ← ServerRuntime.create config workspaceId daemonIdentity? + let runtime ← ServerRuntime.create config workspaceId daemonIdentity? daemonCapability? let transport ← try DaemonTransport.create opts.endpoint @@ -2769,9 +2817,10 @@ private def withDaemonResources (config : BrokerConfig) (workspaceId : WorkspaceId) (daemonIdentity? : Option DaemonIdentity) + (daemonCapability? : Option String) (root : System.FilePath) (act : DaemonResources → IO α) : IO α := do - let resources ← acquireDaemonResources opts config workspaceId daemonIdentity? root + let resources ← acquireDaemonResources opts config workspaceId daemonIdentity? daemonCapability? root try act resources finally @@ -2796,6 +2845,17 @@ def main (args : List String) : IO Unit := do throw <| IO.userError "--daemon-id requires --config-hash" | none, some _ => throw <| IO.userError "--config-hash requires --daemon-id" + let daemonCapability? ← + if opts.sessionOwnerStdin then + let capability := (← (← IO.getStdin).getLine).trimAscii.toString + if capability.isEmpty then + throw <| IO.userError "wrapper-owned Beam daemon received an empty capability" + pure <| some capability + else + pure none + if opts.sessionOwnerStdin && daemonIdentity?.isNone then + throw <| IO.userError + "wrapper-owned Beam daemon identity and stdin capability must be supplied together" let root ← Beam.resolveExistingPath <| System.FilePath.mk root let leanPlugin? ← opts.leanPlugin?.mapM (fun path => Beam.resolveExistingPath <| System.FilePath.mk path) let config : BrokerConfig := { @@ -2804,7 +2864,7 @@ def main (args : List String) : IO Unit := do leanPlugin? := leanPlugin? rocqCmd? := opts.rocqCmd? } - withDaemonResources opts config workspaceId daemonIdentity? root fun resources => + withDaemonResources opts config workspaceId daemonIdentity? daemonCapability? root fun resources => acceptLoop resources.runtime resources.transport end Beam.Broker diff --git a/Beam/Broker/Transport.lean b/Beam/Broker/Transport.lean index 5ddfa3bc..fff17e7a 100644 --- a/Beam/Broker/Transport.lean +++ b/Beam/Broker/Transport.lean @@ -17,6 +17,12 @@ namespace Beam.Broker.Transport open Std.Net open Std.Internal.UV +def maxFrameBytes : Nat := + 16 * 1024 * 1024 + +private def maxFrameHeaderBytes : Nat := + 20 + inductive Endpoint where | tcp (port : UInt16) deriving Repr, BEq @@ -111,6 +117,8 @@ def closeListener (listener : Listener) : IO Unit := do private def sendMsgTcp (client : TCP.Socket) (msg : String) : IO Unit := do let bytes := msg.toUTF8 + if bytes.size > maxFrameBytes then + throw <| IO.userError s!"Beam daemon frame exceeds {maxFrameBytes} bytes" let header := s!"{bytes.size}\n".toUTF8 let promise ← TCP.Socket.send client #[header, bytes] waitTcpPromise promise "Beam daemon connection closed before TCP send completed" @@ -135,19 +143,28 @@ private def recvMsgTcpUsing | .timedOut => return none | .completed none => throw <| IO.userError "Beam daemon connection closed" | .completed (some chunk) => + if chunk.isEmpty then + throw <| IO.userError "Beam daemon received an empty header chunk" if chunk[0]! == '\n'.toUInt8 then break + if header.size >= maxFrameHeaderBytes then + throw <| IO.userError "Beam daemon frame header is too long" header := header ++ chunk let some lenStr := String.fromUTF8? header | throw <| IO.userError "invalid Beam daemon header" let some len := lenStr.toNat? | throw <| IO.userError "invalid Beam daemon length" + if len > maxFrameBytes then + throw <| IO.userError s!"Beam daemon frame exceeds {maxFrameBytes} bytes" let mut payload := ByteArray.empty while payload.size < len do match ← receive (len - payload.size).toUInt64 with | .timedOut => return none | .completed none => throw <| IO.userError "Beam daemon connection closed" - | .completed (some chunk) => payload := payload ++ chunk + | .completed (some chunk) => + if chunk.isEmpty then + throw <| IO.userError "Beam daemon received an empty payload chunk" + payload := payload ++ chunk let some msg := String.fromUTF8? payload | throw <| IO.userError "invalid Beam daemon UTF-8" pure (some msg) diff --git a/Beam/Cli/Broker.lean b/Beam/Cli/Broker.lean index 9cadd173..2e20af90 100644 --- a/Beam/Cli/Broker.lean +++ b/Beam/Cli/Broker.lean @@ -117,29 +117,32 @@ private def withWrapperClientRequestId (req : Request) : IO WrapperBrokerRequest } private def prepareWrapperBrokerRequest - (req : Request) : IO WrapperBrokerRequest := - withWrapperClientRequestId <| inProjectDaemonWorkspace req + (client : ProjectDaemonClient) + (req : Request) : IO WrapperBrokerRequest := do + let wrapper ← withWrapperClientRequestId <| inProjectDaemonWorkspace req + pure { wrapper with request := client.authorize wrapper.request } def decodeCancelAcknowledged? (resp : Response) : Option Bool := do let result ← resp.result? result.getObjValAs? Bool "cancelled" |>.toOption private def sendBrokerCancellation - (endpoint : Transport.Endpoint) + (client : ProjectDaemonClient) (clientRequestId : String) : IO (Option Bool) := do let cancelReq : Request := { op := .cancel cancelRequestId? := some clientRequestId } try - let resp ← sendRequest endpoint (← withEnvClientRequestId cancelReq) + let req ← withEnvClientRequestId cancelReq + let resp ← sendRequest client.endpoint (client.authorize req) pure <| decodeCancelAcknowledged? resp catch _ => pure none private def awaitBrokerResponse (task : Task (Except IO.Error (Except BrokerClientFailure Response))) - (endpoint : Transport.Endpoint) + (client : ProjectDaemonClient) (clientRequestId : String) (visibleClientRequestId? : Option String) (progressSpec? : Option BrokerWaitSpec) @@ -160,7 +163,7 @@ private def awaitBrokerResponse -- SIGINT can arrive after the wrapper starts the request task but before the broker -- has registered the client request id as active. Retry until the broker acknowledges -- cancellation or the original request finishes. - match ← sendBrokerCancellation endpoint clientRequestId with + match ← sendBrokerCancellation client clientRequestId with | some true => cancelAcknowledged := true | some false | none => pure () IO.sleep 500 @@ -182,7 +185,7 @@ private def awaitBrokerResponse pure <| .error failure private def awaitBrokerResponseWithInterrupts - (endpoint : Transport.Endpoint) + (client : ProjectDaemonClient) (clientRequestId : String) (visibleClientRequestId? : Option String) (progressSpec? : Option BrokerWaitSpec) @@ -192,7 +195,7 @@ private def awaitBrokerResponseWithInterrupts -- gives SIGINT cancellation a stable broker key but is kept out of the CLI's public output. withInterruptWatcher fun interruptWatcher => do let task ← IO.asTask (prio := Task.Priority.dedicated) action - awaitBrokerResponse task endpoint clientRequestId visibleClientRequestId? progressSpec? + awaitBrokerResponse task client clientRequestId visibleClientRequestId? progressSpec? interruptWatcher private structure WrapperBrokerResponse where @@ -203,10 +206,10 @@ private def requestBrokerResponse (root : System.FilePath) (client : ProjectDaemonClient) (req : Request) : IO WrapperBrokerResponse := do - let wrapperReq ← prepareWrapperBrokerRequest req + let wrapperReq ← prepareWrapperBrokerRequest client req let req := wrapperReq.request let response ← withBrokerErrorContext root do - awaitBrokerResponseWithInterrupts client.endpoint wrapperReq.clientRequestId + awaitBrokerResponseWithInterrupts client wrapperReq.clientRequestId wrapperReq.visibleClientRequestId? none <| sendRequestWithCallbacksResult client.endpoint req pure { response, visibleClientRequestId? := wrapperReq.visibleClientRequestId? } @@ -433,7 +436,7 @@ def callBrokerWithProgress (client : ProjectDaemonClient) (req : Request) (spec : BrokerWaitSpec) : IO Unit := do - let wrapperReq ← prepareWrapperBrokerRequest req + let wrapperReq ← prepareWrapperBrokerRequest client req let req := wrapperReq.request let visibleClientRequestId? := wrapperReq.visibleClientRequestId? let showProgress ← progressEnabled @@ -446,7 +449,7 @@ def callBrokerWithProgress } let progressSpec? := if showProgress then some spec else none let resp ← withBrokerErrorContext root do - awaitBrokerResponseWithInterrupts client.endpoint wrapperReq.clientRequestId + awaitBrokerResponseWithInterrupts client wrapperReq.clientRequestId visibleClientRequestId? progressSpec? <| sendRequestWithCallbacksResult client.endpoint req callbacks match responseErrorSummary? spec.action spec.failureBoundary resp with diff --git a/Beam/Cli/DaemonManager.lean b/Beam/Cli/DaemonManager.lean index f1d7565d..86520973 100644 --- a/Beam/Cli/DaemonManager.lean +++ b/Beam/Cli/DaemonManager.lean @@ -40,9 +40,6 @@ private def projectControlLockTimeoutMs : IO Nat := do "invalid BEAM_CONTROL_LOCK_TIMEOUT_MS value '0': expected a positive timeout" pure timeoutMs -private def projectControlLockDir (root : System.FilePath) : IO System.FilePath := do - pure ((← controlDir root) / "lock") - /-- Run `act` while holding the per-project daemon control lock. @@ -52,13 +49,37 @@ unbounded lock helper. -/ private structure ProjectControl where root : System.FilePath + dir : System.FilePath + registry : System.FilePath + +private def projectControl (root : System.FilePath) : IO ProjectControl := do + let dir ← controlDir root + pure { root, dir, registry := dir / "beam-daemon.json" } /-- Supply project registry mutation only for the dynamic extent of the project control lock. -/ private def withProjectControl (root : System.FilePath) (act : ProjectControl → IO α) : IO α := do - withLockTimeout (← projectControlLockDir root) (← projectControlLockTimeoutMs) do - act { root } + let control ← projectControl root + withLockTimeout (control.dir / "lock") (← projectControlLockTimeoutMs) do + act control + +/-- +Run teardown under the project lock without recreating a control directory that disappeared with +its project root. +-/ +private def withExistingProjectControl + (root : System.FilePath) + (act : ProjectControl → IO Unit) : IO Unit := do + let control ← projectControl root + unless ← control.dir.isDir do + return + try + withExistingLockTimeout (control.dir / "lock") (← projectControlLockTimeoutMs) do + act control + catch + | .noFileOrDirectory .. => pure () + | err => throw err private def computeConfigHash (root : System.FilePath) @@ -78,25 +99,39 @@ private def computeConfigHash s!"{acc.toNat}" private def writeRegistry (control : ProjectControl) (entry : RegistryEntry) : IO Unit := do - let path ← registryPath control.root - if let some parent := path.parent then + if let some parent := control.registry.parent then IO.FS.createDirAll parent - let tmp := path.withExtension "tmp" + let tmp := control.registry.withExtension "tmp" IO.FS.writeFile tmp ((toJson entry).pretty ++ "\n") - IO.FS.rename tmp path + IO.setAccessRights tmp { + user := { read := true, write := true } + } + IO.FS.rename tmp control.registry + +private def writeExistingRegistry (control : ProjectControl) (entry : RegistryEntry) : IO Unit := do + -- Teardown must not create a path while the project tree is being removed. Rewrite through an + -- already existing file handle; if the registry was concurrently unlinked, this updates only the + -- unlinked inode and cannot recreate the project or control directory. + let handle ← IO.FS.Handle.mk control.registry .readWrite + handle.rewind + handle.putStr ((toJson entry).pretty ++ "\n") + handle.flush + handle.truncate private def removeRegistry (control : ProjectControl) : IO Unit := do - let path ← registryPath control.root - if ← path.pathExists then - IO.FS.removeFile path + if ← control.registry.pathExists then + IO.FS.removeFile control.registry + +private def sameRegistryGeneration (left right : RegistryEntry) : Bool := + left.daemonId == right.daemonId && left.capability == right.capability /-- Remove a registry entry only when it still names the observed daemon generation. -/ -private def removeRegistryGeneration (control : ProjectControl) (daemonId : String) : IO Unit := do - match ← readRegistry? control.root with - | some current => - if current.daemonId == daemonId then +private def removeRegistryGeneration (control : ProjectControl) (entry : RegistryEntry) : IO Unit := do + match ← readRegistry control.root with + | .current current => + if sameRegistryGeneration current entry then removeRegistry control - | none => pure () + | .absent | .legacy | .unsupported _ | .malformed _ => pure () private def daemonShutdownResponseTimeoutMs : Nat := 30000 @@ -104,71 +139,97 @@ private def daemonShutdownResponseTimeoutMs : Nat := /-- Ask a daemon to shut down without allowing its response stream to hold CLI control forever. -/ def requestDaemonShutdown (endpoint : Transport.Endpoint) + (capability : String) (responseTimeoutMs : Nat := daemonShutdownResponseTimeoutMs) : IO (Except BrokerClientFailure Response) := do - sendRequestWithStreamTimeoutResult endpoint { op := .shutdown } + sendRequestWithStreamTimeoutResult endpoint { + op := .shutdown + daemonCapability? := some capability + } responseTimeoutMs (fun _ => pure ()) -private partial def waitForRecordedPidGone - (recorded : Beam.RecordedPid) - (tries : Nat := 20) : IO Unit := do - if tries == 0 then - return - match ← recorded.observe with - | .local true => - IO.sleep 100 - waitForRecordedPidGone recorded (tries - 1) - | .invalid | .local false | .differentDomain | .unknownDomain => - pure () - -private def gracefulDaemonShutdownWaitTries : Nat := - 50 +inductive RegistryUnsafeReason where + | invalidIdentity + | wrongRegistryRoot (recordedRoot : String) + | invalidEndpoint + | ownerDead + | endpointUnavailable + | endpointUnrecognized (detail : String) + | wrongEndpointRoot (daemonRoot : String) + | wrongGeneration (daemonRoot : String) + deriving BEq, Repr + +inductive RegistryObservation where + | absent + | legacy + | unsupported (schemaVersion : Nat) + | malformed (detail : String) + | liveExact (entry : RegistryEntry) + | liveConfigMismatch (entry : RegistryEntry) (expectedHash : String) + | draining (entry : RegistryEntry) + | staleConfirmed (entry : RegistryEntry) + | unusable (entry : RegistryEntry) (reason : RegistryUnsafeReason) + +private def recordedPidGone (pid : Nat) (domain? : Option String) : IO Bool := do + match ← (Beam.RecordedPid.mk pid domain?).observe with + | .local false => pure true + | .invalid | .local true | .differentDomain | .unknownDomain => pure false + +private def registryProcessesGone (entry : RegistryEntry) : IO Bool := do + if !(← recordedPidGone entry.ownerPid entry.ownerPidDomain?) then + return false + recordedPidGone entry.pid entry.pidDomain? -/-- -Finish a graceful daemon shutdown with a PID fallback only when the registry PID belongs to the -current process domain. A PID from another or unknown domain must never be probed or killed. --/ -def finishRegistryDaemonShutdown (entry : RegistryEntry) : IO Unit := do - let recorded : Beam.RecordedPid := { pid := entry.pid, domain? := entry.pidDomain? } - -- A broker may spend up to three seconds completing the bounded LSP shutdown path before its - -- accept loop exits. Do not turn that orderly session close into SIGTERM just before it finishes. - waitForRecordedPidGone recorded gracefulDaemonShutdownWaitTries - match ← recorded.observe with - | .local true => - if ← recorded.terminateIfLocal then - waitForRecordedPidGone recorded - | .invalid | .local false | .differentDomain | .unknownDomain => - pure () +private def registryOwnerKnownDead (entry : RegistryEntry) : IO Bool := + recordedPidGone entry.ownerPid entry.ownerPidDomain? -private def stopDaemonEntry (control : ProjectControl) (entry : RegistryEntry) : IO Unit := do - let releaseGeneration := removeRegistryGeneration control entry.daemonId - let releaseAndFinish := do - try - releaseGeneration - finally - finishRegistryDaemonShutdown entry - match registryEndpoint? entry with - | none => - releaseAndFinish - | some endpoint => - match ← daemonGenerationStatus endpoint projectDaemonWorkspaceId - control.root entry.identity with +def observeProjectRegistry + (root : System.FilePath) + (expectedHash? : Option String := none) : IO RegistryObservation := do + match ← readRegistry root with + | .absent => pure .absent + | .legacy => pure .legacy + | .unsupported schemaVersion => pure <| .unsupported schemaVersion + | .malformed detail => pure <| .malformed detail + | .current entry => + if entry.daemonId.isEmpty || entry.capability.isEmpty then + return .unusable entry .invalidIdentity + unless ← Beam.sameFilePath (System.FilePath.mk entry.root) root do + return .unusable entry (.wrongRegistryRoot entry.root) + if entry.lifecycle == .draining then + if ← registryProcessesGone entry then + return .staleConfirmed entry + return .draining entry + let ownerDead ← registryOwnerKnownDead entry + let some endpoint := registryEndpoint? entry + | if ownerDead && (← registryProcessesGone entry) then + return .staleConfirmed entry + else + return .unusable entry .invalidEndpoint + match ← daemonGenerationStatus endpoint projectDaemonWorkspaceId root + entry.identity entry.capability with | .exact => - try - discard <| requestDaemonShutdown endpoint - finally - releaseAndFinish + if ownerDead then + pure <| .unusable entry .ownerDead + else + match expectedHash? with + | some expectedHash => + if expectedHash == entry.configHash then + pure <| .liveExact entry + else + pure <| .liveConfigMismatch entry expectedHash + | none => pure <| .liveExact entry | .unavailable => - releaseAndFinish - | .unrecognized _ | .wrongRoot _ | .wrongGeneration _ => - releaseGeneration - -private def stopRegisteredDaemon (control : ProjectControl) : IO Unit := do - match ← readRegistry? control.root with - | none => - removeRegistry control - | some entry => - stopDaemonEntry control entry + if ownerDead && (← registryProcessesGone entry) then + pure <| .staleConfirmed entry + else + pure <| .unusable entry .endpointUnavailable + | .unrecognized failure => + pure <| .unusable entry (.endpointUnrecognized failure.detail) + | .wrongRoot daemonRoot => + pure <| .unusable entry (.wrongEndpointRoot daemonRoot) + | .wrongGeneration daemonRoot => + pure <| .unusable entry (.wrongGeneration daemonRoot) private def requestedPortNat? (opts : CliOptions) : Option Nat := opts.requestedPort?.map (·.toNat) @@ -244,7 +305,7 @@ private structure DaemonFailureIncident where root : String controlDir : String registryPath : String - registry : Option RegistryEntry := none + registry : Option Json := none registryPidStatus : Option String := none registryEndpoint : Option String := none startupLogPath : Option String := none @@ -280,7 +341,8 @@ private def writeDaemonFailureIncident? let dir ← daemonFailureIncidentDir root IO.FS.createDirAll dir let registryFile ← registryPath root - let registry ← readRegistry? root + let registryRead ← readRegistry root + let registry := registryRead.entry? let pidStatus ← match registry with | none => pure none @@ -296,7 +358,8 @@ private def writeDaemonFailureIncident? root := root.toString controlDir := control.toString registryPath := registryFile.toString - registry + registry := registry.map fun entry => + (toJson entry).setObjVal! "capability" (toJson "") registryPidStatus := pidStatus registryEndpoint := endpoint startupLogPath := logTail?.map (fun (path, _) => path.toString) @@ -384,7 +447,8 @@ private def startDaemon (desired : DesiredConfig) (endpoint : Transport.Endpoint) (logPath : System.FilePath) - (identity : DaemonIdentity) : IO (IO.Process.Child daemonStdio) := do + (identity : DaemonIdentity) + (capability : String) : IO (IO.Process.Child daemonStdio) := do let mut args : List String := [ "--root", desired.root.toString, "--workspace-id", projectDaemonWorkspaceId, @@ -411,7 +475,10 @@ private def startDaemon cmd := "sh" args := #["-c", shell] cwd := some desired.root + setsid := true } + child.stdin.putStrLn capability + child.stdin.flush pure child private def daemonStartupTimeoutMs : Nat := @@ -423,6 +490,7 @@ private partial def waitForDaemonUntil (logPath : System.FilePath) (root : System.FilePath) (identity : DaemonIdentity) + (capability : String) (deadlineNanos : Nat) (timeoutDetail : String) : IO (Except DaemonStartupFailure Unit) := do if (← child.tryWait).isSome then @@ -437,8 +505,8 @@ private partial def waitForDaemonUntil .error <$> daemonStartupFailure endpoint logPath detail else IO.sleep 100 - waitForDaemonUntil child endpoint logPath root identity deadlineNanos detail - match ← daemonGenerationStatus endpoint projectDaemonWorkspaceId root identity with + waitForDaemonUntil child endpoint logPath root identity capability deadlineNanos detail + match ← daemonGenerationStatus endpoint projectDaemonWorkspaceId root identity capability with | .exact => pure (.ok ()) | .wrongRoot daemonRoot => pure <| .error { @@ -460,9 +528,10 @@ private def waitForDaemon (endpoint : Transport.Endpoint) (logPath : System.FilePath) (root : System.FilePath) - (identity : DaemonIdentity) : IO (Except DaemonStartupFailure Unit) := do + (identity : DaemonIdentity) + (capability : String) : IO (Except DaemonStartupFailure Unit) := do let deadlineNanos := (← IO.monoNanosNow) + daemonStartupTimeoutMs * 1000000 - waitForDaemonUntil child endpoint logPath root identity deadlineNanos + waitForDaemonUntil child endpoint logPath root identity capability deadlineNanos "Beam daemon did not become ready before timeout" private def newDaemonGenerationId (configHash : String) : IO String := do @@ -470,9 +539,23 @@ private def newDaemonGenerationId (configHash : String) : IO String := do let nonce := ByteArray.toUInt64LE! (← IO.getRandomBytes 8) pure s!"{configHash.take 12}-{startedMonoNanos}-{nonce}" +private def hexDigit (n : Nat) : Char := + if n < 10 then + Char.ofNat ('0'.toNat + n) + else + Char.ofNat ('a'.toNat + n - 10) + +private def byteHex (byte : UInt8) : List Char := + [hexDigit (byte.toNat / 16), hexDigit (byte.toNat % 16)] + +private def newDaemonCapability : IO String := do + let bytes ← IO.getRandomBytes 32 + pure <| String.ofList <| bytes.toList.flatMap byteHex + private def registryEntryFor (desired : DesiredConfig) (daemonId : String) + (capability : String) (pid : Nat) (endpoint : Transport.Endpoint) (opts : CliOptions) : IO RegistryEntry := do @@ -482,7 +565,10 @@ private def registryEntryFor let pidDomain? ← Beam.currentPidDomain? let ownerPid ← IO.Process.getPID pure { + schemaVersion := registrySchemaVersion + lifecycle := .live daemonId + capability pid pidDomain? ownerPid := ownerPid.toNat @@ -509,12 +595,13 @@ private partial def startDaemonEntry let logPath ← daemonStartupLogPath desired.root let daemonId ← newDaemonGenerationId desired.configHash let identity : DaemonIdentity := { daemonId, configHash := desired.configHash } - let child ← startDaemon desired endpoint logPath identity + let capability ← newDaemonCapability + let child ← startDaemon desired endpoint logPath identity capability let readiness : Except DaemonStartupFailure RegistryEntry ← try - match ← waitForDaemon child endpoint logPath desired.root identity with + match ← waitForDaemon child endpoint logPath desired.root identity capability with | .ok () => - let entry ← registryEntryFor desired daemonId child.pid.toNat endpoint opts + let entry ← registryEntryFor desired daemonId capability child.pid.toNat endpoint opts pure (.ok entry) | .error failure => pure (.error failure) @@ -585,66 +672,93 @@ def desiredConfig (home root : System.FilePath) (required : Backend) : IO Desire structure ProjectDaemonClient where endpoint : Transport.Endpoint + capability : String + +def ProjectDaemonClient.authorize + (client : ProjectDaemonClient) + (request : Request) : Request := + { request with daemonCapability? := some client.capability } private def projectDaemonClient (entry : RegistryEntry) : IO ProjectDaemonClient := do pure { endpoint := ← Beam.Daemon.endpointFromEntry entry + capability := entry.capability } -private def registryOwnerObservable (entry : RegistryEntry) : IO Bool := do - if entry.ownerPid == 0 then - return false - let recorded : Beam.RecordedPid := { - pid := entry.ownerPid - domain? := entry.ownerPidDomain? - } - match ← recorded.observe with - | .invalid | .local false => pure false - | .local true | .differentDomain | .unknownDomain => pure true - -def registryLiveFor - (root : System.FilePath) - (expectedHash? : Option String := none) : IO (Option RegistryEntry) := do - match ← readRegistry? root with - | none => pure none - | some entry => - let rootOk ← Beam.sameFilePath (System.FilePath.mk entry.root) root - let hashOk := expectedHash?.map (· == entry.configHash) |>.getD true - if !rootOk || !hashOk || !(← registryOwnerObservable entry) then - pure none - else - match registryEndpoint? entry with - | none => pure none - | some endpoint => - -- The owner pipe makes endpoint liveness authoritative across PID domains. A - -- same-domain dead owner is rejected immediately; another domain is never probed. - match ← daemonGenerationStatus endpoint projectDaemonWorkspaceId root entry.identity with - | .exact => pure (some entry) - | .unavailable | .unrecognized _ | .wrongRoot _ | .wrongGeneration _ => pure none +def RegistryUnsafeReason.message : RegistryUnsafeReason → String + | .invalidIdentity => "registry identity or capability is empty" + | .wrongRegistryRoot recordedRoot => s!"registry records another root: {recordedRoot}" + | .invalidEndpoint => "registry endpoint is invalid" + | .ownerDead => "the recorded owner is dead while its daemon still responds" + | .endpointUnavailable => "the recorded daemon endpoint is unavailable" + | .endpointUnrecognized detail => s!"the recorded endpoint is not a recognized Beam generation: {detail}" + | .wrongEndpointRoot daemonRoot => s!"the recorded endpoint serves another root: {daemonRoot}" + | .wrongGeneration daemonRoot => + s!"the recorded endpoint serves another Beam generation for {daemonRoot}" -/-- -Shut down the live wrapper daemon, if any, while holding its project control scope. +private def activeOwnerMessage (root : System.FilePath) (entry : RegistryEntry) : String := + s!"Beam session for {root} is already owned by wrapper pid {entry.ownerPid}; " ++ + "interrupt that 'lean-beam ensure --hold' process before starting another owner" -The generation is unpublished even when the bounded response fails, releasing the foreground owner -without exposing registry mutation or its lock precondition to callers. --/ +private def configMismatchMessage + (root : System.FilePath) + (entry : RegistryEntry) + (expectedHash : String) : String := + s!"the live Beam session for {root} uses configuration {entry.configHash}, " ++ + s!"but this command requires {expectedHash}; the current owner was preserved. " ++ + "Interrupt its 'lean-beam ensure --hold' process, then start a new owner with the desired configuration" + +private def drainingOwnerMessage (root : System.FilePath) (entry : RegistryEntry) : String := + s!"Beam session {entry.daemonId} for {root} is draining; " ++ + "wait for its foreground owner to exit before starting or attaching to another session" + +private def registryRecoveryMessage (root : System.FilePath) (detail : String) : String := + s!"Beam cannot safely use or replace the daemon registry for {root}: {detail}. " ++ + "Preserve the registry, stop the matching foreground owner or daemon explicitly, and retry" + +private def markRegistryDraining (control : ProjectControl) (entry : RegistryEntry) : IO Unit := do + match ← readRegistry control.root with + | .current current => + if sameRegistryGeneration current entry && current.lifecycle == .live then + writeExistingRegistry control { current with lifecycle := .draining } + | .absent | .legacy | .unsupported _ | .malformed _ => pure () + +private inductive ShutdownPlan where + | none + | request (entry : RegistryEntry) + +/-- Fence and request shutdown of the exact wrapper-owned generation without PID signalling. -/ def shutdownRegisteredProjectDaemon (root : System.FilePath) : IO (Except BrokerClientFailure (Option Response)) := do - withProjectControl root fun control => do - match ← registryLiveFor root with - | some entry => - match registryEndpoint? entry with - | some endpoint => - try - pure <| (← requestDaemonShutdown endpoint).map some - finally - removeRegistryGeneration control entry.daemonId - | none => - stopRegisteredDaemon control - pure (.ok none) - | none => - stopRegisteredDaemon control - pure (.ok none) + let plan : ShutdownPlan ← withProjectControl root fun control => do + match ← observeProjectRegistry root with + | .absent => pure ShutdownPlan.none + | .staleConfirmed entry => + removeRegistryGeneration control entry + pure ShutdownPlan.none + | .liveExact entry => + markRegistryDraining control entry + pure <| ShutdownPlan.request entry + | .draining entry => pure <| ShutdownPlan.request entry + | .liveConfigMismatch entry _ => + markRegistryDraining control entry + pure <| ShutdownPlan.request entry + | .legacy => + throw <| IO.userError <| registryRecoveryMessage root <| + RegistryRead.legacy.detail?.getD "legacy registry" + | .unsupported schemaVersion => + throw <| IO.userError <| registryRecoveryMessage root <| + (RegistryRead.unsupported schemaVersion).detail?.getD "unsupported registry" + | .malformed detail => + throw <| IO.userError <| registryRecoveryMessage root detail + | .unusable _ reason => + throw <| IO.userError <| registryRecoveryMessage root reason.message + match plan with + | ShutdownPlan.none => pure <| .ok none + | ShutdownPlan.request entry => + let some endpoint := registryEndpoint? entry + | return .error <| .invalidResponse "draining Beam registry has no valid endpoint" + pure <| (← requestDaemonShutdown endpoint entry.capability).map some private abbrev detachedDaemonStdio : IO.Process.StdioConfig where stdin := .null @@ -674,13 +788,11 @@ def ProjectDaemonOwner.exitCode? (owner : ProjectDaemonOwner) : IO (Option UInt3 /-- Whether this owner generation is still the one published for its project. -/ def ProjectDaemonOwner.registered (owner : ProjectDaemonOwner) : IO Bool := do - match ← readRegistry? owner.root with - | some current => pure (current.daemonId == owner.daemonId) - | none => pure false - -private def activeOwnerMessage (root : System.FilePath) (entry : RegistryEntry) : String := - s!"Beam session for {root} is already owned by wrapper pid {entry.ownerPid}; " ++ - "interrupt that 'lean-beam ensure --hold' process before starting another owner" + match ← readRegistry owner.root with + | .current current => + pure (current.daemonId == owner.daemonId && + current.capability == owner.client.capability && current.lifecycle == .live) + | .absent | .legacy | .unsupported _ | .malformed _ => pure false private def missingOwnerCommand : Option Backend → String | some .rocq => "lean-beam ensure rocq --hold" @@ -694,11 +806,23 @@ private def startOwnedProjectDaemon (control : ProjectControl) (desired : DesiredConfig) (opts : CliOptions) : IO OwnedProjectDaemon := do - if let some live ← registryLiveFor desired.root then - throw <| IO.userError (activeOwnerMessage desired.root live) - -- A non-live registry may refer to a daemon still winding down after owner loss. Ask that exact - -- root-matching endpoint to stop, and use PID fallback only through the typed domain boundary. - stopRegisteredDaemon control + match ← observeProjectRegistry desired.root (some desired.configHash) with + | .absent => pure () + | .staleConfirmed entry => removeRegistryGeneration control entry + | .liveExact entry => throw <| IO.userError (activeOwnerMessage desired.root entry) + | .liveConfigMismatch entry expectedHash => + throw <| IO.userError (configMismatchMessage desired.root entry expectedHash) + | .draining entry => throw <| IO.userError (drainingOwnerMessage desired.root entry) + | .legacy => + throw <| IO.userError <| registryRecoveryMessage desired.root <| + RegistryRead.legacy.detail?.getD "legacy registry" + | .unsupported schemaVersion => + throw <| IO.userError <| registryRecoveryMessage desired.root <| + (RegistryRead.unsupported schemaVersion).detail?.getD "unsupported registry" + | .malformed detail => + throw <| IO.userError <| registryRecoveryMessage desired.root detail + | .unusable _ reason => + throw <| IO.userError <| registryRecoveryMessage desired.root reason.message let (endpoint, entry, child) ← startDaemonEntry desired opts try writeRegistry control entry @@ -706,7 +830,7 @@ private def startOwnedProjectDaemon terminateDaemonChild child throw err pure { - client := { endpoint } + client := { endpoint, capability := entry.capability } entry child } @@ -730,10 +854,10 @@ private partial def waitForOwnedDaemonExit IO.sleep 100 waitForOwnedDaemonExit child exitCodeRef (tries - 1) -private def removeOwnedRegistry (root : System.FilePath) (daemonId : String) : IO Unit := do +private def removeOwnedRegistry (root : System.FilePath) (entry : RegistryEntry) : IO Unit := do try - withProjectControl root fun control => - removeRegistryGeneration control daemonId + withExistingProjectControl root fun control => + removeRegistryGeneration control entry catch _ => pure () @@ -745,27 +869,34 @@ private def attemptCleanup (act : IO Unit) : IO Unit := do private def finishOwnedDaemonChild (owned : OwnedProjectDaemon) - (exitCodeRef : IO.Ref (Option UInt32)) : IO Unit := do + (exitCodeRef : IO.Ref (Option UInt32)) : IO Bool := do try let child ← closeDaemonOwnerPipe owned.child attemptCleanup <| waitForOwnedDaemonExit child exitCodeRef 100 if (← exitCodeRef.get).isNone then + -- `startDaemon` uses `setsid`; Lean's retained child handle therefore kills the complete + -- daemon process group rather than only the broker PID. attemptCleanup child.kill attemptCleanup <| waitForOwnedDaemonExit child exitCodeRef 20 - pure () catch _ => attemptCleanup owned.child.kill attemptCleanup <| waitForOwnedDaemonExit owned.child exitCodeRef 20 + pure (← exitCodeRef.get).isSome + +private def markOwnedRegistryDraining (root : System.FilePath) (entry : RegistryEntry) : IO Unit := do + try + withExistingProjectControl root fun control => + markRegistryDraining control entry + catch _ => + pure () private def finishOwnedProjectDaemon (root : System.FilePath) (owned : OwnedProjectDaemon) (exitCodeRef : IO.Ref (Option UInt32)) : IO Unit := do - -- Stop publishing this generation before waiting for its child to drain. A second removal is an - -- idempotent retry for cleanup failures and cannot remove a replacement generation. - removeOwnedRegistry root owned.entry.daemonId - finishOwnedDaemonChild owned exitCodeRef - removeOwnedRegistry root owned.entry.daemonId + markOwnedRegistryDraining root owned.entry + if ← finishOwnedDaemonChild owned exitCodeRef then + removeOwnedRegistry root owned.entry def withProjectDaemonOwner (home root : System.FilePath) @@ -791,12 +922,24 @@ private def lookupProjectDaemon (root : System.FilePath) (expectedHash? : Option String := none) (backend? : Option Backend := none) : IO ProjectDaemonClient := do - withProjectControl root fun control => do - match ← registryLiveFor root expectedHash? with - | some entry => projectDaemonClient entry - | none => - stopRegisteredDaemon control + withProjectControl root fun _control => do + match ← observeProjectRegistry root expectedHash? with + | .liveExact entry => projectDaemonClient entry + | .absent | .staleConfirmed _ => throw <| IO.userError (missingOwnerMessage root backend?) + | .liveConfigMismatch entry expectedHash => + throw <| IO.userError (configMismatchMessage root entry expectedHash) + | .draining entry => throw <| IO.userError (drainingOwnerMessage root entry) + | .legacy => + throw <| IO.userError <| registryRecoveryMessage root <| + RegistryRead.legacy.detail?.getD "legacy registry" + | .unsupported schemaVersion => + throw <| IO.userError <| registryRecoveryMessage root <| + (RegistryRead.unsupported schemaVersion).detail?.getD "unsupported registry" + | .malformed detail => + throw <| IO.userError <| registryRecoveryMessage root detail + | .unusable _ reason => + throw <| IO.userError <| registryRecoveryMessage root reason.message def withProjectDaemon (home root : System.FilePath) diff --git a/Beam/Cli/Feedback.lean b/Beam/Cli/Feedback.lean index bad6f316..9f5dd7ba 100644 --- a/Beam/Cli/Feedback.lean +++ b/Beam/Cli/Feedback.lean @@ -94,27 +94,40 @@ private def versionIdentityJson (home : System.FilePath) : IO Json := do private def collectDaemonPayload (root : System.FilePath) (warnings : Array String) : IO (Json × Json × Array String) := do - match ← registryLiveFor root with - | none => - pure (Json.null, Json.null, warnings.push "no live Beam daemon was available for stats/open-files") - | some entry => + match ← observeProjectRegistry root with + | .liveExact entry => match Beam.Daemon.registryEndpoint? entry with | none => pure (Json.null, Json.null, warnings.push "Beam daemon registry did not contain a valid endpoint") | some endpoint => - let statsResp ← sendRequest endpoint { + let client : ProjectDaemonClient := { endpoint, capability := entry.capability } + let statsResp ← sendRequest endpoint <| client.authorize { op := .stats workspaceId? := some Beam.Cli.projectDaemonWorkspaceId root? := some root.toString } let (stats, warnings) := Beam.Feedback.responsePayloadOrWarning "stats" statsResp warnings - let openResp ← sendRequest endpoint { + let openResp ← sendRequest endpoint <| client.authorize { op := .openDocs workspaceId? := some Beam.Cli.projectDaemonWorkspaceId root? := some root.toString } let (openDocs, warnings) := Beam.Feedback.responsePayloadOrWarning "open-files" openResp warnings pure (stats, openDocs, warnings) + | .absent | .staleConfirmed _ => + pure (Json.null, Json.null, warnings.push "no live Beam daemon was available for stats/open-files") + | .liveConfigMismatch _ _ => + pure (Json.null, Json.null, warnings.push "the live Beam daemon has a different configuration") + | .draining _ => + pure (Json.null, Json.null, warnings.push "the Beam daemon is draining") + | .legacy => + pure (Json.null, Json.null, warnings.push "the Beam daemon registry is legacy and unsupported") + | .unsupported _ => + pure (Json.null, Json.null, warnings.push "the Beam daemon registry schema is unsupported") + | .malformed detail => + pure (Json.null, Json.null, warnings.push s!"the Beam daemon registry is malformed: {detail}") + | .unusable _ reason => + pure (Json.null, Json.null, warnings.push s!"the Beam daemon registry is unsafe: {reason.message}") private def collectNonConfidential (home : System.FilePath) diff --git a/Beam/Cli/Info.lean b/Beam/Cli/Info.lean index e993380b..9421ec92 100644 --- a/Beam/Cli/Info.lean +++ b/Beam/Cli/Info.lean @@ -167,8 +167,8 @@ def doctor (home : System.FilePath) (opts : CliOptions) (backend : Backend) : IO | .rocq => printRocqDoctorInfo home root let registry ← Beam.Daemon.registryPath root IO.println s!"registry: {registry}" - match ← registryLiveFor root with - | some entry => + match ← observeProjectRegistry root with + | .liveExact entry => IO.println "daemon status: live" IO.println s!"daemon pid: {entry.pid}" if let some pidDomain := entry.pidDomain? then @@ -178,11 +178,25 @@ def doctor (home : System.FilePath) (opts : CliOptions) (backend : Backend) : IO else IO.println "daemon endpoint: invalid" IO.println s!"daemon config hash: {entry.configHash}" - | none => - if ← registry.pathExists then - IO.println "daemon status: stale" - else - IO.println "daemon status: absent" + | .liveConfigMismatch entry expectedHash => + IO.println "daemon status: config mismatch" + IO.println s!"daemon config hash: {entry.configHash}" + IO.println s!"expected config hash: {expectedHash}" + | .draining entry => + IO.println "daemon status: draining" + IO.println s!"daemon generation: {entry.daemonId}" + | .staleConfirmed _ => IO.println "daemon status: stale" + | .absent => IO.println "daemon status: absent" + | .legacy => IO.println "daemon status: legacy registry" + | .unsupported schemaVersion => + IO.println "daemon status: unsupported registry" + IO.println s!"registry schema version: {schemaVersion}" + | .malformed detail => + IO.println "daemon status: malformed registry" + IO.println s!"registry error: {detail}" + | .unusable _ reason => + IO.println "daemon status: unsafe" + IO.println s!"daemon safety error: {reason.message}" printDaemonFailureIncidentDoctorInfo root def printValidatedToolchains (home : System.FilePath) (backendName : String) : IO Unit := do diff --git a/Beam/Cli/InstallPrune.lean b/Beam/Cli/InstallPrune.lean index 863eeeda..b246e91f 100644 --- a/Beam/Cli/InstallPrune.lean +++ b/Beam/Cli/InstallPrune.lean @@ -31,6 +31,59 @@ private structure InstallPrunePlan where oldRuntimes : Array System.FilePath := #[] staleBundles : Array System.FilePath := #[] +private def installLockPollMs : Nat := + 100 + +/-- +Coordinate pruning with the bootstrap shell installer, which owns `.install-lock` as a directory. + +Unlike the old generic directory lock, this compatibility boundary never reaps a purportedly stale +owner: `mkdir` is the only acquisition operation and only the process that created the directory +removes it. A crashed installer therefore fails closed and requires explicit operator recovery. +-/ +private partial def acquireInstallLockUntil + (lockDir : System.FilePath) + (startedNanos deadlineNanos timeoutMs : Nat) : IO Unit := do + let acquired ← + try + IO.FS.createDir lockDir + pure true + catch + | .alreadyExists .. => pure false + | error => throw error + if acquired then + let selfPid ← IO.Process.getPID + IO.FS.writeFile (lockDir / "pid") s!"{selfPid}\n" + if let some pidDomain := ← Beam.currentPidDomain? then + IO.FS.writeFile (lockDir / "pid-domain") s!"{pidDomain}\n" + return + let now ← IO.monoNanosNow + if now >= deadlineNanos then + let waitedMs := (now - startedNanos) / 1000000 + throw <| IO.userError <| + s!"timed out after {waitedMs} ms waiting for Beam lock {lockDir}; timeout: {timeoutMs} ms" + IO.sleep installLockPollMs.toUInt32 + acquireInstallLockUntil lockDir startedNanos deadlineNanos timeoutMs + +private def releaseInstallLock (lockDir : System.FilePath) : IO Unit := do + for name in #["pid", "pid-domain"] do + let path := lockDir / name + if ← path.pathExists then + IO.FS.removeFile path + IO.FS.removeDir lockDir + +private def withInstallLockTimeout + (lockDir : System.FilePath) + (timeoutMs : Nat) + (act : IO α) : IO α := do + let startedNanos ← IO.monoNanosNow + acquireInstallLockUntil lockDir startedNanos + (startedNanos + timeoutMs * 1000000) timeoutMs + try + act + finally + releaseInstallLock lockDir + private def installPruneUsage : String := "usage: lean-beam prune [--apply] [--bundles]" @@ -249,7 +302,7 @@ def runInstallPrune (home : System.FilePath) (args : List String) : IO Unit := d IO.println installPruneHelp return let ctx ← resolveInstallPruneContext home - withLockTimeout (ctx.installRoot / ".install-lock") 1000 do + withInstallLockTimeout (ctx.installRoot / ".install-lock") 1000 do validateInstallPruneContext ctx let plan ← installPrunePlan ctx opts printInstallPrunePlan ctx opts plan diff --git a/Beam/Cli/Lock.lean b/Beam/Cli/Lock.lean index e22becb5..64c73d81 100644 --- a/Beam/Cli/Lock.lean +++ b/Beam/Cli/Lock.lean @@ -20,126 +20,83 @@ private structure LockDeadline where startedNanos : Nat deadlineNanos : Nat -private structure LockOwner where - pid : Nat - pidDomain? : Option String - -private def readRegularFile? (path : System.FilePath) : IO (Option String) := do - try - if ← Beam.regularNonSymlinkFile path then - pure <| some (Beam.trimLine (← IO.FS.readFile path)) - else - pure none - catch _ => - pure none - -private def readLockOwner? (lockDir : System.FilePath) : IO (Option LockOwner) := do - let some pidText ← readRegularFile? (lockDir / "pid") - | return none - let some pid := pidText.toNat? - | return none - let pidDomain? ← readRegularFile? (lockDir / "pid-domain") - pure <| some { pid, pidDomain? := pidDomain?.filter (fun domain => !domain.isEmpty) } - -private def lockOwnerDescription : Option LockOwner → String - | some owner => s!"pid {owner.pid}" - | none => "unknown owner" - private def lockTimeoutMessage - (lockDir : System.FilePath) - (owner? : Option LockOwner) + (lockPath : System.FilePath) (waitedMs timeoutMs : Nat) : String := - s!"timed out after {waitedMs} ms waiting for Beam lock {lockDir}; " ++ - s!"lock owner: {lockOwnerDescription owner?}; timeout: {timeoutMs} ms" + s!"timed out after {waitedMs} ms waiting for Beam lock {lockPath}; " ++ + s!"timeout: {timeoutMs} ms" -private def removeStaleLock? (lockDir : System.FilePath) (owner? : Option LockOwner) : IO Bool := do - match owner? with - | some owner => - match ← (Beam.RecordedPid.mk owner.pid owner.pidDomain?).observe with - | .local false => - if ← lockDir.pathExists then - IO.FS.removeDirAll lockDir - pure true - | .invalid | .local true | .differentDomain | .unknownDomain => pure false - | none => - pure false +/-- +Open the stable file whose kernel lock protects one Beam critical section. -private partial def acquireLockCore - (lockDir : System.FilePath) - (deadline? : Option LockDeadline) : IO Unit := do - if let some parent := lockDir.parent then +The file is deliberately retained after unlock. Removing a lock file would let a later contender +lock a new inode while an earlier waiter still holds or waits on the old one. +-/ +private def openLockHandle (lockPath : System.FilePath) : IO IO.FS.Handle := do + if let some parent := lockPath.parent then IO.FS.createDirAll parent - let selfPid ← IO.Process.getPID - let acquired ← - try - IO.FS.createDir lockDir - pure true - catch - | .alreadyExists .. => - pure false - | error => - throw error - if acquired then - try - IO.FS.writeFile (lockDir / "pid") s!"{selfPid}\n" - if let some pidDomain := ← Beam.currentPidDomain? then - IO.FS.writeFile (lockDir / "pid-domain") s!"{pidDomain}\n" - return - catch error => - try - if ← lockDir.pathExists then - IO.FS.removeDirAll lockDir - catch cleanupError => - throw <| IO.userError <| - s!"failed to publish Beam lock owner at {lockDir}: {error}; " ++ - s!"also failed to remove the acquired lock: {cleanupError}" - throw error - else - let owner? ← readLockOwner? lockDir - if ← removeStaleLock? lockDir owner? then - acquireLockCore lockDir deadline? - else - match deadline? with - | some deadline => - let now ← IO.monoNanosNow - if now >= deadline.deadlineNanos then - let waitedMs := (now - deadline.startedNanos) / 1000000 - throw <| IO.userError <| - lockTimeoutMessage lockDir owner? waitedMs deadline.timeoutMs - | none => - pure () - IO.sleep lockPollMs.toUInt32 - acquireLockCore lockDir deadline? - -private def acquireLock (lockDir : System.FilePath) : IO Unit := - acquireLockCore lockDir none + IO.FS.Handle.mk lockPath .append + +/-- Open a lock without creating a missing parent directory during teardown. -/ +private def openExistingLockHandle (lockPath : System.FilePath) : IO IO.FS.Handle := do + IO.FS.Handle.mk lockPath .readWrite + +private partial def acquireLockUntil + (handle : IO.FS.Handle) + (lockPath : System.FilePath) + (deadline : LockDeadline) : IO Unit := do + if ← handle.tryLock then + return + let now ← IO.monoNanosNow + if now >= deadline.deadlineNanos then + let waitedMs := (now - deadline.startedNanos) / 1000000 + throw <| IO.userError <| + lockTimeoutMessage lockPath waitedMs deadline.timeoutMs + IO.sleep lockPollMs.toUInt32 + acquireLockUntil handle lockPath deadline + +/-- Run `act` while holding an unbounded kernel-backed file lock. -/ +def withLock (lockPath : System.FilePath) (act : IO α) : IO α := do + let handle ← openLockHandle lockPath + handle.lock + try + act + finally + handle.unlock -private def acquireLockTimeout (lockDir : System.FilePath) (timeoutMs : Nat) : IO Unit := do +/-- Run `act` while holding a kernel-backed file lock until an absolute monotonic deadline. -/ +def withLockTimeout (lockPath : System.FilePath) (timeoutMs : Nat) (act : IO α) : IO α := do + let handle ← openLockHandle lockPath let startedNanos ← IO.monoNanosNow - acquireLockCore lockDir <| some { + acquireLockUntil handle lockPath { timeoutMs startedNanos deadlineNanos := startedNanos + timeoutMs * 1000000 } - -private def releaseLock (lockDir : System.FilePath) : IO Unit := do - if ← lockDir.pathExists then - IO.FS.removeDirAll lockDir - -/-- Run `act` while holding an unbounded directory lock. -/ -def withLock (lockDir : System.FilePath) (act : IO α) : IO α := do - acquireLock lockDir try act finally - releaseLock lockDir + handle.unlock + +/-- +Run `act` under a kernel-backed lock without creating the lock's parent directory. -/-- Run `act` while holding a directory lock until an absolute monotonic deadline. -/ -def withLockTimeout (lockDir : System.FilePath) (timeoutMs : Nat) (act : IO α) : IO α := do - acquireLockTimeout lockDir timeoutMs +This is reserved for teardown after an owned project root may have disappeared. +-/ +def withExistingLockTimeout + (lockPath : System.FilePath) + (timeoutMs : Nat) + (act : IO α) : IO α := do + let handle ← openExistingLockHandle lockPath + let startedNanos ← IO.monoNanosNow + acquireLockUntil handle lockPath { + timeoutMs + startedNanos + deadlineNanos := startedNanos + timeoutMs * 1000000 + } try act finally - releaseLock lockDir + handle.unlock end Beam.Cli diff --git a/Beam/Daemon/Debug.lean b/Beam/Daemon/Debug.lean index 47f1704b..fcc0c342 100644 --- a/Beam/Daemon/Debug.lean +++ b/Beam/Daemon/Debug.lean @@ -13,17 +13,54 @@ open Lean namespace Beam.Daemon -def readRegistry? (root : System.FilePath) : IO (Option RegistryEntry) := do +inductive RegistryRead where + | absent + | legacy + | unsupported (schemaVersion : Nat) + | malformed (detail : String) + | current (entry : RegistryEntry) + +def RegistryRead.entry? : RegistryRead → Option RegistryEntry + | .current entry => some entry + | .absent | .legacy | .unsupported _ | .malformed _ => none + +def RegistryRead.status : RegistryRead → String + | .absent => "absent" + | .legacy => "legacy" + | .unsupported _ => "unsupported" + | .malformed _ => "malformed" + | .current _ => "current" + +def RegistryRead.detail? : RegistryRead → Option String + | .legacy => some "legacy registry has no schemaVersion" + | .unsupported version => some s!"unsupported registry schemaVersion {version}" + | .malformed detail => some detail + | .absent | .current _ => none + +def readRegistry (root : System.FilePath) : IO RegistryRead := do let path ← registryPath root unless ← path.pathExists do - return none + return .absent try let text ← IO.FS.readFile path - let json ← IO.ofExcept <| Json.parse text - let entry ← IO.ofExcept <| fromJson? json - pure (some entry) - catch _ => - pure none + let json ← + match Json.parse text with + | .ok json => pure json + | .error err => return .malformed s!"invalid registry JSON: {err}" + match json.getObjVal? "schemaVersion" with + | .error _ => pure .legacy + | .ok schemaJson => + let schemaVersion ← + match fromJson? (α := Nat) schemaJson with + | .ok schemaVersion => pure schemaVersion + | .error err => return .malformed s!"invalid registry schemaVersion: {err}" + unless schemaVersion == registrySchemaVersion do + return .unsupported schemaVersion + match fromJson? json with + | .ok entry => pure <| .current entry + | .error err => pure <| .malformed s!"invalid registry schema: {err}" + catch err => + pure <| .malformed s!"could not read registry: {err}" def daemonFailureIncidentEntries (root : System.FilePath) : IO (Array IO.FS.DirEntry) := do try @@ -139,13 +176,25 @@ private def optionLine (label : String) : Option String → Option String def daemonRegistryContext? (root : System.FilePath) : IO (Option String) := do try - match ← readRegistry? root with - | none => pure none - | some entry => + match ← readRegistry root with + | .absent => pure none + | .legacy => + let path ← registryPath root + pure <| some s!"Beam daemon registry ({path}):\n status: legacy\n detail: legacy registry has no schemaVersion" + | .unsupported schemaVersion => + let path ← registryPath root + let detail := (RegistryRead.unsupported schemaVersion).detail?.getD "unsupported registry" + pure <| some s!"Beam daemon registry ({path}):\n status: unsupported\n detail: {detail}" + | .malformed detail => + let path ← registryPath root + pure <| some s!"Beam daemon registry ({path}):\n status: malformed\n detail: {detail}" + | .current entry => let path ← registryPath root let pidStatus ← registryPidStatus entry let lines := ([ s!"Beam daemon registry ({path}):", + s!" schemaVersion: {entry.schemaVersion}", + s!" lifecycle: {repr entry.lifecycle}", s!" daemonId: {entry.daemonId}", s!" pid: {entry.pid} ({pidStatus})", s!" endpoint: {registryEndpointSummary entry}", @@ -162,7 +211,8 @@ def daemonRegistryContext? (root : System.FilePath) : IO (Option String) := do def daemonDebugContextJson (root : System.FilePath) : IO Json := do let registryFile ← registryPath root - let registry ← readRegistry? root + let registryRead ← readRegistry root + let registry := registryRead.entry? let registryPidStatus ← match registry with | some entry => some <$> registryPidStatus entry @@ -172,7 +222,13 @@ def daemonDebugContextJson (root : System.FilePath) : IO Json := do pure <| Json.mkObj <| [ ("registryPath", toJson registryFile.toString), - ("registry", match registry with | some entry => toJson entry | none => Json.null), + ("registryReadStatus", toJson registryRead.status), + ("registryReadDetail", match registryRead.detail? with + | some detail => toJson detail + | none => Json.null), + ("registry", match registry with + | some entry => (toJson entry).setObjVal! "capability" (toJson "") + | none => Json.null), ("registryPidStatus", match registryPidStatus with | some status => toJson status | none => Json.null), ("registryEndpoint", match registry.map registryEndpointSummary with | some endpoint => toJson endpoint | none => Json.null), ("recentDaemonIncidents", toJson incidents) diff --git a/Beam/Daemon/Protocol.lean b/Beam/Daemon/Protocol.lean index 5ee5f298..044a403a 100644 --- a/Beam/Daemon/Protocol.lean +++ b/Beam/Daemon/Protocol.lean @@ -15,8 +15,30 @@ namespace Beam.Daemon open Beam.Broker +def registrySchemaVersion : Nat := + 1 + +inductive RegistryLifecycle where + | live + | draining + deriving BEq, Repr + +instance : ToJson RegistryLifecycle where + toJson + | .live => "live" + | .draining => "draining" + +instance : FromJson RegistryLifecycle where + fromJson? + | .str "live" => .ok .live + | .str "draining" => .ok .draining + | json => .error s!"expected registry lifecycle 'live' or 'draining', got {json.compress}" + structure RegistryEntry where + schemaVersion : Nat + lifecycle : RegistryLifecycle daemonId : String + capability : String pid : Nat pidDomain? : Option String := none ownerPid : Nat @@ -94,9 +116,10 @@ private def daemonProbeOfResponse (resp : Response) : Except BrokerClientFailure private def daemonProbe (endpoint : Transport.Endpoint) - (workspaceId : WorkspaceId) : IO (Except BrokerClientFailure DaemonProbe) := do + (workspaceId : WorkspaceId) + (capability? : Option String := none) : IO (Except BrokerClientFailure DaemonProbe) := do match ← sendRequestWithStreamTimeoutResult endpoint - { op := .stats, workspaceId? := some workspaceId } + { op := .stats, workspaceId? := some workspaceId, daemonCapability? := capability? } daemonProbeResponseTimeoutMs (fun _ => pure ()) with | .ok resp => pure <| daemonProbeOfResponse resp | .error failure => pure <| .error failure @@ -154,8 +177,9 @@ def daemonGenerationStatus (endpoint : Transport.Endpoint) (workspaceId : WorkspaceId) (root : System.FilePath) - (identity : DaemonIdentity) : IO DaemonGenerationStatus := do - match ← daemonProbe endpoint workspaceId with + (identity : DaemonIdentity) + (capability : String) : IO DaemonGenerationStatus := do + match ← daemonProbe endpoint workspaceId (some capability) with | .error failure => match failure with | .transport _ _ => diff --git a/Beam/System.lean b/Beam/System.lean index a66cac59..13b285cc 100644 --- a/Beam/System.lean +++ b/Beam/System.lean @@ -120,17 +120,6 @@ def RecordedPid.observe (recorded : RecordedPid) : IO RecordedPidObservation := | .different => pure .differentDomain | .unknown => pure .unknownDomain -/-- Send the default termination signal only to a persisted PID in the caller's current domain. -/ -def RecordedPid.terminateIfLocal (recorded : RecordedPid) : IO Bool := do - match ← recorded.domainRelation with - | .local => - try - let out ← IO.Process.output { cmd := (← killCommand), args := #[toString recorded.pid] } - pure (out.exitCode == 0) - catch _ => - pure false - | .invalid | .different | .unknown => pure false - def utcTimestamp : IO String := do readCmdTrim "date" #["-u", "+%Y-%m-%dT%H:%M:%SZ"] diff --git a/docs/COMPATIBILITY.md b/docs/COMPATIBILITY.md index c21832f6..68de4085 100644 --- a/docs/COMPATIBILITY.md +++ b/docs/COMPATIBILITY.md @@ -20,6 +20,10 @@ A Lean release line is the canonical `major.minor` family recorded in - Runtime bundle metadata schema 2 and install manifest schema 3. Install manifest schema 2 is cleanup-only compatibility during the 0.2 release line: identity and `lean-beam prune` may read it, but the installer does not reuse it. Remove the schema-2 decoder when 0.3 development opens. +- Wrapper daemon registry schema 1. The registry is an internal beta coordination boundary: + schema-less and unknown-version records are reported and preserved, but are not decoded, deleted, + or migrated automatically. Stop their matching owner with the runtime that created them before + starting a schema-1 owner. - MCP `2026-07-28` is the preferred stdio protocol revision. MCP `2025-11-25` remains a named transition target for initialization-based clients. Reconsider the legacy path before the 0.3 release once the clients named by the setup guide can all use per-request metadata. diff --git a/docs/DEVELOPMENT.md b/docs/DEVELOPMENT.md index f6f937db..d2538e59 100644 --- a/docs/DEVELOPMENT.md +++ b/docs/DEVELOPMENT.md @@ -402,67 +402,79 @@ This wrapper path is easy to break accidentally, so keep the mental model simple A daemon generation is one concrete daemon start identified by the `daemonId` in `beam-daemon.json`. Exactly one foreground `lean-beam ensure --hold` process owns that generation. -It passes the daemon that identity and the effective configuration hash, starts it with piped stdin, -and retains the pipe's write end. Endpoint attachment requires the root and this exact generation -identity to match. The daemon watches the read end; EOF atomically closes broker admission, marks -admitted requests for cancellation, shuts down backend sessions, and stops the listener. There is -no wrapper heartbeat, lease file, revocation tombstone, or retirement fence. +It starts the daemon in a dedicated process session, passes the daemon identity, effective +configuration hash, and a random per-generation capability through piped stdin, and retains the +pipe's write end. The mode-`0600`, schema-versioned registry publishes `live` or `draining` state. +Every wrapper request, including cancellation, generation probes, and shutdown, must present the +capability. Endpoint attachment also requires the canonical root and exact generation identity to +match. The daemon watches the pipe's read end; EOF atomically closes broker admission, marks admitted +requests for cancellation, shuts down backend sessions, and stops the listener. There is no wrapper +heartbeat, lease file, revocation tombstone, or time-based retirement fence. Ordinary wrapper commands never start a daemon. Under the per-project control lock they require a registry whose root and effective configuration match, whose owner is not known dead in the current PID domain, and whose endpoint answers for the CLI's private workspace, canonical project root, and exact daemon generation identity. Identity probes have a bounded response deadline. An endpoint that accepts a connection but stays silent or returns malformed data is unrecognized, so validation -fails closed and PID fallback is not permitted. +fails closed. A configuration mismatch reports the old and desired hashes without shutting down or +unpublishing the live owner. Ordinary lookup is observation-only: absent, legacy, malformed, +unsupported, stale, draining, or otherwise unsafe registry states are never rewritten by an +attaching command. Endpoint/root validation is authoritative across PID namespaces because numeric PID observations -from another domain are not safe process identity. A same-domain dead owner or a dead endpoint makes -the registry stale; cleanup remains generation-scoped and PID fallback is permitted only through the -typed PID-domain boundary. - -The owner also watches its exact registry generation and daemon child. `lean-beam shutdown` removes -that generation after the typed shutdown response or a bounded response failure, which makes the -holder close its pipe and lets the daemon's stdin watcher finish. On every holder exit path, the -holder removes its exact registry generation before waiting for the daemon child to drain, then -retries the same generation-scoped removal after bounded child cleanup. A draining daemon is -therefore never advertised as attachable, and neither removal can delete a replacement generation. -An unexpected nonzero daemon exit is reported by the holder. Interrupting or killing the holder -closes the pipe by process lifetime. A paused holder keeps the pipe open, so the session remains valid -without time-based expiry. If the project root disappears, the daemon's root watcher and the holder -both converge on the same shutdown path. +from another domain are not safe process identity. Persisted PIDs are conservative liveness +observations, never signal capabilities. Only the foreground owner may force termination, using its +retained child handle and dedicated process group. + +The owner also watches its exact registry generation and daemon child. `lean-beam shutdown` changes +that exact registry from `live` to `draining` under the project lock before sending the authenticated +shutdown request. Every holder exit path likewise publishes `draining`, closes its pipe, waits for +graceful broker/backend teardown, and, after the deadline, terminates the complete owned process +group. Only after the child has been reaped does it remove the exact draining generation. Thus a +paused or wedged old process tree remains fenced and a replacement owner cannot create split-brain +backend sessions. An unexpected nonzero daemon exit is reported by the holder. Killing the holder +closes the pipe by process lifetime. A paused holder keeps the pipe open, so the session remains +valid without time-based expiry. If the project root disappears, cleanup uses the already resolved +control path without recreating the deleted project. This model prevents PID-isolated commands from making contradictory ownership decisions: later commands may attach to a validated endpoint, but none can silently become a replacement owner. -Starting a new session is always an explicit `lean-beam ensure --hold` action. Raw `beam-client` -requests may attach while that owner remains live; they participate in typed broker admission and -disconnect cancellation but do not own the process. A separately launched standalone daemon has -its own explicit process owner. +Starting a new session is always an explicit `lean-beam ensure --hold` action. Wrapper commands read +the private registry and inject its generation capability. A raw `beam-client` request does not gain +authority merely by finding the loopback port; it must explicitly carry that private capability. +The wrapper-owned daemon also rejects dynamic `initWorkspace` and `dropWorkspace`, because its one +bootstrap project is fixed by the owner. A separately launched development daemon has its own +explicit process owner and is not the wrapper security boundary. Keep these invariants covered: - only `ensure --hold` may create and publish a wrapper daemon generation - a second owner is rejected while the current endpoint/root/generation identity is live -- ordinary wrapper commands preserve the owner's generation and fail with the exact recovery command - when no owner is live -- holder teardown unpublishes its exact generation before child drain and cannot remove a replacement +- ordinary wrapper commands are read-only with respect to registry and process lifecycle, including + on configuration mismatch or stale/unsafe state +- holder teardown retains a generation-specific draining fence until the complete process tree is + reaped and cannot remove a replacement - owner EOF, explicit shutdown, and project-root disappearance all close admission before backend teardown and complete with bounded child cleanup -- PID-domain checks gate every PID probe or signal; cross-domain decisions use the validated endpoint +- PID-domain checks gate persisted-PID observations; persisted numeric PIDs are never signalled +- every wrapper request is bound to its random generation capability, and transport frame, initial + request, connection, and task counts are bounded - request IDs and per-admission tokens retain exact disconnect and explicit cancellation semantics - the regressions for this path are [tests/test-beam-wrapper-daemon.sh](../tests/test-beam-wrapper-daemon.sh) and [tests/test-beam-wrapper-sandbox.sh](../tests/test-beam-wrapper-sandbox.sh) Generic process helpers and the typed `RecordedPid.observe` boundary live in -[Beam/System.lean](../Beam/System.lean). Persisted registry and lock-owner PIDs must pass -through that boundary; only a matching recorded/current PID-domain pair permits a local liveness or -termination operation. Generic directory locks live in -[Beam/Cli/Lock.lean](../Beam/Cli/Lock.lean). Their owner metadata records both PID and PID domain; -only a proven dead same-domain owner is reaped, while missing, malformed, unknown-domain, and -different-domain owners fail closed. Project daemon control locks use a bounded wait so a live but -stuck wrapper process produces owner diagnostics instead of making later clients wait silently; +[Beam/System.lean](../Beam/System.lean). Persisted registry PIDs pass through that boundary only for +conservative liveness reporting. Kernel-backed stable file locks live in +[Beam/Cli/Lock.lean](../Beam/Cli/Lock.lean); lock files remain after release so contenders always +coordinate on the same inode, while the kernel releases ownership when a process exits. Project +daemon control locks use a bounded wait so a live but stuck wrapper process produces owner +diagnostics instead of making later clients wait silently; `BEAM_CONTROL_LOCK_TIMEOUT_MS` can shorten or lengthen that wait for local debugging. Bundle build locks intentionally keep the lower-level unbounded helper because another process may legitimately -be compiling a helper bundle. Reusable CLI argument parsing lives in +be compiling a helper bundle. The shell installer's `.install-lock` remains an atomic directory +compatibility boundary: it is never stale-reaped, and a crashed installer requires explicit +operator recovery. Reusable CLI argument parsing lives in [Beam/Cli/Args.lean](../Beam/Cli/Args.lean). Project-root inference, Lean toolchain lookup, and Rocq command discovery live in [Beam/Cli/Project.lean](../Beam/Cli/Project.lean). Shared filesystem path helpers live in [Beam/Path.lean](../Beam/Path.lean). Use them instead of diff --git a/docs/SETUP.md b/docs/SETUP.md index 9f76d331..3d248e42 100644 --- a/docs/SETUP.md +++ b/docs/SETUP.md @@ -163,10 +163,12 @@ not currently in use. Restart active agent and MCP client sessions before any `prune --apply`; otherwise a process may still be running from a runtime selected for removal. A later request rebuilds any needed bundle -that was pruned. Pruning uses the same install lock as the installer and each selected bundle's -build lock. Lock owner metadata includes a PID-domain identity, so cleanup never interprets a -same-numbered PID from an isolated process domain as the local owner. Pruning also refuses symlinked -installed bundle-cache roots or symlinked and unmarked runtime directories. +that was pruned. Pruning uses the same atomic-directory install lock as the shell installer and a +kernel-backed file lock for each selected bundle. Bundle lock files remain as stable coordination +inodes after release; kernel ownership disappears automatically when the holder exits. The install +lock is never stale-reaped, so a crashed installer fails closed and requires explicit recovery. +Pruning also refuses symlinked installed bundle-cache roots or symlinked and unmarked runtime +directories. Apply is incremental rather than transactional: Beam validates and removes one displayed path at a time and reports each successful removal immediately. If a later path fails validation or its lock @@ -222,7 +224,10 @@ lean-beam run-at "Foo.lean" "$version" 10 2 "exact trivial" inherited ownership pipe defines the session lifetime: interrupt the holder, or run `lean-beam shutdown`, to close the daemon and its backend processes. Plain `lean-beam ensure` and all other wrapper commands attach to an existing owner and fail with a recovery command when none -is live. MCP clients do not need a separate holder; the stdio MCP process owns its runtime session. +is live. If the desired bundle or project configuration changes, attaching commands preserve the +old owner and ask you to stop it explicitly. During shutdown the registry reports `draining` and a +replacement owner is refused until the old process tree has exited. MCP clients do not need a +separate holder; the stdio MCP process owns its runtime session. The `python3` line extracts `result.version` for shell examples. You can also copy that version number from the printed `lean-beam update` JSON. diff --git a/docs/STATUS.md b/docs/STATUS.md index 8e056f1e..72cb41fd 100644 --- a/docs/STATUS.md +++ b/docs/STATUS.md @@ -190,8 +190,20 @@ Exact event ordering and examples live in requests drain; a backend success that completed before cancellation remains successful. This happens without heartbeat timeouts or filesystem leases and works across PID namespaces because endpoint, root, and generation-identity validation are authoritative when PID identity is not - locally observable. A paused owner retains the session; a killed owner closes the pipe; explicit - `lean-beam shutdown` unpublishes the registry generation so the holder closes it cleanly. + locally observable. Each wrapper request carries a random per-generation capability from the + mode-`0600` registry. A paused owner retains the session; a killed owner closes the pipe; explicit + `lean-beam shutdown` changes the registry to `draining`, and that fence remains until the holder + has reaped the daemon process tree. Configuration-mismatched and otherwise unsafe ordinary + lookups preserve the current owner and registry. +- After abrupt owner death, a later process in the same PID domain may recognize a registry as stale + only when both recorded processes are proven gone. A client in another PID domain cannot make + that proof and fails closed with the registry preserved; its external sandbox/process supervisor + must establish complete process-tree exit before removing that exact recovery record. +- Wrapper-owned brokers currently use authenticated loopback TCP. The supported trust boundary is + one local OS account with private registry-file permissions; another user who can only discover + the port cannot issue requests without the generation capability. A manually launched standalone + `beam-daemon` has no wrapper registry capability and is maintainer tooling, not a shared-host + service. Unix-domain/per-user native IPC remains a possible later transport improvement. - A startup failure that reports `operation not permitted` through `.beam/beam-daemon-startup.log` is usually an environment restriction, not a bundle-resolution mismatch. - Typed broker transport, invalid-response, and response-timeout failures include registry/log diff --git a/docs/SYNC_AND_DIAGNOSTICS.md b/docs/SYNC_AND_DIAGNOSTICS.md index 89e204a5..3110f351 100644 --- a/docs/SYNC_AND_DIAGNOSTICS.md +++ b/docs/SYNC_AND_DIAGNOSTICS.md @@ -127,12 +127,13 @@ observed it. A request may produce any number of `fileProgress` and `diagnostic` by exactly one terminal `response`; the response is last and no later message belongs to that request. -When `beam-client` targets the per-project daemon managed by `lean-beam` in a PID-reaping command -runner, keep the session's `lean-beam ensure --hold` owner active for the request lifetime. The same -rule applies to wrapper commands: only the holder starts the daemon, while every other command -attaches to its endpoint. Raw broker requests participate in the daemon's typed request admission -and cancellation, but do not own the daemon process. A separately launched standalone daemon has -its own explicit process owner and does not use the wrapper holder. +When `beam-client` targets the per-project daemon managed by `lean-beam`, keep the session's +`lean-beam ensure --hold` owner active for the request lifetime. Only the holder starts the daemon; +wrapper commands attach with the private per-generation capability read from the mode-`0600` +registry. A raw broker request must explicitly include that `daemonCapability`; discovering the +loopback port alone does not authorize it. Raw requests participate in typed request admission and +cancellation but do not own the daemon process. A separately launched standalone development daemon +has its own explicit process owner and does not use the wrapper holder. Every stream variant uses the same `kind`, `payload`, and optional correlation envelope. When the request supplies `clientRequestId`, each message repeats it on that outer stream envelope: diff --git a/docs/TESTING.md b/docs/TESTING.md index 05837c1d..780487e9 100644 --- a/docs/TESTING.md +++ b/docs/TESTING.md @@ -113,17 +113,22 @@ Current Beam coverage includes: - wrapper coverage through [tests/test-beam-wrapper.sh](../tests/test-beam-wrapper.sh), which aggregates focused probe, runtime, sync/save, handle, and diagnostic slices - focused daemon lifecycle coverage in [tests/test-beam-wrapper-daemon.sh](../tests/test-beam-wrapper-daemon.sh), including the no-implicit-start contract, duplicate-owner rejection, Beam and non-Beam endpoint - collision safety, a bounded identity probe against a silent non-Beam listener, cross-root - stale-registry cleanup that preserves the daemon serving the other - root, explicit shutdown, cancellation of requests active during shutdown or owner loss, - exact-generation cleanup that preserves a replacement registry, registry removal before a paused - daemon can finish draining, rejection of attachment to that unpublished draining generation, + collision safety without cross-project disclosure, authenticated generation probes, mode-`0600` + registry publication, oversized-frame and first-message limits, a bounded identity probe against + a silent non-Beam listener, cross-root unsafe-registry preservation that does not affect the daemon + serving the other root, configuration-drift preservation of the owner and active request, + explicit shutdown, cancellation of requests active during shutdown or owner loss, + exact-generation cleanup that preserves a replacement registry, a published draining fence while + a daemon is paused, rejection of attachment or replacement while that generation remains + published, forced process-group cleanup of the daemon and its backend, holder reporting after an unexpected daemon crash, abrupt owner death through inherited-pipe EOF, - stale registry cleanup, and self-termination after the project worktree disappears + read-only stale registry lookup, and self-termination after the project worktree disappears without + recreating it - Linux-only PID-isolated sandbox wrapper coverage in [tests/test-beam-wrapper-sandbox.sh](../tests/test-beam-wrapper-sandbox.sh), including cross-namespace endpoint attachment, duplicate-owner rejection, a paused owner without - time-based expiry, explicit shutdown, killed-owner EOF cleanup, stale-registry recovery, distinct - generation identity, and the absence of legacy lease/retirement artifacts + time-based expiry, explicit shutdown, killed-owner EOF cleanup, fail-closed preservation of an + unavailable foreign-domain registry before supervised recovery, distinct generation identity, + and the absence of legacy lease/retirement artifacts - zero-build save replay, structured-setup support, batch-only-argument rejection, and stale-save race coverage in [tests/test-beam-save-olean.sh](../tests/test-beam-save-olean.sh) diff --git a/tests/lean/BeamTest/Broker/CliDaemonTest.lean b/tests/lean/BeamTest/Broker/CliDaemonTest.lean index b4b8af4f..36b75094 100644 --- a/tests/lean/BeamTest/Broker/CliDaemonTest.lean +++ b/tests/lean/BeamTest/Broker/CliDaemonTest.lean @@ -28,6 +28,7 @@ private def require (label : String) (cond : Bool) : IO Unit := do private def projectDaemonClientForTest (endpoint : Beam.Broker.Transport.Endpoint) : Beam.Cli.ProjectDaemonClient := { endpoint + capability := "test-capability" } private def checkDaemonDebugWarnings : IO Unit := do @@ -175,7 +176,8 @@ private def checkSilentEndpointProbeTimeout : IO Unit := do configHash := "silent-endpoint" } match ← Beam.Daemon.daemonGenerationStatus endpoint - Beam.Cli.projectDaemonWorkspaceId (System.FilePath.mk "/tmp") identity with + Beam.Cli.projectDaemonWorkspaceId (System.FilePath.mk "/tmp") identity + "test-capability" with | .unrecognized (.responseTimeout timeoutMs) => require "silent endpoint should preserve its typed response timeout" (timeoutMs == 2000) @@ -195,7 +197,7 @@ private def checkSilentShutdownTimeout : IO Unit := do let serverTask ← IO.asTask (prio := Task.Priority.dedicated) <| holdAcceptedConnection listener release try - match ← Beam.Cli.requestDaemonShutdown endpoint 50 with + match ← Beam.Cli.requestDaemonShutdown endpoint "test-capability" 50 with | .error (.responseTimeout timeoutMs) => require "silent shutdown should preserve its typed response timeout" (timeoutMs == 50) | .error failure => @@ -586,7 +588,10 @@ private def checkDaemonFailureContext : IO Unit := do IO.FS.createDirAll parent let pidDomain? ← Beam.currentPidDomain? let entry : Beam.Daemon.RegistryEntry := { + schemaVersion := Beam.Daemon.registrySchemaVersion + lifecycle := .live daemonId := "daemon-test" + capability := "test-capability" pid := 999999999 pidDomain? ownerPid := 999999999 @@ -628,6 +633,8 @@ private def checkDaemonFailureContext : IO Unit := do let incidentRegistry ← IO.ofExcept <| fromJson? (α := Beam.Daemon.RegistryEntry) incidentRegistryJson require "daemon failure incident should include daemon id" (incidentRegistry.daemonId == "daemon-test") + require "daemon failure incident must redact the per-generation capability" + (incidentRegistry.capability == "") requireJsonString "daemon failure incident should include registry pid status" "registryPidStatus" "not alive" incidentJson requireJsonString "daemon failure incident should include endpoint summary" @@ -724,7 +731,10 @@ private def writeTestRegistryEntry if let some parent := registryPath.parent then IO.FS.createDirAll parent let entry : Beam.Daemon.RegistryEntry := { + schemaVersion := Beam.Daemon.registrySchemaVersion + lifecycle := .live daemonId := "daemon-test" + capability := "test-capability" pid := 999999999 ownerPid := 999999999 port? @@ -736,6 +746,65 @@ private def writeTestRegistryEntry } IO.FS.writeFile registryPath ((toJson entry).pretty ++ "\n") +private def checkTypedRegistryReads : IO Unit := do + let root := System.FilePath.mk s!"/tmp/beam-typed-registry-test-{← IO.monoNanosNow}" + try + IO.FS.createDirAll root + let registryPath ← Beam.Daemon.registryPath root + if let some parent := registryPath.parent then + IO.FS.createDirAll parent + + match ← Beam.Daemon.readRegistry root with + | .absent => pure () + | state => throw <| IO.userError s!"missing registry was classified as {state.status}" + + IO.FS.writeFile registryPath "{\"daemonId\":\"legacy\"}\n" + match ← Beam.Daemon.readRegistry root with + | .legacy => pure () + | state => throw <| IO.userError s!"legacy registry was classified as {state.status}" + + IO.FS.writeFile registryPath "{\"schemaVersion\":999}\n" + match ← Beam.Daemon.readRegistry root with + | .unsupported 999 => pure () + | state => throw <| IO.userError s!"unsupported registry was classified as {state.status}" + + IO.FS.writeFile registryPath "{\"schemaVersion\":\"one\"}\n" + match ← Beam.Daemon.readRegistry root with + | .malformed detail => + require "mistyped registry schemaVersion should be malformed" + (detail.contains "invalid registry schemaVersion") + | state => throw <| IO.userError s!"mistyped registry version was classified as {state.status}" + + IO.FS.writeFile registryPath "{" + match ← Beam.Daemon.readRegistry root with + | .malformed detail => + require "malformed registry should preserve parse context" (detail.contains "invalid registry JSON") + | state => throw <| IO.userError s!"malformed registry was classified as {state.status}" + + IO.FS.writeFile registryPath "{\"schemaVersion\":1}\n" + match ← Beam.Daemon.readRegistry root with + | .malformed detail => + require "incomplete current registry should preserve schema context" + (detail.contains "invalid registry schema") + | state => throw <| IO.userError s!"incomplete registry was classified as {state.status}" + + writeTestRegistryEntry root + match ← Beam.Daemon.readRegistry root with + | .current entry => + require "current registry should preserve its generation capability" + (entry.capability == "test-capability") + | state => throw <| IO.userError s!"current registry was classified as {state.status}" + let debug ← Beam.Daemon.daemonDebugContextJson root + let debugRegistry ← IO.ofExcept <| debug.getObjVal? "registry" + requireJsonString "daemon debug context must redact its capability" + "capability" "" debugRegistry + finally + try + if ← root.pathExists then + IO.FS.removeDirAll root + catch _ => + pure () + private def checkBrokerConnectionClosedIncident : IO Unit := do let root := System.FilePath.mk s!"/tmp/beam-broker-connection-closed-incident-{← IO.monoNanosNow}" try @@ -937,38 +1006,6 @@ private def checkCurrentPidDomain : IO Unit := do require "Darwin processes should share the explicit host PID domain" (domain? == some "host:Darwin") -private def checkCrossDomainRegistryPidGuard : IO Unit := do - let child ← IO.Process.spawn { - cmd := "sleep" - args := #["30"] - stdin := .null - stdout := .null - stderr := .null - } - try - let entry : Beam.Daemon.RegistryEntry := { - daemonId := "cross-domain-pid-guard" - pid := child.pid.toNat - pidDomain? := some "beam-test-other-pid-domain" - ownerPid := child.pid.toNat - ownerPidDomain? := some "beam-test-other-pid-domain" - root := "/tmp/beam-cross-domain-pid-guard" - configHash := "cross-domain-pid-guard" - startedAt := "2026-08-25T00:00:00Z" - } - Beam.Cli.finishRegistryDaemonShutdown entry - require "a cross-domain registry PID must not be waited on or killed" - (← child.tryWait).isNone - finally - try - child.kill - catch _ => - pure () - try - discard <| child.wait - catch _ => - pure () - private def checkPathCanonicalization : IO Unit := do let stamp ← IO.monoNanosNow let root := System.FilePath.mk s!"/tmp/beam-path-canonical-root-{stamp}" @@ -994,52 +1031,22 @@ private def checkPathCanonicalization : IO Unit := do private def checkLockLifecycle : IO Unit := do let root := System.FilePath.mk s!"/tmp/beam-cli-lock-test-{← IO.monoNanosNow}" - let lockDir := root / "lock" - let some pidDomain := ← Beam.currentPidDomain? - | throw <| IO.userError "lock lifecycle test requires a known PID domain" - let writeOwner := fun (pid : Nat) (domain : String) => do - IO.FS.writeFile (lockDir / "pid") s!"{pid}\n" - IO.FS.writeFile (lockDir / "pid-domain") s!"{domain}\n" + let lockPath := root / "lock" try - Beam.Cli.withLock lockDir do - require "lock directory should exist while lock is held" (← lockDir.pathExists) - require "lock pid file should exist while lock is held" (← (lockDir / "pid").pathExists) - require "lock PID domain file should exist while lock is held" - (← (lockDir / "pid-domain").pathExists) - require "lock directory should be removed after release" (!(← lockDir.pathExists)) - - IO.FS.createDirAll lockDir - writeOwner 999999999 pidDomain - Beam.Cli.withLock lockDir do - let pidText := (← IO.FS.readFile (lockDir / "pid")).trimAscii.toString - require "stale lock should be replaced with this process lock" (pidText != "999999999") - - IO.FS.createDirAll lockDir - let selfPid ← IO.Process.getPID - writeOwner selfPid.toNat pidDomain - expectIoErrorContains "live lock timeout" s!"lock owner: pid {selfPid}" <| - Beam.Cli.withLockTimeout lockDir 100 do - pure () - IO.FS.removeDirAll lockDir - - IO.FS.createDirAll lockDir - writeOwner 999999999 (pidDomain ++ "-other") - expectIoErrorContains "cross-domain dead lock timeout" "lock owner: pid 999999999" <| - Beam.Cli.withLockTimeout lockDir 100 do - pure () - require "a dead PID from another domain should not make a lock stale" - (← lockDir.pathExists) - IO.FS.removeDirAll lockDir - - let deadPidTarget := root / "dead-pid" - IO.FS.writeFile deadPidTarget "999999999\n" - IO.FS.createDirAll lockDir - createSymlink "lock PID fixture" deadPidTarget (lockDir / "pid") - expectIoErrorContains "symlinked lock PID timeout" "lock owner: unknown owner" <| - Beam.Cli.withLockTimeout lockDir 100 do - pure () - require "a lock with a non-regular PID file should not be removed as stale" - (← lockDir.pathExists) + Beam.Cli.withLock lockPath do + require "lock file should exist while lock is held" (← lockPath.pathExists) + expectIoErrorContains "contended kernel lock timeout" "timed out after" <| + Beam.Cli.withLockTimeout lockPath 100 do + pure () + require "stable lock file should remain after release" (← lockPath.pathExists) + + Beam.Cli.withLockTimeout lockPath 100 do + require "released kernel lock should be immediately reusable" true + + Beam.Cli.withLock lockPath do + expectIoErrorContains "second contended kernel lock timeout" "timeout: 100 ms" <| + Beam.Cli.withLockTimeout lockPath 100 do + pure () finally try if ← root.pathExists then @@ -1338,12 +1345,12 @@ def main : IO Unit := do checkSilentShutdownTimeout checkPlainBrokerTaskCancellation checkBrokerConnectionClosedIncident + checkTypedRegistryReads checkDaemonFailureIncidentRetention checkDoctorDaemonFailureIncidentLines checkPathRelativeToRoot checkLeanModuleNamePathHelpers checkCurrentPidDomain - checkCrossDomainRegistryPidGuard checkPathCanonicalization checkLockLifecycle checkLeanToolchainPolicyParsing diff --git a/tests/lean/BeamTest/Broker/ProtocolTest.lean b/tests/lean/BeamTest/Broker/ProtocolTest.lean index 1ff26db5..0f24a35f 100644 --- a/tests/lean/BeamTest/Broker/ProtocolTest.lean +++ b/tests/lean/BeamTest/Broker/ProtocolTest.lean @@ -1069,6 +1069,44 @@ private def checkSessionCloseAdmission : IO Unit := do require "closed admission should leave no active request" ((← ActiveRequestRegistry.count runtime.activeRequests) == 0) +private def checkWrapperDaemonAuthorization : IO Unit := do + let root := System.FilePath.mk "/tmp/beam-wrapper-daemon-authorization" + let capability := "generation-secret" + let runtime ← Beam.Broker.ServerRuntime.create + ({ root } : Beam.Broker.BrokerConfig) "fixture" + (some { daemonId := "generation-a", configHash := "config-a" }) + (some capability) + try + for (label, capability?) in [ + ("missing", none), + ("wrong", some "another-generation-secret") + ] do + let response ← runtime.dispatchRequest { + op := .stats + daemonCapability? := capability? + } + require s!"wrapper daemon should reject {label} capability" + (response.error?.any fun err => + err.code == "invalidParams" && err.message.contains "invalid Beam daemon capability") + + let stats ← runtime.dispatchRequest { + op := .stats + daemonCapability? := some capability + } + require "wrapper daemon should admit the exact generation capability" stats.ok + + for op in [Op.initWorkspace, .dropWorkspace] do + let response ← runtime.dispatchRequest { + op + workspaceId? := some "fixture" + daemonCapability? := some capability + } + require s!"wrapper daemon should disable dynamic {op.key}" + (response.error?.any fun err => + err.code == "invalidParams" && err.message.contains "wrapper-owned daemon mode") + finally + runtime.close + def main : IO Unit := do checkResponseJsonShape checkStreamMessageDecode @@ -1085,6 +1123,7 @@ def main : IO Unit := do checkWorkspaceLifecycleProtocol checkLifecycleTeardownConcurrency checkSessionCloseAdmission + checkWrapperDaemonAuthorization end BeamTest.Broker.ProtocolTest diff --git a/tests/lean/BeamTest/Broker/RequestStreamContractTest.lean b/tests/lean/BeamTest/Broker/RequestStreamContractTest.lean index d605a3d6..1a9a4d2d 100644 --- a/tests/lean/BeamTest/Broker/RequestStreamContractTest.lean +++ b/tests/lean/BeamTest/Broker/RequestStreamContractTest.lean @@ -114,7 +114,7 @@ def main : IO Unit := do let broker ← spawnLeanBroker endpoint root (identity? := some identity) try waitForBrokerReadyForRoot endpoint root - match ← Beam.Daemon.daemonGenerationStatus endpoint testWorkspaceId root identity with + match ← Beam.Daemon.daemonGenerationStatus endpoint testWorkspaceId root identity "test-capability" with | .exact => pure () | status => throw <| IO.userError @@ -123,13 +123,13 @@ def main : IO Unit := do { identity with daemonId := identity.daemonId ++ "-other" }, { identity with configHash := identity.configHash ++ "-other" } ] do - match ← Beam.Daemon.daemonGenerationStatus endpoint testWorkspaceId root mismatched with + match ← Beam.Daemon.daemonGenerationStatus endpoint testWorkspaceId root mismatched "test-capability" with | .wrongGeneration _ => pure () | status => throw <| IO.userError s!"daemon generation mismatch was classified as {repr status}" let otherRoot := root / "other-root" IO.FS.createDirAll otherRoot - match ← Beam.Daemon.daemonGenerationStatus endpoint testWorkspaceId otherRoot identity with + match ← Beam.Daemon.daemonGenerationStatus endpoint testWorkspaceId otherRoot identity "test-capability" with | .wrongRoot _ => pure () | status => throw <| IO.userError s!"daemon root mismatch was classified as {repr status}" @@ -316,7 +316,7 @@ def main : IO Unit := do sendShutdownAndResetConnection port waitForBrokerExit broker - match ← Beam.Daemon.daemonGenerationStatus endpoint testWorkspaceId root identity with + match ← Beam.Daemon.daemonGenerationStatus endpoint testWorkspaceId root identity "test-capability" with | .unavailable => pure () | status => throw <| IO.userError s!"stopped daemon was classified as {repr status}" diff --git a/tests/test-beam-prune.sh b/tests/test-beam-prune.sh index 28d3eb42..aaaed63b 100644 --- a/tests/test-beam-prune.sh +++ b/tests/test-beam-prune.sh @@ -236,14 +236,16 @@ race_lock_held="$tmp_root/race-lock-held" race_lock_release="$tmp_root/race-lock-release" race_err="$tmp_root/race.err" ( + mkdir "$race_lock" touch "$race_lock_held" while [ ! -e "$race_lock_release" ]; do sleep 0.05 done + rm -f "$race_lock/pid" "$race_lock/pid-domain" + rmdir "$race_lock" ) & lock_writer_pid="$!" wait_for_file "$race_lock_held" "prune install-lock holder" 10 -mkdir "$race_lock" write_lock_owner "$race_lock" "$lock_writer_pid" BEAM_HOME="$current_runtime" "$beam_cli" install-prune --apply > /dev/null 2>"$race_err" & race_pid="$!" @@ -310,8 +312,24 @@ PY resolved_partial_runtime="$(beam_test_realpath "$partial_runtime")" stale_bundle_lock="$bundle_root/.locks/200" -mkdir -p "$stale_bundle_lock" -write_lock_owner "$stale_bundle_lock" "$$" +bundle_lock_held="$tmp_root/bundle-lock-held" +bundle_lock_release="$tmp_root/bundle-lock-release" +mkdir -p "$(dirname "$stale_bundle_lock")" +python3 - "$stale_bundle_lock" "$bundle_lock_held" "$bundle_lock_release" <<'PY' & +import fcntl +import pathlib +import sys +import time + +lock_path, held_path, release_path = map(pathlib.Path, sys.argv[1:]) +with lock_path.open("a", encoding="utf-8") as lock_file: + fcntl.flock(lock_file, fcntl.LOCK_EX) + held_path.touch() + while not release_path.exists(): + time.sleep(0.05) +PY +lock_writer_pid="$!" +wait_for_file "$bundle_lock_held" "prune bundle-lock holder" 10 bundle_lock_out="$tmp_root/bundle-lock.out" bundle_lock_err="$tmp_root/bundle-lock.err" if "$install_root/current/bin/lean-beam" prune --apply --bundles \ @@ -327,8 +345,9 @@ assert_contains_literal "$bundle_lock_err" \ assert_contains_literal "$bundle_lock_err" \ 'rerun `lean-beam prune --bundles` to preview the remaining paths' assert_not_exists "$partial_runtime" -rm -f "$stale_bundle_lock/pid" "$stale_bundle_lock/pid-domain" -rmdir "$stale_bundle_lock" +touch "$bundle_lock_release" +wait "$lock_writer_pid" +lock_writer_pid="" assert_file "$stale_bundle/metadata.json" apply_bundle_out="$("$install_root/current/bin/lean-beam" prune --apply --bundles)" diff --git a/tests/test-beam-wrapper-daemon.sh b/tests/test-beam-wrapper-daemon.sh index 573be033..99865411 100644 --- a/tests/test-beam-wrapper-daemon.sh +++ b/tests/test-beam-wrapper-daemon.sh @@ -247,6 +247,56 @@ assert_json_field_equals "owned ensure response" "$ensure_json" ok true stats_json="$("$beam_script" --root "$tmp1" stats)" assert_json_field_equals "owned stats response" "$stats_json" ok true +case "$(uname -s)" in + Darwin) registry_mode="$(stat -f '%Lp' "$registry")" ;; + *) registry_mode="$(stat -c '%a' "$registry")" ;; +esac +if [ "$registry_mode" != "600" ]; then + echo "expected the capability-bearing registry to use mode 600, got $registry_mode" >&2 + exit 1 +fi + +port1="$(read_json_field "$registry" port)" +python3 - "$port1" <<'PY' +import json +import socket +import sys +import time + +port = int(sys.argv[1]) + +def receive_frame(sock): + header = bytearray() + while not header.endswith(b"\n"): + chunk = sock.recv(1) + if not chunk: + raise RuntimeError("daemon closed before returning a framed error") + header.extend(chunk) + size = int(header[:-1]) + payload = bytearray() + while len(payload) < size: + chunk = sock.recv(size - len(payload)) + if not chunk: + raise RuntimeError("daemon closed during its framed error") + payload.extend(chunk) + return json.loads(payload) + +with socket.create_connection(("127.0.0.1", port), timeout=3) as sock: + sock.sendall(b"16777217\n") + response = receive_frame(sock) + if "exceeds 16777216 bytes" not in response.get("payload", {}).get("error", {}).get("message", ""): + raise RuntimeError(f"unexpected oversized-frame response: {response}") + +with socket.create_connection(("127.0.0.1", port), timeout=3) as sock: + time.sleep(5.5) + response = receive_frame(sock) + if "initial request timed out" not in response.get("payload", {}).get("error", {}).get("message", ""): + raise RuntimeError(f"unexpected first-message-timeout response: {response}") +PY + +stats_after_limits_json="$("$beam_script" --root "$tmp1" stats)" +assert_json_field_equals "stats after transport limit probes" "$stats_after_limits_json" ok true + second_owner_out="$tmp1/second-owner.out" second_owner_err="$tmp1/second-owner.err" if "$beam_script" --root "$tmp1" ensure --hold > "$second_owner_out" 2> "$second_owner_err"; then @@ -260,7 +310,6 @@ if ! grep -Fq "already owned" "$second_owner_err"; then exit 1 fi -port1="$(read_json_field "$registry" port)" collision_out="$tmp2/collision.out" collision_err="$tmp2/collision.err" if "$beam_script" --root "$tmp2" --port "$port1" ensure --hold > "$collision_out" 2> "$collision_err"; then @@ -268,8 +317,8 @@ if "$beam_script" --root "$tmp2" --port "$port1" ensure --hold > "$collision_out cat "$collision_out" >&2 exit 1 fi -if ! grep -Fq "already serves Beam root" "$collision_err"; then - echo "expected endpoint collision to identify the served project root" >&2 +if ! grep -Fq "invalid Beam daemon capability" "$collision_err"; then + echo "expected endpoint collision not to disclose an authenticated daemon's project" >&2 cat "$collision_err" >&2 exit 1 fi @@ -293,16 +342,66 @@ with open(replacement, "w", encoding="utf-8") as stream: stream.write("\n") os.replace(replacement, os.environ["STALE_REGISTRY"]) PY -"$beam_script" --root "$tmp2" shutdown > /dev/null -if [ -e "$stale_registry" ]; then - echo "expected shutdown to remove a stale cross-root registry" >&2 - cat "$stale_registry" >&2 +stale_shutdown_out="$tmp2/stale-cross-root-shutdown.out" +stale_shutdown_err="$tmp2/stale-cross-root-shutdown.err" +if "$beam_script" --root "$tmp2" shutdown \ + > "$stale_shutdown_out" 2> "$stale_shutdown_err"; then + echo "expected shutdown to reject a registry whose endpoint serves another root" >&2 + cat "$stale_shutdown_out" >&2 + exit 1 +fi +if ! grep -Fq "serves another root" "$stale_shutdown_err"; then + echo "expected cross-root registry rejection to explain the identity mismatch" >&2 + cat "$stale_shutdown_err" >&2 + exit 1 +fi +if [ ! -e "$stale_registry" ]; then + echo "cross-root registry rejection must preserve the unsafe registry as recovery evidence" >&2 exit 1 fi if ! kill -0 "$owner1_pid" 2>/dev/null || ! kill -0 "$daemon1_pid" 2>/dev/null; then - echo "stale cross-root registry cleanup must not stop the daemon or owner serving the other root" >&2 + echo "cross-root registry rejection must not stop the daemon or owner serving the other root" >&2 exit 1 fi +rm -f -- "$stale_registry" + +LEGACY_REGISTRY="$stale_registry" LEGACY_ROOT="$tmp2" python3 - <<'PY' +import json +import os + +entry = { + "daemonId": "legacy-generation", + "pid": 999999999, + "ownerPid": 999999999, + "port": 42424, + "root": os.path.realpath(os.environ["LEGACY_ROOT"]), + "configHash": "legacy-config", + "startedAt": "2026-08-27T00:00:00Z", +} +with open(os.environ["LEGACY_REGISTRY"], "w", encoding="utf-8") as stream: + json.dump(entry, stream, separators=(",", ":")) + stream.write("\n") +PY +legacy_before="$(cat "$stale_registry")" +legacy_owner_out="$tmp2/legacy-owner.out" +legacy_owner_err="$tmp2/legacy-owner.err" +if "$beam_script" --root "$tmp2" ensure --hold \ + > "$legacy_owner_out" 2> "$legacy_owner_err"; then + echo "expected owner startup to reject a schema-less legacy registry" >&2 + cat "$legacy_owner_out" >&2 + exit 1 +fi +if ! grep -Fq "legacy registry has no schemaVersion" "$legacy_owner_err"; then + echo "expected legacy-registry rejection to explain the unsupported schema" >&2 + cat "$legacy_owner_err" >&2 + exit 1 +fi +if [ "$(cat "$stale_registry")" != "$legacy_before" ]; then + echo "legacy-registry rejection must preserve the recovery evidence" >&2 + cat "$stale_registry" >&2 + exit 1 +fi +rm -f -- "$stale_registry" busy_port_file="$(mktemp "$tmp2/non-beam-port-XXXXXX")" python3 - "$busy_port_file" <<'PY' & @@ -398,6 +497,37 @@ busy_port_file="" start_slow_request "$tmp1" "shutdown-active" "shutdown-active" +# The desired configuration includes the installed bundle paths. Pointing an ordinary command at +# an equivalent bundle in another location creates legitimate desired-hash drift without changing +# the identity of the running generation. The lookup must preserve both the owner and its request. +drift_bundle_dir="$tmp2/config-drift-bundles" +mkdir -p "$drift_bundle_dir" +rsync -a "$BEAM_INSTALL_BUNDLE_DIR/" "$drift_bundle_dir/" +drift_out="$tmp1/config-drift.out" +drift_err="$tmp1/config-drift.err" +if BEAM_INSTALL_BUNDLE_DIR="$drift_bundle_dir" \ + "$beam_script" --root "$tmp1" ensure > "$drift_out" 2> "$drift_err"; then + echo "expected desired configuration drift to reject attachment" >&2 + cat "$drift_out" >&2 + exit 1 +fi +if ! grep -Fq "current owner was preserved" "$drift_err"; then + echo "expected configuration-drift diagnostics to preserve the current owner" >&2 + cat "$drift_err" >&2 + exit 1 +fi +if [ "$(read_json_field "$registry" daemonId)" != "$daemon1_id" ] || \ + [ "$(read_json_field "$registry" pid)" != "$daemon1_pid" ]; then + echo "configuration-drift lookup changed the live daemon generation" >&2 + cat "$registry" >&2 + exit 1 +fi +if ! kill -0 "$owner1_pid" 2>/dev/null || ! kill -0 "$daemon1_pid" 2>/dev/null || \ + ! kill -0 "$active_request_pid" 2>/dev/null; then + echo "configuration-drift lookup terminated the owner, daemon, or active request" >&2 + exit 1 +fi + shutdown_json="$("$beam_script" --root "$tmp1" shutdown)" assert_json_field_equals "explicit session shutdown" "$shutdown_json" ok true expect_slow_request_cancelled "$tmp1" "shutdown-active" "shutdown-active" @@ -457,19 +587,26 @@ if [ -e "$registry" ]; then exit 1 fi -start_owner "$tmp1" "owner-unpublish-before-drain" +start_owner "$tmp1" "owner-draining-fence" draining_daemon_pid="$(read_json_field "$registry" pid)" +draining_daemon_id="$(read_json_field "$registry" daemonId)" +start_slow_request "$tmp1" "draining-process-tree" "draining-process-tree" +draining_backend_pids="$(pgrep -P "$draining_daemon_pid" || true)" +if [ -z "$draining_backend_pids" ]; then + echo "expected the active request to create a daemon-owned backend process" >&2 + exit 1 +fi kill -STOP "$draining_daemon_pid" paused_daemon_pid="$draining_daemon_pid" kill -INT "$hold_pid" for _ in $(seq 1 40); do - if [ ! -e "$registry" ]; then + if [ -e "$registry" ] && [ "$(read_json_field "$registry" lifecycle)" = "draining" ]; then break fi sleep 0.05 done -if [ -e "$registry" ]; then - echo "expected an interrupted owner to unpublish its generation before daemon drain" >&2 +if [ ! -e "$registry" ] || [ "$(read_json_field "$registry" lifecycle)" != "draining" ]; then + echo "expected an interrupted owner to retain a draining generation fence" >&2 cat "$registry" >&2 exit 1 fi @@ -480,24 +617,62 @@ fi draining_lookup_out="$tmp1/draining-lookup.out" draining_lookup_err="$tmp1/draining-lookup.err" if "$beam_script" --root "$tmp1" ensure > "$draining_lookup_out" 2> "$draining_lookup_err"; then - echo "expected an ordinary command not to attach to an unpublished draining generation" >&2 + echo "expected an ordinary command not to attach to a draining generation" >&2 cat "$draining_lookup_out" >&2 exit 1 fi -if ! grep -Fq "lean-beam ensure --hold" "$draining_lookup_err"; then - echo "expected draining-generation recovery to require a new explicit owner" >&2 +if ! grep -Fq "is draining" "$draining_lookup_err"; then + echo "expected ordinary commands to report the draining generation" >&2 cat "$draining_lookup_err" >&2 exit 1 fi -kill -CONT "$draining_daemon_pid" -paused_daemon_pid="" -if ! wait_for_exit "$hold_pid" "owner after unpublish-before-drain check" 200 0.05; then - cat "$tmp1/owner-unpublish-before-drain.err" >&2 +replacement_owner_out="$tmp1/replacement-during-drain.out" +replacement_owner_err="$tmp1/replacement-during-drain.err" +if "$beam_script" --root "$tmp1" ensure --hold \ + > "$replacement_owner_out" 2> "$replacement_owner_err"; then + echo "expected a draining generation to fence out a replacement owner" >&2 + cat "$replacement_owner_out" >&2 + exit 1 +fi +if ! grep -Fq "is draining" "$replacement_owner_err"; then + echo "expected replacement-owner rejection to identify the draining generation" >&2 + cat "$replacement_owner_err" >&2 + exit 1 +fi +if [ "$(read_json_field "$registry" daemonId)" != "$draining_daemon_id" ] || \ + [ "$(read_json_field "$registry" pid)" != "$draining_daemon_pid" ]; then + echo "replacement attempt changed the draining generation fence" >&2 + cat "$registry" >&2 + exit 1 +fi +if ! wait_for_exit "$hold_pid" "owner after forced draining-fence cleanup" 300 0.05; then + cat "$tmp1/owner-draining-fence.err" >&2 exit 1 fi wait "$hold_pid" hold_pid="" -if ! wait_for_exit "$draining_daemon_pid" "daemon after unpublish-before-drain check" 200 0.05; then +paused_daemon_pid="" +if ! wait_for_exit "$draining_daemon_pid" "daemon after forced draining-fence cleanup" 40 0.05; then + exit 1 +fi +for backend_pid in $draining_backend_pids; do + if ! wait_for_exit "$backend_pid" "backend after forced draining-fence cleanup" 40 0.05; then + echo "forced owner cleanup left backend pid $backend_pid alive" >&2 + exit 1 + fi +done +set +e +wait "$active_request_pid" +draining_request_status="$?" +set -e +active_request_pid="" +if [ "$draining_request_status" -eq 0 ]; then + echo "expected the active request to fail when forced drain kills its process group" >&2 + exit 1 +fi +if [ -e "$registry" ]; then + echo "expected the draining fence to disappear only after the daemon was reaped" >&2 + cat "$registry" >&2 exit 1 fi @@ -526,9 +701,8 @@ if ! grep -Fq "lean-beam ensure --hold" "$owner_loss_err"; then cat "$owner_loss_err" >&2 exit 1 fi -if [ -e "$registry" ]; then - echo "expected owner-loss recovery to remove the stale registry" >&2 - cat "$registry" >&2 +if [ ! -e "$registry" ]; then + echo "ordinary owner-loss lookup must not mutate the stale registry" >&2 exit 1 fi @@ -537,15 +711,18 @@ start_owner "$tmp2" "owner-generation" generation_daemon_pid="$(read_json_field "$generation_registry" pid)" generation_id="$(read_json_field "$generation_registry" daemonId)" replacement_generation_id="$generation_id-replacement" -python3 - "$generation_registry" "$replacement_generation_id" <<'PY' +replacement_generation_capability="replacement-generation-capability" +python3 - "$generation_registry" "$replacement_generation_id" \ + "$replacement_generation_capability" <<'PY' import json import os import sys -path, replacement_id = sys.argv[1:] +path, replacement_id, replacement_capability = sys.argv[1:] with open(path, "r", encoding="utf-8") as stream: entry = json.load(stream) entry["daemonId"] = replacement_id +entry["capability"] = replacement_capability replacement = path + ".replacement" with open(replacement, "w", encoding="utf-8") as stream: json.dump(entry, stream, separators=(",", ":")) @@ -566,6 +743,12 @@ if [ "$(read_json_field "$generation_registry" daemonId)" != "$replacement_gener cat "$generation_registry" >&2 exit 1 fi +if [ "$(read_json_field "$generation_registry" capability)" != \ + "$replacement_generation_capability" ]; then + echo "expected old-owner cleanup to preserve the replacement capability" >&2 + cat "$generation_registry" >&2 + exit 1 +fi rm -f -- "$generation_registry" start_owner "$tmp1" "owner-4" @@ -589,3 +772,7 @@ if [ "$root_owner_status" -ne 0 ]; then echo "expected root-disappearance owner to exit cleanly, got $root_owner_status" >&2 exit 1 fi +if [ -e "$tmp1" ]; then + echo "owner cleanup recreated the removed project root" >&2 + exit 1 +fi diff --git a/tests/test-beam-wrapper-sandbox.sh b/tests/test-beam-wrapper-sandbox.sh index 1411ca7f..5b2abf72 100755 --- a/tests/test-beam-wrapper-sandbox.sh +++ b/tests/test-beam-wrapper-sandbox.sh @@ -83,8 +83,8 @@ sandbox_beam() { wait_for_registry() { local remaining=300 while [ "$remaining" -gt 0 ]; do - # The control lock is intentionally short-lived and may disappear while `find` walks the - # per-root directory. Ignore that observational traversal race and keep probing for the file. + # Registry publication is concurrent with this traversal. Ignore transient observational + # misses while the per-root control directory is being created. registry="$(find "$control_root" -name beam-daemon.json -print 2>/dev/null | sed -n '1p' || true)" if [ -n "$registry" ] && [ -f "$registry" ]; then return 0 @@ -280,7 +280,9 @@ if [ "$daemon_id_1" = "$daemon_id_2" ]; then fi # Killing the holder closes the only write end of the inherited owner pipe. The daemon must stop -# without a heartbeat timeout, and the next ordinary command must clean the stale registry. +# without a heartbeat timeout. A later command in another PID namespace cannot prove that the +# foreign-domain process identities are gone, so it must preserve the registry for supervised +# recovery rather than silently treating endpoint unavailability as replacement authority. touch "$owner_kill" if ! wait_for_exit "$owner_pid" "killed sandbox owner" 120 0.1; then sed -n '1,200p' "$owner_err" >&2 @@ -304,16 +306,20 @@ if sandbox_beam ensure >"$after_kill_out" 2>"$after_kill_err"; then sed -n '1,160p' "$after_kill_out" >&2 exit 1 fi -if ! grep -Fq "start 'lean-beam ensure --hold'" "$after_kill_err"; then - echo "expected dead-owner recovery to require a new explicit owner" >&2 +if ! grep -Fq "recorded daemon endpoint is unavailable" "$after_kill_err"; then + echo "expected cross-domain owner loss to fail closed on the unavailable endpoint" >&2 sed -n '1,160p' "$after_kill_err" >&2 exit 1 fi -if find "$control_root" -name beam-daemon.json -print -quit | grep -q .; then - echo "expected dead-owner recovery to remove the stale registry" >&2 +if ! find "$control_root" -name beam-daemon.json -print -quit | grep -q .; then + echo "expected ordinary cross-domain recovery to preserve the unsafe registry" >&2 exit 1 fi +# This test harness supervised the complete bwrap owner namespace and observed its exit, so it can +# now perform the out-of-band recovery that an unsupervised client must refuse to infer. +rm -f -- "$registry" + sandbox_owner owner-3 if ! wait_for_registry || ! wait_for_nonempty_file "$owner_out" "final sandbox owner response"; then sed -n '1,200p' "$owner_err" >&2 From 7b015674f74ec5038bd92402f6f4eb594912f1bf Mon Sep 17 00:00:00 2001 From: Emilio Jesus Gallego Arias Date: Fri, 28 Aug 2026 01:17:44 +0200 Subject: [PATCH 22/28] fix: harden daemon capability boundaries --- Beam/Broker/Server.lean | 11 ++- Beam/Cli/DaemonManager.lean | 111 ++++++++++++++----------- Beam/Cli/Feedback.lean | 4 +- Beam/Cli/Info.lean | 6 +- Beam/Daemon/Debug.lean | 53 +----------- Beam/Daemon/Protocol.lean | 3 + Beam/Daemon/Registry.lean | 65 +++++++++++++++ CHANGELOG.md | 2 +- docs/COMPATIBILITY.md | 5 +- docs/TESTING.md | 8 +- tests/test-beam-wrapper-daemon.sh | 24 +++++- tests/test-beam-wrapper-diagnostics.sh | 15 +++- tests/test-beam-wrapper.sh | 21 +++-- 13 files changed, 203 insertions(+), 125 deletions(-) create mode 100644 Beam/Daemon/Registry.lean diff --git a/Beam/Broker/Server.lean b/Beam/Broker/Server.lean index bb7e3c06..26c1f34a 100644 --- a/Beam/Broker/Server.lean +++ b/Beam/Broker/Server.lean @@ -1055,6 +1055,9 @@ private def awaitRuntimeClose | .ok () => pure () | .error err => throw err +private def ServerRuntime.closeStarted (server : ServerRuntime) : IO Bool := + server.closeMutex.atomically get + /-- Close broker admission, cancel admitted requests, shut down every backend session, and wait for all admitted dispatch scopes to unregister. Concurrent and repeated callers wait for the same @@ -2628,10 +2631,6 @@ private def handleClient | Except.error failure => sendResponse (← clientRequestIdRef.get) failure.toResponse | Except.ok req => - let stopsTransport := - req.op == .shutdown && match req.validateFields with - | .ok () => true - | .error _ => false let emitProgress : SyncFileProgress → IO Unit := fun progress => Transport.sendMsg client (toJson (StreamMessage.fileProgress req.clientRequestId? progress)).compress @@ -2641,6 +2640,10 @@ private def handleClient let resp ← server.dispatchRequestWithHandle req (fun handle => do let _ ← IO.asTask (prio := Task.Priority.dedicated) <| watchClientDisconnect client handle pure true) (some emitProgress) (some emitDiagnostic) + -- Request validation alone does not grant shutdown authority. Only stop the listener once + -- dispatch has authenticated the capability and started runtime closure. + let stopsTransport ← + if req.op == .shutdown then server.closeStarted else pure false if stopsTransport then -- A successful send is the transport's flush boundary. Wake the listener only after the -- terminal response has been handed off, but do so even when the caller disconnected so diff --git a/Beam/Cli/DaemonManager.lean b/Beam/Cli/DaemonManager.lean index 86520973..99b9995e 100644 --- a/Beam/Cli/DaemonManager.lean +++ b/Beam/Cli/DaemonManager.lean @@ -12,6 +12,7 @@ import Beam.Cli.Lock import Beam.Cli.Project import Beam.Daemon.Debug import Beam.Daemon.Paths +import Beam.Daemon.Registry open Lean @@ -102,11 +103,23 @@ private def writeRegistry (control : ProjectControl) (entry : RegistryEntry) : I if let some parent := control.registry.parent then IO.FS.createDirAll parent let tmp := control.registry.withExtension "tmp" - IO.FS.writeFile tmp ((toJson entry).pretty ++ "\n") - IO.setAccessRights tmp { - user := { read := true, write := true } - } - IO.FS.rename tmp control.registry + try + IO.FS.withFile tmp .write fun handle => do + -- The registry contains the daemon capability. Make the inode private before publishing any + -- bytes, rather than relying on a post-write chmod window or the caller's umask. + IO.setAccessRights tmp { + user := { read := true, write := true } + } + handle.putStr ((toJson entry).pretty ++ "\n") + handle.flush + IO.FS.rename tmp control.registry + catch err => + try + if ← tmp.pathExists then + IO.FS.removeFile tmp + catch _ => + pure () + throw err private def writeExistingRegistry (control : ProjectControl) (entry : RegistryEntry) : IO Unit := do -- Teardown must not create a path while the project tree is being removed. Rewrite through an @@ -164,8 +177,7 @@ inductive RegistryObservation where | legacy | unsupported (schemaVersion : Nat) | malformed (detail : String) - | liveExact (entry : RegistryEntry) - | liveConfigMismatch (entry : RegistryEntry) (expectedHash : String) + | live (entry : RegistryEntry) | draining (entry : RegistryEntry) | staleConfirmed (entry : RegistryEntry) | unusable (entry : RegistryEntry) (reason : RegistryUnsafeReason) @@ -183,9 +195,7 @@ private def registryProcessesGone (entry : RegistryEntry) : IO Bool := do private def registryOwnerKnownDead (entry : RegistryEntry) : IO Bool := recordedPidGone entry.ownerPid entry.ownerPidDomain? -def observeProjectRegistry - (root : System.FilePath) - (expectedHash? : Option String := none) : IO RegistryObservation := do +def observeProjectRegistry (root : System.FilePath) : IO RegistryObservation := do match ← readRegistry root with | .absent => pure .absent | .legacy => pure .legacy @@ -212,13 +222,7 @@ def observeProjectRegistry if ownerDead then pure <| .unusable entry .ownerDead else - match expectedHash? with - | some expectedHash => - if expectedHash == entry.configHash then - pure <| .liveExact entry - else - pure <| .liveConfigMismatch entry expectedHash - | none => pure <| .liveExact entry + pure <| .live entry | .unavailable => if ownerDead && (← registryProcessesGone entry) then pure <| .staleConfirmed entry @@ -359,7 +363,7 @@ private def writeDaemonFailureIncident? controlDir := control.toString registryPath := registryFile.toString registry := registry.map fun entry => - (toJson entry).setObjVal! "capability" (toJson "") + entry.redactedJson registryPidStatus := pidStatus registryEndpoint := endpoint startupLogPath := logTail?.map (fun (path, _) => path.toString) @@ -477,9 +481,15 @@ private def startDaemon cwd := some desired.root setsid := true } - child.stdin.putStrLn capability - child.stdin.flush - pure child + try + child.stdin.putStrLn capability + child.stdin.flush + pure child + catch err => + -- Once spawned, the retained child handle owns the whole setsid process group. Do not leak + -- that acquisition when publishing the capability through the owner pipe fails. + terminateDaemonChild child + throw err private def daemonStartupTimeoutMs : Nat := 30000 @@ -716,6 +726,12 @@ private def registryRecoveryMessage (root : System.FilePath) (detail : String) : s!"Beam cannot safely use or replace the daemon registry for {root}: {detail}. " ++ "Preserve the registry, stop the matching foreground owner or daemon explicitly, and retry" +private def registryReadRecoveryMessage + (root : System.FilePath) + (registryRead : RegistryRead) : String := + registryRecoveryMessage root <| + registryRead.detail?.getD s!"unexpected registry state '{registryRead.status}'" + private def markRegistryDraining (control : ProjectControl) (entry : RegistryEntry) : IO Unit := do match ← readRegistry control.root with | .current current => @@ -736,21 +752,16 @@ def shutdownRegisteredProjectDaemon | .staleConfirmed entry => removeRegistryGeneration control entry pure ShutdownPlan.none - | .liveExact entry => + | .live entry => markRegistryDraining control entry pure <| ShutdownPlan.request entry | .draining entry => pure <| ShutdownPlan.request entry - | .liveConfigMismatch entry _ => - markRegistryDraining control entry - pure <| ShutdownPlan.request entry | .legacy => - throw <| IO.userError <| registryRecoveryMessage root <| - RegistryRead.legacy.detail?.getD "legacy registry" + throw <| IO.userError <| registryReadRecoveryMessage root .legacy | .unsupported schemaVersion => - throw <| IO.userError <| registryRecoveryMessage root <| - (RegistryRead.unsupported schemaVersion).detail?.getD "unsupported registry" + throw <| IO.userError <| registryReadRecoveryMessage root (.unsupported schemaVersion) | .malformed detail => - throw <| IO.userError <| registryRecoveryMessage root detail + throw <| IO.userError <| registryReadRecoveryMessage root (.malformed detail) | .unusable _ reason => throw <| IO.userError <| registryRecoveryMessage root reason.message match plan with @@ -806,21 +817,22 @@ private def startOwnedProjectDaemon (control : ProjectControl) (desired : DesiredConfig) (opts : CliOptions) : IO OwnedProjectDaemon := do - match ← observeProjectRegistry desired.root (some desired.configHash) with + match ← observeProjectRegistry desired.root with | .absent => pure () | .staleConfirmed entry => removeRegistryGeneration control entry - | .liveExact entry => throw <| IO.userError (activeOwnerMessage desired.root entry) - | .liveConfigMismatch entry expectedHash => - throw <| IO.userError (configMismatchMessage desired.root entry expectedHash) + | .live entry => + if entry.configHash == desired.configHash then + throw <| IO.userError (activeOwnerMessage desired.root entry) + else + throw <| IO.userError (configMismatchMessage desired.root entry desired.configHash) | .draining entry => throw <| IO.userError (drainingOwnerMessage desired.root entry) | .legacy => - throw <| IO.userError <| registryRecoveryMessage desired.root <| - RegistryRead.legacy.detail?.getD "legacy registry" + throw <| IO.userError <| registryReadRecoveryMessage desired.root .legacy | .unsupported schemaVersion => - throw <| IO.userError <| registryRecoveryMessage desired.root <| - (RegistryRead.unsupported schemaVersion).detail?.getD "unsupported registry" + throw <| IO.userError <| + registryReadRecoveryMessage desired.root (.unsupported schemaVersion) | .malformed detail => - throw <| IO.userError <| registryRecoveryMessage desired.root detail + throw <| IO.userError <| registryReadRecoveryMessage desired.root (.malformed detail) | .unusable _ reason => throw <| IO.userError <| registryRecoveryMessage desired.root reason.message let (endpoint, entry, child) ← startDaemonEntry desired opts @@ -923,21 +935,24 @@ private def lookupProjectDaemon (expectedHash? : Option String := none) (backend? : Option Backend := none) : IO ProjectDaemonClient := do withProjectControl root fun _control => do - match ← observeProjectRegistry root expectedHash? with - | .liveExact entry => projectDaemonClient entry + match ← observeProjectRegistry root with + | .live entry => + match expectedHash? with + | some expectedHash => + if entry.configHash == expectedHash then + projectDaemonClient entry + else + throw <| IO.userError (configMismatchMessage root entry expectedHash) + | none => projectDaemonClient entry | .absent | .staleConfirmed _ => throw <| IO.userError (missingOwnerMessage root backend?) - | .liveConfigMismatch entry expectedHash => - throw <| IO.userError (configMismatchMessage root entry expectedHash) | .draining entry => throw <| IO.userError (drainingOwnerMessage root entry) | .legacy => - throw <| IO.userError <| registryRecoveryMessage root <| - RegistryRead.legacy.detail?.getD "legacy registry" + throw <| IO.userError <| registryReadRecoveryMessage root .legacy | .unsupported schemaVersion => - throw <| IO.userError <| registryRecoveryMessage root <| - (RegistryRead.unsupported schemaVersion).detail?.getD "unsupported registry" + throw <| IO.userError <| registryReadRecoveryMessage root (.unsupported schemaVersion) | .malformed detail => - throw <| IO.userError <| registryRecoveryMessage root detail + throw <| IO.userError <| registryReadRecoveryMessage root (.malformed detail) | .unusable _ reason => throw <| IO.userError <| registryRecoveryMessage root reason.message diff --git a/Beam/Cli/Feedback.lean b/Beam/Cli/Feedback.lean index 9f5dd7ba..25c28730 100644 --- a/Beam/Cli/Feedback.lean +++ b/Beam/Cli/Feedback.lean @@ -95,7 +95,7 @@ private def collectDaemonPayload (root : System.FilePath) (warnings : Array String) : IO (Json × Json × Array String) := do match ← observeProjectRegistry root with - | .liveExact entry => + | .live entry => match Beam.Daemon.registryEndpoint? entry with | none => pure (Json.null, Json.null, warnings.push "Beam daemon registry did not contain a valid endpoint") @@ -116,8 +116,6 @@ private def collectDaemonPayload pure (stats, openDocs, warnings) | .absent | .staleConfirmed _ => pure (Json.null, Json.null, warnings.push "no live Beam daemon was available for stats/open-files") - | .liveConfigMismatch _ _ => - pure (Json.null, Json.null, warnings.push "the live Beam daemon has a different configuration") | .draining _ => pure (Json.null, Json.null, warnings.push "the Beam daemon is draining") | .legacy => diff --git a/Beam/Cli/Info.lean b/Beam/Cli/Info.lean index 9421ec92..19661fd3 100644 --- a/Beam/Cli/Info.lean +++ b/Beam/Cli/Info.lean @@ -168,7 +168,7 @@ def doctor (home : System.FilePath) (opts : CliOptions) (backend : Backend) : IO let registry ← Beam.Daemon.registryPath root IO.println s!"registry: {registry}" match ← observeProjectRegistry root with - | .liveExact entry => + | .live entry => IO.println "daemon status: live" IO.println s!"daemon pid: {entry.pid}" if let some pidDomain := entry.pidDomain? then @@ -178,10 +178,6 @@ def doctor (home : System.FilePath) (opts : CliOptions) (backend : Backend) : IO else IO.println "daemon endpoint: invalid" IO.println s!"daemon config hash: {entry.configHash}" - | .liveConfigMismatch entry expectedHash => - IO.println "daemon status: config mismatch" - IO.println s!"daemon config hash: {entry.configHash}" - IO.println s!"expected config hash: {expectedHash}" | .draining entry => IO.println "daemon status: draining" IO.println s!"daemon generation: {entry.daemonId}" diff --git a/Beam/Daemon/Debug.lean b/Beam/Daemon/Debug.lean index fcc0c342..7d3c7155 100644 --- a/Beam/Daemon/Debug.lean +++ b/Beam/Daemon/Debug.lean @@ -6,62 +6,13 @@ Author: Emilio J. Gallego Arias import Lean import Beam.Daemon.Paths -import Beam.Daemon.Protocol +import Beam.Daemon.Registry import Beam.System open Lean namespace Beam.Daemon -inductive RegistryRead where - | absent - | legacy - | unsupported (schemaVersion : Nat) - | malformed (detail : String) - | current (entry : RegistryEntry) - -def RegistryRead.entry? : RegistryRead → Option RegistryEntry - | .current entry => some entry - | .absent | .legacy | .unsupported _ | .malformed _ => none - -def RegistryRead.status : RegistryRead → String - | .absent => "absent" - | .legacy => "legacy" - | .unsupported _ => "unsupported" - | .malformed _ => "malformed" - | .current _ => "current" - -def RegistryRead.detail? : RegistryRead → Option String - | .legacy => some "legacy registry has no schemaVersion" - | .unsupported version => some s!"unsupported registry schemaVersion {version}" - | .malformed detail => some detail - | .absent | .current _ => none - -def readRegistry (root : System.FilePath) : IO RegistryRead := do - let path ← registryPath root - unless ← path.pathExists do - return .absent - try - let text ← IO.FS.readFile path - let json ← - match Json.parse text with - | .ok json => pure json - | .error err => return .malformed s!"invalid registry JSON: {err}" - match json.getObjVal? "schemaVersion" with - | .error _ => pure .legacy - | .ok schemaJson => - let schemaVersion ← - match fromJson? (α := Nat) schemaJson with - | .ok schemaVersion => pure schemaVersion - | .error err => return .malformed s!"invalid registry schemaVersion: {err}" - unless schemaVersion == registrySchemaVersion do - return .unsupported schemaVersion - match fromJson? json with - | .ok entry => pure <| .current entry - | .error err => pure <| .malformed s!"invalid registry schema: {err}" - catch err => - pure <| .malformed s!"could not read registry: {err}" - def daemonFailureIncidentEntries (root : System.FilePath) : IO (Array IO.FS.DirEntry) := do try let dir ← daemonFailureIncidentDir root @@ -227,7 +178,7 @@ def daemonDebugContextJson (root : System.FilePath) : IO Json := do | some detail => toJson detail | none => Json.null), ("registry", match registry with - | some entry => (toJson entry).setObjVal! "capability" (toJson "") + | some entry => entry.redactedJson | none => Json.null), ("registryPidStatus", match registryPidStatus with | some status => toJson status | none => Json.null), ("registryEndpoint", match registry.map registryEndpointSummary with | some endpoint => toJson endpoint | none => Json.null), diff --git a/Beam/Daemon/Protocol.lean b/Beam/Daemon/Protocol.lean index 044a403a..b60b2ae7 100644 --- a/Beam/Daemon/Protocol.lean +++ b/Beam/Daemon/Protocol.lean @@ -62,6 +62,9 @@ def RegistryEntry.identity (entry : RegistryEntry) : DaemonIdentity := { configHash := entry.configHash } +def RegistryEntry.redactedJson (entry : RegistryEntry) : Json := + (toJson entry).setObjVal! "capability" (toJson "") + structure DesiredConfig where root : System.FilePath leanCmd? : Option String := none diff --git a/Beam/Daemon/Registry.lean b/Beam/Daemon/Registry.lean new file mode 100644 index 00000000..544292a9 --- /dev/null +++ b/Beam/Daemon/Registry.lean @@ -0,0 +1,65 @@ +/- +Copyright (c) 2026 Lean FRO LLC. All rights reserved. +Released under Apache 2.0 license as described in the file LICENSE. +Author: Emilio J. Gallego Arias +-/ + +import Lean +import Beam.Daemon.Paths +import Beam.Daemon.Protocol + +open Lean + +namespace Beam.Daemon + +/-- Typed result of reading the versioned on-disk daemon registry. -/ +inductive RegistryRead where + | absent + | legacy + | unsupported (schemaVersion : Nat) + | malformed (detail : String) + | current (entry : RegistryEntry) + +def RegistryRead.entry? : RegistryRead → Option RegistryEntry + | .current entry => some entry + | .absent | .legacy | .unsupported _ | .malformed _ => none + +def RegistryRead.status : RegistryRead → String + | .absent => "absent" + | .legacy => "legacy" + | .unsupported _ => "unsupported" + | .malformed _ => "malformed" + | .current _ => "current" + +def RegistryRead.detail? : RegistryRead → Option String + | .legacy => some "legacy registry has no schemaVersion" + | .unsupported version => some s!"unsupported registry schemaVersion {version}" + | .malformed detail => some detail + | .absent | .current _ => none + +def readRegistry (root : System.FilePath) : IO RegistryRead := do + let path ← registryPath root + unless ← path.pathExists do + return .absent + try + let text ← IO.FS.readFile path + let json ← + match Json.parse text with + | .ok json => pure json + | .error err => return .malformed s!"invalid registry JSON: {err}" + match json.getObjVal? "schemaVersion" with + | .error _ => pure .legacy + | .ok schemaJson => + let schemaVersion ← + match fromJson? (α := Nat) schemaJson with + | .ok schemaVersion => pure schemaVersion + | .error err => return .malformed s!"invalid registry schemaVersion: {err}" + unless schemaVersion == registrySchemaVersion do + return .unsupported schemaVersion + match fromJson? json with + | .ok entry => pure <| .current entry + | .error err => pure <| .malformed s!"invalid registry schema: {err}" + catch err => + pure <| .malformed s!"could not read registry: {err}" + +end Beam.Daemon diff --git a/CHANGELOG.md b/CHANGELOG.md index 5f7fc81a..ad268338 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -36,7 +36,7 @@ This project keeps a lightweight, reverse-chronological changelog. Dates use `YY - Wrapper daemons now have explicit session ownership: only `lean-beam ensure --hold` starts a generation, ordinary wrapper commands attach to it, `--port` is accepted only by that owner-start command, and holder exit cancels admitted requests before closing the daemon through an inherited - pipe without heartbeat leases or retirement fences + pipe without heartbeat leases or time-based retirement ([#241](https://github.com/leanprover/lean-beam/pull/241), @ejgallego). - Long-running Lean operations now separate liveness status, request progress, and diagnostics. Sync and refresh use the discoverable `diagnostic_scope: "errors" | "all"` and diff --git a/docs/COMPATIBILITY.md b/docs/COMPATIBILITY.md index 68de4085..0e9c1995 100644 --- a/docs/COMPATIBILITY.md +++ b/docs/COMPATIBILITY.md @@ -22,8 +22,9 @@ A Lean release line is the canonical `major.minor` family recorded in but the installer does not reuse it. Remove the schema-2 decoder when 0.3 development opens. - Wrapper daemon registry schema 1. The registry is an internal beta coordination boundary: schema-less and unknown-version records are reported and preserved, but are not decoded, deleted, - or migrated automatically. Stop their matching owner with the runtime that created them before - starting a schema-1 owner. + or migrated automatically. A schema-less generation may not have a foreground wrapper owner; use + the runtime that wrote its record to stop the corresponding daemon, confirm it is gone, and only + then start a schema-1 owner. - MCP `2026-07-28` is the preferred stdio protocol revision. MCP `2025-11-25` remains a named transition target for initialization-based clients. Reconsider the legacy path before the 0.3 release once the clients named by the setup guide can all use per-request metadata. diff --git a/docs/TESTING.md b/docs/TESTING.md index 780487e9..369dbc76 100644 --- a/docs/TESTING.md +++ b/docs/TESTING.md @@ -110,12 +110,14 @@ Current Beam coverage includes: response delivery, shutdown after the requesting TCP client resets its connection, protocol tests, and validated-toolchain/release-line CI policy consistency through [tests/test-beam-fast.sh](../tests/test-beam-fast.sh) -- wrapper coverage through [tests/test-beam-wrapper.sh](../tests/test-beam-wrapper.sh), which aggregates focused probe, runtime, sync/save, handle, and diagnostic slices +- wrapper coverage through [tests/test-beam-wrapper.sh](../tests/test-beam-wrapper.sh), which reports + focused probe, runtime, sync/save, handle, and diagnostic slices independently - focused daemon lifecycle coverage in [tests/test-beam-wrapper-daemon.sh](../tests/test-beam-wrapper-daemon.sh), including the no-implicit-start contract, duplicate-owner rejection, Beam and non-Beam endpoint collision safety without cross-project disclosure, authenticated generation probes, mode-`0600` - registry publication, oversized-frame and first-message limits, a bounded identity probe against - a silent non-Beam listener, cross-root unsafe-registry preservation that does not affect the daemon + registry publication, unauthorized-shutdown rejection without listener teardown, oversized-frame + and first-message limits, a bounded identity probe against a silent non-Beam listener, cross-root + unsafe-registry preservation that does not affect the daemon serving the other root, configuration-drift preservation of the owner and active request, explicit shutdown, cancellation of requests active during shutdown or owner loss, exact-generation cleanup that preserves a replacement registry, a published draining fence while diff --git a/tests/test-beam-wrapper-daemon.sh b/tests/test-beam-wrapper-daemon.sh index 99865411..2cd28823 100644 --- a/tests/test-beam-wrapper-daemon.sh +++ b/tests/test-beam-wrapper-daemon.sh @@ -281,6 +281,20 @@ def receive_frame(sock): payload.extend(chunk) return json.loads(payload) +with socket.create_connection(("127.0.0.1", port), timeout=3) as sock: + request = json.dumps({ + "op": "shutdown", + "daemonCapability": "not-the-owner-capability", + }).encode() + sock.sendall(str(len(request)).encode() + b"\n" + request) + response = receive_frame(sock) + payload = response.get("payload", {}) + if payload.get("ok") is not False: + raise RuntimeError(f"unauthorized shutdown unexpectedly succeeded: {response}") + message = payload.get("error", {}).get("message", "") + if "invalid Beam daemon capability" not in message: + raise RuntimeError(f"unexpected unauthorized-shutdown response: {response}") + with socket.create_connection(("127.0.0.1", port), timeout=3) as sock: sock.sendall(b"16777217\n") response = receive_frame(sock) @@ -294,8 +308,14 @@ with socket.create_connection(("127.0.0.1", port), timeout=3) as sock: raise RuntimeError(f"unexpected first-message-timeout response: {response}") PY -stats_after_limits_json="$("$beam_script" --root "$tmp1" stats)" -assert_json_field_equals "stats after transport limit probes" "$stats_after_limits_json" ok true +stats_after_security_probes_json="$("$beam_script" --root "$tmp1" stats)" +assert_json_field_equals \ + "stats after unauthorized shutdown and transport limit probes" \ + "$stats_after_security_probes_json" ok true +if ! kill -0 "$owner1_pid" 2>/dev/null || ! kill -0 "$daemon1_pid" 2>/dev/null; then + echo "expected unauthorized shutdown to preserve both owner and daemon" >&2 + exit 1 +fi second_owner_out="$tmp1/second-owner.out" second_owner_err="$tmp1/second-owner.err" diff --git a/tests/test-beam-wrapper-diagnostics.sh b/tests/test-beam-wrapper-diagnostics.sh index 6830f96e..1903c241 100755 --- a/tests/test-beam-wrapper-diagnostics.sh +++ b/tests/test-beam-wrapper-diagnostics.sh @@ -105,6 +105,7 @@ expect_sync_result_shape() { expect_json_field_present "$json_payload" "$prefix.readiness" "$label readiness" "$err_file" } +echo "[beam-wrapper:diagnostics] starting blocking-error diagnostics" ( cd "$broken_root" @@ -189,7 +190,9 @@ expect_sync_result_shape() { exit 1 fi ) +echo "[beam-wrapper:diagnostics] passed: blocking-error diagnostics" +echo "[beam-wrapper:diagnostics] starting guard_msgs stderr handling" ( cd "$guard_msgs_io_stderr_root" @@ -232,7 +235,9 @@ EOF exit 1 fi ) +echo "[beam-wrapper:diagnostics] passed: guard_msgs stderr handling" +echo "[beam-wrapper:diagnostics] starting default warning filtering" ( cd "$warn_root" "$beam_script" ensure lean > /dev/null @@ -293,7 +298,9 @@ EOF exit 1 fi ) +echo "[beam-wrapper:diagnostics] passed: default warning filtering" +echo "[beam-wrapper:diagnostics] starting full warning streaming" ( cd "$warn_full_root" "$beam_script" ensure lean > /dev/null @@ -344,12 +351,13 @@ EOF warn_full_registry="$(beam_wrapper_registry_path "$warn_full_root")" beam_wrapper_expect_file "$warn_full_registry" port9="$(read_json_field "$warn_full_registry" port)" + capability9="$(read_json_field "$warn_full_registry" capability)" client9="$(read_json_field "$warn_full_registry" clientBin 2>/dev/null || true)" if [ -z "$client9" ]; then client9="$client" fi - stream_req="$(printf '{"op":"sync_file","workspaceId":"beam-cli-project","root":"%s","path":"SaveSmoke/B.lean","diagnosticScope":"all"}' "$warn_full_root")" + stream_req="$(printf '{"op":"sync_file","workspaceId":"beam-cli-project","root":"%s","path":"SaveSmoke/B.lean","diagnosticScope":"all","daemonCapability":"%s"}' "$warn_full_root" "$capability9")" stream_out="$(beam_wrapper_mktemp_file stream-out)" stream_err="$(beam_wrapper_mktemp_file stream-err)" "$client9" --port "$port9" request-stream "$stream_req" >"$stream_out" 2>"$stream_err" @@ -438,8 +446,10 @@ EOF exit 1 fi ) +echo "[beam-wrapper:diagnostics] passed: full warning streaming" +echo "[beam-wrapper:diagnostics] starting renamed-dependency stale recovery" ( cd "$renamed_stale_root" @@ -637,7 +647,9 @@ EOF exit 1 fi ) +echo "[beam-wrapper:diagnostics] passed: renamed-dependency stale recovery" +echo "[beam-wrapper:diagnostics] starting stale-import recovery" ( cd "$stale_root" lake build SaveSmoke/A.lean > /dev/null @@ -765,3 +777,4 @@ EOF fi assert_json_completed_file_progress "recovered lean-refresh" "$refreshed_a" fileProgress ) +echo "[beam-wrapper:diagnostics] passed: stale-import recovery" diff --git a/tests/test-beam-wrapper.sh b/tests/test-beam-wrapper.sh index d20df962..c0520c7c 100755 --- a/tests/test-beam-wrapper.sh +++ b/tests/test-beam-wrapper.sh @@ -8,8 +8,19 @@ set -euo pipefail cd "$(dirname "$0")/.." -bash tests/test-beam-wrapper-probe.sh -bash tests/test-beam-wrapper-runtime.sh -bash tests/test-beam-wrapper-sync-save.sh -bash tests/test-beam-wrapper-handle.sh -bash tests/test-beam-wrapper-diagnostics.sh +run_wrapper_slice() { + local label="$1" + local script="$2" + echo "[beam-wrapper] starting $label" + if ! bash "$script"; then + echo "[beam-wrapper] failed: $label ($script)" >&2 + return 1 + fi + echo "[beam-wrapper] passed: $label" +} + +run_wrapper_slice "probe" tests/test-beam-wrapper-probe.sh +run_wrapper_slice "runtime" tests/test-beam-wrapper-runtime.sh +run_wrapper_slice "sync/save" tests/test-beam-wrapper-sync-save.sh +run_wrapper_slice "handles" tests/test-beam-wrapper-handle.sh +run_wrapper_slice "diagnostics" tests/test-beam-wrapper-diagnostics.sh From 6bbf86b58ebd04947b139926addfb0b962bf905e Mon Sep 17 00:00:00 2001 From: Emilio Jesus Gallego Arias Date: Fri, 28 Aug 2026 17:32:42 +0200 Subject: [PATCH 23/28] refactor: freeze wrapper session contract --- Beam/Broker/Pending.lean | 36 +- Beam/Broker/Protocol.lean | 68 ++- Beam/Broker/Server.lean | 17 +- Beam/BrokerClient.lean | 10 +- Beam/Cli/Args.lean | 13 + Beam/Cli/Broker.lean | 30 +- Beam/Cli/Commands.lean | 89 ++-- Beam/Cli/DaemonManager.lean | 401 ++++++++++++------ Beam/Cli/Feedback.lean | 29 +- Beam/Cli/Info.lean | 19 +- Beam/Cli/Usage.lean | 7 +- Beam/Daemon/Debug.lean | 75 ++-- Beam/Daemon/Paths.lean | 52 ++- Beam/Daemon/Protocol.lean | 47 +- Beam/Daemon/Registry.lean | 47 +- docs/COMPATIBILITY.md | 11 +- docs/DEVELOPMENT.md | 101 ++--- docs/SETUP.md | 46 +- docs/STATUS.md | 46 +- docs/SYNC_AND_DIAGNOSTICS.md | 31 +- docs/TESTING.md | 7 +- scripts/lean-beam | 6 +- skills/lean-beam/SKILL.md | 20 +- skills/lean-beam/references/anti-patterns.md | 2 +- .../lean-beam/references/workflow-details.md | 2 +- skills/rocq-beam/SKILL.md | 6 +- tests/lean/BeamTest/Broker/CliDaemonTest.lean | 72 +++- tests/lean/BeamTest/Broker/PendingTest.lean | 42 +- tests/lean/BeamTest/Broker/ProtocolTest.lean | 55 ++- tests/test-beam-fast.sh | 16 +- tests/test-beam-toolchain-compat.sh | 2 +- tests/test-beam-wrapper-daemon.sh | 193 ++++++++- tests/test-beam-wrapper-probe.sh | 2 +- tests/test-beam-wrapper-sandbox.sh | 27 +- 34 files changed, 1203 insertions(+), 424 deletions(-) diff --git a/Beam/Broker/Pending.lean b/Beam/Broker/Pending.lean index 5f6826c0..4051a5b7 100644 --- a/Beam/Broker/Pending.lean +++ b/Beam/Broker/Pending.lean @@ -341,7 +341,13 @@ def propagateCancellation end PendingRequestStore +private structure ActiveRequestKey where + workspaceId? : Option WorkspaceId + clientRequestId : String +deriving BEq, Ord + structure ActiveRequest where + workspaceId? : Option WorkspaceId clientRequestId? : Option String token : Nat cancelRef : IO.Ref Bool @@ -349,7 +355,7 @@ structure ActiveRequest where private structure ActiveRequestRegistryState where nextToken : Nat := 1 accepting : Bool := true - requests : Std.TreeMap String ActiveRequest := {} + requests : Std.TreeMap ActiveRequestKey ActiveRequest := {} anonymousRequests : Std.TreeMap Nat ActiveRequest := {} drainedSignaled : Bool := false @@ -384,6 +390,7 @@ private def resolveDrainedIfNeeded def register (registry : ActiveRequestRegistry) + (workspaceId? : Option WorkspaceId) (clientRequestId? : Option String) : IO (Except BrokerFailure ActiveRequest) := do let cancelRef ← IO.mkRef false registry.mutex.atomically do @@ -395,23 +402,28 @@ def register } match clientRequestId? with | none => - let active : ActiveRequest := { clientRequestId?, token := state.nextToken, cancelRef } + let active : ActiveRequest := { + workspaceId?, clientRequestId?, token := state.nextToken, cancelRef + } set { state with nextToken := state.nextToken + 1 anonymousRequests := state.anonymousRequests.insert active.token active } pure <| .ok active | some clientRequestId => - if state.requests.contains clientRequestId then + let key : ActiveRequestKey := { workspaceId?, clientRequestId } + if state.requests.contains key then pure <| .error { code := .invalidParams - message := s!"clientRequestId '{clientRequestId}' is already active" + message := s!"clientRequestId '{clientRequestId}' is already active in this workspace" } else - let active : ActiveRequest := { clientRequestId?, token := state.nextToken, cancelRef } + let active : ActiveRequest := { + workspaceId?, clientRequestId?, token := state.nextToken, cancelRef + } set { state with nextToken := state.nextToken + 1 - requests := state.requests.insert clientRequestId active + requests := state.requests.insert key active } pure <| .ok active @@ -426,10 +438,11 @@ def unregister let state := match active.clientRequestId? with | some clientRequestId => - match state.requests.get? clientRequestId with + let key : ActiveRequestKey := { workspaceId? := active.workspaceId?, clientRequestId } + match state.requests.get? key with | some current => if current.token == active.token then - { state with requests := state.requests.erase clientRequestId } + { state with requests := state.requests.erase key } else state | none => state @@ -474,9 +487,11 @@ def awaitDrained (registry : ActiveRequestRegistry) : IO Unit := do def markCancelled (registry : ActiveRequestRegistry) + (workspaceId? : Option WorkspaceId) (clientRequestId : String) : IO (Option ActiveRequest) := do registry.mutex.atomically do - let active? := (← get).requests.get? clientRequestId + let key : ActiveRequestKey := { workspaceId?, clientRequestId } + let active? := (← get).requests.get? key match active? with | none => pure none @@ -491,7 +506,8 @@ def markCancelledActive let state ← get let current? := match active.clientRequestId? with - | some clientRequestId => state.requests.get? clientRequestId + | some clientRequestId => + state.requests.get? { workspaceId? := active.workspaceId?, clientRequestId } | none => state.anonymousRequests.get? active.token match current? with | none => pure none diff --git a/Beam/Broker/Protocol.lean b/Beam/Broker/Protocol.lean index ef35a74c..d8be71e8 100644 --- a/Beam/Broker/Protocol.lean +++ b/Beam/Broker/Protocol.lean @@ -250,8 +250,9 @@ inductive WorkspaceScope where /-- Describe whether a broker operation is process-wide or resolves one workspace. -/ def Op.workspaceScope : Op → WorkspaceScope - | .cancel | .listWorkspaces | .resetStats | .shutdown => .none + | .listWorkspaces | .resetStats | .shutdown => .none | .openDocs | .stats => .optional + | .cancel | .ensure | .updateFile | .syncFile | .refreshFile | .close | .runAt | .hover | .signatureHelp | .definition | .references | .documentSymbols | .workspaceSymbols | .codeActionResolve | .saveOlean | .goals | .todo | .runWith | .release @@ -437,6 +438,71 @@ instance : FromJson Request where request.validateFields pure request +/-- +A semantic request accepted by the supported project-session client. + +Session routing and authority are supplied by the selected session descriptor, not by caller JSON. +The generic `Request` remains the internal broker protocol used by maintenance tooling. +-/ +structure ProjectRequest where + private request : Request + private requestId : String + +private def projectRequestForbiddenFields : Array String := + #["workspaceId", "workspaceMode", "daemonCapability", "root", "leanCmd", "leanPlugin", "rocqCmd"] + +private def ProjectRequest.supportedOp : Op → Bool + | .initWorkspace | .listWorkspaces | .dropWorkspace => false + | _ => true + +def ProjectRequest.ofRequest (request : Request) : Except String ProjectRequest := do + unless ProjectRequest.supportedOp request.op do + throw s!"broker op '{request.op.key}' is not available through a project session" + if request.workspaceId?.isSome || request.workspaceMode?.isSome || + request.daemonCapability?.isSome || request.root?.isSome || request.leanCmd?.isSome || + request.leanPlugin?.isSome || request.rocqCmd?.isSome then + throw "project requests cannot select session routing, authority, or executable configuration" + let some clientRequestId := request.clientRequestId? + | throw "project requests require a non-empty clientRequestId" + if clientRequestId.isEmpty then + throw "project requests require a non-empty clientRequestId" + request.validateFields + pure { request, requestId := clientRequestId } + +instance : FromJson ProjectRequest where + fromJson? json := do + match json with + | .obj fields => + let forbidden := projectRequestForbiddenFields.filter fields.contains + unless forbidden.isEmpty do + throw s!"project request contains session-owned fields: {String.intercalate ", " forbidden.toList}" + | _ => pure () + ProjectRequest.ofRequest (← fromJson? json) + +def ProjectRequest.op (request : ProjectRequest) : Op := + request.request.op + +def ProjectRequest.clientRequestId (request : ProjectRequest) : String := + request.requestId + +/-- Attach one semantic request to a selected, authenticated workspace session. -/ +def ProjectRequest.attach + (request : ProjectRequest) + (workspaceId : WorkspaceId) + (root capability : String) : Request := + let request := request.request + let request := { request with daemonCapability? := some capability } + match request.op.workspaceScope with + | .none => request + | .optional | .required => + let request := { request with workspaceId? := some workspaceId } + if request.op == .cancel then + request + else { + request with + root? := some root + } + structure Error where code : String message : String := "" diff --git a/Beam/Broker/Server.lean b/Beam/Broker/Server.lean index 26c1f34a..a1115ebc 100644 --- a/Beam/Broker/Server.lean +++ b/Beam/Broker/Server.lean @@ -1261,8 +1261,11 @@ private def cancelRegisteredRequest | some active => let sessions ← server.withState do let state ← get - pure <| state.workspaces.toList.flatMap fun (_, workspace) => - [workspace.lean.session?, workspace.rocq.session?] + pure <| state.workspaces.toList.flatMap fun (workspaceId, workspace) => + if active.workspaceId?.all (fun selected => selected == workspaceId) then + [workspace.lean.session?, workspace.rocq.session?] + else + [] for session? in sessions do if let some session := session? then discard <| PendingRequestStore.cancelMatching session.pending session.stdin active.cancelRef @@ -1271,9 +1274,10 @@ private def cancelRegisteredRequest private def cancelActiveRequest (server : ServerRuntime) + (workspaceId? : Option WorkspaceId) (clientRequestId : String) : IO Bool := cancelRegisteredRequest server <| - ActiveRequestRegistry.markCancelled server.activeRequests clientRequestId + ActiveRequestRegistry.markCancelled server.activeRequests workspaceId? clientRequestId /-- Cancel the exact active admission represented by `handle`. @@ -2414,7 +2418,7 @@ private def handleRequestIO match req.cancelRequestIdArg with | .ok targetClientRequestId => pure targetClientRequestId | .error failure => return failure.toResponse - let cancelled ← cancelActiveRequest server targetClientRequestId + let cancelled ← cancelActiveRequest server req.resolvedWorkspaceId? targetClientRequestId pure <| Response.success (Json.mkObj [("cancelled", toJson cancelled)]) | op => match ← validateRequestWorkspace server req with @@ -2480,7 +2484,7 @@ private def ServerRuntime.withRequestAdmission let resp := errorResponseFor .invalidParams "invalid Beam daemon capability" recordDispatchMetrics server req resp startedAt return resp - if req.op == .initWorkspace || req.op == .dropWorkspace then + if req.op == .initWorkspace || req.op == .listWorkspaces || req.op == .dropWorkspace then let resp := errorResponseFor .invalidParams s!"broker op '{req.op.key}' is unavailable in wrapper-owned daemon mode" recordDispatchMetrics server req resp startedAt @@ -2496,7 +2500,8 @@ private def ServerRuntime.withRequestAdmission try let active? ← if req.op.tracksActiveRequest then - match ← ActiveRequestRegistry.register server.activeRequests req.clientRequestId? with + match ← ActiveRequestRegistry.register + server.activeRequests req.resolvedWorkspaceId? req.clientRequestId? with | .ok active => pure (some active) | .error failure => let resp := BrokerFailure.toResponse failure diff --git a/Beam/BrokerClient.lean b/Beam/BrokerClient.lean index 553db0ef..c8a74911 100644 --- a/Beam/BrokerClient.lean +++ b/Beam/BrokerClient.lean @@ -22,8 +22,12 @@ private def usage : String := String.intercalate "\n" [ "usage: beam-client [--port N] request | request-stream ", "", + "beam-client is raw port-oriented maintainer/debug tooling.", + "For wrapper sessions, use: lean-beam --root PATH [--control-dir DIR] request-stream ", + "That supported machine interface selects the session descriptor and injects routing/authentication.", + "", "request prints the final response on stdout and formats streamed diagnostics for humans on stderr.", - "request-stream is the preferred machine interface: it prints one compact StreamMessage JSON line", + "raw request-stream prints one compact StreamMessage JSON line", "per event on stdout using kind + payload + optional clientRequestId; kinds are", "diagnostic | fileProgress | response, and the final response is last." ] @@ -41,9 +45,9 @@ private def parseRequestArg (json : String) : IO Request := do private def parseRequest (args : List String) : IO (ClientMode × Request) := do match args with - | "request" :: json :: _ => + | ["request", json] => pure (.request, ← parseRequestArg json) - | "request-stream" :: json :: _ => + | ["request-stream", json] => pure (.requestStream, ← parseRequestArg json) | _ => throw <| IO.userError usage diff --git a/Beam/Cli/Args.lean b/Beam/Cli/Args.lean index 1c17b781..47a30cd1 100644 --- a/Beam/Cli/Args.lean +++ b/Beam/Cli/Args.lean @@ -17,6 +17,7 @@ open Beam.Broker structure CliOptions where explicitRoot? : Option System.FilePath := none + explicitControlDir? : Option System.FilePath := none requestedPort? : Option UInt16 := none args : List String := [] @@ -208,11 +209,23 @@ private def resolveExplicitRootArg (root : String) : IO System.FilePath := do catch err => throw <| IO.userError s!"workspace root does not resolve: {err.toString}" +private def resolveControlDirArg (dir : String) : IO System.FilePath := do + let path := System.FilePath.mk dir + if ← path.pathExists then + Beam.resolveExistingPath path + else + let cwd ← IO.currentDir + let absolute := if path.isAbsolute then path else cwd / path + pure absolute.normalize + partial def parseCliOptions (opts : CliOptions) : List String → IO CliOptions | [] => pure opts | "--root" :: root :: rest => do let root ← resolveExplicitRootArg root parseCliOptions { opts with explicitRoot? := some root } rest + | "--control-dir" :: dir :: rest => do + let dir ← resolveControlDirArg dir + parseCliOptions { opts with explicitControlDir? := some dir } rest | "--port" :: port :: rest => do let port ← IO.ofExcept <| parsePortText "port" port parseCliOptions { opts with requestedPort? := some port } rest diff --git a/Beam/Cli/Broker.lean b/Beam/Cli/Broker.lean index 2e20af90..8f67794d 100644 --- a/Beam/Cli/Broker.lean +++ b/Beam/Cli/Broker.lean @@ -14,27 +14,36 @@ namespace Beam.Cli open Beam.Broker +private def inWorkspace (workspaceId : WorkspaceId) (req : Request) : Request := + match req.op.workspaceScope with + | .none => req + | .optional | .required => + if req.workspaceId?.isSome then req + else { req with workspaceId? := some workspaceId } + /-- Address a request to the private workspace of the CLI's one-project daemon. -Process-wide control operations deliberately remain unscoped. An explicitly supplied workspace is +Process-wide control operations deliberately remain unscoped. Optional operations such as +cancellation are scoped to the wrapper workspace by default. An explicitly supplied workspace is preserved so this adapter does not rewrite lower-level test or maintenance requests. -/ def inProjectDaemonWorkspace (req : Request) : Request := - match req.op.workspaceScope with - | .none => req - | .optional | .required => - if req.workspaceId?.isSome then req - else { req with workspaceId? := some projectDaemonWorkspaceId } + inWorkspace projectDaemonWorkspaceId req + +/-- Address a wrapper request to the workspace selected from its session descriptor. -/ +def inSelectedDaemonWorkspace (client : ProjectDaemonClient) (req : Request) : Request := + inWorkspace client.workspaceId req private def withBrokerErrorContext {α} (root : System.FilePath) + (client : ProjectDaemonClient) (action : IO (Except BrokerClientFailure α)) : IO α := do match ← action with | .ok value => pure value | .error failure => - throw <| IO.userError (← daemonFailureMessage root failure) + throw <| IO.userError (← daemonFailureMessage root failure client.controlDir?) structure BrokerWaitSpec where action : String @@ -119,7 +128,7 @@ private def withWrapperClientRequestId (req : Request) : IO WrapperBrokerRequest private def prepareWrapperBrokerRequest (client : ProjectDaemonClient) (req : Request) : IO WrapperBrokerRequest := do - let wrapper ← withWrapperClientRequestId <| inProjectDaemonWorkspace req + let wrapper ← withWrapperClientRequestId <| inSelectedDaemonWorkspace client req pure { wrapper with request := client.authorize wrapper.request } def decodeCancelAcknowledged? (resp : Response) : Option Bool := do @@ -131,6 +140,7 @@ private def sendBrokerCancellation (clientRequestId : String) : IO (Option Bool) := do let cancelReq : Request := { op := .cancel + workspaceId? := some client.workspaceId cancelRequestId? := some clientRequestId } try @@ -208,7 +218,7 @@ private def requestBrokerResponse (req : Request) : IO WrapperBrokerResponse := do let wrapperReq ← prepareWrapperBrokerRequest client req let req := wrapperReq.request - let response ← withBrokerErrorContext root do + let response ← withBrokerErrorContext root client do awaitBrokerResponseWithInterrupts client wrapperReq.clientRequestId wrapperReq.visibleClientRequestId? none <| sendRequestWithCallbacksResult client.endpoint req @@ -448,7 +458,7 @@ def callBrokerWithProgress IO.eprintln <| annotateRunatMessage visibleClientRequestId? (formatStreamDiagnostic diagnostic) } let progressSpec? := if showProgress then some spec else none - let resp ← withBrokerErrorContext root do + let resp ← withBrokerErrorContext root client do awaitBrokerResponseWithInterrupts client wrapperReq.clientRequestId visibleClientRequestId? progressSpec? <| sendRequestWithCallbacksResult client.endpoint req callbacks diff --git a/Beam/Cli/Commands.lean b/Beam/Cli/Commands.lean index 5fa36e8d..d9360bdc 100644 --- a/Beam/Cli/Commands.lean +++ b/Beam/Cli/Commands.lean @@ -34,7 +34,7 @@ private def updateVersionForRocqGoals let resp ← requestBroker root client { op := .updateFile backend := .rocq - workspaceId? := some projectDaemonWorkspaceId + workspaceId? := some client.workspaceId root? := some root.toString path? := some path } @@ -54,7 +54,7 @@ private def runLeanRunAt let line ← parseNatArg "line" lineText let character ← parseNatArg "character" characterText let parsedText ← parseTextArg s!"{action} " textArgs - withProjectDaemon home root .lean fun client => do + withProjectDaemon home root .lean (explicitControlDir? := opts.explicitControlDir?) fun client => do let req ← withEnvClientRequestId <| leanRunAtRequest root path version line character parsedText.text? (storeHandle := storeHandle) maybeEmitTextDebug req.clientRequestId? action parsedText.source parsedText.text? @@ -82,7 +82,7 @@ private def runLeanRunWith let req ← withEnvClientRequestId <| leanRunWithRequest root path handle parsedText.text? (linear := linear) maybeEmitTextDebug req.clientRequestId? action parsedText.source parsedText.text? - withProjectDaemon home root .lean fun client => + withProjectDaemon home root .lean (explicitControlDir? := opts.explicitControlDir?) fun client => callBrokerWithProgress root client req (leanRunWithWaitSpec path (linear := linear)) private def runLeanRelease @@ -95,19 +95,50 @@ private def runLeanRelease let (handle, extra) ← parseHandleInput s!"{action} " args unless extra.isEmpty do throw <| IO.userError (handleArgUsage s!"{action} ") - withProjectDaemon home root .lean fun client => + withProjectDaemon home root .lean (explicitControlDir? := opts.explicitControlDir?) fun client => callBroker root client <| leanReleaseRequest root path handle private def shutdownProjectDaemon (opts : CliOptions) : IO Unit := do let root ← projectRootAny opts - match ← shutdownRegisteredProjectDaemon root with + match ← shutdownRegisteredProjectDaemon root opts.explicitControlDir? with | .ok (some resp) => printResponse resp | .ok none => printJsonLine <| Json.mkObj [ ("result", Json.mkObj [("shutdown", toJson false), ("reason", toJson ("notFound" : String))]) ] | .error failure => - throw <| IO.userError (← daemonFailureMessage root failure) + throw <| IO.userError (← daemonFailureMessage root failure opts.explicitControlDir?) + +private def recoverProjectSession (opts : CliOptions) (args : List String) : IO Unit := do + let root ← projectRootAny opts + let (generation?, forceOpaque) ← + match args with + | ["--generation", generation] => pure (some generation, false) + | ["--force"] => pure (none, true) + | _ => + throw <| IO.userError + "usage: beam [--root PATH] [--control-dir DIR] recover --generation ID | --force" + let result ← recoverProjectDaemon root generation? forceOpaque opts.explicitControlDir? + printJsonLine (toJson result) + +private def parseProjectRequestArg (raw : String) : IO ProjectRequest := do + let text ← if raw == "-" then (← IO.getStdin).readToEnd else pure raw + let json ← parseJsonText "project request json" text + match fromJson? json with + | .ok request => pure request + | .error err => throw <| IO.userError s!"invalid project request payload: {err}" + +private def runProjectRequestStream (opts : CliOptions) (raw : String) : IO Unit := do + let some root := opts.explicitRoot? + | throw <| IO.userError + "request-stream is a machine interface and requires an explicit --root PATH" + let projectRequest ← parseProjectRequestArg raw + withSelectedProjectDaemon root (explicitControlDir? := opts.explicitControlDir?) fun selected => do + let req := projectRequest.attach selected.workspace.workspaceId selected.workspace.root + selected.client.capability + let resp ← sendRequestWithStream selected.client.endpoint req fun stream => + IO.println (toJson stream).compress + failOnError resp private def parseBackendName (name : String) : IO Backend := do match fromJson? (Json.str name) with @@ -152,7 +183,7 @@ private def ensureBackend (← IO.getStdout).flush IO.eprintln "beam: owning Beam session; interrupt this wrapper process when finished" else - withProjectDaemon home root backend fun client => + withProjectDaemon home root backend (explicitControlDir? := opts.explicitControlDir?) fun client => callBroker root client { op := .ensure, backend := backend, root? := some root.toString } def runCommand (home : System.FilePath) (opts : CliOptions) : IO Unit := do @@ -206,7 +237,7 @@ def runCommand (home : System.FilePath) (opts : CliOptions) : IO Unit := do let line ← parseNatArg "line" line let character ← parseNatArg "character" character let action ← wrapperDisplayAction "lean-hover" - withProjectDaemon home root .lean fun client => + withProjectDaemon home root .lean (explicitControlDir? := opts.explicitControlDir?) fun client => callBrokerWithProgress root client (leanHoverRequest root path version line character) (leanHoverWaitSpec path line character action) @@ -216,7 +247,7 @@ def runCommand (home : System.FilePath) (opts : CliOptions) : IO Unit := do let line ← parseNatArg "line" line let character ← parseNatArg "character" character let action ← wrapperDisplayAction "lean-signature-help" - withProjectDaemon home root .lean fun client => + withProjectDaemon home root .lean (explicitControlDir? := opts.explicitControlDir?) fun client => callBrokerWithProgress root client (leanSignatureHelpRequest root path version line character) (leanSignatureHelpWaitSpec path line character action) @@ -226,7 +257,7 @@ def runCommand (home : System.FilePath) (opts : CliOptions) : IO Unit := do let line ← parseNatArg "line" line let character ← parseNatArg "character" character let action ← wrapperDisplayAction "lean-definition" - withProjectDaemon home root .lean fun client => + withProjectDaemon home root .lean (explicitControlDir? := opts.explicitControlDir?) fun client => callBrokerWithProgress root client (leanDefinitionRequest root path version line character) (leanDefinitionWaitSpec path line character action) @@ -237,7 +268,7 @@ def runCommand (home : System.FilePath) (opts : CliOptions) : IO Unit := do let character ← parseNatArg "character" character let includeDeclaration ← parseLeanReferencesArgs extra let action ← wrapperDisplayAction "lean-references" - withProjectDaemon home root .lean fun client => + withProjectDaemon home root .lean (explicitControlDir? := opts.explicitControlDir?) fun client => callBrokerWithProgress root client (leanReferencesRequest root path version line character includeDeclaration) (leanReferencesWaitSpec path line character action) @@ -245,7 +276,7 @@ def runCommand (home : System.FilePath) (opts : CliOptions) : IO Unit := do let root ← projectRoot opts .lean let version ← parseNatArg "version" versionText let action ← wrapperDisplayAction "lean-document-symbols" - withProjectDaemon home root .lean fun client => + withProjectDaemon home root .lean (explicitControlDir? := opts.explicitControlDir?) fun client => callBrokerWithProgress root client (leanDocumentSymbolsRequest root path version) (leanDocumentSymbolsWaitSpec path action) @@ -256,7 +287,7 @@ def runCommand (home : System.FilePath) (opts : CliOptions) : IO Unit := do | some query => pure query | none => throw <| IO.userError "usage: beam [--root PATH] lean-workspace-symbols " let action ← wrapperDisplayAction "lean-workspace-symbols" - withProjectDaemon home root .lean fun client => + withProjectDaemon home root .lean (explicitControlDir? := opts.explicitControlDir?) fun client => callBrokerWithProgress root client (leanWorkspaceSymbolsRequest root query) (leanWorkspaceSymbolsWaitSpec query action) @@ -267,7 +298,7 @@ def runCommand (home : System.FilePath) (opts : CliOptions) : IO Unit := do let line ← parseNatArg "line" line let character ← parseNatArg "character" character let action ← wrapperDisplayAction "lean-goals" - withProjectDaemon home root .lean fun client => + withProjectDaemon home root .lean (explicitControlDir? := opts.explicitControlDir?) fun client => callBrokerWithProgress root client (leanGoalsRequest root path version line character mode) (leanGoalsWaitSpec path line character mode (some action)) @@ -280,7 +311,7 @@ def runCommand (home : System.FilePath) (opts : CliOptions) : IO Unit := do let endCharacter ← parseNatArg "endCharacter" endCharacter let (kinds?, suggest?) ← parseLeanTodoArgs extra let action ← wrapperDisplayAction "lean-todo" - withProjectDaemon home root .lean fun client => + withProjectDaemon home root .lean (explicitControlDir? := opts.explicitControlDir?) fun client => callBrokerWithProgress root client (leanTodoRequest root path version startLine startCharacter endLine endCharacter kinds? suggest?) (leanTodoWaitSpec path startLine startCharacter endLine endCharacter action) @@ -295,19 +326,19 @@ def runCommand (home : System.FilePath) (opts : CliOptions) : IO Unit := do let root ← projectRoot opts .lean let diagnosticScope ← parseLeanSaveArgs extra let action ← wrapperDisplayAction "lean-save" - withProjectDaemon home root .lean fun client => + withProjectDaemon home root .lean (explicitControlDir? := opts.explicitControlDir?) fun client => callBrokerWithProgress root client (leanSaveRequest root path diagnosticScope) (leanSaveWaitSpec path (action? := some action)) | "lean-update" :: path :: [] => let root ← projectRoot opts .lean - withProjectDaemon home root .lean fun client => + withProjectDaemon home root .lean (explicitControlDir? := opts.explicitControlDir?) fun client => callBroker root client <| leanUpdateRequest root path | "lean-sync" :: path :: extra => do let root ← projectRoot opts .lean let diagnosticScope ← parseLeanSyncArgs extra let action ← wrapperDisplayAction "lean-sync" - withProjectDaemon home root .lean fun client => + withProjectDaemon home root .lean (explicitControlDir? := opts.explicitControlDir?) fun client => callBrokerWithProgress root client (leanSyncRequest root path diagnosticScope) (syncWaitSpec path action) @@ -315,25 +346,25 @@ def runCommand (home : System.FilePath) (opts : CliOptions) : IO Unit := do let root ← projectRoot opts .lean let diagnosticScope ← parseLeanRefreshArgs extra let action ← wrapperDisplayAction "lean-refresh" - withProjectDaemon home root .lean fun client => + withProjectDaemon home root .lean (explicitControlDir? := opts.explicitControlDir?) fun client => callBrokerWithProgress root client (leanRefreshRequest root path diagnosticScope) (refreshWaitSpec path action) | "lean-close" :: path :: [] => let root ← projectRoot opts .lean - withProjectDaemon home root .lean fun client => + withProjectDaemon home root .lean (explicitControlDir? := opts.explicitControlDir?) fun client => callBroker root client <| leanCloseRequest root path | "lean-close-save" :: path :: extra => let root ← projectRoot opts .lean let diagnosticScope ← parseLeanCloseSaveArgs extra let action ← wrapperDisplayAction "lean-close-save" - withProjectDaemon home root .lean fun client => + withProjectDaemon home root .lean (explicitControlDir? := opts.explicitControlDir?) fun client => callBrokerWithProgress root client (leanCloseSaveRequest root path diagnosticScope) (leanSaveWaitSpec path (closeAfter := true) (action? := some action)) | "rocq-goals-after" :: path :: line :: character :: text => let root ← projectRoot opts .rocq - withProjectDaemon home root .rocq fun client => do + withProjectDaemon home root .rocq (explicitControlDir? := opts.explicitControlDir?) fun client => do let version ← updateVersionForRocqGoals root client path callBroker root client { op := .goals @@ -350,7 +381,7 @@ def runCommand (home : System.FilePath) (opts : CliOptions) : IO Unit := do } | "rocq-goals-prev" :: path :: line :: character :: text => let root ← projectRoot opts .rocq - withProjectDaemon home root .rocq fun client => do + withProjectDaemon home root .rocq (explicitControlDir? := opts.explicitControlDir?) fun client => do let version ← updateVersionForRocqGoals root client path callBroker root client { op := .goals @@ -369,28 +400,32 @@ def runCommand (home : System.FilePath) (opts : CliOptions) : IO Unit := do doctor home opts (← parseBackendName backend) | "open-files" :: [] => let root ← projectRootAny opts - withExistingProjectDaemon root fun client => + withExistingProjectDaemon root (explicitControlDir? := opts.explicitControlDir?) fun client => callBroker root client { op := .openDocs root? := some root.toString } | "cancel" :: requestId :: [] => let root ← projectRootAny opts - withExistingProjectDaemon root fun client => + withExistingProjectDaemon root (explicitControlDir? := opts.explicitControlDir?) fun client => callBroker root client { op := .cancel cancelRequestId? := some requestId } | "stats" :: [] => let root ← projectRootAny opts - withExistingProjectDaemon root fun client => + withExistingProjectDaemon root (explicitControlDir? := opts.explicitControlDir?) fun client => callBroker root client { op := .stats } | "reset-stats" :: [] => let root ← projectRootAny opts - withExistingProjectDaemon root fun client => + withExistingProjectDaemon root (explicitControlDir? := opts.explicitControlDir?) fun client => callBroker root client { op := .resetStats } | "shutdown" :: [] => shutdownProjectDaemon opts + | "recover" :: args => + recoverProjectSession opts args + | "request-stream" :: raw :: [] => + runProjectRequestStream opts raw | _ => throw <| IO.userError usage diff --git a/Beam/Cli/DaemonManager.lean b/Beam/Cli/DaemonManager.lean index 99b9995e..e429510c 100644 --- a/Beam/Cli/DaemonManager.lean +++ b/Beam/Cli/DaemonManager.lean @@ -53,15 +53,18 @@ private structure ProjectControl where dir : System.FilePath registry : System.FilePath -private def projectControl (root : System.FilePath) : IO ProjectControl := do - let dir ← controlDir root +private def projectControl + (root : System.FilePath) + (explicitControlDir? : Option System.FilePath := none) : IO ProjectControl := do + let dir ← controlDirFor root explicitControlDir? pure { root, dir, registry := dir / "beam-daemon.json" } /-- Supply project registry mutation only for the dynamic extent of the project control lock. -/ private def withProjectControl (root : System.FilePath) - (act : ProjectControl → IO α) : IO α := do - let control ← projectControl root + (act : ProjectControl → IO α) + (explicitControlDir? : Option System.FilePath := none) : IO α := do + let control ← projectControl root explicitControlDir? withLockTimeout (control.dir / "lock") (← projectControlLockTimeoutMs) do act control @@ -71,8 +74,9 @@ its project root. -/ private def withExistingProjectControl (root : System.FilePath) - (act : ProjectControl → IO Unit) : IO Unit := do - let control ← projectControl root + (act : ProjectControl → IO Unit) + (explicitControlDir? : Option System.FilePath := none) : IO Unit := do + let control ← projectControl root explicitControlDir? unless ← control.dir.isDir do return try @@ -99,7 +103,7 @@ private def computeConfigHash acc := mixField acc bundleId s!"{acc.toNat}" -private def writeRegistry (control : ProjectControl) (entry : RegistryEntry) : IO Unit := do +private def writeRegistry (control : ProjectControl) (entry : SessionDescriptor) : IO Unit := do if let some parent := control.registry.parent then IO.FS.createDirAll parent let tmp := control.registry.withExtension "tmp" @@ -121,7 +125,7 @@ private def writeRegistry (control : ProjectControl) (entry : RegistryEntry) : I pure () throw err -private def writeExistingRegistry (control : ProjectControl) (entry : RegistryEntry) : IO Unit := do +private def writeExistingRegistry (control : ProjectControl) (entry : SessionDescriptor) : IO Unit := do -- Teardown must not create a path while the project tree is being removed. Rewrite through an -- already existing file handle; if the registry was concurrently unlinked, this updates only the -- unlinked inode and cannot recreate the project or control directory. @@ -135,12 +139,12 @@ private def removeRegistry (control : ProjectControl) : IO Unit := do if ← control.registry.pathExists then IO.FS.removeFile control.registry -private def sameRegistryGeneration (left right : RegistryEntry) : Bool := +private def sameRegistryGeneration (left right : SessionDescriptor) : Bool := left.daemonId == right.daemonId && left.capability == right.capability /-- Remove a registry entry only when it still names the observed daemon generation. -/ -private def removeRegistryGeneration (control : ProjectControl) (entry : RegistryEntry) : IO Unit := do - match ← readRegistry control.root with +private def removeRegistryGeneration (control : ProjectControl) (entry : SessionDescriptor) : IO Unit := do + match ← readRegistryAt control.registry with | .current current => if sameRegistryGeneration current entry then removeRegistry control @@ -165,7 +169,6 @@ inductive RegistryUnsafeReason where | invalidIdentity | wrongRegistryRoot (recordedRoot : String) | invalidEndpoint - | ownerDead | endpointUnavailable | endpointUnrecognized (detail : String) | wrongEndpointRoot (daemonRoot : String) @@ -177,26 +180,22 @@ inductive RegistryObservation where | legacy | unsupported (schemaVersion : Nat) | malformed (detail : String) - | live (entry : RegistryEntry) - | draining (entry : RegistryEntry) - | staleConfirmed (entry : RegistryEntry) - | unusable (entry : RegistryEntry) (reason : RegistryUnsafeReason) - -private def recordedPidGone (pid : Nat) (domain? : Option String) : IO Bool := do - match ← (Beam.RecordedPid.mk pid domain?).observe with - | .local false => pure true - | .invalid | .local true | .differentDomain | .unknownDomain => pure false - -private def registryProcessesGone (entry : RegistryEntry) : IO Bool := do - if !(← recordedPidGone entry.ownerPid entry.ownerPidDomain?) then - return false - recordedPidGone entry.pid entry.pidDomain? - -private def registryOwnerKnownDead (entry : RegistryEntry) : IO Bool := - recordedPidGone entry.ownerPid entry.ownerPidDomain? - -def observeProjectRegistry (root : System.FilePath) : IO RegistryObservation := do - match ← readRegistry root with + | live (entry : SessionDescriptor) + | draining (entry : SessionDescriptor) + | unusable (entry : SessionDescriptor) (reason : RegistryUnsafeReason) + +/-- Select the unique descriptor binding for a canonical or filesystem-equivalent project root. -/ +def sessionWorkspaceForRoot? + (entry : SessionDescriptor) + (root : System.FilePath) : IO (Option WorkspaceBinding) := do + for workspace in entry.workspaces do + if ← Beam.sameFilePath (System.FilePath.mk workspace.root) root then + return some workspace + pure none + +private def observeProjectRegistryAt + (root registry : System.FilePath) : IO RegistryObservation := do + match ← readRegistryAt registry with | .absent => pure .absent | .legacy => pure .legacy | .unsupported schemaVersion => pure <| .unsupported schemaVersion @@ -204,30 +203,16 @@ def observeProjectRegistry (root : System.FilePath) : IO RegistryObservation := | .current entry => if entry.daemonId.isEmpty || entry.capability.isEmpty then return .unusable entry .invalidIdentity - unless ← Beam.sameFilePath (System.FilePath.mk entry.root) root do - return .unusable entry (.wrongRegistryRoot entry.root) + let some workspace ← sessionWorkspaceForRoot? entry root + | return .unusable entry (.wrongRegistryRoot entry.rootSummary) if entry.lifecycle == .draining then - if ← registryProcessesGone entry then - return .staleConfirmed entry return .draining entry - let ownerDead ← registryOwnerKnownDead entry let some endpoint := registryEndpoint? entry - | if ownerDead && (← registryProcessesGone entry) then - return .staleConfirmed entry - else - return .unusable entry .invalidEndpoint - match ← daemonGenerationStatus endpoint projectDaemonWorkspaceId root + | return .unusable entry .invalidEndpoint + match ← daemonGenerationStatus endpoint workspace.workspaceId root entry.identity entry.capability with - | .exact => - if ownerDead then - pure <| .unusable entry .ownerDead - else - pure <| .live entry - | .unavailable => - if ownerDead && (← registryProcessesGone entry) then - pure <| .staleConfirmed entry - else - pure <| .unusable entry .endpointUnavailable + | .exact => pure <| .live entry + | .unavailable => pure <| .unusable entry .endpointUnavailable | .unrecognized failure => pure <| .unusable entry (.endpointUnrecognized failure.detail) | .wrongRoot daemonRoot => @@ -235,6 +220,11 @@ def observeProjectRegistry (root : System.FilePath) : IO RegistryObservation := | .wrongGeneration daemonRoot => pure <| .unusable entry (.wrongGeneration daemonRoot) +def observeProjectRegistry + (root : System.FilePath) + (explicitControlDir? : Option System.FilePath := none) : IO RegistryObservation := do + observeProjectRegistryAt root (← registryPathFor root explicitControlDir?) + private def requestedPortNat? (opts : CliOptions) : Option Nat := opts.requestedPort?.map (·.toNat) @@ -287,8 +277,10 @@ private partial def selectUnoccupiedEndpoint private def daemonFailureIncidentRetainCount : Nat := 50 -private def pruneDaemonFailureIncidents (root : System.FilePath) : IO Unit := do - let entries ← Beam.Daemon.daemonFailureIncidentEntries root +private def pruneDaemonFailureIncidents + (root : System.FilePath) + (explicitControlDir? : Option System.FilePath := none) : IO Unit := do + let entries ← Beam.Daemon.daemonFailureIncidentEntries root explicitControlDir? let keep := min daemonFailureIncidentRetainCount entries.size let deleteCount := entries.size - keep for entry in entries.toList.take deleteCount do @@ -330,8 +322,9 @@ private def daemonFailureIncidentTimestampLabel (timestamp : String) : String := private def daemonFailureIncidentPath (root : System.FilePath) - (kind observedAt : String) : IO System.FilePath := do - let dir ← daemonFailureIncidentDir root + (kind observedAt : String) + (explicitControlDir? : Option System.FilePath := none) : IO System.FilePath := do + let dir ← daemonFailureIncidentDirFor root explicitControlDir? let pid ← IO.Process.getPID let unique ← IO.monoNanosNow let stamp := daemonFailureIncidentTimestampLabel observedAt @@ -340,19 +333,20 @@ private def daemonFailureIncidentPath private def writeDaemonFailureIncident? (root : System.FilePath) (kind detail : String) - (logTail? : Option (System.FilePath × String)) : IO (Option System.FilePath) := do + (logTail? : Option (System.FilePath × String)) + (explicitControlDir? : Option System.FilePath := none) : IO (Option System.FilePath) := do try - let dir ← daemonFailureIncidentDir root + let dir ← daemonFailureIncidentDirFor root explicitControlDir? IO.FS.createDirAll dir - let registryFile ← registryPath root - let registryRead ← readRegistry root + let registryFile ← registryPathFor root explicitControlDir? + let registryRead ← readRegistryAt registryFile let registry := registryRead.entry? let pidStatus ← match registry with | none => pure none | some entry => some <$> registryPidStatus entry let endpoint := registry.map registryEndpointSummary - let control ← controlDir root + let control ← controlDirFor root explicitControlDir? let observedAt ← Beam.utcTimestamp let incident : DaemonFailureIncident := { schemaVersion := daemonFailureIncidentSchemaVersion @@ -369,12 +363,12 @@ private def writeDaemonFailureIncident? startupLogPath := logTail?.map (fun (path, _) => path.toString) startupLogTail := logTail?.map (fun (_, tail) => tail) } - let path ← daemonFailureIncidentPath root kind observedAt + let path ← daemonFailureIncidentPath root kind observedAt explicitControlDir? let tmp := path.withExtension "tmp" IO.FS.writeFile tmp ((toJson incident).pretty ++ "\n") IO.FS.rename tmp path try - pruneDaemonFailureIncidents root + pruneDaemonFailureIncidents root explicitControlDir? catch _ => pure () pure (some path) @@ -383,19 +377,20 @@ private def writeDaemonFailureIncident? def daemonFailureMessage (root : System.FilePath) - (failure : BrokerClientFailure) : IO String := do + (failure : BrokerClientFailure) + (explicitControlDir? : Option System.FilePath := none) : IO String := do let detail := failure.detail match daemonFailureIncidentKind? failure with | none => pure detail | some kind => - let msg := appendMaybeSection detail (← daemonRegistryContext? root) - let logTail? ← startupLogTail? root + let msg := appendMaybeSection detail (← daemonRegistryContext? root explicitControlDir?) + let logTail? ← startupLogTail? root explicitControlDir? let msg := match logTail? with | none => msg | some (logPath, logTail) => msg ++ s!"\nBeam daemon log tail ({logPath}):\n{logTail}" - let incidentPath? ← writeDaemonFailureIncident? root kind detail logTail? + let incidentPath? ← writeDaemonFailureIncident? root kind detail logTail? explicitControlDir? pure <| appendMaybeSection msg <| incidentPath?.map fun path => s!"Beam daemon incident: {path}" @@ -568,7 +563,7 @@ private def registryEntryFor (capability : String) (pid : Nat) (endpoint : Transport.Endpoint) - (opts : CliOptions) : IO RegistryEntry := do + (opts : CliOptions) : IO SessionDescriptor := do let port? := match endpoint with | .tcp port => some port.toNat @@ -584,15 +579,19 @@ private def registryEntryFor ownerPid := ownerPid.toNat ownerPidDomain? := pidDomain? port? - root := desired.root.toString + workspaces := #[{ + workspaceId := projectDaemonWorkspaceId + root := desired.root.toString + configHash := desired.configHash + leanCmd? := desired.leanCmd? + plugin? := desired.plugin?.map (·.toString) + rocqCmd? := desired.rocqCmd? + toolchain? := desired.toolchain? + bundleId? := some desired.bundleId + }] configHash := desired.configHash - leanCmd? := desired.leanCmd? - plugin? := desired.plugin?.map (·.toString) - rocqCmd? := desired.rocqCmd? - toolchain? := desired.toolchain? clientBin? := some desired.clientBin.toString daemonBin? := some desired.daemonBin.toString - bundleId? := some desired.bundleId startedAt := ← Beam.utcTimestamp requestedPort? := requestedPortNat? opts } @@ -600,14 +599,15 @@ private def registryEntryFor private partial def startDaemonEntry (desired : DesiredConfig) (opts : CliOptions) - (tries : Nat := 10) : IO (Transport.Endpoint × RegistryEntry × IO.Process.Child daemonStdio) := do + (controlDir : System.FilePath) + (tries : Nat := 10) : IO (Transport.Endpoint × SessionDescriptor × IO.Process.Child daemonStdio) := do let endpoint ← selectUnoccupiedEndpoint desired opts - let logPath ← daemonStartupLogPath desired.root + let logPath ← daemonStartupLogPathFor desired.root (some controlDir) let daemonId ← newDaemonGenerationId desired.configHash let identity : DaemonIdentity := { daemonId, configHash := desired.configHash } let capability ← newDaemonCapability let child ← startDaemon desired endpoint logPath identity capability - let readiness : Except DaemonStartupFailure RegistryEntry ← + let readiness : Except DaemonStartupFailure SessionDescriptor ← try match ← waitForDaemon child endpoint logPath desired.root identity capability with | .ok () => @@ -626,7 +626,7 @@ private partial def startDaemonEntry let endpointOccupied ← endpointAcceptsConnection endpoint if shouldRetryAutomaticStartup (usesAutomaticTcpEndpoint opts) tries endpointOccupied failure.endpointInUse then - return ← startDaemonEntry desired opts (tries - 1) + return ← startDaemonEntry desired opts controlDir (tries - 1) throw <| IO.userError failure.message def desiredConfig (home root : System.FilePath) (required : Backend) : IO DesiredConfig := do @@ -683,57 +683,94 @@ def desiredConfig (home root : System.FilePath) (required : Backend) : IO Desire structure ProjectDaemonClient where endpoint : Transport.Endpoint capability : String + workspaceId : WorkspaceId := projectDaemonWorkspaceId + controlDir? : Option System.FilePath := none def ProjectDaemonClient.authorize (client : ProjectDaemonClient) (request : Request) : Request := { request with daemonCapability? := some client.capability } -private def projectDaemonClient (entry : RegistryEntry) : IO ProjectDaemonClient := do +private def projectDaemonClient + (entry : SessionDescriptor) + (workspace : WorkspaceBinding) + (controlDir : System.FilePath) : IO ProjectDaemonClient := do pure { endpoint := ← Beam.Daemon.endpointFromEntry entry capability := entry.capability + workspaceId := workspace.workspaceId + controlDir? := some controlDir } +private def workspaceSupportsBackend (workspace : WorkspaceBinding) : Backend → Bool + | .lean => workspace.leanCmd?.isSome && workspace.plugin?.isSome + | .rocq => workspace.rocqCmd?.isSome + +private def selectWorkspaceBackend + (root : System.FilePath) + (entry : SessionDescriptor) + (backend? : Option Backend) : IO WorkspaceBinding := do + let some workspace ← sessionWorkspaceForRoot? entry root + | throw <| IO.userError s!"the selected Beam session does not contain workspace {root}" + if let some backend := backend? then + unless workspaceSupportsBackend workspace backend do + throw <| IO.userError <| + s!"the owned Beam session for {root} does not provide the {toJson backend |>.compress} backend; " ++ + "interrupt its foreground owner and start a session configured for that backend" + pure workspace + +structure SelectedProjectDaemon where + client : ProjectDaemonClient + workspace : WorkspaceBinding + def RegistryUnsafeReason.message : RegistryUnsafeReason → String | .invalidIdentity => "registry identity or capability is empty" | .wrongRegistryRoot recordedRoot => s!"registry records another root: {recordedRoot}" | .invalidEndpoint => "registry endpoint is invalid" - | .ownerDead => "the recorded owner is dead while its daemon still responds" | .endpointUnavailable => "the recorded daemon endpoint is unavailable" | .endpointUnrecognized detail => s!"the recorded endpoint is not a recognized Beam generation: {detail}" | .wrongEndpointRoot daemonRoot => s!"the recorded endpoint serves another root: {daemonRoot}" | .wrongGeneration daemonRoot => s!"the recorded endpoint serves another Beam generation for {daemonRoot}" -private def activeOwnerMessage (root : System.FilePath) (entry : RegistryEntry) : String := +private def activeOwnerMessage (root : System.FilePath) (entry : SessionDescriptor) : String := s!"Beam session for {root} is already owned by wrapper pid {entry.ownerPid}; " ++ "interrupt that 'lean-beam ensure --hold' process before starting another owner" private def configMismatchMessage (root : System.FilePath) - (entry : RegistryEntry) + (entry : SessionDescriptor) (expectedHash : String) : String := s!"the live Beam session for {root} uses configuration {entry.configHash}, " ++ s!"but this command requires {expectedHash}; the current owner was preserved. " ++ "Interrupt its 'lean-beam ensure --hold' process, then start a new owner with the desired configuration" -private def drainingOwnerMessage (root : System.FilePath) (entry : RegistryEntry) : String := +private def drainingOwnerMessage (root : System.FilePath) (entry : SessionDescriptor) : String := s!"Beam session {entry.daemonId} for {root} is draining; " ++ "wait for its foreground owner to exit before starting or attaching to another session" private def registryRecoveryMessage (root : System.FilePath) (detail : String) : String := s!"Beam cannot safely use or replace the daemon registry for {root}: {detail}. " ++ - "Preserve the registry, stop the matching foreground owner or daemon explicitly, and retry" + "The session remains fenced; preserve its descriptor and use explicit recovery with the same " ++ + "--root and --control-dir selection after stopping the matching owner or daemon" + +private def generationRecoveryMessage + (root : System.FilePath) + (entry : SessionDescriptor) + (detail : String) : String := + registryRecoveryMessage root detail ++ "; run " ++ + s!"'lean-beam --root {root} recover --generation {entry.daemonId}' when recovery is safe" private def registryReadRecoveryMessage (root : System.FilePath) (registryRead : RegistryRead) : String := - registryRecoveryMessage root <| - registryRead.detail?.getD s!"unexpected registry state '{registryRead.status}'" + registryRecoveryMessage root + (registryRead.detail?.getD s!"unexpected registry state '{registryRead.status}'") ++ + "; opaque state can be quarantined explicitly with " ++ + "'lean-beam --root ROOT recover --force'" -private def markRegistryDraining (control : ProjectControl) (entry : RegistryEntry) : IO Unit := do - match ← readRegistry control.root with +private def markRegistryDraining (control : ProjectControl) (entry : SessionDescriptor) : IO Unit := do + match ← readRegistryAt control.registry with | .current current => if sameRegistryGeneration current entry && current.lifecycle == .live then writeExistingRegistry control { current with lifecycle := .draining } @@ -741,17 +778,17 @@ private def markRegistryDraining (control : ProjectControl) (entry : RegistryEnt private inductive ShutdownPlan where | none - | request (entry : RegistryEntry) + | request (entry : SessionDescriptor) /-- Fence and request shutdown of the exact wrapper-owned generation without PID signalling. -/ def shutdownRegisteredProjectDaemon - (root : System.FilePath) : IO (Except BrokerClientFailure (Option Response)) := do - let plan : ShutdownPlan ← withProjectControl root fun control => do - match ← observeProjectRegistry root with + (root : System.FilePath) + (explicitControlDir? : Option System.FilePath := none) : + IO (Except BrokerClientFailure (Option Response)) := do + let plan : ShutdownPlan ← withProjectControl root + (explicitControlDir? := explicitControlDir?) fun control => do + match ← observeProjectRegistryAt root control.registry with | .absent => pure ShutdownPlan.none - | .staleConfirmed entry => - removeRegistryGeneration control entry - pure ShutdownPlan.none | .live entry => markRegistryDraining control entry pure <| ShutdownPlan.request entry @@ -762,8 +799,8 @@ def shutdownRegisteredProjectDaemon throw <| IO.userError <| registryReadRecoveryMessage root (.unsupported schemaVersion) | .malformed detail => throw <| IO.userError <| registryReadRecoveryMessage root (.malformed detail) - | .unusable _ reason => - throw <| IO.userError <| registryRecoveryMessage root reason.message + | .unusable entry reason => + throw <| IO.userError <| generationRecoveryMessage root entry reason.message match plan with | ShutdownPlan.none => pure <| .ok none | ShutdownPlan.request entry => @@ -771,6 +808,70 @@ def shutdownRegisteredProjectDaemon | return .error <| .invalidResponse "draining Beam registry has no valid endpoint" pure <| (← requestDaemonShutdown endpoint entry.capability).map some +structure RecoveryResult where + recovered : Bool + generation? : Option String := none + quarantinedPath? : Option String := none + reason? : Option String := none + deriving ToJson + +private def quarantineRegistry (control : ProjectControl) : IO System.FilePath := do + let nonce ← IO.monoNanosNow + let quarantine := control.dir / s!"beam-daemon.recovered-{nonce}.json" + IO.FS.rename control.registry quarantine + pure quarantine + +private def registeredGenerationResponds + (root : System.FilePath) + (entry : SessionDescriptor) : IO Bool := do + let some workspace ← sessionWorkspaceForRoot? entry root + | pure false + let some endpoint := registryEndpoint? entry + | pure false + match ← daemonGenerationStatus endpoint workspace.workspaceId root entry.identity entry.capability with + | .exact => pure true + | .unavailable | .unrecognized _ | .wrongRoot _ | .wrongGeneration _ => pure false + +/-- +Explicitly quarantine one unusable session descriptor without treating persisted PIDs as signal +capabilities. Current descriptors require their exact generation; opaque descriptors require force. +-/ +def recoverProjectDaemon + (root : System.FilePath) + (generation? : Option String) + (forceOpaque : Bool) + (explicitControlDir? : Option System.FilePath := none) : IO RecoveryResult := do + withProjectControl root (explicitControlDir? := explicitControlDir?) fun control => do + match ← readRegistryAt control.registry with + | .absent => + pure { recovered := false, reason? := some "absent" } + | .current entry => + let some generation := generation? + | throw <| IO.userError + s!"recovery of current session {entry.daemonId} requires --generation {entry.daemonId}" + unless generation == entry.daemonId do + throw <| IO.userError <| + s!"recovery generation '{generation}' does not match recorded generation '{entry.daemonId}'" + if ← registeredGenerationResponds root entry then + throw <| IO.userError <| + s!"Beam session {entry.daemonId} still responds; stop its foreground owner or use authenticated shutdown" + let quarantine ← quarantineRegistry control + pure { + recovered := true + generation? := some entry.daemonId + quarantinedPath? := some quarantine.toString + } + | .legacy | .unsupported _ | .malformed _ => + unless forceOpaque do + throw <| IO.userError + "opaque legacy, unsupported, or malformed session state requires recover --force" + let quarantine ← quarantineRegistry control + pure { + recovered := true + quarantinedPath? := some quarantine.toString + reason? := some "opaque" + } + private abbrev detachedDaemonStdio : IO.Process.StdioConfig where stdin := .null stdout := .null @@ -778,12 +879,13 @@ private abbrev detachedDaemonStdio : IO.Process.StdioConfig where private structure OwnedProjectDaemon where client : ProjectDaemonClient - entry : RegistryEntry + entry : SessionDescriptor child : IO.Process.Child daemonStdio structure ProjectDaemonOwner where client : ProjectDaemonClient private root : System.FilePath + private controlDir : System.FilePath private daemonId : String private child : IO.Process.Child daemonStdio private exitCodeRef : IO.Ref (Option UInt32) @@ -799,7 +901,7 @@ def ProjectDaemonOwner.exitCode? (owner : ProjectDaemonOwner) : IO (Option UInt3 /-- Whether this owner generation is still the one published for its project. -/ def ProjectDaemonOwner.registered (owner : ProjectDaemonOwner) : IO Bool := do - match ← readRegistry owner.root with + match ← readRegistryAt (owner.controlDir / "beam-daemon.json") with | .current current => pure (current.daemonId == owner.daemonId && current.capability == owner.client.capability && current.lifecycle == .live) @@ -817,9 +919,8 @@ private def startOwnedProjectDaemon (control : ProjectControl) (desired : DesiredConfig) (opts : CliOptions) : IO OwnedProjectDaemon := do - match ← observeProjectRegistry desired.root with + match ← observeProjectRegistryAt desired.root control.registry with | .absent => pure () - | .staleConfirmed entry => removeRegistryGeneration control entry | .live entry => if entry.configHash == desired.configHash then throw <| IO.userError (activeOwnerMessage desired.root entry) @@ -833,16 +934,20 @@ private def startOwnedProjectDaemon registryReadRecoveryMessage desired.root (.unsupported schemaVersion) | .malformed detail => throw <| IO.userError <| registryReadRecoveryMessage desired.root (.malformed detail) - | .unusable _ reason => - throw <| IO.userError <| registryRecoveryMessage desired.root reason.message - let (endpoint, entry, child) ← startDaemonEntry desired opts + | .unusable entry reason => + throw <| IO.userError <| generationRecoveryMessage desired.root entry reason.message + let (endpoint, entry, child) ← startDaemonEntry desired opts control.dir try writeRegistry control entry catch err => terminateDaemonChild child throw err pure { - client := { endpoint, capability := entry.capability } + client := { + endpoint + capability := entry.capability + controlDir? := some control.dir + } entry child } @@ -866,9 +971,11 @@ private partial def waitForOwnedDaemonExit IO.sleep 100 waitForOwnedDaemonExit child exitCodeRef (tries - 1) -private def removeOwnedRegistry (root : System.FilePath) (entry : RegistryEntry) : IO Unit := do +private def removeOwnedRegistry + (root controlDir : System.FilePath) + (entry : SessionDescriptor) : IO Unit := do try - withExistingProjectControl root fun control => + withExistingProjectControl root (explicitControlDir? := some controlDir) fun control => removeRegistryGeneration control entry catch _ => pure () @@ -895,20 +1002,41 @@ private def finishOwnedDaemonChild attemptCleanup <| waitForOwnedDaemonExit owned.child exitCodeRef 20 pure (← exitCodeRef.get).isSome -private def markOwnedRegistryDraining (root : System.FilePath) (entry : RegistryEntry) : IO Unit := do +private def markOwnedRegistryDraining + (root controlDir : System.FilePath) + (entry : SessionDescriptor) : IO Unit := do try - withExistingProjectControl root fun control => + withExistingProjectControl root (explicitControlDir? := some controlDir) fun control => markRegistryDraining control entry catch _ => pure () private def finishOwnedProjectDaemon (root : System.FilePath) + (controlDir : System.FilePath) (owned : OwnedProjectDaemon) (exitCodeRef : IO.Ref (Option UInt32)) : IO Unit := do - markOwnedRegistryDraining root owned.entry - if ← finishOwnedDaemonChild owned exitCodeRef then - removeOwnedRegistry root owned.entry + let exitedBeforeOwnerCleanup ← + match ← exitCodeRef.get with + | some _ => pure true + | none => + match ← owned.child.tryWait with + | some exitCode => + exitCodeRef.set (some exitCode) + pure true + | none => pure false + let registryWasDraining ← + match ← readRegistryAt (controlDir / "beam-daemon.json") with + | .current current => + pure (sameRegistryGeneration current owned.entry && current.lifecycle == .draining) + | .absent | .legacy | .unsupported _ | .malformed _ => pure false + markOwnedRegistryDraining root controlDir owned.entry + if exitedBeforeOwnerCleanup && !registryWasDraining then + -- An unexpected daemon exit is not evidence that its complete process tree disappeared. Keep + -- the exact generation fenced for explicit, non-signalling recovery. + pure () + else if ← finishOwnedDaemonChild owned exitCodeRef then + removeOwnedRegistry root controlDir owned.entry def withProjectDaemonOwner (home root : System.FilePath) @@ -916,35 +1044,32 @@ def withProjectDaemonOwner (opts : CliOptions) (act : ProjectDaemonOwner → IO α) : IO α := do let desired ← desiredConfig home root backend - let owned ← withProjectControl root fun control => + let controlDir ← controlDirFor root opts.explicitControlDir? + let owned ← withProjectControl root (explicitControlDir? := some controlDir) fun control => startOwnedProjectDaemon control desired opts let exitCodeRef ← IO.mkRef (none : Option UInt32) try act { client := owned.client root + controlDir daemonId := owned.entry.daemonId child := owned.child exitCodeRef } finally - finishOwnedProjectDaemon root owned exitCodeRef + finishOwnedProjectDaemon root controlDir owned exitCodeRef private def lookupProjectDaemon (root : System.FilePath) - (expectedHash? : Option String := none) - (backend? : Option Backend := none) : IO ProjectDaemonClient := do - withProjectControl root fun _control => do - match ← observeProjectRegistry root with + (backend? : Option Backend := none) + (explicitControlDir? : Option System.FilePath := none) : IO SelectedProjectDaemon := do + withProjectControl root (explicitControlDir? := explicitControlDir?) fun control => do + match ← observeProjectRegistryAt root control.registry with | .live entry => - match expectedHash? with - | some expectedHash => - if entry.configHash == expectedHash then - projectDaemonClient entry - else - throw <| IO.userError (configMismatchMessage root entry expectedHash) - | none => projectDaemonClient entry - | .absent | .staleConfirmed _ => + let workspace ← selectWorkspaceBackend root entry backend? + pure { client := ← projectDaemonClient entry workspace control.dir, workspace } + | .absent => throw <| IO.userError (missingOwnerMessage root backend?) | .draining entry => throw <| IO.userError (drainingOwnerMessage root entry) | .legacy => @@ -953,18 +1078,26 @@ private def lookupProjectDaemon throw <| IO.userError <| registryReadRecoveryMessage root (.unsupported schemaVersion) | .malformed detail => throw <| IO.userError <| registryReadRecoveryMessage root (.malformed detail) - | .unusable _ reason => - throw <| IO.userError <| registryRecoveryMessage root reason.message + | .unusable entry reason => + throw <| IO.userError <| generationRecoveryMessage root entry reason.message def withProjectDaemon - (home root : System.FilePath) + (_home root : System.FilePath) (backend : Backend) - (act : ProjectDaemonClient → IO α) : IO α := do - let desired ← desiredConfig home root backend - act (← lookupProjectDaemon root (some desired.configHash) (some backend)) + (act : ProjectDaemonClient → IO α) + (explicitControlDir? : Option System.FilePath := none) : IO α := do + act (← lookupProjectDaemon root (some backend) explicitControlDir?).client def withExistingProjectDaemon (root : System.FilePath) - (act : ProjectDaemonClient → IO α) : IO α := do - act (← lookupProjectDaemon root) + (act : ProjectDaemonClient → IO α) + (explicitControlDir? : Option System.FilePath := none) : IO α := do + act (← lookupProjectDaemon root (explicitControlDir? := explicitControlDir?)).client + +/-- Select one live workspace session without resolving local toolchain or bundle configuration. -/ +def withSelectedProjectDaemon + (root : System.FilePath) + (act : SelectedProjectDaemon → IO α) + (explicitControlDir? : Option System.FilePath := none) : IO α := do + act (← lookupProjectDaemon root (explicitControlDir? := explicitControlDir?)) end Beam.Cli diff --git a/Beam/Cli/Feedback.lean b/Beam/Cli/Feedback.lean index 25c28730..f09cd8c0 100644 --- a/Beam/Cli/Feedback.lean +++ b/Beam/Cli/Feedback.lean @@ -93,28 +93,37 @@ private def versionIdentityJson (home : System.FilePath) : IO Json := do private def collectDaemonPayload (root : System.FilePath) + (explicitControlDir? : Option System.FilePath) (warnings : Array String) : IO (Json × Json × Array String) := do - match ← observeProjectRegistry root with + match ← observeProjectRegistry root explicitControlDir? with | .live entry => match Beam.Daemon.registryEndpoint? entry with | none => pure (Json.null, Json.null, warnings.push "Beam daemon registry did not contain a valid endpoint") | some endpoint => - let client : ProjectDaemonClient := { endpoint, capability := entry.capability } + let some workspace ← Beam.Cli.sessionWorkspaceForRoot? entry root + | return (Json.null, Json.null, + warnings.push "the Beam session does not contain the selected project root") + let client : ProjectDaemonClient := { + endpoint + capability := entry.capability + workspaceId := workspace.workspaceId + controlDir? := explicitControlDir? + } let statsResp ← sendRequest endpoint <| client.authorize { op := .stats - workspaceId? := some Beam.Cli.projectDaemonWorkspaceId + workspaceId? := some client.workspaceId root? := some root.toString } let (stats, warnings) := Beam.Feedback.responsePayloadOrWarning "stats" statsResp warnings let openResp ← sendRequest endpoint <| client.authorize { op := .openDocs - workspaceId? := some Beam.Cli.projectDaemonWorkspaceId + workspaceId? := some client.workspaceId root? := some root.toString } let (openDocs, warnings) := Beam.Feedback.responsePayloadOrWarning "open-files" openResp warnings pure (stats, openDocs, warnings) - | .absent | .staleConfirmed _ => + | .absent => pure (Json.null, Json.null, warnings.push "no live Beam daemon was available for stats/open-files") | .draining _ => pure (Json.null, Json.null, warnings.push "the Beam daemon is draining") @@ -130,6 +139,7 @@ private def collectDaemonPayload private def collectNonConfidential (home : System.FilePath) (root? : Option System.FilePath) + (explicitControlDir? : Option System.FilePath) (warnings : Array String) : IO Beam.Feedback.Collection := do let generatedAt ← Beam.utcTimestamp let identity ← versionIdentityJson home @@ -139,9 +149,9 @@ private def collectNonConfidential pure (Json.null, Json.null, Json.null, warnings.push "could not infer project root; daemon debug context was not collected") | some root => do - let daemon ← Beam.Daemon.daemonDebugContextJson root + let daemon ← Beam.Daemon.daemonDebugContextJson root explicitControlDir? let warnings := warnings ++ Beam.Daemon.daemonDebugWarnings daemon - let (stats, openDocs, warnings) ← collectDaemonPayload root warnings + let (stats, openDocs, warnings) ← collectDaemonPayload root explicitControlDir? warnings pure (stats, openDocs, daemon, warnings) pure { generatedAt @@ -208,12 +218,13 @@ def run (home : System.FilePath) (cliOpts : CliOptions) (args : List String) : I else pure (none, #[]) let collection ← - if input.confidential then collectConfidential else collectNonConfidential home root? warnings + if input.confidential then collectConfidential + else collectNonConfidential home root? cliOpts.explicitControlDir? warnings let allowedRoots ← if Beam.Feedback.Internal.needsEvidenceRoots input then match root? with | some root => do - let control ← Beam.Daemon.controlDir root + let control ← Beam.Daemon.controlDirFor root cliOpts.explicitControlDir? pure #[root, control] | none => pure #[] else diff --git a/Beam/Cli/Info.lean b/Beam/Cli/Info.lean index 19661fd3..701b7752 100644 --- a/Beam/Cli/Info.lean +++ b/Beam/Cli/Info.lean @@ -145,8 +145,10 @@ private def printRocqDoctorInfo (home root : System.FilePath) : IO Unit := do IO.println s!"daemon binary: {paths.daemon}" IO.println s!"client binary: {paths.client}" -def daemonFailureIncidentDoctorLines (root : System.FilePath) : IO (List String) := do - let incidents ← Beam.Daemon.recentDaemonFailureIncidentPaths root +def daemonFailureIncidentDoctorLines + (root : System.FilePath) + (explicitControlDir? : Option System.FilePath := none) : IO (List String) := do + let incidents ← Beam.Daemon.recentDaemonFailureIncidentPaths root 5 explicitControlDir? if incidents.isEmpty then pure ["daemon incidents: none"] else @@ -154,8 +156,10 @@ def daemonFailureIncidentDoctorLines (root : System.FilePath) : IO (List String) s!"daemon incidents: {incidents.size} recent" :: (incidents.toList.map fun path => s!"daemon incident: {path}") -private def printDaemonFailureIncidentDoctorInfo (root : System.FilePath) : IO Unit := do - for line in ← daemonFailureIncidentDoctorLines root do +private def printDaemonFailureIncidentDoctorInfo + (root : System.FilePath) + (explicitControlDir? : Option System.FilePath := none) : IO Unit := do + for line in ← daemonFailureIncidentDoctorLines root explicitControlDir? do IO.println line def doctor (home : System.FilePath) (opts : CliOptions) (backend : Backend) : IO Unit := do @@ -165,9 +169,9 @@ def doctor (home : System.FilePath) (opts : CliOptions) (backend : Backend) : IO match backend with | .lean => printLeanDoctorInfo home root | .rocq => printRocqDoctorInfo home root - let registry ← Beam.Daemon.registryPath root + let registry ← Beam.Daemon.registryPathFor root opts.explicitControlDir? IO.println s!"registry: {registry}" - match ← observeProjectRegistry root with + match ← observeProjectRegistry root opts.explicitControlDir? with | .live entry => IO.println "daemon status: live" IO.println s!"daemon pid: {entry.pid}" @@ -181,7 +185,6 @@ def doctor (home : System.FilePath) (opts : CliOptions) (backend : Backend) : IO | .draining entry => IO.println "daemon status: draining" IO.println s!"daemon generation: {entry.daemonId}" - | .staleConfirmed _ => IO.println "daemon status: stale" | .absent => IO.println "daemon status: absent" | .legacy => IO.println "daemon status: legacy registry" | .unsupported schemaVersion => @@ -193,7 +196,7 @@ def doctor (home : System.FilePath) (opts : CliOptions) (backend : Backend) : IO | .unusable _ reason => IO.println "daemon status: unsafe" IO.println s!"daemon safety error: {reason.message}" - printDaemonFailureIncidentDoctorInfo root + printDaemonFailureIncidentDoctorInfo root opts.explicitControlDir? def printValidatedToolchains (home : System.FilePath) (backendName : String) : IO Unit := do match backendName with diff --git a/Beam/Cli/Usage.lean b/Beam/Cli/Usage.lean index e2bcc3b0..dd434bf8 100644 --- a/Beam/Cli/Usage.lean +++ b/Beam/Cli/Usage.lean @@ -47,7 +47,11 @@ def usage : String := " beam [--root PATH] stats", " beam [--root PATH] reset-stats", " beam [--root PATH] shutdown", + " beam [--root PATH] [--control-dir DIR] recover --generation ID | --force", + " beam --root PATH [--control-dir DIR] request-stream ", "", + "Project-session commands accept --control-dir DIR as an exact alternate control selection.", + "Use the same --root and --control-dir for owner, attachment, diagnostics, shutdown, and recovery.", "Beam never applies source edits to `.lean` files on disk; the client applies source edits.", "Lean edit loop: save the file, then run lean-update for a broker document version.", "Run lean-sync when you need the diagnostics/readiness barrier. lean-save is lean-sync plus a", @@ -63,6 +67,7 @@ def usage : String := "Wrapper commands require one live session owner. Start ensure --hold in a foreground process", "and keep it running across wrapper invocations; interrupt it or run shutdown when finished.", "Plain ensure checks and warms that owned session but never starts one implicitly.", + "Abnormal session state remains fenced until recover --generation ID quarantines that record.", "Beam does not upload or submit feedback. Use feedback-report to print a pasteable report", "with cheap version, stats, open-files, daemon registry, and daemon incident context.", "Review non-confidential reports before posting because they may contain project context and", @@ -78,7 +83,7 @@ def usage : String := "Wrapper diagnostics and progress are human-facing on stderr.", "Set BEAM_DEBUG_TEXT=1 to print the exact escaped text and UTF-8 bytes sent for text-carrying", "Lean probe requests.", - "For machine-readable streaming diagnostics/progress, use beam-client request-stream.", + "For machine-readable streaming diagnostics/progress, use beam --root PATH request-stream.", "For the Lean workflow contract and anti-patterns, see skills/lean-beam/SKILL.md." ] diff --git a/Beam/Daemon/Debug.lean b/Beam/Daemon/Debug.lean index 7d3c7155..27ce6141 100644 --- a/Beam/Daemon/Debug.lean +++ b/Beam/Daemon/Debug.lean @@ -13,9 +13,11 @@ open Lean namespace Beam.Daemon -def daemonFailureIncidentEntries (root : System.FilePath) : IO (Array IO.FS.DirEntry) := do +def daemonFailureIncidentEntries + (root : System.FilePath) + (explicitControlDir? : Option System.FilePath := none) : IO (Array IO.FS.DirEntry) := do try - let dir ← daemonFailureIncidentDir root + let dir ← daemonFailureIncidentDirFor root explicitControlDir? unless ← dir.pathExists do return #[] let entries ← dir.readDir @@ -24,16 +26,22 @@ def daemonFailureIncidentEntries (root : System.FilePath) : IO (Array IO.FS.DirE catch _ => pure #[] -def recentDaemonFailureIncidentPaths (root : System.FilePath) (limit : Nat := 5) : +def recentDaemonFailureIncidentPaths + (root : System.FilePath) + (limit : Nat := 5) + (explicitControlDir? : Option System.FilePath := none) : IO (Array System.FilePath) := do - let entries ← daemonFailureIncidentEntries root + let entries ← daemonFailureIncidentEntries root explicitControlDir? let keep := min limit entries.size let recent := entries.toList.drop (entries.size - keep) pure <| recent.foldl (fun acc entry => acc.push entry.path) #[] -private def recentDaemonFailureIncidentJson (root : System.FilePath) (limit : Nat := 5) : +private def recentDaemonFailureIncidentJson + (root : System.FilePath) + (limit : Nat := 5) + (explicitControlDir? : Option System.FilePath := none) : IO (Array Json) := do - let paths ← recentDaemonFailureIncidentPaths root limit + let paths ← recentDaemonFailureIncidentPaths root limit explicitControlDir? let mut incidents := #[] for path in paths do let payload ← @@ -59,12 +67,12 @@ private def tailLines (text : String) (count : Nat := 20) : String := let keep := min count lines.length String.intercalate "\n" <| lines.drop (lines.length - keep) -def registryEndpointSummary (entry : RegistryEntry) : String := +def registryEndpointSummary (entry : SessionDescriptor) : String := match registryEndpoint? entry with | some endpoint => endpointSummary endpoint | none => "invalid" -def registryPidStatus (entry : RegistryEntry) : IO String := do +def registryPidStatus (entry : SessionDescriptor) : IO String := do let recorded : Beam.RecordedPid := { pid := entry.pid, domain? := entry.pidDomain? } try match ← recorded.observe with @@ -76,9 +84,12 @@ def registryPidStatus (entry : RegistryEntry) : IO String := do catch _ => pure "unavailable" -def startupLogTail? (root : System.FilePath) : IO (Option (System.FilePath × String)) := do +def startupLogTail? + (root : System.FilePath) + (explicitControlDir? : Option System.FilePath := none) : + IO (Option (System.FilePath × String)) := do try - let logPath ← daemonStartupLogPath root + let logPath ← daemonStartupLogPathFor root explicitControlDir? if ← logPath.pathExists then let logText := Beam.trimLine (← IO.FS.readFile logPath) if logText.isEmpty then @@ -103,7 +114,8 @@ private def jsonNonNullField (json : Json) (field : String) : Bool := def daemonDebugWarnings (debug : Json) : Array String := Id.run do let mut warnings := #[] - let recoveryHint := "Run `lean-beam shutdown`, then start `lean-beam ensure --hold` from the project root to refresh the owned session." + let pidHint := + "Persisted PIDs are diagnostic only; do not reclaim or replace the session from PID status alone." if jsonNonNullField debug "registry" then match jsonStringField? debug "registryPidStatus" with | some "not alive" => @@ -113,10 +125,10 @@ def daemonDebugWarnings (debug : Json) : Array String := Id.run do else "" warnings := warnings.push - s!"Beam daemon registry pid is not alive{detail}; stats/open-files may come from a live endpoint with stale registry metadata. {recoveryHint}" + s!"Beam daemon registry pid is not alive{detail}; stats/open-files may come from a live endpoint with stale registry metadata. {pidHint}" | some "unavailable" => warnings := warnings.push - s!"Beam could not verify the daemon registry pid; stats/open-files may reflect a daemon whose registry metadata cannot be trusted. {recoveryHint}" + s!"Beam could not verify the daemon registry pid; stats/open-files may reflect a daemon whose registry metadata cannot be trusted. {pidHint}" | _ => pure () warnings @@ -125,23 +137,30 @@ private def optionLine (label : String) : Option String → Option String | none => none | some value => some s!" {label}: {value}" -def daemonRegistryContext? (root : System.FilePath) : IO (Option String) := do +def daemonRegistryContext? + (root : System.FilePath) + (explicitControlDir? : Option System.FilePath := none) : IO (Option String) := do try - match ← readRegistry root with + let path ← registryPathFor root explicitControlDir? + match ← readRegistryAt path with | .absent => pure none | .legacy => - let path ← registryPath root pure <| some s!"Beam daemon registry ({path}):\n status: legacy\n detail: legacy registry has no schemaVersion" | .unsupported schemaVersion => - let path ← registryPath root let detail := (RegistryRead.unsupported schemaVersion).detail?.getD "unsupported registry" pure <| some s!"Beam daemon registry ({path}):\n status: unsupported\n detail: {detail}" | .malformed detail => - let path ← registryPath root pure <| some s!"Beam daemon registry ({path}):\n status: malformed\n detail: {detail}" | .current entry => - let path ← registryPath root let pidStatus ← registryPidStatus entry + let workspaceLines := entry.workspaces.toList.flatMap fun workspace => + ([ + s!" workspace: {workspace.workspaceId}", + s!" root: {workspace.root}", + s!" configHash: {workspace.configHash}" + ] ++ + (optionLine " toolchain" workspace.toolchain?).toList ++ + (optionLine " bundleId" workspace.bundleId?).toList) let lines := ([ s!"Beam daemon registry ({path}):", s!" schemaVersion: {entry.schemaVersion}", @@ -150,26 +169,26 @@ def daemonRegistryContext? (root : System.FilePath) : IO (Option String) := do s!" pid: {entry.pid} ({pidStatus})", s!" endpoint: {registryEndpointSummary entry}", s!" startedAt: {entry.startedAt}", - s!" configHash: {entry.configHash}", - s!" root: {entry.root}" + s!" configHash: {entry.configHash}" ] ++ - (optionLine "toolchain" entry.toolchain?).toList ++ - (optionLine "bundleId" entry.bundleId?).toList ++ + workspaceLines ++ (optionLine "pidDomain" entry.pidDomain?).toList) pure <| some <| String.intercalate "\n" lines catch _ => pure none -def daemonDebugContextJson (root : System.FilePath) : IO Json := do - let registryFile ← registryPath root - let registryRead ← readRegistry root +def daemonDebugContextJson + (root : System.FilePath) + (explicitControlDir? : Option System.FilePath := none) : IO Json := do + let registryFile ← registryPathFor root explicitControlDir? + let registryRead ← readRegistryAt registryFile let registry := registryRead.entry? let registryPidStatus ← match registry with | some entry => some <$> registryPidStatus entry | none => pure none - let startupLogTail ← startupLogTail? root - let incidents ← recentDaemonFailureIncidentJson root + let startupLogTail ← startupLogTail? root explicitControlDir? + let incidents ← recentDaemonFailureIncidentJson root 5 explicitControlDir? pure <| Json.mkObj <| [ ("registryPath", toJson registryFile.toString), diff --git a/Beam/Daemon/Paths.lean b/Beam/Daemon/Paths.lean index 1fd5e23e..dec154f8 100644 --- a/Beam/Daemon/Paths.lean +++ b/Beam/Daemon/Paths.lean @@ -11,21 +11,51 @@ namespace Beam.Daemon private def beamStateDir (root : System.FilePath) : System.FilePath := root / ".beam" -def controlDir (root : System.FilePath) : IO System.FilePath := do - match ← IO.getEnv "BEAM_CONTROL_DIR" with - | some dir => - let tag := toString (hash root.toString) - pure (System.FilePath.mk dir / tag) +/-- Stable FNV-1a tag used only for deterministic `BEAM_CONTROL_ROOT` discovery. -/ +private def controlRootTag (root : System.FilePath) : String := + let hash := root.toString.toUTF8.foldl + (fun acc byte => (acc ^^^ byte.toUInt64) * 1099511628211) + (14695981039346656037 : UInt64) + toString hash.toNat + +def controlDirFor + (root : System.FilePath) + (explicitControlDir? : Option System.FilePath := none) : IO System.FilePath := do + match explicitControlDir? with + | some dir => pure dir | none => - pure (beamStateDir root) + match ← IO.getEnv "BEAM_CONTROL_ROOT" with + | some base => + pure (System.FilePath.mk base / controlRootTag root) + | none => + pure (beamStateDir root) + +/-- Resolve the default or environment-selected control directory for one CLI session. -/ +def controlDir (root : System.FilePath) : IO System.FilePath := + controlDirFor root + +def registryPathFor + (root : System.FilePath) + (explicitControlDir? : Option System.FilePath := none) : IO System.FilePath := do + pure ((← controlDirFor root explicitControlDir?) / "beam-daemon.json") def registryPath (root : System.FilePath) : IO System.FilePath := do - pure ((← controlDir root) / "beam-daemon.json") + registryPathFor root + +def daemonStartupLogPathFor + (root : System.FilePath) + (explicitControlDir? : Option System.FilePath := none) : IO System.FilePath := do + pure ((← controlDirFor root explicitControlDir?) / "beam-daemon-startup.log") + +def daemonStartupLogPath (root : System.FilePath) : IO System.FilePath := + daemonStartupLogPathFor root -def daemonStartupLogPath (root : System.FilePath) : IO System.FilePath := do - pure ((← controlDir root) / "beam-daemon-startup.log") +def daemonFailureIncidentDirFor + (root : System.FilePath) + (explicitControlDir? : Option System.FilePath := none) : IO System.FilePath := do + pure ((← controlDirFor root explicitControlDir?) / "daemon-failures") -def daemonFailureIncidentDir (root : System.FilePath) : IO System.FilePath := do - pure ((← controlDir root) / "daemon-failures") +def daemonFailureIncidentDir (root : System.FilePath) : IO System.FilePath := + daemonFailureIncidentDirFor root end Beam.Daemon diff --git a/Beam/Daemon/Protocol.lean b/Beam/Daemon/Protocol.lean index b60b2ae7..15496dfc 100644 --- a/Beam/Daemon/Protocol.lean +++ b/Beam/Daemon/Protocol.lean @@ -16,7 +16,7 @@ namespace Beam.Daemon open Beam.Broker def registrySchemaVersion : Nat := - 1 + 2 inductive RegistryLifecycle where | live @@ -34,7 +34,26 @@ instance : FromJson RegistryLifecycle where | .str "draining" => .ok .draining | json => .error s!"expected registry lifecycle 'live' or 'draining', got {json.compress}" -structure RegistryEntry where +/-- One statically configured workspace owned by a CLI session. -/ +structure WorkspaceBinding where + workspaceId : WorkspaceId + root : String + configHash : String + leanCmd? : Option String := none + plugin? : Option String := none + rocqCmd? : Option String := none + toolchain? : Option String := none + bundleId? : Option String := none + deriving FromJson, ToJson + +/-- +The private descriptor for one wrapper-owned CLI session. + +The descriptor is deliberately shaped as a session with a nonempty workspace collection even +while the public owner command creates one workspace. This keeps session identity separate from +workspace routing and leaves static multi-workspace ownership as an additive CLI feature. +-/ +structure SessionDescriptor where schemaVersion : Nat lifecycle : RegistryLifecycle daemonId : String @@ -44,25 +63,24 @@ structure RegistryEntry where ownerPid : Nat ownerPidDomain? : Option String := none port? : Option Nat := none - root : String + workspaces : Array WorkspaceBinding + /-- Hash of the complete frozen session configuration. -/ configHash : String - leanCmd? : Option String := none - plugin? : Option String := none - rocqCmd? : Option String := none - toolchain? : Option String := none clientBin? : Option String := none daemonBin? : Option String := none - bundleId? : Option String := none startedAt : String requestedPort? : Option Nat := none deriving FromJson, ToJson -def RegistryEntry.identity (entry : RegistryEntry) : DaemonIdentity := { +def SessionDescriptor.rootSummary (entry : SessionDescriptor) : String := + String.intercalate ", " <| entry.workspaces.toList.map (·.root) + +def SessionDescriptor.identity (entry : SessionDescriptor) : DaemonIdentity := { daemonId := entry.daemonId configHash := entry.configHash } -def RegistryEntry.redactedJson (entry : RegistryEntry) : Json := +def SessionDescriptor.redactedJson (entry : SessionDescriptor) : Json := (toJson entry).setObjVal! "capability" (toJson "") structure DesiredConfig where @@ -80,13 +98,16 @@ structure DesiredConfig where def natToPort? (n : Nat) : Option UInt16 := if n < UInt16.size then some n.toUInt16 else none -def registryEndpoint? (entry : RegistryEntry) : Option Transport.Endpoint := do +def registryEndpoint? (entry : SessionDescriptor) : Option Transport.Endpoint := do (natToPort? =<< entry.port?).map Transport.Endpoint.tcp -def endpointFromEntry (entry : RegistryEntry) : IO Transport.Endpoint := do +def endpointFromEntry (entry : SessionDescriptor) : IO Transport.Endpoint := do match registryEndpoint? entry with | some endpoint => pure endpoint - | none => throw <| IO.userError s!"invalid Beam daemon transport data in registry for {entry.root}" + | none => + let message := + s!"invalid Beam daemon transport data for session {entry.daemonId} ({entry.rootSummary})" + throw (IO.userError message) def endpointSummary (endpoint : Transport.Endpoint) : String := Transport.endpointDescription endpoint diff --git a/Beam/Daemon/Registry.lean b/Beam/Daemon/Registry.lean index 544292a9..3c6eb9d7 100644 --- a/Beam/Daemon/Registry.lean +++ b/Beam/Daemon/Registry.lean @@ -18,9 +18,9 @@ inductive RegistryRead where | legacy | unsupported (schemaVersion : Nat) | malformed (detail : String) - | current (entry : RegistryEntry) + | current (entry : SessionDescriptor) -def RegistryRead.entry? : RegistryRead → Option RegistryEntry +def RegistryRead.entry? : RegistryRead → Option SessionDescriptor | .current entry => some entry | .absent | .legacy | .unsupported _ | .malformed _ => none @@ -37,8 +37,39 @@ def RegistryRead.detail? : RegistryRead → Option String | .malformed detail => some detail | .absent | .current _ => none -def readRegistry (root : System.FilePath) : IO RegistryRead := do - let path ← registryPath root +private def validateWorkspaceBindings (workspaces : Array WorkspaceBinding) : Except String Unit := do + if workspaces.isEmpty then + throw "session descriptor must contain at least one workspace" + let mut ids : Std.TreeSet String compare := {} + let mut roots : Std.TreeSet String compare := {} + for workspace in workspaces do + if workspace.workspaceId.isEmpty then + throw "session workspace id must not be empty" + if workspace.root.isEmpty then + throw s!"session workspace '{workspace.workspaceId}' has an empty root" + let rootPath := System.FilePath.mk workspace.root + unless rootPath.isAbsolute do + throw s!"session workspace '{workspace.workspaceId}' root is not absolute" + if workspace.configHash.isEmpty then + throw s!"session workspace '{workspace.workspaceId}' has an empty configuration hash" + if ids.contains workspace.workspaceId then + throw s!"duplicate session workspace id '{workspace.workspaceId}'" + let normalizedRoot := rootPath.normalize.toString + if roots.contains normalizedRoot then + throw s!"duplicate session workspace root '{workspace.root}'" + ids := ids.insert workspace.workspaceId + roots := roots.insert normalizedRoot + +private def validateSessionDescriptor (entry : SessionDescriptor) : Except String Unit := do + if entry.daemonId.isEmpty then + throw "session descriptor daemonId must not be empty" + if entry.capability.isEmpty then + throw "session descriptor capability must not be empty" + if entry.configHash.isEmpty then + throw "session descriptor configuration hash must not be empty" + validateWorkspaceBindings entry.workspaces + +def readRegistryAt (path : System.FilePath) : IO RegistryRead := do unless ← path.pathExists do return .absent try @@ -57,9 +88,15 @@ def readRegistry (root : System.FilePath) : IO RegistryRead := do unless schemaVersion == registrySchemaVersion do return .unsupported schemaVersion match fromJson? json with - | .ok entry => pure <| .current entry + | .ok entry => + match validateSessionDescriptor entry with + | .ok () => pure <| .current entry + | .error err => pure <| .malformed s!"invalid registry schema: {err}" | .error err => pure <| .malformed s!"invalid registry schema: {err}" catch err => pure <| .malformed s!"could not read registry: {err}" +def readRegistry (root : System.FilePath) : IO RegistryRead := do + readRegistryAt (← registryPath root) + end Beam.Daemon diff --git a/docs/COMPATIBILITY.md b/docs/COMPATIBILITY.md index 0e9c1995..29ff8bb6 100644 --- a/docs/COMPATIBILITY.md +++ b/docs/COMPATIBILITY.md @@ -20,11 +20,12 @@ A Lean release line is the canonical `major.minor` family recorded in - Runtime bundle metadata schema 2 and install manifest schema 3. Install manifest schema 2 is cleanup-only compatibility during the 0.2 release line: identity and `lean-beam prune` may read it, but the installer does not reuse it. Remove the schema-2 decoder when 0.3 development opens. -- Wrapper daemon registry schema 1. The registry is an internal beta coordination boundary: - schema-less and unknown-version records are reported and preserved, but are not decoded, deleted, - or migrated automatically. A schema-less generation may not have a foreground wrapper owner; use - the runtime that wrote its record to stop the corresponding daemon, confirm it is gone, and only - then start a schema-1 owner. +- CLI session descriptor schema 2. It freezes a nonempty array of workspace bindings plus one + generation identity, lifecycle, endpoint, and capability. Schema-less, schema-1, and unknown + records are reported and remain fenced; normal startup does not decode, delete, or migrate them. + After independently stopping the generation that wrote an opaque record, an operator may + quarantine it with `lean-beam --root ROOT recover --force`. Current descriptors instead require + their exact generation ID. Persisted PIDs are never recovery signal capabilities. - MCP `2026-07-28` is the preferred stdio protocol revision. MCP `2025-11-25` remains a named transition target for initialization-based clients. Reconsider the legacy path before the 0.3 release once the clients named by the setup guide can all use per-request metadata. diff --git a/docs/DEVELOPMENT.md b/docs/DEVELOPMENT.md index d2538e59..6d4ef260 100644 --- a/docs/DEVELOPMENT.md +++ b/docs/DEVELOPMENT.md @@ -400,65 +400,68 @@ broker-derived decision. This wrapper path is easy to break accidentally, so keep the mental model simple. -A daemon generation is one concrete daemon start identified by the `daemonId` in -`beam-daemon.json`. Exactly one foreground `lean-beam ensure --hold` process owns that generation. -It starts the daemon in a dedicated process session, passes the daemon identity, effective -configuration hash, and a random per-generation capability through piped stdin, and retains the -pipe's write end. The mode-`0600`, schema-versioned registry publishes `live` or `draining` state. -Every wrapper request, including cancellation, generation probes, and shutdown, must present the -capability. Endpoint attachment also requires the canonical root and exact generation identity to -match. The daemon watches the pipe's read end; EOF atomically closes broker admission, marks admitted -requests for cancellation, shuts down backend sessions, and stops the listener. There is no wrapper -heartbeat, lease file, revocation tombstone, or time-based retirement fence. - -Ordinary wrapper commands never start a daemon. Under the per-project control lock they require a -registry whose root and effective configuration match, whose owner is not known dead in the current -PID domain, and whose endpoint answers for the CLI's private workspace, canonical project root, and -exact daemon generation identity. Identity probes have a bounded response deadline. An endpoint -that accepts a connection but stays silent or returns malformed data is unrecognized, so validation -fails closed. A configuration mismatch reports the old and desired hashes without shutting down or -unpublishing the live owner. Ordinary lookup is observation-only: absent, legacy, malformed, -unsupported, stale, draining, or otherwise unsafe registry states are never rewritten by an -attaching command. -Endpoint/root validation is authoritative across PID namespaces because numeric PID observations -from another domain are not safe process identity. Persisted PIDs are conservative liveness -observations, never signal capabilities. Only the foreground owner may force termination, using its -retained child handle and dedicated process group. - -The owner also watches its exact registry generation and daemon child. `lean-beam shutdown` changes -that exact registry from `live` to `draining` under the project lock before sending the authenticated -shutdown request. Every holder exit path likewise publishes `draining`, closes its pipe, waits for -graceful broker/backend teardown, and, after the deadline, terminates the complete owned process -group. Only after the child has been reaped does it remove the exact draining generation. Thus a -paused or wedged old process tree remains fenced and a replacement owner cannot create split-brain -backend sessions. An unexpected nonzero daemon exit is reported by the holder. Killing the holder -closes the pipe by process lifetime. A paused holder keeps the pipe open, so the session remains -valid without time-based expiry. If the project root disappears, cleanup uses the already resolved -control path without recreating the deleted project. - -This model prevents PID-isolated commands from making contradictory ownership decisions: later -commands may attach to a validated endpoint, but none can silently become a replacement owner. -Starting a new session is always an explicit `lean-beam ensure --hold` action. Wrapper commands read -the private registry and inject its generation capability. A raw `beam-client` request does not gain -authority merely by finding the loopback port; it must explicitly carry that private capability. -The wrapper-owned daemon also rejects dynamic `initWorkspace` and `dropWorkspace`, because its one -bootstrap project is fixed by the owner. A separately launched development daemon has its own -explicit process owner and is not the wrapper security boundary. +A CLI session descriptor is the schema-versioned `beam-daemon.json` selected by the control +directory. It contains one generation identity, capability, lifecycle, endpoint, and a nonempty +array of frozen workspace bindings. The public owner command currently creates one binding; the +shape permits a future explicitly configured static multi-workspace session without changing +request routing. Exactly one foreground `lean-beam ensure --hold` process owns the generation. It +starts the daemon in a dedicated process session, passes the identity, effective configuration +hash, and random capability through piped stdin, and retains the pipe's write end. The mode-`0600` +descriptor publishes `live` or `draining`. Every wrapper request, including cancellation, +generation probes, and shutdown, presents the capability. The daemon watches the pipe's read end; +EOF closes admission, marks admitted requests for cancellation, shuts down backend sessions, and +stops the listener. There is no heartbeat, lease, or time-based retirement fence. + +Ordinary wrapper commands never start a daemon or recompute its desired toolchain/bundle +configuration. Under the session control lock they select the canonical root's frozen workspace +binding and require an endpoint that answers for that workspace, root, and exact generation. +Identity probes have a bounded response deadline. A silent or malformed endpoint fails closed. +Ordinary lookup is observation-only: absent, legacy, malformed, unsupported, draining, unreachable, +or otherwise ambiguous descriptor states are never rewritten by an attaching command. Persisted +PIDs are diagnostic observations, never signal capabilities or automatic stale-reclamation proof. +Only the foreground owner may force termination, through its retained child handle and process +group. + +The owner watches its exact descriptor generation and daemon child. `lean-beam shutdown` changes +that generation from `live` to `draining` under the control lock before sending authenticated +shutdown. Normal holder exit likewise publishes `draining`, closes its pipe, waits for graceful +teardown, and, after the deadline, terminates the owned process group. It removes only the exact +generation after owned cleanup completes. An unexpected daemon or owner exit deliberately leaves +the descriptor fenced: startup does not infer complete process-tree exit from persisted PIDs. Once +the operator establishes that the old session is no longer authoritative, +`lean-beam --root ROOT recover --generation ID` quarantines that exact descriptor without signalling +any recorded process. Opaque legacy, unsupported, or malformed state requires `recover --force`. +A paused holder keeps its pipe open and remains valid without expiry. If the project root +disappears, cleanup uses the already resolved control path without recreating the project. + +Human commands may infer the nearest project root. The supported machine stream requires explicit +`--root` and a nonempty `clientRequestId`; its semantic JSON cannot supply `root`, `workspaceId`, +capability, or dynamic workspace operations. The wrapper selects the descriptor binding and injects +session metadata. Raw port-oriented `beam-client` requests are maintainer/debug tooling. A +wrapper-owned daemon rejects `initWorkspace`, `listWorkspaces`, and `dropWorkspace`; a separately +launched broker retains the generic multi-workspace surface and has its own explicit owner. + +The default control directory is `/.beam`, discoverable to project-scoped agents. +`--control-dir DIR` is an exact, stateless selection that every participant must repeat. +`BEAM_CONTROL_ROOT` hashes each canonical root below a writable base for sandboxed/read-only roots. +Do not hide policy inside automatic fallback between these locations. A future multi-root CLI owner +should require a stable explicit control directory and freeze all bindings before publication. Keep these invariants covered: - only `ensure --hold` may create and publish a wrapper daemon generation - a second owner is rejected while the current endpoint/root/generation identity is live - ordinary wrapper commands are read-only with respect to registry and process lifecycle, including - on configuration mismatch or stale/unsafe state -- holder teardown retains a generation-specific draining fence until the complete process tree is - reaped and cannot remove a replacement + in ambiguous or unsafe state; they attach to frozen owner configuration rather than recomputing it +- normal holder teardown retains a generation-specific draining fence until owned cleanup completes + and cannot remove a replacement; abnormal exit leaves the fence for explicit recovery - owner EOF, explicit shutdown, and project-root disappearance all close admission before backend teardown and complete with bounded child cleanup -- PID-domain checks gate persisted-PID observations; persisted numeric PIDs are never signalled +- persisted numeric PIDs are never signalled or used for automatic stale reclamation - every wrapper request is bound to its random generation capability, and transport frame, initial request, connection, and task counts are bounded -- request IDs and per-admission tokens retain exact disconnect and explicit cancellation semantics +- request IDs are unique and cancellation is exact within a workspace; per-admission tokens retain + exact disconnect and close semantics - the regressions for this path are [tests/test-beam-wrapper-daemon.sh](../tests/test-beam-wrapper-daemon.sh) and [tests/test-beam-wrapper-sandbox.sh](../tests/test-beam-wrapper-sandbox.sh) diff --git a/docs/SETUP.md b/docs/SETUP.md index 3d248e42..46a81989 100644 --- a/docs/SETUP.md +++ b/docs/SETUP.md @@ -224,11 +224,53 @@ lean-beam run-at "Foo.lean" "$version" 10 2 "exact trivial" inherited ownership pipe defines the session lifetime: interrupt the holder, or run `lean-beam shutdown`, to close the daemon and its backend processes. Plain `lean-beam ensure` and all other wrapper commands attach to an existing owner and fail with a recovery command when none -is live. If the desired bundle or project configuration changes, attaching commands preserve the -old owner and ask you to stop it explicitly. During shutdown the registry reports `draining` and a +is live. Attaching commands use the owner's frozen configuration and do not rebuild a competing +desired configuration. A second owner does resolve its proposed configuration and reports any +mismatch while preserving the old owner. During shutdown the registry reports `draining` and a replacement owner is refused until the old process tree has exited. MCP clients do not need a separate holder; the stdio MCP process owns its runtime session. +The default session descriptor and lock live in `/.beam`. This is intentional for +project-scoped agent sandboxes: clients that can access the same workspace can discover the same +session. Use an exact alternate directory when the project is read-only or several explicitly +coordinated clients need another stable control plane: + +```bash +lean-beam --root /workspace/a --control-dir /workspace/control ensure --hold +lean-beam --root /workspace/a --control-dir /workspace/control stats +``` + +Every participant must supply the same `--root` and `--control-dir`; Beam does not search alternate +control directories. `BEAM_CONTROL_ROOT=/writable/base` is the sandbox convenience form: Beam +derives a separate hashed directory for each canonical root below that base. An explicit control +directory is also the intended future location for a statically configured multi-workspace CLI +session. The current public owner command still publishes one frozen workspace, and wrapper mode +does not allow runtime `init_workspace`, `list_workspaces`, or `drop_workspace` requests. +Use a stable external control directory when ownership must remain fenced while the project path is +deleted and recreated; deleting a project-local `.beam` necessarily deletes its default fence. + +An abnormal owner or broker exit leaves the descriptor as a safety fence. After independently +establishing that the recorded generation is no longer authoritative, quarantine that exact record +without signalling its recorded PIDs: + +```bash +lean-beam --root /workspace/a recover --generation GENERATION_ID +``` + +Use the same `--control-dir` selection when applicable. `recover --force` is reserved for opaque +legacy, unsupported, or malformed descriptors. Recovery preserves the old file under a +`beam-daemon.recovered-*.json` name for diagnosis. + +Machine clients should avoid root auto-detection and raw port/session fields: + +```bash +lean-beam --root /workspace/a request-stream \ + '{"op":"stats","clientRequestId":"agent-stats-1"}' +``` + +The wrapper selects the frozen workspace and injects root, workspace identity, generation +capability, and endpoint. `beam-client --port ...` is lower-level maintainer/debug tooling. + The `python3` line extracts `result.version` for shell examples. You can also copy that version number from the printed `lean-beam update` JSON. diff --git a/docs/STATUS.md b/docs/STATUS.md index 72cb41fd..61ee5509 100644 --- a/docs/STATUS.md +++ b/docs/STATUS.md @@ -131,18 +131,22 @@ saved accepted text, the intended future direction is for `lean-beam update` or reuse matching speculative execution rather than replaying it from scratch. Beam would still not apply the source edit. -For programmatic local consumers, the preferred machine-readable surface is the JSON stream exposed -by `beam-client request-stream`; wrapper stderr should be treated as human-facing. A wrapper-managed -daemon exists only while its foreground `lean-beam ensure --hold` owner is alive. Keep that owner -active for wrapper and raw-client requests; those requests attach to the session but do not acquire -daemon ownership. A separately launched standalone daemon has its own explicit process owner. Broker +For programmatic local consumers of a wrapper session, the supported machine-readable surface is +`lean-beam --root ROOT [--control-dir DIR] request-stream `. The request JSON contains the +operation, its arguments, and a nonempty `clientRequestId`; it cannot select a workspace, root, or +capability. The wrapper canonicalizes the explicit root, selects its static workspace binding from +the session descriptor, and injects routing and authentication. Wrapper stderr is human-facing. +The port-oriented `beam-client request-stream` remains maintainer/debug tooling for separately +managed brokers. A wrapper daemon exists only while its foreground `lean-beam ensure --hold` owner +is alive. Attaching requests do not acquire daemon ownership. A separately launched standalone +daemon has its own explicit process owner. Broker responses require an explicit top-level `ok` boolean, giving projection layers an unambiguous success/error discriminator. A successful response always includes `result`; response and stream envelopes reject undeclared fields, and typed save/close-save results reject incomplete or extended artifact shapes. All raw stream variants use the same `kind`, `payload`, and optional outer `clientRequestId` fields; the terminal response payload does not duplicate transport correlation. Exact event ordering and examples live in -[SYNC_AND_DIAGNOSTICS.md](SYNC_AND_DIAGNOSTICS.md#raw-broker-stream). +[SYNC_AND_DIAGNOSTICS.md](SYNC_AND_DIAGNOSTICS.md#machine-broker-stream). `lean-beam-mcp` is the experimental stdio MCP entry point. User setup lives in [SETUP.md](SETUP.md#mcp-setup); implementation, protocol, tool-list, and conformance notes live in @@ -192,13 +196,26 @@ Exact event ordering and examples live in endpoint, root, and generation-identity validation are authoritative when PID identity is not locally observable. Each wrapper request carries a random per-generation capability from the mode-`0600` registry. A paused owner retains the session; a killed owner closes the pipe; explicit - `lean-beam shutdown` changes the registry to `draining`, and that fence remains until the holder - has reaped the daemon process tree. Configuration-mismatched and otherwise unsafe ordinary - lookups preserve the current owner and registry. -- After abrupt owner death, a later process in the same PID domain may recognize a registry as stale - only when both recorded processes are proven gone. A client in another PID domain cannot make - that proof and fails closed with the registry preserved; its external sandbox/process supervisor - must establish complete process-tree exit before removing that exact recovery record. + `lean-beam shutdown` changes the descriptor to `draining`, and that fence remains until normal + owner cleanup has reaped the daemon leader after graceful or process-group teardown. + Ordinary lookups use the frozen workspace configuration and preserve unsafe session state. A + competing owner computes its proposed configuration but cannot replace a mismatched live owner. +- Abnormal or ambiguous state is never reclaimed automatically. The descriptor remains as a fence + after an unexpected broker/owner exit. Once the operator has established that the matching + session is no longer authoritative, `lean-beam --root ROOT recover --generation ID` quarantines + that exact descriptor without signalling persisted PIDs. Legacy, unsupported, or malformed + descriptor state requires the deliberately broader `recover --force` form. +- The default authoritative descriptor is `/.beam/beam-daemon.json`, which keeps discovery + available inside project-scoped agent sandboxes. `--control-dir DIR` selects one exact alternate + control directory; callers must repeat the same selection for every owner, request, diagnostic, + shutdown, and recovery command. `BEAM_CONTROL_ROOT` is the sandbox convenience that derives a + per-root subdirectory below a writable base. A stable explicit control directory is also the + intended future boundary for a statically configured multi-workspace CLI session; dynamic + workspace mutation remains unavailable in wrapper mode. +- Deleting the project tree also deletes its default project-local fence. Workflows that may remove + and immediately recreate the same canonical path, and need exclusion to survive that operation, + should select a stable external `--control-dir`; this tradeoff keeps the default usable from + project-scoped agent sandboxes without assuming access to a host-global runtime directory. - Wrapper-owned brokers currently use authenticated loopback TCP. The supported trust boundary is one local OS account with private registry-file permissions; another user who can only discover the port cannot issue requests without the generation capability. A manually launched standalone @@ -207,8 +224,7 @@ Exact event ordering and examples live in - A startup failure that reports `operation not permitted` through `.beam/beam-daemon-startup.log` is usually an environment restriction, not a bundle-resolution mismatch. - Typed broker transport, invalid-response, and response-timeout failures include registry/log - context and write a JSON incident record under `.beam/daemon-failures/` or the per-root - subdirectory of `BEAM_CONTROL_DIR`. Incident kinds are `brokerTransportFailure`, + context and write a JSON incident record below the selected control directory. Incident kinds are `brokerTransportFailure`, `invalidBrokerResponse`, and `brokerResponseTimeout`; callback/display failures do not create daemon incidents. Beam keeps the latest 50 incident records, and `lean-beam doctor` lists recent incident paths. diff --git a/docs/SYNC_AND_DIAGNOSTICS.md b/docs/SYNC_AND_DIAGNOSTICS.md index 3110f351..b0b9308d 100644 --- a/docs/SYNC_AND_DIAGNOSTICS.md +++ b/docs/SYNC_AND_DIAGNOSTICS.md @@ -18,7 +18,7 @@ document version and return `changed: false`. `lean-beam sync` is the diagnostics/readiness barrier for a Lean file. It opens or updates the tracked file, waits for diagnostics for the current document version, streams fresh request diagnostics, and returns a machine-readable JSON verdict for that version. Wrapper stdout uses -stable, agent-oriented field ordering; `beam-client request-stream` is the compact one-line JSON +stable, agent-oriented field ordering; `lean-beam --root ROOT request-stream` is the compact one-line JSON stream for programmatic event consumers. The returned document `version` is the snapshot token for broker, MCP, and wrapper callers. @@ -117,23 +117,24 @@ Their transport types differ by surface. | Streamed diagnostics | Lean-published events observed while a request is pending. | MCP `notifications/message` with logger `lean.diagnostic`; Beam stream `diagnostic` events; CLI stderr diagnostics. | | Current result | Stable synced-state verdict for one document version. | Final broker/CLI `diagnostics`, `readiness`, and `fileProgress` fields; MCP spells the progress field `document_progress`. | -Wrapper stderr is the human-facing surface. Machine consumers should use final stdout JSON or the -broker JSON stream exposed by `beam-client request-stream`. +Wrapper stderr is the human-facing surface. Machine consumers of an owned wrapper session should +use final stdout JSON or `lean-beam --root ROOT [--control-dir DIR] request-stream `. -### Raw Broker Stream +### Machine Broker Stream -`beam-client request-stream` prints one compact JSON object per line, in the order the broker -observed it. A request may produce any number of `fileProgress` and `diagnostic` messages, followed -by exactly one terminal `response`; the response is last and no later message belongs to that -request. +`lean-beam --root ROOT request-stream` prints one compact JSON object per line, in the order the +broker observed it. Its input is a semantic project request with a required nonempty +`clientRequestId`; callers cannot supply `root`, `workspaceId`, `daemonCapability`, executable +configuration, or workspace administration operations. A request may produce any number of +`fileProgress` and `diagnostic` messages, followed by exactly one terminal `response`; the response +is last and no later message belongs to that request. -When `beam-client` targets the per-project daemon managed by `lean-beam`, keep the session's -`lean-beam ensure --hold` owner active for the request lifetime. Only the holder starts the daemon; -wrapper commands attach with the private per-generation capability read from the mode-`0600` -registry. A raw broker request must explicitly include that `daemonCapability`; discovering the -loopback port alone does not authorize it. Raw requests participate in typed request admission and -cancellation but do not own the daemon process. A separately launched standalone development daemon -has its own explicit process owner and does not use the wrapper holder. +Keep the session's `lean-beam ensure --hold` owner active for the request lifetime. Only the holder +starts the daemon; the root-aware machine client reads the mode-`0600` descriptor, selects its +frozen workspace binding, and injects routing and the per-generation capability. Requests +participate in typed request admission and workspace-scoped cancellation but do not own the daemon. +The raw `beam-client --port ... request-stream` surface requires complete internal request fields +and is maintainer/debug tooling for separately managed brokers. Every stream variant uses the same `kind`, `payload`, and optional correlation envelope. When the request supplies `clientRequestId`, each message repeats it on that outer stream envelope: diff --git a/docs/TESTING.md b/docs/TESTING.md index 369dbc76..2177d2a9 100644 --- a/docs/TESTING.md +++ b/docs/TESTING.md @@ -124,12 +124,13 @@ Current Beam coverage includes: a daemon is paused, rejection of attachment or replacement while that generation remains published, forced process-group cleanup of the daemon and its backend, holder reporting after an unexpected daemon crash, abrupt owner death through inherited-pipe EOF, - read-only stale registry lookup, and self-termination after the project worktree disappears without - recreating it + read-only crash-fence lookup, exact-generation non-signalling recovery, explicit control-directory + selection, root-aware machine request routing, and self-termination after the project worktree + disappears without recreating it - Linux-only PID-isolated sandbox wrapper coverage in [tests/test-beam-wrapper-sandbox.sh](../tests/test-beam-wrapper-sandbox.sh), including cross-namespace endpoint attachment, duplicate-owner rejection, a paused owner without time-based expiry, explicit shutdown, killed-owner EOF cleanup, fail-closed preservation of an - unavailable foreign-domain registry before supervised recovery, distinct generation identity, + unavailable foreign-domain descriptor before explicit recovery, distinct generation identity, and the absence of legacy lease/retirement artifacts - zero-build save replay, structured-setup support, batch-only-argument rejection, and stale-save race coverage in diff --git a/scripts/lean-beam b/scripts/lean-beam index e3a3aee3..fdc5cd34 100755 --- a/scripts/lean-beam +++ b/scripts/lean-beam @@ -43,9 +43,12 @@ usage: lean-beam [--root PATH] stats lean-beam [--root PATH] reset-stats lean-beam [--root PATH] shutdown + lean-beam [--root PATH] [--control-dir DIR] recover --generation ID | --force + lean-beam --root PATH [--control-dir DIR] request-stream lean-beam [--root PATH] cancel notes: + - pass --control-dir DIR before the command to select one exact alternate session control directory; repeat it for every participant - lean-beam keeps the Lean wrapper surface primary; optional Rocq goal probes are documented in docs/ROCQ.md - start a Rocq session with `lean-beam ensure rocq --hold`; plain `ensure rocq` only checks and warms that owned session - other common Rocq entry points are `lean-beam doctor rocq`, `lean-beam rocq-goals-after`, and `lean-beam rocq-goals-prev` @@ -55,6 +58,7 @@ notes: - for handle-based commands, use `--handle-file ` when you do not want to inline handle json - wrapper commands require one live session owner; start `ensure --hold` in a foreground process and keep it running - plain `ensure` checks and warms that owned session but never starts one implicitly + - abnormal session state remains fenced; after verifying the old generation is gone, use `recover --generation ID` - set `BEAM_DEBUG_TEXT=1` to print the exact escaped text and UTF-8 bytes sent for text-carrying Lean probes - use `lean-beam --version` for bug reports and installed runtime identity checks - validated-toolchains lists exact CI-validated versions; compatible-release-lines lists canonical RC/patch families qualified locally @@ -130,7 +134,7 @@ prefix=() cmd="" while [ "$#" -gt 0 ]; do case "$1" in - --root|--port) + --root|--control-dir|--port) [ "$#" -ge 2 ] || break prefix+=("$1" "$2") shift 2 diff --git a/skills/lean-beam/SKILL.md b/skills/lean-beam/SKILL.md index 76ef4ede..ccd33d98 100644 --- a/skills/lean-beam/SKILL.md +++ b/skills/lean-beam/SKILL.md @@ -126,11 +126,12 @@ Core workflow contract: to `leanOptions`, or use `lake build` when the arguments are intentionally batch-only - after changing a lakefile or related Lake workspace configuration, run `lean-beam shutdown` before the next command that uses the Lean server; `lean-beam refresh` does not restart it -- treat wrapper `stderr` as human-facing only; use stdout JSON or `beam-client request-stream` +- treat wrapper `stderr` as human-facing only; use stdout JSON or + `lean-beam --root ROOT request-stream` for machine-readable automation -- when `beam-client` targets a wrapper-managed daemon, keep that session's - `lean-beam ensure --hold` owner active for the raw request lifetime; raw broker requests attach to - the session but do not own it, while a separately launched standalone daemon has its own owner +- for a machine-readable wrapper stream, keep `lean-beam ensure --hold` active and use + `lean-beam --root ROOT request-stream`; raw `beam-client --port` requests are maintainer tooling + for separately managed brokers - `lean-beam feedback-report` and `beam_feedback_report` return a report to the caller; Beam does not upload or submit it; before posting non-confidential output, review caller-authored narrative, request/response payloads, local paths, Beam stats, open-file data, daemon logs/incidents, and @@ -293,7 +294,10 @@ Use `lean-beam`, not raw JSON and not raw LSP. - infers the target project root from the current directory or `--root` - keeps one Beam daemon per project root and records it in `/.beam/beam-daemon.json` - - in sandboxed or read-only project trees, set `BEAM_CONTROL_DIR` to a writable directory; `lean-beam` uses a per-root subdirectory there + - in sandboxed or read-only project trees, set `BEAM_CONTROL_ROOT` to a writable directory; + `lean-beam` uses a per-root subdirectory there + - for an exact stable alternate session location, pass the same `--control-dir DIR` to the owner + and every attaching command; Beam does not search alternate control directories - resolves a toolchain-keyed Lean bundle, preferring the installed beam bundle cache and falling back to a project-local runtime bundle under `/.beam/bundles` or `BEAM_BUNDLE_DIR` - fully validates exact Lean toolchains listed in `validated-lean-toolchains`, locally qualifies @@ -310,6 +314,8 @@ Use `lean-beam`, not raw JSON and not raw LSP. `lean-beam compatible-release-lines`, and `lean-beam doctor` to inspect the decision - after effective Lean startup configuration changes, requires shutting down the old session and starting a new `lean-beam ensure --hold` owner +- after abnormal owner/broker exit, preserves the session fence until explicit + `recover --generation ID`; recovery quarantines metadata and never signals its persisted PIDs - `lean-beam shutdown`, `lean-beam stats`, and `lean-beam reset-stats` apply to the current project only - `lean-beam prune` previews old installed runtimes; restart active agents and MCP clients before any `--apply`, and add `--bundles` when stale installed bundle caches should also be removed @@ -443,7 +449,7 @@ Surface rule: - wrapper `stderr` is the human-facing diagnostic surface - wrapper `stderr` may distinguish request-level failures from a completed request whose payload failed inside Lean; use stdout JSON for machine decisions -- `beam-client request-stream ...` is the machine-facing streamed surface +- `lean-beam --root ROOT request-stream ...` is the supported machine-facing wrapper stream - do not parse wrapper `stderr` in tooling - MCP clients can attach `tools/call` `_meta.progressToken` for detailed live updates; without one, Beam keeps fast broker-backed Lean operations, feedback collection, and workspace drops quiet and @@ -468,7 +474,7 @@ Use this when you are deciding between commands: - human after a real saved edit: `lean-beam sync` - human checkpointing one synced module: `lean-beam save` or `lean-beam close-save` - human diagnosing daemon or save-state trouble: `lean-beam open-files` and `lean-beam doctor` -- tooling that wants streamed diagnostics or progress: `beam-client request-stream ...` +- tooling that wants streamed diagnostics or progress: `lean-beam --root ROOT request-stream ...` ## References diff --git a/skills/lean-beam/references/anti-patterns.md b/skills/lean-beam/references/anti-patterns.md index 627273dc..dbbfcda5 100644 --- a/skills/lean-beam/references/anti-patterns.md +++ b/skills/lean-beam/references/anti-patterns.md @@ -29,7 +29,7 @@ Use this reference as a short checklist of what not to assume in Lean `beam` wor - use `lean-beam run-at-handle` plus `lean-beam run-with` / `lean-beam run-with-linear` for exact speculative chaining - use a real edit, save, then `lean-beam sync` when the speculative result should become source - use `lean-beam save` only for a synced workspace module -- use `beam-client request-stream` for machine-readable streaming diagnostics or progress +- use `lean-beam --root ROOT request-stream` for machine-readable streaming diagnostics or progress - use `lake build` when the task has become dependency freshness - rely on successful Beam checkpoints for ordinary local development, ensure CI runs `lake build` from clean artifacts, and use local `lean-beam shutdown`, `lake clean`, and `lake build` once only diff --git a/skills/lean-beam/references/workflow-details.md b/skills/lean-beam/references/workflow-details.md index 17c208f7..7d6cf42c 100644 --- a/skills/lean-beam/references/workflow-details.md +++ b/skills/lean-beam/references/workflow-details.md @@ -188,7 +188,7 @@ What is not a valid checkpoint target: `error.message` includes a compact preview of underlying diagnostics and/or command messages, and `error.data.sync` contains the blocking sync verdict - wrapper `stderr` is the human-facing diagnostic surface -- `beam-client request-stream ...` is the machine-facing streamed surface +- `lean-beam --root ROOT request-stream ...` is the supported machine-facing wrapper stream - streamed diagnostics are request-scoped observations; they may carry `completionBlocking=true`, but save-blocking evidence is attached to the final sync/save verdict - the field-level progress, diagnostic, and readiness contract lives in diff --git a/skills/rocq-beam/SKILL.md b/skills/rocq-beam/SKILL.md index 22bab765..f570b326 100644 --- a/skills/rocq-beam/SKILL.md +++ b/skills/rocq-beam/SKILL.md @@ -87,10 +87,14 @@ Use `lean-beam`, not raw JSON and not raw LSP. - infers the target project root from the current directory or `--root` - keeps one Beam daemon per project root and records it in `/.beam/beam-daemon.json` - - in sandboxed or read-only project trees, set `BEAM_CONTROL_DIR` to a writable directory + - in sandboxed or read-only project trees, set `BEAM_CONTROL_ROOT` to a writable directory + - for an exact stable alternate location, pass the same `--control-dir DIR` to the owner and every + attaching command; Beam does not search alternate control directories - gives daemon startup authority only to `lean-beam ensure rocq --hold`; ordinary commands attach to its registry generation and never start a daemon implicitly - owns shutdown and registry handling +- preserves ambiguous crash state until explicit `recover --generation ID`; recovery does not + signal persisted PIDs - resolves `coq-lsp` from the target project's local `_opam` when available - the explicit owner starts a Rocq-capable Beam daemon with startup args instead of relying on inherited editor state diff --git a/tests/lean/BeamTest/Broker/CliDaemonTest.lean b/tests/lean/BeamTest/Broker/CliDaemonTest.lean index 36b75094..824daa1b 100644 --- a/tests/lean/BeamTest/Broker/CliDaemonTest.lean +++ b/tests/lean/BeamTest/Broker/CliDaemonTest.lean @@ -43,9 +43,9 @@ private def checkDaemonDebugWarnings : IO Unit := do let warnings := Beam.Daemon.daemonDebugWarnings debug require "dead registry pid should produce a feedback warning" (warnings.any (fun warning => warning.contains "registry pid is not alive")) - require "dead registry pid warning should include recovery hint" + require "dead registry pid warning should reject PID-based reclamation" (warnings.any (fun warning => - warning.contains "lean-beam shutdown" && warning.contains "lean-beam ensure --hold")) + warning.contains "diagnostic only" && warning.contains "do not reclaim")) private def expectIoErrorMessage (label : String) (act : IO α) : IO String := do let result ← @@ -294,6 +294,17 @@ private def checkProjectDaemonWorkspaceRouting : IO Unit := do } : Beam.Broker.Request) require "CLI routing should preserve an explicitly selected workspace" (explicitReq.workspaceId? == some "maintenance-fixture") + let selectedClient : Beam.Cli.ProjectDaemonClient := { + endpoint := .tcp 42424 + capability := "test-capability" + workspaceId := "selected-workspace" + } + let selectedCancel := Beam.Cli.inSelectedDaemonWorkspace selectedClient { + op := .cancel + cancelRequestId? := some "request" + } + require "selected descriptor workspace should scope cancellation" + (selectedCancel.workspaceId? == some "selected-workspace") private def checkClientResponsePresentation : IO Unit := do let semantic := Beam.Broker.Response.success Json.null @@ -425,6 +436,22 @@ private def checkCliRootParsing : IO Unit := do "missing explicit CLI root should use the workspace error boundary" "workspace root does not resolve" (Beam.Cli.parseCliOptions {} ["--root", missingRoot.toString, "ensure", "lean"]) + let root := System.FilePath.mk s!"/tmp/beam-cli-root-{← IO.monoNanosNow}" + let control := root / "shared-control" + try + IO.FS.createDirAll control + let opts ← Beam.Cli.parseCliOptions {} [ + "--root", root.toString, + "--control-dir", control.toString, + "stats" + ] + require "explicit CLI root should be canonicalized" (opts.explicitRoot? == some root) + require "explicit control directory should remain an exact selection" + (opts.explicitControlDir? == some control) + require "global selectors should not leak into command arguments" (opts.args == ["stats"]) + finally + if ← root.pathExists then + IO.FS.removeDirAll root private def checkLeanOperationRequests : IO Unit := do let root := System.FilePath.mk "/repo" @@ -587,7 +614,7 @@ private def checkDaemonFailureContext : IO Unit := do if let some parent := registryPath.parent then IO.FS.createDirAll parent let pidDomain? ← Beam.currentPidDomain? - let entry : Beam.Daemon.RegistryEntry := { + let entry : Beam.Daemon.SessionDescriptor := { schemaVersion := Beam.Daemon.registrySchemaVersion lifecycle := .live daemonId := "daemon-test" @@ -597,10 +624,14 @@ private def checkDaemonFailureContext : IO Unit := do ownerPid := 999999999 ownerPidDomain? := pidDomain? port? := some 42424 - root := root.toString + workspaces := #[{ + workspaceId := Beam.Cli.projectDaemonWorkspaceId + root := root.toString + configHash := "config-test" + toolchain? := some "leanprover/lean4:test" + bundleId? := some "bundle-test" + }] configHash := "config-test" - toolchain? := some "leanprover/lean4:test" - bundleId? := some "bundle-test" startedAt := "2026-07-02T00:00:00Z" } IO.FS.writeFile registryPath ((toJson entry).pretty ++ "\n") @@ -630,7 +661,7 @@ private def checkDaemonFailureContext : IO Unit := do requireJsonString "daemon failure incident should include registry path" "registryPath" registryPath.toString incidentJson let incidentRegistryJson ← IO.ofExcept <| incidentJson.getObjVal? "registry" - let incidentRegistry ← IO.ofExcept <| fromJson? (α := Beam.Daemon.RegistryEntry) incidentRegistryJson + let incidentRegistry ← IO.ofExcept <| fromJson? (α := Beam.Daemon.SessionDescriptor) incidentRegistryJson require "daemon failure incident should include daemon id" (incidentRegistry.daemonId == "daemon-test") require "daemon failure incident must redact the per-generation capability" @@ -730,7 +761,7 @@ private def writeTestRegistryEntry let registryPath ← Beam.Daemon.registryPath root if let some parent := registryPath.parent then IO.FS.createDirAll parent - let entry : Beam.Daemon.RegistryEntry := { + let entry : Beam.Daemon.SessionDescriptor := { schemaVersion := Beam.Daemon.registrySchemaVersion lifecycle := .live daemonId := "daemon-test" @@ -738,10 +769,14 @@ private def writeTestRegistryEntry pid := 999999999 ownerPid := 999999999 port? - root := root.toString + workspaces := #[{ + workspaceId := Beam.Cli.projectDaemonWorkspaceId + root := root.toString + configHash := "config-test" + toolchain? := some "leanprover/lean4:test" + bundleId? := some "bundle-test" + }] configHash := "config-test" - toolchain? := some "leanprover/lean4:test" - bundleId? := some "bundle-test" startedAt := "2026-07-05T00:00:00Z" } IO.FS.writeFile registryPath ((toJson entry).pretty ++ "\n") @@ -781,7 +816,8 @@ private def checkTypedRegistryReads : IO Unit := do require "malformed registry should preserve parse context" (detail.contains "invalid registry JSON") | state => throw <| IO.userError s!"malformed registry was classified as {state.status}" - IO.FS.writeFile registryPath "{\"schemaVersion\":1}\n" + IO.FS.writeFile registryPath + ("{\"schemaVersion\":" ++ toString Beam.Daemon.registrySchemaVersion ++ "}\n") match ← Beam.Daemon.readRegistry root with | .malformed detail => require "incomplete current registry should preserve schema context" @@ -794,6 +830,18 @@ private def checkTypedRegistryReads : IO Unit := do require "current registry should preserve its generation capability" (entry.capability == "test-capability") | state => throw <| IO.userError s!"current registry was classified as {state.status}" + + let validText ← IO.FS.readFile registryPath + let validJson ← IO.ofExcept <| Json.parse validText + IO.FS.writeFile registryPath <| + (validJson.setObjVal! "workspaces" (toJson (#[] : Array Beam.Daemon.WorkspaceBinding))).compress + match ← Beam.Daemon.readRegistry root with + | .malformed detail => + require "empty workspace descriptors should fail the typed boundary" + (detail.contains "at least one workspace") + | state => throw <| IO.userError s!"empty workspace descriptor was classified as {state.status}" + + IO.FS.writeFile registryPath validText let debug ← Beam.Daemon.daemonDebugContextJson root let debugRegistry ← IO.ofExcept <| debug.getObjVal? "registry" requireJsonString "daemon debug context must redact its capability" diff --git a/tests/lean/BeamTest/Broker/PendingTest.lean b/tests/lean/BeamTest/Broker/PendingTest.lean index f62813ed..a098855e 100644 --- a/tests/lean/BeamTest/Broker/PendingTest.lean +++ b/tests/lean/BeamTest/Broker/PendingTest.lean @@ -58,7 +58,7 @@ private def expectRegistered private def checkActiveRegistry : IO Unit := do let registry ← ActiveRequestRegistry.create - let noneResult ← ActiveRequestRegistry.register registry none + let noneResult ← ActiveRequestRegistry.register registry none none let anonymous ← expectRegistered "register without clientRequestId" noneResult require "anonymous admission participates in active request count" ((← ActiveRequestRegistry.count registry) == 1) @@ -75,9 +75,9 @@ private def checkActiveRegistry : IO Unit := do require "unregistered anonymous admission leaves no active request" ((← ActiveRequestRegistry.count registry) == 0) - let firstResult ← ActiveRequestRegistry.register registry (some "req-1") + let firstResult ← ActiveRequestRegistry.register registry none (some "req-1") let first ← expectRegistered "register active request" firstResult - match ← ActiveRequestRegistry.register registry (some "req-1") with + match ← ActiveRequestRegistry.register registry none (some "req-1") with | .ok _ => throw <| IO.userError "duplicate clientRequestId registered successfully" | .error failure => @@ -85,7 +85,7 @@ private def checkActiveRegistry : IO Unit := do require "duplicate active request error names id" (failure.message.contains "req-1") require "mark active request cancelled" - (Option.isSome (← ActiveRequestRegistry.markCancelled registry "req-1")) + (Option.isSome (← ActiveRequestRegistry.markCancelled registry none "req-1")) match ← ensureRequestNotCancelled (some (ActiveRequest.cancelRef first)) with | .ok _ => throw <| IO.userError "ensureRequestNotCancelled reports broker cancellation: expected error" @@ -97,9 +97,9 @@ private def checkActiveRegistry : IO Unit := do ActiveRequestRegistry.unregister registry (some first) require "unregistered active request is no longer cancellable" - (Option.isNone (← ActiveRequestRegistry.markCancelled registry "req-1")) + (Option.isNone (← ActiveRequestRegistry.markCancelled registry none "req-1")) - let replacementResult ← ActiveRequestRegistry.register registry (some "req-1") + let replacementResult ← ActiveRequestRegistry.register registry none (some "req-1") let replacement ← expectRegistered "register replacement active request" replacementResult ActiveRequestRegistry.unregister registry (some first) require "stale active handle cannot cancel replacement" @@ -110,7 +110,7 @@ private def checkActiveRegistry : IO Unit := do throw <| IO.userError s!"stale active handle cancelled replacement: {(toJson failure.toResponse).compress}" require "stale unregister preserves replacement active request" - (Option.isSome (← ActiveRequestRegistry.markCancelled registry "req-1")) + (Option.isSome (← ActiveRequestRegistry.markCancelled registry none "req-1")) match ← ensureRequestNotCancelled (some replacement.cancelRef) with | .ok _ => throw <| IO.userError "replacement active request did not observe cancellation" @@ -121,12 +121,30 @@ private def checkActiveRegistry : IO Unit := do failure ActiveRequestRegistry.unregister registry (some replacement) + let workspaceA ← expectRegistered "register workspace A request" <| + ← ActiveRequestRegistry.register registry (some "workspace-a") (some "shared-id") + let workspaceB ← expectRegistered "register workspace B request" <| + ← ActiveRequestRegistry.register registry (some "workspace-b") (some "shared-id") + require "workspace-scoped cancellation finds the selected request" + (Option.isSome (← ActiveRequestRegistry.markCancelled registry + (some "workspace-a") "shared-id")) + match ← ensureRequestNotCancelled (some workspaceA.cancelRef) with + | .ok _ => throw <| IO.userError "workspace A request did not observe cancellation" + | .error _ => pure () + match ← ensureRequestNotCancelled (some workspaceB.cancelRef) with + | .ok _ => pure () + | .error _ => throw <| IO.userError "workspace A cancellation leaked into workspace B" + require "unscoped cancellation does not alias a workspace-scoped request" + (Option.isNone (← ActiveRequestRegistry.markCancelled registry none "shared-id")) + ActiveRequestRegistry.unregister registry (some workspaceA) + ActiveRequestRegistry.unregister registry (some workspaceB) + private def checkActiveRegistryCloseDrain : IO Unit := do let registry ← ActiveRequestRegistry.create let named ← expectRegistered "register named request before close" <| - ← ActiveRequestRegistry.register registry (some "closing-request") + ← ActiveRequestRegistry.register registry none (some "closing-request") let anonymous ← expectRegistered "register anonymous request before close" <| - ← ActiveRequestRegistry.register registry none + ← ActiveRequestRegistry.register registry none none ActiveRequestRegistry.closeAdmission registry for active in #[named, anonymous] do match ← ensureRequestNotCancelled (some active.cancelRef) with @@ -136,7 +154,7 @@ private def checkActiveRegistryCloseDrain : IO Unit := do "admission close cancellation" "requestCancelled" failure - match ← ActiveRequestRegistry.register registry (some "after-close") with + match ← ActiveRequestRegistry.register registry none (some "after-close") with | .ok _ => throw <| IO.userError "closed admission accepted a new request" | .error failure => require "closed admission rejection is typed" (failure.code == .requestCancelled) @@ -155,10 +173,10 @@ private def checkActiveRegistryCloseDrain : IO Unit := do private def checkPendingCancellationIdentity : IO Unit := do let registry ← ActiveRequestRegistry.create - let firstResult ← ActiveRequestRegistry.register registry (some "reused-id") + let firstResult ← ActiveRequestRegistry.register registry none (some "reused-id") let first ← expectRegistered "register first cancellation identity" firstResult ActiveRequestRegistry.unregister registry (some first) - let replacementResult ← ActiveRequestRegistry.register registry (some "reused-id") + let replacementResult ← ActiveRequestRegistry.register registry none (some "reused-id") let replacement ← expectRegistered "register replacement cancellation identity" replacementResult let firstPending ← mkPending (cancelRef? := some first.cancelRef) let replacementPending ← mkPending (cancelRef? := some replacement.cancelRef) diff --git a/tests/lean/BeamTest/Broker/ProtocolTest.lean b/tests/lean/BeamTest/Broker/ProtocolTest.lean index 0f24a35f..015a6531 100644 --- a/tests/lean/BeamTest/Broker/ProtocolTest.lean +++ b/tests/lean/BeamTest/Broker/ProtocolTest.lean @@ -648,7 +648,7 @@ private def checkRequestArgsBoundary : IO Unit := do codeActionResolveRocqUnsupported.codeActionResolveArgs private def checkWorkspaceRoutingFields : IO Unit := do - let processWideOps := #[Op.cancel, .listWorkspaces, .resetStats, .shutdown] + let processWideOps := #[Op.listWorkspaces, .resetStats, .shutdown] let optionallyScopedOps := #[Op.openDocs, .stats] for op in Op.all do @@ -779,6 +779,54 @@ private def checkWorkspaceRoutingFields : IO Unit := do require "unsupported workspace mode error should name accepted values" (err.contains "'set', 'verify', or 'reset'") +private def checkProjectRequestBoundary : IO Unit := do + let semanticJson := Json.mkObj [ + ("op", toJson Op.runAt), + ("backend", toJson Backend.lean), + ("clientRequestId", toJson "project-request"), + ("path", toJson "Demo.lean"), + ("version", toJson (1 : Nat)), + ("line", toJson (0 : Nat)), + ("character", toJson (0 : Nat)), + ("text", toJson "exact rfl") + ] + let projectRequest ← expectOk "semantic project request" <| + fromJson? (α := ProjectRequest) semanticJson + let attached := projectRequest.attach "workspace-a" "/workspace/a" "session-capability" + require "project request attachment injects the selected workspace" + (attached.workspaceId? == some "workspace-a") + require "project request attachment injects the owner-side root" + (attached.root? == some "/workspace/a") + require "project request attachment injects session authority" + (attached.daemonCapability? == some "session-capability") + match fromJson? (α := ProjectRequest) <| Json.mkObj [("op", toJson Op.stats)] with + | .ok _ => throw <| IO.userError "project request unexpectedly accepted a missing request id" + | .error err => + require "project request id rejection should explain the machine identity requirement" + (err.contains "non-empty clientRequestId") + let cancelRequest ← expectOk "semantic cancellation request" <| + fromJson? (α := ProjectRequest) <| Json.mkObj [ + ("op", toJson Op.cancel), + ("clientRequestId", toJson "cancel-command"), + ("cancelRequestId", toJson "project-request") + ] + let attachedCancel := cancelRequest.attach "workspace-a" "/workspace/a" "session-capability" + require "project cancellation is workspace-scoped" + (attachedCancel.workspaceId? == some "workspace-a") + require "project cancellation does not invent an unsupported root field" + attachedCancel.root?.isNone + for field in ["workspaceId", "root", "daemonCapability", "leanCmd"] do + match fromJson? (α := ProjectRequest) (semanticJson.setObjVal! field (toJson "forbidden")) with + | .ok _ => throw <| IO.userError s!"project request unexpectedly accepted session field '{field}'" + | .error _ => pure () + for op in [Op.initWorkspace, .listWorkspaces, .dropWorkspace] do + match fromJson? (α := ProjectRequest) <| Json.mkObj [ + ("op", toJson op), + ("clientRequestId", toJson "admin-request") + ] with + | .ok _ => throw <| IO.userError s!"project request unexpectedly accepted admin op '{op.key}'" + | .error _ => pure () + private def checkWorkspaceLifecycleProtocol : IO Unit := do let root := System.FilePath.mk "/workspace" let previous := System.FilePath.mk "/previous-workspace" @@ -1042,7 +1090,7 @@ private def checkSessionCloseAdmission : IO Unit := do let beforeClose ← runtime.dispatchRequest { op := .stats } require "stats should be admitted before session close" beforeClose.ok let active ← - match ← ActiveRequestRegistry.register runtime.activeRequests (some "close-drain") with + match ← ActiveRequestRegistry.register runtime.activeRequests none (some "close-drain") with | .ok active => pure active | .error failure => throw <| IO.userError failure.message let closeTask ← IO.asTask (prio := Task.Priority.dedicated) runtime.close @@ -1095,7 +1143,7 @@ private def checkWrapperDaemonAuthorization : IO Unit := do } require "wrapper daemon should admit the exact generation capability" stats.ok - for op in [Op.initWorkspace, .dropWorkspace] do + for op in [Op.initWorkspace, .listWorkspaces, .dropWorkspace] do let response ← runtime.dispatchRequest { op workspaceId? := some "fixture" @@ -1120,6 +1168,7 @@ def main : IO Unit := do checkStaleDirectDepHints checkRequestArgsBoundary checkWorkspaceRoutingFields + checkProjectRequestBoundary checkWorkspaceLifecycleProtocol checkLifecycleTeardownConcurrency checkSessionCloseAdmission diff --git a/tests/test-beam-fast.sh b/tests/test-beam-fast.sh index 37547474..44e53b2c 100644 --- a/tests/test-beam-fast.sh +++ b/tests/test-beam-fast.sh @@ -357,8 +357,8 @@ wrapper_todo_owner_out="$(mktemp /tmp/lean-beam-wrapper-todo-owner-out-XXXXXX)" wrapper_todo_owner_err="$(mktemp /tmp/lean-beam-wrapper-todo-owner-err-XXXXXX)" wrapper_todo_owner_pid="" wrapper_todo_cleanup() { - BEAM_CONTROL_DIR="$wrapper_todo_control_dir" \ - scripts/lean-beam --root tests/save_olean_project shutdown > /dev/null 2>&1 || true + scripts/lean-beam --root tests/save_olean_project \ + --control-dir "$wrapper_todo_control_dir" shutdown > /dev/null 2>&1 || true if [ -n "$wrapper_todo_owner_pid" ]; then wait "$wrapper_todo_owner_pid" 2>/dev/null || true fi @@ -367,8 +367,8 @@ wrapper_todo_cleanup() { "$wrapper_todo_owner_out" "$wrapper_todo_owner_err" } -BEAM_CONTROL_DIR="$wrapper_todo_control_dir" \ - scripts/lean-beam --root tests/save_olean_project ensure --hold \ +scripts/lean-beam --root tests/save_olean_project \ + --control-dir "$wrapper_todo_control_dir" ensure --hold \ >"$wrapper_todo_owner_out" 2>"$wrapper_todo_owner_err" & wrapper_todo_owner_pid="$!" for _ in $(seq 1 600); do @@ -390,8 +390,8 @@ if ! grep -Fq "owning Beam session" "$wrapper_todo_owner_err"; then exit 1 fi -if ! BEAM_CONTROL_DIR="$wrapper_todo_control_dir" \ - scripts/lean-beam --root tests/save_olean_project \ +if ! scripts/lean-beam --root tests/save_olean_project \ + --control-dir "$wrapper_todo_control_dir" \ update TodoSmoke.lean \ > "$wrapper_todo_update_out" 2>"$wrapper_todo_update_err"; then echo "expected lean-beam update wrapper smoke to succeed before todo" >&2 @@ -421,8 +421,8 @@ PY exit 1 fi -if ! BEAM_CONTROL_DIR="$wrapper_todo_control_dir" \ - scripts/lean-beam --root tests/save_olean_project \ +if ! scripts/lean-beam --root tests/save_olean_project \ + --control-dir "$wrapper_todo_control_dir" \ todo TodoSmoke.lean "$wrapper_todo_version" 13 0 14 0 --kind sorry --suggest none \ > "$wrapper_todo_out" 2>"$wrapper_todo_err"; then echo "expected lean-beam todo wrapper smoke to succeed" >&2 diff --git a/tests/test-beam-toolchain-compat.sh b/tests/test-beam-toolchain-compat.sh index 5be806a8..55032ada 100644 --- a/tests/test-beam-toolchain-compat.sh +++ b/tests/test-beam-toolchain-compat.sh @@ -165,7 +165,7 @@ PY run_bundle_install() { local rc=0 ( - unset BEAM_HOME BEAM_CONTROL_DIR + unset BEAM_HOME BEAM_CONTROL_ROOT export HOME="$tmp_env_root/home" export CODEX_HOME="$tmp_env_root/codex" export CLAUDE_HOME="$tmp_env_root/claude" diff --git a/tests/test-beam-wrapper-daemon.sh b/tests/test-beam-wrapper-daemon.sh index 2cd28823..ae5745c9 100644 --- a/tests/test-beam-wrapper-daemon.sh +++ b/tests/test-beam-wrapper-daemon.sh @@ -247,6 +247,55 @@ assert_json_field_equals "owned ensure response" "$ensure_json" ok true stats_json="$("$beam_script" --root "$tmp1" stats)" assert_json_field_equals "owned stats response" "$stats_json" ok true +machine_stats_json="$("$beam_script" --root "$tmp1" request-stream \ + '{"op":"stats","clientRequestId":"machine-stats"}')" +assert_json_field_equals "root-aware machine stream kind" "$machine_stats_json" kind response +assert_json_field_equals \ + "root-aware machine stream request id" "$machine_stats_json" clientRequestId machine-stats +assert_json_field_equals \ + "root-aware machine stream response" "$machine_stats_json" payload.ok true +if "$beam_script" request-stream '{"op":"stats","clientRequestId":"missing-root"}' \ + > "$tmp1/machine-missing-root.out" 2> "$tmp1/machine-missing-root.err"; then + echo "expected the machine stream interface to require --root" >&2 + exit 1 +fi +if ! grep -Fq "requires an explicit --root PATH" "$tmp1/machine-missing-root.err"; then + echo "expected missing-root machine diagnostics to explain the explicit selector" >&2 + cat "$tmp1/machine-missing-root.err" >&2 + exit 1 +fi +if "$beam_script" --root "$tmp1" request-stream \ + '{"op":"stats","clientRequestId":"caller-route","workspaceId":"beam-cli-project"}' \ + > "$tmp1/machine-route.out" 2> "$tmp1/machine-route.err"; then + echo "expected machine requests to reject caller-selected session routing" >&2 + exit 1 +fi +if ! grep -Fq "session-owned fields: workspaceId" "$tmp1/machine-route.err"; then + echo "expected machine routing rejection to name the forbidden field" >&2 + cat "$tmp1/machine-route.err" >&2 + exit 1 +fi + +python3 - "$registry" "$tmp1" <<'PY' +import json +import os +import sys + +registry, root = sys.argv[1:] +with open(registry, encoding="utf-8") as stream: + entry = json.load(stream) +if entry.get("schemaVersion") != 2: + raise SystemExit(f"unexpected session schema: {entry.get('schemaVersion')}") +workspaces = entry.get("workspaces") +if not isinstance(workspaces, list) or len(workspaces) != 1: + raise SystemExit(f"unexpected workspace bindings: {workspaces!r}") +workspace = workspaces[0] +if workspace.get("root") != os.path.realpath(root): + raise SystemExit(f"unexpected workspace root: {workspace!r}") +if workspace.get("workspaceId") != "beam-cli-project": + raise SystemExit(f"unexpected workspace id: {workspace!r}") +PY + case "$(uname -s)" in Darwin) registry_mode="$(stat -f '%Lp' "$registry")" ;; *) registry_mode="$(stat -c '%a' "$registry")" ;; @@ -256,6 +305,22 @@ if [ "$registry_mode" != "600" ]; then exit 1 fi +if "$beam_script" --root "$tmp1" recover --generation "$daemon1_id" \ + > "$tmp1/live-recover.out" 2> "$tmp1/live-recover.err"; then + echo "expected explicit recovery to refuse a responding generation" >&2 + exit 1 +fi +if ! grep -Fq "still responds" "$tmp1/live-recover.err"; then + echo "expected live-generation recovery refusal to explain the active endpoint" >&2 + cat "$tmp1/live-recover.err" >&2 + exit 1 +fi +if [ "$(read_json_field "$registry" daemonId)" != "$daemon1_id" ] || \ + ! kill -0 "$owner1_pid" 2>/dev/null || ! kill -0 "$daemon1_pid" 2>/dev/null; then + echo "live-generation recovery refusal must preserve the owner and descriptor" >&2 + exit 1 +fi + port1="$(read_json_field "$registry" port)" python3 - "$port1" <<'PY' import json @@ -355,7 +420,7 @@ import os with open(os.environ["REGISTRY_TEMPLATE"], encoding="utf-8") as stream: entry = json.load(stream) -entry["root"] = os.path.realpath(os.environ["STALE_ROOT"]) +entry["workspaces"][0]["root"] = os.path.realpath(os.environ["STALE_ROOT"]) replacement = os.environ["STALE_REGISTRY"] + ".replacement" with open(replacement, "w", encoding="utf-8") as stream: json.dump(entry, stream, separators=(",", ":")) @@ -421,7 +486,18 @@ if [ "$(cat "$stale_registry")" != "$legacy_before" ]; then cat "$stale_registry" >&2 exit 1 fi -rm -f -- "$stale_registry" +legacy_recover_json="$("$beam_script" --root "$tmp2" recover --force)" +assert_json_field_equals "opaque registry recovery" "$legacy_recover_json" recovered true +if [ -e "$stale_registry" ]; then + echo "expected explicit opaque recovery to quarantine the legacy descriptor" >&2 + exit 1 +fi +legacy_quarantine="$(json_text_field "$legacy_recover_json" quarantinedPath)" +if [ ! -f "$legacy_quarantine" ]; then + echo "expected opaque recovery to preserve quarantined evidence" >&2 + printf '%s\n' "$legacy_recover_json" >&2 + exit 1 +fi busy_port_file="$(mktemp "$tmp2/non-beam-port-XXXXXX")" python3 - "$busy_port_file" <<'PY' & @@ -517,22 +593,40 @@ busy_port_file="" start_slow_request "$tmp1" "shutdown-active" "shutdown-active" -# The desired configuration includes the installed bundle paths. Pointing an ordinary command at -# an equivalent bundle in another location creates legitimate desired-hash drift without changing -# the identity of the running generation. The lookup must preserve both the owner and its request. +# The desired owner configuration includes installed bundle paths. An ordinary attaching command +# must use the descriptor's frozen configuration without rebuilding a local desired hash. A second +# owner still computes its proposed configuration and reports the mismatch without disturbing the +# running generation. drift_bundle_dir="$tmp2/config-drift-bundles" mkdir -p "$drift_bundle_dir" rsync -a "$BEAM_INSTALL_BUNDLE_DIR/" "$drift_bundle_dir/" drift_out="$tmp1/config-drift.out" drift_err="$tmp1/config-drift.err" -if BEAM_INSTALL_BUNDLE_DIR="$drift_bundle_dir" \ +if ! BEAM_INSTALL_BUNDLE_DIR="$drift_bundle_dir" \ "$beam_script" --root "$tmp1" ensure > "$drift_out" 2> "$drift_err"; then - echo "expected desired configuration drift to reject attachment" >&2 - cat "$drift_out" >&2 + echo "expected ordinary attachment to use the owner's frozen configuration" >&2 + cat "$drift_err" >&2 + exit 1 +fi +assert_json_file_field_equals \ + "frozen-configuration attachment" "$drift_out" ok true "$drift_err" +drift_owner_out="$tmp1/config-drift-owner.out" +drift_owner_err="$tmp1/config-drift-owner.err" +if BEAM_INSTALL_BUNDLE_DIR="$drift_bundle_dir" \ + "$beam_script" --root "$tmp1" ensure --hold \ + > "$drift_owner_out" 2> "$drift_owner_err"; then + echo "expected a mismatched replacement owner to be rejected" >&2 + cat "$drift_owner_out" >&2 + exit 1 +fi +if ! grep -Fq "current owner was preserved" "$drift_owner_err"; then + echo "expected replacement-owner drift diagnostics to preserve the current owner" >&2 + cat "$drift_owner_err" >&2 exit 1 fi -if ! grep -Fq "current owner was preserved" "$drift_err"; then - echo "expected configuration-drift diagnostics to preserve the current owner" >&2 +if [ -s "$drift_err" ]; then + echo "ordinary frozen-configuration attachment produced unexpected diagnostics" >&2 + cat "$drift_out" >&2 cat "$drift_err" >&2 exit 1 fi @@ -601,11 +695,27 @@ if ! grep -Fq "owned Beam daemon exited with status" "$tmp1/owner-2.err"; then cat "$tmp1/owner-2.err" >&2 exit 1 fi -if [ -e "$registry" ]; then - echo "expected a crashed daemon's owner to remove its exact registry generation" >&2 +if [ ! -e "$registry" ]; then + echo "expected an unexpected daemon crash to preserve its exact session fence" >&2 + exit 1 +fi +if [ "$(read_json_field "$registry" daemonId)" != "$daemon2_id" ] || \ + [ "$(read_json_field "$registry" lifecycle)" != "draining" ]; then + echo "expected an unexpected daemon crash to leave its generation draining" >&2 cat "$registry" >&2 exit 1 fi +if "$beam_script" --root "$tmp1" ensure --hold \ + > "$tmp1/crash-replacement.out" 2> "$tmp1/crash-replacement.err"; then + echo "expected crash-fenced state to reject a replacement owner" >&2 + exit 1 +fi +crash_recovery_json="$("$beam_script" --root "$tmp1" recover --generation "$daemon2_id")" +assert_json_field_equals "unexpected-crash recovery" "$crash_recovery_json" recovered true +if [ -e "$registry" ]; then + echo "expected exact-generation crash recovery to quarantine the fence" >&2 + exit 1 +fi start_owner "$tmp1" "owner-draining-fence" draining_daemon_pid="$(read_json_field "$registry" pid)" @@ -712,12 +822,13 @@ expect_slow_request_cancelled "$tmp1" "owner-loss-active" "owner-loss-active" owner_loss_out="$tmp1/owner-loss.out" owner_loss_err="$tmp1/owner-loss.err" if "$beam_script" --root "$tmp1" ensure > "$owner_loss_out" 2> "$owner_loss_err"; then - echo "expected a command after owner loss to require a replacement owner" >&2 + echo "expected a command after owner loss to preserve the abnormal-session fence" >&2 cat "$owner_loss_out" >&2 exit 1 fi -if ! grep -Fq "lean-beam ensure --hold" "$owner_loss_err"; then - echo "expected owner-loss recovery to name ensure --hold" >&2 +owner_loss_generation="$(read_json_field "$registry" daemonId)" +if ! grep -Fq "recover --generation $owner_loss_generation" "$owner_loss_err"; then + echo "expected owner-loss diagnostics to name exact-generation recovery" >&2 cat "$owner_loss_err" >&2 exit 1 fi @@ -725,6 +836,58 @@ if [ ! -e "$registry" ]; then echo "ordinary owner-loss lookup must not mutate the stale registry" >&2 exit 1 fi +if "$beam_script" --root "$tmp1" recover --generation wrong-generation \ + > "$tmp1/recover-wrong.out" 2> "$tmp1/recover-wrong.err"; then + echo "expected recovery with the wrong generation to fail closed" >&2 + exit 1 +fi +if [ ! -e "$registry" ]; then + echo "wrong-generation recovery must preserve the session fence" >&2 + exit 1 +fi +recover_json="$("$beam_script" --root "$tmp1" recover --generation "$owner_loss_generation")" +assert_json_field_equals "exact-generation recovery" "$recover_json" recovered true +if [ -e "$registry" ]; then + echo "expected explicit recovery to quarantine the stale session descriptor" >&2 + exit 1 +fi +quarantined_registry="$(json_text_field "$recover_json" quarantinedPath)" +if [ ! -f "$quarantined_registry" ]; then + echo "expected explicit recovery to preserve quarantined evidence" >&2 + printf '%s\n' "$recover_json" >&2 + exit 1 +fi + +explicit_control="$tmp2/shared-control" +mkdir -p "$explicit_control" +"$beam_script" --root "$tmp2" --control-dir "$explicit_control" ensure --hold \ + > "$tmp2/explicit-control-owner.out" 2> "$tmp2/explicit-control-owner.err" & +hold_pid="$!" +explicit_registry="$explicit_control/beam-daemon.json" +if ! wait_for_nonempty_file "$explicit_registry" "explicit control-directory session descriptor"; then + cat "$tmp2/explicit-control-owner.err" >&2 + exit 1 +fi +explicit_stats="$("$beam_script" --root "$tmp2" --control-dir "$explicit_control" stats)" +assert_json_field_equals "explicit control-directory attachment" "$explicit_stats" ok true +if "$beam_script" --root "$tmp2" stats \ + > "$tmp2/default-control-stats.out" 2> "$tmp2/default-control-stats.err"; then + echo "expected the default and explicit control selections to remain distinct" >&2 + exit 1 +fi +if [ -e "$tmp2/.beam/beam-daemon.json" ]; then + echo "explicit control-directory ownership must not publish a project-local descriptor" >&2 + exit 1 +fi +if [ ! -f "$explicit_control/beam-daemon-startup.log" ]; then + echo "expected explicit control-directory startup diagnostics beside the descriptor" >&2 + exit 1 +fi +stop_hold_process true +if [ -e "$explicit_registry" ]; then + echo "expected normal explicit-control teardown to remove its descriptor" >&2 + exit 1 +fi generation_registry="$tmp2/.beam/beam-daemon.json" start_owner "$tmp2" "owner-generation" diff --git a/tests/test-beam-wrapper-probe.sh b/tests/test-beam-wrapper-probe.sh index 508bc10d..0c92a461 100644 --- a/tests/test-beam-wrapper-probe.sh +++ b/tests/test-beam-wrapper-probe.sh @@ -25,7 +25,7 @@ beam_wrapper_expect_file "$registry_path" pid1="$(read_json_field "$registry_path" pid)" port1="$(read_json_field "$registry_path" port)" -root1="$(read_json_field "$registry_path" root)" +root1="$(read_json_field "$registry_path" workspaces.0.root)" client1="$(read_json_field "$registry_path" clientBin 2>/dev/null || true)" if [ -z "$client1" ]; then client1="$client" diff --git a/tests/test-beam-wrapper-sandbox.sh b/tests/test-beam-wrapper-sandbox.sh index 5b2abf72..6ae9b739 100755 --- a/tests/test-beam-wrapper-sandbox.sh +++ b/tests/test-beam-wrapper-sandbox.sh @@ -76,7 +76,7 @@ sandbox_beam() { --proc /proc \ --unshare-pid \ --chdir "$project_root" \ - -- /usr/bin/env BEAM_CONTROL_DIR="$control_root" \ + -- /usr/bin/env BEAM_CONTROL_ROOT="$control_root" \ "$beam_script" --root "$project_root" "$@" } @@ -132,7 +132,7 @@ sandbox_owner() { --unshare-pid \ --chdir "$project_root" \ -- /bin/bash -lc \ - "export BEAM_CONTROL_DIR='$control_root'; \ + "export BEAM_CONTROL_ROOT='$control_root'; \ '$beam_script' --root '$project_root' ensure --hold >'$out' 2>'$err' & \ wrapper_pid=\$!; \ (paused=false; \ @@ -281,7 +281,7 @@ fi # Killing the holder closes the only write end of the inherited owner pipe. The daemon must stop # without a heartbeat timeout. A later command in another PID namespace cannot prove that the -# foreign-domain process identities are gone, so it must preserve the registry for supervised +# foreign-domain process identities are gone, so it must preserve the descriptor for explicit # recovery rather than silently treating endpoint unavailability as replacement authority. touch "$owner_kill" if ! wait_for_exit "$owner_pid" "killed sandbox owner" 120 0.1; then @@ -312,13 +312,28 @@ if ! grep -Fq "recorded daemon endpoint is unavailable" "$after_kill_err"; then exit 1 fi if ! find "$control_root" -name beam-daemon.json -print -quit | grep -q .; then - echo "expected ordinary cross-domain recovery to preserve the unsafe registry" >&2 + echo "expected ordinary cross-domain lookup to preserve the unsafe descriptor" >&2 exit 1 fi # This test harness supervised the complete bwrap owner namespace and observed its exit, so it can -# now perform the out-of-band recovery that an unsupervised client must refuse to infer. -rm -f -- "$registry" +# now authorize exact-generation recovery that an ordinary client must refuse to infer. +recovery_json="$(sandbox_beam recover --generation "$daemon_id_2")" +if [ "$(json_text_field "$recovery_json" recovered)" != "true" ]; then + echo "expected exact-generation sandbox recovery to quarantine the descriptor" >&2 + printf '%s\n' "$recovery_json" >&2 + exit 1 +fi +if [ -e "$registry" ]; then + echo "expected exact-generation recovery to remove the authoritative fence" >&2 + exit 1 +fi +recovery_path="$(json_text_field "$recovery_json" quarantinedPath)" +if [ ! -f "$recovery_path" ]; then + echo "expected sandbox recovery to preserve quarantined evidence" >&2 + printf '%s\n' "$recovery_json" >&2 + exit 1 +fi sandbox_owner owner-3 if ! wait_for_registry || ! wait_for_nonempty_file "$owner_out" "final sandbox owner response"; then From 67411649cd5809a6246796f18291a057f7dd6d9c Mon Sep 17 00:00:00 2001 From: Emilio Jesus Gallego Arias Date: Fri, 28 Aug 2026 18:12:13 +0200 Subject: [PATCH 24/28] test: align session contract checks --- tests/lean/BeamTest/Broker/CliDaemonTest.lean | 6 ++++-- tests/test-beam-install.sh | 14 ++++++++++++-- 2 files changed, 16 insertions(+), 4 deletions(-) diff --git a/tests/lean/BeamTest/Broker/CliDaemonTest.lean b/tests/lean/BeamTest/Broker/CliDaemonTest.lean index 824daa1b..ed530fe2 100644 --- a/tests/lean/BeamTest/Broker/CliDaemonTest.lean +++ b/tests/lean/BeamTest/Broker/CliDaemonTest.lean @@ -440,14 +440,16 @@ private def checkCliRootParsing : IO Unit := do let control := root / "shared-control" try IO.FS.createDirAll control + let expectedRoot ← Beam.resolveExistingPath root + let expectedControl ← Beam.resolveExistingPath control let opts ← Beam.Cli.parseCliOptions {} [ "--root", root.toString, "--control-dir", control.toString, "stats" ] - require "explicit CLI root should be canonicalized" (opts.explicitRoot? == some root) + require "explicit CLI root should be canonicalized" (opts.explicitRoot? == some expectedRoot) require "explicit control directory should remain an exact selection" - (opts.explicitControlDir? == some control) + (opts.explicitControlDir? == some expectedControl) require "global selectors should not leak into command arguments" (opts.args == ["stats"]) finally if ← root.pathExists then diff --git a/tests/test-beam-install.sh b/tests/test-beam-install.sh index 3c040e40..7ad9a287 100644 --- a/tests/test-beam-install.sh +++ b/tests/test-beam-install.sh @@ -1604,8 +1604,18 @@ if ! printf '%s\n' "$unsupported_doctor_out" | grep -q 'bundle toolchain fingerp fi unsupported_err="$(mktemp "$tmp_root/install-unsupported-toolchain-XXXXXX")" -if "$installed_lean_beam" --root "$unsupported_project_root" ensure >"$unsupported_err" 2>&1; then - echo "expected installed wrapper ensure lean to reject an unsupported toolchain" >&2 +"$installed_lean_beam" --root "$unsupported_project_root" ensure --hold >"$unsupported_err" 2>&1 & +unsupported_owner_pid="$!" +if ! wait_for_exit "$unsupported_owner_pid" "unsupported-toolchain session owner" 120 0.1; then + kill -INT "$unsupported_owner_pid" 2>/dev/null || true + wait "$unsupported_owner_pid" 2>/dev/null || true + echo "expected installed wrapper owner admission to reject an unsupported toolchain promptly" >&2 + cat "$unsupported_err" >&2 + remove_tmp_file "$unsupported_err" + exit 1 +fi +if wait "$unsupported_owner_pid"; then + echo "expected installed wrapper owner admission to reject an unsupported toolchain" >&2 cat "$unsupported_err" >&2 remove_tmp_file "$unsupported_err" exit 1 From 4170c09be446d523656ab33a12ab5a924e163e8c Mon Sep 17 00:00:00 2001 From: Emilio Jesus Gallego Arias Date: Fri, 28 Aug 2026 23:06:21 +0200 Subject: [PATCH 25/28] refactor: enforce explicit wrapper session ownership --- Beam/Broker/Protocol.lean | 6 +- Beam/Broker/Server.lean | 91 ++++++++++++------- Beam/Cli/Broker.lean | 12 +-- Beam/Cli/Commands.lean | 53 +++++------ Beam/Cli/DaemonManager.lean | 62 ++++++------- Beam/Cli/Feedback.lean | 4 +- Beam/Cli/Info.lean | 2 - Beam/Cli/InstallPrune.lean | 9 +- Beam/Cli/Lock.lean | 49 ++++------ Beam/Daemon/Debug.lean | 56 +----------- Beam/Daemon/Protocol.lean | 2 - Beam/Feedback.lean | 1 - Beam/Mcp/Server.lean | 3 +- Beam/System.lean | 74 --------------- docs/DEVELOPMENT.md | 39 ++++---- docs/SETUP.md | 4 +- docs/STATUS.md | 16 ++-- docs/SYNC_AND_DIAGNOSTICS.md | 12 ++- scripts/install-beam.sh | 18 +--- tests/lean/BeamTest/Broker/CliDaemonTest.lean | 79 ++-------------- tests/lean/BeamTest/Broker/FeedbackTest.lean | 1 - tests/lean/BeamTest/Broker/ProtocolTest.lean | 9 +- tests/test-beam-prune.sh | 15 +-- tests/test-beam-wrapper-daemon.sh | 34 +++++-- tests/test-beam-wrapper-sandbox.sh | 7 -- 25 files changed, 228 insertions(+), 430 deletions(-) diff --git a/Beam/Broker/Protocol.lean b/Beam/Broker/Protocol.lean index d8be71e8..e83a8974 100644 --- a/Beam/Broker/Protocol.lean +++ b/Beam/Broker/Protocol.lean @@ -452,8 +452,10 @@ private def projectRequestForbiddenFields : Array String := #["workspaceId", "workspaceMode", "daemonCapability", "root", "leanCmd", "leanPlugin", "rocqCmd"] private def ProjectRequest.supportedOp : Op → Bool - | .initWorkspace | .listWorkspaces | .dropWorkspace => false - | _ => true + | .ensure | .openDocs | .cancel | .updateFile | .syncFile | .refreshFile | .close | .runAt + | .hover | .signatureHelp | .definition | .references | .documentSymbols | .workspaceSymbols + | .codeActionResolve | .saveOlean | .goals | .todo | .runWith | .release | .stats => true + | .initWorkspace | .listWorkspaces | .dropWorkspace | .resetStats | .shutdown => false def ProjectRequest.ofRequest (request : Request) : Except String ProjectRequest := do unless ProjectRequest.supportedOp request.op do diff --git a/Beam/Broker/Server.lean b/Beam/Broker/Server.lean index a1115ebc..2c24a692 100644 --- a/Beam/Broker/Server.lean +++ b/Beam/Broker/Server.lean @@ -934,10 +934,30 @@ private def modifyCurrentSessionIfMatching | none => pure () +inductive ServerMode where + /-- A separately managed broker, optionally carrying a public generation identity. -/ + | standalone (identity? : Option DaemonIdentity) + /-- A wrapper-owned broker whose identity and request capability are inseparable. -/ + | wrapper (identity : DaemonIdentity) (capability : String) + +def ServerMode.identity? : ServerMode → Option DaemonIdentity + | .standalone identity? => identity? + | .wrapper identity _ => some identity + +private def ServerMode.validate : ServerMode → Except String Unit + | .standalone none => pure () + | .standalone (some identity) => do + unless !identity.daemonId.isEmpty && !identity.configHash.isEmpty do + throw "daemon identity values must be non-empty" + | .wrapper identity capability => do + unless !identity.daemonId.isEmpty && !identity.configHash.isEmpty do + throw "wrapper-owned daemon identity values must be non-empty" + unless !capability.isEmpty do + throw "wrapper-owned daemon capability must be non-empty" + structure ServerRuntime where state : Std.Mutex State - daemonIdentity? : Option DaemonIdentity - private daemonCapability? : Option String + private mode : ServerMode activeRequests : ActiveRequestRegistry private closeMutex : Std.Mutex Bool private closeDone : IO.Promise (Except IO.Error Unit) @@ -971,7 +991,7 @@ private def ServerRuntime.statsResponse (workspaceId? : Option WorkspaceId := none) : IO Response := do let payload ← server.withState <| statsPayload workspaceId? let payload := - match server.daemonIdentity? with + match server.mode.identity? with | some identity => payload.setObjVal! "daemonIdentity" (toJson identity) | none => payload pure <| Response.success payload @@ -979,16 +999,17 @@ private def ServerRuntime.statsResponse def ServerRuntime.create (config : BrokerConfig) (workspaceId : WorkspaceId) - (daemonIdentity? : Option DaemonIdentity := none) - (daemonCapability? : Option String := none) : IO ServerRuntime := do + (mode : ServerMode := .standalone none) : IO ServerRuntime := do unless validWorkspaceId workspaceId do throw <| IO.userError "workspace id must be non-empty" + match mode.validate with + | .ok () => pure () + | .error err => throw <| IO.userError err let startMonoNanos ← IO.monoNanosNow let state := mkInitialState config workspaceId startMonoNanos pure { state := ← Std.Mutex.new state - daemonIdentity? - daemonCapability? + mode activeRequests := ← ActiveRequestRegistry.create closeMutex := ← Std.Mutex.new false closeDone := ← IO.Promise.new @@ -2479,16 +2500,18 @@ private def ServerRuntime.withRequestAdmission let startedAt ← IO.monoNanosNow traceBroker s!"dispatch start op={req.op.key} clientRequestId={optionLabel req.clientRequestId?}" - if let some expected := server.daemonCapability? then - unless req.daemonCapability? == some expected do - let resp := errorResponseFor .invalidParams "invalid Beam daemon capability" - recordDispatchMetrics server req resp startedAt - return resp - if req.op == .initWorkspace || req.op == .listWorkspaces || req.op == .dropWorkspace then - let resp := errorResponseFor .invalidParams - s!"broker op '{req.op.key}' is unavailable in wrapper-owned daemon mode" - recordDispatchMetrics server req resp startedAt - return resp + match server.mode with + | .standalone _ => pure () + | .wrapper _ expected => + unless req.daemonCapability? == some expected do + let resp := errorResponseFor .invalidParams "invalid Beam daemon capability" + recordDispatchMetrics server req resp startedAt + return resp + if req.op == .initWorkspace || req.op == .listWorkspaces || req.op == .dropWorkspace then + let resp := errorResponseFor .invalidParams + s!"broker op '{req.op.key}' is unavailable in wrapper-owned daemon mode" + recordDispatchMetrics server req resp startedAt + return resp match req.validateFields with | .error err => let resp := errorResponseFor .invalidParams err @@ -2793,10 +2816,9 @@ private def acquireDaemonResources (opts : CliOptions) (config : BrokerConfig) (workspaceId : WorkspaceId) - (daemonIdentity? : Option DaemonIdentity) - (daemonCapability? : Option String) + (mode : ServerMode) (root : System.FilePath) : IO DaemonResources := do - let runtime ← ServerRuntime.create config workspaceId daemonIdentity? daemonCapability? + let runtime ← ServerRuntime.create config workspaceId mode let transport ← try DaemonTransport.create opts.endpoint @@ -2809,11 +2831,11 @@ private def acquireDaemonResources throwAfterBestEffortCleanup err <| closeDaemonParts runtime transport none none let ownerWatcher? ← try - if opts.sessionOwnerStdin then - some <$> IO.asTask (prio := Task.Priority.dedicated) - (watchSessionOwnerStdin runtime transport) - else - pure none + match mode with + | .wrapper _ _ => + some <$> IO.asTask (prio := Task.Priority.dedicated) + (watchSessionOwnerStdin runtime transport) + | .standalone _ => pure none catch err => throwAfterBestEffortCleanup err <| closeDaemonParts runtime transport (some rootWatcher) none @@ -2824,11 +2846,10 @@ private def withDaemonResources (opts : CliOptions) (config : BrokerConfig) (workspaceId : WorkspaceId) - (daemonIdentity? : Option DaemonIdentity) - (daemonCapability? : Option String) + (mode : ServerMode) (root : System.FilePath) (act : DaemonResources → IO α) : IO α := do - let resources ← acquireDaemonResources opts config workspaceId daemonIdentity? daemonCapability? root + let resources ← acquireDaemonResources opts config workspaceId mode root try act resources finally @@ -2853,17 +2874,17 @@ def main (args : List String) : IO Unit := do throw <| IO.userError "--daemon-id requires --config-hash" | none, some _ => throw <| IO.userError "--config-hash requires --daemon-id" - let daemonCapability? ← + let mode : ServerMode ← if opts.sessionOwnerStdin then + let some identity := daemonIdentity? + | throw <| IO.userError + "wrapper-owned Beam daemon identity and stdin capability must be supplied together" let capability := (← (← IO.getStdin).getLine).trimAscii.toString if capability.isEmpty then throw <| IO.userError "wrapper-owned Beam daemon received an empty capability" - pure <| some capability + pure <| .wrapper identity capability else - pure none - if opts.sessionOwnerStdin && daemonIdentity?.isNone then - throw <| IO.userError - "wrapper-owned Beam daemon identity and stdin capability must be supplied together" + pure <| .standalone daemonIdentity? let root ← Beam.resolveExistingPath <| System.FilePath.mk root let leanPlugin? ← opts.leanPlugin?.mapM (fun path => Beam.resolveExistingPath <| System.FilePath.mk path) let config : BrokerConfig := { @@ -2872,7 +2893,7 @@ def main (args : List String) : IO Unit := do leanPlugin? := leanPlugin? rocqCmd? := opts.rocqCmd? } - withDaemonResources opts config workspaceId daemonIdentity? daemonCapability? root fun resources => + withDaemonResources opts config workspaceId mode root fun resources => acceptLoop resources.runtime resources.transport end Beam.Broker diff --git a/Beam/Cli/Broker.lean b/Beam/Cli/Broker.lean index 8f67794d..1ce16619 100644 --- a/Beam/Cli/Broker.lean +++ b/Beam/Cli/Broker.lean @@ -21,16 +21,6 @@ private def inWorkspace (workspaceId : WorkspaceId) (req : Request) : Request := if req.workspaceId?.isSome then req else { req with workspaceId? := some workspaceId } -/-- -Address a request to the private workspace of the CLI's one-project daemon. - -Process-wide control operations deliberately remain unscoped. Optional operations such as -cancellation are scoped to the wrapper workspace by default. An explicitly supplied workspace is -preserved so this adapter does not rewrite lower-level test or maintenance requests. --/ -def inProjectDaemonWorkspace (req : Request) : Request := - inWorkspace projectDaemonWorkspaceId req - /-- Address a wrapper request to the workspace selected from its session descriptor. -/ def inSelectedDaemonWorkspace (client : ProjectDaemonClient) (req : Request) : Request := inWorkspace client.workspaceId req @@ -43,7 +33,7 @@ private def withBrokerErrorContext match ← action with | .ok value => pure value | .error failure => - throw <| IO.userError (← daemonFailureMessage root failure client.controlDir?) + throw <| IO.userError (← daemonFailureMessage root failure (some client.controlDir)) structure BrokerWaitSpec where action : String diff --git a/Beam/Cli/Commands.lean b/Beam/Cli/Commands.lean index d9360bdc..8e637307 100644 --- a/Beam/Cli/Commands.lean +++ b/Beam/Cli/Commands.lean @@ -44,7 +44,6 @@ private def updateVersionForRocqGoals pure result.version private def runLeanRunAt - (home : System.FilePath) (opts : CliOptions) (action path versionText lineText characterText : String) (textArgs : List String) @@ -54,14 +53,13 @@ private def runLeanRunAt let line ← parseNatArg "line" lineText let character ← parseNatArg "character" characterText let parsedText ← parseTextArg s!"{action} " textArgs - withProjectDaemon home root .lean (explicitControlDir? := opts.explicitControlDir?) fun client => do + withProjectDaemon root .lean (explicitControlDir? := opts.explicitControlDir?) fun client => do let req ← withEnvClientRequestId <| leanRunAtRequest root path version line character parsedText.text? (storeHandle := storeHandle) maybeEmitTextDebug req.clientRequestId? action parsedText.source parsedText.text? callBrokerWithProgress root client req (leanRunAtWaitSpec action path line character) private def runLeanRunWith - (home : System.FilePath) (opts : CliOptions) (action path : String) (args : List String) @@ -82,11 +80,10 @@ private def runLeanRunWith let req ← withEnvClientRequestId <| leanRunWithRequest root path handle parsedText.text? (linear := linear) maybeEmitTextDebug req.clientRequestId? action parsedText.source parsedText.text? - withProjectDaemon home root .lean (explicitControlDir? := opts.explicitControlDir?) fun client => + withProjectDaemon root .lean (explicitControlDir? := opts.explicitControlDir?) fun client => callBrokerWithProgress root client req (leanRunWithWaitSpec path (linear := linear)) private def runLeanRelease - (home : System.FilePath) (opts : CliOptions) (action : String) (path : String) @@ -95,7 +92,7 @@ private def runLeanRelease let (handle, extra) ← parseHandleInput s!"{action} " args unless extra.isEmpty do throw <| IO.userError (handleArgUsage s!"{action} ") - withProjectDaemon home root .lean (explicitControlDir? := opts.explicitControlDir?) fun client => + withProjectDaemon root .lean (explicitControlDir? := opts.explicitControlDir?) fun client => callBroker root client <| leanReleaseRequest root path handle private def shutdownProjectDaemon (opts : CliOptions) : IO Unit := do @@ -183,7 +180,7 @@ private def ensureBackend (← IO.getStdout).flush IO.eprintln "beam: owning Beam session; interrupt this wrapper process when finished" else - withProjectDaemon home root backend (explicitControlDir? := opts.explicitControlDir?) fun client => + withProjectDaemon root backend (explicitControlDir? := opts.explicitControlDir?) fun client => callBroker root client { op := .ensure, backend := backend, root? := some root.toString } def runCommand (home : System.FilePath) (opts : CliOptions) : IO Unit := do @@ -227,9 +224,9 @@ def runCommand (home : System.FilePath) (opts : CliOptions) : IO Unit := do | "ensure" :: backend :: "--hold" :: [] => ensureBackend home opts (← parseBackendName backend) (hold := true) | "lean-run-at" :: path :: version :: line :: character :: text => - runLeanRunAt home opts (← wrapperDisplayAction "lean-run-at") path version line character text + runLeanRunAt opts (← wrapperDisplayAction "lean-run-at") path version line character text | "lean-run-at-handle" :: path :: version :: line :: character :: text => - runLeanRunAt home opts (← wrapperDisplayAction "lean-run-at-handle") path version line character text + runLeanRunAt opts (← wrapperDisplayAction "lean-run-at-handle") path version line character text (storeHandle := true) | "lean-hover" :: path :: versionText :: line :: character :: [] => let root ← projectRoot opts .lean @@ -237,7 +234,7 @@ def runCommand (home : System.FilePath) (opts : CliOptions) : IO Unit := do let line ← parseNatArg "line" line let character ← parseNatArg "character" character let action ← wrapperDisplayAction "lean-hover" - withProjectDaemon home root .lean (explicitControlDir? := opts.explicitControlDir?) fun client => + withProjectDaemon root .lean (explicitControlDir? := opts.explicitControlDir?) fun client => callBrokerWithProgress root client (leanHoverRequest root path version line character) (leanHoverWaitSpec path line character action) @@ -247,7 +244,7 @@ def runCommand (home : System.FilePath) (opts : CliOptions) : IO Unit := do let line ← parseNatArg "line" line let character ← parseNatArg "character" character let action ← wrapperDisplayAction "lean-signature-help" - withProjectDaemon home root .lean (explicitControlDir? := opts.explicitControlDir?) fun client => + withProjectDaemon root .lean (explicitControlDir? := opts.explicitControlDir?) fun client => callBrokerWithProgress root client (leanSignatureHelpRequest root path version line character) (leanSignatureHelpWaitSpec path line character action) @@ -257,7 +254,7 @@ def runCommand (home : System.FilePath) (opts : CliOptions) : IO Unit := do let line ← parseNatArg "line" line let character ← parseNatArg "character" character let action ← wrapperDisplayAction "lean-definition" - withProjectDaemon home root .lean (explicitControlDir? := opts.explicitControlDir?) fun client => + withProjectDaemon root .lean (explicitControlDir? := opts.explicitControlDir?) fun client => callBrokerWithProgress root client (leanDefinitionRequest root path version line character) (leanDefinitionWaitSpec path line character action) @@ -268,7 +265,7 @@ def runCommand (home : System.FilePath) (opts : CliOptions) : IO Unit := do let character ← parseNatArg "character" character let includeDeclaration ← parseLeanReferencesArgs extra let action ← wrapperDisplayAction "lean-references" - withProjectDaemon home root .lean (explicitControlDir? := opts.explicitControlDir?) fun client => + withProjectDaemon root .lean (explicitControlDir? := opts.explicitControlDir?) fun client => callBrokerWithProgress root client (leanReferencesRequest root path version line character includeDeclaration) (leanReferencesWaitSpec path line character action) @@ -276,7 +273,7 @@ def runCommand (home : System.FilePath) (opts : CliOptions) : IO Unit := do let root ← projectRoot opts .lean let version ← parseNatArg "version" versionText let action ← wrapperDisplayAction "lean-document-symbols" - withProjectDaemon home root .lean (explicitControlDir? := opts.explicitControlDir?) fun client => + withProjectDaemon root .lean (explicitControlDir? := opts.explicitControlDir?) fun client => callBrokerWithProgress root client (leanDocumentSymbolsRequest root path version) (leanDocumentSymbolsWaitSpec path action) @@ -287,7 +284,7 @@ def runCommand (home : System.FilePath) (opts : CliOptions) : IO Unit := do | some query => pure query | none => throw <| IO.userError "usage: beam [--root PATH] lean-workspace-symbols " let action ← wrapperDisplayAction "lean-workspace-symbols" - withProjectDaemon home root .lean (explicitControlDir? := opts.explicitControlDir?) fun client => + withProjectDaemon root .lean (explicitControlDir? := opts.explicitControlDir?) fun client => callBrokerWithProgress root client (leanWorkspaceSymbolsRequest root query) (leanWorkspaceSymbolsWaitSpec query action) @@ -298,7 +295,7 @@ def runCommand (home : System.FilePath) (opts : CliOptions) : IO Unit := do let line ← parseNatArg "line" line let character ← parseNatArg "character" character let action ← wrapperDisplayAction "lean-goals" - withProjectDaemon home root .lean (explicitControlDir? := opts.explicitControlDir?) fun client => + withProjectDaemon root .lean (explicitControlDir? := opts.explicitControlDir?) fun client => callBrokerWithProgress root client (leanGoalsRequest root path version line character mode) (leanGoalsWaitSpec path line character mode (some action)) @@ -311,34 +308,34 @@ def runCommand (home : System.FilePath) (opts : CliOptions) : IO Unit := do let endCharacter ← parseNatArg "endCharacter" endCharacter let (kinds?, suggest?) ← parseLeanTodoArgs extra let action ← wrapperDisplayAction "lean-todo" - withProjectDaemon home root .lean (explicitControlDir? := opts.explicitControlDir?) fun client => + withProjectDaemon root .lean (explicitControlDir? := opts.explicitControlDir?) fun client => callBrokerWithProgress root client (leanTodoRequest root path version startLine startCharacter endLine endCharacter kinds? suggest?) (leanTodoWaitSpec path startLine startCharacter endLine endCharacter action) | "lean-run-with" :: path :: args => - runLeanRunWith home opts (← wrapperDisplayAction "lean-run-with") path args + runLeanRunWith opts (← wrapperDisplayAction "lean-run-with") path args | "lean-run-with-linear" :: path :: args => - runLeanRunWith home opts (← wrapperDisplayAction "lean-run-with-linear") path args + runLeanRunWith opts (← wrapperDisplayAction "lean-run-with-linear") path args (linear := true) | "lean-release" :: path :: args => - runLeanRelease home opts (← wrapperDisplayAction "lean-release") path args + runLeanRelease opts (← wrapperDisplayAction "lean-release") path args | "lean-save" :: path :: extra => do let root ← projectRoot opts .lean let diagnosticScope ← parseLeanSaveArgs extra let action ← wrapperDisplayAction "lean-save" - withProjectDaemon home root .lean (explicitControlDir? := opts.explicitControlDir?) fun client => + withProjectDaemon root .lean (explicitControlDir? := opts.explicitControlDir?) fun client => callBrokerWithProgress root client (leanSaveRequest root path diagnosticScope) (leanSaveWaitSpec path (action? := some action)) | "lean-update" :: path :: [] => let root ← projectRoot opts .lean - withProjectDaemon home root .lean (explicitControlDir? := opts.explicitControlDir?) fun client => + withProjectDaemon root .lean (explicitControlDir? := opts.explicitControlDir?) fun client => callBroker root client <| leanUpdateRequest root path | "lean-sync" :: path :: extra => do let root ← projectRoot opts .lean let diagnosticScope ← parseLeanSyncArgs extra let action ← wrapperDisplayAction "lean-sync" - withProjectDaemon home root .lean (explicitControlDir? := opts.explicitControlDir?) fun client => + withProjectDaemon root .lean (explicitControlDir? := opts.explicitControlDir?) fun client => callBrokerWithProgress root client (leanSyncRequest root path diagnosticScope) (syncWaitSpec path action) @@ -346,25 +343,25 @@ def runCommand (home : System.FilePath) (opts : CliOptions) : IO Unit := do let root ← projectRoot opts .lean let diagnosticScope ← parseLeanRefreshArgs extra let action ← wrapperDisplayAction "lean-refresh" - withProjectDaemon home root .lean (explicitControlDir? := opts.explicitControlDir?) fun client => + withProjectDaemon root .lean (explicitControlDir? := opts.explicitControlDir?) fun client => callBrokerWithProgress root client (leanRefreshRequest root path diagnosticScope) (refreshWaitSpec path action) | "lean-close" :: path :: [] => let root ← projectRoot opts .lean - withProjectDaemon home root .lean (explicitControlDir? := opts.explicitControlDir?) fun client => + withProjectDaemon root .lean (explicitControlDir? := opts.explicitControlDir?) fun client => callBroker root client <| leanCloseRequest root path | "lean-close-save" :: path :: extra => let root ← projectRoot opts .lean let diagnosticScope ← parseLeanCloseSaveArgs extra let action ← wrapperDisplayAction "lean-close-save" - withProjectDaemon home root .lean (explicitControlDir? := opts.explicitControlDir?) fun client => + withProjectDaemon root .lean (explicitControlDir? := opts.explicitControlDir?) fun client => callBrokerWithProgress root client (leanCloseSaveRequest root path diagnosticScope) (leanSaveWaitSpec path (closeAfter := true) (action? := some action)) | "rocq-goals-after" :: path :: line :: character :: text => let root ← projectRoot opts .rocq - withProjectDaemon home root .rocq (explicitControlDir? := opts.explicitControlDir?) fun client => do + withProjectDaemon root .rocq (explicitControlDir? := opts.explicitControlDir?) fun client => do let version ← updateVersionForRocqGoals root client path callBroker root client { op := .goals @@ -381,7 +378,7 @@ def runCommand (home : System.FilePath) (opts : CliOptions) : IO Unit := do } | "rocq-goals-prev" :: path :: line :: character :: text => let root ← projectRoot opts .rocq - withProjectDaemon home root .rocq (explicitControlDir? := opts.explicitControlDir?) fun client => do + withProjectDaemon root .rocq (explicitControlDir? := opts.explicitControlDir?) fun client => do let version ← updateVersionForRocqGoals root client path callBroker root client { op := .goals diff --git a/Beam/Cli/DaemonManager.lean b/Beam/Cli/DaemonManager.lean index e429510c..ba76223f 100644 --- a/Beam/Cli/DaemonManager.lean +++ b/Beam/Cli/DaemonManager.lean @@ -129,11 +129,11 @@ private def writeExistingRegistry (control : ProjectControl) (entry : SessionDes -- Teardown must not create a path while the project tree is being removed. Rewrite through an -- already existing file handle; if the registry was concurrently unlinked, this updates only the -- unlinked inode and cannot recreate the project or control directory. - let handle ← IO.FS.Handle.mk control.registry .readWrite - handle.rewind - handle.putStr ((toJson entry).pretty ++ "\n") - handle.flush - handle.truncate + IO.FS.withFile control.registry .readWrite fun handle => do + handle.rewind + handle.putStr ((toJson entry).pretty ++ "\n") + handle.flush + handle.truncate private def removeRegistry (control : ProjectControl) : IO Unit := do if ← control.registry.pathExists then @@ -302,7 +302,6 @@ private structure DaemonFailureIncident where controlDir : String registryPath : String registry : Option Json := none - registryPidStatus : Option String := none registryEndpoint : Option String := none startupLogPath : Option String := none startupLogTail : Option String := none @@ -341,10 +340,6 @@ private def writeDaemonFailureIncident? let registryFile ← registryPathFor root explicitControlDir? let registryRead ← readRegistryAt registryFile let registry := registryRead.entry? - let pidStatus ← - match registry with - | none => pure none - | some entry => some <$> registryPidStatus entry let endpoint := registry.map registryEndpointSummary let control ← controlDirFor root explicitControlDir? let observedAt ← Beam.utcTimestamp @@ -358,7 +353,6 @@ private def writeDaemonFailureIncident? registryPath := registryFile.toString registry := registry.map fun entry => entry.redactedJson - registryPidStatus := pidStatus registryEndpoint := endpoint startupLogPath := logTail?.map (fun (path, _) => path.toString) startupLogTail := logTail?.map (fun (_, tail) => tail) @@ -567,7 +561,6 @@ private def registryEntryFor let port? := match endpoint with | .tcp port => some port.toNat - let pidDomain? ← Beam.currentPidDomain? let ownerPid ← IO.Process.getPID pure { schemaVersion := registrySchemaVersion @@ -575,9 +568,7 @@ private def registryEntryFor daemonId capability pid - pidDomain? ownerPid := ownerPid.toNat - ownerPidDomain? := pidDomain? port? workspaces := #[{ workspaceId := projectDaemonWorkspaceId @@ -683,8 +674,8 @@ def desiredConfig (home root : System.FilePath) (required : Backend) : IO Desire structure ProjectDaemonClient where endpoint : Transport.Endpoint capability : String - workspaceId : WorkspaceId := projectDaemonWorkspaceId - controlDir? : Option System.FilePath := none + workspaceId : WorkspaceId + controlDir : System.FilePath def ProjectDaemonClient.authorize (client : ProjectDaemonClient) @@ -699,7 +690,7 @@ private def projectDaemonClient endpoint := ← Beam.Daemon.endpointFromEntry entry capability := entry.capability workspaceId := workspace.workspaceId - controlDir? := some controlDir + controlDir } private def workspaceSupportsBackend (workspace : WorkspaceBinding) : Backend → Bool @@ -946,7 +937,8 @@ private def startOwnedProjectDaemon client := { endpoint capability := entry.capability - controlDir? := some control.dir + workspaceId := projectDaemonWorkspaceId + controlDir := control.dir } entry child @@ -1064,25 +1056,25 @@ private def lookupProjectDaemon (root : System.FilePath) (backend? : Option Backend := none) (explicitControlDir? : Option System.FilePath := none) : IO SelectedProjectDaemon := do - withProjectControl root (explicitControlDir? := explicitControlDir?) fun control => do - match ← observeProjectRegistryAt root control.registry with - | .live entry => - let workspace ← selectWorkspaceBackend root entry backend? - pure { client := ← projectDaemonClient entry workspace control.dir, workspace } - | .absent => - throw <| IO.userError (missingOwnerMessage root backend?) - | .draining entry => throw <| IO.userError (drainingOwnerMessage root entry) - | .legacy => - throw <| IO.userError <| registryReadRecoveryMessage root .legacy - | .unsupported schemaVersion => - throw <| IO.userError <| registryReadRecoveryMessage root (.unsupported schemaVersion) - | .malformed detail => - throw <| IO.userError <| registryReadRecoveryMessage root (.malformed detail) - | .unusable entry reason => - throw <| IO.userError <| generationRecoveryMessage root entry reason.message + let control ← projectControl root explicitControlDir? + match ← observeProjectRegistryAt root control.registry with + | .live entry => + let workspace ← selectWorkspaceBackend root entry backend? + pure { client := ← projectDaemonClient entry workspace control.dir, workspace } + | .absent => + throw <| IO.userError (missingOwnerMessage root backend?) + | .draining entry => throw <| IO.userError (drainingOwnerMessage root entry) + | .legacy => + throw <| IO.userError <| registryReadRecoveryMessage root .legacy + | .unsupported schemaVersion => + throw <| IO.userError <| registryReadRecoveryMessage root (.unsupported schemaVersion) + | .malformed detail => + throw <| IO.userError <| registryReadRecoveryMessage root (.malformed detail) + | .unusable entry reason => + throw <| IO.userError <| generationRecoveryMessage root entry reason.message def withProjectDaemon - (_home root : System.FilePath) + (root : System.FilePath) (backend : Backend) (act : ProjectDaemonClient → IO α) (explicitControlDir? : Option System.FilePath := none) : IO α := do diff --git a/Beam/Cli/Feedback.lean b/Beam/Cli/Feedback.lean index f09cd8c0..f668570e 100644 --- a/Beam/Cli/Feedback.lean +++ b/Beam/Cli/Feedback.lean @@ -104,11 +104,12 @@ private def collectDaemonPayload let some workspace ← Beam.Cli.sessionWorkspaceForRoot? entry root | return (Json.null, Json.null, warnings.push "the Beam session does not contain the selected project root") + let controlDir ← Beam.Daemon.controlDirFor root explicitControlDir? let client : ProjectDaemonClient := { endpoint capability := entry.capability workspaceId := workspace.workspaceId - controlDir? := explicitControlDir? + controlDir } let statsResp ← sendRequest endpoint <| client.authorize { op := .stats @@ -150,7 +151,6 @@ private def collectNonConfidential warnings.push "could not infer project root; daemon debug context was not collected") | some root => do let daemon ← Beam.Daemon.daemonDebugContextJson root explicitControlDir? - let warnings := warnings ++ Beam.Daemon.daemonDebugWarnings daemon let (stats, openDocs, warnings) ← collectDaemonPayload root explicitControlDir? warnings pure (stats, openDocs, daemon, warnings) pure { diff --git a/Beam/Cli/Info.lean b/Beam/Cli/Info.lean index 701b7752..7faa7ee3 100644 --- a/Beam/Cli/Info.lean +++ b/Beam/Cli/Info.lean @@ -175,8 +175,6 @@ def doctor (home : System.FilePath) (opts : CliOptions) (backend : Backend) : IO | .live entry => IO.println "daemon status: live" IO.println s!"daemon pid: {entry.pid}" - if let some pidDomain := entry.pidDomain? then - IO.println s!"daemon pid domain: {pidDomain}" if let some endpoint := Beam.Daemon.registryEndpoint? entry then IO.println s!"daemon endpoint: {Beam.Daemon.endpointSummary endpoint}" else diff --git a/Beam/Cli/InstallPrune.lean b/Beam/Cli/InstallPrune.lean index b246e91f..a6e81367 100644 --- a/Beam/Cli/InstallPrune.lean +++ b/Beam/Cli/InstallPrune.lean @@ -54,8 +54,6 @@ private partial def acquireInstallLockUntil if acquired then let selfPid ← IO.Process.getPID IO.FS.writeFile (lockDir / "pid") s!"{selfPid}\n" - if let some pidDomain := ← Beam.currentPidDomain? then - IO.FS.writeFile (lockDir / "pid-domain") s!"{pidDomain}\n" return let now ← IO.monoNanosNow if now >= deadlineNanos then @@ -66,10 +64,9 @@ private partial def acquireInstallLockUntil acquireInstallLockUntil lockDir startedNanos deadlineNanos timeoutMs private def releaseInstallLock (lockDir : System.FilePath) : IO Unit := do - for name in #["pid", "pid-domain"] do - let path := lockDir / name - if ← path.pathExists then - IO.FS.removeFile path + let pidPath := lockDir / "pid" + if ← pidPath.pathExists then + IO.FS.removeFile pidPath IO.FS.removeDir lockDir private def withInstallLockTimeout diff --git a/Beam/Cli/Lock.lean b/Beam/Cli/Lock.lean index 64c73d81..f5b6f481 100644 --- a/Beam/Cli/Lock.lean +++ b/Beam/Cli/Lock.lean @@ -26,20 +26,22 @@ private def lockTimeoutMessage s!"timed out after {waitedMs} ms waiting for Beam lock {lockPath}; " ++ s!"timeout: {timeoutMs} ms" -/-- -Open the stable file whose kernel lock protects one Beam critical section. - -The file is deliberately retained after unlock. Removing a lock file would let a later contender -lock a new inode while an earlier waiter still holds or waits on the old one. --/ -private def openLockHandle (lockPath : System.FilePath) : IO IO.FS.Handle := do +/-- Ensure the stable lock file can be opened without replacing its inode. -/ +private def ensureLockParent (lockPath : System.FilePath) : IO Unit := do if let some parent := lockPath.parent then IO.FS.createDirAll parent - IO.FS.Handle.mk lockPath .append -/-- Open a lock without creating a missing parent directory during teardown. -/ -private def openExistingLockHandle (lockPath : System.FilePath) : IO IO.FS.Handle := do - IO.FS.Handle.mk lockPath .readWrite +private def withAcquiredLock + (lockPath : System.FilePath) + (mode : IO.FS.Mode) + (acquire : IO.FS.Handle → IO Unit) + (act : IO α) : IO α := do + IO.FS.withFile lockPath mode fun handle => do + acquire handle + try + act + finally + handle.unlock private partial def acquireLockUntil (handle : IO.FS.Handle) @@ -57,26 +59,19 @@ private partial def acquireLockUntil /-- Run `act` while holding an unbounded kernel-backed file lock. -/ def withLock (lockPath : System.FilePath) (act : IO α) : IO α := do - let handle ← openLockHandle lockPath - handle.lock - try - act - finally - handle.unlock + ensureLockParent lockPath + withAcquiredLock lockPath .append (·.lock) act /-- Run `act` while holding a kernel-backed file lock until an absolute monotonic deadline. -/ def withLockTimeout (lockPath : System.FilePath) (timeoutMs : Nat) (act : IO α) : IO α := do - let handle ← openLockHandle lockPath + ensureLockParent lockPath let startedNanos ← IO.monoNanosNow - acquireLockUntil handle lockPath { + let deadline := { timeoutMs startedNanos deadlineNanos := startedNanos + timeoutMs * 1000000 } - try - act - finally - handle.unlock + withAcquiredLock lockPath .append (fun handle => acquireLockUntil handle lockPath deadline) act /-- Run `act` under a kernel-backed lock without creating the lock's parent directory. @@ -87,16 +82,12 @@ def withExistingLockTimeout (lockPath : System.FilePath) (timeoutMs : Nat) (act : IO α) : IO α := do - let handle ← openExistingLockHandle lockPath let startedNanos ← IO.monoNanosNow - acquireLockUntil handle lockPath { + let deadline := { timeoutMs startedNanos deadlineNanos := startedNanos + timeoutMs * 1000000 } - try - act - finally - handle.unlock + withAcquiredLock lockPath .readWrite (fun handle => acquireLockUntil handle lockPath deadline) act end Beam.Cli diff --git a/Beam/Daemon/Debug.lean b/Beam/Daemon/Debug.lean index 27ce6141..80eabc5a 100644 --- a/Beam/Daemon/Debug.lean +++ b/Beam/Daemon/Debug.lean @@ -72,18 +72,6 @@ def registryEndpointSummary (entry : SessionDescriptor) : String := | some endpoint => endpointSummary endpoint | none => "invalid" -def registryPidStatus (entry : SessionDescriptor) : IO String := do - let recorded : Beam.RecordedPid := { pid := entry.pid, domain? := entry.pidDomain? } - try - match ← recorded.observe with - | .invalid => pure "unknown" - | .local true => pure "alive" - | .local false => pure "not alive" - | .differentDomain => pure "different PID domain" - | .unknownDomain => pure "unavailable" - catch _ => - pure "unavailable" - def startupLogTail? (root : System.FilePath) (explicitControlDir? : Option System.FilePath := none) : @@ -101,38 +89,6 @@ def startupLogTail? catch _ => pure none -private def jsonStringField? (json : Json) (field : String) : Option String := - match json.getObjValAs? String field with - | .ok value => some value - | .error _ => none - -private def jsonNonNullField (json : Json) (field : String) : Bool := - match json.getObjVal? field with - | .ok Json.null => false - | .ok _ => true - | .error _ => false - -def daemonDebugWarnings (debug : Json) : Array String := Id.run do - let mut warnings := #[] - let pidHint := - "Persisted PIDs are diagnostic only; do not reclaim or replace the session from PID status alone." - if jsonNonNullField debug "registry" then - match jsonStringField? debug "registryPidStatus" with - | some "not alive" => - let detail := - if jsonNonNullField debug "registryEndpoint" then - " while a registry endpoint is recorded" - else - "" - warnings := warnings.push - s!"Beam daemon registry pid is not alive{detail}; stats/open-files may come from a live endpoint with stale registry metadata. {pidHint}" - | some "unavailable" => - warnings := warnings.push - s!"Beam could not verify the daemon registry pid; stats/open-files may reflect a daemon whose registry metadata cannot be trusted. {pidHint}" - | _ => - pure () - warnings - private def optionLine (label : String) : Option String → Option String | none => none | some value => some s!" {label}: {value}" @@ -152,7 +108,6 @@ def daemonRegistryContext? | .malformed detail => pure <| some s!"Beam daemon registry ({path}):\n status: malformed\n detail: {detail}" | .current entry => - let pidStatus ← registryPidStatus entry let workspaceLines := entry.workspaces.toList.flatMap fun workspace => ([ s!" workspace: {workspace.workspaceId}", @@ -166,13 +121,11 @@ def daemonRegistryContext? s!" schemaVersion: {entry.schemaVersion}", s!" lifecycle: {repr entry.lifecycle}", s!" daemonId: {entry.daemonId}", - s!" pid: {entry.pid} ({pidStatus})", + s!" pid: {entry.pid} (diagnostic only)", s!" endpoint: {registryEndpointSummary entry}", s!" startedAt: {entry.startedAt}", s!" configHash: {entry.configHash}" - ] ++ - workspaceLines ++ - (optionLine "pidDomain" entry.pidDomain?).toList) + ] ++ workspaceLines) pure <| some <| String.intercalate "\n" lines catch _ => pure none @@ -183,10 +136,6 @@ def daemonDebugContextJson let registryFile ← registryPathFor root explicitControlDir? let registryRead ← readRegistryAt registryFile let registry := registryRead.entry? - let registryPidStatus ← - match registry with - | some entry => some <$> registryPidStatus entry - | none => pure none let startupLogTail ← startupLogTail? root explicitControlDir? let incidents ← recentDaemonFailureIncidentJson root 5 explicitControlDir? pure <| Json.mkObj <| @@ -199,7 +148,6 @@ def daemonDebugContextJson ("registry", match registry with | some entry => entry.redactedJson | none => Json.null), - ("registryPidStatus", match registryPidStatus with | some status => toJson status | none => Json.null), ("registryEndpoint", match registry.map registryEndpointSummary with | some endpoint => toJson endpoint | none => Json.null), ("recentDaemonIncidents", toJson incidents) ] ++ diff --git a/Beam/Daemon/Protocol.lean b/Beam/Daemon/Protocol.lean index 15496dfc..50dc0ea6 100644 --- a/Beam/Daemon/Protocol.lean +++ b/Beam/Daemon/Protocol.lean @@ -59,9 +59,7 @@ structure SessionDescriptor where daemonId : String capability : String pid : Nat - pidDomain? : Option String := none ownerPid : Nat - ownerPidDomain? : Option String := none port? : Option Nat := none workspaces : Array WorkspaceBinding /-- Hash of the complete frozen session configuration. -/ diff --git a/Beam/Feedback.lean b/Beam/Feedback.lean index 2c9931be..45ed8f71 100644 --- a/Beam/Feedback.lean +++ b/Beam/Feedback.lean @@ -460,7 +460,6 @@ private def runtimeSummarySection (collection : Collection) : String := optionalLine "runtime current" ((jsonBoolField? identity "runtime_current").map boolText) ++ optionalLine "runtime error" (jsonStringField? identity "runtime_error") ++ optionalLine "source" source? ++ - optionalLine "daemon registry pid" (jsonStringField? daemon "registryPidStatus") ++ optionalLine "daemon endpoint" (jsonStringField? daemon "registryEndpoint") ++ warningLines mdSection "Beam Runtime" (String.intercalate "\n" lines) diff --git a/Beam/Mcp/Server.lean b/Beam/Mcp/Server.lean index edd3d75a..88bcb5c3 100644 --- a/Beam/Mcp/Server.lean +++ b/Beam/Mcp/Server.lean @@ -731,9 +731,8 @@ private def handleBeamFeedback else do let identity ← serverIdentity opts (some root) (some runtime?.isSome) let daemon ← Beam.Daemon.daemonDebugContextJson root - let warnings := Beam.Daemon.daemonDebugWarnings daemon let (stats, openDocs, warnings') ← - collectFeedbackRuntimePayload runtime? workspaceId root warnings + collectFeedbackRuntimePayload runtime? workspaceId root #[] pure { generatedAt activeRoot? := some root.toString diff --git a/Beam/System.lean b/Beam/System.lean index 13b285cc..24b33275 100644 --- a/Beam/System.lean +++ b/Beam/System.lean @@ -46,80 +46,6 @@ def commandAvailable (cmd : String) (args : Array String := #["--help"]) : IO Bo catch _ => pure false -private def killCommand : IO String := do - let candidates := [System.FilePath.mk "/bin/kill", System.FilePath.mk "/usr/bin/kill"] - for candidate in candidates do - if ← candidate.pathExists then - return candidate.toString - if ← commandAvailable "kill" #["-l"] then - pure "kill" - else - throw <| IO.userError "could not find kill command" - -/-- -Test a PID already known to belong to the caller's process domain. - -For PIDs loaded from registries, locks, or other persisted metadata, use `RecordedPid.observe` -instead so a numeric PID from another domain is never probed locally. --/ -private def localPidAlive (pid : Nat) : IO Bool := do - let out ← IO.Process.output { cmd := (← killCommand), args := #["-0", toString pid] } - pure (out.exitCode == 0) - -def currentPidDomain? : IO (Option String) := do - try - let domain ← readCmdTrim "readlink" #["/proc/self/ns/pid"] - pure <| if domain.isEmpty then none else some domain - catch _ => - try - let system ← readCmdTrim "uname" #["-s"] - -- Darwin has no PID namespaces. A stable host-domain marker lets two processes on the same - -- supported platform compare PID observations without weakening Linux's fail-closed fallback - -- when `/proc` namespace identity is unexpectedly unavailable. - pure <| if system == "Darwin" then some "host:Darwin" else none - catch _ => - pure none - -/-- A PID loaded together with the process-domain identity recorded by its owner. -/ -structure RecordedPid where - pid : Nat - domain? : Option String - deriving BEq, Repr - -/-- The only safe outcomes of observing a PID loaded from persisted metadata. -/ -inductive RecordedPidObservation where - | invalid - | local (alive : Bool) - | differentDomain - | unknownDomain - deriving BEq, Repr - -private inductive RecordedPidDomainRelation where - | invalid - | local - | different - | unknown - -private def RecordedPid.domainRelation (recorded : RecordedPid) : IO RecordedPidDomainRelation := do - if recorded.pid == 0 then - return .invalid - match ← currentPidDomain?, recorded.domain? with - | some current, some owner => - pure <| if current == owner then .local else .different - | _, _ => - pure .unknown - -/-- -Observe a persisted PID only when its recorded process domain matches the caller's current domain. -Different and unknown domains never reach the local PID probe. --/ -def RecordedPid.observe (recorded : RecordedPid) : IO RecordedPidObservation := do - match ← recorded.domainRelation with - | .invalid => pure .invalid - | .local => pure <| .local (← localPidAlive recorded.pid) - | .different => pure .differentDomain - | .unknown => pure .unknownDomain - def utcTimestamp : IO String := do readCmdTrim "date" #["-u", "+%Y-%m-%dT%H:%M:%SZ"] diff --git a/docs/DEVELOPMENT.md b/docs/DEVELOPMENT.md index 6d4ef260..6680ad65 100644 --- a/docs/DEVELOPMENT.md +++ b/docs/DEVELOPMENT.md @@ -413,14 +413,15 @@ EOF closes admission, marks admitted requests for cancellation, shuts down backe stops the listener. There is no heartbeat, lease, or time-based retirement fence. Ordinary wrapper commands never start a daemon or recompute its desired toolchain/bundle -configuration. Under the session control lock they select the canonical root's frozen workspace -binding and require an endpoint that answers for that workspace, root, and exact generation. -Identity probes have a bounded response deadline. A silent or malformed endpoint fails closed. -Ordinary lookup is observation-only: absent, legacy, malformed, unsupported, draining, unreachable, -or otherwise ambiguous descriptor states are never rewritten by an attaching command. Persisted -PIDs are diagnostic observations, never signal capabilities or automatic stale-reclamation proof. -Only the foreground owner may force termination, through its retained child handle and process -group. +configuration. Without taking the session mutation lock or creating control files, they select the +canonical root's frozen workspace binding and require an endpoint that answers for that workspace, +root, and exact generation. Atomic descriptor publication plus endpoint authentication makes a +concurrent lifecycle change fail closed. Identity probes have a bounded response deadline. A silent +or malformed endpoint fails closed. Ordinary lookup is observation-only: absent, legacy, malformed, +unsupported, draining, unreachable, or otherwise ambiguous descriptor states are never rewritten +by an attaching command. Persisted PIDs are display-only diagnostics, never probed, signalled, or +used as automatic stale-reclamation proof. Only the foreground owner may force termination, through +its retained child handle and process group. The owner watches its exact descriptor generation and daemon child. `lean-beam shutdown` changes that generation from `live` to `draining` under the control lock before sending authenticated @@ -436,10 +437,13 @@ disappears, cleanup uses the already resolved control path without recreating th Human commands may infer the nearest project root. The supported machine stream requires explicit `--root` and a nonempty `clientRequestId`; its semantic JSON cannot supply `root`, `workspaceId`, -capability, or dynamic workspace operations. The wrapper selects the descriptor binding and injects -session metadata. Raw port-oriented `beam-client` requests are maintainer/debug tooling. A -wrapper-owned daemon rejects `initWorkspace`, `listWorkspaces`, and `dropWorkspace`; a separately -launched broker retains the generic multi-workspace surface and has its own explicit owner. +capability, dynamic workspace operations, or process-wide control operations such as `shutdown` and +`resetStats`. The wrapper selects the descriptor binding and injects session metadata. Lifecycle +shutdown remains the dedicated `lean-beam shutdown` command so it can publish `draining` first. Raw +port-oriented `beam-client` requests are maintainer/debug tooling. A wrapper-owned daemon rejects +`initWorkspace`, `listWorkspaces`, and `dropWorkspace`; a separately launched broker retains the +generic multi-workspace surface and has its own explicit owner. Broker runtime ownership is a typed +`ServerMode`: wrapper identity and capability cannot be constructed independently. The default control directory is `/.beam`, discoverable to project-scoped agents. `--control-dir DIR` is an exact, stateless selection that every participant must repeat. @@ -466,13 +470,12 @@ Keep these invariants covered: [tests/test-beam-wrapper-daemon.sh](../tests/test-beam-wrapper-daemon.sh) and [tests/test-beam-wrapper-sandbox.sh](../tests/test-beam-wrapper-sandbox.sh) -Generic process helpers and the typed `RecordedPid.observe` boundary live in -[Beam/System.lean](../Beam/System.lean). Persisted registry PIDs pass through that boundary only for -conservative liveness reporting. Kernel-backed stable file locks live in +Generic process helpers live in [Beam/System.lean](../Beam/System.lean). Kernel-backed stable file locks live in [Beam/Cli/Lock.lean](../Beam/Cli/Lock.lean); lock files remain after release so contenders always coordinate on the same inode, while the kernel releases ownership when a process exits. Project -daemon control locks use a bounded wait so a live but stuck wrapper process produces owner -diagnostics instead of making later clients wait silently; +daemon control mutations use a bounded wait so a live but stuck wrapper process produces owner +diagnostics instead of making another mutation wait silently; ordinary attachment does not take +this lock. `BEAM_CONTROL_LOCK_TIMEOUT_MS` can shorten or lengthen that wait for local debugging. Bundle build locks intentionally keep the lower-level unbounded helper because another process may legitimately be compiling a helper bundle. The shell installer's `.install-lock` remains an atomic directory @@ -504,7 +507,7 @@ normal-priority work on low-core runners. The cheap regression guard is Shared registry, startup-log, and incident paths live in [Beam/Daemon/Paths.lean](../Beam/Daemon/Paths.lean). Daemon registry management, explicit owner -lifetime, endpoint selection, and typed PID-domain cleanup live in +lifetime, endpoint selection, and explicit non-signalling recovery live in [Beam/Cli/DaemonManager.lean](../Beam/Cli/DaemonManager.lean). Broker request plumbing, progress messages, cancellation-on-interrupt, and response failure notes live in [Beam/Cli/Broker.lean](../Beam/Cli/Broker.lean). User-facing stdout/stderr formatting helpers live diff --git a/docs/SETUP.md b/docs/SETUP.md index 46a81989..491dfb56 100644 --- a/docs/SETUP.md +++ b/docs/SETUP.md @@ -245,7 +245,9 @@ control directories. `BEAM_CONTROL_ROOT=/writable/base` is the sandbox convenien derives a separate hashed directory for each canonical root below that base. An explicit control directory is also the intended future location for a statically configured multi-workspace CLI session. The current public owner command still publishes one frozen workspace, and wrapper mode -does not allow runtime `init_workspace`, `list_workspaces`, or `drop_workspace` requests. +does not allow runtime `init_workspace`, `list_workspaces`, or `drop_workspace` requests. The +supported semantic `request-stream` also excludes process-wide `shutdown` and `reset_stats`; use +the dedicated `lean-beam shutdown` command for lifecycle control. Use a stable external control directory when ownership must remain fenced while the project path is deleted and recreated; deleting a project-local `.beam` necessarily deletes its default fence. diff --git a/docs/STATUS.md b/docs/STATUS.md index 61ee5509..cda03082 100644 --- a/docs/STATUS.md +++ b/docs/STATUS.md @@ -134,8 +134,9 @@ apply the source edit. For programmatic local consumers of a wrapper session, the supported machine-readable surface is `lean-beam --root ROOT [--control-dir DIR] request-stream `. The request JSON contains the operation, its arguments, and a nonempty `clientRequestId`; it cannot select a workspace, root, or -capability. The wrapper canonicalizes the explicit root, selects its static workspace binding from -the session descriptor, and injects routing and authentication. Wrapper stderr is human-facing. +capability, and it cannot issue workspace-administration or process-wide control operations. The +wrapper canonicalizes the explicit root, selects its static workspace binding from the session +descriptor, and injects routing and authentication. Wrapper stderr is human-facing. The port-oriented `beam-client request-stream` remains maintainer/debug tooling for separately managed brokers. A wrapper daemon exists only while its foreground `lean-beam ensure --hold` owner is alive. Attaching requests do not acquire daemon ownership. A separately launched standalone @@ -193,13 +194,14 @@ Exact event ordering and examples live in marks admitted requests for cancellation, and closes backend sessions and the daemon after those requests drain; a backend success that completed before cancellation remains successful. This happens without heartbeat timeouts or filesystem leases and works across PID namespaces because - endpoint, root, and generation-identity validation are authoritative when PID identity is not - locally observable. Each wrapper request carries a random per-generation capability from the - mode-`0600` registry. A paused owner retains the session; a killed owner closes the pipe; explicit + authority does not depend on observing persisted PIDs. Endpoint, root, and generation-identity + validation are authoritative. Each wrapper request carries a random per-generation capability + from the mode-`0600` registry. A paused owner retains the session; a killed owner closes the pipe; explicit `lean-beam shutdown` changes the descriptor to `draining`, and that fence remains until normal owner cleanup has reaped the daemon leader after graceful or process-group teardown. - Ordinary lookups use the frozen workspace configuration and preserve unsafe session state. A - competing owner computes its proposed configuration but cannot replace a mismatched live owner. + Ordinary lookups take no mutation lock, create no control files, use the frozen workspace + configuration, and preserve unsafe session state. A competing owner computes its proposed + configuration but cannot replace a mismatched live owner. - Abnormal or ambiguous state is never reclaimed automatically. The descriptor remains as a fence after an unexpected broker/owner exit. Once the operator has established that the matching session is no longer authoritative, `lean-beam --root ROOT recover --generation ID` quarantines diff --git a/docs/SYNC_AND_DIAGNOSTICS.md b/docs/SYNC_AND_DIAGNOSTICS.md index b0b9308d..56540b82 100644 --- a/docs/SYNC_AND_DIAGNOSTICS.md +++ b/docs/SYNC_AND_DIAGNOSTICS.md @@ -125,9 +125,10 @@ use final stdout JSON or `lean-beam --root ROOT [--control-dir DIR] request-stre `lean-beam --root ROOT request-stream` prints one compact JSON object per line, in the order the broker observed it. Its input is a semantic project request with a required nonempty `clientRequestId`; callers cannot supply `root`, `workspaceId`, `daemonCapability`, executable -configuration, or workspace administration operations. A request may produce any number of -`fileProgress` and `diagnostic` messages, followed by exactly one terminal `response`; the response -is last and no later message belongs to that request. +configuration, workspace administration operations, or process-wide `shutdown` / `reset_stats`. +Use the dedicated `lean-beam shutdown` command for lifecycle control. A request may produce any +number of `fileProgress` and `diagnostic` messages, followed by exactly one terminal `response`; the +response is last and no later message belongs to that request. Keep the session's `lean-beam ensure --hold` owner active for the request lifetime. Only the holder starts the daemon; the root-aware machine client reads the mode-`0600` descriptor, selects its @@ -136,6 +137,11 @@ participate in typed request admission and workspace-scoped cancellation but do The raw `beam-client --port ... request-stream` surface requires complete internal request fields and is maintainer/debug tooling for separately managed brokers. +The selected root chooses a workspace runtime; it is not a filesystem authorization boundary. +Relative paths resolve below that root, while absolute paths may identify dependency sources. Beam +can also execute Lean metaprogramming with IO, so callers that require filesystem isolation must +sandbox the owner process itself. + Every stream variant uses the same `kind`, `payload`, and optional correlation envelope. When the request supplies `clientRequestId`, each message repeats it on that outer stream envelope: diff --git a/scripts/install-beam.sh b/scripts/install-beam.sh index 66f154df..3b36726b 100755 --- a/scripts/install-beam.sh +++ b/scripts/install-beam.sh @@ -453,35 +453,19 @@ ensure_install_root_ready() { release_install_lock() { if [ "$install_lock_owned" -eq 1 ]; then if [ -d "$install_lock_dir" ]; then - rm -f -- "$install_lock_dir/pid" "$install_lock_dir/pid-domain" + rm -f -- "$install_lock_dir/pid" rmdir "$install_lock_dir" 2>/dev/null || true fi install_lock_owned=0 fi } -current_pid_domain() { - case "$(uname -s)" in - Linux) - readlink /proc/self/ns/pid 2>/dev/null || true - ;; - Darwin) - printf '%s\n' 'host:Darwin' - ;; - esac -} - acquire_install_lock() { - local pid_domain="" require_path_within "$install_lock_dir" "$install_root" "install lock" if mkdir "$install_lock_dir"; then install_lock_owned=1 trap 'release_install_lock' EXIT printf '%s\n' "$$" >"$install_lock_dir/pid" - pid_domain="$(current_pid_domain)" - if [ -n "$pid_domain" ]; then - printf '%s\n' "$pid_domain" >"$install_lock_dir/pid-domain" - fi else die "another Beam install appears to be running: $install_lock_dir" fi diff --git a/tests/lean/BeamTest/Broker/CliDaemonTest.lean b/tests/lean/BeamTest/Broker/CliDaemonTest.lean index ed530fe2..7db8ecc8 100644 --- a/tests/lean/BeamTest/Broker/CliDaemonTest.lean +++ b/tests/lean/BeamTest/Broker/CliDaemonTest.lean @@ -26,27 +26,14 @@ private def require (label : String) (cond : Bool) : IO Unit := do throw <| IO.userError label private def projectDaemonClientForTest - (endpoint : Beam.Broker.Transport.Endpoint) : Beam.Cli.ProjectDaemonClient := { + (endpoint : Beam.Broker.Transport.Endpoint) + (controlDir : System.FilePath) : Beam.Cli.ProjectDaemonClient := { endpoint capability := "test-capability" + workspaceId := Beam.Cli.projectDaemonWorkspaceId + controlDir } -private def checkDaemonDebugWarnings : IO Unit := do - let debug := Json.mkObj [ - ("registry", Json.mkObj [ - ("daemonId", toJson "fixture-daemon"), - ("pid", toJson (424242 : Nat)) - ]), - ("registryPidStatus", toJson "not alive"), - ("registryEndpoint", toJson "tcp://127.0.0.1:42424") - ] - let warnings := Beam.Daemon.daemonDebugWarnings debug - require "dead registry pid should produce a feedback warning" - (warnings.any (fun warning => warning.contains "registry pid is not alive")) - require "dead registry pid warning should reject PID-based reclamation" - (warnings.any (fun warning => - warning.contains "diagnostic only" && warning.contains "do not reclaim")) - private def expectIoErrorMessage (label : String) (act : IO α) : IO String := do let result ← try @@ -249,7 +236,7 @@ private def checkPlainBrokerTaskCancellation : IO Unit := do serveCancelablePlainRequest listener requestObserved let requestTask ← IO.asTask (prio := Task.Priority.dedicated) <| Beam.Cli.requestBroker (System.FilePath.mk "/tmp") - (projectDaemonClientForTest endpoint) { op := .stats } + (projectDaemonClientForTest endpoint (System.FilePath.mk "/tmp")) { op := .stats } let some _ ← IO.wait requestObserved.result? | throw <| IO.userError "plain wrapper request observation promise dropped" IO.cancel requestTask @@ -280,24 +267,11 @@ private def sampleBrokerHandle : Beam.Broker.Handle := { } private def checkProjectDaemonWorkspaceRouting : IO Unit := do - let ensureReq := Beam.Cli.inProjectDaemonWorkspace ({ op := .ensure } : Beam.Broker.Request) - require "CLI workspace-bound requests should select the project daemon workspace" - (ensureReq.workspaceId? == some Beam.Cli.projectDaemonWorkspaceId) - let statsReq := Beam.Cli.inProjectDaemonWorkspace ({ op := .stats } : Beam.Broker.Request) - require "CLI stats should be scoped to the project daemon workspace" - (statsReq.workspaceId? == some Beam.Cli.projectDaemonWorkspaceId) - let shutdownReq := Beam.Cli.inProjectDaemonWorkspace ({ op := .shutdown } : Beam.Broker.Request) - require "process-wide CLI shutdown should remain unscoped" shutdownReq.workspaceId?.isNone - let explicitReq := Beam.Cli.inProjectDaemonWorkspace ({ - op := .ensure - workspaceId? := some "maintenance-fixture" - } : Beam.Broker.Request) - require "CLI routing should preserve an explicitly selected workspace" - (explicitReq.workspaceId? == some "maintenance-fixture") let selectedClient : Beam.Cli.ProjectDaemonClient := { endpoint := .tcp 42424 capability := "test-capability" workspaceId := "selected-workspace" + controlDir := System.FilePath.mk "/tmp/beam-selected-control" } let selectedCancel := Beam.Cli.inSelectedDaemonWorkspace selectedClient { op := .cancel @@ -615,16 +589,13 @@ private def checkDaemonFailureContext : IO Unit := do let registryPath ← Beam.Daemon.registryPath root if let some parent := registryPath.parent then IO.FS.createDirAll parent - let pidDomain? ← Beam.currentPidDomain? let entry : Beam.Daemon.SessionDescriptor := { schemaVersion := Beam.Daemon.registrySchemaVersion lifecycle := .live daemonId := "daemon-test" capability := "test-capability" pid := 999999999 - pidDomain? ownerPid := 999999999 - ownerPidDomain? := pidDomain? port? := some 42424 workspaces := #[{ workspaceId := Beam.Cli.projectDaemonWorkspaceId @@ -644,7 +615,8 @@ private def checkDaemonFailureContext : IO Unit := do let msg ← Beam.Cli.daemonFailureMessage root failure requireSubstring "daemon failure context should include registry path" "Beam daemon registry" msg requireSubstring "daemon failure context should include daemon id" "daemonId: daemon-test" msg - requireSubstring "daemon failure context should include dead pid status" "pid: 999999999 (not alive)" msg + requireSubstring "daemon failure context should mark persisted pids as diagnostic" + "pid: 999999999 (diagnostic only)" msg requireSubstring "daemon failure context should include endpoint" "endpoint: tcp://127.0.0.1:42424" msg requireSubstring "daemon failure context should include toolchain" "toolchain: leanprover/lean4:test" msg requireSubstring "daemon failure context should include bundle id" "bundleId: bundle-test" msg @@ -668,8 +640,6 @@ private def checkDaemonFailureContext : IO Unit := do (incidentRegistry.daemonId == "daemon-test") require "daemon failure incident must redact the per-generation capability" (incidentRegistry.capability == "") - requireJsonString "daemon failure incident should include registry pid status" - "registryPidStatus" "not alive" incidentJson requireJsonString "daemon failure incident should include endpoint summary" "registryEndpoint" "tcp://127.0.0.1:42424" incidentJson requireJsonString "daemon failure incident should include startup log path" @@ -864,8 +834,9 @@ private def checkBrokerConnectionClosedIncident : IO Unit := do match endpoint with | .tcp port => some port.toNat writeTestRegistryEntry root port? + let controlDir ← Beam.Daemon.controlDir root let msg ← expectIoErrorMessage "broker connection close should surface daemon failure" <| - Beam.Cli.callBrokerQuiet root (projectDaemonClientForTest endpoint) { op := .stats } + Beam.Cli.callBrokerQuiet root (projectDaemonClientForTest endpoint controlDir) { op := .stats } requireSubstring "broker connection close should preserve transport failure" "Beam daemon receive failed:" msg requireSubstring "broker connection close should include incident path" @@ -1028,34 +999,6 @@ private def createSymlink if out.exitCode != 0 then throw <| IO.userError s!"failed to create {label} symlink\n{out.stderr}" -private def checkCurrentPidDomain : IO Unit := do - let selfPid := (← IO.Process.getPID).toNat - let domain? ← Beam.currentPidDomain? - match domain? with - | some domain => - require "a known PID domain should not be empty" (!domain.isEmpty) - let recordedLocal : Beam.RecordedPid := { pid := selfPid, domain? := some domain } - require "a matching PID domain should permit a local liveness observation" - ((← recordedLocal.observe) == .local true) - let different : Beam.RecordedPid := { - pid := selfPid - domain? := some (domain ++ "-other") - } - require "a different PID domain should prevent a local liveness observation" - ((← different.observe) == .differentDomain) - | none => - pure () - let unknown : Beam.RecordedPid := { pid := selfPid, domain? := none } - require "an unknown recorded PID domain must fail closed" - ((← unknown.observe) == .unknownDomain) - let invalid : Beam.RecordedPid := { pid := 0, domain? } - require "PID zero should be classified before domain observation" - ((← invalid.observe) == .invalid) - let system ← Beam.readCmdTrim "uname" #["-s"] - if system == "Darwin" then - require "Darwin processes should share the explicit host PID domain" - (domain? == some "host:Darwin") - private def checkPathCanonicalization : IO Unit := do let stamp ← IO.monoNanosNow let root := System.FilePath.mk s!"/tmp/beam-path-canonical-root-{stamp}" @@ -1387,7 +1330,6 @@ def main : IO Unit := do checkLeanOperationRequests checkDiagnosticScopeArgs checkStartupRetryPolicy - checkDaemonDebugWarnings checkDaemonFailureContext checkDaemonFailureUnreadableStartupLog checkTypedDaemonFailureClassification @@ -1400,7 +1342,6 @@ def main : IO Unit := do checkDoctorDaemonFailureIncidentLines checkPathRelativeToRoot checkLeanModuleNamePathHelpers - checkCurrentPidDomain checkPathCanonicalization checkLockLifecycle checkLeanToolchainPolicyParsing diff --git a/tests/lean/BeamTest/Broker/FeedbackTest.lean b/tests/lean/BeamTest/Broker/FeedbackTest.lean index 33d32aa2..8f919200 100644 --- a/tests/lean/BeamTest/Broker/FeedbackTest.lean +++ b/tests/lean/BeamTest/Broker/FeedbackTest.lean @@ -111,7 +111,6 @@ private def sampleCollection (home : String) : Beam.Feedback.Collection := { ("stats", Json.mkObj [("requests", toJson (3 : Nat))]), ("openFiles", Json.arr #[Json.mkObj [("path", toJson s!"{home}/project/Demo.lean")]]), ("daemon", Json.mkObj [ - ("registryPidStatus", toJson "alive"), ("registryEndpoint", toJson "127.0.0.1:1234"), ("recentDaemonIncidents", Json.arr #[]) ]) diff --git a/tests/lean/BeamTest/Broker/ProtocolTest.lean b/tests/lean/BeamTest/Broker/ProtocolTest.lean index 015a6531..015f7310 100644 --- a/tests/lean/BeamTest/Broker/ProtocolTest.lean +++ b/tests/lean/BeamTest/Broker/ProtocolTest.lean @@ -819,12 +819,12 @@ private def checkProjectRequestBoundary : IO Unit := do match fromJson? (α := ProjectRequest) (semanticJson.setObjVal! field (toJson "forbidden")) with | .ok _ => throw <| IO.userError s!"project request unexpectedly accepted session field '{field}'" | .error _ => pure () - for op in [Op.initWorkspace, .listWorkspaces, .dropWorkspace] do + for op in [Op.initWorkspace, .listWorkspaces, .dropWorkspace, .resetStats, .shutdown] do match fromJson? (α := ProjectRequest) <| Json.mkObj [ ("op", toJson op), - ("clientRequestId", toJson "admin-request") + ("clientRequestId", toJson "control-request") ] with - | .ok _ => throw <| IO.userError s!"project request unexpectedly accepted admin op '{op.key}'" + | .ok _ => throw <| IO.userError s!"project request unexpectedly accepted control op '{op.key}'" | .error _ => pure () private def checkWorkspaceLifecycleProtocol : IO Unit := do @@ -1122,8 +1122,7 @@ private def checkWrapperDaemonAuthorization : IO Unit := do let capability := "generation-secret" let runtime ← Beam.Broker.ServerRuntime.create ({ root } : Beam.Broker.BrokerConfig) "fixture" - (some { daemonId := "generation-a", configHash := "config-a" }) - (some capability) + (.wrapper { daemonId := "generation-a", configHash := "config-a" } capability) try for (label, capability?) in [ ("missing", none), diff --git a/tests/test-beam-prune.sh b/tests/test-beam-prune.sh index aaaed63b..1b262d47 100644 --- a/tests/test-beam-prune.sh +++ b/tests/test-beam-prune.sh @@ -56,21 +56,10 @@ write_runtime_manifest() { "$beam_cli" install-manifest "$payload" - fixture-toolchain >"$path" } -case "$(uname -s)" in - Linux) test_pid_domain="$(readlink /proc/self/ns/pid 2>/dev/null || true)" ;; - Darwin) test_pid_domain="host:Darwin" ;; - *) test_pid_domain="" ;; -esac -if [ -z "$test_pid_domain" ]; then - echo "prune lock tests require a known PID domain" >&2 - exit 1 -fi - write_lock_owner() { local lock_dir="$1" local pid="$2" printf '%s\n' "$pid" >"$lock_dir/pid" - printf '%s\n' "$test_pid_domain" >"$lock_dir/pid-domain" } assert_lock_timeout() { @@ -241,7 +230,7 @@ race_err="$tmp_root/race.err" while [ ! -e "$race_lock_release" ]; do sleep 0.05 done - rm -f "$race_lock/pid" "$race_lock/pid-domain" + rm -f "$race_lock/pid" rmdir "$race_lock" ) & lock_writer_pid="$!" @@ -277,7 +266,7 @@ if "$install_root/current/bin/lean-beam" prune --apply > /dev/null 2>"$install_l exit 1 fi assert_lock_timeout "$install_lock_err" -rm -f "$install_root/.install-lock/pid" "$install_root/.install-lock/pid-domain" +rm -f "$install_root/.install-lock/pid" rmdir "$install_root/.install-lock" assert_file "$old_runtime/manifest.json" diff --git a/tests/test-beam-wrapper-daemon.sh b/tests/test-beam-wrapper-daemon.sh index ae5745c9..6d89be17 100644 --- a/tests/test-beam-wrapper-daemon.sh +++ b/tests/test-beam-wrapper-daemon.sh @@ -171,6 +171,9 @@ if ! grep -Fq "expected backend 'lean' or 'rocq'" "$invalid_backend_err"; then exit 1 fi +# An attaching command observes descriptor state but does not acquire the mutation lock or create +# control-plane files when no owner exists. +remove_tmp_tree_within "$tmp1/.beam" "$tmp1" missing_owner_out="$tmp1/missing-owner.out" missing_owner_err="$tmp1/missing-owner.err" if "$beam_script" --root "$tmp1" ensure > "$missing_owner_out" 2> "$missing_owner_err"; then @@ -183,6 +186,11 @@ if ! grep -Fq "lean-beam ensure --hold" "$missing_owner_err"; then cat "$missing_owner_err" >&2 exit 1 fi +if [ -e "$tmp1/.beam" ]; then + echo "expected missing-owner attachment not to create the project control directory" >&2 + find "$tmp1/.beam" -maxdepth 2 -print >&2 || true + exit 1 +fi start_owner() { local root="$1" @@ -224,7 +232,6 @@ owner1_pid="$hold_pid" daemon1_pid="$(read_json_field "$registry" pid)" daemon1_id="$(read_json_field "$registry" daemonId)" recorded_owner_pid="$(read_json_field "$registry" ownerPid)" -owner_domain="$(read_json_field "$registry" ownerPidDomain)" case "$recorded_owner_pid" in ''|*[!0-9]*|0) echo "expected registry to record a positive session-owner PID" >&2 @@ -232,11 +239,6 @@ case "$recorded_owner_pid" in exit 1 ;; esac -if [ -z "$owner_domain" ]; then - echo "expected registry to record the session owner's PID domain" >&2 - cat "$registry" >&2 - exit 1 -fi if ! kill -0 "$owner1_pid" 2>/dev/null || ! kill -0 "$daemon1_pid" 2>/dev/null; then echo "expected both the wrapper owner and daemon to remain alive" >&2 exit 1 @@ -275,6 +277,26 @@ if ! grep -Fq "session-owned fields: workspaceId" "$tmp1/machine-route.err"; the cat "$tmp1/machine-route.err" >&2 exit 1 fi +if "$beam_script" --root "$tmp1" request-stream \ + '{"op":"shutdown","clientRequestId":"machine-shutdown"}' \ + > "$tmp1/machine-shutdown.out" 2> "$tmp1/machine-shutdown.err"; then + echo "expected semantic machine requests not to expose process-wide shutdown" >&2 + exit 1 +fi +if ! grep -Fq "not available through a project session" "$tmp1/machine-shutdown.err"; then + echo "expected machine shutdown rejection to explain the project-session boundary" >&2 + cat "$tmp1/machine-shutdown.err" >&2 + exit 1 +fi +machine_after_shutdown_json="$("$beam_script" --root "$tmp1" request-stream \ + '{"op":"stats","clientRequestId":"machine-after-shutdown"}')" +assert_json_field_equals \ + "rejected machine shutdown leaves daemon live" "$machine_after_shutdown_json" payload.ok true +if ! kill -0 "$daemon1_pid" 2>/dev/null || \ + [ "$(read_json_field "$registry" daemonId)" != "$daemon1_id" ]; then + echo "expected rejected machine shutdown to preserve the selected generation" >&2 + exit 1 +fi python3 - "$registry" "$tmp1" <<'PY' import json diff --git a/tests/test-beam-wrapper-sandbox.sh b/tests/test-beam-wrapper-sandbox.sh index 6ae9b739..0a54b4ea 100755 --- a/tests/test-beam-wrapper-sandbox.sh +++ b/tests/test-beam-wrapper-sandbox.sh @@ -191,13 +191,6 @@ fi daemon_id_1="$(read_json_field "$registry" daemonId)" port_1="$(read_json_field "$registry" port)" -pid_domain_1="$(read_json_field "$registry" pidDomain 2>/dev/null || true)" -owner_pid_domain_1="$(read_json_field "$registry" ownerPidDomain 2>/dev/null || true)" -if [ -z "$pid_domain_1" ] || [ -z "$owner_pid_domain_1" ]; then - echo "expected the registry to record daemon and owner PID domains" >&2 - sed -n '1,160p' "$registry" >&2 - exit 1 -fi doctor_out="$(sandbox_beam doctor)" if ! printf '%s\n' "$doctor_out" | grep -q 'daemon status: live'; then From 32bf3e3d630ca3186386b8dca154669a67c75aed Mon Sep 17 00:00:00 2001 From: Emilio Jesus Gallego Arias Date: Sat, 29 Aug 2026 01:06:39 +0200 Subject: [PATCH 26/28] fix: harden wrapper recovery and descriptor publication --- Beam/Cli/DaemonManager.lean | 73 +++++++++++++++++++++---------- Beam/Daemon/Paths.lean | 6 ++- docs/DEVELOPMENT.md | 15 +++++-- docs/SETUP.md | 18 +++++--- docs/STATUS.md | 15 ++++--- docs/TESTING.md | 5 ++- tests/test-beam-wrapper-daemon.sh | 53 +++++++++++++++++++++- 7 files changed, 141 insertions(+), 44 deletions(-) diff --git a/Beam/Cli/DaemonManager.lean b/Beam/Cli/DaemonManager.lean index ba76223f..029bc39a 100644 --- a/Beam/Cli/DaemonManager.lean +++ b/Beam/Cli/DaemonManager.lean @@ -59,12 +59,24 @@ private def projectControl let dir ← controlDirFor root explicitControlDir? pure { root, dir, registry := dir / "beam-daemon.json" } +/-- +Create the selected control directory and make it private before creating its lock or any +capability-bearing descriptor. Wrapper authentication assumes that this directory is shared only +between processes of the same local account. +-/ +private def preparePrivateControlDir (dir : System.FilePath) : IO Unit := do + IO.FS.createDirAll dir + IO.setAccessRights dir { + user := { read := true, write := true, execution := true } + } + /-- Supply project registry mutation only for the dynamic extent of the project control lock. -/ private def withProjectControl (root : System.FilePath) (act : ProjectControl → IO α) (explicitControlDir? : Option System.FilePath := none) : IO α := do let control ← projectControl root explicitControlDir? + preparePrivateControlDir control.dir withLockTimeout (control.dir / "lock") (← projectControlLockTimeoutMs) do act control @@ -103,14 +115,26 @@ private def computeConfigHash acc := mixField acc bundleId s!"{acc.toNat}" +private def hexDigit (n : Nat) : Char := + if n < 10 then + Char.ofNat ('0'.toNat + n) + else + Char.ofNat ('a'.toNat + n - 10) + +private def byteHex (byte : UInt8) : List Char := + [hexDigit (byte.toNat / 16), hexDigit (byte.toNat % 16)] + +private def newRegistryTempPath (control : ProjectControl) : IO System.FilePath := do + let nonce := String.ofList <| (← IO.getRandomBytes 16).toList.flatMap byteHex + pure <| control.dir / s!"beam-daemon-{nonce}.tmp" + private def writeRegistry (control : ProjectControl) (entry : SessionDescriptor) : IO Unit := do - if let some parent := control.registry.parent then - IO.FS.createDirAll parent - let tmp := control.registry.withExtension "tmp" + let tmp ← newRegistryTempPath control try - IO.FS.withFile tmp .write fun handle => do - -- The registry contains the daemon capability. Make the inode private before publishing any - -- bytes, rather than relying on a post-write chmod window or the caller's umask. + IO.FS.withFile tmp .writeNew fun handle => do + -- The private control directory protects the inode from its creation. The exclusive random + -- path also refuses pre-existing files and symlinks; mode 0600 remains defense in depth and + -- protects the descriptor after publication if directory permissions later change. IO.setAccessRights tmp { user := { read := true, write := true } } @@ -538,15 +562,6 @@ private def newDaemonGenerationId (configHash : String) : IO String := do let nonce := ByteArray.toUInt64LE! (← IO.getRandomBytes 8) pure s!"{configHash.take 12}-{startedMonoNanos}-{nonce}" -private def hexDigit (n : Nat) : Char := - if n < 10 then - Char.ofNat ('0'.toNat + n) - else - Char.ofNat ('a'.toNat + n - 10) - -private def byteHex (byte : UInt8) : List Char := - [hexDigit (byte.toNat / 16), hexDigit (byte.toNat % 16)] - private def newDaemonCapability : IO String := do let bytes ← IO.getRandomBytes 32 pure <| String.ofList <| bytes.toList.flatMap byteHex @@ -748,9 +763,16 @@ private def registryRecoveryMessage (root : System.FilePath) (detail : String) : private def generationRecoveryMessage (root : System.FilePath) (entry : SessionDescriptor) - (detail : String) : String := - registryRecoveryMessage root detail ++ "; run " ++ - s!"'lean-beam --root {root} recover --generation {entry.daemonId}' when recovery is safe" + (reason : RegistryUnsafeReason) : String := + let message := registryRecoveryMessage root reason.message + match reason with + | .wrongRegistryRoot recordedRoots => + message ++ "; recovery must select one of the descriptor's recorded workspace roots " ++ + s!"({recordedRoots}) with the same --control-dir; the selected root {root} cannot recover " ++ + s!"session {entry.daemonId}" + | _ => + message ++ "; run " ++ + s!"'lean-beam --root {root} recover --generation {entry.daemonId}' when recovery is safe" private def registryReadRecoveryMessage (root : System.FilePath) @@ -791,7 +813,7 @@ def shutdownRegisteredProjectDaemon | .malformed detail => throw <| IO.userError <| registryReadRecoveryMessage root (.malformed detail) | .unusable entry reason => - throw <| IO.userError <| generationRecoveryMessage root entry reason.message + throw <| IO.userError <| generationRecoveryMessage root entry reason match plan with | ShutdownPlan.none => pure <| .ok none | ShutdownPlan.request entry => @@ -814,9 +836,8 @@ private def quarantineRegistry (control : ProjectControl) : IO System.FilePath : private def registeredGenerationResponds (root : System.FilePath) + (workspace : WorkspaceBinding) (entry : SessionDescriptor) : IO Bool := do - let some workspace ← sessionWorkspaceForRoot? entry root - | pure false let some endpoint := registryEndpoint? entry | pure false match ← daemonGenerationStatus endpoint workspace.workspaceId root entry.identity entry.capability with @@ -843,7 +864,11 @@ def recoverProjectDaemon unless generation == entry.daemonId do throw <| IO.userError <| s!"recovery generation '{generation}' does not match recorded generation '{entry.daemonId}'" - if ← registeredGenerationResponds root entry then + let some workspace ← sessionWorkspaceForRoot? entry root + | throw <| IO.userError <| + s!"selected root {root} is not a workspace in session {entry.daemonId}; " ++ + s!"recorded workspace roots: {entry.rootSummary}" + if ← registeredGenerationResponds root workspace entry then throw <| IO.userError <| s!"Beam session {entry.daemonId} still responds; stop its foreground owner or use authenticated shutdown" let quarantine ← quarantineRegistry control @@ -926,7 +951,7 @@ private def startOwnedProjectDaemon | .malformed detail => throw <| IO.userError <| registryReadRecoveryMessage desired.root (.malformed detail) | .unusable entry reason => - throw <| IO.userError <| generationRecoveryMessage desired.root entry reason.message + throw <| IO.userError <| generationRecoveryMessage desired.root entry reason let (endpoint, entry, child) ← startDaemonEntry desired opts control.dir try writeRegistry control entry @@ -1071,7 +1096,7 @@ private def lookupProjectDaemon | .malformed detail => throw <| IO.userError <| registryReadRecoveryMessage root (.malformed detail) | .unusable entry reason => - throw <| IO.userError <| generationRecoveryMessage root entry reason.message + throw <| IO.userError <| generationRecoveryMessage root entry reason def withProjectDaemon (root : System.FilePath) diff --git a/Beam/Daemon/Paths.lean b/Beam/Daemon/Paths.lean index dec154f8..3bd0c1d7 100644 --- a/Beam/Daemon/Paths.lean +++ b/Beam/Daemon/Paths.lean @@ -26,7 +26,11 @@ def controlDirFor | none => match ← IO.getEnv "BEAM_CONTROL_ROOT" with | some base => - pure (System.FilePath.mk base / controlRootTag root) + let base := System.FilePath.mk base + unless base.isAbsolute do + throw <| IO.userError + s!"BEAM_CONTROL_ROOT must be an absolute path, got '{base}'" + pure (base / controlRootTag root) | none => pure (beamStateDir root) diff --git a/docs/DEVELOPMENT.md b/docs/DEVELOPMENT.md index 6680ad65..78e9ac14 100644 --- a/docs/DEVELOPMENT.md +++ b/docs/DEVELOPMENT.md @@ -406,9 +406,11 @@ array of frozen workspace bindings. The public owner command currently creates o shape permits a future explicitly configured static multi-workspace session without changing request routing. Exactly one foreground `lean-beam ensure --hold` process owns the generation. It starts the daemon in a dedicated process session, passes the identity, effective configuration -hash, and random capability through piped stdin, and retains the pipe's write end. The mode-`0600` -descriptor publishes `live` or `draining`. Every wrapper request, including cancellation, -generation probes, and shutdown, presents the capability. The daemon watches the pipe's read end; +hash, and random capability through piped stdin, and retains the pipe's write end. Before creating +the lock or descriptor, Beam makes the selected control directory `0700`; the mode-`0600` +descriptor is written through an exclusive random temporary path and publishes `live` or +`draining`. Every wrapper request, including cancellation, generation probes, and shutdown, +presents the capability. The daemon watches the pipe's read end; EOF closes admission, marks admitted requests for cancellation, shuts down backend sessions, and stops the listener. There is no heartbeat, lease, or time-based retirement fence. @@ -447,7 +449,10 @@ generic multi-workspace surface and has its own explicit owner. Broker runtime o The default control directory is `/.beam`, discoverable to project-scoped agents. `--control-dir DIR` is an exact, stateless selection that every participant must repeat. -`BEAM_CONTROL_ROOT` hashes each canonical root below a writable base for sandboxed/read-only roots. +`BEAM_CONTROL_ROOT` must be absolute and hashes each canonical root below a writable base for +sandboxed/read-only roots. The selected directory is private to one local account; coordination is +supported between that account's processes, not across a group-shared control directory. + Do not hide policy inside automatic fallback between these locations. A future multi-root CLI owner should require a stable explicit control directory and freeze all bindings before publication. @@ -459,6 +464,8 @@ Keep these invariants covered: in ambiguous or unsafe state; they attach to frozen owner configuration rather than recomputing it - normal holder teardown retains a generation-specific draining fence until owned cleanup completes and cannot remove a replacement; abnormal exit leaves the fence for explicit recovery +- recovery of a current descriptor requires both its exact generation and one of its recorded + workspace roots; a wrong-root caller cannot quarantine another session - owner EOF, explicit shutdown, and project-root disappearance all close admission before backend teardown and complete with bounded child cleanup - persisted numeric PIDs are never signalled or used for automatic stale reclamation diff --git a/docs/SETUP.md b/docs/SETUP.md index 491dfb56..21f2d673 100644 --- a/docs/SETUP.md +++ b/docs/SETUP.md @@ -232,8 +232,11 @@ separate holder; the stdio MCP process owns its runtime session. The default session descriptor and lock live in `/.beam`. This is intentional for project-scoped agent sandboxes: clients that can access the same workspace can discover the same -session. Use an exact alternate directory when the project is read-only or several explicitly -coordinated clients need another stable control plane: +session. Before creating a lock or capability-bearing descriptor, Beam makes the selected control +directory account-private (`0700`); the published descriptor is `0600`. This permits coordination +between sandboxes and agents running as the same local account, but a group-shared or traversable +control directory is not a supported authentication boundary. Use an exact alternate directory +when the project is read-only or several same-account clients need another stable control plane: ```bash lean-beam --root /workspace/a --control-dir /workspace/control ensure --hold @@ -241,10 +244,10 @@ lean-beam --root /workspace/a --control-dir /workspace/control stats ``` Every participant must supply the same `--root` and `--control-dir`; Beam does not search alternate -control directories. `BEAM_CONTROL_ROOT=/writable/base` is the sandbox convenience form: Beam -derives a separate hashed directory for each canonical root below that base. An explicit control -directory is also the intended future location for a statically configured multi-workspace CLI -session. The current public owner command still publishes one frozen workspace, and wrapper mode +control directories. `BEAM_CONTROL_ROOT=/writable/base` is the sandbox convenience form and must be +absolute: Beam derives a separate hashed directory for each canonical root below that base. An +explicit control directory is also the intended future location for a statically configured +multi-workspace CLI session. The current public owner command still publishes one frozen workspace, and wrapper mode does not allow runtime `init_workspace`, `list_workspaces`, or `drop_workspace` requests. The supported semantic `request-stream` also excludes process-wide `shutdown` and `reset_stats`; use the dedicated `lean-beam shutdown` command for lifecycle control. @@ -263,6 +266,9 @@ Use the same `--control-dir` selection when applicable. `recover --force` is res legacy, unsupported, or malformed descriptors. Recovery preserves the old file under a `beam-daemon.recovered-*.json` name for diagnosis. +For a current descriptor, the selected `--root` must be one of its recorded workspace bindings; +using the same control directory with an unrelated root cannot quarantine the session. + Machine clients should avoid root auto-detection and raw port/session fields: ```bash diff --git a/docs/STATUS.md b/docs/STATUS.md index cda03082..36a43596 100644 --- a/docs/STATUS.md +++ b/docs/STATUS.md @@ -196,7 +196,8 @@ Exact event ordering and examples live in happens without heartbeat timeouts or filesystem leases and works across PID namespaces because authority does not depend on observing persisted PIDs. Endpoint, root, and generation-identity validation are authoritative. Each wrapper request carries a random per-generation capability - from the mode-`0600` registry. A paused owner retains the session; a killed owner closes the pipe; explicit + from a mode-`0600` registry inside a mode-`0700` control directory. A paused owner retains the + session; a killed owner closes the pipe; explicit `lean-beam shutdown` changes the descriptor to `draining`, and that fence remains until normal owner cleanup has reaped the daemon leader after graceful or process-group teardown. Ordinary lookups take no mutation lock, create no control files, use the frozen workspace @@ -206,20 +207,24 @@ Exact event ordering and examples live in after an unexpected broker/owner exit. Once the operator has established that the matching session is no longer authoritative, `lean-beam --root ROOT recover --generation ID` quarantines that exact descriptor without signalling persisted PIDs. Legacy, unsupported, or malformed - descriptor state requires the deliberately broader `recover --force` form. + descriptor state requires the deliberately broader `recover --force` form. Current-descriptor + recovery additionally requires a root recorded in that descriptor, so a wrong-root caller using + the same control directory cannot quarantine the session. - The default authoritative descriptor is `/.beam/beam-daemon.json`, which keeps discovery available inside project-scoped agent sandboxes. `--control-dir DIR` selects one exact alternate control directory; callers must repeat the same selection for every owner, request, diagnostic, shutdown, and recovery command. `BEAM_CONTROL_ROOT` is the sandbox convenience that derives a - per-root subdirectory below a writable base. A stable explicit control directory is also the - intended future boundary for a statically configured multi-workspace CLI session; dynamic + per-root subdirectory below an absolute writable base. Beam makes every selected control + directory account-private (`0700`) before creating its lock or capability descriptor. A stable + explicit control directory is also the intended future boundary for a statically configured + multi-workspace CLI session; dynamic workspace mutation remains unavailable in wrapper mode. - Deleting the project tree also deletes its default project-local fence. Workflows that may remove and immediately recreate the same canonical path, and need exclusion to survive that operation, should select a stable external `--control-dir`; this tradeoff keeps the default usable from project-scoped agent sandboxes without assuming access to a host-global runtime directory. - Wrapper-owned brokers currently use authenticated loopback TCP. The supported trust boundary is - one local OS account with private registry-file permissions; another user who can only discover + one local OS account with a private control directory and registry-file permissions; another user who can only discover the port cannot issue requests without the generation capability. A manually launched standalone `beam-daemon` has no wrapper registry capability and is maintainer tooling, not a shared-host service. Unix-domain/per-user native IPC remains a possible later transport improvement. diff --git a/docs/TESTING.md b/docs/TESTING.md index 2177d2a9..f83ff16b 100644 --- a/docs/TESTING.md +++ b/docs/TESTING.md @@ -114,8 +114,9 @@ Current Beam coverage includes: focused probe, runtime, sync/save, handle, and diagnostic slices independently - focused daemon lifecycle coverage in [tests/test-beam-wrapper-daemon.sh](../tests/test-beam-wrapper-daemon.sh), including the no-implicit-start contract, duplicate-owner rejection, Beam and non-Beam endpoint - collision safety without cross-project disclosure, authenticated generation probes, mode-`0600` - registry publication, unauthorized-shutdown rejection without listener teardown, oversized-frame + collision safety without cross-project disclosure, authenticated generation probes, mode-`0700` + control-directory and mode-`0600` registry publication, wrong-root recovery rejection with + byte-for-byte descriptor preservation, unauthorized-shutdown rejection without listener teardown, oversized-frame and first-message limits, a bounded identity probe against a silent non-Beam listener, cross-root unsafe-registry preservation that does not affect the daemon serving the other root, configuration-drift preservation of the owner and active request, diff --git a/tests/test-beam-wrapper-daemon.sh b/tests/test-beam-wrapper-daemon.sh index 6d89be17..55f6765f 100644 --- a/tests/test-beam-wrapper-daemon.sh +++ b/tests/test-beam-wrapper-daemon.sh @@ -319,14 +319,50 @@ if workspace.get("workspaceId") != "beam-cli-project": PY case "$(uname -s)" in - Darwin) registry_mode="$(stat -f '%Lp' "$registry")" ;; - *) registry_mode="$(stat -c '%a' "$registry")" ;; + Darwin) + control_dir_mode="$(stat -f '%Lp' "$tmp1/.beam")" + registry_mode="$(stat -f '%Lp' "$registry")" + ;; + *) + control_dir_mode="$(stat -c '%a' "$tmp1/.beam")" + registry_mode="$(stat -c '%a' "$registry")" + ;; esac +if [ "$control_dir_mode" != "700" ]; then + echo "expected the capability control directory to use mode 700, got $control_dir_mode" >&2 + exit 1 +fi if [ "$registry_mode" != "600" ]; then echo "expected the capability-bearing registry to use mode 600, got $registry_mode" >&2 exit 1 fi +cross_root_descriptor="$tmp2/cross-root-recovery.before" +cp -- "$registry" "$cross_root_descriptor" +if "$beam_script" --root "$tmp2" --control-dir "$tmp1/.beam" \ + recover --generation "$daemon1_id" \ + > "$tmp2/cross-root-recovery.out" 2> "$tmp2/cross-root-recovery.err"; then + echo "expected recovery through a non-member root to fail closed" >&2 + exit 1 +fi +if ! grep -Fq "is not a workspace in session $daemon1_id" \ + "$tmp2/cross-root-recovery.err" || \ + ! grep -Fq "$tmp1" "$tmp2/cross-root-recovery.err"; then + echo "expected cross-root recovery rejection to name the session and its recorded root" >&2 + cat "$tmp2/cross-root-recovery.err" >&2 + exit 1 +fi +if ! cmp -s -- "$cross_root_descriptor" "$registry"; then + echo "cross-root recovery must preserve the descriptor byte-for-byte" >&2 + exit 1 +fi +cross_root_stats_json="$("$beam_script" --root "$tmp1" stats)" +assert_json_field_equals "stats after rejected cross-root recovery" "$cross_root_stats_json" ok true +if ! kill -0 "$owner1_pid" 2>/dev/null || ! kill -0 "$daemon1_pid" 2>/dev/null; then + echo "cross-root recovery rejection must preserve the live owner and daemon" >&2 + exit 1 +fi + if "$beam_script" --root "$tmp1" recover --generation "$daemon1_id" \ > "$tmp1/live-recover.out" 2> "$tmp1/live-recover.err"; then echo "expected explicit recovery to refuse a responding generation" >&2 @@ -343,6 +379,19 @@ if [ "$(read_json_field "$registry" daemonId)" != "$daemon1_id" ] || \ exit 1 fi +if BEAM_CONTROL_ROOT=relative-control-root \ + "$beam_script" --root "$tmp1" stats \ + > "$tmp1/relative-control-root.out" 2> "$tmp1/relative-control-root.err"; then + echo "expected relative BEAM_CONTROL_ROOT to be rejected" >&2 + exit 1 +fi +if ! grep -Fq "BEAM_CONTROL_ROOT must be an absolute path" \ + "$tmp1/relative-control-root.err"; then + echo "expected relative BEAM_CONTROL_ROOT rejection to explain the stable-path requirement" >&2 + cat "$tmp1/relative-control-root.err" >&2 + exit 1 +fi + port1="$(read_json_field "$registry" port)" python3 - "$port1" <<'PY' import json From e18e9834fc9db722b58263fd8c8bc8b9e6766b6e Mon Sep 17 00:00:00 2001 From: Emilio Jesus Gallego Arias Date: Sat, 29 Aug 2026 18:13:16 +0200 Subject: [PATCH 27/28] fix: refuse unsafe control directories --- Beam/Cli/DaemonManager.lean | 99 ++++++++++++++++++++++++++--- Beam/Native/control_dir.c | 26 ++++++++ Beam/System.lean | 7 ++ docs/DEVELOPMENT.md | 16 +++-- docs/SETUP.md | 20 +++--- docs/STATUS.md | 7 +- docs/TESTING.md | 3 +- lakefile.lean | 8 +++ tests/lib/beam-wrapper-common.sh | 1 + tests/test-beam-toolchain-compat.sh | 1 + tests/test-beam-wrapper-daemon.sh | 80 +++++++++++++++++++---- 11 files changed, 233 insertions(+), 35 deletions(-) create mode 100644 Beam/Native/control_dir.c diff --git a/Beam/Cli/DaemonManager.lean b/Beam/Cli/DaemonManager.lean index 029bc39a..fbff39e0 100644 --- a/Beam/Cli/DaemonManager.lean +++ b/Beam/Cli/DaemonManager.lean @@ -59,16 +59,96 @@ private def projectControl let dir ← controlDirFor root explicitControlDir? pure { root, dir, registry := dir / "beam-daemon.json" } +private def privateControlDirRights : IO.FileRight := { + user := { read := true, write := true, execution := true } +} + +private def privateControlDirMode : UInt32 := + privateControlDirRights.flags + +private def invalidControlDirMessage (dir detail : String) : String := + s!"unsafe Beam control directory {dir}: {detail}. Select a dedicated directory that is a real " ++ + "directory with mode 0700; Beam does not change permissions on existing paths" + +private def permissionModeText (mode : UInt32) : String := + let value := mode.toNat + s!"0{value / 64}{(value / 8) % 8}{value % 8}" + +private inductive ControlDirObservation where + | absent + | privateDir + | symlink + | nonPrivate (mode : UInt32) + | notDirectory + +/-- Inspect the exact control leaf without following a final symbolic link. -/ +private def observeControlDir (dir : System.FilePath) : IO ControlDirObservation := do + try + let metadata ← dir.symlinkMetadata + match metadata.type with + | .dir => + let mode ← Beam.fileModeNoFollow dir + if mode == privateControlDirMode then + pure .privateDir + else + pure <| .nonPrivate mode + | .symlink => pure .symlink + | .file | .other => pure .notDirectory + catch + | .noFileOrDirectory .. => pure .absent + | err => throw err + +private def rejectControlDirObservation + (dir : System.FilePath) : ControlDirObservation → IO Unit + | .symlink => + throw <| IO.userError <| + invalidControlDirMessage dir.toString "symbolic links are not accepted" + | .nonPrivate mode => + throw <| IO.userError <| + invalidControlDirMessage dir.toString + s!"existing mode is {permissionModeText mode}, expected 0700" + | .notDirectory => + throw <| IO.userError <| + invalidControlDirMessage dir.toString "the path is not a directory" + | .absent => + throw <| IO.userError <| + invalidControlDirMessage dir.toString "the path disappeared during control preparation" + | .privateDir => pure () + +/-- Accept an existing control path only when it is a real, account-private directory. -/ +private def validatePrivateControlDir (dir : System.FilePath) : IO Unit := do + rejectControlDirObservation dir (← observeControlDir dir) + /-- -Create the selected control directory and make it private before creating its lock or any -capability-bearing descriptor. Wrapper authentication assumes that this directory is shared only -between processes of the same local account. +Create a missing dedicated control leaf as private, or validate an existing path without mutating +it. The directory is ready before Beam creates its lock or any capability-bearing descriptor. -/ private def preparePrivateControlDir (dir : System.FilePath) : IO Unit := do - IO.FS.createDirAll dir - IO.setAccessRights dir { - user := { read := true, write := true, execution := true } - } + match ← observeControlDir dir with + | .privateDir => return + | .absent => pure () + | observation => rejectControlDirObservation dir observation + if let some parent := dir.parent then + IO.FS.createDirAll parent + try + IO.FS.createDir dir + catch + | .alreadyExists .. => + -- A concurrent owner may have created the leaf after our absent observation. Never adopt it + -- implicitly: apply the same read-only validation as any other existing path. + validatePrivateControlDir dir + return + | err => throw err + try + -- This chmod is restricted to the leaf created successfully by this invocation. + IO.setAccessRights dir privateControlDirRights + validatePrivateControlDir dir + catch err => + try + IO.FS.removeDir dir + catch _ => + pure () + throw err /-- Supply project registry mutation only for the dynamic extent of the project control lock. -/ private def withProjectControl @@ -1060,8 +1140,11 @@ def withProjectDaemonOwner (backend : Backend) (opts : CliOptions) (act : ProjectDaemonOwner → IO α) : IO α := do - let desired ← desiredConfig home root backend let controlDir ← controlDirFor root opts.explicitControlDir? + -- Establish or validate the control boundary before bundle resolution can create project-local + -- `.beam` state for a previously unseen toolchain. + preparePrivateControlDir controlDir + let desired ← desiredConfig home root backend let owned ← withProjectControl root (explicitControlDir? := some controlDir) fun control => startOwnedProjectDaemon control desired opts let exitCodeRef ← IO.mkRef (none : Option UInt32) diff --git a/Beam/Native/control_dir.c b/Beam/Native/control_dir.c new file mode 100644 index 00000000..afbdf301 --- /dev/null +++ b/Beam/Native/control_dir.c @@ -0,0 +1,26 @@ +/* +Copyright (c) 2026 Lean FRO LLC. All rights reserved. +Released under Apache 2.0 license as described in the file LICENSE. +Author: Emilio J. Gallego Arias +*/ + +#include +#include +#include +#include + +LEAN_EXPORT lean_obj_res lean_beam_lstat_mode( + b_lean_obj_arg path, + lean_obj_arg world) { + (void)world; +#if defined(_WIN32) + struct _stat status; + if (_stat(lean_string_cstr(path), &status) != 0) { +#else + struct stat status; + if (lstat(lean_string_cstr(path), &status) != 0) { +#endif + return lean_io_result_mk_error(lean_decode_io_error(errno, path)); + } + return lean_io_result_mk_ok(lean_box_uint32((uint32_t)(status.st_mode & 0777))); +} diff --git a/Beam/System.lean b/Beam/System.lean index 24b33275..fb13a9d9 100644 --- a/Beam/System.lean +++ b/Beam/System.lean @@ -10,6 +10,13 @@ open Lean namespace Beam +/-- Return the POSIX permission bits reported by `lstat`, without following the final symlink. -/ +@[extern "lean_beam_lstat_mode"] +private opaque lstatMode (path : @& String) : IO UInt32 + +def fileModeNoFollow (path : System.FilePath) : IO UInt32 := + lstatMode path.toString + def trimLine (text : String) : String := text.trimAscii.toString diff --git a/docs/DEVELOPMENT.md b/docs/DEVELOPMENT.md index 78e9ac14..67d4108f 100644 --- a/docs/DEVELOPMENT.md +++ b/docs/DEVELOPMENT.md @@ -407,10 +407,12 @@ shape permits a future explicitly configured static multi-workspace session with request routing. Exactly one foreground `lean-beam ensure --hold` process owns the generation. It starts the daemon in a dedicated process session, passes the identity, effective configuration hash, and random capability through piped stdin, and retains the pipe's write end. Before creating -the lock or descriptor, Beam makes the selected control directory `0700`; the mode-`0600` -descriptor is written through an exclusive random temporary path and publishes `live` or -`draining`. Every wrapper request, including cancellation, generation probes, and shutdown, -presents the capability. The daemon watches the pipe's read end; +the lock or descriptor, Beam creates a missing control leaf with mode `0700`, or validates that an +existing leaf is a real, non-symlinked directory already using mode `0700`. It never changes an +existing selection's permissions. The mode-`0600` descriptor is written through an exclusive +random temporary path and publishes `live` or `draining`. Every wrapper request, including +cancellation, generation probes, and shutdown, presents the capability. The daemon watches the +pipe's read end; EOF closes admission, marks admitted requests for cancellation, shuts down backend sessions, and stops the listener. There is no heartbeat, lease, or time-based retirement fence. @@ -451,7 +453,9 @@ The default control directory is `/.beam`, discoverable to project-scoped `--control-dir DIR` is an exact, stateless selection that every participant must repeat. `BEAM_CONTROL_ROOT` must be absolute and hashes each canonical root below a writable base for sandboxed/read-only roots. The selected directory is private to one local account; coordination is -supported between that account's processes, not across a group-shared control directory. +supported between that account's processes, not across a group-shared control directory. Existing +directories must be prepared explicitly as mode `0700`; validation rejects symlinks, other file +types, and broader permissions without mutating them. Do not hide policy inside automatic fallback between these locations. A future multi-root CLI owner should require a stable explicit control directory and freeze all bindings before publication. @@ -466,6 +470,8 @@ Keep these invariants covered: and cannot remove a replacement; abnormal exit leaves the fence for explicit recovery - recovery of a current descriptor requires both its exact generation and one of its recorded workspace roots; a wrong-root caller cannot quarantine another session +- control preparation changes permissions only on a leaf Beam just created; an existing control + path must be a non-symlinked mode-`0700` directory and rejection leaves it untouched - owner EOF, explicit shutdown, and project-root disappearance all close admission before backend teardown and complete with bounded child cleanup - persisted numeric PIDs are never signalled or used for automatic stale reclamation diff --git a/docs/SETUP.md b/docs/SETUP.md index 21f2d673..793fb542 100644 --- a/docs/SETUP.md +++ b/docs/SETUP.md @@ -233,10 +233,12 @@ separate holder; the stdio MCP process owns its runtime session. The default session descriptor and lock live in `/.beam`. This is intentional for project-scoped agent sandboxes: clients that can access the same workspace can discover the same session. Before creating a lock or capability-bearing descriptor, Beam makes the selected control -directory account-private (`0700`); the published descriptor is `0600`. This permits coordination -between sandboxes and agents running as the same local account, but a group-shared or traversable -control directory is not a supported authentication boundary. Use an exact alternate directory -when the project is read-only or several same-account clients need another stable control plane: +directory account-private (`0700`) when the leaf does not exist; the published descriptor is +`0600`. An existing selection must already be a real, non-symlinked directory with mode `0700`, or +Beam refuses it without changing its permissions. This permits coordination between sandboxes and +agents running as the same local account, but a group-shared or traversable control directory is +not a supported authentication boundary. Use an exact alternate directory when the project is +read-only or several same-account clients need another stable control plane: ```bash lean-beam --root /workspace/a --control-dir /workspace/control ensure --hold @@ -244,11 +246,13 @@ lean-beam --root /workspace/a --control-dir /workspace/control stats ``` Every participant must supply the same `--root` and `--control-dir`; Beam does not search alternate -control directories. `BEAM_CONTROL_ROOT=/writable/base` is the sandbox convenience form and must be -absolute: Beam derives a separate hashed directory for each canonical root below that base. An +control directories. If the exact directory already exists, prepare its `0700` mode explicitly; +Beam will not adopt it by silently changing permissions. `BEAM_CONTROL_ROOT=/writable/base` is the +sandbox convenience form and must be absolute: Beam derives a separate hashed directory for each +canonical root below that base. An explicit control directory is also the intended future location for a statically configured -multi-workspace CLI session. The current public owner command still publishes one frozen workspace, and wrapper mode -does not allow runtime `init_workspace`, `list_workspaces`, or `drop_workspace` requests. The +multi-workspace CLI session. The current public owner command still publishes one frozen workspace, +and wrapper mode does not allow runtime `init_workspace`, `list_workspaces`, or `drop_workspace` requests. The supported semantic `request-stream` also excludes process-wide `shutdown` and `reset_stats`; use the dedicated `lean-beam shutdown` command for lifecycle control. Use a stable external control directory when ownership must remain fenced while the project path is diff --git a/docs/STATUS.md b/docs/STATUS.md index 36a43596..72abc6f3 100644 --- a/docs/STATUS.md +++ b/docs/STATUS.md @@ -214,8 +214,11 @@ Exact event ordering and examples live in available inside project-scoped agent sandboxes. `--control-dir DIR` selects one exact alternate control directory; callers must repeat the same selection for every owner, request, diagnostic, shutdown, and recovery command. `BEAM_CONTROL_ROOT` is the sandbox convenience that derives a - per-root subdirectory below an absolute writable base. Beam makes every selected control - directory account-private (`0700`) before creating its lock or capability descriptor. A stable + per-root subdirectory below an absolute writable base. Beam requires every selected control + directory to be account-private (`0700`) before creating its lock or capability descriptor: it + creates and privatizes a missing leaf, but accepts an existing path only when it is a real, + non-symlinked mode-`0700` directory. It rejects broader existing permissions without changing + them. A stable explicit control directory is also the intended future boundary for a statically configured multi-workspace CLI session; dynamic workspace mutation remains unavailable in wrapper mode. diff --git a/docs/TESTING.md b/docs/TESTING.md index f83ff16b..0371585d 100644 --- a/docs/TESTING.md +++ b/docs/TESTING.md @@ -115,7 +115,8 @@ Current Beam coverage includes: - focused daemon lifecycle coverage in [tests/test-beam-wrapper-daemon.sh](../tests/test-beam-wrapper-daemon.sh), including the no-implicit-start contract, duplicate-owner rejection, Beam and non-Beam endpoint collision safety without cross-project disclosure, authenticated generation probes, mode-`0700` - control-directory and mode-`0600` registry publication, wrong-root recovery rejection with + control-directory and mode-`0600` registry publication, rejection of symlinked or non-private + existing control paths without mutating their targets, wrong-root recovery rejection with byte-for-byte descriptor preservation, unauthorized-shutdown rejection without listener teardown, oversized-frame and first-message limits, a bounded identity probe against a silent non-Beam listener, cross-root unsafe-registry preservation that does not affect the daemon diff --git a/lakefile.lean b/lakefile.lean index 6a9e94d2..58235290 100644 --- a/lakefile.lean +++ b/lakefile.lean @@ -11,12 +11,20 @@ open System package "beam" where +target beamControlDirObj (pkg) : FilePath := do + let srcFile := pkg.dir / "Beam" / "Native" / "control_dir.c" + let oFile := pkg.buildDir / "native" / "control_dir.o" + let srcTarget ← inputTextFile srcFile + buildFileAfterDep oFile srcTarget fun srcFile => do + compileO oFile srcFile #["-I", (← getLeanIncludeDir).toString, "-fPIC"] + lean_lib Beam.LSP where globs := #[.andSubmodules `Beam.LSP] defaultFacets := #[`shared] lean_lib Beam where defaultFacets := #[`shared] + moreLinkObjs := #[beamControlDirObj] lean_lib BeamTest where srcDir := "tests/lean" diff --git a/tests/lib/beam-wrapper-common.sh b/tests/lib/beam-wrapper-common.sh index 32879688..d1e8f98f 100644 --- a/tests/lib/beam-wrapper-common.sh +++ b/tests/lib/beam-wrapper-common.sh @@ -512,6 +512,7 @@ beam_wrapper_prepare_project_root() { rsync -a --exclude='.beam/' tests/save_olean_project/ "$root"/ rm -rf -- "$root/.beam" mkdir -p "$root/.beam" + chmod 700 "$root/.beam" beam_wrapper_register_root "$root" printf '%s\n' "$root" } diff --git a/tests/test-beam-toolchain-compat.sh b/tests/test-beam-toolchain-compat.sh index 55032ada..44e729eb 100644 --- a/tests/test-beam-toolchain-compat.sh +++ b/tests/test-beam-toolchain-compat.sh @@ -219,6 +219,7 @@ prepare_stale_diagnostic_project() { printf '%s\n' "$toolchain" > "$stale_project_root/lean-toolchain" rm -rf -- "$stale_project_root/.beam" mkdir -p "$stale_project_root/.beam" + chmod 700 "$stale_project_root/.beam" } assert_stale_diagnostic_payload() { diff --git a/tests/test-beam-wrapper-daemon.sh b/tests/test-beam-wrapper-daemon.sh index 55f6765f..ba5d5de9 100644 --- a/tests/test-beam-wrapper-daemon.sh +++ b/tests/test-beam-wrapper-daemon.sh @@ -37,6 +37,13 @@ paused_daemon_pid="" busy_pid="" busy_port_file="" +file_mode() { + case "$(uname -s)" in + Darwin) stat -f '%Lp' "$1" ;; + *) stat -c '%a' "$1" ;; + esac +} + start_slow_request() { local root="$1" local label="$2" @@ -151,6 +158,7 @@ for tmp in "$tmp1" "$tmp2"; do rsync -a --exclude='.beam/' tests/save_olean_project/ "$tmp"/ remove_tmp_tree_within "$tmp/.beam" "$tmp" mkdir -p "$tmp/.beam" + chmod 700 "$tmp/.beam" mkdir -p "$tmp/tests/scenario/docs" cp tests/scenario/docs/SlowPoll.lean "$tmp/tests/scenario/docs/SlowPoll.lean" done @@ -192,6 +200,35 @@ if [ -e "$tmp1/.beam" ]; then exit 1 fi +# A control path is an exact security boundary, not a redirect. Reject a symlinked default path +# without changing the target or creating any control files through it. +symlink_control_target="$tmp2/symlink-control-target" +mkdir -p "$symlink_control_target" +chmod 755 "$symlink_control_target" +symlink_target_mode_before="$(file_mode "$symlink_control_target")" +ln -s "$symlink_control_target" "$tmp1/.beam" +if "$beam_script" --root "$tmp1" recover --force \ + > "$tmp1/symlink-control.out" 2> "$tmp1/symlink-control.err"; then + echo "expected a symlinked default control directory to be rejected" >&2 + exit 1 +fi +if ! grep -Fq "symbolic links are not accepted" "$tmp1/symlink-control.err"; then + echo "expected symlinked control rejection to explain the exact-path boundary" >&2 + cat "$tmp1/symlink-control.err" >&2 + exit 1 +fi +if [ "$(file_mode "$symlink_control_target")" != "$symlink_target_mode_before" ]; then + echo "symlinked control rejection changed the target directory mode" >&2 + exit 1 +fi +if find "$symlink_control_target" -mindepth 1 -print -quit | grep -q .; then + echo "symlinked control rejection created files in the target directory" >&2 + find "$symlink_control_target" -mindepth 1 -maxdepth 2 -print >&2 + exit 1 +fi +rm -f -- "$tmp1/.beam" +rmdir "$symlink_control_target" + start_owner() { local root="$1" local label="$2" @@ -318,16 +355,8 @@ if workspace.get("workspaceId") != "beam-cli-project": raise SystemExit(f"unexpected workspace id: {workspace!r}") PY -case "$(uname -s)" in - Darwin) - control_dir_mode="$(stat -f '%Lp' "$tmp1/.beam")" - registry_mode="$(stat -f '%Lp' "$registry")" - ;; - *) - control_dir_mode="$(stat -c '%a' "$tmp1/.beam")" - registry_mode="$(stat -c '%a' "$registry")" - ;; -esac +control_dir_mode="$(file_mode "$tmp1/.beam")" +registry_mode="$(file_mode "$registry")" if [ "$control_dir_mode" != "700" ]; then echo "expected the capability control directory to use mode 700, got $control_dir_mode" >&2 exit 1 @@ -930,7 +959,31 @@ if [ ! -f "$quarantined_registry" ]; then fi explicit_control="$tmp2/shared-control" -mkdir -p "$explicit_control" +nonprivate_control="$tmp2/nonprivate-control" +mkdir -p "$nonprivate_control" +chmod 755 "$nonprivate_control" +nonprivate_mode_before="$(file_mode "$nonprivate_control")" +if "$beam_script" --root "$tmp2" --control-dir "$nonprivate_control" recover --force \ + > "$tmp2/nonprivate-control.out" 2> "$tmp2/nonprivate-control.err"; then + echo "expected an existing non-private control directory to be rejected" >&2 + exit 1 +fi +if ! grep -Fq "existing mode is 0755, expected 0700" "$tmp2/nonprivate-control.err"; then + echo "expected non-private control rejection to explain the required mode" >&2 + cat "$tmp2/nonprivate-control.err" >&2 + exit 1 +fi +if [ "$(file_mode "$nonprivate_control")" != "$nonprivate_mode_before" ]; then + echo "non-private control rejection changed the existing directory mode" >&2 + exit 1 +fi +if find "$nonprivate_control" -mindepth 1 -print -quit | grep -q .; then + echo "non-private control rejection created files in the existing directory" >&2 + find "$nonprivate_control" -mindepth 1 -maxdepth 2 -print >&2 + exit 1 +fi + +# An absent exact control leaf is safe to create and privatize before descriptor publication. "$beam_script" --root "$tmp2" --control-dir "$explicit_control" ensure --hold \ > "$tmp2/explicit-control-owner.out" 2> "$tmp2/explicit-control-owner.err" & hold_pid="$!" @@ -941,6 +994,11 @@ if ! wait_for_nonempty_file "$explicit_registry" "explicit control-directory ses fi explicit_stats="$("$beam_script" --root "$tmp2" --control-dir "$explicit_control" stats)" assert_json_field_equals "explicit control-directory attachment" "$explicit_stats" ok true +if [ "$(file_mode "$explicit_control")" != "700" ] || \ + [ "$(file_mode "$explicit_registry")" != "600" ]; then + echo "expected a newly created explicit control directory and descriptor to use modes 700/600" >&2 + exit 1 +fi if "$beam_script" --root "$tmp2" stats \ > "$tmp2/default-control-stats.out" 2> "$tmp2/default-control-stats.err"; then echo "expected the default and explicit control selections to remain distinct" >&2 From 0660193dca18f1d325102c6e7c6be3c9d2ef25ab Mon Sep 17 00:00:00 2001 From: Emilio Jesus Gallego Arias Date: Sat, 29 Aug 2026 19:05:08 +0200 Subject: [PATCH 28/28] fix: keep Beam-created project state private --- Beam/Cli/RuntimeBundle/Build.lean | 2 +- Beam/Cli/RuntimeBundle/Paths.lean | 31 +++++++++++++++++++++++++++++++ tests/test-beam-install.sh | 9 ++++++++- 3 files changed, 40 insertions(+), 2 deletions(-) diff --git a/Beam/Cli/RuntimeBundle/Build.lean b/Beam/Cli/RuntimeBundle/Build.lean index 4a006c08..b5755b28 100644 --- a/Beam/Cli/RuntimeBundle/Build.lean +++ b/Beam/Cli/RuntimeBundle/Build.lean @@ -270,7 +270,7 @@ def ensureToolchainBundle (root home : System.FilePath) (toolchain : String) : I match ← existingToolchainBundleInAnyForFingerprint? (← installBundleCacheRoots) home toolchain fingerprint with | some bundle => pure bundle | none => - let cacheRoot ← runtimeBundleCacheRoot root + let cacheRoot ← runtimeBundleCacheRootForWrite root ensureToolchainBundleInForFingerprint cacheRoot home toolchain fingerprint def ensureDefaultDaemonHelpers (home : System.FilePath) : IO BundlePaths := do diff --git a/Beam/Cli/RuntimeBundle/Paths.lean b/Beam/Cli/RuntimeBundle/Paths.lean index 0264a59f..22e89efc 100644 --- a/Beam/Cli/RuntimeBundle/Paths.lean +++ b/Beam/Cli/RuntimeBundle/Paths.lean @@ -103,6 +103,37 @@ def runtimeBundleCacheRoot (root : System.FilePath) : IO System.FilePath := do | some path => pure (System.FilePath.mk path) | none => pure (beamStateDir root / runtimeBundlesDirName) +private def privateBeamStateDirRights : IO.FileRight := { + user := { read := true, write := true, execution := true } +} + +/-- +Create the project-local Beam state leaf privately when runtime bundle construction is the first +Beam operation for a project. Existing state directories are never chmodded: wrapper ownership +applies its stricter control-directory validation separately. +-/ +def runtimeBundleCacheRootForWrite (root : System.FilePath) : IO System.FilePath := do + match ← IO.getEnv "BEAM_BUNDLE_DIR" with + | some path => pure (System.FilePath.mk path) + | none => + let stateDir := beamStateDir root + unless ← stateDir.pathExists do + try + IO.FS.createDir stateDir + catch + | .alreadyExists .. => + return stateDir / runtimeBundlesDirName + | err => throw err + try + IO.setAccessRights stateDir privateBeamStateDirRights + catch err => + try + IO.FS.removeDir stateDir + catch _ => + pure () + throw err + pure (stateDir / runtimeBundlesDirName) + def validatedLeanToolchainsPath (home : System.FilePath) : System.FilePath := home / "validated-lean-toolchains" diff --git a/tests/test-beam-install.sh b/tests/test-beam-install.sh index 7ad9a287..a686b77c 100644 --- a/tests/test-beam-install.sh +++ b/tests/test-beam-install.sh @@ -307,7 +307,7 @@ assert_install_rejects_marker() { remove_tmp_file "$marker_err" } -rsync -a --exclude='.git' ./ "$source_checkout"/ +rsync -a --exclude='.git' --exclude='.beam/' ./ "$source_checkout"/ path_no_elan="$(path_without_elan)" if PATH="$path_no_elan" command -v elan >/dev/null 2>&1; then echo "failed to construct a PATH without elan for the negative install test" >&2 @@ -1581,6 +1581,13 @@ if ! printf '%s\n' "$mcp_self_check_out" | grep -q 'workspace: explicit root des printf '%s\n' "$mcp_self_check_out" >&2 exit 1 fi +if [ -d "$project_root/.beam" ]; then + project_beam_mode="$(python3 -c 'import os, stat, sys; print(format(stat.S_IMODE(os.lstat(sys.argv[1]).st_mode), "o"))' "$project_root/.beam")" + if [ "$project_beam_mode" != "700" ]; then + echo "expected Beam-created project state to be private, got mode $project_beam_mode" >&2 + exit 1 + fi +fi unsupported_project_root="$tmp_root/external-project-unsupported" rsync -a --exclude='.beam/' tests/save_olean_project/ "$unsupported_project_root"/