From 982c75f99eca406290c3e288b46e7c4dcf8377ab Mon Sep 17 00:00:00 2001 From: Coolfan Date: Fri, 7 Aug 2026 22:22:55 +0800 Subject: [PATCH 1/7] feat(api): add terminal session recovery proto messages Introduce TerminalSessionRecoveryCandidate/Result/Report/Ack messages so workers can reconcile terminal sessions with Console after reconnect. --- api/gen/go/registry/v1/registry.pb.go | 592 ++++++++++++++++++++------ api/proto/registry/v1/registry.proto | 26 ++ 2 files changed, 494 insertions(+), 124 deletions(-) diff --git a/api/gen/go/registry/v1/registry.pb.go b/api/gen/go/registry/v1/registry.pb.go index 94038c1..a116238 100644 --- a/api/gen/go/registry/v1/registry.pb.go +++ b/api/gen/go/registry/v1/registry.pb.go @@ -21,6 +21,58 @@ const ( _ = protoimpl.EnforceVersion(protoimpl.MaxVersion - 20) ) +type TerminalSessionRecoveryResult_Status int32 + +const ( + TerminalSessionRecoveryResult_STATUS_UNSPECIFIED TerminalSessionRecoveryResult_Status = 0 + TerminalSessionRecoveryResult_RECOVERED TerminalSessionRecoveryResult_Status = 1 + TerminalSessionRecoveryResult_MISSING TerminalSessionRecoveryResult_Status = 2 + TerminalSessionRecoveryResult_INVALID TerminalSessionRecoveryResult_Status = 3 +) + +// Enum value maps for TerminalSessionRecoveryResult_Status. +var ( + TerminalSessionRecoveryResult_Status_name = map[int32]string{ + 0: "STATUS_UNSPECIFIED", + 1: "RECOVERED", + 2: "MISSING", + 3: "INVALID", + } + TerminalSessionRecoveryResult_Status_value = map[string]int32{ + "STATUS_UNSPECIFIED": 0, + "RECOVERED": 1, + "MISSING": 2, + "INVALID": 3, + } +) + +func (x TerminalSessionRecoveryResult_Status) Enum() *TerminalSessionRecoveryResult_Status { + p := new(TerminalSessionRecoveryResult_Status) + *p = x + return p +} + +func (x TerminalSessionRecoveryResult_Status) String() string { + return protoimpl.X.EnumStringOf(x.Descriptor(), protoreflect.EnumNumber(x)) +} + +func (TerminalSessionRecoveryResult_Status) Descriptor() protoreflect.EnumDescriptor { + return file_registry_v1_registry_proto_enumTypes[0].Descriptor() +} + +func (TerminalSessionRecoveryResult_Status) Type() protoreflect.EnumType { + return &file_registry_v1_registry_proto_enumTypes[0] +} + +func (x TerminalSessionRecoveryResult_Status) Number() protoreflect.EnumNumber { + return protoreflect.EnumNumber(x) +} + +// Deprecated: Use TerminalSessionRecoveryResult_Status.Descriptor instead. +func (TerminalSessionRecoveryResult_Status) EnumDescriptor() ([]byte, []int) { + return file_registry_v1_registry_proto_rawDescGZIP(), []int{6, 0} +} + type CapabilityDeclaration struct { state protoimpl.MessageState `protogen:"open.v1"` Name string `protobuf:"bytes,1,opt,name=name,proto3" json:"name,omitempty"` @@ -294,6 +346,7 @@ type ConnectRequest struct { // *ConnectRequest_Hello // *ConnectRequest_Heartbeat // *ConnectRequest_CommandResult + // *ConnectRequest_TerminalSessionRecoveryReport Payload isConnectRequest_Payload `protobuf_oneof:"payload"` unknownFields protoimpl.UnknownFields sizeCache protoimpl.SizeCache @@ -363,6 +416,15 @@ func (x *ConnectRequest) GetCommandResult() *CommandResult { return nil } +func (x *ConnectRequest) GetTerminalSessionRecoveryReport() *TerminalSessionRecoveryReport { + if x != nil { + if x, ok := x.Payload.(*ConnectRequest_TerminalSessionRecoveryReport); ok { + return x.TerminalSessionRecoveryReport + } + } + return nil +} + type isConnectRequest_Payload interface { isConnectRequest_Payload() } @@ -379,23 +441,214 @@ type ConnectRequest_CommandResult struct { CommandResult *CommandResult `protobuf:"bytes,3,opt,name=command_result,json=commandResult,proto3,oneof"` } +type ConnectRequest_TerminalSessionRecoveryReport struct { + TerminalSessionRecoveryReport *TerminalSessionRecoveryReport `protobuf:"bytes,4,opt,name=terminal_session_recovery_report,json=terminalSessionRecoveryReport,proto3,oneof"` +} + func (*ConnectRequest_Hello) isConnectRequest_Payload() {} func (*ConnectRequest_Heartbeat) isConnectRequest_Payload() {} func (*ConnectRequest_CommandResult) isConnectRequest_Payload() {} +func (*ConnectRequest_TerminalSessionRecoveryReport) isConnectRequest_Payload() {} + +type TerminalSessionRecoveryCandidate struct { + state protoimpl.MessageState `protogen:"open.v1"` + SessionId string `protobuf:"bytes,1,opt,name=session_id,json=sessionId,proto3" json:"session_id,omitempty"` + LeaseExpiresUnixMs int64 `protobuf:"varint,2,opt,name=lease_expires_unix_ms,json=leaseExpiresUnixMs,proto3" json:"lease_expires_unix_ms,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *TerminalSessionRecoveryCandidate) Reset() { + *x = TerminalSessionRecoveryCandidate{} + mi := &file_registry_v1_registry_proto_msgTypes[5] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *TerminalSessionRecoveryCandidate) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*TerminalSessionRecoveryCandidate) ProtoMessage() {} + +func (x *TerminalSessionRecoveryCandidate) ProtoReflect() protoreflect.Message { + mi := &file_registry_v1_registry_proto_msgTypes[5] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use TerminalSessionRecoveryCandidate.ProtoReflect.Descriptor instead. +func (*TerminalSessionRecoveryCandidate) Descriptor() ([]byte, []int) { + return file_registry_v1_registry_proto_rawDescGZIP(), []int{5} +} + +func (x *TerminalSessionRecoveryCandidate) GetSessionId() string { + if x != nil { + return x.SessionId + } + return "" +} + +func (x *TerminalSessionRecoveryCandidate) GetLeaseExpiresUnixMs() int64 { + if x != nil { + return x.LeaseExpiresUnixMs + } + return 0 +} + +type TerminalSessionRecoveryResult struct { + state protoimpl.MessageState `protogen:"open.v1"` + SessionId string `protobuf:"bytes,1,opt,name=session_id,json=sessionId,proto3" json:"session_id,omitempty"` + Status TerminalSessionRecoveryResult_Status `protobuf:"varint,2,opt,name=status,proto3,enum=onlyboxes.registry.v1.TerminalSessionRecoveryResult_Status" json:"status,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *TerminalSessionRecoveryResult) Reset() { + *x = TerminalSessionRecoveryResult{} + mi := &file_registry_v1_registry_proto_msgTypes[6] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *TerminalSessionRecoveryResult) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*TerminalSessionRecoveryResult) ProtoMessage() {} + +func (x *TerminalSessionRecoveryResult) ProtoReflect() protoreflect.Message { + mi := &file_registry_v1_registry_proto_msgTypes[6] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use TerminalSessionRecoveryResult.ProtoReflect.Descriptor instead. +func (*TerminalSessionRecoveryResult) Descriptor() ([]byte, []int) { + return file_registry_v1_registry_proto_rawDescGZIP(), []int{6} +} + +func (x *TerminalSessionRecoveryResult) GetSessionId() string { + if x != nil { + return x.SessionId + } + return "" +} + +func (x *TerminalSessionRecoveryResult) GetStatus() TerminalSessionRecoveryResult_Status { + if x != nil { + return x.Status + } + return TerminalSessionRecoveryResult_STATUS_UNSPECIFIED +} + +type TerminalSessionRecoveryReport struct { + state protoimpl.MessageState `protogen:"open.v1"` + Results []*TerminalSessionRecoveryResult `protobuf:"bytes,1,rep,name=results,proto3" json:"results,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *TerminalSessionRecoveryReport) Reset() { + *x = TerminalSessionRecoveryReport{} + mi := &file_registry_v1_registry_proto_msgTypes[7] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *TerminalSessionRecoveryReport) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*TerminalSessionRecoveryReport) ProtoMessage() {} + +func (x *TerminalSessionRecoveryReport) ProtoReflect() protoreflect.Message { + mi := &file_registry_v1_registry_proto_msgTypes[7] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use TerminalSessionRecoveryReport.ProtoReflect.Descriptor instead. +func (*TerminalSessionRecoveryReport) Descriptor() ([]byte, []int) { + return file_registry_v1_registry_proto_rawDescGZIP(), []int{7} +} + +func (x *TerminalSessionRecoveryReport) GetResults() []*TerminalSessionRecoveryResult { + if x != nil { + return x.Results + } + return nil +} + +type TerminalSessionRecoveryAck struct { + state protoimpl.MessageState `protogen:"open.v1"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *TerminalSessionRecoveryAck) Reset() { + *x = TerminalSessionRecoveryAck{} + mi := &file_registry_v1_registry_proto_msgTypes[8] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *TerminalSessionRecoveryAck) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*TerminalSessionRecoveryAck) ProtoMessage() {} + +func (x *TerminalSessionRecoveryAck) ProtoReflect() protoreflect.Message { + mi := &file_registry_v1_registry_proto_msgTypes[8] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use TerminalSessionRecoveryAck.ProtoReflect.Descriptor instead. +func (*TerminalSessionRecoveryAck) Descriptor() ([]byte, []int) { + return file_registry_v1_registry_proto_rawDescGZIP(), []int{8} +} + type ConnectAck struct { - state protoimpl.MessageState `protogen:"open.v1"` - SessionId string `protobuf:"bytes,1,opt,name=session_id,json=sessionId,proto3" json:"session_id,omitempty"` - HeartbeatIntervalSec int32 `protobuf:"varint,3,opt,name=heartbeat_interval_sec,json=heartbeatIntervalSec,proto3" json:"heartbeat_interval_sec,omitempty"` - unknownFields protoimpl.UnknownFields - sizeCache protoimpl.SizeCache + state protoimpl.MessageState `protogen:"open.v1"` + SessionId string `protobuf:"bytes,1,opt,name=session_id,json=sessionId,proto3" json:"session_id,omitempty"` + HeartbeatIntervalSec int32 `protobuf:"varint,3,opt,name=heartbeat_interval_sec,json=heartbeatIntervalSec,proto3" json:"heartbeat_interval_sec,omitempty"` + TerminalSessionRecoveryCandidates []*TerminalSessionRecoveryCandidate `protobuf:"bytes,4,rep,name=terminal_session_recovery_candidates,json=terminalSessionRecoveryCandidates,proto3" json:"terminal_session_recovery_candidates,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache } func (x *ConnectAck) Reset() { *x = ConnectAck{} - mi := &file_registry_v1_registry_proto_msgTypes[5] + mi := &file_registry_v1_registry_proto_msgTypes[9] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -407,7 +660,7 @@ func (x *ConnectAck) String() string { func (*ConnectAck) ProtoMessage() {} func (x *ConnectAck) ProtoReflect() protoreflect.Message { - mi := &file_registry_v1_registry_proto_msgTypes[5] + mi := &file_registry_v1_registry_proto_msgTypes[9] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -420,7 +673,7 @@ func (x *ConnectAck) ProtoReflect() protoreflect.Message { // Deprecated: Use ConnectAck.ProtoReflect.Descriptor instead. func (*ConnectAck) Descriptor() ([]byte, []int) { - return file_registry_v1_registry_proto_rawDescGZIP(), []int{5} + return file_registry_v1_registry_proto_rawDescGZIP(), []int{9} } func (x *ConnectAck) GetSessionId() string { @@ -437,6 +690,13 @@ func (x *ConnectAck) GetHeartbeatIntervalSec() int32 { return 0 } +func (x *ConnectAck) GetTerminalSessionRecoveryCandidates() []*TerminalSessionRecoveryCandidate { + if x != nil { + return x.TerminalSessionRecoveryCandidates + } + return nil +} + type HeartbeatAck struct { state protoimpl.MessageState `protogen:"open.v1"` HeartbeatIntervalSec int32 `protobuf:"varint,2,opt,name=heartbeat_interval_sec,json=heartbeatIntervalSec,proto3" json:"heartbeat_interval_sec,omitempty"` @@ -446,7 +706,7 @@ type HeartbeatAck struct { func (x *HeartbeatAck) Reset() { *x = HeartbeatAck{} - mi := &file_registry_v1_registry_proto_msgTypes[6] + mi := &file_registry_v1_registry_proto_msgTypes[10] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -458,7 +718,7 @@ func (x *HeartbeatAck) String() string { func (*HeartbeatAck) ProtoMessage() {} func (x *HeartbeatAck) ProtoReflect() protoreflect.Message { - mi := &file_registry_v1_registry_proto_msgTypes[6] + mi := &file_registry_v1_registry_proto_msgTypes[10] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -471,7 +731,7 @@ func (x *HeartbeatAck) ProtoReflect() protoreflect.Message { // Deprecated: Use HeartbeatAck.ProtoReflect.Descriptor instead. func (*HeartbeatAck) Descriptor() ([]byte, []int) { - return file_registry_v1_registry_proto_rawDescGZIP(), []int{6} + return file_registry_v1_registry_proto_rawDescGZIP(), []int{10} } func (x *HeartbeatAck) GetHeartbeatIntervalSec() int32 { @@ -493,7 +753,7 @@ type CommandDispatch struct { func (x *CommandDispatch) Reset() { *x = CommandDispatch{} - mi := &file_registry_v1_registry_proto_msgTypes[7] + mi := &file_registry_v1_registry_proto_msgTypes[11] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -505,7 +765,7 @@ func (x *CommandDispatch) String() string { func (*CommandDispatch) ProtoMessage() {} func (x *CommandDispatch) ProtoReflect() protoreflect.Message { - mi := &file_registry_v1_registry_proto_msgTypes[7] + mi := &file_registry_v1_registry_proto_msgTypes[11] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -518,7 +778,7 @@ func (x *CommandDispatch) ProtoReflect() protoreflect.Message { // Deprecated: Use CommandDispatch.ProtoReflect.Descriptor instead. func (*CommandDispatch) Descriptor() ([]byte, []int) { - return file_registry_v1_registry_proto_rawDescGZIP(), []int{7} + return file_registry_v1_registry_proto_rawDescGZIP(), []int{11} } func (x *CommandDispatch) GetCommandId() string { @@ -559,7 +819,7 @@ type CommandError struct { func (x *CommandError) Reset() { *x = CommandError{} - mi := &file_registry_v1_registry_proto_msgTypes[8] + mi := &file_registry_v1_registry_proto_msgTypes[12] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -571,7 +831,7 @@ func (x *CommandError) String() string { func (*CommandError) ProtoMessage() {} func (x *CommandError) ProtoReflect() protoreflect.Message { - mi := &file_registry_v1_registry_proto_msgTypes[8] + mi := &file_registry_v1_registry_proto_msgTypes[12] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -584,7 +844,7 @@ func (x *CommandError) ProtoReflect() protoreflect.Message { // Deprecated: Use CommandError.ProtoReflect.Descriptor instead. func (*CommandError) Descriptor() ([]byte, []int) { - return file_registry_v1_registry_proto_rawDescGZIP(), []int{8} + return file_registry_v1_registry_proto_rawDescGZIP(), []int{12} } func (x *CommandError) GetCode() string { @@ -613,7 +873,7 @@ type CommandResult struct { func (x *CommandResult) Reset() { *x = CommandResult{} - mi := &file_registry_v1_registry_proto_msgTypes[9] + mi := &file_registry_v1_registry_proto_msgTypes[13] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -625,7 +885,7 @@ func (x *CommandResult) String() string { func (*CommandResult) ProtoMessage() {} func (x *CommandResult) ProtoReflect() protoreflect.Message { - mi := &file_registry_v1_registry_proto_msgTypes[9] + mi := &file_registry_v1_registry_proto_msgTypes[13] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -638,7 +898,7 @@ func (x *CommandResult) ProtoReflect() protoreflect.Message { // Deprecated: Use CommandResult.ProtoReflect.Descriptor instead. func (*CommandResult) Descriptor() ([]byte, []int) { - return file_registry_v1_registry_proto_rawDescGZIP(), []int{9} + return file_registry_v1_registry_proto_rawDescGZIP(), []int{13} } func (x *CommandResult) GetCommandId() string { @@ -676,6 +936,7 @@ type ConnectResponse struct { // *ConnectResponse_ConnectAck // *ConnectResponse_HeartbeatAck // *ConnectResponse_CommandDispatch + // *ConnectResponse_TerminalSessionRecoveryAck Payload isConnectResponse_Payload `protobuf_oneof:"payload"` unknownFields protoimpl.UnknownFields sizeCache protoimpl.SizeCache @@ -683,7 +944,7 @@ type ConnectResponse struct { func (x *ConnectResponse) Reset() { *x = ConnectResponse{} - mi := &file_registry_v1_registry_proto_msgTypes[10] + mi := &file_registry_v1_registry_proto_msgTypes[14] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -695,7 +956,7 @@ func (x *ConnectResponse) String() string { func (*ConnectResponse) ProtoMessage() {} func (x *ConnectResponse) ProtoReflect() protoreflect.Message { - mi := &file_registry_v1_registry_proto_msgTypes[10] + mi := &file_registry_v1_registry_proto_msgTypes[14] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -708,7 +969,7 @@ func (x *ConnectResponse) ProtoReflect() protoreflect.Message { // Deprecated: Use ConnectResponse.ProtoReflect.Descriptor instead. func (*ConnectResponse) Descriptor() ([]byte, []int) { - return file_registry_v1_registry_proto_rawDescGZIP(), []int{10} + return file_registry_v1_registry_proto_rawDescGZIP(), []int{14} } func (x *ConnectResponse) GetPayload() isConnectResponse_Payload { @@ -745,6 +1006,15 @@ func (x *ConnectResponse) GetCommandDispatch() *CommandDispatch { return nil } +func (x *ConnectResponse) GetTerminalSessionRecoveryAck() *TerminalSessionRecoveryAck { + if x != nil { + if x, ok := x.Payload.(*ConnectResponse_TerminalSessionRecoveryAck); ok { + return x.TerminalSessionRecoveryAck + } + } + return nil +} + type isConnectResponse_Payload interface { isConnectResponse_Payload() } @@ -761,12 +1031,18 @@ type ConnectResponse_CommandDispatch struct { CommandDispatch *CommandDispatch `protobuf:"bytes,4,opt,name=command_dispatch,json=commandDispatch,proto3,oneof"` } +type ConnectResponse_TerminalSessionRecoveryAck struct { + TerminalSessionRecoveryAck *TerminalSessionRecoveryAck `protobuf:"bytes,5,opt,name=terminal_session_recovery_ack,json=terminalSessionRecoveryAck,proto3,oneof"` +} + func (*ConnectResponse_ConnectAck) isConnectResponse_Payload() {} func (*ConnectResponse_HeartbeatAck) isConnectResponse_Payload() {} func (*ConnectResponse_CommandDispatch) isConnectResponse_Payload() {} +func (*ConnectResponse_TerminalSessionRecoveryAck) isConnectResponse_Payload() {} + var File_registry_v1_registry_proto protoreflect.FileDescriptor var file_registry_v1_registry_proto_rawDesc = string([]byte{ @@ -824,7 +1100,7 @@ var file_registry_v1_registry_proto_rawDesc = string([]byte{ 0x73, 0x73, 0x69, 0x6f, 0x6e, 0x49, 0x64, 0x12, 0x30, 0x0a, 0x14, 0x61, 0x63, 0x74, 0x69, 0x76, 0x65, 0x5f, 0x73, 0x65, 0x73, 0x73, 0x69, 0x6f, 0x6e, 0x5f, 0x63, 0x6f, 0x75, 0x6e, 0x74, 0x18, 0x03, 0x20, 0x01, 0x28, 0x05, 0x52, 0x12, 0x61, 0x63, 0x74, 0x69, 0x76, 0x65, 0x53, 0x65, 0x73, - 0x73, 0x69, 0x6f, 0x6e, 0x43, 0x6f, 0x75, 0x6e, 0x74, 0x22, 0xee, 0x01, 0x0a, 0x0e, 0x43, 0x6f, + 0x73, 0x69, 0x6f, 0x6e, 0x43, 0x6f, 0x75, 0x6e, 0x74, 0x22, 0xef, 0x02, 0x0a, 0x0e, 0x43, 0x6f, 0x6e, 0x6e, 0x65, 0x63, 0x74, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x3b, 0x0a, 0x05, 0x68, 0x65, 0x6c, 0x6c, 0x6f, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x23, 0x2e, 0x6f, 0x6e, 0x6c, 0x79, 0x62, 0x6f, 0x78, 0x65, 0x73, 0x2e, 0x72, 0x65, 0x67, 0x69, 0x73, 0x74, 0x72, 0x79, @@ -838,73 +1114,127 @@ var file_registry_v1_registry_proto_rawDesc = string([]byte{ 0x6c, 0x74, 0x18, 0x03, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x24, 0x2e, 0x6f, 0x6e, 0x6c, 0x79, 0x62, 0x6f, 0x78, 0x65, 0x73, 0x2e, 0x72, 0x65, 0x67, 0x69, 0x73, 0x74, 0x72, 0x79, 0x2e, 0x76, 0x31, 0x2e, 0x43, 0x6f, 0x6d, 0x6d, 0x61, 0x6e, 0x64, 0x52, 0x65, 0x73, 0x75, 0x6c, 0x74, 0x48, 0x00, - 0x52, 0x0d, 0x63, 0x6f, 0x6d, 0x6d, 0x61, 0x6e, 0x64, 0x52, 0x65, 0x73, 0x75, 0x6c, 0x74, 0x42, - 0x09, 0x0a, 0x07, 0x70, 0x61, 0x79, 0x6c, 0x6f, 0x61, 0x64, 0x22, 0x61, 0x0a, 0x0a, 0x43, 0x6f, - 0x6e, 0x6e, 0x65, 0x63, 0x74, 0x41, 0x63, 0x6b, 0x12, 0x1d, 0x0a, 0x0a, 0x73, 0x65, 0x73, 0x73, - 0x69, 0x6f, 0x6e, 0x5f, 0x69, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x09, 0x73, 0x65, - 0x73, 0x73, 0x69, 0x6f, 0x6e, 0x49, 0x64, 0x12, 0x34, 0x0a, 0x16, 0x68, 0x65, 0x61, 0x72, 0x74, - 0x62, 0x65, 0x61, 0x74, 0x5f, 0x69, 0x6e, 0x74, 0x65, 0x72, 0x76, 0x61, 0x6c, 0x5f, 0x73, 0x65, - 0x63, 0x18, 0x03, 0x20, 0x01, 0x28, 0x05, 0x52, 0x14, 0x68, 0x65, 0x61, 0x72, 0x74, 0x62, 0x65, - 0x61, 0x74, 0x49, 0x6e, 0x74, 0x65, 0x72, 0x76, 0x61, 0x6c, 0x53, 0x65, 0x63, 0x22, 0x44, 0x0a, - 0x0c, 0x48, 0x65, 0x61, 0x72, 0x74, 0x62, 0x65, 0x61, 0x74, 0x41, 0x63, 0x6b, 0x12, 0x34, 0x0a, - 0x16, 0x68, 0x65, 0x61, 0x72, 0x74, 0x62, 0x65, 0x61, 0x74, 0x5f, 0x69, 0x6e, 0x74, 0x65, 0x72, - 0x76, 0x61, 0x6c, 0x5f, 0x73, 0x65, 0x63, 0x18, 0x02, 0x20, 0x01, 0x28, 0x05, 0x52, 0x14, 0x68, - 0x65, 0x61, 0x72, 0x74, 0x62, 0x65, 0x61, 0x74, 0x49, 0x6e, 0x74, 0x65, 0x72, 0x76, 0x61, 0x6c, - 0x53, 0x65, 0x63, 0x22, 0x9d, 0x01, 0x0a, 0x0f, 0x43, 0x6f, 0x6d, 0x6d, 0x61, 0x6e, 0x64, 0x44, - 0x69, 0x73, 0x70, 0x61, 0x74, 0x63, 0x68, 0x12, 0x1d, 0x0a, 0x0a, 0x63, 0x6f, 0x6d, 0x6d, 0x61, - 0x6e, 0x64, 0x5f, 0x69, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x09, 0x63, 0x6f, 0x6d, - 0x6d, 0x61, 0x6e, 0x64, 0x49, 0x64, 0x12, 0x1e, 0x0a, 0x0a, 0x63, 0x61, 0x70, 0x61, 0x62, 0x69, - 0x6c, 0x69, 0x74, 0x79, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0a, 0x63, 0x61, 0x70, 0x61, - 0x62, 0x69, 0x6c, 0x69, 0x74, 0x79, 0x12, 0x21, 0x0a, 0x0c, 0x70, 0x61, 0x79, 0x6c, 0x6f, 0x61, - 0x64, 0x5f, 0x6a, 0x73, 0x6f, 0x6e, 0x18, 0x04, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x0b, 0x70, 0x61, - 0x79, 0x6c, 0x6f, 0x61, 0x64, 0x4a, 0x73, 0x6f, 0x6e, 0x12, 0x28, 0x0a, 0x10, 0x64, 0x65, 0x61, - 0x64, 0x6c, 0x69, 0x6e, 0x65, 0x5f, 0x75, 0x6e, 0x69, 0x78, 0x5f, 0x6d, 0x73, 0x18, 0x05, 0x20, - 0x01, 0x28, 0x03, 0x52, 0x0e, 0x64, 0x65, 0x61, 0x64, 0x6c, 0x69, 0x6e, 0x65, 0x55, 0x6e, 0x69, - 0x78, 0x4d, 0x73, 0x22, 0x3c, 0x0a, 0x0c, 0x43, 0x6f, 0x6d, 0x6d, 0x61, 0x6e, 0x64, 0x45, 0x72, - 0x72, 0x6f, 0x72, 0x12, 0x12, 0x0a, 0x04, 0x63, 0x6f, 0x64, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, - 0x09, 0x52, 0x04, 0x63, 0x6f, 0x64, 0x65, 0x12, 0x18, 0x0a, 0x07, 0x6d, 0x65, 0x73, 0x73, 0x61, - 0x67, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x07, 0x6d, 0x65, 0x73, 0x73, 0x61, 0x67, - 0x65, 0x22, 0xb8, 0x01, 0x0a, 0x0d, 0x43, 0x6f, 0x6d, 0x6d, 0x61, 0x6e, 0x64, 0x52, 0x65, 0x73, - 0x75, 0x6c, 0x74, 0x12, 0x1d, 0x0a, 0x0a, 0x63, 0x6f, 0x6d, 0x6d, 0x61, 0x6e, 0x64, 0x5f, 0x69, - 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x09, 0x63, 0x6f, 0x6d, 0x6d, 0x61, 0x6e, 0x64, - 0x49, 0x64, 0x12, 0x39, 0x0a, 0x05, 0x65, 0x72, 0x72, 0x6f, 0x72, 0x18, 0x03, 0x20, 0x01, 0x28, - 0x0b, 0x32, 0x23, 0x2e, 0x6f, 0x6e, 0x6c, 0x79, 0x62, 0x6f, 0x78, 0x65, 0x73, 0x2e, 0x72, 0x65, - 0x67, 0x69, 0x73, 0x74, 0x72, 0x79, 0x2e, 0x76, 0x31, 0x2e, 0x43, 0x6f, 0x6d, 0x6d, 0x61, 0x6e, - 0x64, 0x45, 0x72, 0x72, 0x6f, 0x72, 0x52, 0x05, 0x65, 0x72, 0x72, 0x6f, 0x72, 0x12, 0x21, 0x0a, - 0x0c, 0x70, 0x61, 0x79, 0x6c, 0x6f, 0x61, 0x64, 0x5f, 0x6a, 0x73, 0x6f, 0x6e, 0x18, 0x04, 0x20, - 0x01, 0x28, 0x0c, 0x52, 0x0b, 0x70, 0x61, 0x79, 0x6c, 0x6f, 0x61, 0x64, 0x4a, 0x73, 0x6f, 0x6e, - 0x12, 0x2a, 0x0a, 0x11, 0x63, 0x6f, 0x6d, 0x70, 0x6c, 0x65, 0x74, 0x65, 0x64, 0x5f, 0x75, 0x6e, - 0x69, 0x78, 0x5f, 0x6d, 0x73, 0x18, 0x05, 0x20, 0x01, 0x28, 0x03, 0x52, 0x0f, 0x63, 0x6f, 0x6d, - 0x70, 0x6c, 0x65, 0x74, 0x65, 0x64, 0x55, 0x6e, 0x69, 0x78, 0x4d, 0x73, 0x22, 0x83, 0x02, 0x0a, - 0x0f, 0x43, 0x6f, 0x6e, 0x6e, 0x65, 0x63, 0x74, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, - 0x12, 0x44, 0x0a, 0x0b, 0x63, 0x6f, 0x6e, 0x6e, 0x65, 0x63, 0x74, 0x5f, 0x61, 0x63, 0x6b, 0x18, - 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x21, 0x2e, 0x6f, 0x6e, 0x6c, 0x79, 0x62, 0x6f, 0x78, 0x65, - 0x73, 0x2e, 0x72, 0x65, 0x67, 0x69, 0x73, 0x74, 0x72, 0x79, 0x2e, 0x76, 0x31, 0x2e, 0x43, 0x6f, - 0x6e, 0x6e, 0x65, 0x63, 0x74, 0x41, 0x63, 0x6b, 0x48, 0x00, 0x52, 0x0a, 0x63, 0x6f, 0x6e, 0x6e, - 0x65, 0x63, 0x74, 0x41, 0x63, 0x6b, 0x12, 0x4a, 0x0a, 0x0d, 0x68, 0x65, 0x61, 0x72, 0x74, 0x62, - 0x65, 0x61, 0x74, 0x5f, 0x61, 0x63, 0x6b, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x23, 0x2e, - 0x6f, 0x6e, 0x6c, 0x79, 0x62, 0x6f, 0x78, 0x65, 0x73, 0x2e, 0x72, 0x65, 0x67, 0x69, 0x73, 0x74, - 0x72, 0x79, 0x2e, 0x76, 0x31, 0x2e, 0x48, 0x65, 0x61, 0x72, 0x74, 0x62, 0x65, 0x61, 0x74, 0x41, - 0x63, 0x6b, 0x48, 0x00, 0x52, 0x0c, 0x68, 0x65, 0x61, 0x72, 0x74, 0x62, 0x65, 0x61, 0x74, 0x41, - 0x63, 0x6b, 0x12, 0x53, 0x0a, 0x10, 0x63, 0x6f, 0x6d, 0x6d, 0x61, 0x6e, 0x64, 0x5f, 0x64, 0x69, - 0x73, 0x70, 0x61, 0x74, 0x63, 0x68, 0x18, 0x04, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x26, 0x2e, 0x6f, - 0x6e, 0x6c, 0x79, 0x62, 0x6f, 0x78, 0x65, 0x73, 0x2e, 0x72, 0x65, 0x67, 0x69, 0x73, 0x74, 0x72, - 0x79, 0x2e, 0x76, 0x31, 0x2e, 0x43, 0x6f, 0x6d, 0x6d, 0x61, 0x6e, 0x64, 0x44, 0x69, 0x73, 0x70, - 0x61, 0x74, 0x63, 0x68, 0x48, 0x00, 0x52, 0x0f, 0x63, 0x6f, 0x6d, 0x6d, 0x61, 0x6e, 0x64, 0x44, - 0x69, 0x73, 0x70, 0x61, 0x74, 0x63, 0x68, 0x42, 0x09, 0x0a, 0x07, 0x70, 0x61, 0x79, 0x6c, 0x6f, - 0x61, 0x64, 0x32, 0x75, 0x0a, 0x15, 0x57, 0x6f, 0x72, 0x6b, 0x65, 0x72, 0x52, 0x65, 0x67, 0x69, - 0x73, 0x74, 0x72, 0x79, 0x53, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x12, 0x5c, 0x0a, 0x07, 0x43, - 0x6f, 0x6e, 0x6e, 0x65, 0x63, 0x74, 0x12, 0x25, 0x2e, 0x6f, 0x6e, 0x6c, 0x79, 0x62, 0x6f, 0x78, - 0x65, 0x73, 0x2e, 0x72, 0x65, 0x67, 0x69, 0x73, 0x74, 0x72, 0x79, 0x2e, 0x76, 0x31, 0x2e, 0x43, - 0x6f, 0x6e, 0x6e, 0x65, 0x63, 0x74, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x26, 0x2e, - 0x6f, 0x6e, 0x6c, 0x79, 0x62, 0x6f, 0x78, 0x65, 0x73, 0x2e, 0x72, 0x65, 0x67, 0x69, 0x73, 0x74, - 0x72, 0x79, 0x2e, 0x76, 0x31, 0x2e, 0x43, 0x6f, 0x6e, 0x6e, 0x65, 0x63, 0x74, 0x52, 0x65, 0x73, - 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x28, 0x01, 0x30, 0x01, 0x42, 0x42, 0x5a, 0x40, 0x67, 0x69, 0x74, - 0x68, 0x75, 0x62, 0x2e, 0x63, 0x6f, 0x6d, 0x2f, 0x6f, 0x6e, 0x6c, 0x79, 0x62, 0x6f, 0x78, 0x65, - 0x73, 0x2f, 0x6f, 0x6e, 0x6c, 0x79, 0x62, 0x6f, 0x78, 0x65, 0x73, 0x2f, 0x61, 0x70, 0x69, 0x2f, - 0x67, 0x65, 0x6e, 0x2f, 0x67, 0x6f, 0x2f, 0x72, 0x65, 0x67, 0x69, 0x73, 0x74, 0x72, 0x79, 0x2f, - 0x76, 0x31, 0x3b, 0x72, 0x65, 0x67, 0x69, 0x73, 0x74, 0x72, 0x79, 0x76, 0x31, 0x62, 0x06, 0x70, - 0x72, 0x6f, 0x74, 0x6f, 0x33, + 0x52, 0x0d, 0x63, 0x6f, 0x6d, 0x6d, 0x61, 0x6e, 0x64, 0x52, 0x65, 0x73, 0x75, 0x6c, 0x74, 0x12, + 0x7f, 0x0a, 0x20, 0x74, 0x65, 0x72, 0x6d, 0x69, 0x6e, 0x61, 0x6c, 0x5f, 0x73, 0x65, 0x73, 0x73, + 0x69, 0x6f, 0x6e, 0x5f, 0x72, 0x65, 0x63, 0x6f, 0x76, 0x65, 0x72, 0x79, 0x5f, 0x72, 0x65, 0x70, + 0x6f, 0x72, 0x74, 0x18, 0x04, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x34, 0x2e, 0x6f, 0x6e, 0x6c, 0x79, + 0x62, 0x6f, 0x78, 0x65, 0x73, 0x2e, 0x72, 0x65, 0x67, 0x69, 0x73, 0x74, 0x72, 0x79, 0x2e, 0x76, + 0x31, 0x2e, 0x54, 0x65, 0x72, 0x6d, 0x69, 0x6e, 0x61, 0x6c, 0x53, 0x65, 0x73, 0x73, 0x69, 0x6f, + 0x6e, 0x52, 0x65, 0x63, 0x6f, 0x76, 0x65, 0x72, 0x79, 0x52, 0x65, 0x70, 0x6f, 0x72, 0x74, 0x48, + 0x00, 0x52, 0x1d, 0x74, 0x65, 0x72, 0x6d, 0x69, 0x6e, 0x61, 0x6c, 0x53, 0x65, 0x73, 0x73, 0x69, + 0x6f, 0x6e, 0x52, 0x65, 0x63, 0x6f, 0x76, 0x65, 0x72, 0x79, 0x52, 0x65, 0x70, 0x6f, 0x72, 0x74, + 0x42, 0x09, 0x0a, 0x07, 0x70, 0x61, 0x79, 0x6c, 0x6f, 0x61, 0x64, 0x22, 0x74, 0x0a, 0x20, 0x54, + 0x65, 0x72, 0x6d, 0x69, 0x6e, 0x61, 0x6c, 0x53, 0x65, 0x73, 0x73, 0x69, 0x6f, 0x6e, 0x52, 0x65, + 0x63, 0x6f, 0x76, 0x65, 0x72, 0x79, 0x43, 0x61, 0x6e, 0x64, 0x69, 0x64, 0x61, 0x74, 0x65, 0x12, + 0x1d, 0x0a, 0x0a, 0x73, 0x65, 0x73, 0x73, 0x69, 0x6f, 0x6e, 0x5f, 0x69, 0x64, 0x18, 0x01, 0x20, + 0x01, 0x28, 0x09, 0x52, 0x09, 0x73, 0x65, 0x73, 0x73, 0x69, 0x6f, 0x6e, 0x49, 0x64, 0x12, 0x31, + 0x0a, 0x15, 0x6c, 0x65, 0x61, 0x73, 0x65, 0x5f, 0x65, 0x78, 0x70, 0x69, 0x72, 0x65, 0x73, 0x5f, + 0x75, 0x6e, 0x69, 0x78, 0x5f, 0x6d, 0x73, 0x18, 0x02, 0x20, 0x01, 0x28, 0x03, 0x52, 0x12, 0x6c, + 0x65, 0x61, 0x73, 0x65, 0x45, 0x78, 0x70, 0x69, 0x72, 0x65, 0x73, 0x55, 0x6e, 0x69, 0x78, 0x4d, + 0x73, 0x22, 0xde, 0x01, 0x0a, 0x1d, 0x54, 0x65, 0x72, 0x6d, 0x69, 0x6e, 0x61, 0x6c, 0x53, 0x65, + 0x73, 0x73, 0x69, 0x6f, 0x6e, 0x52, 0x65, 0x63, 0x6f, 0x76, 0x65, 0x72, 0x79, 0x52, 0x65, 0x73, + 0x75, 0x6c, 0x74, 0x12, 0x1d, 0x0a, 0x0a, 0x73, 0x65, 0x73, 0x73, 0x69, 0x6f, 0x6e, 0x5f, 0x69, + 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x09, 0x73, 0x65, 0x73, 0x73, 0x69, 0x6f, 0x6e, + 0x49, 0x64, 0x12, 0x53, 0x0a, 0x06, 0x73, 0x74, 0x61, 0x74, 0x75, 0x73, 0x18, 0x02, 0x20, 0x01, + 0x28, 0x0e, 0x32, 0x3b, 0x2e, 0x6f, 0x6e, 0x6c, 0x79, 0x62, 0x6f, 0x78, 0x65, 0x73, 0x2e, 0x72, + 0x65, 0x67, 0x69, 0x73, 0x74, 0x72, 0x79, 0x2e, 0x76, 0x31, 0x2e, 0x54, 0x65, 0x72, 0x6d, 0x69, + 0x6e, 0x61, 0x6c, 0x53, 0x65, 0x73, 0x73, 0x69, 0x6f, 0x6e, 0x52, 0x65, 0x63, 0x6f, 0x76, 0x65, + 0x72, 0x79, 0x52, 0x65, 0x73, 0x75, 0x6c, 0x74, 0x2e, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x52, + 0x06, 0x73, 0x74, 0x61, 0x74, 0x75, 0x73, 0x22, 0x49, 0x0a, 0x06, 0x53, 0x74, 0x61, 0x74, 0x75, + 0x73, 0x12, 0x16, 0x0a, 0x12, 0x53, 0x54, 0x41, 0x54, 0x55, 0x53, 0x5f, 0x55, 0x4e, 0x53, 0x50, + 0x45, 0x43, 0x49, 0x46, 0x49, 0x45, 0x44, 0x10, 0x00, 0x12, 0x0d, 0x0a, 0x09, 0x52, 0x45, 0x43, + 0x4f, 0x56, 0x45, 0x52, 0x45, 0x44, 0x10, 0x01, 0x12, 0x0b, 0x0a, 0x07, 0x4d, 0x49, 0x53, 0x53, + 0x49, 0x4e, 0x47, 0x10, 0x02, 0x12, 0x0b, 0x0a, 0x07, 0x49, 0x4e, 0x56, 0x41, 0x4c, 0x49, 0x44, + 0x10, 0x03, 0x22, 0x6f, 0x0a, 0x1d, 0x54, 0x65, 0x72, 0x6d, 0x69, 0x6e, 0x61, 0x6c, 0x53, 0x65, + 0x73, 0x73, 0x69, 0x6f, 0x6e, 0x52, 0x65, 0x63, 0x6f, 0x76, 0x65, 0x72, 0x79, 0x52, 0x65, 0x70, + 0x6f, 0x72, 0x74, 0x12, 0x4e, 0x0a, 0x07, 0x72, 0x65, 0x73, 0x75, 0x6c, 0x74, 0x73, 0x18, 0x01, + 0x20, 0x03, 0x28, 0x0b, 0x32, 0x34, 0x2e, 0x6f, 0x6e, 0x6c, 0x79, 0x62, 0x6f, 0x78, 0x65, 0x73, + 0x2e, 0x72, 0x65, 0x67, 0x69, 0x73, 0x74, 0x72, 0x79, 0x2e, 0x76, 0x31, 0x2e, 0x54, 0x65, 0x72, + 0x6d, 0x69, 0x6e, 0x61, 0x6c, 0x53, 0x65, 0x73, 0x73, 0x69, 0x6f, 0x6e, 0x52, 0x65, 0x63, 0x6f, + 0x76, 0x65, 0x72, 0x79, 0x52, 0x65, 0x73, 0x75, 0x6c, 0x74, 0x52, 0x07, 0x72, 0x65, 0x73, 0x75, + 0x6c, 0x74, 0x73, 0x22, 0x1c, 0x0a, 0x1a, 0x54, 0x65, 0x72, 0x6d, 0x69, 0x6e, 0x61, 0x6c, 0x53, + 0x65, 0x73, 0x73, 0x69, 0x6f, 0x6e, 0x52, 0x65, 0x63, 0x6f, 0x76, 0x65, 0x72, 0x79, 0x41, 0x63, + 0x6b, 0x22, 0xec, 0x01, 0x0a, 0x0a, 0x43, 0x6f, 0x6e, 0x6e, 0x65, 0x63, 0x74, 0x41, 0x63, 0x6b, + 0x12, 0x1d, 0x0a, 0x0a, 0x73, 0x65, 0x73, 0x73, 0x69, 0x6f, 0x6e, 0x5f, 0x69, 0x64, 0x18, 0x01, + 0x20, 0x01, 0x28, 0x09, 0x52, 0x09, 0x73, 0x65, 0x73, 0x73, 0x69, 0x6f, 0x6e, 0x49, 0x64, 0x12, + 0x34, 0x0a, 0x16, 0x68, 0x65, 0x61, 0x72, 0x74, 0x62, 0x65, 0x61, 0x74, 0x5f, 0x69, 0x6e, 0x74, + 0x65, 0x72, 0x76, 0x61, 0x6c, 0x5f, 0x73, 0x65, 0x63, 0x18, 0x03, 0x20, 0x01, 0x28, 0x05, 0x52, + 0x14, 0x68, 0x65, 0x61, 0x72, 0x74, 0x62, 0x65, 0x61, 0x74, 0x49, 0x6e, 0x74, 0x65, 0x72, 0x76, + 0x61, 0x6c, 0x53, 0x65, 0x63, 0x12, 0x88, 0x01, 0x0a, 0x24, 0x74, 0x65, 0x72, 0x6d, 0x69, 0x6e, + 0x61, 0x6c, 0x5f, 0x73, 0x65, 0x73, 0x73, 0x69, 0x6f, 0x6e, 0x5f, 0x72, 0x65, 0x63, 0x6f, 0x76, + 0x65, 0x72, 0x79, 0x5f, 0x63, 0x61, 0x6e, 0x64, 0x69, 0x64, 0x61, 0x74, 0x65, 0x73, 0x18, 0x04, + 0x20, 0x03, 0x28, 0x0b, 0x32, 0x37, 0x2e, 0x6f, 0x6e, 0x6c, 0x79, 0x62, 0x6f, 0x78, 0x65, 0x73, + 0x2e, 0x72, 0x65, 0x67, 0x69, 0x73, 0x74, 0x72, 0x79, 0x2e, 0x76, 0x31, 0x2e, 0x54, 0x65, 0x72, + 0x6d, 0x69, 0x6e, 0x61, 0x6c, 0x53, 0x65, 0x73, 0x73, 0x69, 0x6f, 0x6e, 0x52, 0x65, 0x63, 0x6f, + 0x76, 0x65, 0x72, 0x79, 0x43, 0x61, 0x6e, 0x64, 0x69, 0x64, 0x61, 0x74, 0x65, 0x52, 0x21, 0x74, + 0x65, 0x72, 0x6d, 0x69, 0x6e, 0x61, 0x6c, 0x53, 0x65, 0x73, 0x73, 0x69, 0x6f, 0x6e, 0x52, 0x65, + 0x63, 0x6f, 0x76, 0x65, 0x72, 0x79, 0x43, 0x61, 0x6e, 0x64, 0x69, 0x64, 0x61, 0x74, 0x65, 0x73, + 0x22, 0x44, 0x0a, 0x0c, 0x48, 0x65, 0x61, 0x72, 0x74, 0x62, 0x65, 0x61, 0x74, 0x41, 0x63, 0x6b, + 0x12, 0x34, 0x0a, 0x16, 0x68, 0x65, 0x61, 0x72, 0x74, 0x62, 0x65, 0x61, 0x74, 0x5f, 0x69, 0x6e, + 0x74, 0x65, 0x72, 0x76, 0x61, 0x6c, 0x5f, 0x73, 0x65, 0x63, 0x18, 0x02, 0x20, 0x01, 0x28, 0x05, + 0x52, 0x14, 0x68, 0x65, 0x61, 0x72, 0x74, 0x62, 0x65, 0x61, 0x74, 0x49, 0x6e, 0x74, 0x65, 0x72, + 0x76, 0x61, 0x6c, 0x53, 0x65, 0x63, 0x22, 0x9d, 0x01, 0x0a, 0x0f, 0x43, 0x6f, 0x6d, 0x6d, 0x61, + 0x6e, 0x64, 0x44, 0x69, 0x73, 0x70, 0x61, 0x74, 0x63, 0x68, 0x12, 0x1d, 0x0a, 0x0a, 0x63, 0x6f, + 0x6d, 0x6d, 0x61, 0x6e, 0x64, 0x5f, 0x69, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x09, + 0x63, 0x6f, 0x6d, 0x6d, 0x61, 0x6e, 0x64, 0x49, 0x64, 0x12, 0x1e, 0x0a, 0x0a, 0x63, 0x61, 0x70, + 0x61, 0x62, 0x69, 0x6c, 0x69, 0x74, 0x79, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0a, 0x63, + 0x61, 0x70, 0x61, 0x62, 0x69, 0x6c, 0x69, 0x74, 0x79, 0x12, 0x21, 0x0a, 0x0c, 0x70, 0x61, 0x79, + 0x6c, 0x6f, 0x61, 0x64, 0x5f, 0x6a, 0x73, 0x6f, 0x6e, 0x18, 0x04, 0x20, 0x01, 0x28, 0x0c, 0x52, + 0x0b, 0x70, 0x61, 0x79, 0x6c, 0x6f, 0x61, 0x64, 0x4a, 0x73, 0x6f, 0x6e, 0x12, 0x28, 0x0a, 0x10, + 0x64, 0x65, 0x61, 0x64, 0x6c, 0x69, 0x6e, 0x65, 0x5f, 0x75, 0x6e, 0x69, 0x78, 0x5f, 0x6d, 0x73, + 0x18, 0x05, 0x20, 0x01, 0x28, 0x03, 0x52, 0x0e, 0x64, 0x65, 0x61, 0x64, 0x6c, 0x69, 0x6e, 0x65, + 0x55, 0x6e, 0x69, 0x78, 0x4d, 0x73, 0x22, 0x3c, 0x0a, 0x0c, 0x43, 0x6f, 0x6d, 0x6d, 0x61, 0x6e, + 0x64, 0x45, 0x72, 0x72, 0x6f, 0x72, 0x12, 0x12, 0x0a, 0x04, 0x63, 0x6f, 0x64, 0x65, 0x18, 0x01, + 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x63, 0x6f, 0x64, 0x65, 0x12, 0x18, 0x0a, 0x07, 0x6d, 0x65, + 0x73, 0x73, 0x61, 0x67, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x07, 0x6d, 0x65, 0x73, + 0x73, 0x61, 0x67, 0x65, 0x22, 0xb8, 0x01, 0x0a, 0x0d, 0x43, 0x6f, 0x6d, 0x6d, 0x61, 0x6e, 0x64, + 0x52, 0x65, 0x73, 0x75, 0x6c, 0x74, 0x12, 0x1d, 0x0a, 0x0a, 0x63, 0x6f, 0x6d, 0x6d, 0x61, 0x6e, + 0x64, 0x5f, 0x69, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x09, 0x63, 0x6f, 0x6d, 0x6d, + 0x61, 0x6e, 0x64, 0x49, 0x64, 0x12, 0x39, 0x0a, 0x05, 0x65, 0x72, 0x72, 0x6f, 0x72, 0x18, 0x03, + 0x20, 0x01, 0x28, 0x0b, 0x32, 0x23, 0x2e, 0x6f, 0x6e, 0x6c, 0x79, 0x62, 0x6f, 0x78, 0x65, 0x73, + 0x2e, 0x72, 0x65, 0x67, 0x69, 0x73, 0x74, 0x72, 0x79, 0x2e, 0x76, 0x31, 0x2e, 0x43, 0x6f, 0x6d, + 0x6d, 0x61, 0x6e, 0x64, 0x45, 0x72, 0x72, 0x6f, 0x72, 0x52, 0x05, 0x65, 0x72, 0x72, 0x6f, 0x72, + 0x12, 0x21, 0x0a, 0x0c, 0x70, 0x61, 0x79, 0x6c, 0x6f, 0x61, 0x64, 0x5f, 0x6a, 0x73, 0x6f, 0x6e, + 0x18, 0x04, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x0b, 0x70, 0x61, 0x79, 0x6c, 0x6f, 0x61, 0x64, 0x4a, + 0x73, 0x6f, 0x6e, 0x12, 0x2a, 0x0a, 0x11, 0x63, 0x6f, 0x6d, 0x70, 0x6c, 0x65, 0x74, 0x65, 0x64, + 0x5f, 0x75, 0x6e, 0x69, 0x78, 0x5f, 0x6d, 0x73, 0x18, 0x05, 0x20, 0x01, 0x28, 0x03, 0x52, 0x0f, + 0x63, 0x6f, 0x6d, 0x70, 0x6c, 0x65, 0x74, 0x65, 0x64, 0x55, 0x6e, 0x69, 0x78, 0x4d, 0x73, 0x22, + 0xfb, 0x02, 0x0a, 0x0f, 0x43, 0x6f, 0x6e, 0x6e, 0x65, 0x63, 0x74, 0x52, 0x65, 0x73, 0x70, 0x6f, + 0x6e, 0x73, 0x65, 0x12, 0x44, 0x0a, 0x0b, 0x63, 0x6f, 0x6e, 0x6e, 0x65, 0x63, 0x74, 0x5f, 0x61, + 0x63, 0x6b, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x21, 0x2e, 0x6f, 0x6e, 0x6c, 0x79, 0x62, + 0x6f, 0x78, 0x65, 0x73, 0x2e, 0x72, 0x65, 0x67, 0x69, 0x73, 0x74, 0x72, 0x79, 0x2e, 0x76, 0x31, + 0x2e, 0x43, 0x6f, 0x6e, 0x6e, 0x65, 0x63, 0x74, 0x41, 0x63, 0x6b, 0x48, 0x00, 0x52, 0x0a, 0x63, + 0x6f, 0x6e, 0x6e, 0x65, 0x63, 0x74, 0x41, 0x63, 0x6b, 0x12, 0x4a, 0x0a, 0x0d, 0x68, 0x65, 0x61, + 0x72, 0x74, 0x62, 0x65, 0x61, 0x74, 0x5f, 0x61, 0x63, 0x6b, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, + 0x32, 0x23, 0x2e, 0x6f, 0x6e, 0x6c, 0x79, 0x62, 0x6f, 0x78, 0x65, 0x73, 0x2e, 0x72, 0x65, 0x67, + 0x69, 0x73, 0x74, 0x72, 0x79, 0x2e, 0x76, 0x31, 0x2e, 0x48, 0x65, 0x61, 0x72, 0x74, 0x62, 0x65, + 0x61, 0x74, 0x41, 0x63, 0x6b, 0x48, 0x00, 0x52, 0x0c, 0x68, 0x65, 0x61, 0x72, 0x74, 0x62, 0x65, + 0x61, 0x74, 0x41, 0x63, 0x6b, 0x12, 0x53, 0x0a, 0x10, 0x63, 0x6f, 0x6d, 0x6d, 0x61, 0x6e, 0x64, + 0x5f, 0x64, 0x69, 0x73, 0x70, 0x61, 0x74, 0x63, 0x68, 0x18, 0x04, 0x20, 0x01, 0x28, 0x0b, 0x32, + 0x26, 0x2e, 0x6f, 0x6e, 0x6c, 0x79, 0x62, 0x6f, 0x78, 0x65, 0x73, 0x2e, 0x72, 0x65, 0x67, 0x69, + 0x73, 0x74, 0x72, 0x79, 0x2e, 0x76, 0x31, 0x2e, 0x43, 0x6f, 0x6d, 0x6d, 0x61, 0x6e, 0x64, 0x44, + 0x69, 0x73, 0x70, 0x61, 0x74, 0x63, 0x68, 0x48, 0x00, 0x52, 0x0f, 0x63, 0x6f, 0x6d, 0x6d, 0x61, + 0x6e, 0x64, 0x44, 0x69, 0x73, 0x70, 0x61, 0x74, 0x63, 0x68, 0x12, 0x76, 0x0a, 0x1d, 0x74, 0x65, + 0x72, 0x6d, 0x69, 0x6e, 0x61, 0x6c, 0x5f, 0x73, 0x65, 0x73, 0x73, 0x69, 0x6f, 0x6e, 0x5f, 0x72, + 0x65, 0x63, 0x6f, 0x76, 0x65, 0x72, 0x79, 0x5f, 0x61, 0x63, 0x6b, 0x18, 0x05, 0x20, 0x01, 0x28, + 0x0b, 0x32, 0x31, 0x2e, 0x6f, 0x6e, 0x6c, 0x79, 0x62, 0x6f, 0x78, 0x65, 0x73, 0x2e, 0x72, 0x65, + 0x67, 0x69, 0x73, 0x74, 0x72, 0x79, 0x2e, 0x76, 0x31, 0x2e, 0x54, 0x65, 0x72, 0x6d, 0x69, 0x6e, + 0x61, 0x6c, 0x53, 0x65, 0x73, 0x73, 0x69, 0x6f, 0x6e, 0x52, 0x65, 0x63, 0x6f, 0x76, 0x65, 0x72, + 0x79, 0x41, 0x63, 0x6b, 0x48, 0x00, 0x52, 0x1a, 0x74, 0x65, 0x72, 0x6d, 0x69, 0x6e, 0x61, 0x6c, + 0x53, 0x65, 0x73, 0x73, 0x69, 0x6f, 0x6e, 0x52, 0x65, 0x63, 0x6f, 0x76, 0x65, 0x72, 0x79, 0x41, + 0x63, 0x6b, 0x42, 0x09, 0x0a, 0x07, 0x70, 0x61, 0x79, 0x6c, 0x6f, 0x61, 0x64, 0x32, 0x75, 0x0a, + 0x15, 0x57, 0x6f, 0x72, 0x6b, 0x65, 0x72, 0x52, 0x65, 0x67, 0x69, 0x73, 0x74, 0x72, 0x79, 0x53, + 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x12, 0x5c, 0x0a, 0x07, 0x43, 0x6f, 0x6e, 0x6e, 0x65, 0x63, + 0x74, 0x12, 0x25, 0x2e, 0x6f, 0x6e, 0x6c, 0x79, 0x62, 0x6f, 0x78, 0x65, 0x73, 0x2e, 0x72, 0x65, + 0x67, 0x69, 0x73, 0x74, 0x72, 0x79, 0x2e, 0x76, 0x31, 0x2e, 0x43, 0x6f, 0x6e, 0x6e, 0x65, 0x63, + 0x74, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x26, 0x2e, 0x6f, 0x6e, 0x6c, 0x79, 0x62, + 0x6f, 0x78, 0x65, 0x73, 0x2e, 0x72, 0x65, 0x67, 0x69, 0x73, 0x74, 0x72, 0x79, 0x2e, 0x76, 0x31, + 0x2e, 0x43, 0x6f, 0x6e, 0x6e, 0x65, 0x63, 0x74, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, + 0x28, 0x01, 0x30, 0x01, 0x42, 0x42, 0x5a, 0x40, 0x67, 0x69, 0x74, 0x68, 0x75, 0x62, 0x2e, 0x63, + 0x6f, 0x6d, 0x2f, 0x6f, 0x6e, 0x6c, 0x79, 0x62, 0x6f, 0x78, 0x65, 0x73, 0x2f, 0x6f, 0x6e, 0x6c, + 0x79, 0x62, 0x6f, 0x78, 0x65, 0x73, 0x2f, 0x61, 0x70, 0x69, 0x2f, 0x67, 0x65, 0x6e, 0x2f, 0x67, + 0x6f, 0x2f, 0x72, 0x65, 0x67, 0x69, 0x73, 0x74, 0x72, 0x79, 0x2f, 0x76, 0x31, 0x3b, 0x72, 0x65, + 0x67, 0x69, 0x73, 0x74, 0x72, 0x79, 0x76, 0x31, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33, }) var ( @@ -919,39 +1249,50 @@ func file_registry_v1_registry_proto_rawDescGZIP() []byte { return file_registry_v1_registry_proto_rawDescData } -var file_registry_v1_registry_proto_msgTypes = make([]protoimpl.MessageInfo, 12) +var file_registry_v1_registry_proto_enumTypes = make([]protoimpl.EnumInfo, 1) +var file_registry_v1_registry_proto_msgTypes = make([]protoimpl.MessageInfo, 16) var file_registry_v1_registry_proto_goTypes = []any{ - (*CapabilityDeclaration)(nil), // 0: onlyboxes.registry.v1.CapabilityDeclaration - (*TerminalSessionCapacity)(nil), // 1: onlyboxes.registry.v1.TerminalSessionCapacity - (*ConnectHello)(nil), // 2: onlyboxes.registry.v1.ConnectHello - (*HeartbeatFrame)(nil), // 3: onlyboxes.registry.v1.HeartbeatFrame - (*ConnectRequest)(nil), // 4: onlyboxes.registry.v1.ConnectRequest - (*ConnectAck)(nil), // 5: onlyboxes.registry.v1.ConnectAck - (*HeartbeatAck)(nil), // 6: onlyboxes.registry.v1.HeartbeatAck - (*CommandDispatch)(nil), // 7: onlyboxes.registry.v1.CommandDispatch - (*CommandError)(nil), // 8: onlyboxes.registry.v1.CommandError - (*CommandResult)(nil), // 9: onlyboxes.registry.v1.CommandResult - (*ConnectResponse)(nil), // 10: onlyboxes.registry.v1.ConnectResponse - nil, // 11: onlyboxes.registry.v1.ConnectHello.LabelsEntry + (TerminalSessionRecoveryResult_Status)(0), // 0: onlyboxes.registry.v1.TerminalSessionRecoveryResult.Status + (*CapabilityDeclaration)(nil), // 1: onlyboxes.registry.v1.CapabilityDeclaration + (*TerminalSessionCapacity)(nil), // 2: onlyboxes.registry.v1.TerminalSessionCapacity + (*ConnectHello)(nil), // 3: onlyboxes.registry.v1.ConnectHello + (*HeartbeatFrame)(nil), // 4: onlyboxes.registry.v1.HeartbeatFrame + (*ConnectRequest)(nil), // 5: onlyboxes.registry.v1.ConnectRequest + (*TerminalSessionRecoveryCandidate)(nil), // 6: onlyboxes.registry.v1.TerminalSessionRecoveryCandidate + (*TerminalSessionRecoveryResult)(nil), // 7: onlyboxes.registry.v1.TerminalSessionRecoveryResult + (*TerminalSessionRecoveryReport)(nil), // 8: onlyboxes.registry.v1.TerminalSessionRecoveryReport + (*TerminalSessionRecoveryAck)(nil), // 9: onlyboxes.registry.v1.TerminalSessionRecoveryAck + (*ConnectAck)(nil), // 10: onlyboxes.registry.v1.ConnectAck + (*HeartbeatAck)(nil), // 11: onlyboxes.registry.v1.HeartbeatAck + (*CommandDispatch)(nil), // 12: onlyboxes.registry.v1.CommandDispatch + (*CommandError)(nil), // 13: onlyboxes.registry.v1.CommandError + (*CommandResult)(nil), // 14: onlyboxes.registry.v1.CommandResult + (*ConnectResponse)(nil), // 15: onlyboxes.registry.v1.ConnectResponse + nil, // 16: onlyboxes.registry.v1.ConnectHello.LabelsEntry } var file_registry_v1_registry_proto_depIdxs = []int32{ - 11, // 0: onlyboxes.registry.v1.ConnectHello.labels:type_name -> onlyboxes.registry.v1.ConnectHello.LabelsEntry - 0, // 1: onlyboxes.registry.v1.ConnectHello.capabilities:type_name -> onlyboxes.registry.v1.CapabilityDeclaration - 1, // 2: onlyboxes.registry.v1.ConnectHello.terminal_session_capacity:type_name -> onlyboxes.registry.v1.TerminalSessionCapacity - 2, // 3: onlyboxes.registry.v1.ConnectRequest.hello:type_name -> onlyboxes.registry.v1.ConnectHello - 3, // 4: onlyboxes.registry.v1.ConnectRequest.heartbeat:type_name -> onlyboxes.registry.v1.HeartbeatFrame - 9, // 5: onlyboxes.registry.v1.ConnectRequest.command_result:type_name -> onlyboxes.registry.v1.CommandResult - 8, // 6: onlyboxes.registry.v1.CommandResult.error:type_name -> onlyboxes.registry.v1.CommandError - 5, // 7: onlyboxes.registry.v1.ConnectResponse.connect_ack:type_name -> onlyboxes.registry.v1.ConnectAck - 6, // 8: onlyboxes.registry.v1.ConnectResponse.heartbeat_ack:type_name -> onlyboxes.registry.v1.HeartbeatAck - 7, // 9: onlyboxes.registry.v1.ConnectResponse.command_dispatch:type_name -> onlyboxes.registry.v1.CommandDispatch - 4, // 10: onlyboxes.registry.v1.WorkerRegistryService.Connect:input_type -> onlyboxes.registry.v1.ConnectRequest - 10, // 11: onlyboxes.registry.v1.WorkerRegistryService.Connect:output_type -> onlyboxes.registry.v1.ConnectResponse - 11, // [11:12] is the sub-list for method output_type - 10, // [10:11] is the sub-list for method input_type - 10, // [10:10] is the sub-list for extension type_name - 10, // [10:10] is the sub-list for extension extendee - 0, // [0:10] is the sub-list for field type_name + 16, // 0: onlyboxes.registry.v1.ConnectHello.labels:type_name -> onlyboxes.registry.v1.ConnectHello.LabelsEntry + 1, // 1: onlyboxes.registry.v1.ConnectHello.capabilities:type_name -> onlyboxes.registry.v1.CapabilityDeclaration + 2, // 2: onlyboxes.registry.v1.ConnectHello.terminal_session_capacity:type_name -> onlyboxes.registry.v1.TerminalSessionCapacity + 3, // 3: onlyboxes.registry.v1.ConnectRequest.hello:type_name -> onlyboxes.registry.v1.ConnectHello + 4, // 4: onlyboxes.registry.v1.ConnectRequest.heartbeat:type_name -> onlyboxes.registry.v1.HeartbeatFrame + 14, // 5: onlyboxes.registry.v1.ConnectRequest.command_result:type_name -> onlyboxes.registry.v1.CommandResult + 8, // 6: onlyboxes.registry.v1.ConnectRequest.terminal_session_recovery_report:type_name -> onlyboxes.registry.v1.TerminalSessionRecoveryReport + 0, // 7: onlyboxes.registry.v1.TerminalSessionRecoveryResult.status:type_name -> onlyboxes.registry.v1.TerminalSessionRecoveryResult.Status + 7, // 8: onlyboxes.registry.v1.TerminalSessionRecoveryReport.results:type_name -> onlyboxes.registry.v1.TerminalSessionRecoveryResult + 6, // 9: onlyboxes.registry.v1.ConnectAck.terminal_session_recovery_candidates:type_name -> onlyboxes.registry.v1.TerminalSessionRecoveryCandidate + 13, // 10: onlyboxes.registry.v1.CommandResult.error:type_name -> onlyboxes.registry.v1.CommandError + 10, // 11: onlyboxes.registry.v1.ConnectResponse.connect_ack:type_name -> onlyboxes.registry.v1.ConnectAck + 11, // 12: onlyboxes.registry.v1.ConnectResponse.heartbeat_ack:type_name -> onlyboxes.registry.v1.HeartbeatAck + 12, // 13: onlyboxes.registry.v1.ConnectResponse.command_dispatch:type_name -> onlyboxes.registry.v1.CommandDispatch + 9, // 14: onlyboxes.registry.v1.ConnectResponse.terminal_session_recovery_ack:type_name -> onlyboxes.registry.v1.TerminalSessionRecoveryAck + 5, // 15: onlyboxes.registry.v1.WorkerRegistryService.Connect:input_type -> onlyboxes.registry.v1.ConnectRequest + 15, // 16: onlyboxes.registry.v1.WorkerRegistryService.Connect:output_type -> onlyboxes.registry.v1.ConnectResponse + 16, // [16:17] is the sub-list for method output_type + 15, // [15:16] is the sub-list for method input_type + 15, // [15:15] is the sub-list for extension type_name + 15, // [15:15] is the sub-list for extension extendee + 0, // [0:15] is the sub-list for field type_name } func init() { file_registry_v1_registry_proto_init() } @@ -963,24 +1304,27 @@ func file_registry_v1_registry_proto_init() { (*ConnectRequest_Hello)(nil), (*ConnectRequest_Heartbeat)(nil), (*ConnectRequest_CommandResult)(nil), + (*ConnectRequest_TerminalSessionRecoveryReport)(nil), } - file_registry_v1_registry_proto_msgTypes[10].OneofWrappers = []any{ + file_registry_v1_registry_proto_msgTypes[14].OneofWrappers = []any{ (*ConnectResponse_ConnectAck)(nil), (*ConnectResponse_HeartbeatAck)(nil), (*ConnectResponse_CommandDispatch)(nil), + (*ConnectResponse_TerminalSessionRecoveryAck)(nil), } type x struct{} out := protoimpl.TypeBuilder{ File: protoimpl.DescBuilder{ GoPackagePath: reflect.TypeOf(x{}).PkgPath(), RawDescriptor: unsafe.Slice(unsafe.StringData(file_registry_v1_registry_proto_rawDesc), len(file_registry_v1_registry_proto_rawDesc)), - NumEnums: 0, - NumMessages: 12, + NumEnums: 1, + NumMessages: 16, NumExtensions: 0, NumServices: 1, }, GoTypes: file_registry_v1_registry_proto_goTypes, DependencyIndexes: file_registry_v1_registry_proto_depIdxs, + EnumInfos: file_registry_v1_registry_proto_enumTypes, MessageInfos: file_registry_v1_registry_proto_msgTypes, }.Build() File_registry_v1_registry_proto = out.File diff --git a/api/proto/registry/v1/registry.proto b/api/proto/registry/v1/registry.proto index 6c22682..ec4bf56 100644 --- a/api/proto/registry/v1/registry.proto +++ b/api/proto/registry/v1/registry.proto @@ -38,12 +38,37 @@ message ConnectRequest { ConnectHello hello = 1; HeartbeatFrame heartbeat = 2; CommandResult command_result = 3; + TerminalSessionRecoveryReport terminal_session_recovery_report = 4; } } +message TerminalSessionRecoveryCandidate { + string session_id = 1; + int64 lease_expires_unix_ms = 2; +} + +message TerminalSessionRecoveryResult { + enum Status { + STATUS_UNSPECIFIED = 0; + RECOVERED = 1; + MISSING = 2; + INVALID = 3; + } + + string session_id = 1; + Status status = 2; +} + +message TerminalSessionRecoveryReport { + repeated TerminalSessionRecoveryResult results = 1; +} + +message TerminalSessionRecoveryAck {} + message ConnectAck { string session_id = 1; int32 heartbeat_interval_sec = 3; + repeated TerminalSessionRecoveryCandidate terminal_session_recovery_candidates = 4; } message HeartbeatAck { @@ -74,6 +99,7 @@ message ConnectResponse { ConnectAck connect_ack = 1; HeartbeatAck heartbeat_ack = 2; CommandDispatch command_dispatch = 4; + TerminalSessionRecoveryAck terminal_session_recovery_ack = 5; } } From 61e588af794f4e67846b82ca1c461113e0e1e2a5 Mon Sep 17 00:00:00 2001 From: Coolfan Date: Fri, 7 Aug 2026 22:23:21 +0800 Subject: [PATCH 2/7] feat(console): reconcile terminal sessions on worker reconnect Send recovery candidates in ConnectAck, apply the worker recovery report to refresh leases and drop missing sessions, and gate dispatch until recovery completes. Mark routes unavailable instead of clearing them on disconnect so the bound session_id keeps returning 503 session_unavailable while the worker is offline or reconciling. --- console/README/overview.md | 5 + .../grpcserver/connect_service_test.go | 30 +++ .../internal/grpcserver/service_connect.go | 52 +++- .../internal/grpcserver/service_dispatch.go | 47 +++- .../internal/grpcserver/session_runtime.go | 130 ++++++++- .../terminal_capacity_dispatch_test.go | 1 + .../terminal_session_recovery_test.go | 193 ++++++++++++++ .../grpcserver/terminal_session_routes.go | 246 +++++++++++++++--- console/internal/httpapi/command_handler.go | 39 +-- .../internal/httpapi/command_handler_test.go | 1 + console/internal/httpapi/integration_test.go | 27 ++ console/internal/httpapi/task_handler.go | 8 +- console/internal/httpapi/task_handler_test.go | 10 + 13 files changed, 708 insertions(+), 81 deletions(-) create mode 100644 console/internal/grpcserver/terminal_session_recovery_test.go diff --git a/console/README/overview.md b/console/README/overview.md index 9f92749..cef0927 100644 --- a/console/README/overview.md +++ b/console/README/overview.md @@ -50,6 +50,11 @@ The console service hosts: - an existing session route remains pinned to its original worker even when that worker reports full active-session capacity. - `max_inflight` and active-session capacity are evaluated independently; the worker-local session manager remains the final authority. - an explicit pre-execution `session_capacity_exceeded` can be retried on an untried worker only after the provisional route is safely removed; all attempts share the task deadline. + - terminal session restart recovery: + - confirmed routes retain their worker binding and absolute lease while the worker is disconnected; requests return retryable `session_unavailable` instead of being reassigned. + - a terminal-capable worker must reconcile every Console candidate and receive a recovery acknowledgement before it becomes dispatchable. + - recovered routes keep their original lease; missing, invalid, or expired resources delete the route so later calls follow normal `session_not_found`/creation semantics. + - route and lease state are currently in Console memory and therefore do not survive a Console restart. - MCP Streamable HTTP API (bearer token required): - `POST /mcp` for JSON-RPC requests over Streamable HTTP transport. - recommended request header: `Authorization: Bearer `. diff --git a/console/internal/grpcserver/connect_service_test.go b/console/internal/grpcserver/connect_service_test.go index 1d4e4c8..59f8515 100644 --- a/console/internal/grpcserver/connect_service_test.go +++ b/console/internal/grpcserver/connect_service_test.go @@ -2058,6 +2058,36 @@ func connectWorkerWithHello( if ack == nil { return nil, "", fmt.Errorf("expected connect_ack, got %#v", resp.GetPayload()) } + hasTerminalExec := false + for _, capability := range hello.GetCapabilities() { + if normalizeCapability(capability.GetName()) == taskCapabilityTerminalExec { + hasTerminalExec = true + break + } + } + if hasTerminalExec { + results := make([]*registryv1.TerminalSessionRecoveryResult, 0, len(ack.GetTerminalSessionRecoveryCandidates())) + for _, candidate := range ack.GetTerminalSessionRecoveryCandidates() { + results = append(results, ®istryv1.TerminalSessionRecoveryResult{ + SessionId: candidate.GetSessionId(), + Status: registryv1.TerminalSessionRecoveryResult_RECOVERED, + }) + } + if err := stream.Send(®istryv1.ConnectRequest{ + Payload: ®istryv1.ConnectRequest_TerminalSessionRecoveryReport{ + TerminalSessionRecoveryReport: ®istryv1.TerminalSessionRecoveryReport{Results: results}, + }, + }); err != nil { + return nil, "", err + } + recoveryAck, err := stream.Recv() + if err != nil { + return nil, "", err + } + if recoveryAck.GetTerminalSessionRecoveryAck() == nil { + return nil, "", fmt.Errorf("expected terminal_session_recovery_ack, got %#v", recoveryAck.GetPayload()) + } + } return stream, ack.GetSessionId(), nil } diff --git a/console/internal/grpcserver/service_connect.go b/console/internal/grpcserver/service_connect.go index 6b06309..b4cb3b5 100644 --- a/console/internal/grpcserver/service_connect.go +++ b/console/internal/grpcserver/service_connect.go @@ -71,6 +71,8 @@ func (s *RegistryService) Connect(stream grpc.BidiStreamingServer[registryv1.Con } session := newActiveSessionAt(hello.GetNodeId(), sessionID, hello, now) + recoveryCandidates := s.beginTerminalSessionRecovery(session.nodeID, now) + session.setRecoveryCandidates(recoveryCandidates) logTerminalSessionCapacityInvariant(session.nodeID, session.terminalSessionCapacitySnapshot()) replaced := s.swapSession(session) if replaced != nil { @@ -90,7 +92,7 @@ func (s *RegistryService) Connect(stream grpc.BidiStreamingServer[registryv1.Con writerErrCh <- writerLoop(stream, session) }() - if err := session.enqueueControl(stream.Context(), newConnectAck(sessionID, s.heartbeatIntervalSec)); err != nil { + if err := session.enqueueControl(stream.Context(), newConnectAck(sessionID, s.heartbeatIntervalSec, recoveryCandidates...)); err != nil { return status.Error(codes.Internal, "failed to send connect ack") } @@ -124,6 +126,26 @@ func (s *RegistryService) Connect(stream grpc.BidiStreamingServer[registryv1.Con if err := handleCommandResult(session, req.GetCommandResult()); err != nil { return err } + case req.GetTerminalSessionRecoveryReport() != nil: + if !session.recoveryRequired { + return status.Error(codes.InvalidArgument, "terminal session recovery is not supported by this worker") + } + if session.isReady() { + if !session.matchesRecoveryResults(req.GetTerminalSessionRecoveryReport()) { + return status.Error(codes.FailedPrecondition, "terminal session recovery report changed after completion") + } + if err := session.enqueueControl(stream.Context(), newTerminalSessionRecoveryAck()); err != nil { + return status.Error(codes.Internal, "failed to resend terminal session recovery ack") + } + continue + } + if err := s.applyTerminalSessionRecoveryReport(session, req.GetTerminalSessionRecoveryReport(), s.nowFn()); err != nil { + return status.Error(codes.InvalidArgument, err.Error()) + } + if err := session.enqueueControl(stream.Context(), newTerminalSessionRecoveryAck()); err != nil { + return status.Error(codes.Internal, "failed to send terminal session recovery ack") + } + session.markRecoveryComplete() default: return status.Error(codes.InvalidArgument, "unsupported frame type") } @@ -329,10 +351,6 @@ func (s *RegistryService) swapSession(session *activeSession) *activeSession { s.sessions[session.nodeID] = session return replaced }() - // Release sessionsMu before touching terminal route tables to avoid lock - // inversion with dispatch paths that read terminal routes then sessions. - // This leaves a tiny window where an old route may be observed once. - s.clearTerminalSessionRoutesByNode(session.nodeID) return replaced } @@ -358,9 +376,12 @@ func (s *RegistryService) removeSession(session *activeSession) { if !shouldClearStoreSession { return } - // Keep the same lock order as swapSession: sessions first, then route tables. - // Clearing route mappings outside sessionsMu avoids cross-lock deadlocks. - s.clearTerminalSessionRoutesByNode(session.nodeID) + unavailable := s.markTerminalSessionRoutesUnavailable(session.nodeID) + slog.Info( + "terminal session routes unavailable", + "executor_kind", session.executorKind, + "unavailable_route_count", unavailable, + ) if shouldClearStoreSession && s.store != nil { if err := s.store.ClearSession(session.nodeID, session.sessionID); err != nil { slog.Error( @@ -412,17 +433,26 @@ func writerLoop(stream grpc.BidiStreamingServer[registryv1.ConnectRequest, regis } } -func newConnectAck(sessionID string, heartbeatIntervalSec int32) *registryv1.ConnectResponse { +func newConnectAck(sessionID string, heartbeatIntervalSec int32, recoveryCandidates ...*registryv1.TerminalSessionRecoveryCandidate) *registryv1.ConnectResponse { return ®istryv1.ConnectResponse{ Payload: ®istryv1.ConnectResponse_ConnectAck{ ConnectAck: ®istryv1.ConnectAck{ - SessionId: sessionID, - HeartbeatIntervalSec: heartbeatIntervalSec, + SessionId: sessionID, + HeartbeatIntervalSec: heartbeatIntervalSec, + TerminalSessionRecoveryCandidates: recoveryCandidates, }, }, } } +func newTerminalSessionRecoveryAck() *registryv1.ConnectResponse { + return ®istryv1.ConnectResponse{ + Payload: ®istryv1.ConnectResponse_TerminalSessionRecoveryAck{ + TerminalSessionRecoveryAck: ®istryv1.TerminalSessionRecoveryAck{}, + }, + } +} + func newHeartbeatAck(heartbeatIntervalSec int32) *registryv1.ConnectResponse { return ®istryv1.ConnectResponse{ Payload: ®istryv1.ConnectResponse_HeartbeatAck{ diff --git a/console/internal/grpcserver/service_dispatch.go b/console/internal/grpcserver/service_dispatch.go index 20a3a8d..3b360ea 100644 --- a/console/internal/grpcserver/service_dispatch.go +++ b/console/internal/grpcserver/service_dispatch.go @@ -20,6 +20,7 @@ import ( const ( terminalSessionNotFoundCode = "session_not_found" terminalSessionCapacityExceededCode = "session_capacity_exceeded" + terminalSessionUnavailableCode = "session_unavailable" ) type dispatchOptions struct { @@ -235,7 +236,7 @@ func (s *RegistryService) dispatchCommandAttempt( terminalRouteReservationID, ) } - confirmTerminalRoute := func() { + confirmTerminalRoute := func(resultPayload []byte) { if terminalSessionID != "" { s.confirmTerminalSessionRoute( terminalSessionID, @@ -243,6 +244,9 @@ func (s *RegistryService) dispatchCommandAttempt( terminalRouteReservationID, s.nowFn(), ) + if leaseExpiresUnixMs := terminalSessionLeaseExpiresUnixMs(resultPayload); leaseExpiresUnixMs > 0 { + s.updateTerminalSessionRouteLease(terminalSessionID, session.nodeID, leaseExpiresUnixMs, s.nowFn()) + } } } @@ -293,7 +297,7 @@ func (s *RegistryService) dispatchCommandAttempt( if onDispatched != nil { if err := onDispatched(commandID); err != nil { if terminalRouteReservationID != 0 { - confirmTerminalRoute() + confirmTerminalRoute(nil) } return dispatchAttemptResult{}, err } @@ -304,7 +308,7 @@ func (s *RegistryService) dispatchCommandAttempt( if terminalRouteReservationID != 0 && terminalSessionID != "" { // The dispatch reached the worker stream, so cancellation does not // prove that session creation failed. - confirmTerminalRoute() + confirmTerminalRoute(nil) } if errors.Is(commandCtx.Err(), context.DeadlineExceeded) { return dispatchAttemptResult{}, context.DeadlineExceeded @@ -316,7 +320,7 @@ func (s *RegistryService) dispatchCommandAttempt( return dispatchAttemptResult{}, status.Error(codes.Unavailable, "worker session closed before command result") } if outcome.err == nil && terminalSessionID != "" { - confirmTerminalRoute() + confirmTerminalRoute(outcome.payloadJSON) return dispatchAttemptResult{outcome: outcome}, nil } if outcome.err == nil || terminalSessionID == "" { @@ -340,7 +344,7 @@ func (s *RegistryService) dispatchCommandAttempt( }, nil case terminalRouteReservationID != 0: // Other execution errors do not prove that session creation failed. - confirmTerminalRoute() + confirmTerminalRoute(nil) } return dispatchAttemptResult{outcome: outcome}, nil } @@ -359,6 +363,12 @@ func (s *RegistryService) pickSessionForDispatch( } now := s.nowFn() s.maybePruneTerminalSessionRoutes(now) + if route, exists := s.terminalSessionRouteSnapshot(normalizedTerminalSessionID, now); exists && route.ReservationID == 0 { + boundSession := s.getSession(route.NodeID) + if route.RecoveryState != terminalSessionRecoveryReady || boundSession == nil || !boundSession.isReady() { + return nil, 0, terminalSessionUnavailableError() + } + } nodeID, reservationID, ok := s.claimTerminalSessionRoute(normalizedTerminalSessionID, now) if !ok { @@ -376,6 +386,9 @@ func (s *RegistryService) pickSessionForDispatch( return session, reservationID, nil } if errors.Is(err, ErrNoCapabilityWorker) { + if reservationID == 0 { + return nil, 0, terminalSessionUnavailableError() + } s.clearTerminalSessionRoute(normalizedTerminalSessionID, nodeID) return s.tryReserveAndPickTerminalSession(capability, ownerID, normalizedTerminalSessionID, now, options) } @@ -440,7 +453,7 @@ func (s *RegistryService) pickSessionForNodeAndCapability(nodeID string, capabil } session := s.getSession(normalizedNodeID) - if session == nil || !session.hasCapability(capability) { + if session == nil || !session.isReady() || !session.hasCapability(capability) { return nil, ErrNoCapabilityWorker } @@ -485,7 +498,7 @@ func (s *RegistryService) pickSessionForCapability( continue } session := s.getSession(nodeID) - if session == nil || !session.hasCapability(capability) { + if session == nil || !session.isReady() || !session.hasCapability(capability) { continue } hasSession = true @@ -573,6 +586,26 @@ func isSessionCapacityCommandError(err error) bool { return isCommandErrorCode(err, terminalSessionCapacityExceededCode) } +func terminalSessionUnavailableError() error { + return &CommandExecutionError{ + Code: terminalSessionUnavailableCode, + Message: "terminal session is temporarily unavailable", + } +} + +func terminalSessionLeaseExpiresUnixMs(payload []byte) int64 { + if len(payload) == 0 { + return 0 + } + var decoded struct { + LeaseExpiresUnixMS int64 `json:"lease_expires_unix_ms"` + } + if err := json.Unmarshal(payload, &decoded); err != nil || decoded.LeaseExpiresUnixMS <= 0 { + return 0 + } + return decoded.LeaseExpiresUnixMS +} + func isCommandErrorCode(err error, code string) bool { var commandErr *CommandExecutionError if !errors.As(err, &commandErr) { diff --git a/console/internal/grpcserver/session_runtime.go b/console/internal/grpcserver/session_runtime.go index 26d8ca8..7110fab 100644 --- a/console/internal/grpcserver/session_runtime.go +++ b/console/internal/grpcserver/session_runtime.go @@ -36,8 +36,10 @@ type terminalSessionCapacitySnapshot struct { } type activeSession struct { - nodeID string - sessionID string + nodeID string + sessionID string + executorKind string + connectedAt time.Time capabilitiesMu sync.Mutex capabilities map[string]*sessionCapability @@ -45,6 +47,12 @@ type activeSession struct { terminalCapacityMu sync.RWMutex terminalCapacity terminalSessionCapacitySnapshot + recoveryMu sync.RWMutex + recoveryRequired bool + recoveryComplete bool + recoveryCandidates map[string]int64 + recoveryResults map[string]registryv1.TerminalSessionRecoveryResult_Status + controlOutbound chan *registryv1.ConnectResponse commandOutbound chan *registryv1.ConnectResponse done chan struct{} @@ -57,19 +65,30 @@ type activeSession struct { } func newActiveSession(nodeID string, sessionID string, hello *registryv1.ConnectHello) *activeSession { - return newActiveSessionAt(nodeID, sessionID, hello, time.Now()) + session := newActiveSessionAt(nodeID, sessionID, hello, time.Now()) + // This convenience constructor is used by already-established in-process + // sessions in tests. Real Connect sessions use newActiveSessionAt and must + // complete the recovery handshake before becoming ready. + session.markRecoveryComplete() + return session } func newActiveSessionAt(nodeID string, sessionID string, hello *registryv1.ConnectHello, observedAt time.Time) *activeSession { session := &activeSession{ - nodeID: nodeID, - sessionID: sessionID, - capabilities: capabilitiesFromHello(hello), - controlOutbound: make(chan *registryv1.ConnectResponse, controlOutboundBufferSize), - commandOutbound: make(chan *registryv1.ConnectResponse, commandOutboundBufferSize), - done: make(chan struct{}), - pending: make(map[string]*pendingCommand), - } + nodeID: nodeID, + sessionID: sessionID, + executorKind: strings.TrimSpace(hello.GetExecutorKind()), + connectedAt: observedAt, + capabilities: capabilitiesFromHello(hello), + controlOutbound: make(chan *registryv1.ConnectResponse, controlOutboundBufferSize), + commandOutbound: make(chan *registryv1.ConnectResponse, commandOutboundBufferSize), + done: make(chan struct{}), + pending: make(map[string]*pendingCommand), + recoveryCandidates: make(map[string]int64), + recoveryResults: make(map[string]registryv1.TerminalSessionRecoveryResult_Status), + } + _, session.recoveryRequired = session.capabilities[taskCapabilityTerminalExec] + session.recoveryComplete = !session.recoveryRequired if capacity := hello.GetTerminalSessionCapacity(); capacity != nil { session.terminalCapacity = terminalSessionCapacitySnapshot{ known: true, @@ -81,6 +100,95 @@ func newActiveSessionAt(nodeID string, sessionID string, hello *registryv1.Conne return session } +func (s *activeSession) setRecoveryCandidates(candidates []*registryv1.TerminalSessionRecoveryCandidate) { + if s == nil { + return + } + s.recoveryMu.Lock() + defer s.recoveryMu.Unlock() + s.recoveryCandidates = make(map[string]int64, len(candidates)) + for _, candidate := range candidates { + if candidate == nil { + continue + } + sessionID := strings.TrimSpace(candidate.GetSessionId()) + if sessionID == "" { + continue + } + s.recoveryCandidates[sessionID] = candidate.GetLeaseExpiresUnixMs() + } +} + +func (s *activeSession) recoveryCandidateSnapshot() map[string]int64 { + if s == nil { + return nil + } + s.recoveryMu.RLock() + defer s.recoveryMu.RUnlock() + out := make(map[string]int64, len(s.recoveryCandidates)) + for sessionID, lease := range s.recoveryCandidates { + out[sessionID] = lease + } + return out +} + +func (s *activeSession) markRecoveryComplete() { + if s == nil { + return + } + s.recoveryMu.Lock() + s.recoveryComplete = true + s.recoveryMu.Unlock() +} + +func (s *activeSession) setRecoveryResults(results map[string]registryv1.TerminalSessionRecoveryResult_Status) { + if s == nil { + return + } + s.recoveryMu.Lock() + defer s.recoveryMu.Unlock() + s.recoveryResults = make(map[string]registryv1.TerminalSessionRecoveryResult_Status, len(results)) + for sessionID, status := range results { + s.recoveryResults[sessionID] = status + } +} + +func (s *activeSession) matchesRecoveryResults(report *registryv1.TerminalSessionRecoveryReport) bool { + if s == nil || report == nil { + return false + } + s.recoveryMu.RLock() + defer s.recoveryMu.RUnlock() + if len(report.GetResults()) != len(s.recoveryResults) { + return false + } + seen := make(map[string]struct{}, len(report.GetResults())) + for _, result := range report.GetResults() { + if result == nil { + return false + } + sessionID := strings.TrimSpace(result.GetSessionId()) + status, ok := s.recoveryResults[sessionID] + if !ok || status != result.GetStatus() { + return false + } + if _, duplicate := seen[sessionID]; duplicate { + return false + } + seen[sessionID] = struct{}{} + } + return true +} + +func (s *activeSession) isReady() bool { + if s == nil { + return false + } + s.recoveryMu.RLock() + defer s.recoveryMu.RUnlock() + return !s.recoveryRequired || s.recoveryComplete +} + func (s *activeSession) hasCapability(capability string) bool { normalized := normalizeCapability(capability) if normalized == "" { diff --git a/console/internal/grpcserver/terminal_capacity_dispatch_test.go b/console/internal/grpcserver/terminal_capacity_dispatch_test.go index 48c7588..d7065c1 100644 --- a/console/internal/grpcserver/terminal_capacity_dispatch_test.go +++ b/console/internal/grpcserver/terminal_capacity_dispatch_test.go @@ -33,6 +33,7 @@ func addTerminalCapacityTestWorker( t.Fatalf("upsert worker %s: %v", nodeID, err) } session := newActiveSessionAt(nodeID, "worker-session-"+nodeID, hello, now) + session.markRecoveryComplete() svc.swapSession(session) return session } diff --git a/console/internal/grpcserver/terminal_session_recovery_test.go b/console/internal/grpcserver/terminal_session_recovery_test.go new file mode 100644 index 0000000..0e3d0a7 --- /dev/null +++ b/console/internal/grpcserver/terminal_session_recovery_test.go @@ -0,0 +1,193 @@ +package grpcserver + +import ( + "errors" + "testing" + "time" + + registryv1 "github.com/onlyboxes/onlyboxes/api/gen/go/registry/v1" + "github.com/onlyboxes/onlyboxes/console/internal/testutil/registrytest" +) + +func TestTerminalRouteDisconnectRecoveryRoundTrip(t *testing.T) { + svc := NewRegistryService(registrytest.NewStore(t), nil, 5, 15, time.Minute) + base := time.Unix(1_700_100_000, 0) + lease := base.Add(10 * time.Minute).UnixMilli() + svc.nowFn = func() time.Time { return base.Add(time.Second) } + + svc.bindTerminalSessionRoute("session-a", "node-a", base) + if !svc.updateTerminalSessionRouteLease("session-a", "node-a", lease, base) { + t.Fatal("failed to store terminal lease") + } + if unavailable := svc.markTerminalSessionRoutesUnavailable("node-a"); unavailable != 1 { + t.Fatalf("unavailable route count=%d, want 1", unavailable) + } + + _, _, err := svc.pickSessionForDispatch( + taskCapabilityTerminalExec, + "owner-a", + "session-a", + sessionPickOptions{terminalSessionIntent: terminalSessionIntentKnownNew}, + ) + var commandErr *CommandExecutionError + if !errors.As(err, &commandErr) || commandErr.Code != terminalSessionUnavailableCode { + t.Fatalf("expected session_unavailable while disconnected, got %v", err) + } + + candidates := svc.beginTerminalSessionRecovery("node-a", base.Add(time.Second)) + if len(candidates) != 1 || candidates[0].GetSessionId() != "session-a" || candidates[0].GetLeaseExpiresUnixMs() != lease { + t.Fatalf("unexpected recovery candidates: %#v", candidates) + } + + hello := ®istryv1.ConnectHello{ + NodeId: "node-a", + Capabilities: []*registryv1.CapabilityDeclaration{ + {Name: taskCapabilityTerminalExec, MaxInflight: 1}, + }, + } + session := newActiveSessionAt("node-a", "worker-session-a", hello, base.Add(time.Second)) + session.setRecoveryCandidates(candidates) + err = svc.applyTerminalSessionRecoveryReport(session, ®istryv1.TerminalSessionRecoveryReport{ + Results: []*registryv1.TerminalSessionRecoveryResult{ + {SessionId: "session-a", Status: registryv1.TerminalSessionRecoveryResult_RECOVERED}, + }, + }, base.Add(2*time.Second)) + if err != nil { + t.Fatalf("apply recovery report: %v", err) + } + route, ok := svc.terminalSessionRouteSnapshot("session-a", base.Add(2*time.Second)) + if !ok || route.RecoveryState != terminalSessionRecoveryReady || route.LeaseExpiresUnixMs != lease { + t.Fatalf("unexpected recovered route: %#v ok=%v", route, ok) + } +} + +func TestTerminalRecoveryMissingDeletesRoute(t *testing.T) { + svc := NewRegistryService(registrytest.NewStore(t), nil, 5, 15, time.Minute) + base := time.Unix(1_700_200_000, 0) + svc.bindTerminalSessionRoute("session-missing", "node-a", base) + svc.updateTerminalSessionRouteLease("session-missing", "node-a", base.Add(time.Minute).UnixMilli(), base) + candidates := svc.beginTerminalSessionRecovery("node-a", base.Add(time.Second)) + session := newActiveSessionAt("node-a", "worker-session-a", ®istryv1.ConnectHello{ + Capabilities: []*registryv1.CapabilityDeclaration{{Name: taskCapabilityTerminalExec, MaxInflight: 1}}, + }, base) + session.setRecoveryCandidates(candidates) + + err := svc.applyTerminalSessionRecoveryReport(session, ®istryv1.TerminalSessionRecoveryReport{ + Results: []*registryv1.TerminalSessionRecoveryResult{ + {SessionId: "session-missing", Status: registryv1.TerminalSessionRecoveryResult_MISSING}, + }, + }, base.Add(2*time.Second)) + if err != nil { + t.Fatalf("apply missing report: %v", err) + } + if _, ok := svc.terminalSessionRouteSnapshot("session-missing", base.Add(2*time.Second)); ok { + t.Fatal("missing backend resource must delete route") + } +} + +func TestTerminalRecoveryUsesLegacyRouteTTLFallback(t *testing.T) { + svc := NewRegistryService(registrytest.NewStore(t), nil, 5, 15, time.Minute) + svc.terminalRouteTTL = 30 * time.Minute + base := time.Unix(1_700_300_000, 0) + svc.bindTerminalSessionRoute("legacy-session", "node-a", base) + + candidates := svc.beginTerminalSessionRecovery("node-a", base.Add(time.Minute)) + wantLease := base.Add(30 * time.Minute).UnixMilli() + if len(candidates) != 1 || candidates[0].GetLeaseExpiresUnixMs() != wantLease { + t.Fatalf("legacy route candidate lease=%v, want %d", candidates, wantLease) + } +} + +func TestTerminalRecoveryDropsRouteWhoseLeaseExpiredOffline(t *testing.T) { + svc := NewRegistryService(registrytest.NewStore(t), nil, 5, 15, time.Minute) + base := time.Unix(1_700_350_000, 0) + svc.bindTerminalSessionRoute("expired-session", "node-a", base) + svc.updateTerminalSessionRouteLease("expired-session", "node-a", base.Add(time.Minute).UnixMilli(), base) + svc.markTerminalSessionRoutesUnavailable("node-a") + if candidates := svc.beginTerminalSessionRecovery("node-a", base.Add(2*time.Minute)); len(candidates) != 0 { + t.Fatalf("expired route was offered for recovery: %#v", candidates) + } + if _, ok := svc.terminalSessionRouteSnapshot("expired-session", base.Add(2*time.Minute)); ok { + t.Fatal("expired offline route was not deleted") + } +} + +func TestPruneUsesExactLeaseWhenLegacyTTLDisabled(t *testing.T) { + svc := NewRegistryService(registrytest.NewStore(t), nil, 5, 15, time.Minute) + svc.terminalRouteTTL = 0 + base := time.Unix(1_700_375_000, 0) + svc.bindTerminalSessionRoute("expired-session", "node-a", base) + svc.updateTerminalSessionRouteLease("expired-session", "node-a", base.Add(time.Minute).UnixMilli(), base) + if removed := svc.pruneExpiredTerminalSessionRoutes(base.Add(2 * time.Minute)); removed != 1 { + t.Fatalf("removed=%d, want exact-lease route pruned", removed) + } +} + +func TestTerminalRecoveryReportMustCoverCandidates(t *testing.T) { + svc := NewRegistryService(registrytest.NewStore(t), nil, 5, 15, time.Minute) + base := time.Unix(1_700_400_000, 0) + svc.bindTerminalSessionRoute("session-a", "node-a", base) + svc.updateTerminalSessionRouteLease("session-a", "node-a", base.Add(time.Minute).UnixMilli(), base) + candidates := svc.beginTerminalSessionRecovery("node-a", base) + session := newActiveSessionAt("node-a", "worker-session-a", ®istryv1.ConnectHello{ + Capabilities: []*registryv1.CapabilityDeclaration{{Name: taskCapabilityTerminalExec, MaxInflight: 1}}, + }, base) + session.setRecoveryCandidates(candidates) + + if err := svc.applyTerminalSessionRecoveryReport(session, ®istryv1.TerminalSessionRecoveryReport{}, base); err == nil { + t.Fatal("partial recovery report unexpectedly accepted") + } +} + +func TestTerminalRecoveryReportIsIdempotentOnlyWhenUnchanged(t *testing.T) { + svc := NewRegistryService(registrytest.NewStore(t), nil, 5, 15, time.Minute) + base := time.Unix(1_700_500_000, 0) + svc.bindTerminalSessionRoute("session-a", "node-a", base) + svc.updateTerminalSessionRouteLease("session-a", "node-a", base.Add(time.Minute).UnixMilli(), base) + candidates := svc.beginTerminalSessionRecovery("node-a", base) + session := newActiveSessionAt("node-a", "worker-session-a", ®istryv1.ConnectHello{ + Capabilities: []*registryv1.CapabilityDeclaration{{Name: taskCapabilityTerminalExec, MaxInflight: 1}}, + }, base) + session.setRecoveryCandidates(candidates) + report := ®istryv1.TerminalSessionRecoveryReport{Results: []*registryv1.TerminalSessionRecoveryResult{ + {SessionId: "session-a", Status: registryv1.TerminalSessionRecoveryResult_RECOVERED}, + }} + if err := svc.applyTerminalSessionRecoveryReport(session, report, base.Add(time.Second)); err != nil { + t.Fatalf("apply recovery report: %v", err) + } + session.markRecoveryComplete() + if !session.matchesRecoveryResults(report) { + t.Fatal("unchanged recovery report must be accepted as an idempotent retry") + } + changed := ®istryv1.TerminalSessionRecoveryReport{Results: []*registryv1.TerminalSessionRecoveryResult{ + {SessionId: "session-a", Status: registryv1.TerminalSessionRecoveryResult_MISSING}, + }} + if session.matchesRecoveryResults(changed) { + t.Fatal("changed recovery report must not be accepted as an idempotent retry") + } +} + +func TestTerminalRecoveryRejectsCandidateReassignedToAnotherWorker(t *testing.T) { + svc := NewRegistryService(registrytest.NewStore(t), nil, 5, 15, time.Minute) + base := time.Unix(1_700_600_000, 0) + svc.bindTerminalSessionRoute("session-a", "node-a", base) + svc.updateTerminalSessionRouteLease("session-a", "node-a", base.Add(time.Minute).UnixMilli(), base) + candidates := svc.beginTerminalSessionRecovery("node-a", base) + session := newActiveSessionAt("node-a", "worker-session-a", ®istryv1.ConnectHello{ + Capabilities: []*registryv1.CapabilityDeclaration{{Name: taskCapabilityTerminalExec, MaxInflight: 1}}, + }, base) + session.setRecoveryCandidates(candidates) + svc.bindTerminalSessionRoute("session-a", "node-b", base.Add(time.Second)) + err := svc.applyTerminalSessionRecoveryReport(session, ®istryv1.TerminalSessionRecoveryReport{ + Results: []*registryv1.TerminalSessionRecoveryResult{{ + SessionId: "session-a", Status: registryv1.TerminalSessionRecoveryResult_RECOVERED, + }}, + }, base.Add(2*time.Second)) + if err == nil { + t.Fatal("recovery report unexpectedly changed a route reassigned to another worker") + } + route, ok := svc.terminalSessionRouteSnapshot("session-a", base.Add(2*time.Second)) + if !ok || route.NodeID != "node-b" { + t.Fatalf("reassigned route changed: %#v ok=%v", route, ok) + } +} diff --git a/console/internal/grpcserver/terminal_session_routes.go b/console/internal/grpcserver/terminal_session_routes.go index 13f6ace..5d86152 100644 --- a/console/internal/grpcserver/terminal_session_routes.go +++ b/console/internal/grpcserver/terminal_session_routes.go @@ -1,8 +1,12 @@ package grpcserver import ( + "log/slog" + "sort" "strings" "time" + + registryv1 "github.com/onlyboxes/onlyboxes/api/gen/go/registry/v1" ) type routeReservationReleaseResult int @@ -14,14 +18,24 @@ const ( ) type terminalSessionRoute struct { - NodeID string - LastUsedUnixMs int64 + NodeID string + LastUsedUnixMs int64 + LeaseExpiresUnixMs int64 + RecoveryState terminalSessionRecoveryState // ReservationID is non-zero only while the first dispatch is provisional. // A successful result confirms the route by clearing it. ReservationID uint64 ProvisionalUses uint64 } +type terminalSessionRecoveryState uint8 + +const ( + terminalSessionRecoveryReady terminalSessionRecoveryState = iota + terminalSessionRecoveryUnavailable + terminalSessionRecoveryReconciling +) + func (s *RegistryService) bindTerminalSessionRoute(sessionID string, nodeID string, now time.Time) { if s == nil { return @@ -50,6 +64,7 @@ func (s *RegistryService) bindTerminalSessionRoute(sessionID string, nodeID stri s.terminalSessionToNode[normalizedSessionID] = terminalSessionRoute{ NodeID: normalizedNodeID, LastUsedUnixMs: nowUnixMs, + RecoveryState: terminalSessionRecoveryReady, ReservationID: 0, ProvisionalUses: 0, } @@ -95,6 +110,7 @@ func (s *RegistryService) reserveTerminalSessionRoute(sessionID string, preferre s.terminalSessionToNode[normalizedSessionID] = terminalSessionRoute{ NodeID: normalizedNodeID, LastUsedUnixMs: nowUnixMs, + RecoveryState: terminalSessionRecoveryReady, ReservationID: reservationID, ProvisionalUses: 1, } @@ -194,6 +210,197 @@ func (s *RegistryService) touchTerminalSessionRoute(sessionID string, now time.T return route.NodeID, true } +func (s *RegistryService) updateTerminalSessionRouteLease(sessionID string, expectedNodeID string, leaseExpiresUnixMs int64, now time.Time) bool { + if s == nil || leaseExpiresUnixMs <= 0 { + return false + } + normalizedSessionID := strings.TrimSpace(sessionID) + normalizedNodeID := strings.TrimSpace(expectedNodeID) + if normalizedSessionID == "" || normalizedNodeID == "" { + return false + } + + s.terminalRoutesMu.Lock() + defer s.terminalRoutesMu.Unlock() + route, ok := s.terminalSessionToNode[normalizedSessionID] + if !ok || route.NodeID != normalizedNodeID || route.ReservationID != 0 { + return false + } + route.LeaseExpiresUnixMs = leaseExpiresUnixMs + route.LastUsedUnixMs = routeNowUnixMs(now) + route.RecoveryState = terminalSessionRecoveryReady + s.terminalSessionToNode[normalizedSessionID] = route + return true +} + +func (s *RegistryService) terminalSessionRouteSnapshot(sessionID string, now time.Time) (terminalSessionRoute, bool) { + if s == nil { + return terminalSessionRoute{}, false + } + normalizedSessionID := strings.TrimSpace(sessionID) + if normalizedSessionID == "" { + return terminalSessionRoute{}, false + } + nowUnixMs := routeNowUnixMs(now) + s.terminalRoutesMu.Lock() + defer s.terminalRoutesMu.Unlock() + route, ok := s.terminalSessionToNode[normalizedSessionID] + if !ok { + return terminalSessionRoute{}, false + } + if route.LeaseExpiresUnixMs > 0 && route.LeaseExpiresUnixMs <= nowUnixMs { + s.deleteTerminalSessionRouteLocked(normalizedSessionID, route) + return terminalSessionRoute{}, false + } + return route, true +} + +func (s *RegistryService) beginTerminalSessionRecovery(nodeID string, now time.Time) []*registryv1.TerminalSessionRecoveryCandidate { + if s == nil { + return nil + } + normalizedNodeID := strings.TrimSpace(nodeID) + if normalizedNodeID == "" { + return nil + } + nowUnixMs := routeNowUnixMs(now) + s.terminalRoutesMu.Lock() + defer s.terminalRoutesMu.Unlock() + index := s.terminalNodeToSessionIDIndex[normalizedNodeID] + candidates := make([]*registryv1.TerminalSessionRecoveryCandidate, 0, len(index)) + for sessionID := range index { + route, ok := s.terminalSessionToNode[sessionID] + if !ok || route.NodeID != normalizedNodeID { + continue + } + leaseExpiresUnixMs := route.LeaseExpiresUnixMs + if leaseExpiresUnixMs <= 0 && s.terminalRouteTTL > 0 { + leaseExpiresUnixMs = route.LastUsedUnixMs + s.terminalRouteTTL.Milliseconds() + } + if route.ReservationID != 0 || leaseExpiresUnixMs <= nowUnixMs { + s.deleteTerminalSessionRouteLocked(sessionID, route) + continue + } + route.LeaseExpiresUnixMs = leaseExpiresUnixMs + route.RecoveryState = terminalSessionRecoveryReconciling + s.terminalSessionToNode[sessionID] = route + candidates = append(candidates, ®istryv1.TerminalSessionRecoveryCandidate{ + SessionId: sessionID, + LeaseExpiresUnixMs: leaseExpiresUnixMs, + }) + } + sort.Slice(candidates, func(i, j int) bool { return candidates[i].GetSessionId() < candidates[j].GetSessionId() }) + return candidates +} + +func (s *RegistryService) markTerminalSessionRoutesUnavailable(nodeID string) int { + if s == nil { + return 0 + } + normalizedNodeID := strings.TrimSpace(nodeID) + if normalizedNodeID == "" { + return 0 + } + s.terminalRoutesMu.Lock() + defer s.terminalRoutesMu.Unlock() + unavailable := 0 + for sessionID := range s.terminalNodeToSessionIDIndex[normalizedNodeID] { + route, ok := s.terminalSessionToNode[sessionID] + if !ok || route.NodeID != normalizedNodeID { + continue + } + if route.ReservationID != 0 { + s.deleteTerminalSessionRouteLocked(sessionID, route) + continue + } + route.RecoveryState = terminalSessionRecoveryUnavailable + s.terminalSessionToNode[sessionID] = route + unavailable++ + } + return unavailable +} + +func (s *RegistryService) applyTerminalSessionRecoveryReport(session *activeSession, report *registryv1.TerminalSessionRecoveryReport, now time.Time) error { + if s == nil || session == nil || report == nil { + return nil + } + candidates := session.recoveryCandidateSnapshot() + results := make(map[string]registryv1.TerminalSessionRecoveryResult_Status, len(report.GetResults())) + for _, result := range report.GetResults() { + if result == nil { + continue + } + sessionID := strings.TrimSpace(result.GetSessionId()) + if _, ok := candidates[sessionID]; !ok { + return &terminalRecoveryValidationError{message: "recovery result is not a candidate"} + } + if _, duplicate := results[sessionID]; duplicate { + return &terminalRecoveryValidationError{message: "duplicate recovery result"} + } + status := result.GetStatus() + if status != registryv1.TerminalSessionRecoveryResult_RECOVERED && status != registryv1.TerminalSessionRecoveryResult_MISSING && status != registryv1.TerminalSessionRecoveryResult_INVALID { + return &terminalRecoveryValidationError{message: "invalid recovery status"} + } + results[sessionID] = status + } + if len(results) != len(candidates) { + return &terminalRecoveryValidationError{message: "recovery report must cover every candidate"} + } + + nowUnixMs := routeNowUnixMs(now) + recovered := 0 + failures := 0 + s.terminalRoutesMu.Lock() + for sessionID := range candidates { + if route, ok := s.terminalSessionToNode[sessionID]; ok && route.NodeID != session.nodeID { + s.terminalRoutesMu.Unlock() + return &terminalRecoveryValidationError{message: "recovery candidate no longer belongs to worker"} + } + } + for sessionID, leaseExpiresUnixMs := range candidates { + route, ok := s.terminalSessionToNode[sessionID] + if !ok || route.NodeID != session.nodeID { + continue + } + status := results[sessionID] + if status != registryv1.TerminalSessionRecoveryResult_RECOVERED || (leaseExpiresUnixMs > 0 && leaseExpiresUnixMs <= nowUnixMs) { + s.deleteTerminalSessionRouteLocked(sessionID, route) + failures++ + continue + } + route.RecoveryState = terminalSessionRecoveryReady + route.LastUsedUnixMs = nowUnixMs + s.terminalSessionToNode[sessionID] = route + recovered++ + } + unavailable := s.countTerminalSessionRoutesByStateLocked(terminalSessionRecoveryUnavailable) + s.terminalRoutesMu.Unlock() + session.setRecoveryResults(results) + slog.Info( + "terminal session recovery metrics", + "executor_kind", session.executorKind, + "recovery_duration_ms", now.Sub(session.connectedAt).Milliseconds(), + "recovered_session_count", recovered, + "recovery_failures", failures, + "unavailable_route_count", unavailable, + ) + return nil +} + +func (s *RegistryService) countTerminalSessionRoutesByStateLocked(state terminalSessionRecoveryState) int { + count := 0 + for _, route := range s.terminalSessionToNode { + if route.RecoveryState == state && route.ReservationID == 0 { + count++ + } + } + return count +} + +type terminalRecoveryValidationError struct{ message string } + +func (e *terminalRecoveryValidationError) Error() string { return e.message } + // clearTerminalSessionRouteReservation releases one provisional dispatch. The // route is removed only after every dispatch sharing the reservation failed. func (s *RegistryService) clearTerminalSessionRouteReservation(sessionID string, expectedNodeID string, reservationID uint64) routeReservationReleaseResult { @@ -258,40 +465,11 @@ func (s *RegistryService) deleteTerminalSessionRouteLocked(sessionID string, rou } } -func (s *RegistryService) clearTerminalSessionRoutesByNode(nodeID string) { - if s == nil { - return - } - normalizedNodeID := strings.TrimSpace(nodeID) - if normalizedNodeID == "" { - return - } - - s.terminalRoutesMu.Lock() - defer s.terminalRoutesMu.Unlock() - - index := s.terminalNodeToSessionIDIndex[normalizedNodeID] - if index == nil { - return - } - for sessionID := range index { - route, ok := s.terminalSessionToNode[sessionID] - if !ok || route.NodeID != normalizedNodeID { - continue - } - delete(s.terminalSessionToNode, sessionID) - } - delete(s.terminalNodeToSessionIDIndex, normalizedNodeID) -} - func (s *RegistryService) pruneExpiredTerminalSessionRoutes(now time.Time) int { if s == nil { return 0 } ttl := s.terminalRouteTTL - if ttl <= 0 { - return 0 - } nowUnixMs := routeNowUnixMs(now) expireBefore := nowUnixMs - ttl.Milliseconds() @@ -300,7 +478,11 @@ func (s *RegistryService) pruneExpiredTerminalSessionRoutes(now time.Time) int { defer s.terminalRoutesMu.Unlock() for sessionID, route := range s.terminalSessionToNode { - if route.LastUsedUnixMs > expireBefore { + if route.LeaseExpiresUnixMs > 0 { + if route.LeaseExpiresUnixMs > nowUnixMs { + continue + } + } else if ttl <= 0 || route.LastUsedUnixMs > expireBefore { continue } delete(s.terminalSessionToNode, sessionID) diff --git a/console/internal/httpapi/command_handler.go b/console/internal/httpapi/command_handler.go index 0c7bf20..ce1695f 100644 --- a/console/internal/httpapi/command_handler.go +++ b/console/internal/httpapi/command_handler.go @@ -13,24 +13,25 @@ import ( ) const ( - defaultEchoTimeoutMS = 5000 - minEchoTimeoutMS = 1 - maxEchoTimeoutMS = 60000 - defaultTerminalTimeoutMS = defaultTaskTimeoutMS - minTerminalTimeoutMS = 1 - maxTerminalTimeoutMS = maxTaskTimeoutMS - defaultComputerUseTimeoutMS = defaultTaskTimeoutMS - minComputerUseTimeoutMS = 1 - maxComputerUseTimeoutMS = maxTaskTimeoutMS - terminalExecCapability = "terminalExec" - computerUseCapability = "computerUse" - terminalExecSessionNotFoundCode = "session_not_found" - terminalExecSessionBusyCode = "session_busy" - terminalExecSessionCapacityCode = "session_capacity_exceeded" - terminalExecInvalidPayloadCode = "invalid_payload" - terminalTaskNoWorkerCode = "no_worker" - terminalTaskNoCapacityCode = "no_capacity" - terminalTaskTimeoutCode = "timeout" + defaultEchoTimeoutMS = 5000 + minEchoTimeoutMS = 1 + maxEchoTimeoutMS = 60000 + defaultTerminalTimeoutMS = defaultTaskTimeoutMS + minTerminalTimeoutMS = 1 + maxTerminalTimeoutMS = maxTaskTimeoutMS + defaultComputerUseTimeoutMS = defaultTaskTimeoutMS + minComputerUseTimeoutMS = 1 + maxComputerUseTimeoutMS = maxTaskTimeoutMS + terminalExecCapability = "terminalExec" + computerUseCapability = "computerUse" + terminalExecSessionNotFoundCode = "session_not_found" + terminalExecSessionBusyCode = "session_busy" + terminalExecSessionCapacityCode = "session_capacity_exceeded" + terminalExecSessionUnavailableCode = "session_unavailable" + terminalExecInvalidPayloadCode = "invalid_payload" + terminalTaskNoWorkerCode = "no_worker" + terminalTaskNoCapacityCode = "no_capacity" + terminalTaskTimeoutCode = "timeout" ) type EchoDispatcher interface { @@ -320,6 +321,8 @@ func mapTerminalTaskFailure(task grpcserver.TaskSnapshot) (int, string) { return http.StatusConflict, message case terminalExecSessionCapacityCode: return http.StatusTooManyRequests, message + case terminalExecSessionUnavailableCode: + return http.StatusServiceUnavailable, message case terminalTaskNoWorkerCode: return http.StatusServiceUnavailable, "no online worker supports requested capability" case terminalTaskNoCapacityCode: diff --git a/console/internal/httpapi/command_handler_test.go b/console/internal/httpapi/command_handler_test.go index e1263cf..d6891fb 100644 --- a/console/internal/httpapi/command_handler_test.go +++ b/console/internal/httpapi/command_handler_test.go @@ -260,6 +260,7 @@ func TestTerminalCommandStatusMappings(t *testing.T) { {name: "session_not_found", errorCode: terminalExecSessionNotFoundCode, statusCode: http.StatusNotFound}, {name: "session_busy", errorCode: terminalExecSessionBusyCode, statusCode: http.StatusConflict}, {name: "session_capacity_exceeded", errorCode: terminalExecSessionCapacityCode, statusCode: http.StatusTooManyRequests}, + {name: "session_unavailable", errorCode: terminalExecSessionUnavailableCode, statusCode: http.StatusServiceUnavailable}, {name: "invalid_payload", errorCode: terminalExecInvalidPayloadCode, statusCode: http.StatusBadRequest}, {name: "no_capacity", errorCode: terminalTaskNoCapacityCode, statusCode: http.StatusTooManyRequests}, {name: "no_worker", errorCode: terminalTaskNoWorkerCode, statusCode: http.StatusServiceUnavailable}, diff --git a/console/internal/httpapi/integration_test.go b/console/internal/httpapi/integration_test.go index d5c8b10..616132c 100644 --- a/console/internal/httpapi/integration_test.go +++ b/console/internal/httpapi/integration_test.go @@ -615,6 +615,7 @@ func TestTerminalLifecycle(t *testing.T) { if connectResp.GetConnectAck() == nil { t.Fatalf("expected connect_ack, got %#v", connectResp.GetPayload()) } + completeEmptyTerminalRecovery(t, stream, connectResp.GetConnectAck()) go func() { sessionContent := map[string]string{} @@ -1116,6 +1117,7 @@ func TestTokenIsolationLifecycle(t *testing.T) { if connectResp.GetConnectAck() == nil { t.Fatalf("expected connect_ack, got %#v", connectResp.GetPayload()) } + completeEmptyTerminalRecovery(t, stream, connectResp.GetConnectAck()) go func() { sessionContent := map[string]string{} @@ -1943,6 +1945,31 @@ func requestList(t *testing.T, client *http.Client, url string) listWorkersRespo return payload } +func completeEmptyTerminalRecovery( + t *testing.T, + stream grpc.BidiStreamingClient[registryv1.ConnectRequest, registryv1.ConnectResponse], + ack *registryv1.ConnectAck, +) { + t.Helper() + if len(ack.GetTerminalSessionRecoveryCandidates()) != 0 { + t.Fatalf("new test worker unexpectedly received recovery candidates: %#v", ack.GetTerminalSessionRecoveryCandidates()) + } + if err := stream.Send(®istryv1.ConnectRequest{ + Payload: ®istryv1.ConnectRequest_TerminalSessionRecoveryReport{ + TerminalSessionRecoveryReport: ®istryv1.TerminalSessionRecoveryReport{}, + }, + }); err != nil { + t.Fatalf("send empty terminal recovery report: %v", err) + } + resp, err := stream.Recv() + if err != nil { + t.Fatalf("receive terminal recovery ack: %v", err) + } + if resp.GetTerminalSessionRecoveryAck() == nil { + t.Fatalf("expected terminal recovery ack, got %#v", resp.GetPayload()) + } +} + func newAuthenticatedClient(t *testing.T, server *httptest.Server) *http.Client { t.Helper() diff --git a/console/internal/httpapi/task_handler.go b/console/internal/httpapi/task_handler.go index e69d551..b588654 100644 --- a/console/internal/httpapi/task_handler.go +++ b/console/internal/httpapi/task_handler.go @@ -170,7 +170,11 @@ func (h *WorkerHandler) writeTaskSubmitError(c *gin.Context, err error) { case errors.Is(err, grpcserver.ErrNoWorkerCapacity): c.JSON(http.StatusTooManyRequests, gin.H{"error": "no online worker capacity for requested capability"}) case errors.As(err, &commandErr): - c.JSON(http.StatusBadGateway, gin.H{"error": commandErr.Error()}) + if strings.EqualFold(strings.TrimSpace(commandErr.Code), terminalExecSessionUnavailableCode) { + c.JSON(http.StatusServiceUnavailable, gin.H{"error": commandErr.Error()}) + } else { + c.JSON(http.StatusBadGateway, gin.H{"error": commandErr.Error()}) + } case errors.Is(err, context.DeadlineExceeded): c.JSON(http.StatusGatewayTimeout, gin.H{"error": "task timed out"}) case status.Code(err) == codes.InvalidArgument: @@ -192,7 +196,7 @@ func mapTaskTerminalStatusToHTTP(task grpcserver.TaskSnapshot) int { switch task.ErrorCode { case "no_capacity", terminalExecSessionCapacityCode: return http.StatusTooManyRequests - case "no_worker": + case "no_worker", terminalExecSessionUnavailableCode: return http.StatusServiceUnavailable default: return http.StatusBadGateway diff --git a/console/internal/httpapi/task_handler_test.go b/console/internal/httpapi/task_handler_test.go index 6278eb9..e14b1e5 100644 --- a/console/internal/httpapi/task_handler_test.go +++ b/console/internal/httpapi/task_handler_test.go @@ -313,6 +313,16 @@ func TestMapTaskTerminalStatusToHTTPSessionCapacity(t *testing.T) { } } +func TestMapTaskTerminalStatusToHTTPSessionUnavailable(t *testing.T) { + task := grpcserver.TaskSnapshot{ + Status: grpcserver.TaskStatusFailed, + ErrorCode: terminalExecSessionUnavailableCode, + } + if got := mapTaskTerminalStatusToHTTP(task); got != http.StatusServiceUnavailable { + t.Fatalf("expected 503, got %d", got) + } +} + func TestCancelTaskTerminalConflict(t *testing.T) { now := time.Unix(1_700_000_000, 0) handler := NewWorkerHandler(registrytest.NewStore(t), 15*time.Second, &fakeTaskDispatcher{ From 439da48718ff91b9ce3c189c62003bf311daad2d Mon Sep 17 00:00:00 2001 From: Coolfan Date: Fri, 7 Aug 2026 22:23:29 +0800 Subject: [PATCH 3/7] feat(worker-docker): recover terminal sessions after restart Use deterministic container names and schema/session-hash labels so terminal containers survive worker exit. On reconnect, reconcile Console candidates before accepting commands: restart stopped containers, restore the exact lease, and remove local orphans Console no longer recognizes. PreserveOnClose keeps containers running on graceful shutdown. --- worker/worker-docker/README/overview.md | 5 +- .../worker-docker/internal/runner/runner.go | 4 + .../internal/runner/runner_test.go | 27 ++ .../internal/runner/session_client.go | 54 ++++ .../internal/runner/session_recovery.go | 209 ++++++++++++++ .../session_recovery_integration_test.go | 184 ++++++++++++ .../internal/runner/session_recovery_test.go | 263 ++++++++++++++++++ .../internal/runner/terminal_exec.go | 37 ++- .../internal/runner/terminal_exec_test.go | 7 +- 9 files changed, 775 insertions(+), 15 deletions(-) create mode 100644 worker/worker-docker/internal/runner/session_recovery.go create mode 100644 worker/worker-docker/internal/runner/session_recovery_integration_test.go create mode 100644 worker/worker-docker/internal/runner/session_recovery_test.go diff --git a/worker/worker-docker/README/overview.md b/worker/worker-docker/README/overview.md index 45442d2..5f0cdf6 100644 --- a/worker/worker-docker/README/overview.md +++ b/worker/worker-docker/README/overview.md @@ -70,8 +70,9 @@ Capability behavior: - `terminalExec` cleanup behavior: - command timeout/cancel marks the session for destruction and stops it accepting new commands; the container is removed once in-flight commands drain, so one command's timeout does not kill its siblings. - idle sessions are reaped after lease expiry by an internal janitor loop; a session with in-flight commands is never reaped. - - worker shutdown force-removes all managed terminal containers. - - `SIGINT`/`SIGTERM` (for example Ctrl+C) performs best-effort cleanup; `SIGKILL`/process crash does not guarantee cleanup. + - terminal containers use deterministic `onlyboxes-terminal-v1-` names plus schema/session-hash labels and are preserved when the worker exits. + - after reconnect, the worker reconciles Console candidates before accepting commands, restarts stopped containers, restores the exact lease, and removes local Onlyboxes terminal orphans that Console no longer recognizes. + - lease expiry, explicit destruction, unsafe command timeout, and invalid resource identity still remove the container; one-shot `pythonExec` containers remain per-call resources. - `terminalExec` result uses JSON payload: - `{"session_id":"...","created":true,"stdout":"...","stderr":"...","exit_code":0,"stdout_truncated":false,"stderr_truncated":false,"lease_expires_unix_ms":...}` - output truncation: diff --git a/worker/worker-docker/internal/runner/runner.go b/worker/worker-docker/internal/runner/runner.go index d017c77..c1fd44f 100644 --- a/worker/worker-docker/internal/runner/runner.go +++ b/worker/worker-docker/internal/runner/runner.go @@ -69,6 +69,7 @@ func Run(ctx context.Context, cfg config.Config) error { PidsLimit: cfg.TerminalExecPidsLimit, SessionMaxInflight: cfg.TerminalSessionMaxInflight, MaxActiveSessions: cfg.TerminalMaxActiveSessions, + PreserveOnClose: true, }) pythonRunner := newPythonExecRunner( cfg.PythonExecDockerImage, @@ -84,11 +85,14 @@ func Run(ctx context.Context, cfg config.Config) error { runTerminalResource = terminalManager.ResolveResource originalActiveSessionCountFn := activeSessionCountFn activeSessionCountFn = terminalManager.ActiveSessionCount + originalRecoverTerminalSessionsFn := recoverTerminalSessionsFn + recoverTerminalSessionsFn = terminalManager.Recover defer func() { runPythonExec = originalRunPythonExec runTerminalExec = originalRunTerminalExec runTerminalResource = originalRunTerminalResource activeSessionCountFn = originalActiveSessionCountFn + recoverTerminalSessionsFn = originalRecoverTerminalSessionsFn terminalManager.Close() }() diff --git a/worker/worker-docker/internal/runner/runner_test.go b/worker/worker-docker/internal/runner/runner_test.go index b38679c..bb93b49 100644 --- a/worker/worker-docker/internal/runner/runner_test.go +++ b/worker/worker-docker/internal/runner/runner_test.go @@ -29,6 +29,19 @@ func TestRunReturnsContextCanceled(t *testing.T) { } } +func TestTerminalRecoveryTimesOutBeforeWorkerBecomesReady(t *testing.T) { + original := recoverTerminalSessionsFn + t.Cleanup(func() { recoverTerminalSessionsFn = original }) + recoverTerminalSessionsFn = func(ctx context.Context, _ []*registryv1.TerminalSessionRecoveryCandidate) []*registryv1.TerminalSessionRecoveryResult { + <-ctx.Done() + return nil + } + _, err := recoverTerminalSessionsWithTimeout(context.Background(), 10*time.Millisecond, nil) + if !errors.Is(err, context.DeadlineExceeded) { + t.Fatalf("expected recovery deadline, got %v", err) + } +} + func TestRunWaitsBeforeReconnectOnSessionFailure(t *testing.T) { originalWaitReconnect := waitReconnect waitCalls := 0 @@ -922,6 +935,20 @@ func (s *fakeRegistryService) Connect(stream grpc.BidiStreamingServer[registryv1 }); err != nil { return err } + recoveryReq, err := stream.Recv() + if err != nil { + return err + } + if recoveryReq.GetTerminalSessionRecoveryReport() == nil { + return status.Error(codes.InvalidArgument, "terminal session recovery report is required") + } + if err := stream.Send(®istryv1.ConnectResponse{ + Payload: ®istryv1.ConnectResponse_TerminalSessionRecoveryAck{ + TerminalSessionRecoveryAck: ®istryv1.TerminalSessionRecoveryAck{}, + }, + }); err != nil { + return err + } for { req, err := stream.Recv() diff --git a/worker/worker-docker/internal/runner/session_client.go b/worker/worker-docker/internal/runner/session_client.go index d1d45d9..003ab43 100644 --- a/worker/worker-docker/internal/runner/session_client.go +++ b/worker/worker-docker/internal/runner/session_client.go @@ -60,6 +60,25 @@ func runSession(ctx context.Context, cfg config.Config) error { } heartbeatInterval := durationFromServer(ack.GetHeartbeatIntervalSec(), cfg.HeartbeatInterval) + recoveryResults, err := recoverTerminalSessionsWithTimeout(ctx, cfg.CallTimeout, ack.GetTerminalSessionRecoveryCandidates()) + if err != nil { + return fmt.Errorf("recover terminal sessions: %w", err) + } + if err := stream.Send(®istryv1.ConnectRequest{ + Payload: ®istryv1.ConnectRequest_TerminalSessionRecoveryReport{ + TerminalSessionRecoveryReport: ®istryv1.TerminalSessionRecoveryReport{Results: recoveryResults}, + }, + }); err != nil { + return fmt.Errorf("send terminal session recovery report: %w", err) + } + recoveryResp, err := recvWithTimeout(ctx, cfg.CallTimeout, stream.Recv) + if err != nil { + return fmt.Errorf("recv terminal session recovery ack: %w", err) + } + if recoveryResp.GetTerminalSessionRecoveryAck() == nil { + return fmt.Errorf("unexpected response while waiting for terminal session recovery ack") + } + logTerminalRecoverySummary(recoveryResults) logging.Infof("worker connected: node_id=%s node_name=%s session_id=%s", hello.GetNodeId(), hello.GetNodeName(), sessionID) sessionCtx, cancel := context.WithCancel(ctx) @@ -75,6 +94,41 @@ func runSession(ctx context.Context, cfg config.Config) error { return heartbeatLoop(sessionCtx, outbound, heartbeatAckCh, sessionErrCh, cfg, sessionID, heartbeatInterval) } +func recoverTerminalSessionsWithTimeout( + ctx context.Context, + timeout time.Duration, + candidates []*registryv1.TerminalSessionRecoveryCandidate, +) ([]*registryv1.TerminalSessionRecoveryResult, error) { + recoveryCtx, cancel := context.WithTimeout(ctx, timeout) + defer cancel() + resultCh := make(chan []*registryv1.TerminalSessionRecoveryResult, 1) + go func() { + resultCh <- recoverTerminalSessionsFn(recoveryCtx, candidates) + }() + select { + case <-recoveryCtx.Done(): + return nil, recoveryCtx.Err() + case results := <-resultCh: + return results, nil + } +} + +func logTerminalRecoverySummary(results []*registryv1.TerminalSessionRecoveryResult) { + counts := map[registryv1.TerminalSessionRecoveryResult_Status]int{} + for _, result := range results { + if result != nil { + counts[result.GetStatus()]++ + } + } + logging.Infof( + "terminal session recovery completed: candidates=%d recovered=%d missing=%d invalid=%d", + len(results), + counts[registryv1.TerminalSessionRecoveryResult_RECOVERED], + counts[registryv1.TerminalSessionRecoveryResult_MISSING], + counts[registryv1.TerminalSessionRecoveryResult_INVALID], + ) +} + func dial(ctx context.Context, cfg config.Config) (*grpc.ClientConn, error) { if err := ctx.Err(); err != nil { return nil, err diff --git a/worker/worker-docker/internal/runner/session_recovery.go b/worker/worker-docker/internal/runner/session_recovery.go new file mode 100644 index 0000000..e2454ce --- /dev/null +++ b/worker/worker-docker/internal/runner/session_recovery.go @@ -0,0 +1,209 @@ +package runner + +import ( + "context" + "encoding/json" + "strings" + "time" + + registryv1 "github.com/onlyboxes/onlyboxes/api/gen/go/registry/v1" + "github.com/onlyboxes/onlyboxes/worker/worker-docker/internal/logging" +) + +var recoverTerminalSessionsFn = func( + _ context.Context, + candidates []*registryv1.TerminalSessionRecoveryCandidate, +) []*registryv1.TerminalSessionRecoveryResult { + results := make([]*registryv1.TerminalSessionRecoveryResult, 0, len(candidates)) + for _, candidate := range candidates { + if candidate == nil { + continue + } + results = append(results, ®istryv1.TerminalSessionRecoveryResult{ + SessionId: candidate.GetSessionId(), + Status: registryv1.TerminalSessionRecoveryResult_MISSING, + }) + } + return results +} + +func (m *terminalSessionManager) Recover( + ctx context.Context, + candidates []*registryv1.TerminalSessionRecoveryCandidate, +) []*registryv1.TerminalSessionRecoveryResult { + startedAt := time.Now() + results := make([]*registryv1.TerminalSessionRecoveryResult, 0, len(candidates)) + expectedNames := make(map[string]struct{}, len(candidates)) + now := time.Now() + recoveredCount := 0 + missingCount := 0 + invalidCount := 0 + + for _, candidate := range candidates { + if candidate == nil { + continue + } + sessionID := strings.TrimSpace(candidate.GetSessionId()) + name := terminalSessionResourceName(sessionID) + status := registryv1.TerminalSessionRecoveryResult_INVALID + leaseExpiresAt := time.UnixMilli(candidate.GetLeaseExpiresUnixMs()) + if sessionID != "" && candidate.GetLeaseExpiresUnixMs() > 0 && leaseExpiresAt.After(now) { + status = m.recoverOne(ctx, sessionID, name, leaseExpiresAt) + } else if sessionID != "" { + m.forceRemoveContainer(name) + } + switch status { + case registryv1.TerminalSessionRecoveryResult_RECOVERED: + expectedNames[name] = struct{}{} + recoveredCount++ + case registryv1.TerminalSessionRecoveryResult_MISSING: + missingCount++ + default: + invalidCount++ + } + results = append(results, ®istryv1.TerminalSessionRecoveryResult{ + SessionId: sessionID, + Status: status, + }) + } + + discovered, orphanCleaned := m.cleanupOrphanContainers(ctx, expectedNames) + logging.Infof( + "terminal recovery completed: executor_kind=docker discovered=%d candidate=%d recovered=%d missing=%d invalid=%d orphan_cleaned=%d duration_ms=%d recovery_failures=%d", + discovered, len(results), recoveredCount, missingCount, invalidCount, orphanCleaned, + time.Since(startedAt).Milliseconds(), invalidCount, + ) + return results +} + +func (m *terminalSessionManager) recoverOne( + ctx context.Context, + sessionID string, + containerName string, + leaseExpiresAt time.Time, +) registryv1.TerminalSessionRecoveryResult_Status { + m.mu.Lock() + if m.closed { + m.mu.Unlock() + return registryv1.TerminalSessionRecoveryResult_INVALID + } + if existing := m.sessions[sessionID]; existing != nil && !existing.destroying { + existing.leaseExpiresAt = leaseExpiresAt + m.mu.Unlock() + return registryv1.TerminalSessionRecoveryResult_RECOVERED + } + m.mu.Unlock() + + inspect := runDockerCommand(ctx, + "inspect", + "--format", "{{json .Config.Labels}}\t{{.State.Status}}", + containerName, + ) + if inspect.Err != nil || inspect.ExitCode != 0 { + if inspect.Err == nil && isNoSuchContainerMessage(inspect.Stderr) { + return registryv1.TerminalSessionRecoveryResult_MISSING + } + return registryv1.TerminalSessionRecoveryResult_MISSING + } + parts := strings.SplitN(strings.TrimSpace(inspect.Stdout), "\t", 2) + if len(parts) != 2 { + return registryv1.TerminalSessionRecoveryResult_INVALID + } + labels := map[string]string{} + if err := json.Unmarshal([]byte(parts[0]), &labels); err != nil || + labels[terminalExecSessionLabelKey] != terminalSessionIDHash(sessionID) || + labels[terminalExecSchemaLabelKey] != terminalExecSchemaVersion { + m.forceRemoveContainer(containerName) + return registryv1.TerminalSessionRecoveryResult_INVALID + } + matching := runDockerCommand(ctx, + "ps", "-a", + "--filter", "label="+terminalExecSessionLabelKey+"="+terminalSessionIDHash(sessionID), + "--filter", "label="+terminalExecSchemaLabelKey+"="+terminalExecSchemaVersion, + "--format", "{{.Names}}", + ) + if matching.Err != nil || matching.ExitCode != 0 { + return registryv1.TerminalSessionRecoveryResult_INVALID + } + matchingNames := nonEmptyLines(matching.Stdout) + if len(matchingNames) != 1 || matchingNames[0] != containerName { + for _, name := range matchingNames { + m.forceRemoveContainer(name) + } + return registryv1.TerminalSessionRecoveryResult_INVALID + } + + switch strings.TrimSpace(strings.ToLower(parts[1])) { + case "running": + case "created", "exited", "stopped": + start := runDockerCommand(ctx, terminalExecDockerStartArgs(containerName)...) + if start.Err != nil || start.ExitCode != 0 { + return registryv1.TerminalSessionRecoveryResult_INVALID + } + default: + m.forceRemoveContainer(containerName) + return registryv1.TerminalSessionRecoveryResult_INVALID + } + + ready := make(chan struct{}) + close(ready) + m.mu.Lock() + defer m.mu.Unlock() + if m.closed { + return registryv1.TerminalSessionRecoveryResult_INVALID + } + if existing := m.sessions[sessionID]; existing != nil { + if existing.containerName != containerName || existing.destroying { + return registryv1.TerminalSessionRecoveryResult_INVALID + } + existing.leaseExpiresAt = leaseExpiresAt + return registryv1.TerminalSessionRecoveryResult_RECOVERED + } + m.sessions[sessionID] = &terminalSession{ + sessionID: sessionID, + containerName: containerName, + leaseExpiresAt: leaseExpiresAt, + ready: ready, + capacityReserved: true, + } + m.activeSessionReservations++ + return registryv1.TerminalSessionRecoveryResult_RECOVERED +} + +func nonEmptyLines(value string) []string { + lines := make([]string, 0) + for _, line := range strings.Split(value, "\n") { + if trimmed := strings.TrimSpace(line); trimmed != "" { + lines = append(lines, trimmed) + } + } + return lines +} + +func (m *terminalSessionManager) cleanupOrphanContainers(ctx context.Context, expectedNames map[string]struct{}) (int, int) { + listed := runDockerCommand(ctx, + "ps", "-a", + "--filter", "label="+terminalExecSessionLabelKey, + "--filter", "label="+terminalExecSchemaLabelKey+"="+terminalExecSchemaVersion, + "--format", "{{.Names}}", + ) + if listed.Err != nil || listed.ExitCode != 0 { + logging.Warnf("terminal recovery orphan discovery failed") + return 0, 0 + } + discovered := 0 + cleaned := 0 + for _, line := range strings.Split(listed.Stdout, "\n") { + name := strings.TrimSpace(line) + if !strings.HasPrefix(name, terminalExecContainerPrefix) { + continue + } + discovered++ + if _, expected := expectedNames[name]; expected { + continue + } + m.forceRemoveContainer(name) + cleaned++ + } + return discovered, cleaned +} diff --git a/worker/worker-docker/internal/runner/session_recovery_integration_test.go b/worker/worker-docker/internal/runner/session_recovery_integration_test.go new file mode 100644 index 0000000..3260dfd --- /dev/null +++ b/worker/worker-docker/internal/runner/session_recovery_integration_test.go @@ -0,0 +1,184 @@ +package runner + +import ( + "context" + "fmt" + "os" + "os/exec" + "strings" + "testing" + "time" + + registryv1 "github.com/onlyboxes/onlyboxes/api/gen/go/registry/v1" +) + +func TestIntegrationTerminalContainerSurvivesManagerRestart(t *testing.T) { + if os.Getenv("DOCKER_INTEGRATION") != "1" { + t.Skip("set DOCKER_INTEGRATION=1 to run against Docker") + } + sessionID := "integration:docker-session-recovery" + containerName := terminalSessionResourceName(sessionID) + t.Cleanup(func() { + cleanupCtx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + defer cancel() + _ = runDockerCommand(cleanupCtx, pythonExecDockerRemoveArgs(containerName)...) + }) + + baseConfig := terminalSessionManagerConfig{ + LeaseMinSec: 1, + LeaseMaxSec: 600, + LeaseDefaultSec: 300, + DockerImage: defaultTerminalExecDockerImage, + SessionMaxInflight: 1, + } + firstConfig := baseConfig + firstConfig.PreserveOnClose = true + first := newTerminalSessionManager(firstConfig) + ctx, cancel := context.WithTimeout(context.Background(), time.Minute) + defer cancel() + created, err := first.Execute(ctx, terminalExecRequest{ + Command: "printf recovery-ok > /tmp/onlyboxes-recovery.txt", + SessionID: sessionID, + CreateIfMissing: true, + LeaseTTLSec: intPointerForDockerRecovery(300), + }) + if err != nil { + first.Close() + t.Fatal(err) + } + if !created.Created { + first.Close() + t.Fatal("initial session was not created") + } + first.Close() + + lease := time.UnixMilli(created.LeaseExpiresUnixMS) + second := newTerminalSessionManager(baseConfig) + defer second.Close() + if status := second.recoverOne(ctx, sessionID, containerName, lease); status != registryv1.TerminalSessionRecoveryResult_RECOVERED { + t.Fatalf("recover status=%s", status) + } + recovered, err := second.Execute(ctx, terminalExecRequest{ + Command: "cat /tmp/onlyboxes-recovery.txt", + SessionID: sessionID, + LeaseTTLSec: intPointerForDockerRecovery(300), + }) + if err != nil { + t.Fatal(err) + } + if recovered.Created || recovered.Stdout != "recovery-ok" { + t.Fatalf("unexpected recovered result: %#v", recovered) + } +} + +func intPointerForDockerRecovery(value int) *int { return &value } + +func TestIntegrationTerminalContainerSurvivesForcedWorkerTermination(t *testing.T) { + if os.Getenv("DOCKER_INTEGRATION") != "1" { + t.Skip("set DOCKER_INTEGRATION=1 to run against Docker") + } + if os.Getenv("DOCKER_RECOVERY_CRASH_HELPER") == "1" { + runDockerRecoveryCrashHelper(t) + return + } + + sessionID := fmt.Sprintf("integration:docker-crash-recovery-%d", time.Now().UnixNano()) + containerName := terminalSessionResourceName(sessionID) + readyFile, err := os.CreateTemp("", "onlyboxes-docker-crash-ready-*") + if err != nil { + t.Fatal(err) + } + readyPath := readyFile.Name() + _ = readyFile.Close() + _ = os.Remove(readyPath) + t.Cleanup(func() { + _ = os.Remove(readyPath) + cleanupCtx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + defer cancel() + _ = runDockerCommand(cleanupCtx, pythonExecDockerRemoveArgs(containerName)...) + }) + cmd := exec.Command(os.Args[0], "-test.run=^TestIntegrationTerminalContainerSurvivesForcedWorkerTermination$", "-test.v") + cmd.Env = append(os.Environ(), "DOCKER_RECOVERY_CRASH_HELPER=1", "DOCKER_RECOVERY_SESSION_ID="+sessionID, "DOCKER_RECOVERY_READY_FILE="+readyPath) + cmd.Stdout = os.Stderr + cmd.Stderr = os.Stderr + if err := cmd.Start(); err != nil { + t.Fatal(err) + } + waitCh := make(chan error, 1) + go func() { waitCh <- cmd.Wait() }() + leaseUnixMs := int64(0) + deadline := time.NewTimer(time.Minute) + defer deadline.Stop() + ticker := time.NewTicker(25 * time.Millisecond) + defer ticker.Stop() + for leaseUnixMs == 0 { + select { + case err := <-waitCh: + t.Fatalf("crash helper exited before creating container: %v", err) + case <-deadline.C: + _ = cmd.Process.Kill() + <-waitCh + t.Fatal("timed out waiting for crash helper") + case <-ticker.C: + content, readErr := os.ReadFile(readyPath) + if readErr != nil { + continue + } + if _, scanErr := fmt.Sscan(strings.TrimSpace(string(content)), &leaseUnixMs); scanErr != nil { + t.Fatalf("parse helper lease: %v", scanErr) + } + } + } + if err := cmd.Process.Kill(); err != nil { + t.Fatal(err) + } + if err := <-waitCh; err == nil { + t.Fatal("crash helper was not force terminated") + } + + manager := newTerminalSessionManager(terminalSessionManagerConfig{ + LeaseMinSec: 1, LeaseMaxSec: 600, LeaseDefaultSec: 300, + DockerImage: defaultTerminalExecDockerImage, SessionMaxInflight: 1, + }) + defer manager.Close() + ctx, cancel := context.WithTimeout(context.Background(), time.Minute) + defer cancel() + results := manager.Recover(ctx, []*registryv1.TerminalSessionRecoveryCandidate{{ + SessionId: sessionID, LeaseExpiresUnixMs: leaseUnixMs, + }}) + if len(results) != 1 || results[0].GetStatus() != registryv1.TerminalSessionRecoveryResult_RECOVERED { + t.Fatalf("unexpected recovery result: %#v", results) + } + recovered, err := manager.Execute(ctx, terminalExecRequest{ + Command: "cat /tmp/onlyboxes-forced-recovery.txt", SessionID: sessionID, + LeaseTTLSec: intPointerForDockerRecovery(1), + }) + if err != nil { + t.Fatal(err) + } + if recovered.Created || recovered.Stdout != "forced-recovery-ok" { + t.Fatalf("unexpected recovered result: %#v", recovered) + } +} + +func runDockerRecoveryCrashHelper(t *testing.T) { + sessionID := strings.TrimSpace(os.Getenv("DOCKER_RECOVERY_SESSION_ID")) + manager := newTerminalSessionManager(terminalSessionManagerConfig{ + LeaseMinSec: 1, LeaseMaxSec: 600, LeaseDefaultSec: 300, + DockerImage: defaultTerminalExecDockerImage, SessionMaxInflight: 1, PreserveOnClose: true, + }) + ctx, cancel := context.WithTimeout(context.Background(), time.Minute) + defer cancel() + created, err := manager.Execute(ctx, terminalExecRequest{ + Command: "printf forced-recovery-ok > /tmp/onlyboxes-forced-recovery.txt", + SessionID: sessionID, CreateIfMissing: true, LeaseTTLSec: intPointerForDockerRecovery(300), + }) + if err != nil { + t.Fatal(err) + } + readyPath := strings.TrimSpace(os.Getenv("DOCKER_RECOVERY_READY_FILE")) + if err := os.WriteFile(readyPath, []byte(fmt.Sprintf("%d", created.LeaseExpiresUnixMS)), 0o600); err != nil { + t.Fatal(err) + } + select {} +} diff --git a/worker/worker-docker/internal/runner/session_recovery_test.go b/worker/worker-docker/internal/runner/session_recovery_test.go new file mode 100644 index 0000000..18a3a50 --- /dev/null +++ b/worker/worker-docker/internal/runner/session_recovery_test.go @@ -0,0 +1,263 @@ +package runner + +import ( + "context" + "fmt" + "strings" + "testing" + "time" + + registryv1 "github.com/onlyboxes/onlyboxes/api/gen/go/registry/v1" +) + +func TestTerminalRecoveryResourceNameUsesFullHashWithoutSessionID(t *testing.T) { + sessionID := "owner-secret:session-secret" + name := terminalSessionResourceName(sessionID) + if !strings.HasPrefix(name, terminalExecContainerPrefix) || len(name) != len(terminalExecContainerPrefix)+64 { + t.Fatalf("unexpected recovery resource name %q", name) + } + if strings.Contains(name, "owner-secret") || name != terminalSessionResourceName(sessionID) { + t.Fatalf("resource name exposed or did not deterministically map the session: %q", name) + } +} + +func TestTerminalRecoveryRestoresContainerAndExactLease(t *testing.T) { + original := runDockerCommand + t.Cleanup(func() { runDockerCommand = original }) + + sessionID := "owner-a:session-a" + name := terminalSessionResourceName(sessionID) + hash := terminalSessionIDHash(sessionID) + runDockerCommand = func(_ context.Context, args ...string) dockerCommandResult { + joined := strings.Join(args, " ") + switch { + case len(args) > 0 && args[0] == "inspect": + return dockerCommandResult{ExitCode: 0, Stdout: fmt.Sprintf("{\"%s\":\"%s\",\"%s\":\"1\"}\trunning\n", terminalExecSessionLabelKey, hash, terminalExecSchemaLabelKey)} + case strings.Contains(joined, "label="+terminalExecSessionLabelKey+"="+hash): + return dockerCommandResult{ExitCode: 0, Stdout: name + "\n"} + case len(args) > 0 && args[0] == "ps": + return dockerCommandResult{ExitCode: 0, Stdout: name + "\n"} + default: + t.Fatalf("unexpected docker command: %v", args) + return dockerCommandResult{ExitCode: 1} + } + } + + manager := newTerminalSessionManager(terminalSessionManagerConfig{PreserveOnClose: true}) + defer manager.Close() + lease := time.Now().Add(10 * time.Minute).Truncate(time.Millisecond) + results := manager.Recover(context.Background(), []*registryv1.TerminalSessionRecoveryCandidate{ + {SessionId: sessionID, LeaseExpiresUnixMs: lease.UnixMilli()}, + }) + if len(results) != 1 || results[0].GetStatus() != registryv1.TerminalSessionRecoveryResult_RECOVERED { + t.Fatalf("unexpected recovery result: %#v", results) + } + if manager.ActiveSessionCount() != 1 { + t.Fatalf("active session count=%d, want 1", manager.ActiveSessionCount()) + } + manager.mu.Lock() + recovered := manager.sessions[sessionID] + manager.mu.Unlock() + if recovered == nil || !recovered.leaseExpiresAt.Equal(lease) || recovered.inflight != 0 { + t.Fatalf("unexpected recovered session: %#v", recovered) + } +} + +func TestTerminalRecoveryRejectsAndRemovesDuplicateContainers(t *testing.T) { + original := runDockerCommand + t.Cleanup(func() { runDockerCommand = original }) + + sessionID := "owner-a:session-duplicate" + name := terminalSessionResourceName(sessionID) + hash := terminalSessionIDHash(sessionID) + removed := map[string]bool{} + runDockerCommand = func(_ context.Context, args ...string) dockerCommandResult { + joined := strings.Join(args, " ") + switch { + case len(args) > 0 && args[0] == "inspect": + return dockerCommandResult{ExitCode: 0, Stdout: fmt.Sprintf("{\"%s\":\"%s\",\"%s\":\"1\"}\trunning\n", terminalExecSessionLabelKey, hash, terminalExecSchemaLabelKey)} + case strings.Contains(joined, "label="+terminalExecSessionLabelKey+"="+hash): + return dockerCommandResult{ExitCode: 0, Stdout: name + "\n" + name + "-duplicate\n"} + case len(args) > 0 && args[0] == "rm": + removed[args[len(args)-1]] = true + return dockerCommandResult{ExitCode: 0} + case len(args) > 0 && args[0] == "ps": + return dockerCommandResult{ExitCode: 0} + default: + t.Fatalf("unexpected docker command: %v", args) + return dockerCommandResult{ExitCode: 1} + } + } + + manager := newTerminalSessionManager(terminalSessionManagerConfig{PreserveOnClose: true}) + defer manager.Close() + results := manager.Recover(context.Background(), []*registryv1.TerminalSessionRecoveryCandidate{ + {SessionId: sessionID, LeaseExpiresUnixMs: time.Now().Add(time.Minute).UnixMilli()}, + }) + if len(results) != 1 || results[0].GetStatus() != registryv1.TerminalSessionRecoveryResult_INVALID { + t.Fatalf("unexpected recovery result: %#v", results) + } + if !removed[name] || !removed[name+"-duplicate"] { + t.Fatalf("duplicate containers were not isolated: %#v", removed) + } +} + +func TestTerminalRecoveryRemovesContainerWithInvalidLabels(t *testing.T) { + original := runDockerCommand + t.Cleanup(func() { runDockerCommand = original }) + + sessionID := "owner-a:session-invalid" + name := terminalSessionResourceName(sessionID) + removed := false + runDockerCommand = func(_ context.Context, args ...string) dockerCommandResult { + switch { + case len(args) > 0 && args[0] == "inspect": + return dockerCommandResult{ExitCode: 0, Stdout: fmt.Sprintf("{\"%s\":\"wrong\",\"%s\":\"1\"}\trunning\n", terminalExecSessionLabelKey, terminalExecSchemaLabelKey)} + case len(args) > 0 && args[0] == "rm": + removed = args[len(args)-1] == name + return dockerCommandResult{ExitCode: 0} + case len(args) > 0 && args[0] == "ps": + return dockerCommandResult{ExitCode: 0} + default: + t.Fatalf("unexpected docker command: %v", args) + return dockerCommandResult{ExitCode: 1} + } + } + + manager := newTerminalSessionManager(terminalSessionManagerConfig{PreserveOnClose: true}) + defer manager.Close() + results := manager.Recover(context.Background(), []*registryv1.TerminalSessionRecoveryCandidate{ + {SessionId: sessionID, LeaseExpiresUnixMs: time.Now().Add(time.Minute).UnixMilli()}, + }) + if len(results) != 1 || results[0].GetStatus() != registryv1.TerminalSessionRecoveryResult_INVALID { + t.Fatalf("unexpected recovery result: %#v", results) + } + if !removed { + t.Fatal("container with invalid recovery labels was not isolated") + } +} + +func TestTerminalRecoveryIsIdempotentAndCleansOnlyOwnedOrphans(t *testing.T) { + original := runDockerCommand + t.Cleanup(func() { runDockerCommand = original }) + + sessionID := "owner-a:session-idempotent" + name := terminalSessionResourceName(sessionID) + orphan := terminalExecContainerPrefix + "orphan" + hash := terminalSessionIDHash(sessionID) + inspectCalls := 0 + removed := map[string]bool{} + runDockerCommand = func(_ context.Context, args ...string) dockerCommandResult { + joined := strings.Join(args, " ") + switch { + case len(args) > 0 && args[0] == "inspect": + inspectCalls++ + return dockerCommandResult{ExitCode: 0, Stdout: fmt.Sprintf("{\"%s\":\"%s\",\"%s\":\"1\"}\trunning\n", terminalExecSessionLabelKey, hash, terminalExecSchemaLabelKey)} + case strings.Contains(joined, "label="+terminalExecSessionLabelKey+"="+hash): + return dockerCommandResult{ExitCode: 0, Stdout: name + "\n"} + case len(args) > 0 && args[0] == "ps": + return dockerCommandResult{ExitCode: 0, Stdout: name + "\n" + orphan + "\nforeign-container\n"} + case len(args) > 0 && args[0] == "rm": + removed[args[len(args)-1]] = true + return dockerCommandResult{ExitCode: 0} + default: + t.Fatalf("unexpected docker command: %v", args) + return dockerCommandResult{ExitCode: 1} + } + } + + manager := newTerminalSessionManager(terminalSessionManagerConfig{PreserveOnClose: true}) + defer manager.Close() + lease := time.Now().Add(time.Minute).Truncate(time.Millisecond) + candidate := []*registryv1.TerminalSessionRecoveryCandidate{{SessionId: sessionID, LeaseExpiresUnixMs: lease.UnixMilli()}} + for range 2 { + results := manager.Recover(context.Background(), candidate) + if len(results) != 1 || results[0].GetStatus() != registryv1.TerminalSessionRecoveryResult_RECOVERED { + t.Fatalf("unexpected recovery result: %#v", results) + } + } + if inspectCalls != 1 || manager.ActiveSessionCount() != 1 { + t.Fatalf("recovery was not idempotent: inspect_calls=%d active=%d", inspectCalls, manager.ActiveSessionCount()) + } + if !removed[orphan] || removed[name] || removed["foreign-container"] { + t.Fatalf("unsafe orphan cleanup: %#v", removed) + } +} + +func TestTerminalRecoveryReportsMissingAndCleansExpiredCandidate(t *testing.T) { + original := runDockerCommand + t.Cleanup(func() { runDockerCommand = original }) + + missingID := "owner-a:session-missing" + expiredID := "owner-a:session-expired" + expiredName := terminalSessionResourceName(expiredID) + removedExpired := false + runDockerCommand = func(_ context.Context, args ...string) dockerCommandResult { + switch { + case len(args) > 0 && args[0] == "inspect": + return dockerCommandResult{ExitCode: 1, Stderr: "No such container"} + case len(args) > 0 && args[0] == "rm": + if args[len(args)-1] == expiredName { + removedExpired = true + } + return dockerCommandResult{ExitCode: 0} + case len(args) > 0 && args[0] == "ps": + return dockerCommandResult{ExitCode: 0} + default: + t.Fatalf("unexpected docker command: %v", args) + return dockerCommandResult{ExitCode: 1} + } + } + manager := newTerminalSessionManager(terminalSessionManagerConfig{PreserveOnClose: true}) + defer manager.Close() + results := manager.Recover(context.Background(), []*registryv1.TerminalSessionRecoveryCandidate{ + {SessionId: missingID, LeaseExpiresUnixMs: time.Now().Add(time.Minute).UnixMilli()}, + {SessionId: expiredID, LeaseExpiresUnixMs: time.Now().Add(-time.Minute).UnixMilli()}, + }) + if results[0].GetStatus() != registryv1.TerminalSessionRecoveryResult_MISSING || + results[1].GetStatus() != registryv1.TerminalSessionRecoveryResult_INVALID { + t.Fatalf("unexpected recovery results: %#v", results) + } + if !removedExpired || manager.ActiveSessionCount() != 0 { + t.Fatalf("expired resource was not cleaned: removed=%v active=%d", removedExpired, manager.ActiveSessionCount()) + } +} + +func TestConcurrentTerminalRecoveryUsesOneReservation(t *testing.T) { + original := runDockerCommand + t.Cleanup(func() { runDockerCommand = original }) + sessionID := "owner-a:session-concurrent" + name := terminalSessionResourceName(sessionID) + hash := terminalSessionIDHash(sessionID) + runDockerCommand = func(_ context.Context, args ...string) dockerCommandResult { + joined := strings.Join(args, " ") + switch { + case len(args) > 0 && args[0] == "inspect": + return dockerCommandResult{ExitCode: 0, Stdout: fmt.Sprintf("{\"%s\":\"%s\",\"%s\":\"1\"}\trunning\n", terminalExecSessionLabelKey, hash, terminalExecSchemaLabelKey)} + case strings.Contains(joined, "label="+terminalExecSessionLabelKey+"="+hash): + return dockerCommandResult{ExitCode: 0, Stdout: name + "\n"} + case len(args) > 0 && args[0] == "ps": + return dockerCommandResult{ExitCode: 0, Stdout: name + "\n"} + default: + t.Fatalf("unexpected docker command: %v", args) + return dockerCommandResult{ExitCode: 1} + } + } + manager := newTerminalSessionManager(terminalSessionManagerConfig{PreserveOnClose: true}) + defer manager.Close() + candidate := []*registryv1.TerminalSessionRecoveryCandidate{{ + SessionId: sessionID, LeaseExpiresUnixMs: time.Now().Add(time.Minute).UnixMilli(), + }} + done := make(chan []*registryv1.TerminalSessionRecoveryResult, 2) + for range 2 { + go func() { done <- manager.Recover(context.Background(), candidate) }() + } + for range 2 { + if result := <-done; len(result) != 1 || result[0].GetStatus() != registryv1.TerminalSessionRecoveryResult_RECOVERED { + t.Fatalf("unexpected concurrent recovery result: %#v", result) + } + } + if manager.ActiveSessionCount() != 1 { + t.Fatalf("concurrent recovery reserved %d sessions, want 1", manager.ActiveSessionCount()) + } +} diff --git a/worker/worker-docker/internal/runner/terminal_exec.go b/worker/worker-docker/internal/runner/terminal_exec.go index 07819ab..513aa55 100644 --- a/worker/worker-docker/internal/runner/terminal_exec.go +++ b/worker/worker-docker/internal/runner/terminal_exec.go @@ -2,6 +2,8 @@ package runner import ( "context" + "crypto/sha256" + "encoding/hex" "errors" "fmt" "strconv" @@ -16,8 +18,11 @@ import ( const ( terminalExecCapabilityName = "terminalexec" terminalExecCapabilityDeclared = "terminalExec" - terminalExecContainerPrefix = "onlyboxes-terminalexec-" + terminalExecContainerPrefix = "onlyboxes-terminal-v1-" terminalExecCapabilityLabel = "onlyboxes.capability=terminalExec" + terminalExecSessionLabelKey = "onlyboxes.session_id_hash" + terminalExecSchemaLabelKey = "onlyboxes.schema_version" + terminalExecSchemaVersion = "1" terminalExecIdleCommand = "while true; do sleep 3600; done" terminalExecCleanupTimeout = 3 * time.Second terminalExecJanitorInterval = 5 * time.Second @@ -124,6 +129,9 @@ type terminalSessionManagerConfig struct { // MaxActiveSessions caps terminal sandbox reservations across all sessions. // Zero preserves the existing unlimited behaviour. MaxActiveSessions int + // PreserveOnClose leaves terminal containers running for process restart + // recovery. Tests and explicit teardown may leave this false. + PreserveOnClose bool } type terminalSessionManager struct { @@ -142,6 +150,7 @@ type terminalSessionManager struct { sessionMaxInflight int maxActiveSessions int activeSessionReservations int + preserveOnClose bool stopCh chan struct{} doneCh chan struct{} @@ -232,6 +241,7 @@ func newTerminalSessionManager(cfg terminalSessionManagerConfig) *terminalSessio pidsLimit: pidsLimit, sessionMaxInflight: sessionMaxInflight, maxActiveSessions: maxActiveSessions, + preserveOnClose: cfg.PreserveOnClose, stopCh: make(chan struct{}), doneCh: make(chan struct{}), } @@ -265,6 +275,10 @@ func (m *terminalSessionManager) Close() { m.cleanupWG.Wait() for _, session := range sessions { + if m.preserveOnClose { + m.releaseCapacityReservation(session) + continue + } m.cleanupSession(session) } }) @@ -459,10 +473,7 @@ func (m *terminalSessionManager) newSessionLocked(sessionID string, leaseTarget ) } - containerName, err := newTerminalExecContainerName() - if err != nil { - return nil, fmt.Errorf("allocate terminal container name: %w", err) - } + containerName := terminalSessionResourceName(sessionID) session := &terminalSession{ sessionID: sessionID, @@ -628,12 +639,15 @@ func (m *terminalSessionManager) forceRemoveContainer(containerName string) { } func terminalExecDockerCreateArgs(containerName string, dockerImage string, memoryLimit string, cpuLimit string, pidsLimit int) []string { + sessionHash := strings.TrimPrefix(strings.TrimSpace(containerName), terminalExecContainerPrefix) return []string{ "create", "--name", containerName, "--label", pythonExecManagedLabel, "--label", terminalExecCapabilityLabel, "--label", pythonExecRuntimeLabel, + "--label", terminalExecSessionLabelKey + "=" + sessionHash, + "--label", terminalExecSchemaLabelKey + "=" + terminalExecSchemaVersion, "--memory", memoryLimit, "--cpus", cpuLimit, "--pids-limit", strconv.Itoa(pidsLimit), @@ -652,12 +666,13 @@ func terminalExecDockerExecArgs(containerName string, command string) []string { return []string{"exec", containerName, "sh", "-lc", command} } -func newTerminalExecContainerName() (string, error) { - suffix, err := randomHex(8) - if err != nil { - return "", err - } - return terminalExecContainerPrefix + suffix, nil +func terminalSessionIDHash(sessionID string) string { + sum := sha256.Sum256([]byte(strings.TrimSpace(sessionID))) + return hex.EncodeToString(sum[:]) +} + +func terminalSessionResourceName(sessionID string) string { + return terminalExecContainerPrefix + terminalSessionIDHash(sessionID) } func truncateByBytes(value string, maxBytes int) (string, bool) { diff --git a/worker/worker-docker/internal/runner/terminal_exec_test.go b/worker/worker-docker/internal/runner/terminal_exec_test.go index 5c0f6dd..5f1869a 100644 --- a/worker/worker-docker/internal/runner/terminal_exec_test.go +++ b/worker/worker-docker/internal/runner/terminal_exec_test.go @@ -476,13 +476,16 @@ func TestNewTerminalSessionManagerUsesConfiguredResourceLimits(t *testing.T) { } func TestTerminalExecDockerCreateArgs(t *testing.T) { - got := terminalExecDockerCreateArgs("container-a", "python:slim", "256m", "1.0", 128) + containerName := terminalSessionResourceName("session-a") + got := terminalExecDockerCreateArgs(containerName, "python:slim", "256m", "1.0", 128) want := []string{ "create", - "--name", "container-a", + "--name", containerName, "--label", pythonExecManagedLabel, "--label", terminalExecCapabilityLabel, "--label", pythonExecRuntimeLabel, + "--label", terminalExecSessionLabelKey + "=" + terminalSessionIDHash("session-a"), + "--label", terminalExecSchemaLabelKey + "=" + terminalExecSchemaVersion, "--memory", "256m", "--cpus", "1.0", "--pids-limit", "128", From b7070084a3b81450043a19789df031e50f55710d Mon Sep 17 00:00:00 2001 From: Coolfan Date: Fri, 7 Aug 2026 22:23:35 +0800 Subject: [PATCH 4/7] feat(worker-bridge-e2b): recover terminal sessions after restart Tag terminal sandboxes with session-id metadata, add E2B List/Connect helpers to rediscover paused sandboxes, and reconcile Console recovery candidates on reconnect before accepting commands. Restarted sandboxes keep their original session_id and lease. --- worker/worker-bridge-e2b/README/overview.md | 4 +- .../worker-bridge-e2b/internal/e2b/client.go | 78 +++++- .../internal/e2b/client_integration_test.go | 217 +++++++++++++++ .../internal/e2b/client_test.go | 57 ++++ .../internal/runner/console_session_test.go | 34 +++ .../internal/runner/e2b_backend.go | 6 + .../runner/full_worker_integration_test.go | 3 + .../internal/runner/runner.go | 4 + .../internal/runner/session_client.go | 54 ++++ .../internal/runner/session_recovery.go | 159 +++++++++++ .../internal/runner/session_recovery_test.go | 253 ++++++++++++++++++ .../internal/runner/terminal_exec.go | 39 ++- .../terminal_session_integration_test.go | 120 +++++++++ .../internal/runner/terminal_session_test.go | 50 +++- 14 files changed, 1062 insertions(+), 16 deletions(-) create mode 100644 worker/worker-bridge-e2b/internal/runner/session_recovery.go create mode 100644 worker/worker-bridge-e2b/internal/runner/session_recovery_test.go diff --git a/worker/worker-bridge-e2b/README/overview.md b/worker/worker-bridge-e2b/README/overview.md index 4aaf9d7..a29d7ac 100644 --- a/worker/worker-bridge-e2b/README/overview.md +++ b/worker/worker-bridge-e2b/README/overview.md @@ -104,7 +104,9 @@ worker 在 hello 中声明以下四项能力： - 新建 terminal session 受 `WORKER_TERMINAL_MAX_ACTIVE_SESSIONS` 限制（`0` 表示不限），超出正数上限返回 `session_capacity_exceeded`；容量已满时已有 session 仍可执行。创建中、可用、销毁中和 E2B cleanup 进行中的 session 都计入容量；该限制只适用于当前 worker 进程。 - 创建中的 session 会阻塞后续调用，所有等待者共享创建结果。 - 某个命令超时会把 session 标记为待销毁；已有并发命令继续完成，最后一个调用退出后才销毁沙箱。 -- 空闲 session 到期后由 janitor 销毁；worker 正常退出时销毁所有仍管理的沙箱。 +- terminal sandbox 在通用 `onlyboxes.worker` metadata 上增加 `onlyboxes.session_id_hash` 与 `onlyboxes.schema_version`；worker 正常退出时保留尚未过期的 terminal sandbox，`pythonExec` 仍按次销毁。 +- worker 重连 Console 后先按 candidate metadata 精确查询 sandbox，再通过 E2B connect API 重新取得 envd access token；恢复完成前不接收命令，恢复不会改变 Console 保存的 lease。 +- 空闲 session 到期后由 janitor 销毁；E2B 账号中不属于本次 Console candidates 的 sandbox 不会被扫描或删除，由各自远端 timeout 回收。 每个命令由独立的 `/bin/bash -l -c` 进程执行，因此共享文件系统，但不共享 cwd、shell 变量或当前进程环境。 diff --git a/worker/worker-bridge-e2b/internal/e2b/client.go b/worker/worker-bridge-e2b/internal/e2b/client.go index 81247ec..a607a64 100644 --- a/worker/worker-bridge-e2b/internal/e2b/client.go +++ b/worker/worker-bridge-e2b/internal/e2b/client.go @@ -12,6 +12,7 @@ import ( "net/http" "net/url" "path/filepath" + "sort" "strconv" "strings" "time" @@ -53,6 +54,13 @@ type Sandbox struct { AccessToken string } +type SandboxInfo struct { + ID string `json:"sandboxID"` + State string `json:"state"` + EnvdVersion string `json:"envdVersion"` + Metadata map[string]string `json:"metadata"` +} + type CommandResult struct { Stdout string Stderr string @@ -129,6 +137,10 @@ func NewClient(cfg Config) (*Client, error) { } func (c *Client) Create(ctx context.Context, template string, timeoutSec int) (*Sandbox, error) { + return c.CreateWithMetadata(ctx, template, timeoutSec, map[string]string{"onlyboxes.worker": "worker-bridge-e2b"}) +} + +func (c *Client) CreateWithMetadata(ctx context.Context, template string, timeoutSec int, metadata map[string]string) (*Sandbox, error) { template = strings.TrimSpace(template) if template == "" { return nil, errors.New("E2B template is required") @@ -150,7 +162,7 @@ func (c *Client) Create(ctx context.Context, template string, timeoutSec int) (* AutoPause: false, Secure: true, AllowInternetAccess: true, - Metadata: map[string]string{"onlyboxes.worker": "worker-bridge-e2b"}, + Metadata: cloneMetadata(metadata), EnvVars: map[string]string{}, } var response struct { @@ -177,6 +189,70 @@ func (c *Client) Create(ctx context.Context, template string, timeoutSec int) (* }, nil } +func (c *Client) List(ctx context.Context, metadata map[string]string) ([]SandboxInfo, error) { + query := url.Values{} + if len(metadata) > 0 { + keys := make([]string, 0, len(metadata)) + for key := range metadata { + keys = append(keys, key) + } + sort.Strings(keys) + parts := make([]string, 0, len(keys)) + for _, key := range keys { + parts = append(parts, url.QueryEscape(key)+"="+url.QueryEscape(metadata[key])) + } + query.Set("metadata", strings.Join(parts, "&")) + } + path := "/sandboxes" + if encoded := query.Encode(); encoded != "" { + path += "?" + encoded + } + var response []SandboxInfo + if err := c.controlJSON(ctx, http.MethodGet, path, nil, &response); err != nil { + return nil, err + } + return response, nil +} + +func (c *Client) Connect(ctx context.Context, sandboxID string, timeoutSec int) (*Sandbox, error) { + if timeoutSec <= 0 { + return nil, errors.New("E2B sandbox timeout must be positive") + } + var response struct { + SandboxID string `json:"sandboxID"` + EnvdVersion string `json:"envdVersion"` + EnvdAccessToken string `json:"envdAccessToken"` + Domain *string `json:"domain"` + } + path := "/sandboxes/" + url.PathEscape(strings.TrimSpace(sandboxID)) + "/connect" + err := c.controlJSON(ctx, http.MethodPost, path, struct { + Timeout int `json:"timeout"` + }{Timeout: timeoutSec}, &response) + if isHTTPStatus(err, http.StatusNotFound) { + return nil, fmt.Errorf("%w: %v", ErrSandboxNotFound, err) + } + if err != nil { + return nil, err + } + domain := c.domain + if response.Domain != nil && strings.TrimSpace(*response.Domain) != "" { + domain = strings.TrimSpace(*response.Domain) + } + id := strings.TrimSpace(response.SandboxID) + if id == "" { + id = strings.TrimSpace(sandboxID) + } + return &Sandbox{ID: id, Domain: domain, EnvdVersion: response.EnvdVersion, AccessToken: response.EnvdAccessToken}, nil +} + +func cloneMetadata(metadata map[string]string) map[string]string { + out := make(map[string]string, len(metadata)) + for key, value := range metadata { + out[key] = value + } + return out +} + func (c *Client) SetTimeout(ctx context.Context, sandboxID string, timeoutSec int) error { if timeoutSec <= 0 { return errors.New("E2B sandbox timeout must be positive") diff --git a/worker/worker-bridge-e2b/internal/e2b/client_integration_test.go b/worker/worker-bridge-e2b/internal/e2b/client_integration_test.go index cffbbda..fa4fc4c 100644 --- a/worker/worker-bridge-e2b/internal/e2b/client_integration_test.go +++ b/worker/worker-bridge-e2b/internal/e2b/client_integration_test.go @@ -2,7 +2,10 @@ package e2b import ( "context" + "encoding/json" + "fmt" "os" + "os/exec" "strings" "testing" "time" @@ -62,3 +65,217 @@ func TestIntegrationSandboxCommandAndFile(t *testing.T) { t.Fatal(err) } } + +func TestIntegrationSandboxRecoveryByMetadata(t *testing.T) { + if os.Getenv("E2B_INTEGRATION") != "1" { + t.Skip("set E2B_INTEGRATION=1 to run against E2B") + } + apiKey := strings.TrimSpace(os.Getenv("E2B_API_KEY")) + template := strings.TrimSpace(os.Getenv("E2B_TERMINAL_TEMPLATE")) + if apiKey == "" || template == "" { + t.Fatal("E2B_API_KEY and E2B_TERMINAL_TEMPLATE are required") + } + client, err := NewClient(Config{ + APIKey: apiKey, + APIURL: strings.TrimSpace(os.Getenv("E2B_API_URL")), + Domain: strings.TrimSpace(os.Getenv("E2B_DOMAIN")), + RequestTimeout: 60 * time.Second, + }) + if err != nil { + t.Fatal(err) + } + ctx, cancel := context.WithTimeout(context.Background(), 3*time.Minute) + defer cancel() + marker := fmt.Sprintf("integration-%d", time.Now().UnixNano()) + metadata := map[string]string{ + "onlyboxes.session_id_hash": marker, + "onlyboxes.schema_version": "1", + } + sandbox, err := client.CreateWithMetadata(ctx, template, 120, metadata) + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { + cleanupCtx, cleanupCancel := context.WithTimeout(context.Background(), 30*time.Second) + defer cleanupCancel() + if err := client.Kill(cleanupCtx, sandbox.ID); err != nil { + t.Errorf("cleanup sandbox: %v", err) + } + }) + if _, err := client.Run(ctx, sandbox, `printf 'recovery-ok' > /tmp/onlyboxes-recovery.txt`, 1024); err != nil { + t.Fatal(err) + } + + infos, err := client.List(ctx, metadata) + if err != nil { + t.Fatal(err) + } + var found *SandboxInfo + for index := range infos { + if infos[index].ID == sandbox.ID { + found = &infos[index] + break + } + } + if found == nil || found.Metadata["onlyboxes.session_id_hash"] != marker { + t.Fatalf("created sandbox was not discoverable by metadata: %#v", infos) + } + + reconnected, err := client.Connect(ctx, sandbox.ID, 90) + if err != nil { + t.Fatal(err) + } + if reconnected.ID != sandbox.ID || strings.TrimSpace(reconnected.AccessToken) == "" { + t.Fatalf("connect did not return fresh sandbox credentials: %#v", reconnected) + } + result, err := client.Run(ctx, reconnected, `cat /tmp/onlyboxes-recovery.txt`, 1024) + if err != nil { + t.Fatal(err) + } + if result.ExitCode != 0 || result.Stdout != "recovery-ok" { + t.Fatalf("reconnected sandbox lost filesystem state: %#v", result) + } +} + +func TestIntegrationSandboxSurvivesForcedWorkerTermination(t *testing.T) { + if os.Getenv("E2B_INTEGRATION") != "1" { + t.Skip("set E2B_INTEGRATION=1 to run against E2B") + } + if os.Getenv("E2B_RECOVERY_CRASH_HELPER") == "1" { + runE2BRecoveryCrashHelper(t) + return + } + apiKey := strings.TrimSpace(os.Getenv("E2B_API_KEY")) + template := strings.TrimSpace(os.Getenv("E2B_TERMINAL_TEMPLATE")) + if apiKey == "" || template == "" { + t.Fatal("E2B_API_KEY and E2B_TERMINAL_TEMPLATE are required") + } + client, err := NewClient(Config{ + APIKey: apiKey, APIURL: strings.TrimSpace(os.Getenv("E2B_API_URL")), + Domain: strings.TrimSpace(os.Getenv("E2B_DOMAIN")), RequestTimeout: 60 * time.Second, + }) + if err != nil { + t.Fatal(err) + } + marker := fmt.Sprintf("forced-integration-%d", time.Now().UnixNano()) + metadata := map[string]string{ + "onlyboxes.worker": "worker-bridge-e2b", + "onlyboxes.session_id_hash": marker, + "onlyboxes.schema_version": "1", + } + readyFile, err := os.CreateTemp("", "onlyboxes-e2b-crash-ready-*") + if err != nil { + t.Fatal(err) + } + readyPath := readyFile.Name() + _ = readyFile.Close() + _ = os.Remove(readyPath) + t.Cleanup(func() { _ = os.Remove(readyPath) }) + metadataJSON, err := json.Marshal(metadata) + if err != nil { + t.Fatal(err) + } + cmd := exec.Command(os.Args[0], "-test.run=^TestIntegrationSandboxSurvivesForcedWorkerTermination$", "-test.v") + cmd.Env = append(os.Environ(), + "E2B_RECOVERY_CRASH_HELPER=1", + "E2B_RECOVERY_READY_FILE="+readyPath, + "E2B_RECOVERY_METADATA="+string(metadataJSON), + ) + cmd.Stdout = os.Stderr + cmd.Stderr = os.Stderr + if err := cmd.Start(); err != nil { + t.Fatal(err) + } + waitCh := make(chan error, 1) + go func() { waitCh <- cmd.Wait() }() + sandboxID := "" + deadline := time.NewTimer(3 * time.Minute) + defer deadline.Stop() + ticker := time.NewTicker(50 * time.Millisecond) + defer ticker.Stop() + for sandboxID == "" { + select { + case err := <-waitCh: + t.Fatalf("crash helper exited before creating sandbox: %v", err) + case <-deadline.C: + _ = cmd.Process.Kill() + <-waitCh + t.Fatal("timed out waiting for E2B crash helper") + case <-ticker.C: + content, readErr := os.ReadFile(readyPath) + if readErr == nil { + sandboxID = strings.TrimSpace(string(content)) + } + } + } + if err := cmd.Process.Kill(); err != nil { + t.Fatal(err) + } + if err := <-waitCh; err == nil { + t.Fatal("crash helper was not force terminated") + } + t.Cleanup(func() { + cleanupCtx, cancel := context.WithTimeout(context.Background(), 30*time.Second) + defer cancel() + if err := client.Kill(cleanupCtx, sandboxID); err != nil { + t.Errorf("cleanup sandbox: %v", err) + } + }) + + ctx, cancel := context.WithTimeout(context.Background(), 3*time.Minute) + defer cancel() + infos, err := client.List(ctx, metadata) + if err != nil { + t.Fatal(err) + } + found := false + for _, info := range infos { + if info.ID == sandboxID { + found = true + break + } + } + if !found { + t.Fatalf("sandbox %q was not discoverable after worker termination", sandboxID) + } + reconnected, err := client.Connect(ctx, sandboxID, 90) + if err != nil { + t.Fatal(err) + } + result, err := client.Run(ctx, reconnected, "cat /tmp/onlyboxes-forced-recovery.txt", 1024) + if err != nil { + t.Fatal(err) + } + if result.ExitCode != 0 || result.Stdout != "forced-recovery-ok" { + t.Fatalf("sandbox lost state after forced worker termination: %#v", result) + } +} + +func runE2BRecoveryCrashHelper(t *testing.T) { + apiKey := strings.TrimSpace(os.Getenv("E2B_API_KEY")) + template := strings.TrimSpace(os.Getenv("E2B_TERMINAL_TEMPLATE")) + metadata := map[string]string{} + if err := json.Unmarshal([]byte(os.Getenv("E2B_RECOVERY_METADATA")), &metadata); err != nil { + t.Fatal(err) + } + client, err := NewClient(Config{ + APIKey: apiKey, APIURL: strings.TrimSpace(os.Getenv("E2B_API_URL")), + Domain: strings.TrimSpace(os.Getenv("E2B_DOMAIN")), RequestTimeout: 60 * time.Second, + }) + if err != nil { + t.Fatal(err) + } + ctx, cancel := context.WithTimeout(context.Background(), 2*time.Minute) + defer cancel() + sandbox, err := client.CreateWithMetadata(ctx, template, 180, metadata) + if err != nil { + t.Fatal(err) + } + if _, err := client.Run(ctx, sandbox, "printf forced-recovery-ok > /tmp/onlyboxes-forced-recovery.txt", 1024); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(strings.TrimSpace(os.Getenv("E2B_RECOVERY_READY_FILE")), []byte(sandbox.ID), 0o600); err != nil { + t.Fatal(err) + } + select {} +} diff --git a/worker/worker-bridge-e2b/internal/e2b/client_test.go b/worker/worker-bridge-e2b/internal/e2b/client_test.go index ab3b66d..3e6effd 100644 --- a/worker/worker-bridge-e2b/internal/e2b/client_test.go +++ b/worker/worker-bridge-e2b/internal/e2b/client_test.go @@ -217,6 +217,63 @@ func TestControlPlaneLifecycle(t *testing.T) { } } +func TestRecoveryControlPlaneMetadataListAndConnect(t *testing.T) { + t.Parallel() + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.Header.Get("X-API-Key") != "test-key" { + t.Errorf("missing API key") + } + switch { + case r.Method == http.MethodGet && r.URL.Path == "/sandboxes": + if got := r.URL.Query().Get("metadata"); got != "onlyboxes.schema_version=1&onlyboxes.session_id_hash=abc+123" { + t.Errorf("unexpected metadata query: %q", got) + } + _ = json.NewEncoder(w).Encode([]map[string]any{{ + "sandboxID": "sb-recover", + "state": "running", + "envdVersion": "0.6.2", + "metadata": map[string]string{ + "onlyboxes.schema_version": "1", + "onlyboxes.session_id_hash": "abc 123", + }, + }}) + case r.Method == http.MethodPost && r.URL.Path == "/sandboxes/sb-recover/connect": + var body map[string]any + if err := json.NewDecoder(r.Body).Decode(&body); err != nil || body["timeout"] != float64(90) { + t.Errorf("unexpected connect body: %#v err=%v", body, err) + } + _ = json.NewEncoder(w).Encode(map[string]any{ + "sandboxID": "sb-recover", + "envdVersion": "0.6.2", + "envdAccessToken": "fresh-token", + }) + default: + http.NotFound(w, r) + } + })) + defer server.Close() + + client, err := NewClient(Config{APIKey: "test-key", APIURL: server.URL, RequestTimeout: time.Second}) + if err != nil { + t.Fatal(err) + } + metadata := map[string]string{ + "onlyboxes.session_id_hash": "abc 123", + "onlyboxes.schema_version": "1", + } + infos, err := client.List(context.Background(), metadata) + if err != nil || len(infos) != 1 || infos[0].ID != "sb-recover" { + t.Fatalf("unexpected list result: infos=%#v err=%v", infos, err) + } + sandbox, err := client.Connect(context.Background(), "sb-recover", 90) + if err != nil { + t.Fatal(err) + } + if sandbox.ID != "sb-recover" || sandbox.AccessToken != "fresh-token" { + t.Fatalf("unexpected connected sandbox: %#v", sandbox) + } +} + func TestReadFileUsesSandboxRoutingHeadersAndLimit(t *testing.T) { t.Parallel() server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { diff --git a/worker/worker-bridge-e2b/internal/runner/console_session_test.go b/worker/worker-bridge-e2b/internal/runner/console_session_test.go index aa9a287..23a0876 100644 --- a/worker/worker-bridge-e2b/internal/runner/console_session_test.go +++ b/worker/worker-bridge-e2b/internal/runner/console_session_test.go @@ -17,6 +17,19 @@ import ( "google.golang.org/grpc/status" ) +func TestTerminalRecoveryTimesOutBeforeWorkerBecomesReady(t *testing.T) { + original := recoverTerminalSessionsFn + t.Cleanup(func() { recoverTerminalSessionsFn = original }) + recoverTerminalSessionsFn = func(ctx context.Context, _ []*registryv1.TerminalSessionRecoveryCandidate) []*registryv1.TerminalSessionRecoveryResult { + <-ctx.Done() + return nil + } + _, err := recoverTerminalSessionsWithTimeout(context.Background(), 10*time.Millisecond, nil) + if !errors.Is(err, context.DeadlineExceeded) { + t.Fatalf("expected recovery deadline, got %v", err) + } +} + func TestConsoleSessionContract(t *testing.T) { originalActiveSessionCount := activeSessionCountFn activeSessionCountFn = func() int32 { return 3 } @@ -217,6 +230,9 @@ func (s *allCapabilityContractService) Connect(stream grpc.BidiStreamingServer[r }); err != nil { return err } + if err := acceptEmptyRecovery(stream); err != nil { + return err + } firstHeartbeat, err := stream.Recv() if err != nil || firstHeartbeat.GetHeartbeat() == nil { return status.Error(codes.InvalidArgument, "heartbeat required") @@ -295,6 +311,9 @@ func (s *consoleContractService) Connect(stream grpc.BidiStreamingServer[registr }); err != nil { return err } + if err := acceptEmptyRecovery(stream); err != nil { + return err + } heartbeatFrame, err := stream.Recv() if err != nil { @@ -344,3 +363,18 @@ func (s *consoleContractService) Connect(stream grpc.BidiStreamingServer[registr } } } + +func acceptEmptyRecovery(stream grpc.BidiStreamingServer[registryv1.ConnectRequest, registryv1.ConnectResponse]) error { + req, err := stream.Recv() + if err != nil { + return err + } + if req.GetTerminalSessionRecoveryReport() == nil { + return status.Error(codes.InvalidArgument, "terminal session recovery report required") + } + return stream.Send(®istryv1.ConnectResponse{ + Payload: ®istryv1.ConnectResponse_TerminalSessionRecoveryAck{ + TerminalSessionRecoveryAck: ®istryv1.TerminalSessionRecoveryAck{}, + }, + }) +} diff --git a/worker/worker-bridge-e2b/internal/runner/e2b_backend.go b/worker/worker-bridge-e2b/internal/runner/e2b_backend.go index 1c2c74c..4d21a9c 100644 --- a/worker/worker-bridge-e2b/internal/runner/e2b_backend.go +++ b/worker/worker-bridge-e2b/internal/runner/e2b_backend.go @@ -14,3 +14,9 @@ type e2bBackend interface { ReadFile(context.Context, *e2b.Sandbox, string, int64) (e2b.File, error) OpenFile(context.Context, *e2b.Sandbox, string) (e2b.FileReader, error) } + +type e2bRecoveryBackend interface { + CreateWithMetadata(context.Context, string, int, map[string]string) (*e2b.Sandbox, error) + List(context.Context, map[string]string) ([]e2b.SandboxInfo, error) + Connect(context.Context, string, int) (*e2b.Sandbox, error) +} diff --git a/worker/worker-bridge-e2b/internal/runner/full_worker_integration_test.go b/worker/worker-bridge-e2b/internal/runner/full_worker_integration_test.go index d57f1bb..e48a236 100644 --- a/worker/worker-bridge-e2b/internal/runner/full_worker_integration_test.go +++ b/worker/worker-bridge-e2b/internal/runner/full_worker_integration_test.go @@ -106,6 +106,9 @@ func (s *liveAllCapabilityService) Connect(stream grpc.BidiStreamingServer[regis }); err != nil { return err } + if err := acceptEmptyRecovery(stream); err != nil { + return err + } if err := s.waitForHeartbeat(stream); err != nil { return err } diff --git a/worker/worker-bridge-e2b/internal/runner/runner.go b/worker/worker-bridge-e2b/internal/runner/runner.go index f8513fc..4aac6f5 100644 --- a/worker/worker-bridge-e2b/internal/runner/runner.go +++ b/worker/worker-bridge-e2b/internal/runner/runner.go @@ -75,6 +75,7 @@ func Run(ctx context.Context, cfg config.Config) error { ExportMode: cfg.TerminalExportMode, SessionMaxInflight: cfg.TerminalSessionMaxInflight, MaxActiveSessions: cfg.TerminalMaxActiveSessions, + PreserveOnClose: true, }) pythonRunner := newPythonExecRunner( backend, @@ -89,11 +90,14 @@ func Run(ctx context.Context, cfg config.Config) error { runTerminalResource = terminalManager.ResolveResource originalActiveSessionCountFn := activeSessionCountFn activeSessionCountFn = terminalManager.ActiveSessionCount + originalRecoverTerminalSessionsFn := recoverTerminalSessionsFn + recoverTerminalSessionsFn = terminalManager.Recover defer func() { runPythonExec = originalRunPythonExec runTerminalExec = originalRunTerminalExec runTerminalResource = originalRunTerminalResource activeSessionCountFn = originalActiveSessionCountFn + recoverTerminalSessionsFn = originalRecoverTerminalSessionsFn terminalManager.Close() }() diff --git a/worker/worker-bridge-e2b/internal/runner/session_client.go b/worker/worker-bridge-e2b/internal/runner/session_client.go index 716aad2..deeb076 100644 --- a/worker/worker-bridge-e2b/internal/runner/session_client.go +++ b/worker/worker-bridge-e2b/internal/runner/session_client.go @@ -65,6 +65,25 @@ func runSessionWithStatus(ctx context.Context, cfg config.Config) (bool, error) } heartbeatInterval := durationFromServer(ack.GetHeartbeatIntervalSec(), cfg.HeartbeatInterval) + recoveryResults, err := recoverTerminalSessionsWithTimeout(ctx, cfg.CallTimeout, ack.GetTerminalSessionRecoveryCandidates()) + if err != nil { + return true, fmt.Errorf("recover terminal sessions: %w", err) + } + if err := stream.Send(®istryv1.ConnectRequest{ + Payload: ®istryv1.ConnectRequest_TerminalSessionRecoveryReport{ + TerminalSessionRecoveryReport: ®istryv1.TerminalSessionRecoveryReport{Results: recoveryResults}, + }, + }); err != nil { + return true, fmt.Errorf("send terminal session recovery report: %w", err) + } + recoveryResp, err := recvWithTimeout(ctx, cfg.CallTimeout, stream.Recv) + if err != nil { + return true, fmt.Errorf("recv terminal session recovery ack: %w", err) + } + if recoveryResp.GetTerminalSessionRecoveryAck() == nil { + return true, fmt.Errorf("unexpected response while waiting for terminal session recovery ack") + } + logTerminalRecoverySummary(recoveryResults) logging.Infof("worker connected: node_id=%s node_name=%s session_id=%s", hello.GetNodeId(), hello.GetNodeName(), sessionID) sessionCtx, cancel := context.WithCancel(ctx) @@ -80,6 +99,41 @@ func runSessionWithStatus(ctx context.Context, cfg config.Config) (bool, error) return true, heartbeatLoop(sessionCtx, outbound, heartbeatAckCh, sessionErrCh, cfg, sessionID, heartbeatInterval) } +func recoverTerminalSessionsWithTimeout( + ctx context.Context, + timeout time.Duration, + candidates []*registryv1.TerminalSessionRecoveryCandidate, +) ([]*registryv1.TerminalSessionRecoveryResult, error) { + recoveryCtx, cancel := context.WithTimeout(ctx, timeout) + defer cancel() + resultCh := make(chan []*registryv1.TerminalSessionRecoveryResult, 1) + go func() { + resultCh <- recoverTerminalSessionsFn(recoveryCtx, candidates) + }() + select { + case <-recoveryCtx.Done(): + return nil, recoveryCtx.Err() + case results := <-resultCh: + return results, nil + } +} + +func logTerminalRecoverySummary(results []*registryv1.TerminalSessionRecoveryResult) { + counts := map[registryv1.TerminalSessionRecoveryResult_Status]int{} + for _, result := range results { + if result != nil { + counts[result.GetStatus()]++ + } + } + logging.Infof( + "terminal session recovery completed: candidates=%d recovered=%d missing=%d invalid=%d", + len(results), + counts[registryv1.TerminalSessionRecoveryResult_RECOVERED], + counts[registryv1.TerminalSessionRecoveryResult_MISSING], + counts[registryv1.TerminalSessionRecoveryResult_INVALID], + ) +} + func dial(ctx context.Context, cfg config.Config) (*grpc.ClientConn, error) { if err := ctx.Err(); err != nil { return nil, err diff --git a/worker/worker-bridge-e2b/internal/runner/session_recovery.go b/worker/worker-bridge-e2b/internal/runner/session_recovery.go new file mode 100644 index 0000000..c47aab9 --- /dev/null +++ b/worker/worker-bridge-e2b/internal/runner/session_recovery.go @@ -0,0 +1,159 @@ +package runner + +import ( + "context" + "errors" + "strings" + "time" + + registryv1 "github.com/onlyboxes/onlyboxes/api/gen/go/registry/v1" + "github.com/onlyboxes/onlyboxes/worker/worker-bridge-e2b/internal/e2b" + "github.com/onlyboxes/onlyboxes/worker/worker-bridge-e2b/internal/logging" +) + +var recoverTerminalSessionsFn = func( + _ context.Context, + candidates []*registryv1.TerminalSessionRecoveryCandidate, +) []*registryv1.TerminalSessionRecoveryResult { + return missingTerminalRecoveryResults(candidates) +} + +func missingTerminalRecoveryResults(candidates []*registryv1.TerminalSessionRecoveryCandidate) []*registryv1.TerminalSessionRecoveryResult { + results := make([]*registryv1.TerminalSessionRecoveryResult, 0, len(candidates)) + for _, candidate := range candidates { + if candidate == nil { + continue + } + results = append(results, ®istryv1.TerminalSessionRecoveryResult{ + SessionId: candidate.GetSessionId(), + Status: registryv1.TerminalSessionRecoveryResult_MISSING, + }) + } + return results +} + +func (m *terminalSessionManager) Recover( + ctx context.Context, + candidates []*registryv1.TerminalSessionRecoveryCandidate, +) []*registryv1.TerminalSessionRecoveryResult { + startedAt := time.Now() + recoveryBackend, ok := m.backend.(e2bRecoveryBackend) + if !ok { + return missingTerminalRecoveryResults(candidates) + } + results := make([]*registryv1.TerminalSessionRecoveryResult, 0, len(candidates)) + discovered := 0 + now := time.Now() + for _, candidate := range candidates { + if candidate == nil { + continue + } + sessionID := strings.TrimSpace(candidate.GetSessionId()) + leaseExpiresAt := time.UnixMilli(candidate.GetLeaseExpiresUnixMs()) + status := registryv1.TerminalSessionRecoveryResult_INVALID + if sessionID != "" && candidate.GetLeaseExpiresUnixMs() > 0 && leaseExpiresAt.After(now) { + var found int + status, found = m.recoverOne(ctx, recoveryBackend, sessionID, leaseExpiresAt) + discovered += found + } + results = append(results, ®istryv1.TerminalSessionRecoveryResult{SessionId: sessionID, Status: status}) + } + counts := map[registryv1.TerminalSessionRecoveryResult_Status]int{} + for _, result := range results { + counts[result.GetStatus()]++ + } + logging.Infof( + "terminal recovery completed: executor_kind=e2b discovered=%d candidate=%d recovered=%d missing=%d invalid=%d orphan_cleaned=0 duration_ms=%d recovery_failures=%d", + discovered, + len(results), + counts[registryv1.TerminalSessionRecoveryResult_RECOVERED], + counts[registryv1.TerminalSessionRecoveryResult_MISSING], + counts[registryv1.TerminalSessionRecoveryResult_INVALID], + time.Since(startedAt).Milliseconds(), + counts[registryv1.TerminalSessionRecoveryResult_INVALID], + ) + return results +} + +func (m *terminalSessionManager) recoverOne( + ctx context.Context, + backend e2bRecoveryBackend, + sessionID string, + leaseExpiresAt time.Time, +) (registryv1.TerminalSessionRecoveryResult_Status, int) { + m.mu.Lock() + if m.closed { + m.mu.Unlock() + return registryv1.TerminalSessionRecoveryResult_INVALID, 0 + } + if existing := m.sessions[sessionID]; existing != nil && !existing.destroying { + existing.desiredLeaseExpiresAt = leaseExpiresAt + existing.confirmedLeaseExpiresAt = leaseExpiresAt + existing.remoteTimeoutExpiresAt = leaseExpiresAt + m.mu.Unlock() + return registryv1.TerminalSessionRecoveryResult_RECOVERED, 1 + } + m.mu.Unlock() + + infos, err := backend.List(ctx, terminalSessionMetadata(sessionID)) + if err != nil { + return registryv1.TerminalSessionRecoveryResult_INVALID, 0 + } + if len(infos) == 0 { + return registryv1.TerminalSessionRecoveryResult_MISSING, 0 + } + matches := make([]e2b.SandboxInfo, 0, len(infos)) + for _, info := range infos { + if info.Metadata[terminalSessionMetadataKey] == terminalSessionIDHash(sessionID) && + info.Metadata[terminalSessionSchemaKey] == terminalSessionSchemaVersion && + info.Metadata[terminalSessionWorkerMetadataKey] == terminalSessionWorkerMetadata { + matches = append(matches, info) + } + } + if len(matches) == 0 { + return registryv1.TerminalSessionRecoveryResult_INVALID, len(infos) + } + if len(matches) != 1 { + for _, info := range matches { + if strings.TrimSpace(info.ID) != "" { + _ = m.backend.Kill(ctx, info.ID) + } + } + return registryv1.TerminalSessionRecoveryResult_INVALID, len(matches) + } + sandbox, err := backend.Connect(ctx, matches[0].ID, secondsUntil(leaseExpiresAt)) + if errors.Is(err, e2b.ErrSandboxNotFound) { + return registryv1.TerminalSessionRecoveryResult_MISSING, 1 + } + if err != nil || sandbox == nil || strings.TrimSpace(sandbox.ID) == "" { + return registryv1.TerminalSessionRecoveryResult_INVALID, 1 + } + + ready := make(chan struct{}) + close(ready) + m.mu.Lock() + defer m.mu.Unlock() + if m.closed { + return registryv1.TerminalSessionRecoveryResult_INVALID, 1 + } + if existing := m.sessions[sessionID]; existing != nil { + if existing.sandbox == nil || existing.sandbox.ID != sandbox.ID || existing.destroying { + return registryv1.TerminalSessionRecoveryResult_INVALID, 1 + } + existing.desiredLeaseExpiresAt = leaseExpiresAt + existing.confirmedLeaseExpiresAt = leaseExpiresAt + existing.remoteTimeoutExpiresAt = leaseExpiresAt + return registryv1.TerminalSessionRecoveryResult_RECOVERED, 1 + } + m.sessions[sessionID] = &terminalSession{ + sessionID: sessionID, + sandbox: sandbox, + desiredLeaseExpiresAt: leaseExpiresAt, + confirmedLeaseExpiresAt: leaseExpiresAt, + remoteTimeoutExpiresAt: leaseExpiresAt, + capacityReserved: true, + ready: ready, + } + m.activeSessionReservations++ + return registryv1.TerminalSessionRecoveryResult_RECOVERED, 1 +} diff --git a/worker/worker-bridge-e2b/internal/runner/session_recovery_test.go b/worker/worker-bridge-e2b/internal/runner/session_recovery_test.go new file mode 100644 index 0000000..50b9504 --- /dev/null +++ b/worker/worker-bridge-e2b/internal/runner/session_recovery_test.go @@ -0,0 +1,253 @@ +package runner + +import ( + "context" + "strings" + "testing" + "time" + + registryv1 "github.com/onlyboxes/onlyboxes/api/gen/go/registry/v1" + "github.com/onlyboxes/onlyboxes/worker/worker-bridge-e2b/internal/e2b" +) + +func TestE2BTerminalRecoveryMetadataUsesFullHashWithoutSessionID(t *testing.T) { + sessionID := "owner-secret:session-secret" + metadata := terminalSessionMetadata(sessionID) + hash := metadata[terminalSessionMetadataKey] + if len(hash) != 64 || strings.Contains(hash, "owner-secret") || hash != terminalSessionIDHash(sessionID) { + t.Fatalf("metadata exposed or did not deterministically map the session: %#v", metadata) + } + if metadata[terminalSessionSchemaKey] != terminalSessionSchemaVersion || + metadata[terminalSessionWorkerMetadataKey] != terminalSessionWorkerMetadata { + t.Fatalf("metadata markers missing: %#v", metadata) + } +} + +func TestE2BTerminalRecoveryUsesMetadataAndFreshConnection(t *testing.T) { + sessionID := "owner-a:session-a" + wantMetadata := terminalSessionMetadata(sessionID) + var listedMetadata map[string]string + var connectedID string + var connectedTimeout int + backend := &fakeE2BBackend{ + listFn: func(_ context.Context, metadata map[string]string) ([]e2b.SandboxInfo, error) { + listedMetadata = metadata + return []e2b.SandboxInfo{{ + ID: "sandbox-a", + State: "running", + Metadata: wantMetadata, + }}, nil + }, + connectFn: func(_ context.Context, sandboxID string, timeout int) (*e2b.Sandbox, error) { + connectedID = sandboxID + connectedTimeout = timeout + return &e2b.Sandbox{ID: sandboxID, Domain: "e2b.app", AccessToken: "fresh-token"}, nil + }, + } + manager := newTestTerminalManager(backend, 1) + manager.preserveOnClose = true + defer manager.Close() + + lease := time.Now().Add(10 * time.Minute).Truncate(time.Millisecond) + results := manager.Recover(context.Background(), []*registryv1.TerminalSessionRecoveryCandidate{ + {SessionId: sessionID, LeaseExpiresUnixMs: lease.UnixMilli()}, + }) + if len(results) != 1 || results[0].GetStatus() != registryv1.TerminalSessionRecoveryResult_RECOVERED { + t.Fatalf("unexpected recovery result: %#v", results) + } + if listedMetadata[terminalSessionMetadataKey] != wantMetadata[terminalSessionMetadataKey] || + listedMetadata[terminalSessionSchemaKey] != terminalSessionSchemaVersion { + t.Fatalf("unexpected metadata filter: %#v", listedMetadata) + } + if connectedID != "sandbox-a" || connectedTimeout < 599 || connectedTimeout > 601 { + t.Fatalf("unexpected reconnect: id=%q timeout=%d", connectedID, connectedTimeout) + } + manager.mu.Lock() + recovered := manager.sessions[sessionID] + manager.mu.Unlock() + if recovered == nil || recovered.sandbox.AccessToken != "fresh-token" || + !recovered.confirmedLeaseExpiresAt.Equal(lease) || recovered.inflight != 0 { + t.Fatalf("unexpected recovered session: %#v", recovered) + } +} + +func TestE2BTerminalRecoveryRejectsDuplicateMetadataMatches(t *testing.T) { + sessionID := "owner-a:session-duplicate" + metadata := terminalSessionMetadata(sessionID) + connectCalled := false + killed := map[string]bool{} + backend := &fakeE2BBackend{ + listFn: func(_ context.Context, _ map[string]string) ([]e2b.SandboxInfo, error) { + return []e2b.SandboxInfo{ + {ID: "sandbox-a", Metadata: metadata}, + {ID: "sandbox-b", Metadata: metadata}, + }, nil + }, + connectFn: func(_ context.Context, sandboxID string, timeout int) (*e2b.Sandbox, error) { + connectCalled = true + return nil, nil + }, + killFn: func(_ context.Context, sandboxID string) error { + killed[sandboxID] = true + return nil + }, + } + manager := newTestTerminalManager(backend, 1) + defer manager.Close() + + results := manager.Recover(context.Background(), []*registryv1.TerminalSessionRecoveryCandidate{ + {SessionId: sessionID, LeaseExpiresUnixMs: time.Now().Add(time.Minute).UnixMilli()}, + }) + if len(results) != 1 || results[0].GetStatus() != registryv1.TerminalSessionRecoveryResult_INVALID { + t.Fatalf("unexpected recovery result: %#v", results) + } + if connectCalled { + t.Fatal("duplicate metadata matches must not reconnect to an arbitrary sandbox") + } + if !killed["sandbox-a"] || !killed["sandbox-b"] { + t.Fatalf("duplicate sandboxes were not isolated: %#v", killed) + } +} + +func TestE2BTerminalRecoveryNeverKillsNonCandidateSandbox(t *testing.T) { + sessionID := "owner-a:session-safe-filter" + killCalled := false + backend := &fakeE2BBackend{ + listFn: func(_ context.Context, _ map[string]string) ([]e2b.SandboxInfo, error) { + return []e2b.SandboxInfo{{ + ID: "unrelated-sandbox", + Metadata: map[string]string{ + terminalSessionMetadataKey: "different-hash", + terminalSessionSchemaKey: terminalSessionSchemaVersion, + }, + }}, nil + }, + killFn: func(_ context.Context, _ string) error { + killCalled = true + return nil + }, + } + manager := newTestTerminalManager(backend, 1) + defer manager.Close() + results := manager.Recover(context.Background(), []*registryv1.TerminalSessionRecoveryCandidate{{ + SessionId: sessionID, LeaseExpiresUnixMs: time.Now().Add(time.Minute).UnixMilli(), + }}) + if len(results) != 1 || results[0].GetStatus() != registryv1.TerminalSessionRecoveryResult_INVALID { + t.Fatalf("unexpected recovery result: %#v", results) + } + if killCalled { + t.Fatal("non-candidate E2B sandbox was killed") + } +} + +func TestE2BTerminalCreationAddsRecoveryMetadata(t *testing.T) { + var metadata map[string]string + backend := &fakeE2BBackend{ + createWithMetadataFn: func(_ context.Context, _ string, _ int, values map[string]string) (*e2b.Sandbox, error) { + metadata = values + return &e2b.Sandbox{ID: "sandbox-created", Domain: "test"}, nil + }, + } + manager := newTestTerminalManager(backend, 1) + defer manager.Close() + + result, err := manager.Execute(context.Background(), terminalExecRequest{ + Command: "pwd", + SessionID: "owner-a:session-create", + CreateIfMissing: true, + LeaseTTLSec: intPointer(10), + }) + if err != nil { + t.Fatalf("create terminal session: %v", err) + } + if !result.Created || metadata[terminalSessionWorkerMetadataKey] != terminalSessionWorkerMetadata || + metadata[terminalSessionMetadataKey] != terminalSessionIDHash("owner-a:session-create") || + metadata[terminalSessionSchemaKey] != terminalSessionSchemaVersion { + t.Fatalf("unexpected recovery metadata: %#v", metadata) + } +} + +func TestE2BTerminalRecoveryIsIdempotent(t *testing.T) { + sessionID := "owner-a:session-idempotent" + metadata := terminalSessionMetadata(sessionID) + listCalls := 0 + connectCalls := 0 + backend := &fakeE2BBackend{ + listFn: func(_ context.Context, _ map[string]string) ([]e2b.SandboxInfo, error) { + listCalls++ + return []e2b.SandboxInfo{{ID: "sandbox-a", Metadata: metadata}}, nil + }, + connectFn: func(_ context.Context, sandboxID string, _ int) (*e2b.Sandbox, error) { + connectCalls++ + return &e2b.Sandbox{ID: sandboxID, AccessToken: "fresh-token"}, nil + }, + } + manager := newTestTerminalManager(backend, 1) + defer manager.Close() + lease := time.Now().Add(time.Minute).Truncate(time.Millisecond) + candidates := []*registryv1.TerminalSessionRecoveryCandidate{{SessionId: sessionID, LeaseExpiresUnixMs: lease.UnixMilli()}} + for range 2 { + results := manager.Recover(context.Background(), candidates) + if len(results) != 1 || results[0].GetStatus() != registryv1.TerminalSessionRecoveryResult_RECOVERED { + t.Fatalf("unexpected recovery result: %#v", results) + } + } + if listCalls != 1 || connectCalls != 1 || manager.ActiveSessionCount() != 1 { + t.Fatalf("recovery was not idempotent: list=%d connect=%d active=%d", listCalls, connectCalls, manager.ActiveSessionCount()) + } +} + +func TestE2BTerminalRecoveryReportsMissingAndSkipsExpiredCandidate(t *testing.T) { + listCalls := 0 + backend := &fakeE2BBackend{ + listFn: func(_ context.Context, _ map[string]string) ([]e2b.SandboxInfo, error) { + listCalls++ + return nil, nil + }, + } + manager := newTestTerminalManager(backend, 1) + defer manager.Close() + results := manager.Recover(context.Background(), []*registryv1.TerminalSessionRecoveryCandidate{ + {SessionId: "owner-a:session-missing", LeaseExpiresUnixMs: time.Now().Add(time.Minute).UnixMilli()}, + {SessionId: "owner-a:session-expired", LeaseExpiresUnixMs: time.Now().Add(-time.Minute).UnixMilli()}, + }) + if results[0].GetStatus() != registryv1.TerminalSessionRecoveryResult_MISSING || + results[1].GetStatus() != registryv1.TerminalSessionRecoveryResult_INVALID { + t.Fatalf("unexpected recovery results: %#v", results) + } + if listCalls != 1 || manager.ActiveSessionCount() != 0 { + t.Fatalf("expired candidate contacted E2B or consumed capacity: list=%d active=%d", listCalls, manager.ActiveSessionCount()) + } +} + +func TestConcurrentE2BTerminalRecoveryUsesOneReservation(t *testing.T) { + sessionID := "owner-a:session-concurrent" + metadata := terminalSessionMetadata(sessionID) + backend := &fakeE2BBackend{ + listFn: func(_ context.Context, _ map[string]string) ([]e2b.SandboxInfo, error) { + return []e2b.SandboxInfo{{ID: "sandbox-a", Metadata: metadata}}, nil + }, + connectFn: func(_ context.Context, sandboxID string, _ int) (*e2b.Sandbox, error) { + return &e2b.Sandbox{ID: sandboxID, AccessToken: "fresh-token"}, nil + }, + } + manager := newTestTerminalManager(backend, 1) + defer manager.Close() + candidate := []*registryv1.TerminalSessionRecoveryCandidate{{ + SessionId: sessionID, LeaseExpiresUnixMs: time.Now().Add(time.Minute).UnixMilli(), + }} + done := make(chan []*registryv1.TerminalSessionRecoveryResult, 2) + for range 2 { + go func() { done <- manager.Recover(context.Background(), candidate) }() + } + for range 2 { + if result := <-done; len(result) != 1 || result[0].GetStatus() != registryv1.TerminalSessionRecoveryResult_RECOVERED { + t.Fatalf("unexpected concurrent recovery result: %#v", result) + } + } + if manager.ActiveSessionCount() != 1 { + t.Fatalf("concurrent recovery reserved %d sessions, want 1", manager.ActiveSessionCount()) + } +} + +func intPointer(value int) *int { return &value } diff --git a/worker/worker-bridge-e2b/internal/runner/terminal_exec.go b/worker/worker-bridge-e2b/internal/runner/terminal_exec.go index e62f643..fd684d1 100644 --- a/worker/worker-bridge-e2b/internal/runner/terminal_exec.go +++ b/worker/worker-bridge-e2b/internal/runner/terminal_exec.go @@ -2,6 +2,8 @@ package runner import ( "context" + "crypto/sha256" + "encoding/hex" "errors" "fmt" "strings" @@ -27,6 +29,11 @@ const ( defaultTerminalLeaseSec = 300 defaultTerminalOutputLimitBytes = 1024 * 1024 defaultTerminalSessionMaxInflight = 128 + terminalSessionWorkerMetadataKey = "onlyboxes.worker" + terminalSessionWorkerMetadata = "worker-bridge-e2b" + terminalSessionMetadataKey = "onlyboxes.session_id_hash" + terminalSessionSchemaKey = "onlyboxes.schema_version" + terminalSessionSchemaVersion = "1" ) const ( @@ -109,6 +116,7 @@ type terminalSessionManagerConfig struct { ExportMode string SessionMaxInflight int MaxActiveSessions int + PreserveOnClose bool // JanitorInterval is test-only tuning in practice; zero selects the // production interval. JanitorInterval time.Duration @@ -133,6 +141,7 @@ type terminalSessionManager struct { maxActiveSessions int activeSessionReservations int janitorInterval time.Duration + preserveOnClose bool stopCh chan struct{} doneCh chan struct{} @@ -185,6 +194,7 @@ func newTerminalSessionManager(cfg terminalSessionManagerConfig) *terminalSessio sessionMaxInflight: maxInflight, maxActiveSessions: maxActiveSessions, janitorInterval: janitorInterval, + preserveOnClose: cfg.PreserveOnClose, stopCh: make(chan struct{}), doneCh: make(chan struct{}), } @@ -398,7 +408,13 @@ func (m *terminalSessionManager) awaitSessionReady(ctx context.Context, session leaseExpiresAt := session.desiredLeaseExpiresAt m.mu.Unlock() timeout := secondsUntil(leaseExpiresAt) - sandbox, err := m.backend.Create(ctx, m.template, timeout) + var sandbox *e2b.Sandbox + var err error + if recoveryBackend, ok := m.backend.(e2bRecoveryBackend); ok { + sandbox, err = recoveryBackend.CreateWithMetadata(ctx, m.template, timeout, terminalSessionMetadata(session.sessionID)) + } else { + sandbox, err = m.backend.Create(ctx, m.template, timeout) + } m.mu.Lock() session.sandbox = sandbox session.initErr = err @@ -601,11 +617,32 @@ func (m *terminalSessionManager) Close() { m.mu.Unlock() m.cleanupWG.Wait() for _, session := range sessions { + if m.preserveOnClose { + m.mu.Lock() + if session.capacityReserved && m.activeSessionReservations > 0 { + m.activeSessionReservations-- + } + m.mu.Unlock() + continue + } m.cleanupRetiredSession(session) } }) } +func terminalSessionIDHash(sessionID string) string { + sum := sha256.Sum256([]byte(strings.TrimSpace(sessionID))) + return hex.EncodeToString(sum[:]) +} + +func terminalSessionMetadata(sessionID string) map[string]string { + return map[string]string{ + terminalSessionWorkerMetadataKey: terminalSessionWorkerMetadata, + terminalSessionMetadataKey: terminalSessionIDHash(sessionID), + terminalSessionSchemaKey: terminalSessionSchemaVersion, + } +} + func (m *terminalSessionManager) killSandbox(sandbox *e2b.Sandbox) { if sandbox == nil { return diff --git a/worker/worker-bridge-e2b/internal/runner/terminal_session_integration_test.go b/worker/worker-bridge-e2b/internal/runner/terminal_session_integration_test.go index 4f8358a..e429aee 100644 --- a/worker/worker-bridge-e2b/internal/runner/terminal_session_integration_test.go +++ b/worker/worker-bridge-e2b/internal/runner/terminal_session_integration_test.go @@ -3,6 +3,7 @@ package runner import ( "context" "errors" + "fmt" "io" "net/http" "net/http/httptest" @@ -12,6 +13,7 @@ import ( "testing" "time" + registryv1 "github.com/onlyboxes/onlyboxes/api/gen/go/registry/v1" "github.com/onlyboxes/onlyboxes/worker/worker-bridge-e2b/internal/e2b" ) @@ -29,6 +31,26 @@ func (b *recordingE2BBackend) Create(ctx context.Context, template string, timeo return b.e2bBackend.Create(ctx, template, timeoutSec) } +func (b *recordingE2BBackend) CreateWithMetadata( + ctx context.Context, + template string, + timeoutSec int, + metadata map[string]string, +) (*e2b.Sandbox, error) { + b.mu.Lock() + b.createCalls++ + b.mu.Unlock() + return b.e2bBackend.(e2bRecoveryBackend).CreateWithMetadata(ctx, template, timeoutSec, metadata) +} + +func (b *recordingE2BBackend) List(ctx context.Context, metadata map[string]string) ([]e2b.SandboxInfo, error) { + return b.e2bBackend.(e2bRecoveryBackend).List(ctx, metadata) +} + +func (b *recordingE2BBackend) Connect(ctx context.Context, sandboxID string, timeoutSec int) (*e2b.Sandbox, error) { + return b.e2bBackend.(e2bRecoveryBackend).Connect(ctx, sandboxID, timeoutSec) +} + func (b *recordingE2BBackend) createCount() int { b.mu.Lock() defer b.mu.Unlock() @@ -335,6 +357,104 @@ func TestIntegrationJanitorExpiresTerminalSession(t *testing.T) { } } +func TestIntegrationTerminalSessionRecoversAcrossManagerRestart(t *testing.T) { + backend, template := liveTerminalBackend(t) + manager := newTerminalSessionManager(terminalSessionManagerConfig{ + Backend: backend, + Template: template, + LeaseMinSec: 1, + LeaseMaxSec: 120, + LeaseDefaultSec: 60, + OutputLimitBytes: 1024, + SessionMaxInflight: 1, + MaxActiveSessions: 1, + PreserveOnClose: true, + }) + ctx, cancel := context.WithTimeout(context.Background(), 3*time.Minute) + defer cancel() + sessionID := fmt.Sprintf("integration:e2b-manager-recovery-%d", time.Now().UnixNano()) + leaseTTL := 90 + created, err := manager.Execute(ctx, terminalExecRequest{ + Command: "printf manager-recovery-ok > /tmp/onlyboxes-manager-recovery.txt", + SessionID: sessionID, + CreateIfMissing: true, + LeaseTTLSec: &leaseTTL, + }) + if err != nil { + manager.Close() + t.Fatal(err) + } + if !created.Created || created.SessionID != sessionID { + manager.Close() + t.Fatalf("unexpected created session: %#v", created) + } + manager.mu.Lock() + sandboxID := manager.sessions[sessionID].sandbox.ID + manager.mu.Unlock() + t.Cleanup(func() { + cleanupCtx, cleanupCancel := context.WithTimeout(context.Background(), 30*time.Second) + defer cleanupCancel() + _ = backend.Kill(cleanupCtx, sandboxID) + }) + + manager.Close() + if backend.killCount() != 0 { + t.Fatalf("preserving manager killed sandbox during restart: kills=%d", backend.killCount()) + } + + recoveredManager := newTerminalSessionManager(terminalSessionManagerConfig{ + Backend: backend, + Template: template, + LeaseMinSec: 1, + LeaseMaxSec: 120, + LeaseDefaultSec: 60, + OutputLimitBytes: 1024, + SessionMaxInflight: 1, + MaxActiveSessions: 1, + }) + defer recoveredManager.Close() + results := recoveredManager.Recover(ctx, []*registryv1.TerminalSessionRecoveryCandidate{{ + SessionId: sessionID, + LeaseExpiresUnixMs: created.LeaseExpiresUnixMS, + }}) + if len(results) != 1 || results[0].GetStatus() != registryv1.TerminalSessionRecoveryResult_RECOVERED { + t.Fatalf("unexpected recovery results: %#v", results) + } + if recoveredManager.ActiveSessionCount() != 1 { + t.Fatalf("recovery did not restore capacity reservation: active=%d", recoveredManager.ActiveSessionCount()) + } + recoveredManager.mu.Lock() + recovered := recoveredManager.sessions[sessionID] + if recovered == nil { + recoveredManager.mu.Unlock() + t.Fatal("recovered session is not registered") + } + gotLeaseExpiresUnixMS := recovered.confirmedLeaseExpiresAt.UnixMilli() + gotSandboxID := recovered.sandbox.ID + gotInflight := recovered.inflight + recoveredManager.mu.Unlock() + if gotLeaseExpiresUnixMS != created.LeaseExpiresUnixMS { + t.Fatalf("recovery changed lease: got=%d want=%d", gotLeaseExpiresUnixMS, created.LeaseExpiresUnixMS) + } + if gotSandboxID != sandboxID || gotInflight != 0 { + t.Fatalf("unexpected recovered state: sandbox=%q inflight=%d", gotSandboxID, gotInflight) + } + + reused, err := recoveredManager.Execute(ctx, terminalExecRequest{ + Command: "cat /tmp/onlyboxes-manager-recovery.txt", + SessionID: sessionID, + }) + if err != nil { + t.Fatal(err) + } + if reused.Created || reused.Stdout != "manager-recovery-ok" || reused.SessionID != sessionID { + t.Fatalf("unexpected recovered execution: %#v", reused) + } + if backend.createCount() != 1 { + t.Fatalf("recovered execution created a second sandbox: creates=%d", backend.createCount()) + } +} + func liveTerminalBackend(t *testing.T) (*recordingE2BBackend, string) { t.Helper() if os.Getenv("E2B_INTEGRATION") != "1" { diff --git a/worker/worker-bridge-e2b/internal/runner/terminal_session_test.go b/worker/worker-bridge-e2b/internal/runner/terminal_session_test.go index 18e4853..1164069 100644 --- a/worker/worker-bridge-e2b/internal/runner/terminal_session_test.go +++ b/worker/worker-bridge-e2b/internal/runner/terminal_session_test.go @@ -20,19 +20,43 @@ import ( ) type fakeE2BBackend struct { - mu sync.Mutex - created int - killed int - killedIDs []string - timeouts []int - runStarted chan struct{} - runRelease chan struct{} - createFn func(context.Context, string, int) (*e2b.Sandbox, error) - timeoutFn func(context.Context, string, int) error - runFn func(context.Context, *e2b.Sandbox, string, int) (e2b.CommandResult, error) - readFn func(context.Context, *e2b.Sandbox, string, int64) (e2b.File, error) - openFn func(context.Context, *e2b.Sandbox, string) (e2b.FileReader, error) - killFn func(context.Context, string) error + mu sync.Mutex + created int + killed int + killedIDs []string + timeouts []int + runStarted chan struct{} + runRelease chan struct{} + createFn func(context.Context, string, int) (*e2b.Sandbox, error) + createWithMetadataFn func(context.Context, string, int, map[string]string) (*e2b.Sandbox, error) + listFn func(context.Context, map[string]string) ([]e2b.SandboxInfo, error) + connectFn func(context.Context, string, int) (*e2b.Sandbox, error) + timeoutFn func(context.Context, string, int) error + runFn func(context.Context, *e2b.Sandbox, string, int) (e2b.CommandResult, error) + readFn func(context.Context, *e2b.Sandbox, string, int64) (e2b.File, error) + openFn func(context.Context, *e2b.Sandbox, string) (e2b.FileReader, error) + killFn func(context.Context, string) error +} + +func (f *fakeE2BBackend) CreateWithMetadata(ctx context.Context, template string, timeout int, metadata map[string]string) (*e2b.Sandbox, error) { + if f.createWithMetadataFn != nil { + return f.createWithMetadataFn(ctx, template, timeout, metadata) + } + return f.Create(ctx, template, timeout) +} + +func (f *fakeE2BBackend) List(ctx context.Context, metadata map[string]string) ([]e2b.SandboxInfo, error) { + if f.listFn != nil { + return f.listFn(ctx, metadata) + } + return nil, nil +} + +func (f *fakeE2BBackend) Connect(ctx context.Context, sandboxID string, timeout int) (*e2b.Sandbox, error) { + if f.connectFn != nil { + return f.connectFn(ctx, sandboxID, timeout) + } + return &e2b.Sandbox{ID: sandboxID, Domain: "test", AccessToken: "fresh-token"}, nil } func (f *fakeE2BBackend) Create(ctx context.Context, template string, timeout int) (*e2b.Sandbox, error) { From f8e0ae2315ee1dd9e9d27e6af3636b7f2341a9d1 Mon Sep 17 00:00:00 2001 From: Coolfan Date: Fri, 7 Aug 2026 22:23:40 +0800 Subject: [PATCH 5/7] feat(worker-boxlite): recover terminal sessions after restart Derive deterministic box names from session-id hashes so boxes survive worker exit, and reconcile Console recovery candidates on reconnect before accepting commands. Restarted boxes keep their original session_id and lease. --- worker/worker-boxlite/Cargo.lock | 1 + worker/worker-boxlite/Cargo.toml | 1 + .../README/implementation-plan.md | 2 +- worker/worker-boxlite/README/overview.md | 5 +- worker/worker-boxlite/src/boxlite_runtime.rs | 102 +++- .../src/runner/session_client.rs | 104 +++- .../src/runner/terminal_session_manager.rs | 453 +++++++++++++++++- worker/worker-boxlite/tests/integration.rs | 191 +++++++- 8 files changed, 839 insertions(+), 20 deletions(-) diff --git a/worker/worker-boxlite/Cargo.lock b/worker/worker-boxlite/Cargo.lock index 6780327..62197ea 100644 --- a/worker/worker-boxlite/Cargo.lock +++ b/worker/worker-boxlite/Cargo.lock @@ -4095,6 +4095,7 @@ dependencies = [ "reqwest", "serde", "serde_json", + "sha2", "thiserror 2.0.18", "tokio", "tokio-stream", diff --git a/worker/worker-boxlite/Cargo.toml b/worker/worker-boxlite/Cargo.toml index 89a4f58..528e9dc 100644 --- a/worker/worker-boxlite/Cargo.toml +++ b/worker/worker-boxlite/Cargo.toml @@ -13,6 +13,7 @@ rand = "0.8" reqwest = { version = "0.12", default-features = false, features = ["rustls-tls"] } serde = { version = "1", features = ["derive"] } serde_json = "1" +sha2 = "0.10" thiserror = "2" toml = "0.8" tokio = { version = "1", features = ["macros", "rt-multi-thread", "signal", "sync", "time"] } diff --git a/worker/worker-boxlite/README/implementation-plan.md b/worker/worker-boxlite/README/implementation-plan.md index 709cc08..a5c8670 100644 --- a/worker/worker-boxlite/README/implementation-plan.md +++ b/worker/worker-boxlite/README/implementation-plan.md @@ -732,7 +732,7 @@ struct MockLiteBox { ... } - [ ] gRPC 握手流程（Fake gRPC Server via tonic） - [ ] 心跳容忍（单次超时恢复） - [ ] 心跳失败（双次超时触发重连） -- [ ] 优雅关闭清理所有 VM +- [ ] 优雅关闭清理一次性 VM，并保留可恢复的 detached terminal VM ### 7.3 集成测试 diff --git a/worker/worker-boxlite/README/overview.md b/worker/worker-boxlite/README/overview.md index 9a9e797..4f53970 100644 --- a/worker/worker-boxlite/README/overview.md +++ b/worker/worker-boxlite/README/overview.md @@ -68,8 +68,9 @@ Capability behavior: - `terminalExec` cleanup behavior: - command timeout/cancel marks the session for destruction and stops it accepting new commands; the box is removed once in-flight commands drain, so one command's timeout does not kill its siblings. - idle sessions are reaped after lease expiry by an internal janitor loop; a session with in-flight commands is never reaped. - - worker shutdown cancels pending Box creation, waits up to five seconds for it to stop, then force-removes all managed terminal boxes; a Box returned after the wait is removed as a late cleanup. - - `SIGINT`/`SIGTERM` performs best-effort cleanup; `SIGKILL`/process crash does not guarantee cleanup. + - terminal Boxes use deterministic `onlyboxes-terminal-v1-` names with `auto_remove=false` and `detach=true`, so normal exit and process termination preserve them. + - after reconnect, the worker reconciles Console candidates before accepting commands, reattaches or starts matching Boxes from the configured `WORKER_BOXLITE_HOME`, restores the exact lease, and removes local Onlyboxes terminal orphans. + - lease expiry, explicit destruction, unsafe command timeout, and invalid Box state still remove the Box; one-shot `pythonExec` Boxes remain per-call resources. - `terminalExec` result uses JSON payload: - `{"session_id":"...","created":true,"stdout":"...","stderr":"...","exit_code":0,"stdout_truncated":false,"stderr_truncated":false,"lease_expires_unix_ms":...}` - output truncation: diff --git a/worker/worker-boxlite/src/boxlite_runtime.rs b/worker/worker-boxlite/src/boxlite_runtime.rs index b2c43aa..973b413 100644 --- a/worker/worker-boxlite/src/boxlite_runtime.rs +++ b/worker/worker-boxlite/src/boxlite_runtime.rs @@ -1,11 +1,11 @@ -use std::collections::HashMap; +use std::collections::{HashMap, HashSet}; use std::path::{Path, PathBuf}; use std::sync::{Arc, Mutex, OnceLock}; use std::time::Duration; use boxlite::{ - BoxCommand, BoxOptions, BoxliteOptions, BoxliteRuntime, CopyOptions, ExecStderr, ExecStdout, - LiteBox, RootfsSpec, + BoxCommand, BoxOptions, BoxStatus, BoxliteOptions, BoxliteRuntime, CopyOptions, ExecStderr, + ExecStdout, LiteBox, RootfsSpec, }; use tokio::task::JoinHandle; use tokio_stream::StreamExt; @@ -108,7 +108,7 @@ pub(crate) async fn create_terminal_session_box( let runtime = runtime(cfg)?; let litebox = { - let create = runtime.create( + let create = runtime.get_or_create( build_terminal_exec_box_options(cfg), Some(box_name.trim().to_owned()), ); @@ -120,7 +120,7 @@ pub(crate) async fn create_terminal_session_box( TERMINAL_SESSION_CREATE_CANCELLED_MESSAGE.to_owned(), )), _ = tokio::time::sleep(remaining) => Err(BoxliteCommandError::DeadlineExceeded), - result = &mut create => result.map_err(|err| { + result = &mut create => result.map(|(litebox, _)| litebox).map_err(|err| { BoxliteCommandError::ExecutionFailed(format!("terminalExec create failed: {err}")) }), }, @@ -128,7 +128,7 @@ pub(crate) async fn create_terminal_session_box( _ = shutdown.cancelled() => Err(BoxliteCommandError::ExecutionFailed( TERMINAL_SESSION_CREATE_CANCELLED_MESSAGE.to_owned(), )), - result = &mut create => result.map_err(|err| { + result = &mut create => result.map(|(litebox, _)| litebox).map_err(|err| { BoxliteCommandError::ExecutionFailed(format!("terminalExec create failed: {err}")) }), }, @@ -180,6 +180,79 @@ pub(crate) async fn create_terminal_session_box( Ok(box_id) } +pub(crate) async fn recover_terminal_session_box( + cfg: &Config, + box_name: &str, +) -> Result, BoxliteCommandError> { + let runtime = runtime(cfg)?; + let Some(litebox) = runtime + .get(box_name.trim()) + .await + .map_err(|err| BoxliteCommandError::ExecutionFailed(format!("lookup box failed: {err}")))? + else { + return Ok(None); + }; + + match litebox.info().status { + BoxStatus::Running => {} + BoxStatus::Configured | BoxStatus::Stopped => { + litebox.start().await.map_err(|err| { + BoxliteCommandError::ExecutionFailed(format!("restart terminalExec box: {err}")) + })?; + } + BoxStatus::Unknown | BoxStatus::Stopping | BoxStatus::Paused | BoxStatus::Failed => { + return Err(BoxliteCommandError::ExecutionFailed(format!( + "terminalExec box has invalid status: {:?}", + litebox.info().status + ))); + } + } + + let box_id = litebox.id().as_str().to_owned(); + cache_terminal_session_box(litebox); + Ok(Some(box_id)) +} + +pub(crate) async fn cleanup_orphan_terminal_session_boxes( + cfg: &Config, + expected_names: &HashSet, +) -> usize { + let Ok(runtime) = runtime(cfg) else { + return 0; + }; + let infos = match runtime.list_info().await { + Ok(infos) => infos, + Err(err) => { + tracing::warn!(error = %err, "list terminal Box orphans failed"); + return 0; + } + }; + + let mut cleaned = 0; + for info in infos { + let Some(name) = info.name.as_deref() else { + continue; + }; + if !is_onlyboxes_terminal_session_box_name(name) || expected_names.contains(name) { + continue; + } + remove_box(cfg, info.id.as_str()).await; + cleaned += 1; + } + cleaned +} + +fn is_onlyboxes_terminal_session_box_name(name: &str) -> bool { + const PREFIX: &str = "onlyboxes-terminal-v1-"; + let Some(hash) = name.strip_prefix(PREFIX) else { + return false; + }; + hash.len() == 64 + && hash + .bytes() + .all(|byte| byte.is_ascii_digit() || (b'a'..=b'f').contains(&byte)) +} + pub(crate) async fn exec_terminal_shell( cfg: &Config, box_id: &str, @@ -348,6 +421,7 @@ fn build_terminal_exec_box_options(cfg: &Config) -> BoxOptions { memory_mib: Some(cfg.terminal_exec_memory_mib), rootfs: RootfsSpec::Image(cfg.terminal_exec_image.clone()), auto_remove: false, + detach: true, entrypoint: Some(vec!["sh".to_owned(), "-lc".to_owned()]), cmd: Some(vec![TERMINAL_EXEC_IDLE_COMMAND.to_owned()]), ..Default::default() @@ -605,6 +679,20 @@ mod tests { assert_eq!(resolve_cpus(999), u8::MAX); } + #[test] + fn orphan_cleanup_name_filter_rejects_foreign_boxes() { + let valid = format!("onlyboxes-terminal-v1-{}", "a".repeat(64)); + assert!(is_onlyboxes_terminal_session_box_name(&valid)); + assert!(!is_onlyboxes_terminal_session_box_name("developer-box")); + assert!(!is_onlyboxes_terminal_session_box_name( + "onlyboxes-terminal-v1-not-a-session-hash" + )); + assert!(!is_onlyboxes_terminal_session_box_name(&format!( + "onlyboxes-terminal-v1-{}", + "A".repeat(64) + ))); + } + #[test] fn terminal_exec_box_options_override_entrypoint_to_idle_loop() { let cfg = Config { @@ -653,6 +741,8 @@ mod tests { options.cmd, Some(vec![TERMINAL_EXEC_IDLE_COMMAND.to_owned()]) ); + assert!(!options.auto_remove); + assert!(options.detach); } #[test] diff --git a/worker/worker-boxlite/src/runner/session_client.rs b/worker/worker-boxlite/src/runner/session_client.rs index 6e55025..08d2f75 100644 --- a/worker/worker-boxlite/src/runner/session_client.rs +++ b/worker/worker-boxlite/src/runner/session_client.rs @@ -11,11 +11,14 @@ use tonic::transport::{Channel, ClientTlsConfig, Endpoint}; use crate::config::Config; use crate::proto::registryv1::{ - connect_request, connect_response, worker_registry_service_client::WorkerRegistryServiceClient, - CommandDispatch, ConnectRequest, ConnectResponse, HeartbeatAck, HeartbeatFrame, + connect_request, connect_response, terminal_session_recovery_result, + worker_registry_service_client::WorkerRegistryServiceClient, CommandDispatch, ConnectRequest, + ConnectResponse, HeartbeatAck, HeartbeatFrame, TerminalSessionRecoveryReport, }; -use super::terminal_session_manager::shared_active_session_count; +use super::terminal_session_manager::{ + shared_active_session_count, shared_recover_terminal_sessions, +}; use super::{ build_command_result, build_hello, command_dispatch_summary_for_log, duration_from_server, RunnerError, @@ -66,6 +69,63 @@ async fn run_session_with_builder( )); } + let recovery_results = tokio::select! { + _ = shutdown.cancelled() => return Err(RunnerError::Message("worker shutdown during terminal session recovery".to_owned())), + result = tokio::time::timeout( + cfg.call_timeout, + shared_recover_terminal_sessions(cfg, &ack.terminal_session_recovery_candidates), + ) => result.map_err(|_| RunnerError::Message("terminal session recovery deadline exceeded".to_owned()))?, + }; + outbound_tx + .send(ConnectRequest { + payload: Some(connect_request::Payload::TerminalSessionRecoveryReport( + TerminalSessionRecoveryReport { + results: recovery_results.clone(), + }, + )), + }) + .await + .map_err(|_| { + RunnerError::Message("send terminal session recovery report failed".to_owned()) + })?; + let recovery_ack = recv_with_timeout(shutdown.clone(), cfg.call_timeout, inbound.message()) + .await? + .ok_or_else(|| { + RunnerError::Message("stream closed before terminal session recovery ack".to_owned()) + })?; + if !matches!( + recovery_ack.payload, + Some(connect_response::Payload::TerminalSessionRecoveryAck(_)) + ) { + return Err(RunnerError::Message( + "unexpected response while waiting for terminal session recovery ack".to_owned(), + )); + } + + tracing::info!( + executor_kind = "boxlite", + candidates = recovery_results.len(), + recovered = recovery_results + .iter() + .filter(|result| { + result.status == terminal_session_recovery_result::Status::Recovered as i32 + }) + .count(), + missing = recovery_results + .iter() + .filter(|result| { + result.status == terminal_session_recovery_result::Status::Missing as i32 + }) + .count(), + invalid = recovery_results + .iter() + .filter(|result| { + result.status == terminal_session_recovery_result::Status::Invalid as i32 + }) + .count(), + "terminal session recovery acknowledged" + ); + let heartbeat_interval = duration_from_server(ack.heartbeat_interval_sec, cfg.heartbeat_interval); tracing::info!( @@ -341,6 +401,7 @@ mod tests { connect_request as test_connect_request, connect_response as test_connect_response, CommandDispatch as TestCommandDispatch, CommandResult, ConnectAck, ConnectHello, ConnectRequest as TestConnectRequest, ConnectResponse, HeartbeatAck, HeartbeatFrame, + TerminalSessionRecoveryAck, }; use super::*; @@ -444,12 +505,33 @@ mod tests { payload: Some(test_connect_response::Payload::ConnectAck(ConnectAck { session_id: TEST_SESSION_ID.to_owned(), heartbeat_interval_sec: shared.heartbeat_interval_sec, + terminal_session_recovery_candidates: Vec::new(), })), })) .await .map_err(|_| Status::internal("failed to enqueue connect ack"))?; tokio::spawn(async move { + match inbound.message().await { + Ok(Some(frame)) + if matches!( + frame.payload, + Some(test_connect_request::Payload::TerminalSessionRecoveryReport(_)) + ) => {} + _ => return, + } + if response_tx + .send(Ok(ConnectResponse { + payload: Some(test_connect_response::Payload::TerminalSessionRecoveryAck( + TerminalSessionRecoveryAck {}, + )), + })) + .await + .is_err() + { + return; + } + for dispatch in &shared.dispatches { if response_tx .send(Ok(ConnectResponse { @@ -487,7 +569,11 @@ mod tests { Some(test_connect_request::Payload::CommandResult(result)) => { let _ = shared.command_result_tx.send(result); } - Some(test_connect_request::Payload::Hello(_)) | None => return, + Some(test_connect_request::Payload::Hello(_)) + | Some(test_connect_request::Payload::TerminalSessionRecoveryReport( + _, + )) + | None => return, }, Ok(None) | Err(_) => return, } @@ -556,11 +642,17 @@ mod tests { worker_secret: "secret".to_owned(), heartbeat_interval: Duration::from_millis(25), heartbeat_jitter_pct: 0, - call_timeout: Duration::from_millis(25), + call_timeout: Duration::from_millis(250), node_name: "worker-boxlite-test".to_owned(), executor_kind: "boxlite".to_owned(), labels: BTreeMap::new(), - boxlite_home: String::new(), + boxlite_home: std::env::temp_dir() + .join(format!( + "onlyboxes-worker-boxlite-session-client-tests-{}", + std::process::id() + )) + .to_string_lossy() + .into_owned(), python_exec_image: "ghcr.io/astral-sh/uv:python3.12-bookworm-slim".to_owned(), python_exec_memory_mib: 256, python_exec_cpus: 1, diff --git a/worker/worker-boxlite/src/runner/terminal_session_manager.rs b/worker/worker-boxlite/src/runner/terminal_session_manager.rs index a63ed8d..5135957 100644 --- a/worker/worker-boxlite/src/runner/terminal_session_manager.rs +++ b/worker/worker-boxlite/src/runner/terminal_session_manager.rs @@ -1,4 +1,4 @@ -use std::collections::HashMap; +use std::collections::{HashMap, HashSet}; use std::path::{Path, PathBuf}; use std::sync::atomic::{AtomicBool, AtomicU32, Ordering}; use std::sync::Arc; @@ -9,6 +9,7 @@ use async_trait::async_trait; use base64::Engine; use reqwest::StatusCode; use serde::{Deserialize, Serialize}; +use sha2::{Digest, Sha256}; use tokio::sync::{watch, Mutex, Notify}; use tokio_util::io::ReaderStream; use tokio_util::sync::CancellationToken; @@ -16,6 +17,10 @@ use uuid::Uuid; use crate::boxlite_runtime::{self, BoxliteCommandError, CollectedExecOutput}; use crate::config::Config; +use crate::proto::registryv1::{ + terminal_session_recovery_result, TerminalSessionRecoveryCandidate, + TerminalSessionRecoveryResult, +}; pub(crate) const TERMINAL_EXEC_JANITOR_INTERVAL: Duration = Duration::from_secs(5); #[cfg(not(test))] @@ -204,6 +209,23 @@ pub(crate) trait TerminalBackend: Send + Sync { shutdown: CancellationToken, deadline_unix_ms: i64, ) -> Result; + async fn create_session_box_for_session( + &self, + _session_id: &str, + shutdown: CancellationToken, + deadline_unix_ms: i64, + ) -> Result { + self.create_session_box(shutdown, deadline_unix_ms).await + } + async fn recover_session_box( + &self, + _session_id: &str, + ) -> Result, TerminalOperationError> { + Ok(None) + } + async fn cleanup_orphan_session_boxes(&self, _expected_names: &HashSet) -> usize { + 0 + } async fn exec_shell_command( &self, box_id: &str, @@ -308,7 +330,7 @@ pub(crate) fn shared_terminal_session_manager(cfg: &Config) -> Arc i32 { } } +pub(crate) async fn shared_recover_terminal_sessions( + cfg: &Config, + candidates: &[TerminalSessionRecoveryCandidate], +) -> Vec { + shared_terminal_session_manager(cfg) + .recover_sessions(candidates) + .await +} + struct RetiredSession { box_id: String, capacity_reserved: bool, @@ -441,7 +472,11 @@ impl TerminalSessionManager { let _creation_guard = PendingCreationGuard { manager: self }; match self .backend - .create_session_box(self.shutdown.clone(), req.deadline_unix_ms) + .create_session_box_for_session( + &claimed.session_id, + self.shutdown.clone(), + req.deadline_unix_ms, + ) .await { Ok(box_id) => { @@ -713,7 +748,16 @@ impl TerminalSessionManager { Ok(TerminalResourceRunResult { blob, ..result }) } + #[cfg(test)] pub(crate) async fn close(&self) { + self.close_inner(false).await; + } + + pub(crate) async fn close_preserving(&self) { + self.close_inner(true).await; + } + + async fn close_inner(&self, preserve: bool) { if self.closed.swap(true, Ordering::SeqCst) { return; } @@ -751,7 +795,14 @@ impl TerminalSessionManager { self.wait_for_pending_cleanups().await; for session in sessions { - self.cleanup_retired_session(session).await; + if preserve { + if session.capacity_reserved { + self.active_session_reservations + .fetch_sub(1, Ordering::SeqCst); + } + } else { + self.cleanup_retired_session(session).await; + } } } @@ -770,6 +821,129 @@ impl TerminalSessionManager { .min(i32::MAX as u32) as i32 } + pub(crate) async fn recover_sessions( + &self, + candidates: &[TerminalSessionRecoveryCandidate], + ) -> Vec { + let started_at = std::time::Instant::now(); + let now = SystemTime::now(); + let mut results = Vec::with_capacity(candidates.len()); + let mut recovered_names = HashSet::with_capacity(candidates.len()); + + for candidate in candidates { + let session_id = candidate.session_id.trim(); + let lease_expires_at = system_time_from_unix_millis(candidate.lease_expires_unix_ms); + let status = if session_id.is_empty() + || lease_expires_at.is_none() + || lease_expires_at.is_some_and(|lease| lease <= now) + { + terminal_session_recovery_result::Status::Invalid + } else { + let lease_expires_at = lease_expires_at.expect("validated lease"); + match self.recover_one_session(session_id, lease_expires_at).await { + Ok(true) => { + recovered_names.insert(terminal_session_resource_name(session_id)); + terminal_session_recovery_result::Status::Recovered + } + Ok(false) => terminal_session_recovery_result::Status::Missing, + Err(err) => { + tracing::warn!( + session_id_hash = %terminal_session_id_hash(session_id), + error = %err, + "terminal Box recovery failed" + ); + terminal_session_recovery_result::Status::Invalid + } + } + }; + results.push(TerminalSessionRecoveryResult { + session_id: candidate.session_id.clone(), + status: status as i32, + }); + } + + let orphan_cleaned = self + .backend + .cleanup_orphan_session_boxes(&recovered_names) + .await; + let missing = results + .iter() + .filter(|result| { + result.status == terminal_session_recovery_result::Status::Missing as i32 + }) + .count(); + let invalid = results + .iter() + .filter(|result| { + result.status == terminal_session_recovery_result::Status::Invalid as i32 + }) + .count(); + tracing::info!( + executor_kind = "boxlite", + discovered = recovered_names.len() + orphan_cleaned, + candidate = candidates.len(), + recovered = recovered_names.len(), + missing, + invalid, + orphan_cleaned, + duration_ms = started_at.elapsed().as_millis(), + recovery_failures = invalid, + "terminal session recovery completed" + ); + results + } + + async fn recover_one_session( + &self, + session_id: &str, + lease_expires_at: SystemTime, + ) -> Result { + { + let mut sessions = self.sessions.lock().await; + if let Some(session) = sessions.get_mut(session_id) { + if session.destroying || session.box_id.trim().is_empty() { + return Err(TerminalOperationError::ExecutionFailed( + "terminal session is not recoverable".to_owned(), + )); + } + session.lease_expires_at = lease_expires_at; + return Ok(true); + } + } + + let Some(box_id) = self.backend.recover_session_box(session_id).await? else { + return Ok(false); + }; + + let mut sessions = self.sessions.lock().await; + if self.closed.load(Ordering::SeqCst) { + return Err(TerminalOperationError::ExecutionFailed( + TERMINAL_MANAGER_CLOSED_MESSAGE.to_owned(), + )); + } + if let Some(session) = sessions.get_mut(session_id) { + session.lease_expires_at = lease_expires_at; + return Ok(true); + } + + let (ready_tx, _) = watch::channel(SessionReadyState::Ready(box_id.clone())); + sessions.insert( + session_id.to_owned(), + TerminalSession { + session_id: session_id.to_owned(), + box_id, + lease_expires_at, + inflight: 0, + destroying: false, + capacity_reserved: true, + ready_tx, + }, + ); + self.active_session_reservations + .fetch_add(1, Ordering::SeqCst); + Ok(true) + } + pub(crate) async fn cleanup_expired_sessions(&self) { let now = SystemTime::now(); let expired = { @@ -1219,6 +1393,34 @@ impl TerminalBackend for BoxliteTerminalBackend { }) } + async fn create_session_box_for_session( + &self, + session_id: &str, + shutdown: CancellationToken, + deadline_unix_ms: i64, + ) -> Result { + let name = terminal_session_resource_name(session_id); + boxlite_runtime::create_terminal_session_box(&self.cfg, &name, shutdown, deadline_unix_ms) + .await + .map_err(map_boxlite_terminal_error) + } + + async fn recover_session_box( + &self, + session_id: &str, + ) -> Result, TerminalOperationError> { + boxlite_runtime::recover_terminal_session_box( + &self.cfg, + &terminal_session_resource_name(session_id), + ) + .await + .map_err(map_boxlite_terminal_error) + } + + async fn cleanup_orphan_session_boxes(&self, expected_names: &HashSet) -> usize { + boxlite_runtime::cleanup_orphan_terminal_session_boxes(&self.cfg, expected_names).await + } + async fn exec_shell_command( &self, box_id: &str, @@ -1269,6 +1471,29 @@ impl TerminalBackend for BoxliteTerminalBackend { } } +fn map_boxlite_terminal_error(err: BoxliteCommandError) -> TerminalOperationError { + match err { + BoxliteCommandError::DeadlineExceeded => TerminalOperationError::DeadlineExceeded, + BoxliteCommandError::MissingBox => { + TerminalOperationError::ExecutionFailed("terminalExec box not found".to_owned()) + } + BoxliteCommandError::ExecutionFailed(message) => { + TerminalOperationError::ExecutionFailed(message) + } + } +} + +fn terminal_session_id_hash(session_id: &str) -> String { + format!("{:x}", Sha256::digest(session_id.as_bytes())) +} + +fn terminal_session_resource_name(session_id: &str) -> String { + format!( + "onlyboxes-terminal-v1-{}", + terminal_session_id_hash(session_id) + ) +} + fn truncate_by_bytes(value: &str, max_bytes: usize) -> (String, bool) { if max_bytes == 0 || value.len() <= max_bytes { return (value.to_owned(), false); @@ -1386,6 +1611,13 @@ fn to_unix_millis(time: SystemTime) -> i64 { .unwrap_or_default() } +fn system_time_from_unix_millis(value: i64) -> Option { + if value <= 0 { + return None; + } + SystemTime::UNIX_EPOCH.checked_add(Duration::from_millis(value as u64)) +} + fn serialize_resource_blob(blob: &[u8], serializer: S) -> Result where S: serde::Serializer, @@ -1431,6 +1663,85 @@ mod tests { use tokio::net::TcpListener; use tokio::sync::Notify; + struct RecoveryBackend { + boxes: Mutex>, + expected_names: Mutex>, + } + + impl RecoveryBackend { + fn new(session_id: &str, box_id: &str) -> Arc { + Arc::new(Self { + boxes: Mutex::new(HashMap::from([(session_id.to_owned(), box_id.to_owned())])), + expected_names: Mutex::new(HashSet::new()), + }) + } + } + + #[async_trait] + impl TerminalBackend for RecoveryBackend { + async fn create_session_box( + &self, + _shutdown: CancellationToken, + _deadline_unix_ms: i64, + ) -> Result { + Err(TerminalOperationError::ExecutionFailed( + "unexpected create".to_owned(), + )) + } + + async fn recover_session_box( + &self, + session_id: &str, + ) -> Result, TerminalOperationError> { + Ok(self.boxes.lock().await.get(session_id).cloned()) + } + + async fn cleanup_orphan_session_boxes(&self, expected_names: &HashSet) -> usize { + *self.expected_names.lock().await = expected_names.clone(); + 0 + } + + async fn exec_shell_command( + &self, + _box_id: &str, + _command: &str, + _deadline_unix_ms: i64, + ) -> Result { + Ok(CollectedExecOutput { + stdout: String::new(), + stderr: String::new(), + exit_code: 0, + }) + } + + async fn exec_resource_probe( + &self, + _box_id: &str, + _action: &str, + _file_path: &str, + _max_read_bytes: usize, + _deadline_unix_ms: i64, + ) -> Result { + Err(BoxliteCommandError::ExecutionFailed( + "unexpected resource probe".to_owned(), + )) + } + + async fn copy_out_file( + &self, + _box_id: &str, + _container_src: &str, + _host_dst: &Path, + _deadline_unix_ms: i64, + ) -> Result<(), BoxliteCommandError> { + Err(BoxliteCommandError::ExecutionFailed( + "unexpected copy".to_owned(), + )) + } + + async fn remove_box(&self, _box_id: &str) {} + } + struct StatefulShellBackend { next_box: Mutex, persisted: Mutex>, @@ -4147,4 +4458,138 @@ mod tests { manager.close().await; } + + #[tokio::test] + async fn recovery_restores_box_with_exact_lease_and_capacity() { + let session_id = "owner-a:session-recover"; + let backend = RecoveryBackend::new(session_id, "box-recover"); + let manager = manager_with_limit(backend.clone(), 1); + let lease = SystemTime::now() + Duration::from_secs(600); + let lease_unix_ms = to_unix_millis(lease); + + let results = manager + .recover_sessions(&[TerminalSessionRecoveryCandidate { + session_id: session_id.to_owned(), + lease_expires_unix_ms: lease_unix_ms, + }]) + .await; + assert_eq!(results.len(), 1); + assert_eq!( + results[0].status, + terminal_session_recovery_result::Status::Recovered as i32 + ); + assert_eq!(manager.active_session_count().await, 1); + + let sessions = manager.sessions.lock().await; + let recovered = sessions.get(session_id).expect("recovered session"); + assert_eq!(recovered.box_id, "box-recover"); + assert_eq!(recovered.inflight, 0); + assert_eq!(to_unix_millis(recovered.lease_expires_at), lease_unix_ms); + drop(sessions); + + let expected_names = backend.expected_names.lock().await; + assert_eq!( + &*expected_names, + &HashSet::from([terminal_session_resource_name(session_id)]) + ); + drop(expected_names); + + let reused = manager + .execute(exec_req(session_id, "printf recovered", false)) + .await + .expect("execute recovered session"); + assert!(!reused.created); + assert_eq!(reused.session_id, session_id); + + let repeated = manager + .recover_sessions(&[TerminalSessionRecoveryCandidate { + session_id: session_id.to_owned(), + lease_expires_unix_ms: lease_unix_ms, + }]) + .await; + assert_eq!( + repeated[0].status, + terminal_session_recovery_result::Status::Recovered as i32 + ); + assert_eq!(manager.active_session_count().await, 1); + assert_eq!( + to_unix_millis( + manager + .sessions + .lock() + .await + .get(session_id) + .expect("recovered session") + .lease_expires_at + ), + lease_unix_ms + ); + manager.close_preserving().await; + } + + #[tokio::test] + async fn recovery_reports_missing_and_skips_expired_candidate() { + let backend = RecoveryBackend::new("different-session", "box-other"); + let manager = manager_with_limit(backend.clone(), 1); + let results = manager + .recover_sessions(&[ + TerminalSessionRecoveryCandidate { + session_id: "session-missing".to_owned(), + lease_expires_unix_ms: to_unix_millis( + SystemTime::now() + Duration::from_secs(60), + ), + }, + TerminalSessionRecoveryCandidate { + session_id: "session-expired".to_owned(), + lease_expires_unix_ms: to_unix_millis( + SystemTime::now() - Duration::from_secs(60), + ), + }, + ]) + .await; + assert_eq!( + results[0].status, + terminal_session_recovery_result::Status::Missing as i32 + ); + assert_eq!( + results[1].status, + terminal_session_recovery_result::Status::Invalid as i32 + ); + assert_eq!(manager.active_session_count().await, 0); + assert!(backend.expected_names.lock().await.is_empty()); + manager.close_preserving().await; + } + + #[tokio::test] + async fn concurrent_recovery_uses_one_reservation() { + let session_id = "session-concurrent"; + let backend = RecoveryBackend::new(session_id, "box-concurrent"); + let manager = manager_with_limit(backend, 1); + let candidates = vec![TerminalSessionRecoveryCandidate { + session_id: session_id.to_owned(), + lease_expires_unix_ms: to_unix_millis(SystemTime::now() + Duration::from_secs(60)), + }]; + let (first, second) = tokio::join!( + manager.recover_sessions(&candidates), + manager.recover_sessions(&candidates) + ); + for result in [first, second] { + assert_eq!( + result[0].status, + terminal_session_recovery_result::Status::Recovered as i32 + ); + } + assert_eq!(manager.active_session_count().await, 1); + manager.close_preserving().await; + } + + #[test] + fn recovery_resource_name_is_deterministic_and_does_not_expose_session_id() { + let session_id = "owner-secret:session-secret"; + let name = terminal_session_resource_name(session_id); + assert!(name.starts_with("onlyboxes-terminal-v1-")); + assert_eq!(name.len(), "onlyboxes-terminal-v1-".len() + 64); + assert!(!name.contains("owner-secret")); + assert_eq!(name, terminal_session_resource_name(session_id)); + } } diff --git a/worker/worker-boxlite/tests/integration.rs b/worker/worker-boxlite/tests/integration.rs index c022d86..db5ed13 100644 --- a/worker/worker-boxlite/tests/integration.rs +++ b/worker/worker-boxlite/tests/integration.rs @@ -5,7 +5,9 @@ //! These tests require a working Boxlite runtime (macOS Hypervisor.framework or Linux KVM). //! Run with: `cargo test --features integration` -use boxlite::{BoxCommand, BoxOptions, BoxliteOptions, BoxliteRuntime, RootfsSpec}; +use boxlite::{BoxCommand, BoxOptions, BoxStatus, BoxliteOptions, BoxliteRuntime, RootfsSpec}; +use std::io::{BufRead, BufReader}; +use std::process::{Command, Stdio}; use std::sync::{Arc, OnceLock}; use tokio_stream::StreamExt; @@ -148,3 +150,190 @@ async fn test_vm_stop_and_cleanup() { rt.remove(&name, true).await.expect("remove"); assert!(!rt.exists(&name).await.expect("exists check after remove")); } + +#[tokio::test] +async fn test_detached_terminal_survives_runtime_restart_and_keeps_files() { + let home = std::path::PathBuf::from(format!( + "/tmp/obx-recovery-it-{}-{:08x}", + std::process::id(), + rand::random::() + )); + let name = random_name("onlyboxes-terminal-v1-recovery"); + let mut options = BoxliteOptions::default(); + options.home_dir = home.clone(); + let first_runtime = Arc::new(BoxliteRuntime::new(options.clone()).expect("first runtime")); + let litebox = first_runtime + .get_or_create( + BoxOptions { + cpus: Some(1), + memory_mib: Some(512), + rootfs: RootfsSpec::Image("alpine:latest".into()), + auto_remove: false, + detach: true, + entrypoint: Some(vec!["sh".into(), "-lc".into()]), + cmd: Some(vec!["while true; do sleep 3600; done".into()]), + ..Default::default() + }, + Some(name.clone()), + ) + .await + .expect("create detached terminal") + .0; + litebox.start().await.expect("start detached terminal"); + let write = litebox + .exec( + BoxCommand::new("sh").args(["-lc", "printf recovery-ok > /tmp/onlyboxes-recovery.txt"]), + ) + .await + .expect("write persistent file"); + assert_eq!(write.wait().await.expect("wait for write").exit_code, 0); + drop(litebox); + first_runtime + .shutdown(None) + .await + .expect("shutdown first runtime"); + drop(first_runtime); + + let second_runtime = Arc::new(BoxliteRuntime::new(options).expect("second runtime")); + let recovered = second_runtime + .get(&name) + .await + .expect("lookup by deterministic name") + .expect("detached terminal survived runtime shutdown"); + match recovered.info().status { + BoxStatus::Running => {} + BoxStatus::Configured | BoxStatus::Stopped => { + recovered.start().await.expect("restart recovered terminal") + } + status => panic!("unexpected recovered status: {status:?}"), + } + let mut read = recovered + .exec(BoxCommand::new("sh").args(["-lc", "cat /tmp/onlyboxes-recovery.txt"])) + .await + .expect("read persistent file"); + let mut stdout = String::new(); + if let Some(mut stream) = read.stdout() { + while let Some(chunk) = stream.next().await { + stdout.push_str(&chunk); + } + } + assert_eq!(read.wait().await.expect("wait for read").exit_code, 0); + assert_eq!(stdout, "recovery-ok"); + second_runtime + .remove(&name, true) + .await + .expect("remove recovered terminal"); +} + +#[tokio::test] +async fn boxlite_forced_termination_helper() { + if std::env::var("BOXLITE_CRASH_HELPER").ok().as_deref() != Some("1") { + return; + } + let home = + std::path::PathBuf::from(std::env::var("BOXLITE_CRASH_HOME").expect("BOXLITE_CRASH_HOME")); + let name = std::env::var("BOXLITE_CRASH_NAME").expect("BOXLITE_CRASH_NAME"); + let mut options = BoxliteOptions::default(); + options.home_dir = home; + let runtime = BoxliteRuntime::new(options).expect("crash helper runtime"); + let litebox = runtime + .get_or_create(detached_terminal_options(), Some(name)) + .await + .expect("crash helper create") + .0; + litebox.start().await.expect("crash helper start"); + let write = litebox + .exec(BoxCommand::new("sh").args([ + "-lc", + "printf forced-recovery-ok > /tmp/onlyboxes-forced-recovery.txt", + ])) + .await + .expect("crash helper write"); + assert_eq!(write.wait().await.expect("crash helper wait").exit_code, 0); + println!("BOXLITE_CRASH_HELPER_READY"); + std::io::Write::flush(&mut std::io::stdout()).expect("flush helper readiness"); + std::future::pending::<()>().await; +} + +#[tokio::test] +async fn test_detached_terminal_survives_forced_worker_termination() { + let home = std::path::PathBuf::from(format!( + "/tmp/obx-crash-it-{}-{:08x}", + std::process::id(), + rand::random::() + )); + let name = random_name("onlyboxes-terminal-v1-crash-recovery"); + let mut child = Command::new(std::env::current_exe().expect("current test executable")) + .args([ + "--exact", + "boxlite_forced_termination_helper", + "--nocapture", + ]) + .env("BOXLITE_CRASH_HELPER", "1") + .env("BOXLITE_CRASH_HOME", &home) + .env("BOXLITE_CRASH_NAME", &name) + .stdout(Stdio::piped()) + .spawn() + .expect("spawn crash helper"); + let stdout = child.stdout.take().expect("crash helper stdout"); + let mut ready = false; + for line in BufReader::new(stdout).lines() { + let line = line.expect("read crash helper output"); + if line.contains("BOXLITE_CRASH_HELPER_READY") { + ready = true; + break; + } + } + assert!( + ready, + "crash helper exited before creating the detached Box" + ); + child.kill().expect("force terminate crash helper"); + let status = child.wait().expect("wait for crash helper termination"); + assert!(!status.success(), "crash helper was not force terminated"); + + let mut options = BoxliteOptions::default(); + options.home_dir = home; + let runtime = BoxliteRuntime::new(options).expect("recovery runtime after forced termination"); + let recovered = runtime + .get(&name) + .await + .expect("lookup after forced termination") + .expect("detached Box survived forced worker termination"); + match recovered.info().status { + BoxStatus::Running => {} + BoxStatus::Configured | BoxStatus::Stopped => { + recovered.start().await.expect("restart recovered Box") + } + status => panic!("unexpected recovered status: {status:?}"), + } + let mut read = recovered + .exec(BoxCommand::new("sh").args(["-lc", "cat /tmp/onlyboxes-forced-recovery.txt"])) + .await + .expect("read after forced termination"); + let mut stdout = String::new(); + if let Some(mut stream) = read.stdout() { + while let Some(chunk) = stream.next().await { + stdout.push_str(&chunk); + } + } + assert_eq!(read.wait().await.expect("wait for read").exit_code, 0); + assert_eq!(stdout, "forced-recovery-ok"); + runtime + .remove(&name, true) + .await + .expect("remove recovered Box"); +} + +fn detached_terminal_options() -> BoxOptions { + BoxOptions { + cpus: Some(1), + memory_mib: Some(512), + rootfs: RootfsSpec::Image("alpine:latest".into()), + auto_remove: false, + detach: true, + entrypoint: Some(vec!["sh".into(), "-lc".into()]), + cmd: Some(vec!["while true; do sleep 3600; done".into()]), + ..Default::default() + } +} From 4193ab2d042d923034b381e0493eac1ba4e5237a Mon Sep 17 00:00:00 2001 From: Coolfan Date: Fri, 7 Aug 2026 22:23:43 +0800 Subject: [PATCH 6/7] docs: document session_unavailable recovery error --- README/API.md | 1 + README/API.zh-CN.md | 1 + 2 files changed, 2 insertions(+) diff --git a/README/API.md b/README/API.md index d453f32..b7e703a 100644 --- a/README/API.md +++ b/README/API.md @@ -574,6 +574,7 @@ Errors: - `400` invalid body/params or `invalid_payload` - `404` `session_not_found` +- `503` `session_unavailable` while the session's bound worker is offline or reconciling after restart; retry the same `session_id` without rerouting it. - `409` `session_busy` or canceled - `session_busy` means the request exceeded the per-session concurrency limit. Workers default to one command per session; the worker must raise `WORKER_TERMINAL_SESSION_MAX_INFLIGHT` to allow concurrent commands on one `session_id`. - `terminalExec` and `terminalResource` share that per-session limit. diff --git a/README/API.zh-CN.md b/README/API.zh-CN.md index 3411aac..288ea3e 100644 --- a/README/API.zh-CN.md +++ b/README/API.zh-CN.md @@ -576,6 +576,7 @@ Worker 类型： - `400` 请求参数非法或 `invalid_payload` - `404` `session_not_found` +- `503` `session_unavailable`：session 绑定的 Worker 离线或重启后正在核对资源；客户端应使用同一 `session_id` 重试，Console 不会将其改派。 - `409` `session_busy` 或任务被取消 - `session_busy` 表示请求超出了单 session 的并发上限。worker 默认每个 session 只允许一条命令，需由 worker 调大 `WORKER_TERMINAL_SESSION_MAX_INFLIGHT` 才能在同一 `session_id` 上并发执行。 - `terminalExec` 与 `terminalResource` 共用该单 session 上限。 From 864636613dd9dee6fef863a7a37995478ce804e8 Mon Sep 17 00:00:00 2001 From: Coolfan Date: Sat, 8 Aug 2026 15:22:30 +0800 Subject: [PATCH 7/7] fix(worker-bridge-e2b): avoid recovery timeout race --- worker/worker-bridge-e2b/internal/runner/session_client.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/worker/worker-bridge-e2b/internal/runner/session_client.go b/worker/worker-bridge-e2b/internal/runner/session_client.go index deeb076..c898c3f 100644 --- a/worker/worker-bridge-e2b/internal/runner/session_client.go +++ b/worker/worker-bridge-e2b/internal/runner/session_client.go @@ -107,8 +107,9 @@ func recoverTerminalSessionsWithTimeout( recoveryCtx, cancel := context.WithTimeout(ctx, timeout) defer cancel() resultCh := make(chan []*registryv1.TerminalSessionRecoveryResult, 1) + recoverTerminalSessions := recoverTerminalSessionsFn go func() { - resultCh <- recoverTerminalSessionsFn(recoveryCtx, candidates) + resultCh <- recoverTerminalSessions(recoveryCtx, candidates) }() select { case <-recoveryCtx.Done():