diff --git a/README/API.md b/README/API.md index 7db0306..88a7e9f 100644 --- a/README/API.md +++ b/README/API.md @@ -632,6 +632,7 @@ Errors: - `400` invalid body/params or `invalid_payload` - `404` `session_not_found` +- `503` `session_unavailable` while the session's bound worker is offline or reconciling after restart; retry the same `session_id` without rerouting it. - `409` `session_busy` or canceled - `session_busy` means the request exceeded the per-session concurrency limit. Workers default to one command per session; the worker must raise `WORKER_TERMINAL_SESSION_MAX_INFLIGHT` to allow concurrent commands on one `session_id`. - `terminalExec` and `terminalResource` share that per-session limit. diff --git a/README/API.zh-CN.md b/README/API.zh-CN.md index eeb276e..b038a4f 100644 --- a/README/API.zh-CN.md +++ b/README/API.zh-CN.md @@ -634,6 +634,7 @@ Worker 类型: - `400` 请求参数非法或 `invalid_payload` - `404` `session_not_found` +- `503` `session_unavailable`:session 绑定的 Worker 离线或重启后正在核对资源;客户端应使用同一 `session_id` 重试,Console 不会将其改派。 - `409` `session_busy` 或任务被取消 - `session_busy` 表示请求超出了单 session 的并发上限。worker 默认每个 session 只允许一条命令,需由 worker 调大 `WORKER_TERMINAL_SESSION_MAX_INFLIGHT` 才能在同一 `session_id` 上并发执行。 - `terminalExec` 与 `terminalResource` 共用该单 session 上限。 diff --git a/api/gen/go/registry/v1/registry.pb.go b/api/gen/go/registry/v1/registry.pb.go index 94038c1..a116238 100644 --- a/api/gen/go/registry/v1/registry.pb.go +++ b/api/gen/go/registry/v1/registry.pb.go @@ -21,6 +21,58 @@ const ( _ = protoimpl.EnforceVersion(protoimpl.MaxVersion - 20) ) +type TerminalSessionRecoveryResult_Status int32 + +const ( + TerminalSessionRecoveryResult_STATUS_UNSPECIFIED TerminalSessionRecoveryResult_Status = 0 + TerminalSessionRecoveryResult_RECOVERED TerminalSessionRecoveryResult_Status = 1 + TerminalSessionRecoveryResult_MISSING TerminalSessionRecoveryResult_Status = 2 + TerminalSessionRecoveryResult_INVALID TerminalSessionRecoveryResult_Status = 3 +) + +// Enum value maps for TerminalSessionRecoveryResult_Status. +var ( + TerminalSessionRecoveryResult_Status_name = map[int32]string{ + 0: "STATUS_UNSPECIFIED", + 1: "RECOVERED", + 2: "MISSING", + 3: "INVALID", + } + TerminalSessionRecoveryResult_Status_value = map[string]int32{ + "STATUS_UNSPECIFIED": 0, + "RECOVERED": 1, + "MISSING": 2, + "INVALID": 3, + } +) + +func (x TerminalSessionRecoveryResult_Status) Enum() *TerminalSessionRecoveryResult_Status { + p := new(TerminalSessionRecoveryResult_Status) + *p = x + return p +} + +func (x TerminalSessionRecoveryResult_Status) String() string { + return protoimpl.X.EnumStringOf(x.Descriptor(), protoreflect.EnumNumber(x)) +} + +func (TerminalSessionRecoveryResult_Status) Descriptor() protoreflect.EnumDescriptor { + return file_registry_v1_registry_proto_enumTypes[0].Descriptor() +} + +func (TerminalSessionRecoveryResult_Status) Type() protoreflect.EnumType { + return &file_registry_v1_registry_proto_enumTypes[0] +} + +func (x TerminalSessionRecoveryResult_Status) Number() protoreflect.EnumNumber { + return protoreflect.EnumNumber(x) +} + +// Deprecated: Use TerminalSessionRecoveryResult_Status.Descriptor instead. +func (TerminalSessionRecoveryResult_Status) EnumDescriptor() ([]byte, []int) { + return file_registry_v1_registry_proto_rawDescGZIP(), []int{6, 0} +} + type CapabilityDeclaration struct { state protoimpl.MessageState `protogen:"open.v1"` Name string `protobuf:"bytes,1,opt,name=name,proto3" json:"name,omitempty"` @@ -294,6 +346,7 @@ type ConnectRequest struct { // *ConnectRequest_Hello // *ConnectRequest_Heartbeat // *ConnectRequest_CommandResult + // *ConnectRequest_TerminalSessionRecoveryReport Payload isConnectRequest_Payload `protobuf_oneof:"payload"` unknownFields protoimpl.UnknownFields sizeCache protoimpl.SizeCache @@ -363,6 +416,15 @@ func (x *ConnectRequest) GetCommandResult() *CommandResult { return nil } +func (x *ConnectRequest) GetTerminalSessionRecoveryReport() *TerminalSessionRecoveryReport { + if x != nil { + if x, ok := x.Payload.(*ConnectRequest_TerminalSessionRecoveryReport); ok { + return x.TerminalSessionRecoveryReport + } + } + return nil +} + type isConnectRequest_Payload interface { isConnectRequest_Payload() } @@ -379,23 +441,214 @@ type ConnectRequest_CommandResult struct { CommandResult *CommandResult `protobuf:"bytes,3,opt,name=command_result,json=commandResult,proto3,oneof"` } +type ConnectRequest_TerminalSessionRecoveryReport struct { + TerminalSessionRecoveryReport *TerminalSessionRecoveryReport `protobuf:"bytes,4,opt,name=terminal_session_recovery_report,json=terminalSessionRecoveryReport,proto3,oneof"` +} + func (*ConnectRequest_Hello) isConnectRequest_Payload() {} func (*ConnectRequest_Heartbeat) isConnectRequest_Payload() {} func (*ConnectRequest_CommandResult) isConnectRequest_Payload() {} +func (*ConnectRequest_TerminalSessionRecoveryReport) isConnectRequest_Payload() {} + +type TerminalSessionRecoveryCandidate struct { + state protoimpl.MessageState `protogen:"open.v1"` + SessionId string `protobuf:"bytes,1,opt,name=session_id,json=sessionId,proto3" json:"session_id,omitempty"` + LeaseExpiresUnixMs int64 `protobuf:"varint,2,opt,name=lease_expires_unix_ms,json=leaseExpiresUnixMs,proto3" json:"lease_expires_unix_ms,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *TerminalSessionRecoveryCandidate) Reset() { + *x = TerminalSessionRecoveryCandidate{} + mi := &file_registry_v1_registry_proto_msgTypes[5] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *TerminalSessionRecoveryCandidate) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*TerminalSessionRecoveryCandidate) ProtoMessage() {} + +func (x *TerminalSessionRecoveryCandidate) ProtoReflect() protoreflect.Message { + mi := &file_registry_v1_registry_proto_msgTypes[5] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use TerminalSessionRecoveryCandidate.ProtoReflect.Descriptor instead. +func (*TerminalSessionRecoveryCandidate) Descriptor() ([]byte, []int) { + return file_registry_v1_registry_proto_rawDescGZIP(), []int{5} +} + +func (x *TerminalSessionRecoveryCandidate) GetSessionId() string { + if x != nil { + return x.SessionId + } + return "" +} + +func (x *TerminalSessionRecoveryCandidate) GetLeaseExpiresUnixMs() int64 { + if x != nil { + return x.LeaseExpiresUnixMs + } + return 0 +} + +type TerminalSessionRecoveryResult struct { + state protoimpl.MessageState `protogen:"open.v1"` + SessionId string `protobuf:"bytes,1,opt,name=session_id,json=sessionId,proto3" json:"session_id,omitempty"` + Status TerminalSessionRecoveryResult_Status `protobuf:"varint,2,opt,name=status,proto3,enum=onlyboxes.registry.v1.TerminalSessionRecoveryResult_Status" json:"status,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *TerminalSessionRecoveryResult) Reset() { + *x = TerminalSessionRecoveryResult{} + mi := &file_registry_v1_registry_proto_msgTypes[6] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *TerminalSessionRecoveryResult) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*TerminalSessionRecoveryResult) ProtoMessage() {} + +func (x *TerminalSessionRecoveryResult) ProtoReflect() protoreflect.Message { + mi := &file_registry_v1_registry_proto_msgTypes[6] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use TerminalSessionRecoveryResult.ProtoReflect.Descriptor instead. +func (*TerminalSessionRecoveryResult) Descriptor() ([]byte, []int) { + return file_registry_v1_registry_proto_rawDescGZIP(), []int{6} +} + +func (x *TerminalSessionRecoveryResult) GetSessionId() string { + if x != nil { + return x.SessionId + } + return "" +} + +func (x *TerminalSessionRecoveryResult) GetStatus() TerminalSessionRecoveryResult_Status { + if x != nil { + return x.Status + } + return TerminalSessionRecoveryResult_STATUS_UNSPECIFIED +} + +type TerminalSessionRecoveryReport struct { + state protoimpl.MessageState `protogen:"open.v1"` + Results []*TerminalSessionRecoveryResult `protobuf:"bytes,1,rep,name=results,proto3" json:"results,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *TerminalSessionRecoveryReport) Reset() { + *x = TerminalSessionRecoveryReport{} + mi := &file_registry_v1_registry_proto_msgTypes[7] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *TerminalSessionRecoveryReport) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*TerminalSessionRecoveryReport) ProtoMessage() {} + +func (x *TerminalSessionRecoveryReport) ProtoReflect() protoreflect.Message { + mi := &file_registry_v1_registry_proto_msgTypes[7] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use TerminalSessionRecoveryReport.ProtoReflect.Descriptor instead. +func (*TerminalSessionRecoveryReport) Descriptor() ([]byte, []int) { + return file_registry_v1_registry_proto_rawDescGZIP(), []int{7} +} + +func (x *TerminalSessionRecoveryReport) GetResults() []*TerminalSessionRecoveryResult { + if x != nil { + return x.Results + } + return nil +} + +type TerminalSessionRecoveryAck struct { + state protoimpl.MessageState `protogen:"open.v1"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *TerminalSessionRecoveryAck) Reset() { + *x = TerminalSessionRecoveryAck{} + mi := &file_registry_v1_registry_proto_msgTypes[8] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *TerminalSessionRecoveryAck) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*TerminalSessionRecoveryAck) ProtoMessage() {} + +func (x *TerminalSessionRecoveryAck) ProtoReflect() protoreflect.Message { + mi := &file_registry_v1_registry_proto_msgTypes[8] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use TerminalSessionRecoveryAck.ProtoReflect.Descriptor instead. +func (*TerminalSessionRecoveryAck) Descriptor() ([]byte, []int) { + return file_registry_v1_registry_proto_rawDescGZIP(), []int{8} +} + type ConnectAck struct { - state protoimpl.MessageState `protogen:"open.v1"` - SessionId string `protobuf:"bytes,1,opt,name=session_id,json=sessionId,proto3" json:"session_id,omitempty"` - HeartbeatIntervalSec int32 `protobuf:"varint,3,opt,name=heartbeat_interval_sec,json=heartbeatIntervalSec,proto3" json:"heartbeat_interval_sec,omitempty"` - unknownFields protoimpl.UnknownFields - sizeCache protoimpl.SizeCache + state protoimpl.MessageState `protogen:"open.v1"` + SessionId string `protobuf:"bytes,1,opt,name=session_id,json=sessionId,proto3" json:"session_id,omitempty"` + HeartbeatIntervalSec int32 `protobuf:"varint,3,opt,name=heartbeat_interval_sec,json=heartbeatIntervalSec,proto3" json:"heartbeat_interval_sec,omitempty"` + TerminalSessionRecoveryCandidates []*TerminalSessionRecoveryCandidate `protobuf:"bytes,4,rep,name=terminal_session_recovery_candidates,json=terminalSessionRecoveryCandidates,proto3" json:"terminal_session_recovery_candidates,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache } func (x *ConnectAck) Reset() { *x = ConnectAck{} - mi := &file_registry_v1_registry_proto_msgTypes[5] + mi := &file_registry_v1_registry_proto_msgTypes[9] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -407,7 +660,7 @@ func (x *ConnectAck) String() string { func (*ConnectAck) ProtoMessage() {} func (x *ConnectAck) ProtoReflect() protoreflect.Message { - mi := &file_registry_v1_registry_proto_msgTypes[5] + mi := &file_registry_v1_registry_proto_msgTypes[9] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -420,7 +673,7 @@ func (x *ConnectAck) ProtoReflect() protoreflect.Message { // Deprecated: Use ConnectAck.ProtoReflect.Descriptor instead. func (*ConnectAck) Descriptor() ([]byte, []int) { - return file_registry_v1_registry_proto_rawDescGZIP(), []int{5} + return file_registry_v1_registry_proto_rawDescGZIP(), []int{9} } func (x *ConnectAck) GetSessionId() string { @@ -437,6 +690,13 @@ func (x *ConnectAck) GetHeartbeatIntervalSec() int32 { return 0 } +func (x *ConnectAck) GetTerminalSessionRecoveryCandidates() []*TerminalSessionRecoveryCandidate { + if x != nil { + return x.TerminalSessionRecoveryCandidates + } + return nil +} + type HeartbeatAck struct { state protoimpl.MessageState `protogen:"open.v1"` HeartbeatIntervalSec int32 `protobuf:"varint,2,opt,name=heartbeat_interval_sec,json=heartbeatIntervalSec,proto3" json:"heartbeat_interval_sec,omitempty"` @@ -446,7 +706,7 @@ type HeartbeatAck struct { func (x *HeartbeatAck) Reset() { *x = HeartbeatAck{} - mi := &file_registry_v1_registry_proto_msgTypes[6] + mi := &file_registry_v1_registry_proto_msgTypes[10] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -458,7 +718,7 @@ func (x *HeartbeatAck) String() string { func (*HeartbeatAck) ProtoMessage() {} func (x *HeartbeatAck) ProtoReflect() protoreflect.Message { - mi := &file_registry_v1_registry_proto_msgTypes[6] + mi := &file_registry_v1_registry_proto_msgTypes[10] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -471,7 +731,7 @@ func (x *HeartbeatAck) ProtoReflect() protoreflect.Message { // Deprecated: Use HeartbeatAck.ProtoReflect.Descriptor instead. func (*HeartbeatAck) Descriptor() ([]byte, []int) { - return file_registry_v1_registry_proto_rawDescGZIP(), []int{6} + return file_registry_v1_registry_proto_rawDescGZIP(), []int{10} } func (x *HeartbeatAck) GetHeartbeatIntervalSec() int32 { @@ -493,7 +753,7 @@ type CommandDispatch struct { func (x *CommandDispatch) Reset() { *x = CommandDispatch{} - mi := &file_registry_v1_registry_proto_msgTypes[7] + mi := &file_registry_v1_registry_proto_msgTypes[11] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -505,7 +765,7 @@ func (x *CommandDispatch) String() string { func (*CommandDispatch) ProtoMessage() {} func (x *CommandDispatch) ProtoReflect() protoreflect.Message { - mi := &file_registry_v1_registry_proto_msgTypes[7] + mi := &file_registry_v1_registry_proto_msgTypes[11] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -518,7 +778,7 @@ func (x *CommandDispatch) ProtoReflect() protoreflect.Message { // Deprecated: Use CommandDispatch.ProtoReflect.Descriptor instead. func (*CommandDispatch) Descriptor() ([]byte, []int) { - return file_registry_v1_registry_proto_rawDescGZIP(), []int{7} + return file_registry_v1_registry_proto_rawDescGZIP(), []int{11} } func (x *CommandDispatch) GetCommandId() string { @@ -559,7 +819,7 @@ type CommandError struct { func (x *CommandError) Reset() { *x = CommandError{} - mi := &file_registry_v1_registry_proto_msgTypes[8] + mi := &file_registry_v1_registry_proto_msgTypes[12] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -571,7 +831,7 @@ func (x *CommandError) String() string { func (*CommandError) ProtoMessage() {} func (x *CommandError) ProtoReflect() protoreflect.Message { - mi := &file_registry_v1_registry_proto_msgTypes[8] + mi := &file_registry_v1_registry_proto_msgTypes[12] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -584,7 +844,7 @@ func (x *CommandError) ProtoReflect() protoreflect.Message { // Deprecated: Use CommandError.ProtoReflect.Descriptor instead. func (*CommandError) Descriptor() ([]byte, []int) { - return file_registry_v1_registry_proto_rawDescGZIP(), []int{8} + return file_registry_v1_registry_proto_rawDescGZIP(), []int{12} } func (x *CommandError) GetCode() string { @@ -613,7 +873,7 @@ type CommandResult struct { func (x *CommandResult) Reset() { *x = CommandResult{} - mi := &file_registry_v1_registry_proto_msgTypes[9] + mi := &file_registry_v1_registry_proto_msgTypes[13] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -625,7 +885,7 @@ func (x *CommandResult) String() string { func (*CommandResult) ProtoMessage() {} func (x *CommandResult) ProtoReflect() protoreflect.Message { - mi := &file_registry_v1_registry_proto_msgTypes[9] + mi := &file_registry_v1_registry_proto_msgTypes[13] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -638,7 +898,7 @@ func (x *CommandResult) ProtoReflect() protoreflect.Message { // Deprecated: Use CommandResult.ProtoReflect.Descriptor instead. func (*CommandResult) Descriptor() ([]byte, []int) { - return file_registry_v1_registry_proto_rawDescGZIP(), []int{9} + return file_registry_v1_registry_proto_rawDescGZIP(), []int{13} } func (x *CommandResult) GetCommandId() string { @@ -676,6 +936,7 @@ type ConnectResponse struct { // *ConnectResponse_ConnectAck // *ConnectResponse_HeartbeatAck // *ConnectResponse_CommandDispatch + // *ConnectResponse_TerminalSessionRecoveryAck Payload isConnectResponse_Payload `protobuf_oneof:"payload"` unknownFields protoimpl.UnknownFields sizeCache protoimpl.SizeCache @@ -683,7 +944,7 @@ type ConnectResponse struct { func (x *ConnectResponse) Reset() { *x = ConnectResponse{} - mi := &file_registry_v1_registry_proto_msgTypes[10] + mi := &file_registry_v1_registry_proto_msgTypes[14] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -695,7 +956,7 @@ func (x *ConnectResponse) String() string { func (*ConnectResponse) ProtoMessage() {} func (x *ConnectResponse) ProtoReflect() protoreflect.Message { - mi := &file_registry_v1_registry_proto_msgTypes[10] + mi := &file_registry_v1_registry_proto_msgTypes[14] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -708,7 +969,7 @@ func (x *ConnectResponse) ProtoReflect() protoreflect.Message { // Deprecated: Use ConnectResponse.ProtoReflect.Descriptor instead. func (*ConnectResponse) Descriptor() ([]byte, []int) { - return file_registry_v1_registry_proto_rawDescGZIP(), []int{10} + return file_registry_v1_registry_proto_rawDescGZIP(), []int{14} } func (x *ConnectResponse) GetPayload() isConnectResponse_Payload { @@ -745,6 +1006,15 @@ func (x *ConnectResponse) GetCommandDispatch() *CommandDispatch { return nil } +func (x *ConnectResponse) GetTerminalSessionRecoveryAck() *TerminalSessionRecoveryAck { + if x != nil { + if x, ok := x.Payload.(*ConnectResponse_TerminalSessionRecoveryAck); ok { + return x.TerminalSessionRecoveryAck + } + } + return nil +} + type isConnectResponse_Payload interface { isConnectResponse_Payload() } @@ -761,12 +1031,18 @@ type ConnectResponse_CommandDispatch struct { CommandDispatch *CommandDispatch `protobuf:"bytes,4,opt,name=command_dispatch,json=commandDispatch,proto3,oneof"` } +type ConnectResponse_TerminalSessionRecoveryAck struct { + TerminalSessionRecoveryAck *TerminalSessionRecoveryAck `protobuf:"bytes,5,opt,name=terminal_session_recovery_ack,json=terminalSessionRecoveryAck,proto3,oneof"` +} + func (*ConnectResponse_ConnectAck) isConnectResponse_Payload() {} func (*ConnectResponse_HeartbeatAck) isConnectResponse_Payload() {} func (*ConnectResponse_CommandDispatch) isConnectResponse_Payload() {} +func (*ConnectResponse_TerminalSessionRecoveryAck) isConnectResponse_Payload() {} + var File_registry_v1_registry_proto protoreflect.FileDescriptor var file_registry_v1_registry_proto_rawDesc = string([]byte{ @@ -824,7 +1100,7 @@ var file_registry_v1_registry_proto_rawDesc = string([]byte{ 0x73, 0x73, 0x69, 0x6f, 0x6e, 0x49, 0x64, 0x12, 0x30, 0x0a, 0x14, 0x61, 0x63, 0x74, 0x69, 0x76, 0x65, 0x5f, 0x73, 0x65, 0x73, 0x73, 0x69, 0x6f, 0x6e, 0x5f, 0x63, 0x6f, 0x75, 0x6e, 0x74, 0x18, 0x03, 0x20, 0x01, 0x28, 0x05, 0x52, 0x12, 0x61, 0x63, 0x74, 0x69, 0x76, 0x65, 0x53, 0x65, 0x73, - 0x73, 0x69, 0x6f, 0x6e, 0x43, 0x6f, 0x75, 0x6e, 0x74, 0x22, 0xee, 0x01, 0x0a, 0x0e, 0x43, 0x6f, + 0x73, 0x69, 0x6f, 0x6e, 0x43, 0x6f, 0x75, 0x6e, 0x74, 0x22, 0xef, 0x02, 0x0a, 0x0e, 0x43, 0x6f, 0x6e, 0x6e, 0x65, 0x63, 0x74, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x3b, 0x0a, 0x05, 0x68, 0x65, 0x6c, 0x6c, 0x6f, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x23, 0x2e, 0x6f, 0x6e, 0x6c, 0x79, 0x62, 0x6f, 0x78, 0x65, 0x73, 0x2e, 0x72, 0x65, 0x67, 0x69, 0x73, 0x74, 0x72, 0x79, @@ -838,73 +1114,127 @@ var file_registry_v1_registry_proto_rawDesc = string([]byte{ 0x6c, 0x74, 0x18, 0x03, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x24, 0x2e, 0x6f, 0x6e, 0x6c, 0x79, 0x62, 0x6f, 0x78, 0x65, 0x73, 0x2e, 0x72, 0x65, 0x67, 0x69, 0x73, 0x74, 0x72, 0x79, 0x2e, 0x76, 0x31, 0x2e, 0x43, 0x6f, 0x6d, 0x6d, 0x61, 0x6e, 0x64, 0x52, 0x65, 0x73, 0x75, 0x6c, 0x74, 0x48, 0x00, - 0x52, 0x0d, 0x63, 0x6f, 0x6d, 0x6d, 0x61, 0x6e, 0x64, 0x52, 0x65, 0x73, 0x75, 0x6c, 0x74, 0x42, - 0x09, 0x0a, 0x07, 0x70, 0x61, 0x79, 0x6c, 0x6f, 0x61, 0x64, 0x22, 0x61, 0x0a, 0x0a, 0x43, 0x6f, - 0x6e, 0x6e, 0x65, 0x63, 0x74, 0x41, 0x63, 0x6b, 0x12, 0x1d, 0x0a, 0x0a, 0x73, 0x65, 0x73, 0x73, - 0x69, 0x6f, 0x6e, 0x5f, 0x69, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x09, 0x73, 0x65, - 0x73, 0x73, 0x69, 0x6f, 0x6e, 0x49, 0x64, 0x12, 0x34, 0x0a, 0x16, 0x68, 0x65, 0x61, 0x72, 0x74, - 0x62, 0x65, 0x61, 0x74, 0x5f, 0x69, 0x6e, 0x74, 0x65, 0x72, 0x76, 0x61, 0x6c, 0x5f, 0x73, 0x65, - 0x63, 0x18, 0x03, 0x20, 0x01, 0x28, 0x05, 0x52, 0x14, 0x68, 0x65, 0x61, 0x72, 0x74, 0x62, 0x65, - 0x61, 0x74, 0x49, 0x6e, 0x74, 0x65, 0x72, 0x76, 0x61, 0x6c, 0x53, 0x65, 0x63, 0x22, 0x44, 0x0a, - 0x0c, 0x48, 0x65, 0x61, 0x72, 0x74, 0x62, 0x65, 0x61, 0x74, 0x41, 0x63, 0x6b, 0x12, 0x34, 0x0a, - 0x16, 0x68, 0x65, 0x61, 0x72, 0x74, 0x62, 0x65, 0x61, 0x74, 0x5f, 0x69, 0x6e, 0x74, 0x65, 0x72, - 0x76, 0x61, 0x6c, 0x5f, 0x73, 0x65, 0x63, 0x18, 0x02, 0x20, 0x01, 0x28, 0x05, 0x52, 0x14, 0x68, - 0x65, 0x61, 0x72, 0x74, 0x62, 0x65, 0x61, 0x74, 0x49, 0x6e, 0x74, 0x65, 0x72, 0x76, 0x61, 0x6c, - 0x53, 0x65, 0x63, 0x22, 0x9d, 0x01, 0x0a, 0x0f, 0x43, 0x6f, 0x6d, 0x6d, 0x61, 0x6e, 0x64, 0x44, - 0x69, 0x73, 0x70, 0x61, 0x74, 0x63, 0x68, 0x12, 0x1d, 0x0a, 0x0a, 0x63, 0x6f, 0x6d, 0x6d, 0x61, - 0x6e, 0x64, 0x5f, 0x69, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x09, 0x63, 0x6f, 0x6d, - 0x6d, 0x61, 0x6e, 0x64, 0x49, 0x64, 0x12, 0x1e, 0x0a, 0x0a, 0x63, 0x61, 0x70, 0x61, 0x62, 0x69, - 0x6c, 0x69, 0x74, 0x79, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0a, 0x63, 0x61, 0x70, 0x61, - 0x62, 0x69, 0x6c, 0x69, 0x74, 0x79, 0x12, 0x21, 0x0a, 0x0c, 0x70, 0x61, 0x79, 0x6c, 0x6f, 0x61, - 0x64, 0x5f, 0x6a, 0x73, 0x6f, 0x6e, 0x18, 0x04, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x0b, 0x70, 0x61, - 0x79, 0x6c, 0x6f, 0x61, 0x64, 0x4a, 0x73, 0x6f, 0x6e, 0x12, 0x28, 0x0a, 0x10, 0x64, 0x65, 0x61, - 0x64, 0x6c, 0x69, 0x6e, 0x65, 0x5f, 0x75, 0x6e, 0x69, 0x78, 0x5f, 0x6d, 0x73, 0x18, 0x05, 0x20, - 0x01, 0x28, 0x03, 0x52, 0x0e, 0x64, 0x65, 0x61, 0x64, 0x6c, 0x69, 0x6e, 0x65, 0x55, 0x6e, 0x69, - 0x78, 0x4d, 0x73, 0x22, 0x3c, 0x0a, 0x0c, 0x43, 0x6f, 0x6d, 0x6d, 0x61, 0x6e, 0x64, 0x45, 0x72, - 0x72, 0x6f, 0x72, 0x12, 0x12, 0x0a, 0x04, 0x63, 0x6f, 0x64, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, - 0x09, 0x52, 0x04, 0x63, 0x6f, 0x64, 0x65, 0x12, 0x18, 0x0a, 0x07, 0x6d, 0x65, 0x73, 0x73, 0x61, - 0x67, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x07, 0x6d, 0x65, 0x73, 0x73, 0x61, 0x67, - 0x65, 0x22, 0xb8, 0x01, 0x0a, 0x0d, 0x43, 0x6f, 0x6d, 0x6d, 0x61, 0x6e, 0x64, 0x52, 0x65, 0x73, - 0x75, 0x6c, 0x74, 0x12, 0x1d, 0x0a, 0x0a, 0x63, 0x6f, 0x6d, 0x6d, 0x61, 0x6e, 0x64, 0x5f, 0x69, - 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x09, 0x63, 0x6f, 0x6d, 0x6d, 0x61, 0x6e, 0x64, - 0x49, 0x64, 0x12, 0x39, 0x0a, 0x05, 0x65, 0x72, 0x72, 0x6f, 0x72, 0x18, 0x03, 0x20, 0x01, 0x28, - 0x0b, 0x32, 0x23, 0x2e, 0x6f, 0x6e, 0x6c, 0x79, 0x62, 0x6f, 0x78, 0x65, 0x73, 0x2e, 0x72, 0x65, - 0x67, 0x69, 0x73, 0x74, 0x72, 0x79, 0x2e, 0x76, 0x31, 0x2e, 0x43, 0x6f, 0x6d, 0x6d, 0x61, 0x6e, - 0x64, 0x45, 0x72, 0x72, 0x6f, 0x72, 0x52, 0x05, 0x65, 0x72, 0x72, 0x6f, 0x72, 0x12, 0x21, 0x0a, - 0x0c, 0x70, 0x61, 0x79, 0x6c, 0x6f, 0x61, 0x64, 0x5f, 0x6a, 0x73, 0x6f, 0x6e, 0x18, 0x04, 0x20, - 0x01, 0x28, 0x0c, 0x52, 0x0b, 0x70, 0x61, 0x79, 0x6c, 0x6f, 0x61, 0x64, 0x4a, 0x73, 0x6f, 0x6e, - 0x12, 0x2a, 0x0a, 0x11, 0x63, 0x6f, 0x6d, 0x70, 0x6c, 0x65, 0x74, 0x65, 0x64, 0x5f, 0x75, 0x6e, - 0x69, 0x78, 0x5f, 0x6d, 0x73, 0x18, 0x05, 0x20, 0x01, 0x28, 0x03, 0x52, 0x0f, 0x63, 0x6f, 0x6d, - 0x70, 0x6c, 0x65, 0x74, 0x65, 0x64, 0x55, 0x6e, 0x69, 0x78, 0x4d, 0x73, 0x22, 0x83, 0x02, 0x0a, - 0x0f, 0x43, 0x6f, 0x6e, 0x6e, 0x65, 0x63, 0x74, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, - 0x12, 0x44, 0x0a, 0x0b, 0x63, 0x6f, 0x6e, 0x6e, 0x65, 0x63, 0x74, 0x5f, 0x61, 0x63, 0x6b, 0x18, - 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x21, 0x2e, 0x6f, 0x6e, 0x6c, 0x79, 0x62, 0x6f, 0x78, 0x65, - 0x73, 0x2e, 0x72, 0x65, 0x67, 0x69, 0x73, 0x74, 0x72, 0x79, 0x2e, 0x76, 0x31, 0x2e, 0x43, 0x6f, - 0x6e, 0x6e, 0x65, 0x63, 0x74, 0x41, 0x63, 0x6b, 0x48, 0x00, 0x52, 0x0a, 0x63, 0x6f, 0x6e, 0x6e, - 0x65, 0x63, 0x74, 0x41, 0x63, 0x6b, 0x12, 0x4a, 0x0a, 0x0d, 0x68, 0x65, 0x61, 0x72, 0x74, 0x62, - 0x65, 0x61, 0x74, 0x5f, 0x61, 0x63, 0x6b, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x23, 0x2e, - 0x6f, 0x6e, 0x6c, 0x79, 0x62, 0x6f, 0x78, 0x65, 0x73, 0x2e, 0x72, 0x65, 0x67, 0x69, 0x73, 0x74, - 0x72, 0x79, 0x2e, 0x76, 0x31, 0x2e, 0x48, 0x65, 0x61, 0x72, 0x74, 0x62, 0x65, 0x61, 0x74, 0x41, - 0x63, 0x6b, 0x48, 0x00, 0x52, 0x0c, 0x68, 0x65, 0x61, 0x72, 0x74, 0x62, 0x65, 0x61, 0x74, 0x41, - 0x63, 0x6b, 0x12, 0x53, 0x0a, 0x10, 0x63, 0x6f, 0x6d, 0x6d, 0x61, 0x6e, 0x64, 0x5f, 0x64, 0x69, - 0x73, 0x70, 0x61, 0x74, 0x63, 0x68, 0x18, 0x04, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x26, 0x2e, 0x6f, - 0x6e, 0x6c, 0x79, 0x62, 0x6f, 0x78, 0x65, 0x73, 0x2e, 0x72, 0x65, 0x67, 0x69, 0x73, 0x74, 0x72, - 0x79, 0x2e, 0x76, 0x31, 0x2e, 0x43, 0x6f, 0x6d, 0x6d, 0x61, 0x6e, 0x64, 0x44, 0x69, 0x73, 0x70, - 0x61, 0x74, 0x63, 0x68, 0x48, 0x00, 0x52, 0x0f, 0x63, 0x6f, 0x6d, 0x6d, 0x61, 0x6e, 0x64, 0x44, - 0x69, 0x73, 0x70, 0x61, 0x74, 0x63, 0x68, 0x42, 0x09, 0x0a, 0x07, 0x70, 0x61, 0x79, 0x6c, 0x6f, - 0x61, 0x64, 0x32, 0x75, 0x0a, 0x15, 0x57, 0x6f, 0x72, 0x6b, 0x65, 0x72, 0x52, 0x65, 0x67, 0x69, - 0x73, 0x74, 0x72, 0x79, 0x53, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x12, 0x5c, 0x0a, 0x07, 0x43, - 0x6f, 0x6e, 0x6e, 0x65, 0x63, 0x74, 0x12, 0x25, 0x2e, 0x6f, 0x6e, 0x6c, 0x79, 0x62, 0x6f, 0x78, - 0x65, 0x73, 0x2e, 0x72, 0x65, 0x67, 0x69, 0x73, 0x74, 0x72, 0x79, 0x2e, 0x76, 0x31, 0x2e, 0x43, - 0x6f, 0x6e, 0x6e, 0x65, 0x63, 0x74, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x26, 0x2e, - 0x6f, 0x6e, 0x6c, 0x79, 0x62, 0x6f, 0x78, 0x65, 0x73, 0x2e, 0x72, 0x65, 0x67, 0x69, 0x73, 0x74, - 0x72, 0x79, 0x2e, 0x76, 0x31, 0x2e, 0x43, 0x6f, 0x6e, 0x6e, 0x65, 0x63, 0x74, 0x52, 0x65, 0x73, - 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x28, 0x01, 0x30, 0x01, 0x42, 0x42, 0x5a, 0x40, 0x67, 0x69, 0x74, - 0x68, 0x75, 0x62, 0x2e, 0x63, 0x6f, 0x6d, 0x2f, 0x6f, 0x6e, 0x6c, 0x79, 0x62, 0x6f, 0x78, 0x65, - 0x73, 0x2f, 0x6f, 0x6e, 0x6c, 0x79, 0x62, 0x6f, 0x78, 0x65, 0x73, 0x2f, 0x61, 0x70, 0x69, 0x2f, - 0x67, 0x65, 0x6e, 0x2f, 0x67, 0x6f, 0x2f, 0x72, 0x65, 0x67, 0x69, 0x73, 0x74, 0x72, 0x79, 0x2f, - 0x76, 0x31, 0x3b, 0x72, 0x65, 0x67, 0x69, 0x73, 0x74, 0x72, 0x79, 0x76, 0x31, 0x62, 0x06, 0x70, - 0x72, 0x6f, 0x74, 0x6f, 0x33, + 0x52, 0x0d, 0x63, 0x6f, 0x6d, 0x6d, 0x61, 0x6e, 0x64, 0x52, 0x65, 0x73, 0x75, 0x6c, 0x74, 0x12, + 0x7f, 0x0a, 0x20, 0x74, 0x65, 0x72, 0x6d, 0x69, 0x6e, 0x61, 0x6c, 0x5f, 0x73, 0x65, 0x73, 0x73, + 0x69, 0x6f, 0x6e, 0x5f, 0x72, 0x65, 0x63, 0x6f, 0x76, 0x65, 0x72, 0x79, 0x5f, 0x72, 0x65, 0x70, + 0x6f, 0x72, 0x74, 0x18, 0x04, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x34, 0x2e, 0x6f, 0x6e, 0x6c, 0x79, + 0x62, 0x6f, 0x78, 0x65, 0x73, 0x2e, 0x72, 0x65, 0x67, 0x69, 0x73, 0x74, 0x72, 0x79, 0x2e, 0x76, + 0x31, 0x2e, 0x54, 0x65, 0x72, 0x6d, 0x69, 0x6e, 0x61, 0x6c, 0x53, 0x65, 0x73, 0x73, 0x69, 0x6f, + 0x6e, 0x52, 0x65, 0x63, 0x6f, 0x76, 0x65, 0x72, 0x79, 0x52, 0x65, 0x70, 0x6f, 0x72, 0x74, 0x48, + 0x00, 0x52, 0x1d, 0x74, 0x65, 0x72, 0x6d, 0x69, 0x6e, 0x61, 0x6c, 0x53, 0x65, 0x73, 0x73, 0x69, + 0x6f, 0x6e, 0x52, 0x65, 0x63, 0x6f, 0x76, 0x65, 0x72, 0x79, 0x52, 0x65, 0x70, 0x6f, 0x72, 0x74, + 0x42, 0x09, 0x0a, 0x07, 0x70, 0x61, 0x79, 0x6c, 0x6f, 0x61, 0x64, 0x22, 0x74, 0x0a, 0x20, 0x54, + 0x65, 0x72, 0x6d, 0x69, 0x6e, 0x61, 0x6c, 0x53, 0x65, 0x73, 0x73, 0x69, 0x6f, 0x6e, 0x52, 0x65, + 0x63, 0x6f, 0x76, 0x65, 0x72, 0x79, 0x43, 0x61, 0x6e, 0x64, 0x69, 0x64, 0x61, 0x74, 0x65, 0x12, + 0x1d, 0x0a, 0x0a, 0x73, 0x65, 0x73, 0x73, 0x69, 0x6f, 0x6e, 0x5f, 0x69, 0x64, 0x18, 0x01, 0x20, + 0x01, 0x28, 0x09, 0x52, 0x09, 0x73, 0x65, 0x73, 0x73, 0x69, 0x6f, 0x6e, 0x49, 0x64, 0x12, 0x31, + 0x0a, 0x15, 0x6c, 0x65, 0x61, 0x73, 0x65, 0x5f, 0x65, 0x78, 0x70, 0x69, 0x72, 0x65, 0x73, 0x5f, + 0x75, 0x6e, 0x69, 0x78, 0x5f, 0x6d, 0x73, 0x18, 0x02, 0x20, 0x01, 0x28, 0x03, 0x52, 0x12, 0x6c, + 0x65, 0x61, 0x73, 0x65, 0x45, 0x78, 0x70, 0x69, 0x72, 0x65, 0x73, 0x55, 0x6e, 0x69, 0x78, 0x4d, + 0x73, 0x22, 0xde, 0x01, 0x0a, 0x1d, 0x54, 0x65, 0x72, 0x6d, 0x69, 0x6e, 0x61, 0x6c, 0x53, 0x65, + 0x73, 0x73, 0x69, 0x6f, 0x6e, 0x52, 0x65, 0x63, 0x6f, 0x76, 0x65, 0x72, 0x79, 0x52, 0x65, 0x73, + 0x75, 0x6c, 0x74, 0x12, 0x1d, 0x0a, 0x0a, 0x73, 0x65, 0x73, 0x73, 0x69, 0x6f, 0x6e, 0x5f, 0x69, + 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x09, 0x73, 0x65, 0x73, 0x73, 0x69, 0x6f, 0x6e, + 0x49, 0x64, 0x12, 0x53, 0x0a, 0x06, 0x73, 0x74, 0x61, 0x74, 0x75, 0x73, 0x18, 0x02, 0x20, 0x01, + 0x28, 0x0e, 0x32, 0x3b, 0x2e, 0x6f, 0x6e, 0x6c, 0x79, 0x62, 0x6f, 0x78, 0x65, 0x73, 0x2e, 0x72, + 0x65, 0x67, 0x69, 0x73, 0x74, 0x72, 0x79, 0x2e, 0x76, 0x31, 0x2e, 0x54, 0x65, 0x72, 0x6d, 0x69, + 0x6e, 0x61, 0x6c, 0x53, 0x65, 0x73, 0x73, 0x69, 0x6f, 0x6e, 0x52, 0x65, 0x63, 0x6f, 0x76, 0x65, + 0x72, 0x79, 0x52, 0x65, 0x73, 0x75, 0x6c, 0x74, 0x2e, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x52, + 0x06, 0x73, 0x74, 0x61, 0x74, 0x75, 0x73, 0x22, 0x49, 0x0a, 0x06, 0x53, 0x74, 0x61, 0x74, 0x75, + 0x73, 0x12, 0x16, 0x0a, 0x12, 0x53, 0x54, 0x41, 0x54, 0x55, 0x53, 0x5f, 0x55, 0x4e, 0x53, 0x50, + 0x45, 0x43, 0x49, 0x46, 0x49, 0x45, 0x44, 0x10, 0x00, 0x12, 0x0d, 0x0a, 0x09, 0x52, 0x45, 0x43, + 0x4f, 0x56, 0x45, 0x52, 0x45, 0x44, 0x10, 0x01, 0x12, 0x0b, 0x0a, 0x07, 0x4d, 0x49, 0x53, 0x53, + 0x49, 0x4e, 0x47, 0x10, 0x02, 0x12, 0x0b, 0x0a, 0x07, 0x49, 0x4e, 0x56, 0x41, 0x4c, 0x49, 0x44, + 0x10, 0x03, 0x22, 0x6f, 0x0a, 0x1d, 0x54, 0x65, 0x72, 0x6d, 0x69, 0x6e, 0x61, 0x6c, 0x53, 0x65, + 0x73, 0x73, 0x69, 0x6f, 0x6e, 0x52, 0x65, 0x63, 0x6f, 0x76, 0x65, 0x72, 0x79, 0x52, 0x65, 0x70, + 0x6f, 0x72, 0x74, 0x12, 0x4e, 0x0a, 0x07, 0x72, 0x65, 0x73, 0x75, 0x6c, 0x74, 0x73, 0x18, 0x01, + 0x20, 0x03, 0x28, 0x0b, 0x32, 0x34, 0x2e, 0x6f, 0x6e, 0x6c, 0x79, 0x62, 0x6f, 0x78, 0x65, 0x73, + 0x2e, 0x72, 0x65, 0x67, 0x69, 0x73, 0x74, 0x72, 0x79, 0x2e, 0x76, 0x31, 0x2e, 0x54, 0x65, 0x72, + 0x6d, 0x69, 0x6e, 0x61, 0x6c, 0x53, 0x65, 0x73, 0x73, 0x69, 0x6f, 0x6e, 0x52, 0x65, 0x63, 0x6f, + 0x76, 0x65, 0x72, 0x79, 0x52, 0x65, 0x73, 0x75, 0x6c, 0x74, 0x52, 0x07, 0x72, 0x65, 0x73, 0x75, + 0x6c, 0x74, 0x73, 0x22, 0x1c, 0x0a, 0x1a, 0x54, 0x65, 0x72, 0x6d, 0x69, 0x6e, 0x61, 0x6c, 0x53, + 0x65, 0x73, 0x73, 0x69, 0x6f, 0x6e, 0x52, 0x65, 0x63, 0x6f, 0x76, 0x65, 0x72, 0x79, 0x41, 0x63, + 0x6b, 0x22, 0xec, 0x01, 0x0a, 0x0a, 0x43, 0x6f, 0x6e, 0x6e, 0x65, 0x63, 0x74, 0x41, 0x63, 0x6b, + 0x12, 0x1d, 0x0a, 0x0a, 0x73, 0x65, 0x73, 0x73, 0x69, 0x6f, 0x6e, 0x5f, 0x69, 0x64, 0x18, 0x01, + 0x20, 0x01, 0x28, 0x09, 0x52, 0x09, 0x73, 0x65, 0x73, 0x73, 0x69, 0x6f, 0x6e, 0x49, 0x64, 0x12, + 0x34, 0x0a, 0x16, 0x68, 0x65, 0x61, 0x72, 0x74, 0x62, 0x65, 0x61, 0x74, 0x5f, 0x69, 0x6e, 0x74, + 0x65, 0x72, 0x76, 0x61, 0x6c, 0x5f, 0x73, 0x65, 0x63, 0x18, 0x03, 0x20, 0x01, 0x28, 0x05, 0x52, + 0x14, 0x68, 0x65, 0x61, 0x72, 0x74, 0x62, 0x65, 0x61, 0x74, 0x49, 0x6e, 0x74, 0x65, 0x72, 0x76, + 0x61, 0x6c, 0x53, 0x65, 0x63, 0x12, 0x88, 0x01, 0x0a, 0x24, 0x74, 0x65, 0x72, 0x6d, 0x69, 0x6e, + 0x61, 0x6c, 0x5f, 0x73, 0x65, 0x73, 0x73, 0x69, 0x6f, 0x6e, 0x5f, 0x72, 0x65, 0x63, 0x6f, 0x76, + 0x65, 0x72, 0x79, 0x5f, 0x63, 0x61, 0x6e, 0x64, 0x69, 0x64, 0x61, 0x74, 0x65, 0x73, 0x18, 0x04, + 0x20, 0x03, 0x28, 0x0b, 0x32, 0x37, 0x2e, 0x6f, 0x6e, 0x6c, 0x79, 0x62, 0x6f, 0x78, 0x65, 0x73, + 0x2e, 0x72, 0x65, 0x67, 0x69, 0x73, 0x74, 0x72, 0x79, 0x2e, 0x76, 0x31, 0x2e, 0x54, 0x65, 0x72, + 0x6d, 0x69, 0x6e, 0x61, 0x6c, 0x53, 0x65, 0x73, 0x73, 0x69, 0x6f, 0x6e, 0x52, 0x65, 0x63, 0x6f, + 0x76, 0x65, 0x72, 0x79, 0x43, 0x61, 0x6e, 0x64, 0x69, 0x64, 0x61, 0x74, 0x65, 0x52, 0x21, 0x74, + 0x65, 0x72, 0x6d, 0x69, 0x6e, 0x61, 0x6c, 0x53, 0x65, 0x73, 0x73, 0x69, 0x6f, 0x6e, 0x52, 0x65, + 0x63, 0x6f, 0x76, 0x65, 0x72, 0x79, 0x43, 0x61, 0x6e, 0x64, 0x69, 0x64, 0x61, 0x74, 0x65, 0x73, + 0x22, 0x44, 0x0a, 0x0c, 0x48, 0x65, 0x61, 0x72, 0x74, 0x62, 0x65, 0x61, 0x74, 0x41, 0x63, 0x6b, + 0x12, 0x34, 0x0a, 0x16, 0x68, 0x65, 0x61, 0x72, 0x74, 0x62, 0x65, 0x61, 0x74, 0x5f, 0x69, 0x6e, + 0x74, 0x65, 0x72, 0x76, 0x61, 0x6c, 0x5f, 0x73, 0x65, 0x63, 0x18, 0x02, 0x20, 0x01, 0x28, 0x05, + 0x52, 0x14, 0x68, 0x65, 0x61, 0x72, 0x74, 0x62, 0x65, 0x61, 0x74, 0x49, 0x6e, 0x74, 0x65, 0x72, + 0x76, 0x61, 0x6c, 0x53, 0x65, 0x63, 0x22, 0x9d, 0x01, 0x0a, 0x0f, 0x43, 0x6f, 0x6d, 0x6d, 0x61, + 0x6e, 0x64, 0x44, 0x69, 0x73, 0x70, 0x61, 0x74, 0x63, 0x68, 0x12, 0x1d, 0x0a, 0x0a, 0x63, 0x6f, + 0x6d, 0x6d, 0x61, 0x6e, 0x64, 0x5f, 0x69, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x09, + 0x63, 0x6f, 0x6d, 0x6d, 0x61, 0x6e, 0x64, 0x49, 0x64, 0x12, 0x1e, 0x0a, 0x0a, 0x63, 0x61, 0x70, + 0x61, 0x62, 0x69, 0x6c, 0x69, 0x74, 0x79, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0a, 0x63, + 0x61, 0x70, 0x61, 0x62, 0x69, 0x6c, 0x69, 0x74, 0x79, 0x12, 0x21, 0x0a, 0x0c, 0x70, 0x61, 0x79, + 0x6c, 0x6f, 0x61, 0x64, 0x5f, 0x6a, 0x73, 0x6f, 0x6e, 0x18, 0x04, 0x20, 0x01, 0x28, 0x0c, 0x52, + 0x0b, 0x70, 0x61, 0x79, 0x6c, 0x6f, 0x61, 0x64, 0x4a, 0x73, 0x6f, 0x6e, 0x12, 0x28, 0x0a, 0x10, + 0x64, 0x65, 0x61, 0x64, 0x6c, 0x69, 0x6e, 0x65, 0x5f, 0x75, 0x6e, 0x69, 0x78, 0x5f, 0x6d, 0x73, + 0x18, 0x05, 0x20, 0x01, 0x28, 0x03, 0x52, 0x0e, 0x64, 0x65, 0x61, 0x64, 0x6c, 0x69, 0x6e, 0x65, + 0x55, 0x6e, 0x69, 0x78, 0x4d, 0x73, 0x22, 0x3c, 0x0a, 0x0c, 0x43, 0x6f, 0x6d, 0x6d, 0x61, 0x6e, + 0x64, 0x45, 0x72, 0x72, 0x6f, 0x72, 0x12, 0x12, 0x0a, 0x04, 0x63, 0x6f, 0x64, 0x65, 0x18, 0x01, + 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x63, 0x6f, 0x64, 0x65, 0x12, 0x18, 0x0a, 0x07, 0x6d, 0x65, + 0x73, 0x73, 0x61, 0x67, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x07, 0x6d, 0x65, 0x73, + 0x73, 0x61, 0x67, 0x65, 0x22, 0xb8, 0x01, 0x0a, 0x0d, 0x43, 0x6f, 0x6d, 0x6d, 0x61, 0x6e, 0x64, + 0x52, 0x65, 0x73, 0x75, 0x6c, 0x74, 0x12, 0x1d, 0x0a, 0x0a, 0x63, 0x6f, 0x6d, 0x6d, 0x61, 0x6e, + 0x64, 0x5f, 0x69, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x09, 0x63, 0x6f, 0x6d, 0x6d, + 0x61, 0x6e, 0x64, 0x49, 0x64, 0x12, 0x39, 0x0a, 0x05, 0x65, 0x72, 0x72, 0x6f, 0x72, 0x18, 0x03, + 0x20, 0x01, 0x28, 0x0b, 0x32, 0x23, 0x2e, 0x6f, 0x6e, 0x6c, 0x79, 0x62, 0x6f, 0x78, 0x65, 0x73, + 0x2e, 0x72, 0x65, 0x67, 0x69, 0x73, 0x74, 0x72, 0x79, 0x2e, 0x76, 0x31, 0x2e, 0x43, 0x6f, 0x6d, + 0x6d, 0x61, 0x6e, 0x64, 0x45, 0x72, 0x72, 0x6f, 0x72, 0x52, 0x05, 0x65, 0x72, 0x72, 0x6f, 0x72, + 0x12, 0x21, 0x0a, 0x0c, 0x70, 0x61, 0x79, 0x6c, 0x6f, 0x61, 0x64, 0x5f, 0x6a, 0x73, 0x6f, 0x6e, + 0x18, 0x04, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x0b, 0x70, 0x61, 0x79, 0x6c, 0x6f, 0x61, 0x64, 0x4a, + 0x73, 0x6f, 0x6e, 0x12, 0x2a, 0x0a, 0x11, 0x63, 0x6f, 0x6d, 0x70, 0x6c, 0x65, 0x74, 0x65, 0x64, + 0x5f, 0x75, 0x6e, 0x69, 0x78, 0x5f, 0x6d, 0x73, 0x18, 0x05, 0x20, 0x01, 0x28, 0x03, 0x52, 0x0f, + 0x63, 0x6f, 0x6d, 0x70, 0x6c, 0x65, 0x74, 0x65, 0x64, 0x55, 0x6e, 0x69, 0x78, 0x4d, 0x73, 0x22, + 0xfb, 0x02, 0x0a, 0x0f, 0x43, 0x6f, 0x6e, 0x6e, 0x65, 0x63, 0x74, 0x52, 0x65, 0x73, 0x70, 0x6f, + 0x6e, 0x73, 0x65, 0x12, 0x44, 0x0a, 0x0b, 0x63, 0x6f, 0x6e, 0x6e, 0x65, 0x63, 0x74, 0x5f, 0x61, + 0x63, 0x6b, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x21, 0x2e, 0x6f, 0x6e, 0x6c, 0x79, 0x62, + 0x6f, 0x78, 0x65, 0x73, 0x2e, 0x72, 0x65, 0x67, 0x69, 0x73, 0x74, 0x72, 0x79, 0x2e, 0x76, 0x31, + 0x2e, 0x43, 0x6f, 0x6e, 0x6e, 0x65, 0x63, 0x74, 0x41, 0x63, 0x6b, 0x48, 0x00, 0x52, 0x0a, 0x63, + 0x6f, 0x6e, 0x6e, 0x65, 0x63, 0x74, 0x41, 0x63, 0x6b, 0x12, 0x4a, 0x0a, 0x0d, 0x68, 0x65, 0x61, + 0x72, 0x74, 0x62, 0x65, 0x61, 0x74, 0x5f, 0x61, 0x63, 0x6b, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, + 0x32, 0x23, 0x2e, 0x6f, 0x6e, 0x6c, 0x79, 0x62, 0x6f, 0x78, 0x65, 0x73, 0x2e, 0x72, 0x65, 0x67, + 0x69, 0x73, 0x74, 0x72, 0x79, 0x2e, 0x76, 0x31, 0x2e, 0x48, 0x65, 0x61, 0x72, 0x74, 0x62, 0x65, + 0x61, 0x74, 0x41, 0x63, 0x6b, 0x48, 0x00, 0x52, 0x0c, 0x68, 0x65, 0x61, 0x72, 0x74, 0x62, 0x65, + 0x61, 0x74, 0x41, 0x63, 0x6b, 0x12, 0x53, 0x0a, 0x10, 0x63, 0x6f, 0x6d, 0x6d, 0x61, 0x6e, 0x64, + 0x5f, 0x64, 0x69, 0x73, 0x70, 0x61, 0x74, 0x63, 0x68, 0x18, 0x04, 0x20, 0x01, 0x28, 0x0b, 0x32, + 0x26, 0x2e, 0x6f, 0x6e, 0x6c, 0x79, 0x62, 0x6f, 0x78, 0x65, 0x73, 0x2e, 0x72, 0x65, 0x67, 0x69, + 0x73, 0x74, 0x72, 0x79, 0x2e, 0x76, 0x31, 0x2e, 0x43, 0x6f, 0x6d, 0x6d, 0x61, 0x6e, 0x64, 0x44, + 0x69, 0x73, 0x70, 0x61, 0x74, 0x63, 0x68, 0x48, 0x00, 0x52, 0x0f, 0x63, 0x6f, 0x6d, 0x6d, 0x61, + 0x6e, 0x64, 0x44, 0x69, 0x73, 0x70, 0x61, 0x74, 0x63, 0x68, 0x12, 0x76, 0x0a, 0x1d, 0x74, 0x65, + 0x72, 0x6d, 0x69, 0x6e, 0x61, 0x6c, 0x5f, 0x73, 0x65, 0x73, 0x73, 0x69, 0x6f, 0x6e, 0x5f, 0x72, + 0x65, 0x63, 0x6f, 0x76, 0x65, 0x72, 0x79, 0x5f, 0x61, 0x63, 0x6b, 0x18, 0x05, 0x20, 0x01, 0x28, + 0x0b, 0x32, 0x31, 0x2e, 0x6f, 0x6e, 0x6c, 0x79, 0x62, 0x6f, 0x78, 0x65, 0x73, 0x2e, 0x72, 0x65, + 0x67, 0x69, 0x73, 0x74, 0x72, 0x79, 0x2e, 0x76, 0x31, 0x2e, 0x54, 0x65, 0x72, 0x6d, 0x69, 0x6e, + 0x61, 0x6c, 0x53, 0x65, 0x73, 0x73, 0x69, 0x6f, 0x6e, 0x52, 0x65, 0x63, 0x6f, 0x76, 0x65, 0x72, + 0x79, 0x41, 0x63, 0x6b, 0x48, 0x00, 0x52, 0x1a, 0x74, 0x65, 0x72, 0x6d, 0x69, 0x6e, 0x61, 0x6c, + 0x53, 0x65, 0x73, 0x73, 0x69, 0x6f, 0x6e, 0x52, 0x65, 0x63, 0x6f, 0x76, 0x65, 0x72, 0x79, 0x41, + 0x63, 0x6b, 0x42, 0x09, 0x0a, 0x07, 0x70, 0x61, 0x79, 0x6c, 0x6f, 0x61, 0x64, 0x32, 0x75, 0x0a, + 0x15, 0x57, 0x6f, 0x72, 0x6b, 0x65, 0x72, 0x52, 0x65, 0x67, 0x69, 0x73, 0x74, 0x72, 0x79, 0x53, + 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x12, 0x5c, 0x0a, 0x07, 0x43, 0x6f, 0x6e, 0x6e, 0x65, 0x63, + 0x74, 0x12, 0x25, 0x2e, 0x6f, 0x6e, 0x6c, 0x79, 0x62, 0x6f, 0x78, 0x65, 0x73, 0x2e, 0x72, 0x65, + 0x67, 0x69, 0x73, 0x74, 0x72, 0x79, 0x2e, 0x76, 0x31, 0x2e, 0x43, 0x6f, 0x6e, 0x6e, 0x65, 0x63, + 0x74, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x26, 0x2e, 0x6f, 0x6e, 0x6c, 0x79, 0x62, + 0x6f, 0x78, 0x65, 0x73, 0x2e, 0x72, 0x65, 0x67, 0x69, 0x73, 0x74, 0x72, 0x79, 0x2e, 0x76, 0x31, + 0x2e, 0x43, 0x6f, 0x6e, 0x6e, 0x65, 0x63, 0x74, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, + 0x28, 0x01, 0x30, 0x01, 0x42, 0x42, 0x5a, 0x40, 0x67, 0x69, 0x74, 0x68, 0x75, 0x62, 0x2e, 0x63, + 0x6f, 0x6d, 0x2f, 0x6f, 0x6e, 0x6c, 0x79, 0x62, 0x6f, 0x78, 0x65, 0x73, 0x2f, 0x6f, 0x6e, 0x6c, + 0x79, 0x62, 0x6f, 0x78, 0x65, 0x73, 0x2f, 0x61, 0x70, 0x69, 0x2f, 0x67, 0x65, 0x6e, 0x2f, 0x67, + 0x6f, 0x2f, 0x72, 0x65, 0x67, 0x69, 0x73, 0x74, 0x72, 0x79, 0x2f, 0x76, 0x31, 0x3b, 0x72, 0x65, + 0x67, 0x69, 0x73, 0x74, 0x72, 0x79, 0x76, 0x31, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33, }) var ( @@ -919,39 +1249,50 @@ func file_registry_v1_registry_proto_rawDescGZIP() []byte { return file_registry_v1_registry_proto_rawDescData } -var file_registry_v1_registry_proto_msgTypes = make([]protoimpl.MessageInfo, 12) +var file_registry_v1_registry_proto_enumTypes = make([]protoimpl.EnumInfo, 1) +var file_registry_v1_registry_proto_msgTypes = make([]protoimpl.MessageInfo, 16) var file_registry_v1_registry_proto_goTypes = []any{ - (*CapabilityDeclaration)(nil), // 0: onlyboxes.registry.v1.CapabilityDeclaration - (*TerminalSessionCapacity)(nil), // 1: onlyboxes.registry.v1.TerminalSessionCapacity - (*ConnectHello)(nil), // 2: onlyboxes.registry.v1.ConnectHello - (*HeartbeatFrame)(nil), // 3: onlyboxes.registry.v1.HeartbeatFrame - (*ConnectRequest)(nil), // 4: onlyboxes.registry.v1.ConnectRequest - (*ConnectAck)(nil), // 5: onlyboxes.registry.v1.ConnectAck - (*HeartbeatAck)(nil), // 6: onlyboxes.registry.v1.HeartbeatAck - (*CommandDispatch)(nil), // 7: onlyboxes.registry.v1.CommandDispatch - (*CommandError)(nil), // 8: onlyboxes.registry.v1.CommandError - (*CommandResult)(nil), // 9: onlyboxes.registry.v1.CommandResult - (*ConnectResponse)(nil), // 10: onlyboxes.registry.v1.ConnectResponse - nil, // 11: onlyboxes.registry.v1.ConnectHello.LabelsEntry + (TerminalSessionRecoveryResult_Status)(0), // 0: onlyboxes.registry.v1.TerminalSessionRecoveryResult.Status + (*CapabilityDeclaration)(nil), // 1: onlyboxes.registry.v1.CapabilityDeclaration + (*TerminalSessionCapacity)(nil), // 2: onlyboxes.registry.v1.TerminalSessionCapacity + (*ConnectHello)(nil), // 3: onlyboxes.registry.v1.ConnectHello + (*HeartbeatFrame)(nil), // 4: onlyboxes.registry.v1.HeartbeatFrame + (*ConnectRequest)(nil), // 5: onlyboxes.registry.v1.ConnectRequest + (*TerminalSessionRecoveryCandidate)(nil), // 6: onlyboxes.registry.v1.TerminalSessionRecoveryCandidate + (*TerminalSessionRecoveryResult)(nil), // 7: onlyboxes.registry.v1.TerminalSessionRecoveryResult + (*TerminalSessionRecoveryReport)(nil), // 8: onlyboxes.registry.v1.TerminalSessionRecoveryReport + (*TerminalSessionRecoveryAck)(nil), // 9: onlyboxes.registry.v1.TerminalSessionRecoveryAck + (*ConnectAck)(nil), // 10: onlyboxes.registry.v1.ConnectAck + (*HeartbeatAck)(nil), // 11: onlyboxes.registry.v1.HeartbeatAck + (*CommandDispatch)(nil), // 12: onlyboxes.registry.v1.CommandDispatch + (*CommandError)(nil), // 13: onlyboxes.registry.v1.CommandError + (*CommandResult)(nil), // 14: onlyboxes.registry.v1.CommandResult + (*ConnectResponse)(nil), // 15: onlyboxes.registry.v1.ConnectResponse + nil, // 16: onlyboxes.registry.v1.ConnectHello.LabelsEntry } var file_registry_v1_registry_proto_depIdxs = []int32{ - 11, // 0: onlyboxes.registry.v1.ConnectHello.labels:type_name -> onlyboxes.registry.v1.ConnectHello.LabelsEntry - 0, // 1: onlyboxes.registry.v1.ConnectHello.capabilities:type_name -> onlyboxes.registry.v1.CapabilityDeclaration - 1, // 2: onlyboxes.registry.v1.ConnectHello.terminal_session_capacity:type_name -> onlyboxes.registry.v1.TerminalSessionCapacity - 2, // 3: onlyboxes.registry.v1.ConnectRequest.hello:type_name -> onlyboxes.registry.v1.ConnectHello - 3, // 4: onlyboxes.registry.v1.ConnectRequest.heartbeat:type_name -> onlyboxes.registry.v1.HeartbeatFrame - 9, // 5: onlyboxes.registry.v1.ConnectRequest.command_result:type_name -> onlyboxes.registry.v1.CommandResult - 8, // 6: onlyboxes.registry.v1.CommandResult.error:type_name -> onlyboxes.registry.v1.CommandError - 5, // 7: onlyboxes.registry.v1.ConnectResponse.connect_ack:type_name -> onlyboxes.registry.v1.ConnectAck - 6, // 8: onlyboxes.registry.v1.ConnectResponse.heartbeat_ack:type_name -> onlyboxes.registry.v1.HeartbeatAck - 7, // 9: onlyboxes.registry.v1.ConnectResponse.command_dispatch:type_name -> onlyboxes.registry.v1.CommandDispatch - 4, // 10: onlyboxes.registry.v1.WorkerRegistryService.Connect:input_type -> onlyboxes.registry.v1.ConnectRequest - 10, // 11: onlyboxes.registry.v1.WorkerRegistryService.Connect:output_type -> onlyboxes.registry.v1.ConnectResponse - 11, // [11:12] is the sub-list for method output_type - 10, // [10:11] is the sub-list for method input_type - 10, // [10:10] is the sub-list for extension type_name - 10, // [10:10] is the sub-list for extension extendee - 0, // [0:10] is the sub-list for field type_name + 16, // 0: onlyboxes.registry.v1.ConnectHello.labels:type_name -> onlyboxes.registry.v1.ConnectHello.LabelsEntry + 1, // 1: onlyboxes.registry.v1.ConnectHello.capabilities:type_name -> onlyboxes.registry.v1.CapabilityDeclaration + 2, // 2: onlyboxes.registry.v1.ConnectHello.terminal_session_capacity:type_name -> onlyboxes.registry.v1.TerminalSessionCapacity + 3, // 3: onlyboxes.registry.v1.ConnectRequest.hello:type_name -> onlyboxes.registry.v1.ConnectHello + 4, // 4: onlyboxes.registry.v1.ConnectRequest.heartbeat:type_name -> onlyboxes.registry.v1.HeartbeatFrame + 14, // 5: onlyboxes.registry.v1.ConnectRequest.command_result:type_name -> onlyboxes.registry.v1.CommandResult + 8, // 6: onlyboxes.registry.v1.ConnectRequest.terminal_session_recovery_report:type_name -> onlyboxes.registry.v1.TerminalSessionRecoveryReport + 0, // 7: onlyboxes.registry.v1.TerminalSessionRecoveryResult.status:type_name -> onlyboxes.registry.v1.TerminalSessionRecoveryResult.Status + 7, // 8: onlyboxes.registry.v1.TerminalSessionRecoveryReport.results:type_name -> onlyboxes.registry.v1.TerminalSessionRecoveryResult + 6, // 9: onlyboxes.registry.v1.ConnectAck.terminal_session_recovery_candidates:type_name -> onlyboxes.registry.v1.TerminalSessionRecoveryCandidate + 13, // 10: onlyboxes.registry.v1.CommandResult.error:type_name -> onlyboxes.registry.v1.CommandError + 10, // 11: onlyboxes.registry.v1.ConnectResponse.connect_ack:type_name -> onlyboxes.registry.v1.ConnectAck + 11, // 12: onlyboxes.registry.v1.ConnectResponse.heartbeat_ack:type_name -> onlyboxes.registry.v1.HeartbeatAck + 12, // 13: onlyboxes.registry.v1.ConnectResponse.command_dispatch:type_name -> onlyboxes.registry.v1.CommandDispatch + 9, // 14: onlyboxes.registry.v1.ConnectResponse.terminal_session_recovery_ack:type_name -> onlyboxes.registry.v1.TerminalSessionRecoveryAck + 5, // 15: onlyboxes.registry.v1.WorkerRegistryService.Connect:input_type -> onlyboxes.registry.v1.ConnectRequest + 15, // 16: onlyboxes.registry.v1.WorkerRegistryService.Connect:output_type -> onlyboxes.registry.v1.ConnectResponse + 16, // [16:17] is the sub-list for method output_type + 15, // [15:16] is the sub-list for method input_type + 15, // [15:15] is the sub-list for extension type_name + 15, // [15:15] is the sub-list for extension extendee + 0, // [0:15] is the sub-list for field type_name } func init() { file_registry_v1_registry_proto_init() } @@ -963,24 +1304,27 @@ func file_registry_v1_registry_proto_init() { (*ConnectRequest_Hello)(nil), (*ConnectRequest_Heartbeat)(nil), (*ConnectRequest_CommandResult)(nil), + (*ConnectRequest_TerminalSessionRecoveryReport)(nil), } - file_registry_v1_registry_proto_msgTypes[10].OneofWrappers = []any{ + file_registry_v1_registry_proto_msgTypes[14].OneofWrappers = []any{ (*ConnectResponse_ConnectAck)(nil), (*ConnectResponse_HeartbeatAck)(nil), (*ConnectResponse_CommandDispatch)(nil), + (*ConnectResponse_TerminalSessionRecoveryAck)(nil), } type x struct{} out := protoimpl.TypeBuilder{ File: protoimpl.DescBuilder{ GoPackagePath: reflect.TypeOf(x{}).PkgPath(), RawDescriptor: unsafe.Slice(unsafe.StringData(file_registry_v1_registry_proto_rawDesc), len(file_registry_v1_registry_proto_rawDesc)), - NumEnums: 0, - NumMessages: 12, + NumEnums: 1, + NumMessages: 16, NumExtensions: 0, NumServices: 1, }, GoTypes: file_registry_v1_registry_proto_goTypes, DependencyIndexes: file_registry_v1_registry_proto_depIdxs, + EnumInfos: file_registry_v1_registry_proto_enumTypes, MessageInfos: file_registry_v1_registry_proto_msgTypes, }.Build() File_registry_v1_registry_proto = out.File diff --git a/api/proto/registry/v1/registry.proto b/api/proto/registry/v1/registry.proto index 6c22682..ec4bf56 100644 --- a/api/proto/registry/v1/registry.proto +++ b/api/proto/registry/v1/registry.proto @@ -38,12 +38,37 @@ message ConnectRequest { ConnectHello hello = 1; HeartbeatFrame heartbeat = 2; CommandResult command_result = 3; + TerminalSessionRecoveryReport terminal_session_recovery_report = 4; } } +message TerminalSessionRecoveryCandidate { + string session_id = 1; + int64 lease_expires_unix_ms = 2; +} + +message TerminalSessionRecoveryResult { + enum Status { + STATUS_UNSPECIFIED = 0; + RECOVERED = 1; + MISSING = 2; + INVALID = 3; + } + + string session_id = 1; + Status status = 2; +} + +message TerminalSessionRecoveryReport { + repeated TerminalSessionRecoveryResult results = 1; +} + +message TerminalSessionRecoveryAck {} + message ConnectAck { string session_id = 1; int32 heartbeat_interval_sec = 3; + repeated TerminalSessionRecoveryCandidate terminal_session_recovery_candidates = 4; } message HeartbeatAck { @@ -74,6 +99,7 @@ message ConnectResponse { ConnectAck connect_ack = 1; HeartbeatAck heartbeat_ack = 2; CommandDispatch command_dispatch = 4; + TerminalSessionRecoveryAck terminal_session_recovery_ack = 5; } } diff --git a/console/README/overview.md b/console/README/overview.md index 4802d39..fd97159 100644 --- a/console/README/overview.md +++ b/console/README/overview.md @@ -57,6 +57,11 @@ The console service hosts: - an existing session route remains pinned to its original worker even when that worker reports full active-session capacity. - `max_inflight` and active-session capacity are evaluated independently; the worker-local session manager remains the final authority. - an explicit pre-execution `session_capacity_exceeded` can be retried on an untried worker only after the provisional route is safely removed; all attempts share the task deadline. + - terminal session restart recovery: + - confirmed routes retain their worker binding and absolute lease while the worker is disconnected; requests return retryable `session_unavailable` instead of being reassigned. + - a terminal-capable worker must reconcile every Console candidate and receive a recovery acknowledgement before it becomes dispatchable. + - recovered routes keep their original lease; missing, invalid, or expired resources delete the route so later calls follow normal `session_not_found`/creation semantics. + - route and lease state are currently in Console memory and therefore do not survive a Console restart. - MCP Streamable HTTP API (bearer token required): - `POST /mcp` for JSON-RPC requests over Streamable HTTP transport. - recommended request header: `Authorization: Bearer `. diff --git a/console/internal/grpcserver/connect_service_test.go b/console/internal/grpcserver/connect_service_test.go index 1d4e4c8..59f8515 100644 --- a/console/internal/grpcserver/connect_service_test.go +++ b/console/internal/grpcserver/connect_service_test.go @@ -2058,6 +2058,36 @@ func connectWorkerWithHello( if ack == nil { return nil, "", fmt.Errorf("expected connect_ack, got %#v", resp.GetPayload()) } + hasTerminalExec := false + for _, capability := range hello.GetCapabilities() { + if normalizeCapability(capability.GetName()) == taskCapabilityTerminalExec { + hasTerminalExec = true + break + } + } + if hasTerminalExec { + results := make([]*registryv1.TerminalSessionRecoveryResult, 0, len(ack.GetTerminalSessionRecoveryCandidates())) + for _, candidate := range ack.GetTerminalSessionRecoveryCandidates() { + results = append(results, ®istryv1.TerminalSessionRecoveryResult{ + SessionId: candidate.GetSessionId(), + Status: registryv1.TerminalSessionRecoveryResult_RECOVERED, + }) + } + if err := stream.Send(®istryv1.ConnectRequest{ + Payload: ®istryv1.ConnectRequest_TerminalSessionRecoveryReport{ + TerminalSessionRecoveryReport: ®istryv1.TerminalSessionRecoveryReport{Results: results}, + }, + }); err != nil { + return nil, "", err + } + recoveryAck, err := stream.Recv() + if err != nil { + return nil, "", err + } + if recoveryAck.GetTerminalSessionRecoveryAck() == nil { + return nil, "", fmt.Errorf("expected terminal_session_recovery_ack, got %#v", recoveryAck.GetPayload()) + } + } return stream, ack.GetSessionId(), nil } diff --git a/console/internal/grpcserver/proxy_routes_test.go b/console/internal/grpcserver/proxy_routes_test.go index 42822a2..20e0628 100644 --- a/console/internal/grpcserver/proxy_routes_test.go +++ b/console/internal/grpcserver/proxy_routes_test.go @@ -25,6 +25,7 @@ func TestDirectProxyAuthorizationResolvesE2BOriginThroughWorker(t *testing.T) { Capabilities: []*registryv1.CapabilityDeclaration{{Name: taskCapabilityTerminalExec}, {Name: taskCapabilityTerminalProxy, MaxInflight: 4}}, } session := newActiveSessionAt("worker-e2b", "connection-e2b", hello, now) + session.markRecoveryComplete() if err := service.configureSessionProxy(session, hello, "worker-secret"); err != nil { t.Fatalf("configure direct proxy: %v", err) } diff --git a/console/internal/grpcserver/service_connect.go b/console/internal/grpcserver/service_connect.go index 0a466a5..c1d5bcb 100644 --- a/console/internal/grpcserver/service_connect.go +++ b/console/internal/grpcserver/service_connect.go @@ -71,6 +71,8 @@ func (s *RegistryService) Connect(stream grpc.BidiStreamingServer[registryv1.Con } session := newActiveSessionAt(hello.GetNodeId(), sessionID, hello, now) + recoveryCandidates := s.beginTerminalSessionRecovery(session.nodeID, now) + session.setRecoveryCandidates(recoveryCandidates) if err := s.configureSessionProxy(session, hello, workerSecret); err != nil { return status.Errorf(codes.InvalidArgument, "invalid proxy endpoint: %v", err) } @@ -93,7 +95,7 @@ func (s *RegistryService) Connect(stream grpc.BidiStreamingServer[registryv1.Con writerErrCh <- writerLoop(stream, session) }() - if err := session.enqueueControl(stream.Context(), newConnectAck(sessionID, s.heartbeatIntervalSec)); err != nil { + if err := session.enqueueControl(stream.Context(), newConnectAck(sessionID, s.heartbeatIntervalSec, recoveryCandidates...)); err != nil { return status.Error(codes.Internal, "failed to send connect ack") } @@ -127,6 +129,26 @@ func (s *RegistryService) Connect(stream grpc.BidiStreamingServer[registryv1.Con if err := handleCommandResult(session, req.GetCommandResult()); err != nil { return err } + case req.GetTerminalSessionRecoveryReport() != nil: + if !session.recoveryRequired { + return status.Error(codes.InvalidArgument, "terminal session recovery is not supported by this worker") + } + if session.isReady() { + if !session.matchesRecoveryResults(req.GetTerminalSessionRecoveryReport()) { + return status.Error(codes.FailedPrecondition, "terminal session recovery report changed after completion") + } + if err := session.enqueueControl(stream.Context(), newTerminalSessionRecoveryAck()); err != nil { + return status.Error(codes.Internal, "failed to resend terminal session recovery ack") + } + continue + } + if err := s.applyTerminalSessionRecoveryReport(session, req.GetTerminalSessionRecoveryReport(), s.nowFn()); err != nil { + return status.Error(codes.InvalidArgument, err.Error()) + } + if err := session.enqueueControl(stream.Context(), newTerminalSessionRecoveryAck()); err != nil { + return status.Error(codes.Internal, "failed to send terminal session recovery ack") + } + session.markRecoveryComplete() default: return status.Error(codes.InvalidArgument, "unsupported frame type") } @@ -332,10 +354,6 @@ func (s *RegistryService) swapSession(session *activeSession) *activeSession { s.sessions[session.nodeID] = session return replaced }() - // Release sessionsMu before touching terminal route tables to avoid lock - // inversion with dispatch paths that read terminal routes then sessions. - // This leaves a tiny window where an old route may be observed once. - s.clearTerminalSessionRoutesByNode(session.nodeID) return replaced } @@ -361,9 +379,12 @@ func (s *RegistryService) removeSession(session *activeSession) { if !shouldClearStoreSession { return } - // Keep the same lock order as swapSession: sessions first, then route tables. - // Clearing route mappings outside sessionsMu avoids cross-lock deadlocks. - s.clearTerminalSessionRoutesByNode(session.nodeID) + unavailable := s.markTerminalSessionRoutesUnavailable(session.nodeID) + slog.Info( + "terminal session routes unavailable", + "executor_kind", session.executorKind, + "unavailable_route_count", unavailable, + ) if shouldClearStoreSession && s.store != nil { if err := s.store.ClearSession(session.nodeID, session.sessionID); err != nil { slog.Error( @@ -415,17 +436,26 @@ func writerLoop(stream grpc.BidiStreamingServer[registryv1.ConnectRequest, regis } } -func newConnectAck(sessionID string, heartbeatIntervalSec int32) *registryv1.ConnectResponse { +func newConnectAck(sessionID string, heartbeatIntervalSec int32, recoveryCandidates ...*registryv1.TerminalSessionRecoveryCandidate) *registryv1.ConnectResponse { return ®istryv1.ConnectResponse{ Payload: ®istryv1.ConnectResponse_ConnectAck{ ConnectAck: ®istryv1.ConnectAck{ - SessionId: sessionID, - HeartbeatIntervalSec: heartbeatIntervalSec, + SessionId: sessionID, + HeartbeatIntervalSec: heartbeatIntervalSec, + TerminalSessionRecoveryCandidates: recoveryCandidates, }, }, } } +func newTerminalSessionRecoveryAck() *registryv1.ConnectResponse { + return ®istryv1.ConnectResponse{ + Payload: ®istryv1.ConnectResponse_TerminalSessionRecoveryAck{ + TerminalSessionRecoveryAck: ®istryv1.TerminalSessionRecoveryAck{}, + }, + } +} + func newHeartbeatAck(heartbeatIntervalSec int32) *registryv1.ConnectResponse { return ®istryv1.ConnectResponse{ Payload: ®istryv1.ConnectResponse_HeartbeatAck{ diff --git a/console/internal/grpcserver/service_dispatch.go b/console/internal/grpcserver/service_dispatch.go index 20a3a8d..3b360ea 100644 --- a/console/internal/grpcserver/service_dispatch.go +++ b/console/internal/grpcserver/service_dispatch.go @@ -20,6 +20,7 @@ import ( const ( terminalSessionNotFoundCode = "session_not_found" terminalSessionCapacityExceededCode = "session_capacity_exceeded" + terminalSessionUnavailableCode = "session_unavailable" ) type dispatchOptions struct { @@ -235,7 +236,7 @@ func (s *RegistryService) dispatchCommandAttempt( terminalRouteReservationID, ) } - confirmTerminalRoute := func() { + confirmTerminalRoute := func(resultPayload []byte) { if terminalSessionID != "" { s.confirmTerminalSessionRoute( terminalSessionID, @@ -243,6 +244,9 @@ func (s *RegistryService) dispatchCommandAttempt( terminalRouteReservationID, s.nowFn(), ) + if leaseExpiresUnixMs := terminalSessionLeaseExpiresUnixMs(resultPayload); leaseExpiresUnixMs > 0 { + s.updateTerminalSessionRouteLease(terminalSessionID, session.nodeID, leaseExpiresUnixMs, s.nowFn()) + } } } @@ -293,7 +297,7 @@ func (s *RegistryService) dispatchCommandAttempt( if onDispatched != nil { if err := onDispatched(commandID); err != nil { if terminalRouteReservationID != 0 { - confirmTerminalRoute() + confirmTerminalRoute(nil) } return dispatchAttemptResult{}, err } @@ -304,7 +308,7 @@ func (s *RegistryService) dispatchCommandAttempt( if terminalRouteReservationID != 0 && terminalSessionID != "" { // The dispatch reached the worker stream, so cancellation does not // prove that session creation failed. - confirmTerminalRoute() + confirmTerminalRoute(nil) } if errors.Is(commandCtx.Err(), context.DeadlineExceeded) { return dispatchAttemptResult{}, context.DeadlineExceeded @@ -316,7 +320,7 @@ func (s *RegistryService) dispatchCommandAttempt( return dispatchAttemptResult{}, status.Error(codes.Unavailable, "worker session closed before command result") } if outcome.err == nil && terminalSessionID != "" { - confirmTerminalRoute() + confirmTerminalRoute(outcome.payloadJSON) return dispatchAttemptResult{outcome: outcome}, nil } if outcome.err == nil || terminalSessionID == "" { @@ -340,7 +344,7 @@ func (s *RegistryService) dispatchCommandAttempt( }, nil case terminalRouteReservationID != 0: // Other execution errors do not prove that session creation failed. - confirmTerminalRoute() + confirmTerminalRoute(nil) } return dispatchAttemptResult{outcome: outcome}, nil } @@ -359,6 +363,12 @@ func (s *RegistryService) pickSessionForDispatch( } now := s.nowFn() s.maybePruneTerminalSessionRoutes(now) + if route, exists := s.terminalSessionRouteSnapshot(normalizedTerminalSessionID, now); exists && route.ReservationID == 0 { + boundSession := s.getSession(route.NodeID) + if route.RecoveryState != terminalSessionRecoveryReady || boundSession == nil || !boundSession.isReady() { + return nil, 0, terminalSessionUnavailableError() + } + } nodeID, reservationID, ok := s.claimTerminalSessionRoute(normalizedTerminalSessionID, now) if !ok { @@ -376,6 +386,9 @@ func (s *RegistryService) pickSessionForDispatch( return session, reservationID, nil } if errors.Is(err, ErrNoCapabilityWorker) { + if reservationID == 0 { + return nil, 0, terminalSessionUnavailableError() + } s.clearTerminalSessionRoute(normalizedTerminalSessionID, nodeID) return s.tryReserveAndPickTerminalSession(capability, ownerID, normalizedTerminalSessionID, now, options) } @@ -440,7 +453,7 @@ func (s *RegistryService) pickSessionForNodeAndCapability(nodeID string, capabil } session := s.getSession(normalizedNodeID) - if session == nil || !session.hasCapability(capability) { + if session == nil || !session.isReady() || !session.hasCapability(capability) { return nil, ErrNoCapabilityWorker } @@ -485,7 +498,7 @@ func (s *RegistryService) pickSessionForCapability( continue } session := s.getSession(nodeID) - if session == nil || !session.hasCapability(capability) { + if session == nil || !session.isReady() || !session.hasCapability(capability) { continue } hasSession = true @@ -573,6 +586,26 @@ func isSessionCapacityCommandError(err error) bool { return isCommandErrorCode(err, terminalSessionCapacityExceededCode) } +func terminalSessionUnavailableError() error { + return &CommandExecutionError{ + Code: terminalSessionUnavailableCode, + Message: "terminal session is temporarily unavailable", + } +} + +func terminalSessionLeaseExpiresUnixMs(payload []byte) int64 { + if len(payload) == 0 { + return 0 + } + var decoded struct { + LeaseExpiresUnixMS int64 `json:"lease_expires_unix_ms"` + } + if err := json.Unmarshal(payload, &decoded); err != nil || decoded.LeaseExpiresUnixMS <= 0 { + return 0 + } + return decoded.LeaseExpiresUnixMS +} + func isCommandErrorCode(err error, code string) bool { var commandErr *CommandExecutionError if !errors.As(err, &commandErr) { diff --git a/console/internal/grpcserver/session_runtime.go b/console/internal/grpcserver/session_runtime.go index fd3a3bd..f355ea6 100644 --- a/console/internal/grpcserver/session_runtime.go +++ b/console/internal/grpcserver/session_runtime.go @@ -36,8 +36,10 @@ type terminalSessionCapacitySnapshot struct { } type activeSession struct { - nodeID string - sessionID string + nodeID string + sessionID string + executorKind string + connectedAt time.Time proxyEndpoint string routeTokenKey []byte @@ -49,6 +51,12 @@ type activeSession struct { terminalCapacityMu sync.RWMutex terminalCapacity terminalSessionCapacitySnapshot + recoveryMu sync.RWMutex + recoveryRequired bool + recoveryComplete bool + recoveryCandidates map[string]int64 + recoveryResults map[string]registryv1.TerminalSessionRecoveryResult_Status + controlOutbound chan *registryv1.ConnectResponse commandOutbound chan *registryv1.ConnectResponse done chan struct{} @@ -61,19 +69,30 @@ type activeSession struct { } func newActiveSession(nodeID string, sessionID string, hello *registryv1.ConnectHello) *activeSession { - return newActiveSessionAt(nodeID, sessionID, hello, time.Now()) + session := newActiveSessionAt(nodeID, sessionID, hello, time.Now()) + // This convenience constructor is used by already-established in-process + // sessions in tests. Real Connect sessions use newActiveSessionAt and must + // complete the recovery handshake before becoming ready. + session.markRecoveryComplete() + return session } func newActiveSessionAt(nodeID string, sessionID string, hello *registryv1.ConnectHello, observedAt time.Time) *activeSession { session := &activeSession{ - nodeID: nodeID, - sessionID: sessionID, - capabilities: capabilitiesFromHello(hello), - controlOutbound: make(chan *registryv1.ConnectResponse, controlOutboundBufferSize), - commandOutbound: make(chan *registryv1.ConnectResponse, commandOutboundBufferSize), - done: make(chan struct{}), - pending: make(map[string]*pendingCommand), - } + nodeID: nodeID, + sessionID: sessionID, + executorKind: strings.TrimSpace(hello.GetExecutorKind()), + connectedAt: observedAt, + capabilities: capabilitiesFromHello(hello), + controlOutbound: make(chan *registryv1.ConnectResponse, controlOutboundBufferSize), + commandOutbound: make(chan *registryv1.ConnectResponse, commandOutboundBufferSize), + done: make(chan struct{}), + pending: make(map[string]*pendingCommand), + recoveryCandidates: make(map[string]int64), + recoveryResults: make(map[string]registryv1.TerminalSessionRecoveryResult_Status), + } + _, session.recoveryRequired = session.capabilities[taskCapabilityTerminalExec] + session.recoveryComplete = !session.recoveryRequired if capacity := hello.GetTerminalSessionCapacity(); capacity != nil { session.terminalCapacity = terminalSessionCapacitySnapshot{ known: true, @@ -85,6 +104,95 @@ func newActiveSessionAt(nodeID string, sessionID string, hello *registryv1.Conne return session } +func (s *activeSession) setRecoveryCandidates(candidates []*registryv1.TerminalSessionRecoveryCandidate) { + if s == nil { + return + } + s.recoveryMu.Lock() + defer s.recoveryMu.Unlock() + s.recoveryCandidates = make(map[string]int64, len(candidates)) + for _, candidate := range candidates { + if candidate == nil { + continue + } + sessionID := strings.TrimSpace(candidate.GetSessionId()) + if sessionID == "" { + continue + } + s.recoveryCandidates[sessionID] = candidate.GetLeaseExpiresUnixMs() + } +} + +func (s *activeSession) recoveryCandidateSnapshot() map[string]int64 { + if s == nil { + return nil + } + s.recoveryMu.RLock() + defer s.recoveryMu.RUnlock() + out := make(map[string]int64, len(s.recoveryCandidates)) + for sessionID, lease := range s.recoveryCandidates { + out[sessionID] = lease + } + return out +} + +func (s *activeSession) markRecoveryComplete() { + if s == nil { + return + } + s.recoveryMu.Lock() + s.recoveryComplete = true + s.recoveryMu.Unlock() +} + +func (s *activeSession) setRecoveryResults(results map[string]registryv1.TerminalSessionRecoveryResult_Status) { + if s == nil { + return + } + s.recoveryMu.Lock() + defer s.recoveryMu.Unlock() + s.recoveryResults = make(map[string]registryv1.TerminalSessionRecoveryResult_Status, len(results)) + for sessionID, status := range results { + s.recoveryResults[sessionID] = status + } +} + +func (s *activeSession) matchesRecoveryResults(report *registryv1.TerminalSessionRecoveryReport) bool { + if s == nil || report == nil { + return false + } + s.recoveryMu.RLock() + defer s.recoveryMu.RUnlock() + if len(report.GetResults()) != len(s.recoveryResults) { + return false + } + seen := make(map[string]struct{}, len(report.GetResults())) + for _, result := range report.GetResults() { + if result == nil { + return false + } + sessionID := strings.TrimSpace(result.GetSessionId()) + status, ok := s.recoveryResults[sessionID] + if !ok || status != result.GetStatus() { + return false + } + if _, duplicate := seen[sessionID]; duplicate { + return false + } + seen[sessionID] = struct{}{} + } + return true +} + +func (s *activeSession) isReady() bool { + if s == nil { + return false + } + s.recoveryMu.RLock() + defer s.recoveryMu.RUnlock() + return !s.recoveryRequired || s.recoveryComplete +} + func (s *activeSession) hasCapability(capability string) bool { normalized := normalizeCapability(capability) if normalized == "" { diff --git a/console/internal/grpcserver/terminal_capacity_dispatch_test.go b/console/internal/grpcserver/terminal_capacity_dispatch_test.go index 48c7588..d7065c1 100644 --- a/console/internal/grpcserver/terminal_capacity_dispatch_test.go +++ b/console/internal/grpcserver/terminal_capacity_dispatch_test.go @@ -33,6 +33,7 @@ func addTerminalCapacityTestWorker( t.Fatalf("upsert worker %s: %v", nodeID, err) } session := newActiveSessionAt(nodeID, "worker-session-"+nodeID, hello, now) + session.markRecoveryComplete() svc.swapSession(session) return session } diff --git a/console/internal/grpcserver/terminal_session_recovery_test.go b/console/internal/grpcserver/terminal_session_recovery_test.go new file mode 100644 index 0000000..0e3d0a7 --- /dev/null +++ b/console/internal/grpcserver/terminal_session_recovery_test.go @@ -0,0 +1,193 @@ +package grpcserver + +import ( + "errors" + "testing" + "time" + + registryv1 "github.com/onlyboxes/onlyboxes/api/gen/go/registry/v1" + "github.com/onlyboxes/onlyboxes/console/internal/testutil/registrytest" +) + +func TestTerminalRouteDisconnectRecoveryRoundTrip(t *testing.T) { + svc := NewRegistryService(registrytest.NewStore(t), nil, 5, 15, time.Minute) + base := time.Unix(1_700_100_000, 0) + lease := base.Add(10 * time.Minute).UnixMilli() + svc.nowFn = func() time.Time { return base.Add(time.Second) } + + svc.bindTerminalSessionRoute("session-a", "node-a", base) + if !svc.updateTerminalSessionRouteLease("session-a", "node-a", lease, base) { + t.Fatal("failed to store terminal lease") + } + if unavailable := svc.markTerminalSessionRoutesUnavailable("node-a"); unavailable != 1 { + t.Fatalf("unavailable route count=%d, want 1", unavailable) + } + + _, _, err := svc.pickSessionForDispatch( + taskCapabilityTerminalExec, + "owner-a", + "session-a", + sessionPickOptions{terminalSessionIntent: terminalSessionIntentKnownNew}, + ) + var commandErr *CommandExecutionError + if !errors.As(err, &commandErr) || commandErr.Code != terminalSessionUnavailableCode { + t.Fatalf("expected session_unavailable while disconnected, got %v", err) + } + + candidates := svc.beginTerminalSessionRecovery("node-a", base.Add(time.Second)) + if len(candidates) != 1 || candidates[0].GetSessionId() != "session-a" || candidates[0].GetLeaseExpiresUnixMs() != lease { + t.Fatalf("unexpected recovery candidates: %#v", candidates) + } + + hello := ®istryv1.ConnectHello{ + NodeId: "node-a", + Capabilities: []*registryv1.CapabilityDeclaration{ + {Name: taskCapabilityTerminalExec, MaxInflight: 1}, + }, + } + session := newActiveSessionAt("node-a", "worker-session-a", hello, base.Add(time.Second)) + session.setRecoveryCandidates(candidates) + err = svc.applyTerminalSessionRecoveryReport(session, ®istryv1.TerminalSessionRecoveryReport{ + Results: []*registryv1.TerminalSessionRecoveryResult{ + {SessionId: "session-a", Status: registryv1.TerminalSessionRecoveryResult_RECOVERED}, + }, + }, base.Add(2*time.Second)) + if err != nil { + t.Fatalf("apply recovery report: %v", err) + } + route, ok := svc.terminalSessionRouteSnapshot("session-a", base.Add(2*time.Second)) + if !ok || route.RecoveryState != terminalSessionRecoveryReady || route.LeaseExpiresUnixMs != lease { + t.Fatalf("unexpected recovered route: %#v ok=%v", route, ok) + } +} + +func TestTerminalRecoveryMissingDeletesRoute(t *testing.T) { + svc := NewRegistryService(registrytest.NewStore(t), nil, 5, 15, time.Minute) + base := time.Unix(1_700_200_000, 0) + svc.bindTerminalSessionRoute("session-missing", "node-a", base) + svc.updateTerminalSessionRouteLease("session-missing", "node-a", base.Add(time.Minute).UnixMilli(), base) + candidates := svc.beginTerminalSessionRecovery("node-a", base.Add(time.Second)) + session := newActiveSessionAt("node-a", "worker-session-a", ®istryv1.ConnectHello{ + Capabilities: []*registryv1.CapabilityDeclaration{{Name: taskCapabilityTerminalExec, MaxInflight: 1}}, + }, base) + session.setRecoveryCandidates(candidates) + + err := svc.applyTerminalSessionRecoveryReport(session, ®istryv1.TerminalSessionRecoveryReport{ + Results: []*registryv1.TerminalSessionRecoveryResult{ + {SessionId: "session-missing", Status: registryv1.TerminalSessionRecoveryResult_MISSING}, + }, + }, base.Add(2*time.Second)) + if err != nil { + t.Fatalf("apply missing report: %v", err) + } + if _, ok := svc.terminalSessionRouteSnapshot("session-missing", base.Add(2*time.Second)); ok { + t.Fatal("missing backend resource must delete route") + } +} + +func TestTerminalRecoveryUsesLegacyRouteTTLFallback(t *testing.T) { + svc := NewRegistryService(registrytest.NewStore(t), nil, 5, 15, time.Minute) + svc.terminalRouteTTL = 30 * time.Minute + base := time.Unix(1_700_300_000, 0) + svc.bindTerminalSessionRoute("legacy-session", "node-a", base) + + candidates := svc.beginTerminalSessionRecovery("node-a", base.Add(time.Minute)) + wantLease := base.Add(30 * time.Minute).UnixMilli() + if len(candidates) != 1 || candidates[0].GetLeaseExpiresUnixMs() != wantLease { + t.Fatalf("legacy route candidate lease=%v, want %d", candidates, wantLease) + } +} + +func TestTerminalRecoveryDropsRouteWhoseLeaseExpiredOffline(t *testing.T) { + svc := NewRegistryService(registrytest.NewStore(t), nil, 5, 15, time.Minute) + base := time.Unix(1_700_350_000, 0) + svc.bindTerminalSessionRoute("expired-session", "node-a", base) + svc.updateTerminalSessionRouteLease("expired-session", "node-a", base.Add(time.Minute).UnixMilli(), base) + svc.markTerminalSessionRoutesUnavailable("node-a") + if candidates := svc.beginTerminalSessionRecovery("node-a", base.Add(2*time.Minute)); len(candidates) != 0 { + t.Fatalf("expired route was offered for recovery: %#v", candidates) + } + if _, ok := svc.terminalSessionRouteSnapshot("expired-session", base.Add(2*time.Minute)); ok { + t.Fatal("expired offline route was not deleted") + } +} + +func TestPruneUsesExactLeaseWhenLegacyTTLDisabled(t *testing.T) { + svc := NewRegistryService(registrytest.NewStore(t), nil, 5, 15, time.Minute) + svc.terminalRouteTTL = 0 + base := time.Unix(1_700_375_000, 0) + svc.bindTerminalSessionRoute("expired-session", "node-a", base) + svc.updateTerminalSessionRouteLease("expired-session", "node-a", base.Add(time.Minute).UnixMilli(), base) + if removed := svc.pruneExpiredTerminalSessionRoutes(base.Add(2 * time.Minute)); removed != 1 { + t.Fatalf("removed=%d, want exact-lease route pruned", removed) + } +} + +func TestTerminalRecoveryReportMustCoverCandidates(t *testing.T) { + svc := NewRegistryService(registrytest.NewStore(t), nil, 5, 15, time.Minute) + base := time.Unix(1_700_400_000, 0) + svc.bindTerminalSessionRoute("session-a", "node-a", base) + svc.updateTerminalSessionRouteLease("session-a", "node-a", base.Add(time.Minute).UnixMilli(), base) + candidates := svc.beginTerminalSessionRecovery("node-a", base) + session := newActiveSessionAt("node-a", "worker-session-a", ®istryv1.ConnectHello{ + Capabilities: []*registryv1.CapabilityDeclaration{{Name: taskCapabilityTerminalExec, MaxInflight: 1}}, + }, base) + session.setRecoveryCandidates(candidates) + + if err := svc.applyTerminalSessionRecoveryReport(session, ®istryv1.TerminalSessionRecoveryReport{}, base); err == nil { + t.Fatal("partial recovery report unexpectedly accepted") + } +} + +func TestTerminalRecoveryReportIsIdempotentOnlyWhenUnchanged(t *testing.T) { + svc := NewRegistryService(registrytest.NewStore(t), nil, 5, 15, time.Minute) + base := time.Unix(1_700_500_000, 0) + svc.bindTerminalSessionRoute("session-a", "node-a", base) + svc.updateTerminalSessionRouteLease("session-a", "node-a", base.Add(time.Minute).UnixMilli(), base) + candidates := svc.beginTerminalSessionRecovery("node-a", base) + session := newActiveSessionAt("node-a", "worker-session-a", ®istryv1.ConnectHello{ + Capabilities: []*registryv1.CapabilityDeclaration{{Name: taskCapabilityTerminalExec, MaxInflight: 1}}, + }, base) + session.setRecoveryCandidates(candidates) + report := ®istryv1.TerminalSessionRecoveryReport{Results: []*registryv1.TerminalSessionRecoveryResult{ + {SessionId: "session-a", Status: registryv1.TerminalSessionRecoveryResult_RECOVERED}, + }} + if err := svc.applyTerminalSessionRecoveryReport(session, report, base.Add(time.Second)); err != nil { + t.Fatalf("apply recovery report: %v", err) + } + session.markRecoveryComplete() + if !session.matchesRecoveryResults(report) { + t.Fatal("unchanged recovery report must be accepted as an idempotent retry") + } + changed := ®istryv1.TerminalSessionRecoveryReport{Results: []*registryv1.TerminalSessionRecoveryResult{ + {SessionId: "session-a", Status: registryv1.TerminalSessionRecoveryResult_MISSING}, + }} + if session.matchesRecoveryResults(changed) { + t.Fatal("changed recovery report must not be accepted as an idempotent retry") + } +} + +func TestTerminalRecoveryRejectsCandidateReassignedToAnotherWorker(t *testing.T) { + svc := NewRegistryService(registrytest.NewStore(t), nil, 5, 15, time.Minute) + base := time.Unix(1_700_600_000, 0) + svc.bindTerminalSessionRoute("session-a", "node-a", base) + svc.updateTerminalSessionRouteLease("session-a", "node-a", base.Add(time.Minute).UnixMilli(), base) + candidates := svc.beginTerminalSessionRecovery("node-a", base) + session := newActiveSessionAt("node-a", "worker-session-a", ®istryv1.ConnectHello{ + Capabilities: []*registryv1.CapabilityDeclaration{{Name: taskCapabilityTerminalExec, MaxInflight: 1}}, + }, base) + session.setRecoveryCandidates(candidates) + svc.bindTerminalSessionRoute("session-a", "node-b", base.Add(time.Second)) + err := svc.applyTerminalSessionRecoveryReport(session, ®istryv1.TerminalSessionRecoveryReport{ + Results: []*registryv1.TerminalSessionRecoveryResult{{ + SessionId: "session-a", Status: registryv1.TerminalSessionRecoveryResult_RECOVERED, + }}, + }, base.Add(2*time.Second)) + if err == nil { + t.Fatal("recovery report unexpectedly changed a route reassigned to another worker") + } + route, ok := svc.terminalSessionRouteSnapshot("session-a", base.Add(2*time.Second)) + if !ok || route.NodeID != "node-b" { + t.Fatalf("reassigned route changed: %#v ok=%v", route, ok) + } +} diff --git a/console/internal/grpcserver/terminal_session_routes.go b/console/internal/grpcserver/terminal_session_routes.go index 13f6ace..5d86152 100644 --- a/console/internal/grpcserver/terminal_session_routes.go +++ b/console/internal/grpcserver/terminal_session_routes.go @@ -1,8 +1,12 @@ package grpcserver import ( + "log/slog" + "sort" "strings" "time" + + registryv1 "github.com/onlyboxes/onlyboxes/api/gen/go/registry/v1" ) type routeReservationReleaseResult int @@ -14,14 +18,24 @@ const ( ) type terminalSessionRoute struct { - NodeID string - LastUsedUnixMs int64 + NodeID string + LastUsedUnixMs int64 + LeaseExpiresUnixMs int64 + RecoveryState terminalSessionRecoveryState // ReservationID is non-zero only while the first dispatch is provisional. // A successful result confirms the route by clearing it. ReservationID uint64 ProvisionalUses uint64 } +type terminalSessionRecoveryState uint8 + +const ( + terminalSessionRecoveryReady terminalSessionRecoveryState = iota + terminalSessionRecoveryUnavailable + terminalSessionRecoveryReconciling +) + func (s *RegistryService) bindTerminalSessionRoute(sessionID string, nodeID string, now time.Time) { if s == nil { return @@ -50,6 +64,7 @@ func (s *RegistryService) bindTerminalSessionRoute(sessionID string, nodeID stri s.terminalSessionToNode[normalizedSessionID] = terminalSessionRoute{ NodeID: normalizedNodeID, LastUsedUnixMs: nowUnixMs, + RecoveryState: terminalSessionRecoveryReady, ReservationID: 0, ProvisionalUses: 0, } @@ -95,6 +110,7 @@ func (s *RegistryService) reserveTerminalSessionRoute(sessionID string, preferre s.terminalSessionToNode[normalizedSessionID] = terminalSessionRoute{ NodeID: normalizedNodeID, LastUsedUnixMs: nowUnixMs, + RecoveryState: terminalSessionRecoveryReady, ReservationID: reservationID, ProvisionalUses: 1, } @@ -194,6 +210,197 @@ func (s *RegistryService) touchTerminalSessionRoute(sessionID string, now time.T return route.NodeID, true } +func (s *RegistryService) updateTerminalSessionRouteLease(sessionID string, expectedNodeID string, leaseExpiresUnixMs int64, now time.Time) bool { + if s == nil || leaseExpiresUnixMs <= 0 { + return false + } + normalizedSessionID := strings.TrimSpace(sessionID) + normalizedNodeID := strings.TrimSpace(expectedNodeID) + if normalizedSessionID == "" || normalizedNodeID == "" { + return false + } + + s.terminalRoutesMu.Lock() + defer s.terminalRoutesMu.Unlock() + route, ok := s.terminalSessionToNode[normalizedSessionID] + if !ok || route.NodeID != normalizedNodeID || route.ReservationID != 0 { + return false + } + route.LeaseExpiresUnixMs = leaseExpiresUnixMs + route.LastUsedUnixMs = routeNowUnixMs(now) + route.RecoveryState = terminalSessionRecoveryReady + s.terminalSessionToNode[normalizedSessionID] = route + return true +} + +func (s *RegistryService) terminalSessionRouteSnapshot(sessionID string, now time.Time) (terminalSessionRoute, bool) { + if s == nil { + return terminalSessionRoute{}, false + } + normalizedSessionID := strings.TrimSpace(sessionID) + if normalizedSessionID == "" { + return terminalSessionRoute{}, false + } + nowUnixMs := routeNowUnixMs(now) + s.terminalRoutesMu.Lock() + defer s.terminalRoutesMu.Unlock() + route, ok := s.terminalSessionToNode[normalizedSessionID] + if !ok { + return terminalSessionRoute{}, false + } + if route.LeaseExpiresUnixMs > 0 && route.LeaseExpiresUnixMs <= nowUnixMs { + s.deleteTerminalSessionRouteLocked(normalizedSessionID, route) + return terminalSessionRoute{}, false + } + return route, true +} + +func (s *RegistryService) beginTerminalSessionRecovery(nodeID string, now time.Time) []*registryv1.TerminalSessionRecoveryCandidate { + if s == nil { + return nil + } + normalizedNodeID := strings.TrimSpace(nodeID) + if normalizedNodeID == "" { + return nil + } + nowUnixMs := routeNowUnixMs(now) + s.terminalRoutesMu.Lock() + defer s.terminalRoutesMu.Unlock() + index := s.terminalNodeToSessionIDIndex[normalizedNodeID] + candidates := make([]*registryv1.TerminalSessionRecoveryCandidate, 0, len(index)) + for sessionID := range index { + route, ok := s.terminalSessionToNode[sessionID] + if !ok || route.NodeID != normalizedNodeID { + continue + } + leaseExpiresUnixMs := route.LeaseExpiresUnixMs + if leaseExpiresUnixMs <= 0 && s.terminalRouteTTL > 0 { + leaseExpiresUnixMs = route.LastUsedUnixMs + s.terminalRouteTTL.Milliseconds() + } + if route.ReservationID != 0 || leaseExpiresUnixMs <= nowUnixMs { + s.deleteTerminalSessionRouteLocked(sessionID, route) + continue + } + route.LeaseExpiresUnixMs = leaseExpiresUnixMs + route.RecoveryState = terminalSessionRecoveryReconciling + s.terminalSessionToNode[sessionID] = route + candidates = append(candidates, ®istryv1.TerminalSessionRecoveryCandidate{ + SessionId: sessionID, + LeaseExpiresUnixMs: leaseExpiresUnixMs, + }) + } + sort.Slice(candidates, func(i, j int) bool { return candidates[i].GetSessionId() < candidates[j].GetSessionId() }) + return candidates +} + +func (s *RegistryService) markTerminalSessionRoutesUnavailable(nodeID string) int { + if s == nil { + return 0 + } + normalizedNodeID := strings.TrimSpace(nodeID) + if normalizedNodeID == "" { + return 0 + } + s.terminalRoutesMu.Lock() + defer s.terminalRoutesMu.Unlock() + unavailable := 0 + for sessionID := range s.terminalNodeToSessionIDIndex[normalizedNodeID] { + route, ok := s.terminalSessionToNode[sessionID] + if !ok || route.NodeID != normalizedNodeID { + continue + } + if route.ReservationID != 0 { + s.deleteTerminalSessionRouteLocked(sessionID, route) + continue + } + route.RecoveryState = terminalSessionRecoveryUnavailable + s.terminalSessionToNode[sessionID] = route + unavailable++ + } + return unavailable +} + +func (s *RegistryService) applyTerminalSessionRecoveryReport(session *activeSession, report *registryv1.TerminalSessionRecoveryReport, now time.Time) error { + if s == nil || session == nil || report == nil { + return nil + } + candidates := session.recoveryCandidateSnapshot() + results := make(map[string]registryv1.TerminalSessionRecoveryResult_Status, len(report.GetResults())) + for _, result := range report.GetResults() { + if result == nil { + continue + } + sessionID := strings.TrimSpace(result.GetSessionId()) + if _, ok := candidates[sessionID]; !ok { + return &terminalRecoveryValidationError{message: "recovery result is not a candidate"} + } + if _, duplicate := results[sessionID]; duplicate { + return &terminalRecoveryValidationError{message: "duplicate recovery result"} + } + status := result.GetStatus() + if status != registryv1.TerminalSessionRecoveryResult_RECOVERED && status != registryv1.TerminalSessionRecoveryResult_MISSING && status != registryv1.TerminalSessionRecoveryResult_INVALID { + return &terminalRecoveryValidationError{message: "invalid recovery status"} + } + results[sessionID] = status + } + if len(results) != len(candidates) { + return &terminalRecoveryValidationError{message: "recovery report must cover every candidate"} + } + + nowUnixMs := routeNowUnixMs(now) + recovered := 0 + failures := 0 + s.terminalRoutesMu.Lock() + for sessionID := range candidates { + if route, ok := s.terminalSessionToNode[sessionID]; ok && route.NodeID != session.nodeID { + s.terminalRoutesMu.Unlock() + return &terminalRecoveryValidationError{message: "recovery candidate no longer belongs to worker"} + } + } + for sessionID, leaseExpiresUnixMs := range candidates { + route, ok := s.terminalSessionToNode[sessionID] + if !ok || route.NodeID != session.nodeID { + continue + } + status := results[sessionID] + if status != registryv1.TerminalSessionRecoveryResult_RECOVERED || (leaseExpiresUnixMs > 0 && leaseExpiresUnixMs <= nowUnixMs) { + s.deleteTerminalSessionRouteLocked(sessionID, route) + failures++ + continue + } + route.RecoveryState = terminalSessionRecoveryReady + route.LastUsedUnixMs = nowUnixMs + s.terminalSessionToNode[sessionID] = route + recovered++ + } + unavailable := s.countTerminalSessionRoutesByStateLocked(terminalSessionRecoveryUnavailable) + s.terminalRoutesMu.Unlock() + session.setRecoveryResults(results) + slog.Info( + "terminal session recovery metrics", + "executor_kind", session.executorKind, + "recovery_duration_ms", now.Sub(session.connectedAt).Milliseconds(), + "recovered_session_count", recovered, + "recovery_failures", failures, + "unavailable_route_count", unavailable, + ) + return nil +} + +func (s *RegistryService) countTerminalSessionRoutesByStateLocked(state terminalSessionRecoveryState) int { + count := 0 + for _, route := range s.terminalSessionToNode { + if route.RecoveryState == state && route.ReservationID == 0 { + count++ + } + } + return count +} + +type terminalRecoveryValidationError struct{ message string } + +func (e *terminalRecoveryValidationError) Error() string { return e.message } + // clearTerminalSessionRouteReservation releases one provisional dispatch. The // route is removed only after every dispatch sharing the reservation failed. func (s *RegistryService) clearTerminalSessionRouteReservation(sessionID string, expectedNodeID string, reservationID uint64) routeReservationReleaseResult { @@ -258,40 +465,11 @@ func (s *RegistryService) deleteTerminalSessionRouteLocked(sessionID string, rou } } -func (s *RegistryService) clearTerminalSessionRoutesByNode(nodeID string) { - if s == nil { - return - } - normalizedNodeID := strings.TrimSpace(nodeID) - if normalizedNodeID == "" { - return - } - - s.terminalRoutesMu.Lock() - defer s.terminalRoutesMu.Unlock() - - index := s.terminalNodeToSessionIDIndex[normalizedNodeID] - if index == nil { - return - } - for sessionID := range index { - route, ok := s.terminalSessionToNode[sessionID] - if !ok || route.NodeID != normalizedNodeID { - continue - } - delete(s.terminalSessionToNode, sessionID) - } - delete(s.terminalNodeToSessionIDIndex, normalizedNodeID) -} - func (s *RegistryService) pruneExpiredTerminalSessionRoutes(now time.Time) int { if s == nil { return 0 } ttl := s.terminalRouteTTL - if ttl <= 0 { - return 0 - } nowUnixMs := routeNowUnixMs(now) expireBefore := nowUnixMs - ttl.Milliseconds() @@ -300,7 +478,11 @@ func (s *RegistryService) pruneExpiredTerminalSessionRoutes(now time.Time) int { defer s.terminalRoutesMu.Unlock() for sessionID, route := range s.terminalSessionToNode { - if route.LastUsedUnixMs > expireBefore { + if route.LeaseExpiresUnixMs > 0 { + if route.LeaseExpiresUnixMs > nowUnixMs { + continue + } + } else if ttl <= 0 || route.LastUsedUnixMs > expireBefore { continue } delete(s.terminalSessionToNode, sessionID) diff --git a/console/internal/httpapi/command_handler.go b/console/internal/httpapi/command_handler.go index 0c7bf20..ce1695f 100644 --- a/console/internal/httpapi/command_handler.go +++ b/console/internal/httpapi/command_handler.go @@ -13,24 +13,25 @@ import ( ) const ( - defaultEchoTimeoutMS = 5000 - minEchoTimeoutMS = 1 - maxEchoTimeoutMS = 60000 - defaultTerminalTimeoutMS = defaultTaskTimeoutMS - minTerminalTimeoutMS = 1 - maxTerminalTimeoutMS = maxTaskTimeoutMS - defaultComputerUseTimeoutMS = defaultTaskTimeoutMS - minComputerUseTimeoutMS = 1 - maxComputerUseTimeoutMS = maxTaskTimeoutMS - terminalExecCapability = "terminalExec" - computerUseCapability = "computerUse" - terminalExecSessionNotFoundCode = "session_not_found" - terminalExecSessionBusyCode = "session_busy" - terminalExecSessionCapacityCode = "session_capacity_exceeded" - terminalExecInvalidPayloadCode = "invalid_payload" - terminalTaskNoWorkerCode = "no_worker" - terminalTaskNoCapacityCode = "no_capacity" - terminalTaskTimeoutCode = "timeout" + defaultEchoTimeoutMS = 5000 + minEchoTimeoutMS = 1 + maxEchoTimeoutMS = 60000 + defaultTerminalTimeoutMS = defaultTaskTimeoutMS + minTerminalTimeoutMS = 1 + maxTerminalTimeoutMS = maxTaskTimeoutMS + defaultComputerUseTimeoutMS = defaultTaskTimeoutMS + minComputerUseTimeoutMS = 1 + maxComputerUseTimeoutMS = maxTaskTimeoutMS + terminalExecCapability = "terminalExec" + computerUseCapability = "computerUse" + terminalExecSessionNotFoundCode = "session_not_found" + terminalExecSessionBusyCode = "session_busy" + terminalExecSessionCapacityCode = "session_capacity_exceeded" + terminalExecSessionUnavailableCode = "session_unavailable" + terminalExecInvalidPayloadCode = "invalid_payload" + terminalTaskNoWorkerCode = "no_worker" + terminalTaskNoCapacityCode = "no_capacity" + terminalTaskTimeoutCode = "timeout" ) type EchoDispatcher interface { @@ -320,6 +321,8 @@ func mapTerminalTaskFailure(task grpcserver.TaskSnapshot) (int, string) { return http.StatusConflict, message case terminalExecSessionCapacityCode: return http.StatusTooManyRequests, message + case terminalExecSessionUnavailableCode: + return http.StatusServiceUnavailable, message case terminalTaskNoWorkerCode: return http.StatusServiceUnavailable, "no online worker supports requested capability" case terminalTaskNoCapacityCode: diff --git a/console/internal/httpapi/command_handler_test.go b/console/internal/httpapi/command_handler_test.go index e1263cf..d6891fb 100644 --- a/console/internal/httpapi/command_handler_test.go +++ b/console/internal/httpapi/command_handler_test.go @@ -260,6 +260,7 @@ func TestTerminalCommandStatusMappings(t *testing.T) { {name: "session_not_found", errorCode: terminalExecSessionNotFoundCode, statusCode: http.StatusNotFound}, {name: "session_busy", errorCode: terminalExecSessionBusyCode, statusCode: http.StatusConflict}, {name: "session_capacity_exceeded", errorCode: terminalExecSessionCapacityCode, statusCode: http.StatusTooManyRequests}, + {name: "session_unavailable", errorCode: terminalExecSessionUnavailableCode, statusCode: http.StatusServiceUnavailable}, {name: "invalid_payload", errorCode: terminalExecInvalidPayloadCode, statusCode: http.StatusBadRequest}, {name: "no_capacity", errorCode: terminalTaskNoCapacityCode, statusCode: http.StatusTooManyRequests}, {name: "no_worker", errorCode: terminalTaskNoWorkerCode, statusCode: http.StatusServiceUnavailable}, diff --git a/console/internal/httpapi/integration_test.go b/console/internal/httpapi/integration_test.go index d5c8b10..616132c 100644 --- a/console/internal/httpapi/integration_test.go +++ b/console/internal/httpapi/integration_test.go @@ -615,6 +615,7 @@ func TestTerminalLifecycle(t *testing.T) { if connectResp.GetConnectAck() == nil { t.Fatalf("expected connect_ack, got %#v", connectResp.GetPayload()) } + completeEmptyTerminalRecovery(t, stream, connectResp.GetConnectAck()) go func() { sessionContent := map[string]string{} @@ -1116,6 +1117,7 @@ func TestTokenIsolationLifecycle(t *testing.T) { if connectResp.GetConnectAck() == nil { t.Fatalf("expected connect_ack, got %#v", connectResp.GetPayload()) } + completeEmptyTerminalRecovery(t, stream, connectResp.GetConnectAck()) go func() { sessionContent := map[string]string{} @@ -1943,6 +1945,31 @@ func requestList(t *testing.T, client *http.Client, url string) listWorkersRespo return payload } +func completeEmptyTerminalRecovery( + t *testing.T, + stream grpc.BidiStreamingClient[registryv1.ConnectRequest, registryv1.ConnectResponse], + ack *registryv1.ConnectAck, +) { + t.Helper() + if len(ack.GetTerminalSessionRecoveryCandidates()) != 0 { + t.Fatalf("new test worker unexpectedly received recovery candidates: %#v", ack.GetTerminalSessionRecoveryCandidates()) + } + if err := stream.Send(®istryv1.ConnectRequest{ + Payload: ®istryv1.ConnectRequest_TerminalSessionRecoveryReport{ + TerminalSessionRecoveryReport: ®istryv1.TerminalSessionRecoveryReport{}, + }, + }); err != nil { + t.Fatalf("send empty terminal recovery report: %v", err) + } + resp, err := stream.Recv() + if err != nil { + t.Fatalf("receive terminal recovery ack: %v", err) + } + if resp.GetTerminalSessionRecoveryAck() == nil { + t.Fatalf("expected terminal recovery ack, got %#v", resp.GetPayload()) + } +} + func newAuthenticatedClient(t *testing.T, server *httptest.Server) *http.Client { t.Helper() diff --git a/console/internal/httpapi/task_handler.go b/console/internal/httpapi/task_handler.go index e69d551..b588654 100644 --- a/console/internal/httpapi/task_handler.go +++ b/console/internal/httpapi/task_handler.go @@ -170,7 +170,11 @@ func (h *WorkerHandler) writeTaskSubmitError(c *gin.Context, err error) { case errors.Is(err, grpcserver.ErrNoWorkerCapacity): c.JSON(http.StatusTooManyRequests, gin.H{"error": "no online worker capacity for requested capability"}) case errors.As(err, &commandErr): - c.JSON(http.StatusBadGateway, gin.H{"error": commandErr.Error()}) + if strings.EqualFold(strings.TrimSpace(commandErr.Code), terminalExecSessionUnavailableCode) { + c.JSON(http.StatusServiceUnavailable, gin.H{"error": commandErr.Error()}) + } else { + c.JSON(http.StatusBadGateway, gin.H{"error": commandErr.Error()}) + } case errors.Is(err, context.DeadlineExceeded): c.JSON(http.StatusGatewayTimeout, gin.H{"error": "task timed out"}) case status.Code(err) == codes.InvalidArgument: @@ -192,7 +196,7 @@ func mapTaskTerminalStatusToHTTP(task grpcserver.TaskSnapshot) int { switch task.ErrorCode { case "no_capacity", terminalExecSessionCapacityCode: return http.StatusTooManyRequests - case "no_worker": + case "no_worker", terminalExecSessionUnavailableCode: return http.StatusServiceUnavailable default: return http.StatusBadGateway diff --git a/console/internal/httpapi/task_handler_test.go b/console/internal/httpapi/task_handler_test.go index 6278eb9..e14b1e5 100644 --- a/console/internal/httpapi/task_handler_test.go +++ b/console/internal/httpapi/task_handler_test.go @@ -313,6 +313,16 @@ func TestMapTaskTerminalStatusToHTTPSessionCapacity(t *testing.T) { } } +func TestMapTaskTerminalStatusToHTTPSessionUnavailable(t *testing.T) { + task := grpcserver.TaskSnapshot{ + Status: grpcserver.TaskStatusFailed, + ErrorCode: terminalExecSessionUnavailableCode, + } + if got := mapTaskTerminalStatusToHTTP(task); got != http.StatusServiceUnavailable { + t.Fatalf("expected 503, got %d", got) + } +} + func TestCancelTaskTerminalConflict(t *testing.T) { now := time.Unix(1_700_000_000, 0) handler := NewWorkerHandler(registrytest.NewStore(t), 15*time.Second, &fakeTaskDispatcher{ diff --git a/worker/worker-boxlite/Cargo.toml b/worker/worker-boxlite/Cargo.toml index 41d47cb..af8386d 100644 --- a/worker/worker-boxlite/Cargo.toml +++ b/worker/worker-boxlite/Cargo.toml @@ -14,8 +14,8 @@ rand = "0.8" reqwest = { version = "0.12", default-features = false, features = ["rustls-tls"] } serde = { version = "1", features = ["derive"] } serde_json = "1" -thiserror = "2" sha2 = "0.10" +thiserror = "2" toml = "0.8" tokio = { version = "1", features = ["io-util", "macros", "net", "rt-multi-thread", "signal", "sync", "time"] } tokio-stream = { version = "0.1", features = ["net"] } diff --git a/worker/worker-boxlite/README/implementation-plan.md b/worker/worker-boxlite/README/implementation-plan.md index 709cc08..a5c8670 100644 --- a/worker/worker-boxlite/README/implementation-plan.md +++ b/worker/worker-boxlite/README/implementation-plan.md @@ -732,7 +732,7 @@ struct MockLiteBox { ... } - [ ] gRPC 握手流程(Fake gRPC Server via tonic) - [ ] 心跳容忍(单次超时恢复) - [ ] 心跳失败(双次超时触发重连) -- [ ] 优雅关闭清理所有 VM +- [ ] 优雅关闭清理一次性 VM,并保留可恢复的 detached terminal VM ### 7.3 集成测试 diff --git a/worker/worker-boxlite/README/overview.md b/worker/worker-boxlite/README/overview.md index 31b161a..b3e8f7a 100644 --- a/worker/worker-boxlite/README/overview.md +++ b/worker/worker-boxlite/README/overview.md @@ -69,8 +69,9 @@ Capability behavior: - `terminalExec` cleanup behavior: - command timeout/cancel marks the session for destruction and stops it accepting new commands; the box is removed once in-flight commands drain, so one command's timeout does not kill its siblings. - idle sessions are reaped after lease expiry by an internal janitor loop; a session with in-flight commands is never reaped. - - worker shutdown cancels pending Box creation, waits up to five seconds for it to stop, then force-removes all managed terminal boxes; a Box returned after the wait is removed as a late cleanup. - - `SIGINT`/`SIGTERM` performs best-effort cleanup; `SIGKILL`/process crash does not guarantee cleanup. + - terminal Boxes use deterministic `onlyboxes-terminal-v1-` names with `auto_remove=false` and `detach=true`, so normal exit and process termination preserve them. + - after reconnect, the worker reconciles Console candidates before accepting commands, reattaches or starts matching Boxes from the configured `WORKER_BOXLITE_HOME`, restores the exact lease, and removes local Onlyboxes terminal orphans. + - lease expiry, explicit destruction, unsafe command timeout, and invalid Box state still remove the Box; one-shot `pythonExec` Boxes remain per-call resources. - `terminalExec` result uses JSON payload: - `{"session_id":"...","created":true,"stdout":"...","stderr":"...","exit_code":0,"stdout_truncated":false,"stderr_truncated":false,"lease_expires_unix_ms":...}` - output truncation: diff --git a/worker/worker-boxlite/src/boxlite_runtime.rs b/worker/worker-boxlite/src/boxlite_runtime.rs index 5d0e97c..f76b64d 100644 --- a/worker/worker-boxlite/src/boxlite_runtime.rs +++ b/worker/worker-boxlite/src/boxlite_runtime.rs @@ -1,12 +1,12 @@ -use std::collections::HashMap; +use std::collections::{HashMap, HashSet}; use std::path::{Path, PathBuf}; use std::sync::{Arc, Mutex, OnceLock}; use std::time::Duration; use boxlite::runtime::options::PortSpec; use boxlite::{ - BoxCommand, BoxOptions, BoxliteOptions, BoxliteRuntime, CopyOptions, ExecStderr, ExecStdout, - LiteBox, RootfsSpec, + BoxCommand, BoxOptions, BoxStatus, BoxliteOptions, BoxliteRuntime, CopyOptions, ExecStderr, + ExecStdout, LiteBox, RootfsSpec, }; use tokio::task::JoinHandle; use tokio_stream::StreamExt; @@ -211,6 +211,79 @@ async fn create_terminal_session_box_once( Ok(box_id) } +pub(crate) async fn recover_terminal_session_box( + cfg: &Config, + box_name: &str, +) -> Result, BoxliteCommandError> { + let runtime = runtime(cfg)?; + let Some(litebox) = runtime + .get(box_name.trim()) + .await + .map_err(|err| BoxliteCommandError::ExecutionFailed(format!("lookup box failed: {err}")))? + else { + return Ok(None); + }; + + match litebox.info().status { + BoxStatus::Running => {} + BoxStatus::Configured | BoxStatus::Stopped => { + litebox.start().await.map_err(|err| { + BoxliteCommandError::ExecutionFailed(format!("restart terminalExec box: {err}")) + })?; + } + BoxStatus::Unknown | BoxStatus::Stopping | BoxStatus::Paused | BoxStatus::Failed => { + return Err(BoxliteCommandError::ExecutionFailed(format!( + "terminalExec box has invalid status: {:?}", + litebox.info().status + ))); + } + } + + let box_id = litebox.id().as_str().to_owned(); + cache_terminal_session_box(litebox); + Ok(Some(box_id)) +} + +pub(crate) async fn cleanup_orphan_terminal_session_boxes( + cfg: &Config, + expected_names: &HashSet, +) -> usize { + let Ok(runtime) = runtime(cfg) else { + return 0; + }; + let infos = match runtime.list_info().await { + Ok(infos) => infos, + Err(err) => { + tracing::warn!(error = %err, "list terminal Box orphans failed"); + return 0; + } + }; + + let mut cleaned = 0; + for info in infos { + let Some(name) = info.name.as_deref() else { + continue; + }; + if !is_onlyboxes_terminal_session_box_name(name) || expected_names.contains(name) { + continue; + } + remove_box(cfg, info.id.as_str()).await; + cleaned += 1; + } + cleaned +} + +fn is_onlyboxes_terminal_session_box_name(name: &str) -> bool { + const PREFIX: &str = "onlyboxes-terminal-v1-"; + let Some(hash) = name.strip_prefix(PREFIX) else { + return false; + }; + hash.len() == 64 + && hash + .bytes() + .all(|byte| byte.is_ascii_digit() || (b'a'..=b'f').contains(&byte)) +} + fn is_proxy_port_conflict(err: &BoxliteCommandError) -> bool { matches!( err, @@ -423,6 +496,7 @@ fn build_terminal_exec_box_options( memory_mib: Some(cfg.terminal_exec_memory_mib), rootfs: RootfsSpec::Image(cfg.terminal_exec_image.clone()), auto_remove: false, + detach: true, entrypoint: Some(vec!["sh".to_owned(), "-lc".to_owned()]), cmd: Some(vec![TERMINAL_EXEC_IDLE_COMMAND.to_owned()]), ports, @@ -728,6 +802,20 @@ mod tests { assert_eq!(resolve_cpus(999), u8::MAX); } + #[test] + fn orphan_cleanup_name_filter_rejects_foreign_boxes() { + let valid = format!("onlyboxes-terminal-v1-{}", "a".repeat(64)); + assert!(is_onlyboxes_terminal_session_box_name(&valid)); + assert!(!is_onlyboxes_terminal_session_box_name("developer-box")); + assert!(!is_onlyboxes_terminal_session_box_name( + "onlyboxes-terminal-v1-not-a-session-hash" + )); + assert!(!is_onlyboxes_terminal_session_box_name(&format!( + "onlyboxes-terminal-v1-{}", + "A".repeat(64) + ))); + } + #[test] fn terminal_exec_box_options_override_entrypoint_to_idle_loop() { let cfg = Config { @@ -780,6 +868,8 @@ mod tests { options.cmd, Some(vec![TERMINAL_EXEC_IDLE_COMMAND.to_owned()]) ); + assert!(!options.auto_remove); + assert!(options.detach); } #[test] diff --git a/worker/worker-boxlite/src/runner/session_client.rs b/worker/worker-boxlite/src/runner/session_client.rs index d898483..dffce96 100644 --- a/worker/worker-boxlite/src/runner/session_client.rs +++ b/worker/worker-boxlite/src/runner/session_client.rs @@ -11,11 +11,14 @@ use tonic::transport::{Channel, ClientTlsConfig, Endpoint}; use crate::config::Config; use crate::proto::registryv1::{ - connect_request, connect_response, worker_registry_service_client::WorkerRegistryServiceClient, - CommandDispatch, ConnectRequest, ConnectResponse, HeartbeatAck, HeartbeatFrame, + connect_request, connect_response, terminal_session_recovery_result, + worker_registry_service_client::WorkerRegistryServiceClient, CommandDispatch, ConnectRequest, + ConnectResponse, HeartbeatAck, HeartbeatFrame, TerminalSessionRecoveryReport, }; -use super::terminal_session_manager::shared_active_session_count; +use super::terminal_session_manager::{ + shared_active_session_count, shared_recover_terminal_sessions, +}; use super::{ build_command_result, build_hello, command_dispatch_summary_for_log, duration_from_server, RunnerError, @@ -66,6 +69,63 @@ async fn run_session_with_builder( )); } + let recovery_results = tokio::select! { + _ = shutdown.cancelled() => return Err(RunnerError::Message("worker shutdown during terminal session recovery".to_owned())), + result = tokio::time::timeout( + cfg.call_timeout, + shared_recover_terminal_sessions(cfg, &ack.terminal_session_recovery_candidates), + ) => result.map_err(|_| RunnerError::Message("terminal session recovery deadline exceeded".to_owned()))?, + }; + outbound_tx + .send(ConnectRequest { + payload: Some(connect_request::Payload::TerminalSessionRecoveryReport( + TerminalSessionRecoveryReport { + results: recovery_results.clone(), + }, + )), + }) + .await + .map_err(|_| { + RunnerError::Message("send terminal session recovery report failed".to_owned()) + })?; + let recovery_ack = recv_with_timeout(shutdown.clone(), cfg.call_timeout, inbound.message()) + .await? + .ok_or_else(|| { + RunnerError::Message("stream closed before terminal session recovery ack".to_owned()) + })?; + if !matches!( + recovery_ack.payload, + Some(connect_response::Payload::TerminalSessionRecoveryAck(_)) + ) { + return Err(RunnerError::Message( + "unexpected response while waiting for terminal session recovery ack".to_owned(), + )); + } + + tracing::info!( + executor_kind = "boxlite", + candidates = recovery_results.len(), + recovered = recovery_results + .iter() + .filter(|result| { + result.status == terminal_session_recovery_result::Status::Recovered as i32 + }) + .count(), + missing = recovery_results + .iter() + .filter(|result| { + result.status == terminal_session_recovery_result::Status::Missing as i32 + }) + .count(), + invalid = recovery_results + .iter() + .filter(|result| { + result.status == terminal_session_recovery_result::Status::Invalid as i32 + }) + .count(), + "terminal session recovery acknowledged" + ); + let heartbeat_interval = duration_from_server(ack.heartbeat_interval_sec, cfg.heartbeat_interval); tracing::info!( @@ -341,6 +401,7 @@ mod tests { connect_request as test_connect_request, connect_response as test_connect_response, CommandDispatch as TestCommandDispatch, CommandResult, ConnectAck, ConnectHello, ConnectRequest as TestConnectRequest, ConnectResponse, HeartbeatAck, HeartbeatFrame, + TerminalSessionRecoveryAck, }; use super::*; @@ -444,12 +505,33 @@ mod tests { payload: Some(test_connect_response::Payload::ConnectAck(ConnectAck { session_id: TEST_SESSION_ID.to_owned(), heartbeat_interval_sec: shared.heartbeat_interval_sec, + terminal_session_recovery_candidates: Vec::new(), })), })) .await .map_err(|_| Status::internal("failed to enqueue connect ack"))?; tokio::spawn(async move { + match inbound.message().await { + Ok(Some(frame)) + if matches!( + frame.payload, + Some(test_connect_request::Payload::TerminalSessionRecoveryReport(_)) + ) => {} + _ => return, + } + if response_tx + .send(Ok(ConnectResponse { + payload: Some(test_connect_response::Payload::TerminalSessionRecoveryAck( + TerminalSessionRecoveryAck {}, + )), + })) + .await + .is_err() + { + return; + } + for dispatch in &shared.dispatches { if response_tx .send(Ok(ConnectResponse { @@ -487,7 +569,11 @@ mod tests { Some(test_connect_request::Payload::CommandResult(result)) => { let _ = shared.command_result_tx.send(result); } - Some(test_connect_request::Payload::Hello(_)) | None => return, + Some(test_connect_request::Payload::Hello(_)) + | Some(test_connect_request::Payload::TerminalSessionRecoveryReport( + _, + )) + | None => return, }, Ok(None) | Err(_) => return, } @@ -556,11 +642,17 @@ mod tests { worker_secret: "secret".to_owned(), heartbeat_interval: Duration::from_millis(25), heartbeat_jitter_pct: 0, - call_timeout: Duration::from_millis(25), + call_timeout: Duration::from_millis(250), node_name: "worker-boxlite-test".to_owned(), executor_kind: "boxlite".to_owned(), labels: BTreeMap::new(), - boxlite_home: String::new(), + boxlite_home: std::env::temp_dir() + .join(format!( + "onlyboxes-worker-boxlite-session-client-tests-{}", + std::process::id() + )) + .to_string_lossy() + .into_owned(), python_exec_image: "ghcr.io/astral-sh/uv:python3.12-bookworm-slim".to_owned(), python_exec_memory_mib: 256, python_exec_cpus: 1, diff --git a/worker/worker-boxlite/src/runner/terminal_session_manager.rs b/worker/worker-boxlite/src/runner/terminal_session_manager.rs index e83e87c..5bb95fb 100644 --- a/worker/worker-boxlite/src/runner/terminal_session_manager.rs +++ b/worker/worker-boxlite/src/runner/terminal_session_manager.rs @@ -1,4 +1,4 @@ -use std::collections::HashMap; +use std::collections::{HashMap, HashSet}; use std::path::{Path, PathBuf}; use std::sync::atomic::{AtomicBool, AtomicU32, Ordering}; use std::sync::Arc; @@ -9,6 +9,7 @@ use async_trait::async_trait; use base64::Engine; use reqwest::StatusCode; use serde::{Deserialize, Serialize}; +use sha2::{Digest, Sha256}; use tokio::sync::{watch, Mutex, Notify}; use tokio_util::io::ReaderStream; use tokio_util::sync::CancellationToken; @@ -16,6 +17,10 @@ use uuid::Uuid; use crate::boxlite_runtime::{self, BoxliteCommandError, CollectedExecOutput}; use crate::config::Config; +use crate::proto::registryv1::{ + terminal_session_recovery_result, TerminalSessionRecoveryCandidate, + TerminalSessionRecoveryResult, +}; pub(crate) const TERMINAL_EXEC_JANITOR_INTERVAL: Duration = Duration::from_secs(5); #[cfg(not(test))] @@ -231,6 +236,23 @@ pub(crate) trait TerminalBackend: Send + Sync { shutdown: CancellationToken, deadline_unix_ms: i64, ) -> Result; + async fn create_session_box_for_session( + &self, + _session_id: &str, + shutdown: CancellationToken, + deadline_unix_ms: i64, + ) -> Result { + self.create_session_box(shutdown, deadline_unix_ms).await + } + async fn recover_session_box( + &self, + _session_id: &str, + ) -> Result, TerminalOperationError> { + Ok(None) + } + async fn cleanup_orphan_session_boxes(&self, _expected_names: &HashSet) -> usize { + 0 + } async fn exec_shell_command( &self, box_id: &str, @@ -342,7 +364,7 @@ pub(crate) fn shared_terminal_session_manager(cfg: &Config) -> Arc i32 { } } +pub(crate) async fn shared_recover_terminal_sessions( + cfg: &Config, + candidates: &[TerminalSessionRecoveryCandidate], +) -> Vec { + shared_terminal_session_manager(cfg) + .recover_sessions(candidates) + .await +} + struct RetiredSession { box_id: String, capacity_reserved: bool, @@ -475,7 +506,11 @@ impl TerminalSessionManager { let _creation_guard = PendingCreationGuard { manager: self }; match self .backend - .create_session_box(self.shutdown.clone(), req.deadline_unix_ms) + .create_session_box_for_session( + &claimed.session_id, + self.shutdown.clone(), + req.deadline_unix_ms, + ) .await { Ok(box_id) => { @@ -778,7 +813,16 @@ impl TerminalSessionManager { Ok(TerminalResourceRunResult { blob, ..result }) } + #[cfg(test)] pub(crate) async fn close(&self) { + self.close_inner(false).await; + } + + pub(crate) async fn close_preserving(&self) { + self.close_inner(true).await; + } + + async fn close_inner(&self, preserve: bool) { if self.closed.swap(true, Ordering::SeqCst) { return; } @@ -816,7 +860,14 @@ impl TerminalSessionManager { self.wait_for_pending_cleanups().await; for session in sessions { - self.cleanup_retired_session(session).await; + if preserve { + if session.capacity_reserved { + self.active_session_reservations + .fetch_sub(1, Ordering::SeqCst); + } + } else { + self.cleanup_retired_session(session).await; + } } } @@ -835,6 +886,129 @@ impl TerminalSessionManager { .min(i32::MAX as u32) as i32 } + pub(crate) async fn recover_sessions( + &self, + candidates: &[TerminalSessionRecoveryCandidate], + ) -> Vec { + let started_at = std::time::Instant::now(); + let now = SystemTime::now(); + let mut results = Vec::with_capacity(candidates.len()); + let mut recovered_names = HashSet::with_capacity(candidates.len()); + + for candidate in candidates { + let session_id = candidate.session_id.trim(); + let lease_expires_at = system_time_from_unix_millis(candidate.lease_expires_unix_ms); + let status = if session_id.is_empty() + || lease_expires_at.is_none() + || lease_expires_at.is_some_and(|lease| lease <= now) + { + terminal_session_recovery_result::Status::Invalid + } else { + let lease_expires_at = lease_expires_at.expect("validated lease"); + match self.recover_one_session(session_id, lease_expires_at).await { + Ok(true) => { + recovered_names.insert(terminal_session_resource_name(session_id)); + terminal_session_recovery_result::Status::Recovered + } + Ok(false) => terminal_session_recovery_result::Status::Missing, + Err(err) => { + tracing::warn!( + session_id_hash = %terminal_session_id_hash(session_id), + error = %err, + "terminal Box recovery failed" + ); + terminal_session_recovery_result::Status::Invalid + } + } + }; + results.push(TerminalSessionRecoveryResult { + session_id: candidate.session_id.clone(), + status: status as i32, + }); + } + + let orphan_cleaned = self + .backend + .cleanup_orphan_session_boxes(&recovered_names) + .await; + let missing = results + .iter() + .filter(|result| { + result.status == terminal_session_recovery_result::Status::Missing as i32 + }) + .count(); + let invalid = results + .iter() + .filter(|result| { + result.status == terminal_session_recovery_result::Status::Invalid as i32 + }) + .count(); + tracing::info!( + executor_kind = "boxlite", + discovered = recovered_names.len() + orphan_cleaned, + candidate = candidates.len(), + recovered = recovered_names.len(), + missing, + invalid, + orphan_cleaned, + duration_ms = started_at.elapsed().as_millis(), + recovery_failures = invalid, + "terminal session recovery completed" + ); + results + } + + async fn recover_one_session( + &self, + session_id: &str, + lease_expires_at: SystemTime, + ) -> Result { + { + let mut sessions = self.sessions.lock().await; + if let Some(session) = sessions.get_mut(session_id) { + if session.destroying || session.box_id.trim().is_empty() { + return Err(TerminalOperationError::ExecutionFailed( + "terminal session is not recoverable".to_owned(), + )); + } + session.lease_expires_at = lease_expires_at; + return Ok(true); + } + } + + let Some(box_id) = self.backend.recover_session_box(session_id).await? else { + return Ok(false); + }; + + let mut sessions = self.sessions.lock().await; + if self.closed.load(Ordering::SeqCst) { + return Err(TerminalOperationError::ExecutionFailed( + TERMINAL_MANAGER_CLOSED_MESSAGE.to_owned(), + )); + } + if let Some(session) = sessions.get_mut(session_id) { + session.lease_expires_at = lease_expires_at; + return Ok(true); + } + + let (ready_tx, _) = watch::channel(SessionReadyState::Ready(box_id.clone())); + sessions.insert( + session_id.to_owned(), + TerminalSession { + session_id: session_id.to_owned(), + box_id, + lease_expires_at, + inflight: 0, + destroying: false, + capacity_reserved: true, + ready_tx, + }, + ); + self.active_session_reservations + .fetch_add(1, Ordering::SeqCst); + Ok(true) + } + pub(crate) async fn cleanup_expired_sessions(&self) { let now = SystemTime::now(); let expired = { @@ -1284,6 +1458,34 @@ impl TerminalBackend for BoxliteTerminalBackend { }) } + async fn create_session_box_for_session( + &self, + session_id: &str, + shutdown: CancellationToken, + deadline_unix_ms: i64, + ) -> Result { + let name = terminal_session_resource_name(session_id); + boxlite_runtime::create_terminal_session_box(&self.cfg, &name, shutdown, deadline_unix_ms) + .await + .map_err(map_boxlite_terminal_error) + } + + async fn recover_session_box( + &self, + session_id: &str, + ) -> Result, TerminalOperationError> { + boxlite_runtime::recover_terminal_session_box( + &self.cfg, + &terminal_session_resource_name(session_id), + ) + .await + .map_err(map_boxlite_terminal_error) + } + + async fn cleanup_orphan_session_boxes(&self, expected_names: &HashSet) -> usize { + boxlite_runtime::cleanup_orphan_terminal_session_boxes(&self.cfg, expected_names).await + } + async fn exec_shell_command( &self, box_id: &str, @@ -1342,6 +1544,29 @@ impl TerminalBackend for BoxliteTerminalBackend { } } +fn map_boxlite_terminal_error(err: BoxliteCommandError) -> TerminalOperationError { + match err { + BoxliteCommandError::DeadlineExceeded => TerminalOperationError::DeadlineExceeded, + BoxliteCommandError::MissingBox => { + TerminalOperationError::ExecutionFailed("terminalExec box not found".to_owned()) + } + BoxliteCommandError::ExecutionFailed(message) => { + TerminalOperationError::ExecutionFailed(message) + } + } +} + +fn terminal_session_id_hash(session_id: &str) -> String { + format!("{:x}", Sha256::digest(session_id.as_bytes())) +} + +fn terminal_session_resource_name(session_id: &str) -> String { + format!( + "onlyboxes-terminal-v1-{}", + terminal_session_id_hash(session_id) + ) +} + fn truncate_by_bytes(value: &str, max_bytes: usize) -> (String, bool) { if max_bytes == 0 || value.len() <= max_bytes { return (value.to_owned(), false); @@ -1459,6 +1684,13 @@ fn to_unix_millis(time: SystemTime) -> i64 { .unwrap_or_default() } +fn system_time_from_unix_millis(value: i64) -> Option { + if value <= 0 { + return None; + } + SystemTime::UNIX_EPOCH.checked_add(Duration::from_millis(value as u64)) +} + fn serialize_resource_blob(blob: &[u8], serializer: S) -> Result where S: serde::Serializer, @@ -1504,6 +1736,85 @@ mod tests { use tokio::net::TcpListener; use tokio::sync::Notify; + struct RecoveryBackend { + boxes: Mutex>, + expected_names: Mutex>, + } + + impl RecoveryBackend { + fn new(session_id: &str, box_id: &str) -> Arc { + Arc::new(Self { + boxes: Mutex::new(HashMap::from([(session_id.to_owned(), box_id.to_owned())])), + expected_names: Mutex::new(HashSet::new()), + }) + } + } + + #[async_trait] + impl TerminalBackend for RecoveryBackend { + async fn create_session_box( + &self, + _shutdown: CancellationToken, + _deadline_unix_ms: i64, + ) -> Result { + Err(TerminalOperationError::ExecutionFailed( + "unexpected create".to_owned(), + )) + } + + async fn recover_session_box( + &self, + session_id: &str, + ) -> Result, TerminalOperationError> { + Ok(self.boxes.lock().await.get(session_id).cloned()) + } + + async fn cleanup_orphan_session_boxes(&self, expected_names: &HashSet) -> usize { + *self.expected_names.lock().await = expected_names.clone(); + 0 + } + + async fn exec_shell_command( + &self, + _box_id: &str, + _command: &str, + _deadline_unix_ms: i64, + ) -> Result { + Ok(CollectedExecOutput { + stdout: String::new(), + stderr: String::new(), + exit_code: 0, + }) + } + + async fn exec_resource_probe( + &self, + _box_id: &str, + _action: &str, + _file_path: &str, + _max_read_bytes: usize, + _deadline_unix_ms: i64, + ) -> Result { + Err(BoxliteCommandError::ExecutionFailed( + "unexpected resource probe".to_owned(), + )) + } + + async fn copy_out_file( + &self, + _box_id: &str, + _container_src: &str, + _host_dst: &Path, + _deadline_unix_ms: i64, + ) -> Result<(), BoxliteCommandError> { + Err(BoxliteCommandError::ExecutionFailed( + "unexpected copy".to_owned(), + )) + } + + async fn remove_box(&self, _box_id: &str) {} + } + struct StatefulShellBackend { next_box: Mutex, persisted: Mutex>, @@ -4293,4 +4604,138 @@ mod tests { manager.close().await; } + + #[tokio::test] + async fn recovery_restores_box_with_exact_lease_and_capacity() { + let session_id = "owner-a:session-recover"; + let backend = RecoveryBackend::new(session_id, "box-recover"); + let manager = manager_with_limit(backend.clone(), 1); + let lease = SystemTime::now() + Duration::from_secs(600); + let lease_unix_ms = to_unix_millis(lease); + + let results = manager + .recover_sessions(&[TerminalSessionRecoveryCandidate { + session_id: session_id.to_owned(), + lease_expires_unix_ms: lease_unix_ms, + }]) + .await; + assert_eq!(results.len(), 1); + assert_eq!( + results[0].status, + terminal_session_recovery_result::Status::Recovered as i32 + ); + assert_eq!(manager.active_session_count().await, 1); + + let sessions = manager.sessions.lock().await; + let recovered = sessions.get(session_id).expect("recovered session"); + assert_eq!(recovered.box_id, "box-recover"); + assert_eq!(recovered.inflight, 0); + assert_eq!(to_unix_millis(recovered.lease_expires_at), lease_unix_ms); + drop(sessions); + + let expected_names = backend.expected_names.lock().await; + assert_eq!( + &*expected_names, + &HashSet::from([terminal_session_resource_name(session_id)]) + ); + drop(expected_names); + + let reused = manager + .execute(exec_req(session_id, "printf recovered", false)) + .await + .expect("execute recovered session"); + assert!(!reused.created); + assert_eq!(reused.session_id, session_id); + + let repeated = manager + .recover_sessions(&[TerminalSessionRecoveryCandidate { + session_id: session_id.to_owned(), + lease_expires_unix_ms: lease_unix_ms, + }]) + .await; + assert_eq!( + repeated[0].status, + terminal_session_recovery_result::Status::Recovered as i32 + ); + assert_eq!(manager.active_session_count().await, 1); + assert_eq!( + to_unix_millis( + manager + .sessions + .lock() + .await + .get(session_id) + .expect("recovered session") + .lease_expires_at + ), + lease_unix_ms + ); + manager.close_preserving().await; + } + + #[tokio::test] + async fn recovery_reports_missing_and_skips_expired_candidate() { + let backend = RecoveryBackend::new("different-session", "box-other"); + let manager = manager_with_limit(backend.clone(), 1); + let results = manager + .recover_sessions(&[ + TerminalSessionRecoveryCandidate { + session_id: "session-missing".to_owned(), + lease_expires_unix_ms: to_unix_millis( + SystemTime::now() + Duration::from_secs(60), + ), + }, + TerminalSessionRecoveryCandidate { + session_id: "session-expired".to_owned(), + lease_expires_unix_ms: to_unix_millis( + SystemTime::now() - Duration::from_secs(60), + ), + }, + ]) + .await; + assert_eq!( + results[0].status, + terminal_session_recovery_result::Status::Missing as i32 + ); + assert_eq!( + results[1].status, + terminal_session_recovery_result::Status::Invalid as i32 + ); + assert_eq!(manager.active_session_count().await, 0); + assert!(backend.expected_names.lock().await.is_empty()); + manager.close_preserving().await; + } + + #[tokio::test] + async fn concurrent_recovery_uses_one_reservation() { + let session_id = "session-concurrent"; + let backend = RecoveryBackend::new(session_id, "box-concurrent"); + let manager = manager_with_limit(backend, 1); + let candidates = vec![TerminalSessionRecoveryCandidate { + session_id: session_id.to_owned(), + lease_expires_unix_ms: to_unix_millis(SystemTime::now() + Duration::from_secs(60)), + }]; + let (first, second) = tokio::join!( + manager.recover_sessions(&candidates), + manager.recover_sessions(&candidates) + ); + for result in [first, second] { + assert_eq!( + result[0].status, + terminal_session_recovery_result::Status::Recovered as i32 + ); + } + assert_eq!(manager.active_session_count().await, 1); + manager.close_preserving().await; + } + + #[test] + fn recovery_resource_name_is_deterministic_and_does_not_expose_session_id() { + let session_id = "owner-secret:session-secret"; + let name = terminal_session_resource_name(session_id); + assert!(name.starts_with("onlyboxes-terminal-v1-")); + assert_eq!(name.len(), "onlyboxes-terminal-v1-".len() + 64); + assert!(!name.contains("owner-secret")); + assert_eq!(name, terminal_session_resource_name(session_id)); + } } diff --git a/worker/worker-boxlite/tests/integration.rs b/worker/worker-boxlite/tests/integration.rs index c022d86..db5ed13 100644 --- a/worker/worker-boxlite/tests/integration.rs +++ b/worker/worker-boxlite/tests/integration.rs @@ -5,7 +5,9 @@ //! These tests require a working Boxlite runtime (macOS Hypervisor.framework or Linux KVM). //! Run with: `cargo test --features integration` -use boxlite::{BoxCommand, BoxOptions, BoxliteOptions, BoxliteRuntime, RootfsSpec}; +use boxlite::{BoxCommand, BoxOptions, BoxStatus, BoxliteOptions, BoxliteRuntime, RootfsSpec}; +use std::io::{BufRead, BufReader}; +use std::process::{Command, Stdio}; use std::sync::{Arc, OnceLock}; use tokio_stream::StreamExt; @@ -148,3 +150,190 @@ async fn test_vm_stop_and_cleanup() { rt.remove(&name, true).await.expect("remove"); assert!(!rt.exists(&name).await.expect("exists check after remove")); } + +#[tokio::test] +async fn test_detached_terminal_survives_runtime_restart_and_keeps_files() { + let home = std::path::PathBuf::from(format!( + "/tmp/obx-recovery-it-{}-{:08x}", + std::process::id(), + rand::random::() + )); + let name = random_name("onlyboxes-terminal-v1-recovery"); + let mut options = BoxliteOptions::default(); + options.home_dir = home.clone(); + let first_runtime = Arc::new(BoxliteRuntime::new(options.clone()).expect("first runtime")); + let litebox = first_runtime + .get_or_create( + BoxOptions { + cpus: Some(1), + memory_mib: Some(512), + rootfs: RootfsSpec::Image("alpine:latest".into()), + auto_remove: false, + detach: true, + entrypoint: Some(vec!["sh".into(), "-lc".into()]), + cmd: Some(vec!["while true; do sleep 3600; done".into()]), + ..Default::default() + }, + Some(name.clone()), + ) + .await + .expect("create detached terminal") + .0; + litebox.start().await.expect("start detached terminal"); + let write = litebox + .exec( + BoxCommand::new("sh").args(["-lc", "printf recovery-ok > /tmp/onlyboxes-recovery.txt"]), + ) + .await + .expect("write persistent file"); + assert_eq!(write.wait().await.expect("wait for write").exit_code, 0); + drop(litebox); + first_runtime + .shutdown(None) + .await + .expect("shutdown first runtime"); + drop(first_runtime); + + let second_runtime = Arc::new(BoxliteRuntime::new(options).expect("second runtime")); + let recovered = second_runtime + .get(&name) + .await + .expect("lookup by deterministic name") + .expect("detached terminal survived runtime shutdown"); + match recovered.info().status { + BoxStatus::Running => {} + BoxStatus::Configured | BoxStatus::Stopped => { + recovered.start().await.expect("restart recovered terminal") + } + status => panic!("unexpected recovered status: {status:?}"), + } + let mut read = recovered + .exec(BoxCommand::new("sh").args(["-lc", "cat /tmp/onlyboxes-recovery.txt"])) + .await + .expect("read persistent file"); + let mut stdout = String::new(); + if let Some(mut stream) = read.stdout() { + while let Some(chunk) = stream.next().await { + stdout.push_str(&chunk); + } + } + assert_eq!(read.wait().await.expect("wait for read").exit_code, 0); + assert_eq!(stdout, "recovery-ok"); + second_runtime + .remove(&name, true) + .await + .expect("remove recovered terminal"); +} + +#[tokio::test] +async fn boxlite_forced_termination_helper() { + if std::env::var("BOXLITE_CRASH_HELPER").ok().as_deref() != Some("1") { + return; + } + let home = + std::path::PathBuf::from(std::env::var("BOXLITE_CRASH_HOME").expect("BOXLITE_CRASH_HOME")); + let name = std::env::var("BOXLITE_CRASH_NAME").expect("BOXLITE_CRASH_NAME"); + let mut options = BoxliteOptions::default(); + options.home_dir = home; + let runtime = BoxliteRuntime::new(options).expect("crash helper runtime"); + let litebox = runtime + .get_or_create(detached_terminal_options(), Some(name)) + .await + .expect("crash helper create") + .0; + litebox.start().await.expect("crash helper start"); + let write = litebox + .exec(BoxCommand::new("sh").args([ + "-lc", + "printf forced-recovery-ok > /tmp/onlyboxes-forced-recovery.txt", + ])) + .await + .expect("crash helper write"); + assert_eq!(write.wait().await.expect("crash helper wait").exit_code, 0); + println!("BOXLITE_CRASH_HELPER_READY"); + std::io::Write::flush(&mut std::io::stdout()).expect("flush helper readiness"); + std::future::pending::<()>().await; +} + +#[tokio::test] +async fn test_detached_terminal_survives_forced_worker_termination() { + let home = std::path::PathBuf::from(format!( + "/tmp/obx-crash-it-{}-{:08x}", + std::process::id(), + rand::random::() + )); + let name = random_name("onlyboxes-terminal-v1-crash-recovery"); + let mut child = Command::new(std::env::current_exe().expect("current test executable")) + .args([ + "--exact", + "boxlite_forced_termination_helper", + "--nocapture", + ]) + .env("BOXLITE_CRASH_HELPER", "1") + .env("BOXLITE_CRASH_HOME", &home) + .env("BOXLITE_CRASH_NAME", &name) + .stdout(Stdio::piped()) + .spawn() + .expect("spawn crash helper"); + let stdout = child.stdout.take().expect("crash helper stdout"); + let mut ready = false; + for line in BufReader::new(stdout).lines() { + let line = line.expect("read crash helper output"); + if line.contains("BOXLITE_CRASH_HELPER_READY") { + ready = true; + break; + } + } + assert!( + ready, + "crash helper exited before creating the detached Box" + ); + child.kill().expect("force terminate crash helper"); + let status = child.wait().expect("wait for crash helper termination"); + assert!(!status.success(), "crash helper was not force terminated"); + + let mut options = BoxliteOptions::default(); + options.home_dir = home; + let runtime = BoxliteRuntime::new(options).expect("recovery runtime after forced termination"); + let recovered = runtime + .get(&name) + .await + .expect("lookup after forced termination") + .expect("detached Box survived forced worker termination"); + match recovered.info().status { + BoxStatus::Running => {} + BoxStatus::Configured | BoxStatus::Stopped => { + recovered.start().await.expect("restart recovered Box") + } + status => panic!("unexpected recovered status: {status:?}"), + } + let mut read = recovered + .exec(BoxCommand::new("sh").args(["-lc", "cat /tmp/onlyboxes-forced-recovery.txt"])) + .await + .expect("read after forced termination"); + let mut stdout = String::new(); + if let Some(mut stream) = read.stdout() { + while let Some(chunk) = stream.next().await { + stdout.push_str(&chunk); + } + } + assert_eq!(read.wait().await.expect("wait for read").exit_code, 0); + assert_eq!(stdout, "forced-recovery-ok"); + runtime + .remove(&name, true) + .await + .expect("remove recovered Box"); +} + +fn detached_terminal_options() -> BoxOptions { + BoxOptions { + cpus: Some(1), + memory_mib: Some(512), + rootfs: RootfsSpec::Image("alpine:latest".into()), + auto_remove: false, + detach: true, + entrypoint: Some(vec!["sh".into(), "-lc".into()]), + cmd: Some(vec!["while true; do sleep 3600; done".into()]), + ..Default::default() + } +} diff --git a/worker/worker-bridge-e2b/README/overview.md b/worker/worker-bridge-e2b/README/overview.md index a9c8b5d..23ad909 100644 --- a/worker/worker-bridge-e2b/README/overview.md +++ b/worker/worker-bridge-e2b/README/overview.md @@ -105,7 +105,9 @@ worker 在 hello 中声明以下四项能力: - 新建 terminal session 受 `WORKER_TERMINAL_MAX_ACTIVE_SESSIONS` 限制(`0` 表示不限),超出正数上限返回 `session_capacity_exceeded`;容量已满时已有 session 仍可执行。创建中、可用、销毁中和 E2B cleanup 进行中的 session 都计入容量;该限制只适用于当前 worker 进程。 - 创建中的 session 会阻塞后续调用,所有等待者共享创建结果。 - 某个命令超时会把 session 标记为待销毁;已有并发命令继续完成,最后一个调用退出后才销毁沙箱。 -- 空闲 session 到期后由 janitor 销毁;worker 正常退出时销毁所有仍管理的沙箱。 +- terminal sandbox 在通用 `onlyboxes.worker` metadata 上增加 `onlyboxes.session_id_hash` 与 `onlyboxes.schema_version`;worker 正常退出时保留尚未过期的 terminal sandbox,`pythonExec` 仍按次销毁。 +- worker 重连 Console 后先按 candidate metadata 精确查询 sandbox,再通过 E2B connect API 重新取得 envd access token;恢复完成前不接收命令,恢复不会改变 Console 保存的 lease。 +- 空闲 session 到期后由 janitor 销毁;E2B 账号中不属于本次 Console candidates 的 sandbox 不会被扫描或删除,由各自远端 timeout 回收。 每个命令由独立的 `/bin/bash -l -c` 进程执行,因此共享文件系统,但不共享 cwd、shell 变量或当前进程环境。 diff --git a/worker/worker-bridge-e2b/internal/e2b/client.go b/worker/worker-bridge-e2b/internal/e2b/client.go index 5b86cce..5dc1794 100644 --- a/worker/worker-bridge-e2b/internal/e2b/client.go +++ b/worker/worker-bridge-e2b/internal/e2b/client.go @@ -12,6 +12,7 @@ import ( "net/http" "net/url" "path/filepath" + "sort" "strconv" "strings" "time" @@ -56,6 +57,13 @@ type Sandbox struct { TrafficAccessToken string } +type SandboxInfo struct { + ID string `json:"sandboxID"` + State string `json:"state"` + EnvdVersion string `json:"envdVersion"` + Metadata map[string]string `json:"metadata"` +} + type CommandResult struct { Stdout string Stderr string @@ -133,6 +141,10 @@ func NewClient(cfg Config) (*Client, error) { } func (c *Client) Create(ctx context.Context, template string, timeoutSec int) (*Sandbox, error) { + return c.CreateWithMetadata(ctx, template, timeoutSec, map[string]string{"onlyboxes.worker": "worker-bridge-e2b"}) +} + +func (c *Client) CreateWithMetadata(ctx context.Context, template string, timeoutSec int, metadata map[string]string) (*Sandbox, error) { template = strings.TrimSpace(template) if template == "" { return nil, errors.New("E2B template is required") @@ -155,7 +167,7 @@ func (c *Client) Create(ctx context.Context, template string, timeoutSec int) (* AutoPause: false, Secure: true, AllowInternetAccess: true, - Metadata: map[string]string{"onlyboxes.worker": "worker-bridge-e2b"}, + Metadata: cloneMetadata(metadata), EnvVars: map[string]string{}, } if c.restrictPublicTraffic { @@ -187,6 +199,77 @@ func (c *Client) Create(ctx context.Context, template string, timeoutSec int) (* }, nil } +func (c *Client) List(ctx context.Context, metadata map[string]string) ([]SandboxInfo, error) { + query := url.Values{} + if len(metadata) > 0 { + keys := make([]string, 0, len(metadata)) + for key := range metadata { + keys = append(keys, key) + } + sort.Strings(keys) + parts := make([]string, 0, len(keys)) + for _, key := range keys { + parts = append(parts, url.QueryEscape(key)+"="+url.QueryEscape(metadata[key])) + } + query.Set("metadata", strings.Join(parts, "&")) + } + path := "/sandboxes" + if encoded := query.Encode(); encoded != "" { + path += "?" + encoded + } + var response []SandboxInfo + if err := c.controlJSON(ctx, http.MethodGet, path, nil, &response); err != nil { + return nil, err + } + return response, nil +} + +func (c *Client) Connect(ctx context.Context, sandboxID string, timeoutSec int) (*Sandbox, error) { + if timeoutSec <= 0 { + return nil, errors.New("E2B sandbox timeout must be positive") + } + var response struct { + SandboxID string `json:"sandboxID"` + EnvdVersion string `json:"envdVersion"` + EnvdAccessToken string `json:"envdAccessToken"` + TrafficAccessToken string `json:"trafficAccessToken"` + Domain *string `json:"domain"` + } + path := "/sandboxes/" + url.PathEscape(strings.TrimSpace(sandboxID)) + "/connect" + err := c.controlJSON(ctx, http.MethodPost, path, struct { + Timeout int `json:"timeout"` + }{Timeout: timeoutSec}, &response) + if isHTTPStatus(err, http.StatusNotFound) { + return nil, fmt.Errorf("%w: %v", ErrSandboxNotFound, err) + } + if err != nil { + return nil, err + } + domain := c.domain + if response.Domain != nil && strings.TrimSpace(*response.Domain) != "" { + domain = strings.TrimSpace(*response.Domain) + } + id := strings.TrimSpace(response.SandboxID) + if id == "" { + id = strings.TrimSpace(sandboxID) + } + return &Sandbox{ + ID: id, + Domain: domain, + EnvdVersion: response.EnvdVersion, + AccessToken: response.EnvdAccessToken, + TrafficAccessToken: response.TrafficAccessToken, + }, nil +} + +func cloneMetadata(metadata map[string]string) map[string]string { + out := make(map[string]string, len(metadata)) + for key, value := range metadata { + out[key] = value + } + return out +} + func (sandbox *Sandbox) ProxyURL(port int) (string, string, error) { if sandbox == nil || strings.TrimSpace(sandbox.ID) == "" { return "", "", errors.New("sandbox is required") diff --git a/worker/worker-bridge-e2b/internal/e2b/client_integration_test.go b/worker/worker-bridge-e2b/internal/e2b/client_integration_test.go index cffbbda..fa4fc4c 100644 --- a/worker/worker-bridge-e2b/internal/e2b/client_integration_test.go +++ b/worker/worker-bridge-e2b/internal/e2b/client_integration_test.go @@ -2,7 +2,10 @@ package e2b import ( "context" + "encoding/json" + "fmt" "os" + "os/exec" "strings" "testing" "time" @@ -62,3 +65,217 @@ func TestIntegrationSandboxCommandAndFile(t *testing.T) { t.Fatal(err) } } + +func TestIntegrationSandboxRecoveryByMetadata(t *testing.T) { + if os.Getenv("E2B_INTEGRATION") != "1" { + t.Skip("set E2B_INTEGRATION=1 to run against E2B") + } + apiKey := strings.TrimSpace(os.Getenv("E2B_API_KEY")) + template := strings.TrimSpace(os.Getenv("E2B_TERMINAL_TEMPLATE")) + if apiKey == "" || template == "" { + t.Fatal("E2B_API_KEY and E2B_TERMINAL_TEMPLATE are required") + } + client, err := NewClient(Config{ + APIKey: apiKey, + APIURL: strings.TrimSpace(os.Getenv("E2B_API_URL")), + Domain: strings.TrimSpace(os.Getenv("E2B_DOMAIN")), + RequestTimeout: 60 * time.Second, + }) + if err != nil { + t.Fatal(err) + } + ctx, cancel := context.WithTimeout(context.Background(), 3*time.Minute) + defer cancel() + marker := fmt.Sprintf("integration-%d", time.Now().UnixNano()) + metadata := map[string]string{ + "onlyboxes.session_id_hash": marker, + "onlyboxes.schema_version": "1", + } + sandbox, err := client.CreateWithMetadata(ctx, template, 120, metadata) + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { + cleanupCtx, cleanupCancel := context.WithTimeout(context.Background(), 30*time.Second) + defer cleanupCancel() + if err := client.Kill(cleanupCtx, sandbox.ID); err != nil { + t.Errorf("cleanup sandbox: %v", err) + } + }) + if _, err := client.Run(ctx, sandbox, `printf 'recovery-ok' > /tmp/onlyboxes-recovery.txt`, 1024); err != nil { + t.Fatal(err) + } + + infos, err := client.List(ctx, metadata) + if err != nil { + t.Fatal(err) + } + var found *SandboxInfo + for index := range infos { + if infos[index].ID == sandbox.ID { + found = &infos[index] + break + } + } + if found == nil || found.Metadata["onlyboxes.session_id_hash"] != marker { + t.Fatalf("created sandbox was not discoverable by metadata: %#v", infos) + } + + reconnected, err := client.Connect(ctx, sandbox.ID, 90) + if err != nil { + t.Fatal(err) + } + if reconnected.ID != sandbox.ID || strings.TrimSpace(reconnected.AccessToken) == "" { + t.Fatalf("connect did not return fresh sandbox credentials: %#v", reconnected) + } + result, err := client.Run(ctx, reconnected, `cat /tmp/onlyboxes-recovery.txt`, 1024) + if err != nil { + t.Fatal(err) + } + if result.ExitCode != 0 || result.Stdout != "recovery-ok" { + t.Fatalf("reconnected sandbox lost filesystem state: %#v", result) + } +} + +func TestIntegrationSandboxSurvivesForcedWorkerTermination(t *testing.T) { + if os.Getenv("E2B_INTEGRATION") != "1" { + t.Skip("set E2B_INTEGRATION=1 to run against E2B") + } + if os.Getenv("E2B_RECOVERY_CRASH_HELPER") == "1" { + runE2BRecoveryCrashHelper(t) + return + } + apiKey := strings.TrimSpace(os.Getenv("E2B_API_KEY")) + template := strings.TrimSpace(os.Getenv("E2B_TERMINAL_TEMPLATE")) + if apiKey == "" || template == "" { + t.Fatal("E2B_API_KEY and E2B_TERMINAL_TEMPLATE are required") + } + client, err := NewClient(Config{ + APIKey: apiKey, APIURL: strings.TrimSpace(os.Getenv("E2B_API_URL")), + Domain: strings.TrimSpace(os.Getenv("E2B_DOMAIN")), RequestTimeout: 60 * time.Second, + }) + if err != nil { + t.Fatal(err) + } + marker := fmt.Sprintf("forced-integration-%d", time.Now().UnixNano()) + metadata := map[string]string{ + "onlyboxes.worker": "worker-bridge-e2b", + "onlyboxes.session_id_hash": marker, + "onlyboxes.schema_version": "1", + } + readyFile, err := os.CreateTemp("", "onlyboxes-e2b-crash-ready-*") + if err != nil { + t.Fatal(err) + } + readyPath := readyFile.Name() + _ = readyFile.Close() + _ = os.Remove(readyPath) + t.Cleanup(func() { _ = os.Remove(readyPath) }) + metadataJSON, err := json.Marshal(metadata) + if err != nil { + t.Fatal(err) + } + cmd := exec.Command(os.Args[0], "-test.run=^TestIntegrationSandboxSurvivesForcedWorkerTermination$", "-test.v") + cmd.Env = append(os.Environ(), + "E2B_RECOVERY_CRASH_HELPER=1", + "E2B_RECOVERY_READY_FILE="+readyPath, + "E2B_RECOVERY_METADATA="+string(metadataJSON), + ) + cmd.Stdout = os.Stderr + cmd.Stderr = os.Stderr + if err := cmd.Start(); err != nil { + t.Fatal(err) + } + waitCh := make(chan error, 1) + go func() { waitCh <- cmd.Wait() }() + sandboxID := "" + deadline := time.NewTimer(3 * time.Minute) + defer deadline.Stop() + ticker := time.NewTicker(50 * time.Millisecond) + defer ticker.Stop() + for sandboxID == "" { + select { + case err := <-waitCh: + t.Fatalf("crash helper exited before creating sandbox: %v", err) + case <-deadline.C: + _ = cmd.Process.Kill() + <-waitCh + t.Fatal("timed out waiting for E2B crash helper") + case <-ticker.C: + content, readErr := os.ReadFile(readyPath) + if readErr == nil { + sandboxID = strings.TrimSpace(string(content)) + } + } + } + if err := cmd.Process.Kill(); err != nil { + t.Fatal(err) + } + if err := <-waitCh; err == nil { + t.Fatal("crash helper was not force terminated") + } + t.Cleanup(func() { + cleanupCtx, cancel := context.WithTimeout(context.Background(), 30*time.Second) + defer cancel() + if err := client.Kill(cleanupCtx, sandboxID); err != nil { + t.Errorf("cleanup sandbox: %v", err) + } + }) + + ctx, cancel := context.WithTimeout(context.Background(), 3*time.Minute) + defer cancel() + infos, err := client.List(ctx, metadata) + if err != nil { + t.Fatal(err) + } + found := false + for _, info := range infos { + if info.ID == sandboxID { + found = true + break + } + } + if !found { + t.Fatalf("sandbox %q was not discoverable after worker termination", sandboxID) + } + reconnected, err := client.Connect(ctx, sandboxID, 90) + if err != nil { + t.Fatal(err) + } + result, err := client.Run(ctx, reconnected, "cat /tmp/onlyboxes-forced-recovery.txt", 1024) + if err != nil { + t.Fatal(err) + } + if result.ExitCode != 0 || result.Stdout != "forced-recovery-ok" { + t.Fatalf("sandbox lost state after forced worker termination: %#v", result) + } +} + +func runE2BRecoveryCrashHelper(t *testing.T) { + apiKey := strings.TrimSpace(os.Getenv("E2B_API_KEY")) + template := strings.TrimSpace(os.Getenv("E2B_TERMINAL_TEMPLATE")) + metadata := map[string]string{} + if err := json.Unmarshal([]byte(os.Getenv("E2B_RECOVERY_METADATA")), &metadata); err != nil { + t.Fatal(err) + } + client, err := NewClient(Config{ + APIKey: apiKey, APIURL: strings.TrimSpace(os.Getenv("E2B_API_URL")), + Domain: strings.TrimSpace(os.Getenv("E2B_DOMAIN")), RequestTimeout: 60 * time.Second, + }) + if err != nil { + t.Fatal(err) + } + ctx, cancel := context.WithTimeout(context.Background(), 2*time.Minute) + defer cancel() + sandbox, err := client.CreateWithMetadata(ctx, template, 180, metadata) + if err != nil { + t.Fatal(err) + } + if _, err := client.Run(ctx, sandbox, "printf forced-recovery-ok > /tmp/onlyboxes-forced-recovery.txt", 1024); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(strings.TrimSpace(os.Getenv("E2B_RECOVERY_READY_FILE")), []byte(sandbox.ID), 0o600); err != nil { + t.Fatal(err) + } + select {} +} diff --git a/worker/worker-bridge-e2b/internal/e2b/client_test.go b/worker/worker-bridge-e2b/internal/e2b/client_test.go index 04e8924..55eaec3 100644 --- a/worker/worker-bridge-e2b/internal/e2b/client_test.go +++ b/worker/worker-bridge-e2b/internal/e2b/client_test.go @@ -223,6 +223,64 @@ func TestControlPlaneLifecycle(t *testing.T) { } } +func TestRecoveryControlPlaneMetadataListAndConnect(t *testing.T) { + t.Parallel() + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.Header.Get("X-API-Key") != "test-key" { + t.Errorf("missing API key") + } + switch { + case r.Method == http.MethodGet && r.URL.Path == "/sandboxes": + if got := r.URL.Query().Get("metadata"); got != "onlyboxes.schema_version=1&onlyboxes.session_id_hash=abc+123" { + t.Errorf("unexpected metadata query: %q", got) + } + _ = json.NewEncoder(w).Encode([]map[string]any{{ + "sandboxID": "sb-recover", + "state": "running", + "envdVersion": "0.6.2", + "metadata": map[string]string{ + "onlyboxes.schema_version": "1", + "onlyboxes.session_id_hash": "abc 123", + }, + }}) + case r.Method == http.MethodPost && r.URL.Path == "/sandboxes/sb-recover/connect": + var body map[string]any + if err := json.NewDecoder(r.Body).Decode(&body); err != nil || body["timeout"] != float64(90) { + t.Errorf("unexpected connect body: %#v err=%v", body, err) + } + _ = json.NewEncoder(w).Encode(map[string]any{ + "sandboxID": "sb-recover", + "envdVersion": "0.6.2", + "envdAccessToken": "fresh-token", + "trafficAccessToken": "fresh-traffic-token", + }) + default: + http.NotFound(w, r) + } + })) + defer server.Close() + + client, err := NewClient(Config{APIKey: "test-key", APIURL: server.URL, RequestTimeout: time.Second}) + if err != nil { + t.Fatal(err) + } + metadata := map[string]string{ + "onlyboxes.session_id_hash": "abc 123", + "onlyboxes.schema_version": "1", + } + infos, err := client.List(context.Background(), metadata) + if err != nil || len(infos) != 1 || infos[0].ID != "sb-recover" { + t.Fatalf("unexpected list result: infos=%#v err=%v", infos, err) + } + sandbox, err := client.Connect(context.Background(), "sb-recover", 90) + if err != nil { + t.Fatal(err) + } + if sandbox.ID != "sb-recover" || sandbox.AccessToken != "fresh-token" || sandbox.TrafficAccessToken != "fresh-traffic-token" { + t.Fatalf("unexpected connected sandbox: %#v", sandbox) + } +} + func TestSandboxProxyURL(t *testing.T) { sandbox := &Sandbox{ID: "sandbox-1", Domain: "e2b.app", TrafficAccessToken: "traffic-secret"} gotURL, gotToken, err := sandbox.ProxyURL(8080) diff --git a/worker/worker-bridge-e2b/internal/runner/console_session_test.go b/worker/worker-bridge-e2b/internal/runner/console_session_test.go index fba71f2..3b51583 100644 --- a/worker/worker-bridge-e2b/internal/runner/console_session_test.go +++ b/worker/worker-bridge-e2b/internal/runner/console_session_test.go @@ -18,6 +18,19 @@ import ( "google.golang.org/grpc/status" ) +func TestTerminalRecoveryTimesOutBeforeWorkerBecomesReady(t *testing.T) { + original := recoverTerminalSessionsFn + t.Cleanup(func() { recoverTerminalSessionsFn = original }) + recoverTerminalSessionsFn = func(ctx context.Context, _ []*registryv1.TerminalSessionRecoveryCandidate) []*registryv1.TerminalSessionRecoveryResult { + <-ctx.Done() + return nil + } + _, err := recoverTerminalSessionsWithTimeout(context.Background(), 10*time.Millisecond, nil) + if !errors.Is(err, context.DeadlineExceeded) { + t.Fatalf("expected recovery deadline, got %v", err) + } +} + func TestConsoleSessionContract(t *testing.T) { originalActiveSessionCount := activeSessionCountFn activeSessionCountFn = func() int32 { return 3 } @@ -239,6 +252,9 @@ func (s *allCapabilityContractService) Connect(stream grpc.BidiStreamingServer[r }); err != nil { return err } + if err := acceptEmptyRecovery(stream); err != nil { + return err + } firstHeartbeat, err := stream.Recv() if err != nil || firstHeartbeat.GetHeartbeat() == nil { return status.Error(codes.InvalidArgument, "heartbeat required") @@ -317,6 +333,9 @@ func (s *consoleContractService) Connect(stream grpc.BidiStreamingServer[registr }); err != nil { return err } + if err := acceptEmptyRecovery(stream); err != nil { + return err + } heartbeatFrame, err := stream.Recv() if err != nil { @@ -366,3 +385,18 @@ func (s *consoleContractService) Connect(stream grpc.BidiStreamingServer[registr } } } + +func acceptEmptyRecovery(stream grpc.BidiStreamingServer[registryv1.ConnectRequest, registryv1.ConnectResponse]) error { + req, err := stream.Recv() + if err != nil { + return err + } + if req.GetTerminalSessionRecoveryReport() == nil { + return status.Error(codes.InvalidArgument, "terminal session recovery report required") + } + return stream.Send(®istryv1.ConnectResponse{ + Payload: ®istryv1.ConnectResponse_TerminalSessionRecoveryAck{ + TerminalSessionRecoveryAck: ®istryv1.TerminalSessionRecoveryAck{}, + }, + }) +} diff --git a/worker/worker-bridge-e2b/internal/runner/e2b_backend.go b/worker/worker-bridge-e2b/internal/runner/e2b_backend.go index 1c2c74c..4d21a9c 100644 --- a/worker/worker-bridge-e2b/internal/runner/e2b_backend.go +++ b/worker/worker-bridge-e2b/internal/runner/e2b_backend.go @@ -14,3 +14,9 @@ type e2bBackend interface { ReadFile(context.Context, *e2b.Sandbox, string, int64) (e2b.File, error) OpenFile(context.Context, *e2b.Sandbox, string) (e2b.FileReader, error) } + +type e2bRecoveryBackend interface { + CreateWithMetadata(context.Context, string, int, map[string]string) (*e2b.Sandbox, error) + List(context.Context, map[string]string) ([]e2b.SandboxInfo, error) + Connect(context.Context, string, int) (*e2b.Sandbox, error) +} diff --git a/worker/worker-bridge-e2b/internal/runner/full_worker_integration_test.go b/worker/worker-bridge-e2b/internal/runner/full_worker_integration_test.go index d57f1bb..e48a236 100644 --- a/worker/worker-bridge-e2b/internal/runner/full_worker_integration_test.go +++ b/worker/worker-bridge-e2b/internal/runner/full_worker_integration_test.go @@ -106,6 +106,9 @@ func (s *liveAllCapabilityService) Connect(stream grpc.BidiStreamingServer[regis }); err != nil { return err } + if err := acceptEmptyRecovery(stream); err != nil { + return err + } if err := s.waitForHeartbeat(stream); err != nil { return err } diff --git a/worker/worker-bridge-e2b/internal/runner/runner.go b/worker/worker-bridge-e2b/internal/runner/runner.go index c994de9..32512d4 100644 --- a/worker/worker-bridge-e2b/internal/runner/runner.go +++ b/worker/worker-bridge-e2b/internal/runner/runner.go @@ -79,6 +79,7 @@ func Run(ctx context.Context, cfg config.Config) error { ExportMode: cfg.TerminalExportMode, SessionMaxInflight: cfg.TerminalSessionMaxInflight, MaxActiveSessions: cfg.TerminalMaxActiveSessions, + PreserveOnClose: true, }) pythonRunner := newPythonExecRunner( backend, @@ -95,12 +96,15 @@ func Run(ctx context.Context, cfg config.Config) error { runTerminalProxy = terminalManager.ResolveProxy originalActiveSessionCountFn := activeSessionCountFn activeSessionCountFn = terminalManager.ActiveSessionCount + originalRecoverTerminalSessionsFn := recoverTerminalSessionsFn + recoverTerminalSessionsFn = terminalManager.Recover defer func() { runPythonExec = originalRunPythonExec runTerminalExec = originalRunTerminalExec runTerminalResource = originalRunTerminalResource runTerminalProxy = originalRunTerminalProxy activeSessionCountFn = originalActiveSessionCountFn + recoverTerminalSessionsFn = originalRecoverTerminalSessionsFn terminalManager.Close() }() diff --git a/worker/worker-bridge-e2b/internal/runner/session_client.go b/worker/worker-bridge-e2b/internal/runner/session_client.go index 716aad2..c898c3f 100644 --- a/worker/worker-bridge-e2b/internal/runner/session_client.go +++ b/worker/worker-bridge-e2b/internal/runner/session_client.go @@ -65,6 +65,25 @@ func runSessionWithStatus(ctx context.Context, cfg config.Config) (bool, error) } heartbeatInterval := durationFromServer(ack.GetHeartbeatIntervalSec(), cfg.HeartbeatInterval) + recoveryResults, err := recoverTerminalSessionsWithTimeout(ctx, cfg.CallTimeout, ack.GetTerminalSessionRecoveryCandidates()) + if err != nil { + return true, fmt.Errorf("recover terminal sessions: %w", err) + } + if err := stream.Send(®istryv1.ConnectRequest{ + Payload: ®istryv1.ConnectRequest_TerminalSessionRecoveryReport{ + TerminalSessionRecoveryReport: ®istryv1.TerminalSessionRecoveryReport{Results: recoveryResults}, + }, + }); err != nil { + return true, fmt.Errorf("send terminal session recovery report: %w", err) + } + recoveryResp, err := recvWithTimeout(ctx, cfg.CallTimeout, stream.Recv) + if err != nil { + return true, fmt.Errorf("recv terminal session recovery ack: %w", err) + } + if recoveryResp.GetTerminalSessionRecoveryAck() == nil { + return true, fmt.Errorf("unexpected response while waiting for terminal session recovery ack") + } + logTerminalRecoverySummary(recoveryResults) logging.Infof("worker connected: node_id=%s node_name=%s session_id=%s", hello.GetNodeId(), hello.GetNodeName(), sessionID) sessionCtx, cancel := context.WithCancel(ctx) @@ -80,6 +99,42 @@ func runSessionWithStatus(ctx context.Context, cfg config.Config) (bool, error) return true, heartbeatLoop(sessionCtx, outbound, heartbeatAckCh, sessionErrCh, cfg, sessionID, heartbeatInterval) } +func recoverTerminalSessionsWithTimeout( + ctx context.Context, + timeout time.Duration, + candidates []*registryv1.TerminalSessionRecoveryCandidate, +) ([]*registryv1.TerminalSessionRecoveryResult, error) { + recoveryCtx, cancel := context.WithTimeout(ctx, timeout) + defer cancel() + resultCh := make(chan []*registryv1.TerminalSessionRecoveryResult, 1) + recoverTerminalSessions := recoverTerminalSessionsFn + go func() { + resultCh <- recoverTerminalSessions(recoveryCtx, candidates) + }() + select { + case <-recoveryCtx.Done(): + return nil, recoveryCtx.Err() + case results := <-resultCh: + return results, nil + } +} + +func logTerminalRecoverySummary(results []*registryv1.TerminalSessionRecoveryResult) { + counts := map[registryv1.TerminalSessionRecoveryResult_Status]int{} + for _, result := range results { + if result != nil { + counts[result.GetStatus()]++ + } + } + logging.Infof( + "terminal session recovery completed: candidates=%d recovered=%d missing=%d invalid=%d", + len(results), + counts[registryv1.TerminalSessionRecoveryResult_RECOVERED], + counts[registryv1.TerminalSessionRecoveryResult_MISSING], + counts[registryv1.TerminalSessionRecoveryResult_INVALID], + ) +} + func dial(ctx context.Context, cfg config.Config) (*grpc.ClientConn, error) { if err := ctx.Err(); err != nil { return nil, err diff --git a/worker/worker-bridge-e2b/internal/runner/session_recovery.go b/worker/worker-bridge-e2b/internal/runner/session_recovery.go new file mode 100644 index 0000000..c47aab9 --- /dev/null +++ b/worker/worker-bridge-e2b/internal/runner/session_recovery.go @@ -0,0 +1,159 @@ +package runner + +import ( + "context" + "errors" + "strings" + "time" + + registryv1 "github.com/onlyboxes/onlyboxes/api/gen/go/registry/v1" + "github.com/onlyboxes/onlyboxes/worker/worker-bridge-e2b/internal/e2b" + "github.com/onlyboxes/onlyboxes/worker/worker-bridge-e2b/internal/logging" +) + +var recoverTerminalSessionsFn = func( + _ context.Context, + candidates []*registryv1.TerminalSessionRecoveryCandidate, +) []*registryv1.TerminalSessionRecoveryResult { + return missingTerminalRecoveryResults(candidates) +} + +func missingTerminalRecoveryResults(candidates []*registryv1.TerminalSessionRecoveryCandidate) []*registryv1.TerminalSessionRecoveryResult { + results := make([]*registryv1.TerminalSessionRecoveryResult, 0, len(candidates)) + for _, candidate := range candidates { + if candidate == nil { + continue + } + results = append(results, ®istryv1.TerminalSessionRecoveryResult{ + SessionId: candidate.GetSessionId(), + Status: registryv1.TerminalSessionRecoveryResult_MISSING, + }) + } + return results +} + +func (m *terminalSessionManager) Recover( + ctx context.Context, + candidates []*registryv1.TerminalSessionRecoveryCandidate, +) []*registryv1.TerminalSessionRecoveryResult { + startedAt := time.Now() + recoveryBackend, ok := m.backend.(e2bRecoveryBackend) + if !ok { + return missingTerminalRecoveryResults(candidates) + } + results := make([]*registryv1.TerminalSessionRecoveryResult, 0, len(candidates)) + discovered := 0 + now := time.Now() + for _, candidate := range candidates { + if candidate == nil { + continue + } + sessionID := strings.TrimSpace(candidate.GetSessionId()) + leaseExpiresAt := time.UnixMilli(candidate.GetLeaseExpiresUnixMs()) + status := registryv1.TerminalSessionRecoveryResult_INVALID + if sessionID != "" && candidate.GetLeaseExpiresUnixMs() > 0 && leaseExpiresAt.After(now) { + var found int + status, found = m.recoverOne(ctx, recoveryBackend, sessionID, leaseExpiresAt) + discovered += found + } + results = append(results, ®istryv1.TerminalSessionRecoveryResult{SessionId: sessionID, Status: status}) + } + counts := map[registryv1.TerminalSessionRecoveryResult_Status]int{} + for _, result := range results { + counts[result.GetStatus()]++ + } + logging.Infof( + "terminal recovery completed: executor_kind=e2b discovered=%d candidate=%d recovered=%d missing=%d invalid=%d orphan_cleaned=0 duration_ms=%d recovery_failures=%d", + discovered, + len(results), + counts[registryv1.TerminalSessionRecoveryResult_RECOVERED], + counts[registryv1.TerminalSessionRecoveryResult_MISSING], + counts[registryv1.TerminalSessionRecoveryResult_INVALID], + time.Since(startedAt).Milliseconds(), + counts[registryv1.TerminalSessionRecoveryResult_INVALID], + ) + return results +} + +func (m *terminalSessionManager) recoverOne( + ctx context.Context, + backend e2bRecoveryBackend, + sessionID string, + leaseExpiresAt time.Time, +) (registryv1.TerminalSessionRecoveryResult_Status, int) { + m.mu.Lock() + if m.closed { + m.mu.Unlock() + return registryv1.TerminalSessionRecoveryResult_INVALID, 0 + } + if existing := m.sessions[sessionID]; existing != nil && !existing.destroying { + existing.desiredLeaseExpiresAt = leaseExpiresAt + existing.confirmedLeaseExpiresAt = leaseExpiresAt + existing.remoteTimeoutExpiresAt = leaseExpiresAt + m.mu.Unlock() + return registryv1.TerminalSessionRecoveryResult_RECOVERED, 1 + } + m.mu.Unlock() + + infos, err := backend.List(ctx, terminalSessionMetadata(sessionID)) + if err != nil { + return registryv1.TerminalSessionRecoveryResult_INVALID, 0 + } + if len(infos) == 0 { + return registryv1.TerminalSessionRecoveryResult_MISSING, 0 + } + matches := make([]e2b.SandboxInfo, 0, len(infos)) + for _, info := range infos { + if info.Metadata[terminalSessionMetadataKey] == terminalSessionIDHash(sessionID) && + info.Metadata[terminalSessionSchemaKey] == terminalSessionSchemaVersion && + info.Metadata[terminalSessionWorkerMetadataKey] == terminalSessionWorkerMetadata { + matches = append(matches, info) + } + } + if len(matches) == 0 { + return registryv1.TerminalSessionRecoveryResult_INVALID, len(infos) + } + if len(matches) != 1 { + for _, info := range matches { + if strings.TrimSpace(info.ID) != "" { + _ = m.backend.Kill(ctx, info.ID) + } + } + return registryv1.TerminalSessionRecoveryResult_INVALID, len(matches) + } + sandbox, err := backend.Connect(ctx, matches[0].ID, secondsUntil(leaseExpiresAt)) + if errors.Is(err, e2b.ErrSandboxNotFound) { + return registryv1.TerminalSessionRecoveryResult_MISSING, 1 + } + if err != nil || sandbox == nil || strings.TrimSpace(sandbox.ID) == "" { + return registryv1.TerminalSessionRecoveryResult_INVALID, 1 + } + + ready := make(chan struct{}) + close(ready) + m.mu.Lock() + defer m.mu.Unlock() + if m.closed { + return registryv1.TerminalSessionRecoveryResult_INVALID, 1 + } + if existing := m.sessions[sessionID]; existing != nil { + if existing.sandbox == nil || existing.sandbox.ID != sandbox.ID || existing.destroying { + return registryv1.TerminalSessionRecoveryResult_INVALID, 1 + } + existing.desiredLeaseExpiresAt = leaseExpiresAt + existing.confirmedLeaseExpiresAt = leaseExpiresAt + existing.remoteTimeoutExpiresAt = leaseExpiresAt + return registryv1.TerminalSessionRecoveryResult_RECOVERED, 1 + } + m.sessions[sessionID] = &terminalSession{ + sessionID: sessionID, + sandbox: sandbox, + desiredLeaseExpiresAt: leaseExpiresAt, + confirmedLeaseExpiresAt: leaseExpiresAt, + remoteTimeoutExpiresAt: leaseExpiresAt, + capacityReserved: true, + ready: ready, + } + m.activeSessionReservations++ + return registryv1.TerminalSessionRecoveryResult_RECOVERED, 1 +} diff --git a/worker/worker-bridge-e2b/internal/runner/session_recovery_test.go b/worker/worker-bridge-e2b/internal/runner/session_recovery_test.go new file mode 100644 index 0000000..50b9504 --- /dev/null +++ b/worker/worker-bridge-e2b/internal/runner/session_recovery_test.go @@ -0,0 +1,253 @@ +package runner + +import ( + "context" + "strings" + "testing" + "time" + + registryv1 "github.com/onlyboxes/onlyboxes/api/gen/go/registry/v1" + "github.com/onlyboxes/onlyboxes/worker/worker-bridge-e2b/internal/e2b" +) + +func TestE2BTerminalRecoveryMetadataUsesFullHashWithoutSessionID(t *testing.T) { + sessionID := "owner-secret:session-secret" + metadata := terminalSessionMetadata(sessionID) + hash := metadata[terminalSessionMetadataKey] + if len(hash) != 64 || strings.Contains(hash, "owner-secret") || hash != terminalSessionIDHash(sessionID) { + t.Fatalf("metadata exposed or did not deterministically map the session: %#v", metadata) + } + if metadata[terminalSessionSchemaKey] != terminalSessionSchemaVersion || + metadata[terminalSessionWorkerMetadataKey] != terminalSessionWorkerMetadata { + t.Fatalf("metadata markers missing: %#v", metadata) + } +} + +func TestE2BTerminalRecoveryUsesMetadataAndFreshConnection(t *testing.T) { + sessionID := "owner-a:session-a" + wantMetadata := terminalSessionMetadata(sessionID) + var listedMetadata map[string]string + var connectedID string + var connectedTimeout int + backend := &fakeE2BBackend{ + listFn: func(_ context.Context, metadata map[string]string) ([]e2b.SandboxInfo, error) { + listedMetadata = metadata + return []e2b.SandboxInfo{{ + ID: "sandbox-a", + State: "running", + Metadata: wantMetadata, + }}, nil + }, + connectFn: func(_ context.Context, sandboxID string, timeout int) (*e2b.Sandbox, error) { + connectedID = sandboxID + connectedTimeout = timeout + return &e2b.Sandbox{ID: sandboxID, Domain: "e2b.app", AccessToken: "fresh-token"}, nil + }, + } + manager := newTestTerminalManager(backend, 1) + manager.preserveOnClose = true + defer manager.Close() + + lease := time.Now().Add(10 * time.Minute).Truncate(time.Millisecond) + results := manager.Recover(context.Background(), []*registryv1.TerminalSessionRecoveryCandidate{ + {SessionId: sessionID, LeaseExpiresUnixMs: lease.UnixMilli()}, + }) + if len(results) != 1 || results[0].GetStatus() != registryv1.TerminalSessionRecoveryResult_RECOVERED { + t.Fatalf("unexpected recovery result: %#v", results) + } + if listedMetadata[terminalSessionMetadataKey] != wantMetadata[terminalSessionMetadataKey] || + listedMetadata[terminalSessionSchemaKey] != terminalSessionSchemaVersion { + t.Fatalf("unexpected metadata filter: %#v", listedMetadata) + } + if connectedID != "sandbox-a" || connectedTimeout < 599 || connectedTimeout > 601 { + t.Fatalf("unexpected reconnect: id=%q timeout=%d", connectedID, connectedTimeout) + } + manager.mu.Lock() + recovered := manager.sessions[sessionID] + manager.mu.Unlock() + if recovered == nil || recovered.sandbox.AccessToken != "fresh-token" || + !recovered.confirmedLeaseExpiresAt.Equal(lease) || recovered.inflight != 0 { + t.Fatalf("unexpected recovered session: %#v", recovered) + } +} + +func TestE2BTerminalRecoveryRejectsDuplicateMetadataMatches(t *testing.T) { + sessionID := "owner-a:session-duplicate" + metadata := terminalSessionMetadata(sessionID) + connectCalled := false + killed := map[string]bool{} + backend := &fakeE2BBackend{ + listFn: func(_ context.Context, _ map[string]string) ([]e2b.SandboxInfo, error) { + return []e2b.SandboxInfo{ + {ID: "sandbox-a", Metadata: metadata}, + {ID: "sandbox-b", Metadata: metadata}, + }, nil + }, + connectFn: func(_ context.Context, sandboxID string, timeout int) (*e2b.Sandbox, error) { + connectCalled = true + return nil, nil + }, + killFn: func(_ context.Context, sandboxID string) error { + killed[sandboxID] = true + return nil + }, + } + manager := newTestTerminalManager(backend, 1) + defer manager.Close() + + results := manager.Recover(context.Background(), []*registryv1.TerminalSessionRecoveryCandidate{ + {SessionId: sessionID, LeaseExpiresUnixMs: time.Now().Add(time.Minute).UnixMilli()}, + }) + if len(results) != 1 || results[0].GetStatus() != registryv1.TerminalSessionRecoveryResult_INVALID { + t.Fatalf("unexpected recovery result: %#v", results) + } + if connectCalled { + t.Fatal("duplicate metadata matches must not reconnect to an arbitrary sandbox") + } + if !killed["sandbox-a"] || !killed["sandbox-b"] { + t.Fatalf("duplicate sandboxes were not isolated: %#v", killed) + } +} + +func TestE2BTerminalRecoveryNeverKillsNonCandidateSandbox(t *testing.T) { + sessionID := "owner-a:session-safe-filter" + killCalled := false + backend := &fakeE2BBackend{ + listFn: func(_ context.Context, _ map[string]string) ([]e2b.SandboxInfo, error) { + return []e2b.SandboxInfo{{ + ID: "unrelated-sandbox", + Metadata: map[string]string{ + terminalSessionMetadataKey: "different-hash", + terminalSessionSchemaKey: terminalSessionSchemaVersion, + }, + }}, nil + }, + killFn: func(_ context.Context, _ string) error { + killCalled = true + return nil + }, + } + manager := newTestTerminalManager(backend, 1) + defer manager.Close() + results := manager.Recover(context.Background(), []*registryv1.TerminalSessionRecoveryCandidate{{ + SessionId: sessionID, LeaseExpiresUnixMs: time.Now().Add(time.Minute).UnixMilli(), + }}) + if len(results) != 1 || results[0].GetStatus() != registryv1.TerminalSessionRecoveryResult_INVALID { + t.Fatalf("unexpected recovery result: %#v", results) + } + if killCalled { + t.Fatal("non-candidate E2B sandbox was killed") + } +} + +func TestE2BTerminalCreationAddsRecoveryMetadata(t *testing.T) { + var metadata map[string]string + backend := &fakeE2BBackend{ + createWithMetadataFn: func(_ context.Context, _ string, _ int, values map[string]string) (*e2b.Sandbox, error) { + metadata = values + return &e2b.Sandbox{ID: "sandbox-created", Domain: "test"}, nil + }, + } + manager := newTestTerminalManager(backend, 1) + defer manager.Close() + + result, err := manager.Execute(context.Background(), terminalExecRequest{ + Command: "pwd", + SessionID: "owner-a:session-create", + CreateIfMissing: true, + LeaseTTLSec: intPointer(10), + }) + if err != nil { + t.Fatalf("create terminal session: %v", err) + } + if !result.Created || metadata[terminalSessionWorkerMetadataKey] != terminalSessionWorkerMetadata || + metadata[terminalSessionMetadataKey] != terminalSessionIDHash("owner-a:session-create") || + metadata[terminalSessionSchemaKey] != terminalSessionSchemaVersion { + t.Fatalf("unexpected recovery metadata: %#v", metadata) + } +} + +func TestE2BTerminalRecoveryIsIdempotent(t *testing.T) { + sessionID := "owner-a:session-idempotent" + metadata := terminalSessionMetadata(sessionID) + listCalls := 0 + connectCalls := 0 + backend := &fakeE2BBackend{ + listFn: func(_ context.Context, _ map[string]string) ([]e2b.SandboxInfo, error) { + listCalls++ + return []e2b.SandboxInfo{{ID: "sandbox-a", Metadata: metadata}}, nil + }, + connectFn: func(_ context.Context, sandboxID string, _ int) (*e2b.Sandbox, error) { + connectCalls++ + return &e2b.Sandbox{ID: sandboxID, AccessToken: "fresh-token"}, nil + }, + } + manager := newTestTerminalManager(backend, 1) + defer manager.Close() + lease := time.Now().Add(time.Minute).Truncate(time.Millisecond) + candidates := []*registryv1.TerminalSessionRecoveryCandidate{{SessionId: sessionID, LeaseExpiresUnixMs: lease.UnixMilli()}} + for range 2 { + results := manager.Recover(context.Background(), candidates) + if len(results) != 1 || results[0].GetStatus() != registryv1.TerminalSessionRecoveryResult_RECOVERED { + t.Fatalf("unexpected recovery result: %#v", results) + } + } + if listCalls != 1 || connectCalls != 1 || manager.ActiveSessionCount() != 1 { + t.Fatalf("recovery was not idempotent: list=%d connect=%d active=%d", listCalls, connectCalls, manager.ActiveSessionCount()) + } +} + +func TestE2BTerminalRecoveryReportsMissingAndSkipsExpiredCandidate(t *testing.T) { + listCalls := 0 + backend := &fakeE2BBackend{ + listFn: func(_ context.Context, _ map[string]string) ([]e2b.SandboxInfo, error) { + listCalls++ + return nil, nil + }, + } + manager := newTestTerminalManager(backend, 1) + defer manager.Close() + results := manager.Recover(context.Background(), []*registryv1.TerminalSessionRecoveryCandidate{ + {SessionId: "owner-a:session-missing", LeaseExpiresUnixMs: time.Now().Add(time.Minute).UnixMilli()}, + {SessionId: "owner-a:session-expired", LeaseExpiresUnixMs: time.Now().Add(-time.Minute).UnixMilli()}, + }) + if results[0].GetStatus() != registryv1.TerminalSessionRecoveryResult_MISSING || + results[1].GetStatus() != registryv1.TerminalSessionRecoveryResult_INVALID { + t.Fatalf("unexpected recovery results: %#v", results) + } + if listCalls != 1 || manager.ActiveSessionCount() != 0 { + t.Fatalf("expired candidate contacted E2B or consumed capacity: list=%d active=%d", listCalls, manager.ActiveSessionCount()) + } +} + +func TestConcurrentE2BTerminalRecoveryUsesOneReservation(t *testing.T) { + sessionID := "owner-a:session-concurrent" + metadata := terminalSessionMetadata(sessionID) + backend := &fakeE2BBackend{ + listFn: func(_ context.Context, _ map[string]string) ([]e2b.SandboxInfo, error) { + return []e2b.SandboxInfo{{ID: "sandbox-a", Metadata: metadata}}, nil + }, + connectFn: func(_ context.Context, sandboxID string, _ int) (*e2b.Sandbox, error) { + return &e2b.Sandbox{ID: sandboxID, AccessToken: "fresh-token"}, nil + }, + } + manager := newTestTerminalManager(backend, 1) + defer manager.Close() + candidate := []*registryv1.TerminalSessionRecoveryCandidate{{ + SessionId: sessionID, LeaseExpiresUnixMs: time.Now().Add(time.Minute).UnixMilli(), + }} + done := make(chan []*registryv1.TerminalSessionRecoveryResult, 2) + for range 2 { + go func() { done <- manager.Recover(context.Background(), candidate) }() + } + for range 2 { + if result := <-done; len(result) != 1 || result[0].GetStatus() != registryv1.TerminalSessionRecoveryResult_RECOVERED { + t.Fatalf("unexpected concurrent recovery result: %#v", result) + } + } + if manager.ActiveSessionCount() != 1 { + t.Fatalf("concurrent recovery reserved %d sessions, want 1", manager.ActiveSessionCount()) + } +} + +func intPointer(value int) *int { return &value } diff --git a/worker/worker-bridge-e2b/internal/runner/terminal_exec.go b/worker/worker-bridge-e2b/internal/runner/terminal_exec.go index f014d35..55a2dcb 100644 --- a/worker/worker-bridge-e2b/internal/runner/terminal_exec.go +++ b/worker/worker-bridge-e2b/internal/runner/terminal_exec.go @@ -2,6 +2,8 @@ package runner import ( "context" + "crypto/sha256" + "encoding/hex" "errors" "fmt" "strings" @@ -27,6 +29,11 @@ const ( defaultTerminalLeaseSec = 300 defaultTerminalOutputLimitBytes = 1024 * 1024 defaultTerminalSessionMaxInflight = 128 + terminalSessionWorkerMetadataKey = "onlyboxes.worker" + terminalSessionWorkerMetadata = "worker-bridge-e2b" + terminalSessionMetadataKey = "onlyboxes.session_id_hash" + terminalSessionSchemaKey = "onlyboxes.schema_version" + terminalSessionSchemaVersion = "1" ) const ( @@ -114,6 +121,7 @@ type terminalSessionManagerConfig struct { ExportMode string SessionMaxInflight int MaxActiveSessions int + PreserveOnClose bool // JanitorInterval is test-only tuning in practice; zero selects the // production interval. JanitorInterval time.Duration @@ -138,6 +146,7 @@ type terminalSessionManager struct { maxActiveSessions int activeSessionReservations int janitorInterval time.Duration + preserveOnClose bool stopCh chan struct{} doneCh chan struct{} @@ -190,6 +199,7 @@ func newTerminalSessionManager(cfg terminalSessionManagerConfig) *terminalSessio sessionMaxInflight: maxInflight, maxActiveSessions: maxActiveSessions, janitorInterval: janitorInterval, + preserveOnClose: cfg.PreserveOnClose, stopCh: make(chan struct{}), doneCh: make(chan struct{}), } @@ -429,7 +439,13 @@ func (m *terminalSessionManager) awaitSessionReady(ctx context.Context, session leaseExpiresAt := session.desiredLeaseExpiresAt m.mu.Unlock() timeout := secondsUntil(leaseExpiresAt) - sandbox, err := m.backend.Create(ctx, m.template, timeout) + var sandbox *e2b.Sandbox + var err error + if recoveryBackend, ok := m.backend.(e2bRecoveryBackend); ok { + sandbox, err = recoveryBackend.CreateWithMetadata(ctx, m.template, timeout, terminalSessionMetadata(session.sessionID)) + } else { + sandbox, err = m.backend.Create(ctx, m.template, timeout) + } m.mu.Lock() session.sandbox = sandbox session.initErr = err @@ -632,11 +648,32 @@ func (m *terminalSessionManager) Close() { m.mu.Unlock() m.cleanupWG.Wait() for _, session := range sessions { + if m.preserveOnClose { + m.mu.Lock() + if session.capacityReserved && m.activeSessionReservations > 0 { + m.activeSessionReservations-- + } + m.mu.Unlock() + continue + } m.cleanupRetiredSession(session) } }) } +func terminalSessionIDHash(sessionID string) string { + sum := sha256.Sum256([]byte(strings.TrimSpace(sessionID))) + return hex.EncodeToString(sum[:]) +} + +func terminalSessionMetadata(sessionID string) map[string]string { + return map[string]string{ + terminalSessionWorkerMetadataKey: terminalSessionWorkerMetadata, + terminalSessionMetadataKey: terminalSessionIDHash(sessionID), + terminalSessionSchemaKey: terminalSessionSchemaVersion, + } +} + func (m *terminalSessionManager) killSandbox(sandbox *e2b.Sandbox) { if sandbox == nil { return diff --git a/worker/worker-bridge-e2b/internal/runner/terminal_session_integration_test.go b/worker/worker-bridge-e2b/internal/runner/terminal_session_integration_test.go index 4f8358a..e429aee 100644 --- a/worker/worker-bridge-e2b/internal/runner/terminal_session_integration_test.go +++ b/worker/worker-bridge-e2b/internal/runner/terminal_session_integration_test.go @@ -3,6 +3,7 @@ package runner import ( "context" "errors" + "fmt" "io" "net/http" "net/http/httptest" @@ -12,6 +13,7 @@ import ( "testing" "time" + registryv1 "github.com/onlyboxes/onlyboxes/api/gen/go/registry/v1" "github.com/onlyboxes/onlyboxes/worker/worker-bridge-e2b/internal/e2b" ) @@ -29,6 +31,26 @@ func (b *recordingE2BBackend) Create(ctx context.Context, template string, timeo return b.e2bBackend.Create(ctx, template, timeoutSec) } +func (b *recordingE2BBackend) CreateWithMetadata( + ctx context.Context, + template string, + timeoutSec int, + metadata map[string]string, +) (*e2b.Sandbox, error) { + b.mu.Lock() + b.createCalls++ + b.mu.Unlock() + return b.e2bBackend.(e2bRecoveryBackend).CreateWithMetadata(ctx, template, timeoutSec, metadata) +} + +func (b *recordingE2BBackend) List(ctx context.Context, metadata map[string]string) ([]e2b.SandboxInfo, error) { + return b.e2bBackend.(e2bRecoveryBackend).List(ctx, metadata) +} + +func (b *recordingE2BBackend) Connect(ctx context.Context, sandboxID string, timeoutSec int) (*e2b.Sandbox, error) { + return b.e2bBackend.(e2bRecoveryBackend).Connect(ctx, sandboxID, timeoutSec) +} + func (b *recordingE2BBackend) createCount() int { b.mu.Lock() defer b.mu.Unlock() @@ -335,6 +357,104 @@ func TestIntegrationJanitorExpiresTerminalSession(t *testing.T) { } } +func TestIntegrationTerminalSessionRecoversAcrossManagerRestart(t *testing.T) { + backend, template := liveTerminalBackend(t) + manager := newTerminalSessionManager(terminalSessionManagerConfig{ + Backend: backend, + Template: template, + LeaseMinSec: 1, + LeaseMaxSec: 120, + LeaseDefaultSec: 60, + OutputLimitBytes: 1024, + SessionMaxInflight: 1, + MaxActiveSessions: 1, + PreserveOnClose: true, + }) + ctx, cancel := context.WithTimeout(context.Background(), 3*time.Minute) + defer cancel() + sessionID := fmt.Sprintf("integration:e2b-manager-recovery-%d", time.Now().UnixNano()) + leaseTTL := 90 + created, err := manager.Execute(ctx, terminalExecRequest{ + Command: "printf manager-recovery-ok > /tmp/onlyboxes-manager-recovery.txt", + SessionID: sessionID, + CreateIfMissing: true, + LeaseTTLSec: &leaseTTL, + }) + if err != nil { + manager.Close() + t.Fatal(err) + } + if !created.Created || created.SessionID != sessionID { + manager.Close() + t.Fatalf("unexpected created session: %#v", created) + } + manager.mu.Lock() + sandboxID := manager.sessions[sessionID].sandbox.ID + manager.mu.Unlock() + t.Cleanup(func() { + cleanupCtx, cleanupCancel := context.WithTimeout(context.Background(), 30*time.Second) + defer cleanupCancel() + _ = backend.Kill(cleanupCtx, sandboxID) + }) + + manager.Close() + if backend.killCount() != 0 { + t.Fatalf("preserving manager killed sandbox during restart: kills=%d", backend.killCount()) + } + + recoveredManager := newTerminalSessionManager(terminalSessionManagerConfig{ + Backend: backend, + Template: template, + LeaseMinSec: 1, + LeaseMaxSec: 120, + LeaseDefaultSec: 60, + OutputLimitBytes: 1024, + SessionMaxInflight: 1, + MaxActiveSessions: 1, + }) + defer recoveredManager.Close() + results := recoveredManager.Recover(ctx, []*registryv1.TerminalSessionRecoveryCandidate{{ + SessionId: sessionID, + LeaseExpiresUnixMs: created.LeaseExpiresUnixMS, + }}) + if len(results) != 1 || results[0].GetStatus() != registryv1.TerminalSessionRecoveryResult_RECOVERED { + t.Fatalf("unexpected recovery results: %#v", results) + } + if recoveredManager.ActiveSessionCount() != 1 { + t.Fatalf("recovery did not restore capacity reservation: active=%d", recoveredManager.ActiveSessionCount()) + } + recoveredManager.mu.Lock() + recovered := recoveredManager.sessions[sessionID] + if recovered == nil { + recoveredManager.mu.Unlock() + t.Fatal("recovered session is not registered") + } + gotLeaseExpiresUnixMS := recovered.confirmedLeaseExpiresAt.UnixMilli() + gotSandboxID := recovered.sandbox.ID + gotInflight := recovered.inflight + recoveredManager.mu.Unlock() + if gotLeaseExpiresUnixMS != created.LeaseExpiresUnixMS { + t.Fatalf("recovery changed lease: got=%d want=%d", gotLeaseExpiresUnixMS, created.LeaseExpiresUnixMS) + } + if gotSandboxID != sandboxID || gotInflight != 0 { + t.Fatalf("unexpected recovered state: sandbox=%q inflight=%d", gotSandboxID, gotInflight) + } + + reused, err := recoveredManager.Execute(ctx, terminalExecRequest{ + Command: "cat /tmp/onlyboxes-manager-recovery.txt", + SessionID: sessionID, + }) + if err != nil { + t.Fatal(err) + } + if reused.Created || reused.Stdout != "manager-recovery-ok" || reused.SessionID != sessionID { + t.Fatalf("unexpected recovered execution: %#v", reused) + } + if backend.createCount() != 1 { + t.Fatalf("recovered execution created a second sandbox: creates=%d", backend.createCount()) + } +} + func liveTerminalBackend(t *testing.T) (*recordingE2BBackend, string) { t.Helper() if os.Getenv("E2B_INTEGRATION") != "1" { diff --git a/worker/worker-bridge-e2b/internal/runner/terminal_session_test.go b/worker/worker-bridge-e2b/internal/runner/terminal_session_test.go index 25c8e18..bad5394 100644 --- a/worker/worker-bridge-e2b/internal/runner/terminal_session_test.go +++ b/worker/worker-bridge-e2b/internal/runner/terminal_session_test.go @@ -20,19 +20,43 @@ import ( ) type fakeE2BBackend struct { - mu sync.Mutex - created int - killed int - killedIDs []string - timeouts []int - runStarted chan struct{} - runRelease chan struct{} - createFn func(context.Context, string, int) (*e2b.Sandbox, error) - timeoutFn func(context.Context, string, int) error - runFn func(context.Context, *e2b.Sandbox, string, int) (e2b.CommandResult, error) - readFn func(context.Context, *e2b.Sandbox, string, int64) (e2b.File, error) - openFn func(context.Context, *e2b.Sandbox, string) (e2b.FileReader, error) - killFn func(context.Context, string) error + mu sync.Mutex + created int + killed int + killedIDs []string + timeouts []int + runStarted chan struct{} + runRelease chan struct{} + createFn func(context.Context, string, int) (*e2b.Sandbox, error) + createWithMetadataFn func(context.Context, string, int, map[string]string) (*e2b.Sandbox, error) + listFn func(context.Context, map[string]string) ([]e2b.SandboxInfo, error) + connectFn func(context.Context, string, int) (*e2b.Sandbox, error) + timeoutFn func(context.Context, string, int) error + runFn func(context.Context, *e2b.Sandbox, string, int) (e2b.CommandResult, error) + readFn func(context.Context, *e2b.Sandbox, string, int64) (e2b.File, error) + openFn func(context.Context, *e2b.Sandbox, string) (e2b.FileReader, error) + killFn func(context.Context, string) error +} + +func (f *fakeE2BBackend) CreateWithMetadata(ctx context.Context, template string, timeout int, metadata map[string]string) (*e2b.Sandbox, error) { + if f.createWithMetadataFn != nil { + return f.createWithMetadataFn(ctx, template, timeout, metadata) + } + return f.Create(ctx, template, timeout) +} + +func (f *fakeE2BBackend) List(ctx context.Context, metadata map[string]string) ([]e2b.SandboxInfo, error) { + if f.listFn != nil { + return f.listFn(ctx, metadata) + } + return nil, nil +} + +func (f *fakeE2BBackend) Connect(ctx context.Context, sandboxID string, timeout int) (*e2b.Sandbox, error) { + if f.connectFn != nil { + return f.connectFn(ctx, sandboxID, timeout) + } + return &e2b.Sandbox{ID: sandboxID, Domain: "test", AccessToken: "fresh-token"}, nil } func (f *fakeE2BBackend) Create(ctx context.Context, template string, timeout int) (*e2b.Sandbox, error) { diff --git a/worker/worker-docker/README/overview.md b/worker/worker-docker/README/overview.md index bc65125..f01f49a 100644 --- a/worker/worker-docker/README/overview.md +++ b/worker/worker-docker/README/overview.md @@ -70,8 +70,9 @@ Capability behavior: - `terminalExec` cleanup behavior: - command timeout/cancel marks the session for destruction and stops it accepting new commands; the container is removed once in-flight commands drain, so one command's timeout does not kill its siblings. - lease expiry is a hard boundary: a per-session deadline timer removes the session even when commands are in flight, cancels active proxy requests, and force-removes the container; the janitor remains a cleanup fallback. - - worker shutdown force-removes all managed terminal containers. - - `SIGINT`/`SIGTERM` (for example Ctrl+C) performs best-effort cleanup; `SIGKILL`/process crash does not guarantee cleanup. + - terminal containers use deterministic `onlyboxes-terminal-v1-` names plus schema/session-hash labels and are preserved when the worker exits. + - after reconnect, the worker reconciles Console candidates before accepting commands, restarts stopped containers, restores the exact lease, and removes local Onlyboxes terminal orphans that Console no longer recognizes. + - lease expiry, explicit destruction, unsafe command timeout, and invalid resource identity still remove the container; one-shot `pythonExec` containers remain per-call resources. - `terminalExec` result uses JSON payload: - `{"session_id":"...","created":true,"stdout":"...","stderr":"...","exit_code":0,"stdout_truncated":false,"stderr_truncated":false,"lease_expires_unix_ms":...}` - output truncation: diff --git a/worker/worker-docker/internal/runner/runner.go b/worker/worker-docker/internal/runner/runner.go index 9d8e675..61f00f1 100644 --- a/worker/worker-docker/internal/runner/runner.go +++ b/worker/worker-docker/internal/runner/runner.go @@ -84,6 +84,7 @@ func Run(ctx context.Context, cfg config.Config) error { DockerNetwork: dockerNetwork, SessionMaxInflight: cfg.TerminalSessionMaxInflight, MaxActiveSessions: cfg.TerminalMaxActiveSessions, + PreserveOnClose: true, }) pythonRunner := newPythonExecRunner( cfg.PythonExecDockerImage, @@ -99,11 +100,14 @@ func Run(ctx context.Context, cfg config.Config) error { runTerminalResource = terminalManager.ResolveResource originalActiveSessionCountFn := activeSessionCountFn activeSessionCountFn = terminalManager.ActiveSessionCount + originalRecoverTerminalSessionsFn := recoverTerminalSessionsFn + recoverTerminalSessionsFn = terminalManager.Recover defer func() { runPythonExec = originalRunPythonExec runTerminalExec = originalRunTerminalExec runTerminalResource = originalRunTerminalResource activeSessionCountFn = originalActiveSessionCountFn + recoverTerminalSessionsFn = originalRecoverTerminalSessionsFn terminalManager.Close() }() diff --git a/worker/worker-docker/internal/runner/runner_test.go b/worker/worker-docker/internal/runner/runner_test.go index 679a511..0aa6272 100644 --- a/worker/worker-docker/internal/runner/runner_test.go +++ b/worker/worker-docker/internal/runner/runner_test.go @@ -30,6 +30,19 @@ func TestRunReturnsContextCanceled(t *testing.T) { } } +func TestTerminalRecoveryTimesOutBeforeWorkerBecomesReady(t *testing.T) { + original := recoverTerminalSessionsFn + t.Cleanup(func() { recoverTerminalSessionsFn = original }) + recoverTerminalSessionsFn = func(ctx context.Context, _ []*registryv1.TerminalSessionRecoveryCandidate) []*registryv1.TerminalSessionRecoveryResult { + <-ctx.Done() + return nil + } + _, err := recoverTerminalSessionsWithTimeout(context.Background(), 10*time.Millisecond, nil) + if !errors.Is(err, context.DeadlineExceeded) { + t.Fatalf("expected recovery deadline, got %v", err) + } +} + func TestRunWaitsBeforeReconnectOnSessionFailure(t *testing.T) { originalWaitReconnect := waitReconnect waitCalls := 0 @@ -984,6 +997,20 @@ func (s *fakeRegistryService) Connect(stream grpc.BidiStreamingServer[registryv1 }); err != nil { return err } + recoveryReq, err := stream.Recv() + if err != nil { + return err + } + if recoveryReq.GetTerminalSessionRecoveryReport() == nil { + return status.Error(codes.InvalidArgument, "terminal session recovery report is required") + } + if err := stream.Send(®istryv1.ConnectResponse{ + Payload: ®istryv1.ConnectResponse_TerminalSessionRecoveryAck{ + TerminalSessionRecoveryAck: ®istryv1.TerminalSessionRecoveryAck{}, + }, + }); err != nil { + return err + } for { req, err := stream.Recv() diff --git a/worker/worker-docker/internal/runner/session_client.go b/worker/worker-docker/internal/runner/session_client.go index d1d45d9..003ab43 100644 --- a/worker/worker-docker/internal/runner/session_client.go +++ b/worker/worker-docker/internal/runner/session_client.go @@ -60,6 +60,25 @@ func runSession(ctx context.Context, cfg config.Config) error { } heartbeatInterval := durationFromServer(ack.GetHeartbeatIntervalSec(), cfg.HeartbeatInterval) + recoveryResults, err := recoverTerminalSessionsWithTimeout(ctx, cfg.CallTimeout, ack.GetTerminalSessionRecoveryCandidates()) + if err != nil { + return fmt.Errorf("recover terminal sessions: %w", err) + } + if err := stream.Send(®istryv1.ConnectRequest{ + Payload: ®istryv1.ConnectRequest_TerminalSessionRecoveryReport{ + TerminalSessionRecoveryReport: ®istryv1.TerminalSessionRecoveryReport{Results: recoveryResults}, + }, + }); err != nil { + return fmt.Errorf("send terminal session recovery report: %w", err) + } + recoveryResp, err := recvWithTimeout(ctx, cfg.CallTimeout, stream.Recv) + if err != nil { + return fmt.Errorf("recv terminal session recovery ack: %w", err) + } + if recoveryResp.GetTerminalSessionRecoveryAck() == nil { + return fmt.Errorf("unexpected response while waiting for terminal session recovery ack") + } + logTerminalRecoverySummary(recoveryResults) logging.Infof("worker connected: node_id=%s node_name=%s session_id=%s", hello.GetNodeId(), hello.GetNodeName(), sessionID) sessionCtx, cancel := context.WithCancel(ctx) @@ -75,6 +94,41 @@ func runSession(ctx context.Context, cfg config.Config) error { return heartbeatLoop(sessionCtx, outbound, heartbeatAckCh, sessionErrCh, cfg, sessionID, heartbeatInterval) } +func recoverTerminalSessionsWithTimeout( + ctx context.Context, + timeout time.Duration, + candidates []*registryv1.TerminalSessionRecoveryCandidate, +) ([]*registryv1.TerminalSessionRecoveryResult, error) { + recoveryCtx, cancel := context.WithTimeout(ctx, timeout) + defer cancel() + resultCh := make(chan []*registryv1.TerminalSessionRecoveryResult, 1) + go func() { + resultCh <- recoverTerminalSessionsFn(recoveryCtx, candidates) + }() + select { + case <-recoveryCtx.Done(): + return nil, recoveryCtx.Err() + case results := <-resultCh: + return results, nil + } +} + +func logTerminalRecoverySummary(results []*registryv1.TerminalSessionRecoveryResult) { + counts := map[registryv1.TerminalSessionRecoveryResult_Status]int{} + for _, result := range results { + if result != nil { + counts[result.GetStatus()]++ + } + } + logging.Infof( + "terminal session recovery completed: candidates=%d recovered=%d missing=%d invalid=%d", + len(results), + counts[registryv1.TerminalSessionRecoveryResult_RECOVERED], + counts[registryv1.TerminalSessionRecoveryResult_MISSING], + counts[registryv1.TerminalSessionRecoveryResult_INVALID], + ) +} + func dial(ctx context.Context, cfg config.Config) (*grpc.ClientConn, error) { if err := ctx.Err(); err != nil { return nil, err diff --git a/worker/worker-docker/internal/runner/session_recovery.go b/worker/worker-docker/internal/runner/session_recovery.go new file mode 100644 index 0000000..e2454ce --- /dev/null +++ b/worker/worker-docker/internal/runner/session_recovery.go @@ -0,0 +1,209 @@ +package runner + +import ( + "context" + "encoding/json" + "strings" + "time" + + registryv1 "github.com/onlyboxes/onlyboxes/api/gen/go/registry/v1" + "github.com/onlyboxes/onlyboxes/worker/worker-docker/internal/logging" +) + +var recoverTerminalSessionsFn = func( + _ context.Context, + candidates []*registryv1.TerminalSessionRecoveryCandidate, +) []*registryv1.TerminalSessionRecoveryResult { + results := make([]*registryv1.TerminalSessionRecoveryResult, 0, len(candidates)) + for _, candidate := range candidates { + if candidate == nil { + continue + } + results = append(results, ®istryv1.TerminalSessionRecoveryResult{ + SessionId: candidate.GetSessionId(), + Status: registryv1.TerminalSessionRecoveryResult_MISSING, + }) + } + return results +} + +func (m *terminalSessionManager) Recover( + ctx context.Context, + candidates []*registryv1.TerminalSessionRecoveryCandidate, +) []*registryv1.TerminalSessionRecoveryResult { + startedAt := time.Now() + results := make([]*registryv1.TerminalSessionRecoveryResult, 0, len(candidates)) + expectedNames := make(map[string]struct{}, len(candidates)) + now := time.Now() + recoveredCount := 0 + missingCount := 0 + invalidCount := 0 + + for _, candidate := range candidates { + if candidate == nil { + continue + } + sessionID := strings.TrimSpace(candidate.GetSessionId()) + name := terminalSessionResourceName(sessionID) + status := registryv1.TerminalSessionRecoveryResult_INVALID + leaseExpiresAt := time.UnixMilli(candidate.GetLeaseExpiresUnixMs()) + if sessionID != "" && candidate.GetLeaseExpiresUnixMs() > 0 && leaseExpiresAt.After(now) { + status = m.recoverOne(ctx, sessionID, name, leaseExpiresAt) + } else if sessionID != "" { + m.forceRemoveContainer(name) + } + switch status { + case registryv1.TerminalSessionRecoveryResult_RECOVERED: + expectedNames[name] = struct{}{} + recoveredCount++ + case registryv1.TerminalSessionRecoveryResult_MISSING: + missingCount++ + default: + invalidCount++ + } + results = append(results, ®istryv1.TerminalSessionRecoveryResult{ + SessionId: sessionID, + Status: status, + }) + } + + discovered, orphanCleaned := m.cleanupOrphanContainers(ctx, expectedNames) + logging.Infof( + "terminal recovery completed: executor_kind=docker discovered=%d candidate=%d recovered=%d missing=%d invalid=%d orphan_cleaned=%d duration_ms=%d recovery_failures=%d", + discovered, len(results), recoveredCount, missingCount, invalidCount, orphanCleaned, + time.Since(startedAt).Milliseconds(), invalidCount, + ) + return results +} + +func (m *terminalSessionManager) recoverOne( + ctx context.Context, + sessionID string, + containerName string, + leaseExpiresAt time.Time, +) registryv1.TerminalSessionRecoveryResult_Status { + m.mu.Lock() + if m.closed { + m.mu.Unlock() + return registryv1.TerminalSessionRecoveryResult_INVALID + } + if existing := m.sessions[sessionID]; existing != nil && !existing.destroying { + existing.leaseExpiresAt = leaseExpiresAt + m.mu.Unlock() + return registryv1.TerminalSessionRecoveryResult_RECOVERED + } + m.mu.Unlock() + + inspect := runDockerCommand(ctx, + "inspect", + "--format", "{{json .Config.Labels}}\t{{.State.Status}}", + containerName, + ) + if inspect.Err != nil || inspect.ExitCode != 0 { + if inspect.Err == nil && isNoSuchContainerMessage(inspect.Stderr) { + return registryv1.TerminalSessionRecoveryResult_MISSING + } + return registryv1.TerminalSessionRecoveryResult_MISSING + } + parts := strings.SplitN(strings.TrimSpace(inspect.Stdout), "\t", 2) + if len(parts) != 2 { + return registryv1.TerminalSessionRecoveryResult_INVALID + } + labels := map[string]string{} + if err := json.Unmarshal([]byte(parts[0]), &labels); err != nil || + labels[terminalExecSessionLabelKey] != terminalSessionIDHash(sessionID) || + labels[terminalExecSchemaLabelKey] != terminalExecSchemaVersion { + m.forceRemoveContainer(containerName) + return registryv1.TerminalSessionRecoveryResult_INVALID + } + matching := runDockerCommand(ctx, + "ps", "-a", + "--filter", "label="+terminalExecSessionLabelKey+"="+terminalSessionIDHash(sessionID), + "--filter", "label="+terminalExecSchemaLabelKey+"="+terminalExecSchemaVersion, + "--format", "{{.Names}}", + ) + if matching.Err != nil || matching.ExitCode != 0 { + return registryv1.TerminalSessionRecoveryResult_INVALID + } + matchingNames := nonEmptyLines(matching.Stdout) + if len(matchingNames) != 1 || matchingNames[0] != containerName { + for _, name := range matchingNames { + m.forceRemoveContainer(name) + } + return registryv1.TerminalSessionRecoveryResult_INVALID + } + + switch strings.TrimSpace(strings.ToLower(parts[1])) { + case "running": + case "created", "exited", "stopped": + start := runDockerCommand(ctx, terminalExecDockerStartArgs(containerName)...) + if start.Err != nil || start.ExitCode != 0 { + return registryv1.TerminalSessionRecoveryResult_INVALID + } + default: + m.forceRemoveContainer(containerName) + return registryv1.TerminalSessionRecoveryResult_INVALID + } + + ready := make(chan struct{}) + close(ready) + m.mu.Lock() + defer m.mu.Unlock() + if m.closed { + return registryv1.TerminalSessionRecoveryResult_INVALID + } + if existing := m.sessions[sessionID]; existing != nil { + if existing.containerName != containerName || existing.destroying { + return registryv1.TerminalSessionRecoveryResult_INVALID + } + existing.leaseExpiresAt = leaseExpiresAt + return registryv1.TerminalSessionRecoveryResult_RECOVERED + } + m.sessions[sessionID] = &terminalSession{ + sessionID: sessionID, + containerName: containerName, + leaseExpiresAt: leaseExpiresAt, + ready: ready, + capacityReserved: true, + } + m.activeSessionReservations++ + return registryv1.TerminalSessionRecoveryResult_RECOVERED +} + +func nonEmptyLines(value string) []string { + lines := make([]string, 0) + for _, line := range strings.Split(value, "\n") { + if trimmed := strings.TrimSpace(line); trimmed != "" { + lines = append(lines, trimmed) + } + } + return lines +} + +func (m *terminalSessionManager) cleanupOrphanContainers(ctx context.Context, expectedNames map[string]struct{}) (int, int) { + listed := runDockerCommand(ctx, + "ps", "-a", + "--filter", "label="+terminalExecSessionLabelKey, + "--filter", "label="+terminalExecSchemaLabelKey+"="+terminalExecSchemaVersion, + "--format", "{{.Names}}", + ) + if listed.Err != nil || listed.ExitCode != 0 { + logging.Warnf("terminal recovery orphan discovery failed") + return 0, 0 + } + discovered := 0 + cleaned := 0 + for _, line := range strings.Split(listed.Stdout, "\n") { + name := strings.TrimSpace(line) + if !strings.HasPrefix(name, terminalExecContainerPrefix) { + continue + } + discovered++ + if _, expected := expectedNames[name]; expected { + continue + } + m.forceRemoveContainer(name) + cleaned++ + } + return discovered, cleaned +} diff --git a/worker/worker-docker/internal/runner/session_recovery_integration_test.go b/worker/worker-docker/internal/runner/session_recovery_integration_test.go new file mode 100644 index 0000000..3260dfd --- /dev/null +++ b/worker/worker-docker/internal/runner/session_recovery_integration_test.go @@ -0,0 +1,184 @@ +package runner + +import ( + "context" + "fmt" + "os" + "os/exec" + "strings" + "testing" + "time" + + registryv1 "github.com/onlyboxes/onlyboxes/api/gen/go/registry/v1" +) + +func TestIntegrationTerminalContainerSurvivesManagerRestart(t *testing.T) { + if os.Getenv("DOCKER_INTEGRATION") != "1" { + t.Skip("set DOCKER_INTEGRATION=1 to run against Docker") + } + sessionID := "integration:docker-session-recovery" + containerName := terminalSessionResourceName(sessionID) + t.Cleanup(func() { + cleanupCtx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + defer cancel() + _ = runDockerCommand(cleanupCtx, pythonExecDockerRemoveArgs(containerName)...) + }) + + baseConfig := terminalSessionManagerConfig{ + LeaseMinSec: 1, + LeaseMaxSec: 600, + LeaseDefaultSec: 300, + DockerImage: defaultTerminalExecDockerImage, + SessionMaxInflight: 1, + } + firstConfig := baseConfig + firstConfig.PreserveOnClose = true + first := newTerminalSessionManager(firstConfig) + ctx, cancel := context.WithTimeout(context.Background(), time.Minute) + defer cancel() + created, err := first.Execute(ctx, terminalExecRequest{ + Command: "printf recovery-ok > /tmp/onlyboxes-recovery.txt", + SessionID: sessionID, + CreateIfMissing: true, + LeaseTTLSec: intPointerForDockerRecovery(300), + }) + if err != nil { + first.Close() + t.Fatal(err) + } + if !created.Created { + first.Close() + t.Fatal("initial session was not created") + } + first.Close() + + lease := time.UnixMilli(created.LeaseExpiresUnixMS) + second := newTerminalSessionManager(baseConfig) + defer second.Close() + if status := second.recoverOne(ctx, sessionID, containerName, lease); status != registryv1.TerminalSessionRecoveryResult_RECOVERED { + t.Fatalf("recover status=%s", status) + } + recovered, err := second.Execute(ctx, terminalExecRequest{ + Command: "cat /tmp/onlyboxes-recovery.txt", + SessionID: sessionID, + LeaseTTLSec: intPointerForDockerRecovery(300), + }) + if err != nil { + t.Fatal(err) + } + if recovered.Created || recovered.Stdout != "recovery-ok" { + t.Fatalf("unexpected recovered result: %#v", recovered) + } +} + +func intPointerForDockerRecovery(value int) *int { return &value } + +func TestIntegrationTerminalContainerSurvivesForcedWorkerTermination(t *testing.T) { + if os.Getenv("DOCKER_INTEGRATION") != "1" { + t.Skip("set DOCKER_INTEGRATION=1 to run against Docker") + } + if os.Getenv("DOCKER_RECOVERY_CRASH_HELPER") == "1" { + runDockerRecoveryCrashHelper(t) + return + } + + sessionID := fmt.Sprintf("integration:docker-crash-recovery-%d", time.Now().UnixNano()) + containerName := terminalSessionResourceName(sessionID) + readyFile, err := os.CreateTemp("", "onlyboxes-docker-crash-ready-*") + if err != nil { + t.Fatal(err) + } + readyPath := readyFile.Name() + _ = readyFile.Close() + _ = os.Remove(readyPath) + t.Cleanup(func() { + _ = os.Remove(readyPath) + cleanupCtx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + defer cancel() + _ = runDockerCommand(cleanupCtx, pythonExecDockerRemoveArgs(containerName)...) + }) + cmd := exec.Command(os.Args[0], "-test.run=^TestIntegrationTerminalContainerSurvivesForcedWorkerTermination$", "-test.v") + cmd.Env = append(os.Environ(), "DOCKER_RECOVERY_CRASH_HELPER=1", "DOCKER_RECOVERY_SESSION_ID="+sessionID, "DOCKER_RECOVERY_READY_FILE="+readyPath) + cmd.Stdout = os.Stderr + cmd.Stderr = os.Stderr + if err := cmd.Start(); err != nil { + t.Fatal(err) + } + waitCh := make(chan error, 1) + go func() { waitCh <- cmd.Wait() }() + leaseUnixMs := int64(0) + deadline := time.NewTimer(time.Minute) + defer deadline.Stop() + ticker := time.NewTicker(25 * time.Millisecond) + defer ticker.Stop() + for leaseUnixMs == 0 { + select { + case err := <-waitCh: + t.Fatalf("crash helper exited before creating container: %v", err) + case <-deadline.C: + _ = cmd.Process.Kill() + <-waitCh + t.Fatal("timed out waiting for crash helper") + case <-ticker.C: + content, readErr := os.ReadFile(readyPath) + if readErr != nil { + continue + } + if _, scanErr := fmt.Sscan(strings.TrimSpace(string(content)), &leaseUnixMs); scanErr != nil { + t.Fatalf("parse helper lease: %v", scanErr) + } + } + } + if err := cmd.Process.Kill(); err != nil { + t.Fatal(err) + } + if err := <-waitCh; err == nil { + t.Fatal("crash helper was not force terminated") + } + + manager := newTerminalSessionManager(terminalSessionManagerConfig{ + LeaseMinSec: 1, LeaseMaxSec: 600, LeaseDefaultSec: 300, + DockerImage: defaultTerminalExecDockerImage, SessionMaxInflight: 1, + }) + defer manager.Close() + ctx, cancel := context.WithTimeout(context.Background(), time.Minute) + defer cancel() + results := manager.Recover(ctx, []*registryv1.TerminalSessionRecoveryCandidate{{ + SessionId: sessionID, LeaseExpiresUnixMs: leaseUnixMs, + }}) + if len(results) != 1 || results[0].GetStatus() != registryv1.TerminalSessionRecoveryResult_RECOVERED { + t.Fatalf("unexpected recovery result: %#v", results) + } + recovered, err := manager.Execute(ctx, terminalExecRequest{ + Command: "cat /tmp/onlyboxes-forced-recovery.txt", SessionID: sessionID, + LeaseTTLSec: intPointerForDockerRecovery(1), + }) + if err != nil { + t.Fatal(err) + } + if recovered.Created || recovered.Stdout != "forced-recovery-ok" { + t.Fatalf("unexpected recovered result: %#v", recovered) + } +} + +func runDockerRecoveryCrashHelper(t *testing.T) { + sessionID := strings.TrimSpace(os.Getenv("DOCKER_RECOVERY_SESSION_ID")) + manager := newTerminalSessionManager(terminalSessionManagerConfig{ + LeaseMinSec: 1, LeaseMaxSec: 600, LeaseDefaultSec: 300, + DockerImage: defaultTerminalExecDockerImage, SessionMaxInflight: 1, PreserveOnClose: true, + }) + ctx, cancel := context.WithTimeout(context.Background(), time.Minute) + defer cancel() + created, err := manager.Execute(ctx, terminalExecRequest{ + Command: "printf forced-recovery-ok > /tmp/onlyboxes-forced-recovery.txt", + SessionID: sessionID, CreateIfMissing: true, LeaseTTLSec: intPointerForDockerRecovery(300), + }) + if err != nil { + t.Fatal(err) + } + readyPath := strings.TrimSpace(os.Getenv("DOCKER_RECOVERY_READY_FILE")) + if err := os.WriteFile(readyPath, []byte(fmt.Sprintf("%d", created.LeaseExpiresUnixMS)), 0o600); err != nil { + t.Fatal(err) + } + select {} +} diff --git a/worker/worker-docker/internal/runner/session_recovery_test.go b/worker/worker-docker/internal/runner/session_recovery_test.go new file mode 100644 index 0000000..18a3a50 --- /dev/null +++ b/worker/worker-docker/internal/runner/session_recovery_test.go @@ -0,0 +1,263 @@ +package runner + +import ( + "context" + "fmt" + "strings" + "testing" + "time" + + registryv1 "github.com/onlyboxes/onlyboxes/api/gen/go/registry/v1" +) + +func TestTerminalRecoveryResourceNameUsesFullHashWithoutSessionID(t *testing.T) { + sessionID := "owner-secret:session-secret" + name := terminalSessionResourceName(sessionID) + if !strings.HasPrefix(name, terminalExecContainerPrefix) || len(name) != len(terminalExecContainerPrefix)+64 { + t.Fatalf("unexpected recovery resource name %q", name) + } + if strings.Contains(name, "owner-secret") || name != terminalSessionResourceName(sessionID) { + t.Fatalf("resource name exposed or did not deterministically map the session: %q", name) + } +} + +func TestTerminalRecoveryRestoresContainerAndExactLease(t *testing.T) { + original := runDockerCommand + t.Cleanup(func() { runDockerCommand = original }) + + sessionID := "owner-a:session-a" + name := terminalSessionResourceName(sessionID) + hash := terminalSessionIDHash(sessionID) + runDockerCommand = func(_ context.Context, args ...string) dockerCommandResult { + joined := strings.Join(args, " ") + switch { + case len(args) > 0 && args[0] == "inspect": + return dockerCommandResult{ExitCode: 0, Stdout: fmt.Sprintf("{\"%s\":\"%s\",\"%s\":\"1\"}\trunning\n", terminalExecSessionLabelKey, hash, terminalExecSchemaLabelKey)} + case strings.Contains(joined, "label="+terminalExecSessionLabelKey+"="+hash): + return dockerCommandResult{ExitCode: 0, Stdout: name + "\n"} + case len(args) > 0 && args[0] == "ps": + return dockerCommandResult{ExitCode: 0, Stdout: name + "\n"} + default: + t.Fatalf("unexpected docker command: %v", args) + return dockerCommandResult{ExitCode: 1} + } + } + + manager := newTerminalSessionManager(terminalSessionManagerConfig{PreserveOnClose: true}) + defer manager.Close() + lease := time.Now().Add(10 * time.Minute).Truncate(time.Millisecond) + results := manager.Recover(context.Background(), []*registryv1.TerminalSessionRecoveryCandidate{ + {SessionId: sessionID, LeaseExpiresUnixMs: lease.UnixMilli()}, + }) + if len(results) != 1 || results[0].GetStatus() != registryv1.TerminalSessionRecoveryResult_RECOVERED { + t.Fatalf("unexpected recovery result: %#v", results) + } + if manager.ActiveSessionCount() != 1 { + t.Fatalf("active session count=%d, want 1", manager.ActiveSessionCount()) + } + manager.mu.Lock() + recovered := manager.sessions[sessionID] + manager.mu.Unlock() + if recovered == nil || !recovered.leaseExpiresAt.Equal(lease) || recovered.inflight != 0 { + t.Fatalf("unexpected recovered session: %#v", recovered) + } +} + +func TestTerminalRecoveryRejectsAndRemovesDuplicateContainers(t *testing.T) { + original := runDockerCommand + t.Cleanup(func() { runDockerCommand = original }) + + sessionID := "owner-a:session-duplicate" + name := terminalSessionResourceName(sessionID) + hash := terminalSessionIDHash(sessionID) + removed := map[string]bool{} + runDockerCommand = func(_ context.Context, args ...string) dockerCommandResult { + joined := strings.Join(args, " ") + switch { + case len(args) > 0 && args[0] == "inspect": + return dockerCommandResult{ExitCode: 0, Stdout: fmt.Sprintf("{\"%s\":\"%s\",\"%s\":\"1\"}\trunning\n", terminalExecSessionLabelKey, hash, terminalExecSchemaLabelKey)} + case strings.Contains(joined, "label="+terminalExecSessionLabelKey+"="+hash): + return dockerCommandResult{ExitCode: 0, Stdout: name + "\n" + name + "-duplicate\n"} + case len(args) > 0 && args[0] == "rm": + removed[args[len(args)-1]] = true + return dockerCommandResult{ExitCode: 0} + case len(args) > 0 && args[0] == "ps": + return dockerCommandResult{ExitCode: 0} + default: + t.Fatalf("unexpected docker command: %v", args) + return dockerCommandResult{ExitCode: 1} + } + } + + manager := newTerminalSessionManager(terminalSessionManagerConfig{PreserveOnClose: true}) + defer manager.Close() + results := manager.Recover(context.Background(), []*registryv1.TerminalSessionRecoveryCandidate{ + {SessionId: sessionID, LeaseExpiresUnixMs: time.Now().Add(time.Minute).UnixMilli()}, + }) + if len(results) != 1 || results[0].GetStatus() != registryv1.TerminalSessionRecoveryResult_INVALID { + t.Fatalf("unexpected recovery result: %#v", results) + } + if !removed[name] || !removed[name+"-duplicate"] { + t.Fatalf("duplicate containers were not isolated: %#v", removed) + } +} + +func TestTerminalRecoveryRemovesContainerWithInvalidLabels(t *testing.T) { + original := runDockerCommand + t.Cleanup(func() { runDockerCommand = original }) + + sessionID := "owner-a:session-invalid" + name := terminalSessionResourceName(sessionID) + removed := false + runDockerCommand = func(_ context.Context, args ...string) dockerCommandResult { + switch { + case len(args) > 0 && args[0] == "inspect": + return dockerCommandResult{ExitCode: 0, Stdout: fmt.Sprintf("{\"%s\":\"wrong\",\"%s\":\"1\"}\trunning\n", terminalExecSessionLabelKey, terminalExecSchemaLabelKey)} + case len(args) > 0 && args[0] == "rm": + removed = args[len(args)-1] == name + return dockerCommandResult{ExitCode: 0} + case len(args) > 0 && args[0] == "ps": + return dockerCommandResult{ExitCode: 0} + default: + t.Fatalf("unexpected docker command: %v", args) + return dockerCommandResult{ExitCode: 1} + } + } + + manager := newTerminalSessionManager(terminalSessionManagerConfig{PreserveOnClose: true}) + defer manager.Close() + results := manager.Recover(context.Background(), []*registryv1.TerminalSessionRecoveryCandidate{ + {SessionId: sessionID, LeaseExpiresUnixMs: time.Now().Add(time.Minute).UnixMilli()}, + }) + if len(results) != 1 || results[0].GetStatus() != registryv1.TerminalSessionRecoveryResult_INVALID { + t.Fatalf("unexpected recovery result: %#v", results) + } + if !removed { + t.Fatal("container with invalid recovery labels was not isolated") + } +} + +func TestTerminalRecoveryIsIdempotentAndCleansOnlyOwnedOrphans(t *testing.T) { + original := runDockerCommand + t.Cleanup(func() { runDockerCommand = original }) + + sessionID := "owner-a:session-idempotent" + name := terminalSessionResourceName(sessionID) + orphan := terminalExecContainerPrefix + "orphan" + hash := terminalSessionIDHash(sessionID) + inspectCalls := 0 + removed := map[string]bool{} + runDockerCommand = func(_ context.Context, args ...string) dockerCommandResult { + joined := strings.Join(args, " ") + switch { + case len(args) > 0 && args[0] == "inspect": + inspectCalls++ + return dockerCommandResult{ExitCode: 0, Stdout: fmt.Sprintf("{\"%s\":\"%s\",\"%s\":\"1\"}\trunning\n", terminalExecSessionLabelKey, hash, terminalExecSchemaLabelKey)} + case strings.Contains(joined, "label="+terminalExecSessionLabelKey+"="+hash): + return dockerCommandResult{ExitCode: 0, Stdout: name + "\n"} + case len(args) > 0 && args[0] == "ps": + return dockerCommandResult{ExitCode: 0, Stdout: name + "\n" + orphan + "\nforeign-container\n"} + case len(args) > 0 && args[0] == "rm": + removed[args[len(args)-1]] = true + return dockerCommandResult{ExitCode: 0} + default: + t.Fatalf("unexpected docker command: %v", args) + return dockerCommandResult{ExitCode: 1} + } + } + + manager := newTerminalSessionManager(terminalSessionManagerConfig{PreserveOnClose: true}) + defer manager.Close() + lease := time.Now().Add(time.Minute).Truncate(time.Millisecond) + candidate := []*registryv1.TerminalSessionRecoveryCandidate{{SessionId: sessionID, LeaseExpiresUnixMs: lease.UnixMilli()}} + for range 2 { + results := manager.Recover(context.Background(), candidate) + if len(results) != 1 || results[0].GetStatus() != registryv1.TerminalSessionRecoveryResult_RECOVERED { + t.Fatalf("unexpected recovery result: %#v", results) + } + } + if inspectCalls != 1 || manager.ActiveSessionCount() != 1 { + t.Fatalf("recovery was not idempotent: inspect_calls=%d active=%d", inspectCalls, manager.ActiveSessionCount()) + } + if !removed[orphan] || removed[name] || removed["foreign-container"] { + t.Fatalf("unsafe orphan cleanup: %#v", removed) + } +} + +func TestTerminalRecoveryReportsMissingAndCleansExpiredCandidate(t *testing.T) { + original := runDockerCommand + t.Cleanup(func() { runDockerCommand = original }) + + missingID := "owner-a:session-missing" + expiredID := "owner-a:session-expired" + expiredName := terminalSessionResourceName(expiredID) + removedExpired := false + runDockerCommand = func(_ context.Context, args ...string) dockerCommandResult { + switch { + case len(args) > 0 && args[0] == "inspect": + return dockerCommandResult{ExitCode: 1, Stderr: "No such container"} + case len(args) > 0 && args[0] == "rm": + if args[len(args)-1] == expiredName { + removedExpired = true + } + return dockerCommandResult{ExitCode: 0} + case len(args) > 0 && args[0] == "ps": + return dockerCommandResult{ExitCode: 0} + default: + t.Fatalf("unexpected docker command: %v", args) + return dockerCommandResult{ExitCode: 1} + } + } + manager := newTerminalSessionManager(terminalSessionManagerConfig{PreserveOnClose: true}) + defer manager.Close() + results := manager.Recover(context.Background(), []*registryv1.TerminalSessionRecoveryCandidate{ + {SessionId: missingID, LeaseExpiresUnixMs: time.Now().Add(time.Minute).UnixMilli()}, + {SessionId: expiredID, LeaseExpiresUnixMs: time.Now().Add(-time.Minute).UnixMilli()}, + }) + if results[0].GetStatus() != registryv1.TerminalSessionRecoveryResult_MISSING || + results[1].GetStatus() != registryv1.TerminalSessionRecoveryResult_INVALID { + t.Fatalf("unexpected recovery results: %#v", results) + } + if !removedExpired || manager.ActiveSessionCount() != 0 { + t.Fatalf("expired resource was not cleaned: removed=%v active=%d", removedExpired, manager.ActiveSessionCount()) + } +} + +func TestConcurrentTerminalRecoveryUsesOneReservation(t *testing.T) { + original := runDockerCommand + t.Cleanup(func() { runDockerCommand = original }) + sessionID := "owner-a:session-concurrent" + name := terminalSessionResourceName(sessionID) + hash := terminalSessionIDHash(sessionID) + runDockerCommand = func(_ context.Context, args ...string) dockerCommandResult { + joined := strings.Join(args, " ") + switch { + case len(args) > 0 && args[0] == "inspect": + return dockerCommandResult{ExitCode: 0, Stdout: fmt.Sprintf("{\"%s\":\"%s\",\"%s\":\"1\"}\trunning\n", terminalExecSessionLabelKey, hash, terminalExecSchemaLabelKey)} + case strings.Contains(joined, "label="+terminalExecSessionLabelKey+"="+hash): + return dockerCommandResult{ExitCode: 0, Stdout: name + "\n"} + case len(args) > 0 && args[0] == "ps": + return dockerCommandResult{ExitCode: 0, Stdout: name + "\n"} + default: + t.Fatalf("unexpected docker command: %v", args) + return dockerCommandResult{ExitCode: 1} + } + } + manager := newTerminalSessionManager(terminalSessionManagerConfig{PreserveOnClose: true}) + defer manager.Close() + candidate := []*registryv1.TerminalSessionRecoveryCandidate{{ + SessionId: sessionID, LeaseExpiresUnixMs: time.Now().Add(time.Minute).UnixMilli(), + }} + done := make(chan []*registryv1.TerminalSessionRecoveryResult, 2) + for range 2 { + go func() { done <- manager.Recover(context.Background(), candidate) }() + } + for range 2 { + if result := <-done; len(result) != 1 || result[0].GetStatus() != registryv1.TerminalSessionRecoveryResult_RECOVERED { + t.Fatalf("unexpected concurrent recovery result: %#v", result) + } + } + if manager.ActiveSessionCount() != 1 { + t.Fatalf("concurrent recovery reserved %d sessions, want 1", manager.ActiveSessionCount()) + } +} diff --git a/worker/worker-docker/internal/runner/terminal_exec.go b/worker/worker-docker/internal/runner/terminal_exec.go index 72f0ef7..e5d5998 100644 --- a/worker/worker-docker/internal/runner/terminal_exec.go +++ b/worker/worker-docker/internal/runner/terminal_exec.go @@ -2,6 +2,8 @@ package runner import ( "context" + "crypto/sha256" + "encoding/hex" "errors" "fmt" "net/netip" @@ -17,8 +19,11 @@ import ( const ( terminalExecCapabilityName = "terminalexec" terminalExecCapabilityDeclared = "terminalExec" - terminalExecContainerPrefix = "onlyboxes-terminalexec-" + terminalExecContainerPrefix = "onlyboxes-terminal-v1-" terminalExecCapabilityLabel = "onlyboxes.capability=terminalExec" + terminalExecSessionLabelKey = "onlyboxes.session_id_hash" + terminalExecSchemaLabelKey = "onlyboxes.schema_version" + terminalExecSchemaVersion = "1" terminalExecIdleCommand = "while true; do sleep 3600; done" terminalExecCleanupTimeout = 3 * time.Second terminalExecInspectTimeout = 2 * time.Second @@ -131,6 +136,9 @@ type terminalSessionManagerConfig struct { // MaxActiveSessions caps terminal sandbox reservations across all sessions. // Zero preserves the existing unlimited behaviour. MaxActiveSessions int + // PreserveOnClose leaves terminal containers running for process restart + // recovery. Tests and explicit teardown may leave this false. + PreserveOnClose bool } type terminalSessionManager struct { @@ -150,6 +158,7 @@ type terminalSessionManager struct { sessionMaxInflight int maxActiveSessions int activeSessionReservations int + preserveOnClose bool stopCh chan struct{} doneCh chan struct{} @@ -241,6 +250,7 @@ func newTerminalSessionManager(cfg terminalSessionManagerConfig) *terminalSessio dockerNetwork: strings.TrimSpace(cfg.DockerNetwork), sessionMaxInflight: sessionMaxInflight, maxActiveSessions: maxActiveSessions, + preserveOnClose: cfg.PreserveOnClose, stopCh: make(chan struct{}), doneCh: make(chan struct{}), } @@ -275,6 +285,10 @@ func (m *terminalSessionManager) Close() { m.cleanupWG.Wait() for _, session := range sessions { + if m.preserveOnClose { + m.releaseCapacityReservation(session) + continue + } m.cleanupSession(session) } }) @@ -637,10 +651,7 @@ func (m *terminalSessionManager) newSessionLocked(sessionID string, leaseTarget ) } - containerName, err := newTerminalExecContainerName() - if err != nil { - return nil, fmt.Errorf("allocate terminal container name: %w", err) - } + containerName := terminalSessionResourceName(sessionID) proxyCtx, proxyCancel := context.WithCancel(context.Background()) session := &terminalSession{ @@ -865,12 +876,15 @@ func terminalExecDockerCreateArgs(containerName string, dockerImage string, memo } func terminalExecDockerCreateArgsWithNetwork(containerName string, dockerImage string, memoryLimit string, cpuLimit string, pidsLimit int, dockerNetwork string) []string { + sessionHash := strings.TrimPrefix(strings.TrimSpace(containerName), terminalExecContainerPrefix) args := []string{ "create", "--name", containerName, "--label", pythonExecManagedLabel, "--label", terminalExecCapabilityLabel, "--label", pythonExecRuntimeLabel, + "--label", terminalExecSessionLabelKey + "=" + sessionHash, + "--label", terminalExecSchemaLabelKey + "=" + terminalExecSchemaVersion, "--memory", memoryLimit, "--cpus", cpuLimit, "--pids-limit", strconv.Itoa(pidsLimit), @@ -894,12 +908,13 @@ func terminalExecDockerExecArgs(containerName string, command string) []string { return []string{"exec", containerName, "sh", "-lc", command} } -func newTerminalExecContainerName() (string, error) { - suffix, err := randomHex(8) - if err != nil { - return "", err - } - return terminalExecContainerPrefix + suffix, nil +func terminalSessionIDHash(sessionID string) string { + sum := sha256.Sum256([]byte(strings.TrimSpace(sessionID))) + return hex.EncodeToString(sum[:]) +} + +func terminalSessionResourceName(sessionID string) string { + return terminalExecContainerPrefix + terminalSessionIDHash(sessionID) } func truncateByBytes(value string, maxBytes int) (string, bool) { diff --git a/worker/worker-docker/internal/runner/terminal_exec_test.go b/worker/worker-docker/internal/runner/terminal_exec_test.go index ff1cd2d..68d0e39 100644 --- a/worker/worker-docker/internal/runner/terminal_exec_test.go +++ b/worker/worker-docker/internal/runner/terminal_exec_test.go @@ -480,13 +480,16 @@ func TestNewTerminalSessionManagerUsesConfiguredResourceLimits(t *testing.T) { } func TestTerminalExecDockerCreateArgs(t *testing.T) { - got := terminalExecDockerCreateArgs("container-a", "python:slim", "256m", "1.0", 128) + containerName := terminalSessionResourceName("session-a") + got := terminalExecDockerCreateArgs(containerName, "python:slim", "256m", "1.0", 128) want := []string{ "create", - "--name", "container-a", + "--name", containerName, "--label", pythonExecManagedLabel, "--label", terminalExecCapabilityLabel, "--label", pythonExecRuntimeLabel, + "--label", terminalExecSessionLabelKey + "=" + terminalSessionIDHash("session-a"), + "--label", terminalExecSchemaLabelKey + "=" + terminalExecSchemaVersion, "--memory", "256m", "--cpus", "1.0", "--pids-limit", "128",