From 9e9c74942c01e6a24f7b5177e755b100f9293aa5 Mon Sep 17 00:00:00 2001 From: Stephan Arenswald Date: Tue, 11 Aug 2026 09:19:52 +0200 Subject: [PATCH] feat: act on the app's own controls in-process (elementActionRequest) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Where launch parameters put an app into a state, a plan sometimes needs to drive its UI — MacPacker's search field has to be focused for the search screenshot. Doing that from outside would mean synthesizing input, which needs the Accessibility grant; doing it inside the target does not, because a process may read and drive its own accessibility tree freely. elementActionRequest names a control by accessibility identifier and says what to do to it. ElementLocator resolves it with the same in-process self read as screenFrame, and replies with elementActionResult saying whether it worked and, if not, why — so a plan naming a control the app does not expose fails legibly instead of producing a screenshot of the wrong thing. One action kind, focus. Neither the kit nor SandboxPilot learns anything about any app's UI: the identifier comes from whoever wrote the plan. --- .../SandboxPilotKit/Core/ElementLocator.swift | 23 +++++++++ .../Core/SandboxPilotCore.swift | 3 ++ .../Network/PilotMessage.swift | 48 +++++++++++++++++++ .../SandboxPilotKitTests.swift | 17 +++++++ 4 files changed, 91 insertions(+) diff --git a/Sources/SandboxPilotKit/Core/ElementLocator.swift b/Sources/SandboxPilotKit/Core/ElementLocator.swift index 58ba17d..00062fa 100644 --- a/Sources/SandboxPilotKit/Core/ElementLocator.swift +++ b/Sources/SandboxPilotKit/Core/ElementLocator.swift @@ -36,6 +36,29 @@ enum ElementLocator { } } + /// Carries out `action` on this app's control with that accessibility + /// identifier. Same in-process self-read as `screenFrame`, so it needs no + /// Accessibility grant, and no knowledge of what the app's UI contains. + static func perform(_ action: ElementAction) async -> ElementActionResult { + await MainActor.run { + let app = AXUIElementCreateApplication(getpid()) + guard let element = find(in: app, identifier: action.identifier, depth: 0) else { + return ElementActionResult( + identifier: action.identifier, kind: action.kind, performed: false, + failure: "no control with accessibility identifier \"\(action.identifier)\"" + ) + } + switch action.kind { + case .focus: + let code = AXUIElementSetAttributeValue(element, kAXFocusedAttribute as CFString, kCFBooleanTrue) + return ElementActionResult( + identifier: action.identifier, kind: action.kind, performed: code == .success, + failure: code == .success ? nil : "AXFocused could not be set (AXError \(code.rawValue))" + ) + } + } + } + private static let skippedRoles: Set = ["AXTable", "AXOutline", "AXList"] @MainActor diff --git a/Sources/SandboxPilotKit/Core/SandboxPilotCore.swift b/Sources/SandboxPilotKit/Core/SandboxPilotCore.swift index b3b9a4d..ba4e71a 100644 --- a/Sources/SandboxPilotKit/Core/SandboxPilotCore.swift +++ b/Sources/SandboxPilotKit/Core/SandboxPilotCore.swift @@ -122,6 +122,9 @@ final actor SandboxPilotCore { case .elementFrameRequest(let identifier): await net.send(.elementFrame(ElementLocator.screenFrame(identifier: identifier))) + case .elementActionRequest(let action): + await net.send(.elementActionResult(ElementLocator.perform(action))) + case .ack, .error: break } diff --git a/Sources/SandboxPilotKit/Network/PilotMessage.swift b/Sources/SandboxPilotKit/Network/PilotMessage.swift index f9d4372..3429482 100644 --- a/Sources/SandboxPilotKit/Network/PilotMessage.swift +++ b/Sources/SandboxPilotKit/Network/PilotMessage.swift @@ -24,6 +24,47 @@ public struct ElementFrame: Codable, Sendable { public var found: Bool { x != nil && width != nil } } +/// Something to do *to* one of the app's controls, named by accessibility +/// identifier and carried out in-process by the kit. +/// +/// Doing it inside the target is what keeps this permission-free: a process +/// reading and driving its own accessibility tree needs no Accessibility grant, +/// where synthesizing input from outside would. It also keeps both sides +/// app-agnostic — which control, and what to do to it, come from whoever wrote +/// the plan; neither SandboxPilot nor the kit knows what any app's UI contains. +public struct ElementAction: Codable, Sendable { + /// Deliberately small. Add a case when a plan needs one, not before. + public enum Kind: String, Codable, Sendable { + /// Make the control the focused element of its window (`AXFocused`). + case focus + } + + public let identifier: String + public let kind: Kind + + public init(identifier: String, kind: Kind) { + self.identifier = identifier + self.kind = kind + } +} + +/// The outcome of an `elementActionRequest`. `failure` says why not, so a plan +/// that names a control the app doesn't expose fails legibly instead of just +/// producing a screenshot of the wrong thing. +public struct ElementActionResult: Codable, Sendable { + public let identifier: String + public let kind: ElementAction.Kind + public let performed: Bool + public let failure: String? + + public init(identifier: String, kind: ElementAction.Kind, performed: Bool, failure: String? = nil) { + self.identifier = identifier + self.kind = kind + self.performed = performed + self.failure = failure + } +} + /// Messages sent FROM a controlled app TO the SandboxPilot companion app. public enum PilotClientMessage: Codable, Sendable { case ack(String) @@ -35,6 +76,8 @@ public enum PilotClientMessage: Codable, Sendable { case defaults([PrefPatch]) /// The resolved frame for an `elementFrameRequest` (or a not-found reply). case elementFrame(ElementFrame) + /// The outcome of an `elementActionRequest`. + case elementActionResult(ElementActionResult) } /// Messages sent FROM the SandboxPilot companion app TO a controlled app. @@ -57,6 +100,11 @@ public enum PilotServerMessage: Codable, Sendable { /// blocks) and without any SandboxPilot-specific code in the target — the app only /// needs to label its controls with `.accessibilityIdentifier(_:)`. case elementFrameRequest(String) + /// Ask the app to act on one of its controls (by accessibility identifier) + /// in-process, and reply with `elementActionResult`. Where launch parameters + /// put the app into a *state*, this drives its *UI* — and doing it inside the + /// target needs no Accessibility grant, unlike synthesizing input from outside. + case elementActionRequest(ElementAction) /// Set the app's SandboxPilot launch parameters. They are written to a /// dedicated UserDefaults suite (never the standard domain), so the host app /// can read them as a fallback for real command-line launch arguments diff --git a/Tests/SandboxPilotKitTests/SandboxPilotKitTests.swift b/Tests/SandboxPilotKitTests/SandboxPilotKitTests.swift index 4e86870..76264f6 100644 --- a/Tests/SandboxPilotKitTests/SandboxPilotKitTests.swift +++ b/Tests/SandboxPilotKitTests/SandboxPilotKitTests.swift @@ -23,6 +23,8 @@ struct WireProtocolTests { .windowAsKeyRequest(42), .windowResizeRequest(WindowResizeRequest(windowNumber: 7, frame: CGRect(x: 0, y: 0, width: 800, height: 500))), .userDefaultsPatch([PrefPatch(key: "k", value: .int(1))]), + .elementFrameRequest("search-field"), + .elementActionRequest(ElementAction(identifier: "search-field", kind: .focus)), ] for message in messages { let data = try encoder.encode(message) @@ -41,6 +43,8 @@ struct WireProtocolTests { .environment(env), .windows([window]), .defaults([PrefPatch(key: "flag", value: .bool(true))]), + .elementActionResult(ElementActionResult(identifier: "search-field", kind: .focus, performed: true)), + .elementActionResult(ElementActionResult(identifier: "nope", kind: .focus, performed: false, failure: "not found")), ] for message in messages { let data = try encoder.encode(message) @@ -81,6 +85,19 @@ struct LaunchParameterTests { #expect(AppearanceControl().change(to: .light) == false) #expect(AppearanceControl().change(to: .system) == false) } + + // A plan naming a control the app does not expose has to say so, rather than + // quietly succeeding and leaving whoever wrote it to work out why the shot + // looks wrong. Nothing in this process publishes that identifier. + @Test("acting on an unknown identifier reports why it failed") + func unknownElementActionExplainsItself() async { + let result = await ElementLocator.perform( + ElementAction(identifier: "no.such.control", kind: .focus) + ) + #expect(result.performed == false) + #expect(result.identifier == "no.such.control") + #expect(result.failure?.contains("no.such.control") == true) + } } @Suite("PrefPatch.Value")