feat: act on the app's own controls in-process (elementActionRequest)

Sources/SandboxPilotKit/Core/ElementLocator.swift

@@ -36,6 +36,29 @@ enum ElementLocator {
         }
     }
 
+    /// Carries out `action` on this app's control with that accessibility
+    /// identifier. Same in-process self-read as `screenFrame`, so it needs no
+    /// Accessibility grant, and no knowledge of what the app's UI contains.
+    static func perform(_ action: ElementAction) async -> ElementActionResult {
+        await MainActor.run {
+            let app = AXUIElementCreateApplication(getpid())
+            guard let element = find(in: app, identifier: action.identifier, depth: 0) else {
+                return ElementActionResult(
+                    identifier: action.identifier, kind: action.kind, performed: false,
+                    failure: "no control with accessibility identifier \"\(action.identifier)\""
+                )
+            }
+            switch action.kind {
+            case .focus:
+                let code = AXUIElementSetAttributeValue(element, kAXFocusedAttribute as CFString, kCFBooleanTrue)
+                return ElementActionResult(
+                    identifier: action.identifier, kind: action.kind, performed: code == .success,
+                    failure: code == .success ? nil : "AXFocused could not be set (AXError \(code.rawValue))"
+                )
+            }
+        }
+    }
+
     private static let skippedRoles: Set<String> = ["AXTable", "AXOutline", "AXList"]
 
     @MainActor

Sources/SandboxPilotKit/Core/SandboxPilotCore.swift

@@ -122,6 +122,9 @@ final actor SandboxPilotCore {
         case .elementFrameRequest(let identifier):
             await net.send(.elementFrame(ElementLocator.screenFrame(identifier: identifier)))
 
+        case .elementActionRequest(let action):
+            await net.send(.elementActionResult(ElementLocator.perform(action)))
+
         case .ack, .error:
             break
         }

Sources/SandboxPilotKit/Network/PilotMessage.swift

@@ -24,6 +24,47 @@ public struct ElementFrame: Codable, Sendable {
     public var found: Bool { x != nil && width != nil }
 }
 
+/// Something to do *to* one of the app's controls, named by accessibility
+/// identifier and carried out in-process by the kit.
+///
+/// Doing it inside the target is what keeps this permission-free: a process
+/// reading and driving its own accessibility tree needs no Accessibility grant,
+/// where synthesizing input from outside would. It also keeps both sides
+/// app-agnostic — which control, and what to do to it, come from whoever wrote
+/// the plan; neither SandboxPilot nor the kit knows what any app's UI contains.
+public struct ElementAction: Codable, Sendable {
+    /// Deliberately small. Add a case when a plan needs one, not before.
+    public enum Kind: String, Codable, Sendable {
+        /// Make the control the focused element of its window (`AXFocused`).
+        case focus
+    }
+
+    public let identifier: String
+    public let kind: Kind
+
+    public init(identifier: String, kind: Kind) {
+        self.identifier = identifier
+        self.kind = kind
+    }
+}
+
+/// The outcome of an `elementActionRequest`. `failure` says why not, so a plan
+/// that names a control the app doesn't expose fails legibly instead of just
+/// producing a screenshot of the wrong thing.
+public struct ElementActionResult: Codable, Sendable {
+    public let identifier: String
+    public let kind: ElementAction.Kind
+    public let performed: Bool
+    public let failure: String?
+
+    public init(identifier: String, kind: ElementAction.Kind, performed: Bool, failure: String? = nil) {
+        self.identifier = identifier
+        self.kind = kind
+        self.performed = performed
+        self.failure = failure
+    }
+}
+
 /// Messages sent FROM a controlled app TO the SandboxPilot companion app.
 public enum PilotClientMessage: Codable, Sendable {
     case ack(String)
@@ -35,6 +76,8 @@ public enum PilotClientMessage: Codable, Sendable {
     case defaults([PrefPatch])
     /// The resolved frame for an `elementFrameRequest` (or a not-found reply).
     case elementFrame(ElementFrame)
+    /// The outcome of an `elementActionRequest`.
+    case elementActionResult(ElementActionResult)
 }
 
 /// Messages sent FROM the SandboxPilot companion app TO a controlled app.
@@ -57,6 +100,11 @@ public enum PilotServerMessage: Codable, Sendable {
     /// blocks) and without any SandboxPilot-specific code in the target — the app only
     /// needs to label its controls with `.accessibilityIdentifier(_:)`.
     case elementFrameRequest(String)
+    /// Ask the app to act on one of its controls (by accessibility identifier)
+    /// in-process, and reply with `elementActionResult`. Where launch parameters
+    /// put the app into a *state*, this drives its *UI* — and doing it inside the
+    /// target needs no Accessibility grant, unlike synthesizing input from outside.
+    case elementActionRequest(ElementAction)
     /// Set the app's SandboxPilot launch parameters. They are written to a
     /// dedicated UserDefaults suite (never the standard domain), so the host app
     /// can read them as a fallback for real command-line launch arguments

Tests/SandboxPilotKitTests/SandboxPilotKitTests.swift

@@ -23,6 +23,8 @@ struct WireProtocolTests {
             .windowAsKeyRequest(42),
             .windowResizeRequest(WindowResizeRequest(windowNumber: 7, frame: CGRect(x: 0, y: 0, width: 800, height: 500))),
             .userDefaultsPatch([PrefPatch(key: "k", value: .int(1))]),
+            .elementFrameRequest("search-field"),
+            .elementActionRequest(ElementAction(identifier: "search-field", kind: .focus)),
         ]
         for message in messages {
             let data = try encoder.encode(message)
@@ -41,6 +43,8 @@ struct WireProtocolTests {
             .environment(env),
             .windows([window]),
             .defaults([PrefPatch(key: "flag", value: .bool(true))]),
+            .elementActionResult(ElementActionResult(identifier: "search-field", kind: .focus, performed: true)),
+            .elementActionResult(ElementActionResult(identifier: "nope", kind: .focus, performed: false, failure: "not found")),
         ]
         for message in messages {
             let data = try encoder.encode(message)
@@ -81,6 +85,19 @@ struct LaunchParameterTests {
         #expect(AppearanceControl().change(to: .light) == false)
         #expect(AppearanceControl().change(to: .system) == false)
     }
+
+    // A plan naming a control the app does not expose has to say so, rather than
+    // quietly succeeding and leaving whoever wrote it to work out why the shot
+    // looks wrong. Nothing in this process publishes that identifier.
+    @Test("acting on an unknown identifier reports why it failed")
+    func unknownElementActionExplainsItself() async {
+        let result = await ElementLocator.perform(
+            ElementAction(identifier: "no.such.control", kind: .focus)
+        )
+        #expect(result.performed == false)
+        #expect(result.identifier == "no.such.control")
+        #expect(result.failure?.contains("no.such.control") == true)
+    }
 }
 
 @Suite("PrefPatch.Value")