feat(desktop): add targeted native actions

apps/app/src/native-settings.tsx

@@ -8,6 +8,7 @@ type Permission = Extract<NativeAction, { op: "permission" }>["permission"];
 const permissions = [
 	{ key: "accessibility", label: "Accessibility" },
 	{ key: "screenRecording", label: "Screen Recording" },
+	{ key: "eventSynthesizing", label: "Keyboard input" },
 ] as const;
 
 const states: Record<NativeState["state"], string> = {
@@ -68,24 +69,24 @@ export function NativeStatus(
 
 	const value = result.value;
 	const needsPermission = value?.state === "ready"
-		&& (!value.accessibility || !value.screenRecording);
+		&& permissions.some(({ key }) => !value[key]);
 	const error = action.error || result.error || value?.error;
 
 	return (
-		<section className="space-y-2" aria-label="Native inspection">
-			<h3 className="font-medium">Native inspection</h3>
+		<section className="space-y-2" aria-label="Computer use">
+			<h3 className="font-medium">Computer use</h3>
 			<p role="status">
 				{needsPermission
 					? "Permissions needed"
 					: value
 					? states[value.state]
 					: result.error
-					? "Could not check native inspection"
-					: "Checking native inspection…"}
+					? "Could not check computer use"
+					: "Checking computer use…"}
 			</p>
 			<p className="text-muted-foreground">
 				Keep Ace open on the host running the channel's tools. For a hosted channel, that is its
-				workspace. The channel's collaborator agent setting also applies to native inspection.
+				workspace. The channel's collaborator agent setting also applies to computer use.
 			</p>
 			<div className="space-y-2">
 				{permissions.map(({ key, label }) => (

apps/desktop/native/Package.swift

@@ -20,7 +20,11 @@ let package = Package(
 		),
 		.target(
 			name: "AceDesktop",
-			dependencies: ["AceSigning", .product(name: "PeekabooBridge", package: "Peekaboo")],
+			dependencies: [
+				"AceSigning",
+				.product(name: "PeekabooBridge", package: "Peekaboo"),
+				.product(name: "PeekabooAutomationKit", package: "Peekaboo"),
+			],
 			path: "sources/desktop"
 		),
 		.executableTarget(
@@ -29,6 +33,7 @@ let package = Package(
 				"AceSigning",
 				.product(name: "PeekabooBridge", package: "Peekaboo"),
 				.product(name: "PeekabooAutomationKit", package: "Peekaboo"),
+				.product(name: "PeekabooFoundation", package: "Peekaboo"),
 			],
 			path: "sources/client"
 		),

apps/desktop/native/patches/README.md

@@ -0,0 +1,22 @@
+# Native dependency patches
+
+`peekaboo-click.patch` applies to Peekaboo 4.8.0, revision
+`4d43dc9d80cd2aa3787a27f54b76d692db1dcf8f`. It addresses
+[self-targeted Accessibility clicks blocking Ace's native bridge](https://github.com/githubnext/ace2/issues/61).
+
+The patch moves semantic `AXPress` off MainActor so Ace can service its own Accessibility request.
+It captures result metadata before dispatch and avoids querying removed non-tab controls after a
+press. Exact target validation remains in place. The operation keeps its coordinator lane until
+the native call returns, including after client cancellation; no timeout is treated as completed
+input.
+Ambiguous native press failures retain an indeterminate outcome, preventing Peekaboo from
+falling back to another click after input may already have been delivered.
+
+The desktop build resolves only `Package.resolved` versions, checks the pin and checkout revision,
+and applies the patch before compiling Swift. A repeated build accepts it only when the reverse
+patch check succeeds. If neither direction applies, the build fails with Git's diagnostics rather
+than building an unexpected source state. Patch files are excluded from formatting.
+
+When updating Peekaboo, review the upstream fix and this patch together. Update the revision guard
+and patch deliberately, or remove both when the dependency includes the fix. Revalidate real
+button clicks, target refusal, and interruption before shipping the update.

apps/desktop/native/patches/peekaboo-click.patch

@@ -0,0 +1,143 @@
+diff --git a/Core/PeekabooAutomationKit/Sources/PeekabooAutomationKit/Services/UI/ActionInputDriver.swift b/Core/PeekabooAutomationKit/Sources/PeekabooAutomationKit/Services/UI/ActionInputDriver.swift
+--- a/Core/PeekabooAutomationKit/Sources/PeekabooAutomationKit/Services/UI/ActionInputDriver.swift
++++ b/Core/PeekabooAutomationKit/Sources/PeekabooAutomationKit/Services/UI/ActionInputDriver.swift
+@@ -172,7 +172,7 @@
+         beforeMutation: @MainActor () throws -> Void = {}) async throws -> UIInputExecutionResult.Action
+     {
+         do {
+-            return try self.performAction(AXActionNames.kAXPressAction, on: element, beforeMutation: beforeMutation)
++            return try await self.performPressAction(on: element, beforeMutation: beforeMutation)
+         } catch let error as ActionInputError
+             where error == .unsupported(.actionUnsupported) &&
+             allowAccessibilityValueFallback &&
+@@ -204,7 +204,52 @@
+         }
+     }
+
+-    /// Issues `AXShowMenu` without waiting on the menu's tracking runloop.
++    /// Keeps the operation lane owned until native completion while allowing a self-targeted app to service AX.
++    private func performPressAction(
++        on element: any AutomationElementRepresenting,
++        beforeMutation: @MainActor () throws -> Void) async throws -> UIInputExecutionResult.Action
++    {
++        guard let axElement = element.underlyingAXElement else {
++            return try self.performAction(AXActionNames.kAXPressAction, on: element, beforeMutation: beforeMutation)
++        }
++        try Task.checkCancellation()
++        guard element.supportsAction(AXActionNames.kAXPressAction) else {
++            throw ActionInputError.unsupported(.actionUnsupported)
++        }
++        // The action may remove this element; result metadata must not query it after dispatch.
++        let anchorPoint = element.anchorPoint
++        let elementRole = element.role
++        try beforeMutation()
++        try Task.checkCancellation()
++        do {
++            _ = try await DetachedAXActionRunner.perform(
++                action: AXActionNames.kAXPressAction,
++                on: axElement,
++                gracePeriod: nil)
++            return UIInputExecutionResult.Action(
++                outcome: .dispatchedUnverified(
++                    delivery: Self.accessibilityActionDelivery,
++                    evidence: .deliveryAccepted,
++                    unitCount: .one),
++                actionName: AXActionNames.kAXPressAction,
++                anchorPoint: anchorPoint,
++                elementRole: elementRole)
++        } catch {
++            let classified = Self.classify(error)
++            if Self.nativeMutationFailureMayHaveDispatched(classified) {
++                throw DesktopActionFailure.indeterminate(
++                    delivery: Self.accessibilityActionDelivery,
++                    evidence: .completionUnknown,
++                    unitCount: .one,
++                    message: "Accessibility press returned without reliable completion evidence.",
++                    hint: "Observe the exact target before retrying this click.",
++                    causeDescription: error.localizedDescription)
++            }
++            throw classified
++        }
++    }
++
++    /// Issues `AXShowMenu` without waiting on the menu's tracking runloop.
+     ///
+     /// A successful `AXShowMenu` opens an NSMenu whose tracking loop is a nested runloop on the
+     /// target app's main thread, so `AXUIElementPerformAction` does not return until the menu is
+diff --git a/Core/PeekabooAutomationKit/Sources/PeekabooAutomationKit/Services/UI/DetachedAXActionRunner.swift b/Core/PeekabooAutomationKit/Sources/PeekabooAutomationKit/Services/UI/DetachedAXActionRunner.swift
+--- a/Core/PeekabooAutomationKit/Sources/PeekabooAutomationKit/Services/UI/DetachedAXActionRunner.swift
++++ b/Core/PeekabooAutomationKit/Sources/PeekabooAutomationKit/Services/UI/DetachedAXActionRunner.swift
+@@ -24,10 +24,11 @@
+     /// modal loop should still be reported as delivered.
+     static let pressGracePeriod: TimeInterval = 2.0
+
+-    static func perform(
+-        action actionName: String,
+-        on element: AXUIElement,
+-        gracePeriod: TimeInterval) async throws -> DesktopActionOutcome
++    /// A nil grace period retains the awaiting operation and its lane until the native call actually returns.
++    static func perform(
++        action actionName: String,
++        on element: AXUIElement,
++        gracePeriod: TimeInterval?) async throws -> DesktopActionOutcome
+     {
+         let box = UncheckedAXElementBox(element: element)
+         return try await self.run(gracePeriod: gracePeriod) {
+@@ -39,8 +40,8 @@
+     /// it does not return within `gracePeriod`. Factored over a closure so tests can exercise the
+     /// race without a live accessibility element.
+     static func run(
+-        gracePeriod: TimeInterval,
+-        operation: @escaping @Sendable () -> AXError) async throws -> DesktopActionOutcome
++        gracePeriod: TimeInterval?,
++        operation: @escaping @Sendable () -> AXError) async throws -> DesktopActionOutcome
+     {
+         let gate = OneShotOutcomeGate()
+         let result = await withCheckedContinuation { (continuation: CheckedContinuation<
+@@ -58,11 +59,13 @@
+                     gate.resume(with: .failure(AccessibilitySystemError(result)))
+                 }
+             }
+-            DispatchQueue.global().asyncAfter(deadline: .now() + gracePeriod) {
+-                gate.resume(with: .success(.dispatchedUnverified(
+-                    delivery: self.delivery,
+-                    evidence: .operationStillRunning)))
+-            }
++            if let gracePeriod {
++                DispatchQueue.global().asyncAfter(deadline: .now() + gracePeriod) {
++                    gate.resume(with: .success(.dispatchedUnverified(
++                        delivery: self.delivery,
++                        evidence: .operationStillRunning)))
++                }
++            }
+         }
+         return try result.get()
+     }
+diff --git a/Core/PeekabooAutomationKit/Sources/PeekabooAutomationKit/Services/UI/ClickService.swift b/Core/PeekabooAutomationKit/Sources/PeekabooAutomationKit/Services/UI/ClickService.swift
+--- a/Core/PeekabooAutomationKit/Sources/PeekabooAutomationKit/Services/UI/ClickService.swift
++++ b/Core/PeekabooAutomationKit/Sources/PeekabooAutomationKit/Services/UI/ClickService.swift
+@@ -193,8 +193,9 @@
+             validateProcessIdentity: validatesProcessIdentity)
+         switch clickType {
+         case .single:
+-            let valueBefore = element.intAttribute(AXAttributeNames.kAXValueAttribute)
+-            let originalIdentity = element.focusedElementIdentity
++            let valueBefore = element.intAttribute(AXAttributeNames.kAXValueAttribute)
++            let requiresTabVerification = element.subrole == "AXTabButton" && valueBefore == 0
++            let originalIdentity = element.focusedElementIdentity
+             let originalRole = element.role
+             let result = try await self.actionInputDriver.tryClick(
+                 element: element,
+@@ -228,7 +229,11 @@
+             }
+             let tabPressDidNotSelect: Bool
+             do {
+-                tabPressDidNotSelect = try await self.tabSelectionVerifier(element, valueBefore)
++                if requiresTabVerification {
++                    tabPressDidNotSelect = try await self.tabSelectionVerifier(element, valueBefore)
++                } else {
++                    tabPressDidNotSelect = false
++                }
+             } catch {
+                 guard result.outcome.dispatchState.mutationDispatched else { throw error }
+                 throw Self.unconfirmedTabPressFailure(result, cause: error)