Embed native desktop inspection in Ace

.github/workflows/desktop-release.yml

@@ -30,6 +30,7 @@ jobs:
     timeout-minutes: 60
     environment: desktop-${{ inputs.channel }}
     env:
+      DEVELOPER_DIR: /Applications/Xcode_26.2.app/Contents/Developer
       ACE_CODESIGN_IDENTITY: ${{ vars.ACE_CODESIGN_IDENTITY }}
       ACE_UPDATE_URL: ${{ vars.ACE_UPDATE_URL }}
       ACE_UPDATE_PUBLIC_KEY: ${{ vars.ACE_UPDATE_PUBLIC_KEY }}

.gitignore

@@ -10,3 +10,5 @@ artifacts/
 .dev.vars
 .wrangler/
 dist-web/
+apps/desktop/native/.build/
+apps/desktop/native/.swiftpm/

apps/app/src/desktop.ts

@@ -1,7 +1,7 @@
 import { Electroview } from "electrobun/view";
 import type { MouseEvent } from "react";
 
-import type { DesktopRPC, HelperAction, UpdateAction } from "@ace/desktop/protocol";
+import type { DesktopRPC, HelperAction, NativeAction, UpdateAction } from "@ace/desktop/protocol";
 
 import { nativeToken } from "./host";
 
@@ -20,6 +20,7 @@ export const desktop = view && token
 		lights: (expanded: boolean) => rpc.request.lights({ token, expanded }),
 		zoom: () => rpc.request.zoom({ token }),
 		helper: (action: HelperAction) => rpc.request.helper({ token, action }),
+		native: (action: NativeAction) => rpc.request.native({ token, action }),
 		updates: (action: UpdateAction) => rpc.request.updates({ token, action }),
 	}
 	: undefined;

apps/app/src/host-settings.tsx

@@ -6,6 +6,7 @@ import type { Diagnostics } from "@ace/host/protocol";
 
 import { desktop } from "./desktop";
 import { host } from "./host";
+import { NativeStatus } from "./native-settings";
 
 const tailnet = {
 	missing: "Tailscale is not installed",
@@ -103,7 +104,11 @@ export function HostStatus() {
 									>
 										Restart Ace Helper
 									</Button>
-									<Button variant="outline" disabled={!available} onClick={() => setConfirm(true)}>
+									<Button
+										variant="outline"
+										disabled={!available}
+										onClick={() => setConfirm(true)}
+									>
 										Stop Ace Helper…
 									</Button>
 								</>
@@ -141,6 +146,7 @@ export function HostStatus() {
 					</div>
 				</section>
 			)}
+			{desktop && <NativeStatus native={desktop.native} refresh={retry} />}
 			{status !== "open"
 				? <p role="status">Connect to Ace Helper to check tools and team connectivity.</p>
 				: checks.value

apps/app/src/native-settings.tsx

@@ -0,0 +1,110 @@
+import { useEffect, useState } from "react";
+
+import { Button } from "@ace/ui";
+import type { NativeAction, NativeState } from "@ace/desktop/protocol";
+
+type Permission = Extract<NativeAction, { op: "permission" }>["permission"];
+
+const permissions = [
+	{ key: "accessibility", label: "Accessibility" },
+	{ key: "screenRecording", label: "Screen Recording" },
+] as const;
+
+const states: Record<NativeState["state"], string> = {
+	stopped: "Not running",
+	starting: "Starting…",
+	ready: "Ready",
+	stopping: "Stopping…",
+	error: "Needs attention",
+};
+
+export function NativeStatus(
+	{ native, refresh }: { native: (action: NativeAction) => Promise<NativeState>; refresh: number },
+) {
+	const [result, setResult] = useState<{ value?: NativeState; error?: string }>({});
+	const [action, setAction] = useState<{ busy?: Permission; error?: string }>({});
+	const [retry, setRetry] = useState(0);
+
+	useEffect(() => {
+		let active = true;
+		let pending = false;
+		const check = () => {
+			if (document.visibilityState !== "visible" || pending) return;
+			pending = true;
+			native({ op: "status" }).then(
+				(value) => {
+					if (active) setResult({ value });
+				},
+				(error: Error) => {
+					if (active) setResult({ error: error.message });
+				},
+			).finally(() => {
+				pending = false;
+			});
+		};
+		check();
+		const timer = window.setInterval(check, 3000);
+		window.addEventListener("focus", check);
+		document.addEventListener("visibilitychange", check);
+		return () => {
+			active = false;
+			window.clearInterval(timer);
+			window.removeEventListener("focus", check);
+			document.removeEventListener("visibilitychange", check);
+		};
+	}, [native, refresh, retry]);
+
+	async function grant(permission: Permission) {
+		setAction({ busy: permission });
+		try {
+			const value = await native({ op: "permission", permission });
+			setResult({ value });
+			setAction(value.error ? { error: value.error } : {});
+		} catch (error) {
+			setAction({ error: (error as Error).message });
+		}
+		setRetry(value => value + 1);
+	}
+
+	const value = result.value;
+	const needsPermission = value?.state === "ready"
+		&& (!value.accessibility || !value.screenRecording);
+	const error = action.error || result.error || value?.error;
+
+	return (
+		<section className="space-y-2" aria-label="Native inspection">
+			<h3 className="font-medium">Native inspection</h3>
+			<p role="status">
+				{needsPermission
+					? "Permissions needed"
+					: value
+					? states[value.state]
+					: result.error
+					? "Could not check native inspection"
+					: "Checking native inspection…"}
+			</p>
+			<p className="text-muted-foreground">
+				Keep Ace open on the host running the channel's tools. For a hosted channel, that is its
+				workspace. The channel's collaborator agent setting also applies to native inspection.
+			</p>
+			<div className="space-y-2">
+				{permissions.map(({ key, label }) => (
+					<div key={key} className="flex items-center justify-between gap-3">
+						<p>{label}: {value ? value[key] ? "Granted" : "Not granted" : "Not checked"}</p>
+						{!value?.[key] && (
+							<Button
+								variant="outline"
+								disabled={!!action.busy}
+								onClick={() => void grant(key)}
+							>
+								Enable {label}
+							</Button>
+						)}
+					</div>
+				))}
+			</div>
+			{action.busy && <p role="status">Requesting permission…</p>}
+			{error && <p role="alert" className="text-destructive">{error}</p>}
+		</section>
+	);
+}

apps/app/src/timeline.ts

@@ -22,7 +22,20 @@ function tool(tool: Tool, busy: boolean): Event.Message.Content.Tool {
 		name: tool.name,
 		arguments: (tool.args ?? {}) as Record<string, unknown>,
 		status,
-		...(tool.result ? { result: { content: tool.result.text } } : {}),
+		...(tool.result
+			? {
+				result: {
+					content: tool.result.text,
+					...(tool.result.images
+						? {
+							images: tool.result.images.map((image) =>
+								`data:${image.mimeType};base64,${image.data}`
+							),
+						}
+						: {}),
+				},
+			}
+			: {}),
 	};
 }
 

apps/app/src/transcript.ts

@@ -9,7 +9,7 @@ export type Tool = {
 	call: string;
 	name: string;
 	args: unknown;
-	result?: { error: boolean; text: string };
+	result?: { error: boolean; text: string; images?: Image[] };
 };
 
 export type Item =
@@ -99,7 +99,14 @@ export function apply(state: Transcript, event: Event): Transcript {
 							...(event.stopped ? { stopped: true } : {}),
 							tools: item.tools.map((tool) =>
 								tool.call === event.call
-									? { ...tool, result: { error: event.error, text: event.text } }
+									? {
+										...tool,
+										result: {
+											error: event.error,
+											text: event.text,
+											...(event.images ? { images: event.images } : {}),
+										},
+									}
 									: tool
 							),
 						}

apps/desktop/native/Package.swift

@@ -0,0 +1,37 @@
+// swift-tools-version: 6.2
+
+import PackageDescription
+
+let package = Package(
+	name: "AceNative",
+	platforms: [.macOS(.v15)],
+	products: [
+		.library(name: "AceDesktop", type: .dynamic, targets: ["AceDesktop"]),
+		.executable(name: "ace-desktop-client", targets: ["AceDesktopClient"]),
+	],
+	dependencies: [
+		.package(url: "https://github.com/openclaw/Peekaboo.git", exact: "4.8.0"),
+	],
+	targets: [
+		.target(
+			name: "AceSigning",
+			path: "sources/signing",
+			linkerSettings: [.linkedFramework("Security")]
+		),
+		.target(
+			name: "AceDesktop",
+			dependencies: ["AceSigning", .product(name: "PeekabooBridge", package: "Peekaboo")],
+			path: "sources/desktop"
+		),
+		.executableTarget(
+			name: "AceDesktopClient",
+			dependencies: [
+				"AceSigning",
+				.product(name: "PeekabooBridge", package: "Peekaboo"),
+				.product(name: "PeekabooAutomationKit", package: "Peekaboo"),
+			],
+			path: "sources/client"
+		),
+	],
+	swiftLanguageModes: [.v6]
+)