From 81b83afac7f75d6e28c75ccc5b5c7207bab7cd52 Mon Sep 17 00:00:00 2001 From: iamnbutler Date: Sat, 17 Oct 2026 04:53:53 -0400 Subject: [PATCH] feat: turn on Chromium and Electron web accessibility before inspect and read, once per app launch The host sets AXManualAccessibility (never AXEnhancedUserInterface) through a new enableWebAccessibility Bridge operation, before reading an app that ships Chromium or after a read finds a nearly empty web area, then reads again. Results report web_accessibility; web_accessibility false on inspect and read opts out. It runs while input is paused and is listed once per launch in the activity list. Also lists the #5 background-paste spike questions as a manual live check. --- dist/build.js | 1 + dist/client.js | 13 + dist/protocol.d.ts | 4 + dist/tools.js | 11 +- docs/app.md | 4 +- docs/live-checks.md | 3 + docs/tools.md | 9 + native/patches/README.md | 6 + .../patches/peekaboo-web-accessibility.patch | 300 ++++++++++++++++++ native/sources/client/client.swift | 70 ++-- native/sources/client/read.swift | 43 ++- native/sources/client/webaccessibility.swift | 90 ++++++ native/sources/runtime/activity.swift | 32 ++ native/sources/runtime/runtime.swift | 2 +- scripts/live-check.mjs | 35 ++ src/build.ts | 1 + src/client.ts | 13 + src/protocol.ts | 4 + src/tools.ts | 12 +- test/webaccessibility.test.mjs | 28 ++ 20 files changed, 638 insertions(+), 43 deletions(-) create mode 100644 native/patches/peekaboo-web-accessibility.patch create mode 100644 native/sources/client/webaccessibility.swift create mode 100644 test/webaccessibility.test.mjs diff --git a/dist/build.js b/dist/build.js index 01e231c..0f84879 100644 --- a/dist/build.js +++ b/dist/build.js @@ -67,6 +67,7 @@ const patches = [ "peekaboo-operation-record.patch", "peekaboo-input-source.patch", "peekaboo-attachment-insert.patch", + "peekaboo-web-accessibility.patch", ].map((name) => join(native, "patches", name)); function spawn(command, env) { return spawnSync(command[0], command.slice(1), { encoding: "utf8", env: env ?? process.env }); diff --git a/dist/client.js b/dist/client.js index 1b0ca9c..3d038a5 100644 --- a/dist/client.js +++ b/dist/client.js @@ -129,6 +129,13 @@ function bounded(data, field, limit = MAX_TEXT) { } return text; } +/** The native argument that leaves a Chromium or Electron app's web accessibility as it is. */ +const KEEP_WEB_ACCESSIBILITY = "keep-web-accessibility"; +function validateWebAccessibility(request) { + if (request.web_accessibility !== undefined && typeof request.web_accessibility !== "boolean") { + throw new Error("Choose web_accessibility true or false."); + } +} function validateScope(scope) { if (!scope || typeof scope !== "object" || Object.keys(scope).length !== 2 || !("snapshot" in scope) || !("element" in scope) @@ -275,6 +282,7 @@ async function inspect(native, request, signal) { if (mode !== "accessibility") throw new Error("A scope needs accessibility mode."); } + validateWebAccessibility(request); const savePath = await validateSave(request); const directory = await mkdtemp(join(tmpdir(), "desktop-tools-")); try { @@ -284,6 +292,8 @@ async function inspect(native, request, signal) { args.push("ocr"); if (request.scope) args.push("scope", request.scope.snapshot, request.scope.element); + if (request.web_accessibility === false) + args.push(KEEP_WEB_ACCESSIBILITY); const data = await native(args, signal, { target: request }); if (request.scope && data.scope?.from_element !== request.scope.element) { throw new Error("Native inspection did not honor the requested scope."); @@ -370,6 +380,9 @@ async function read(native, request, signal) { validateScope(request.scope); args.push(request.scope.snapshot, request.scope.element); } + validateWebAccessibility(request); + if (request.web_accessibility === false) + args.push(KEEP_WEB_ACCESSIBILITY); if (request.table !== undefined || request.row_offset !== undefined || request.max_rows !== undefined) { if (request.table !== true || !request.scope) { throw new Error("A table read needs table true and scope: the table, or the area that holds it, from an accessibility desktop_inspect."); diff --git a/dist/protocol.d.ts b/dist/protocol.d.ts index 7611de0..f2de20d 100644 --- a/dist/protocol.d.ts +++ b/dist/protocol.d.ts @@ -181,11 +181,15 @@ export type DesktopRequest = DesktopManagement | DesktopLaunch | DesktopOpen | D mode?: "accessibility" | "pixels"; ocr?: boolean; scope?: DesktopScope; + /** false leaves a Chromium or Electron app's web accessibility as it is. */ + web_accessibility?: boolean; } & DesktopCaptureSave) | { op: "read"; pid: number; window: number; scope?: DesktopScope; + /** false leaves a Chromium or Electron app's web accessibility as it is. */ + web_accessibility?: boolean; /** Reads the scope as a table of rows and cells instead of text. */ table?: boolean; row_offset?: number; diff --git a/dist/tools.js b/dist/tools.js index e855f23..1a92eb9 100644 --- a/dist/tools.js +++ b/dist/tools.js @@ -159,6 +159,11 @@ const saveProperties = { }, }, }; +const webAccessibility = { + type: "boolean", + description: "false leaves a Chromium or Electron app's web accessibility as it is, so its web content may be missing.", +}; +const webAccessibilityNote = "in a Chromium or Electron app, or a window whose web area is nearly empty, it first turns on the app's web accessibility so web content appears, once per app launch, and reports web_accessibility; it stays on until the app quits. Pass web_accessibility false to leave the app as it is."; const saveNote = "save_path, an absolute .png path in an existing folder, keeps the full-resolution capture; region, in global points, saves only part of it; an existing file needs overwrite true."; export function desktopTools({ place, app }) { return [ @@ -540,7 +545,7 @@ export function desktopTools({ place, app }) { { name: "desktop_inspect", op: "inspect", - description: `Read one window's Accessibility elements and a screenshot by pid and window, returning the single-use snapshot_id that input tools take. Read-only; never activates or focuses. mode pixels, for windows Accessibility can't read, gives a snapshot for point clicks, scrolls and drags only. A large window lists its first 200 elements and reports truncated: pass scope (a container's snapshot and element from an earlier inspection) to list its subtree with a much larger budget. ocr true adds recognized_text, whose box centers are points for this snapshot. ${saveNote}`, + description: `Read one window's Accessibility elements and a screenshot by pid and window, returning the single-use snapshot_id that input tools take. Read-only and never activates or focuses, except that ${webAccessibilityNote} mode pixels, for windows Accessibility can't read, gives a snapshot for point clicks, scrolls and drags only. A large window lists its first 200 elements and reports truncated: pass scope (a container's snapshot and element from an earlier inspection) to list its subtree with a much larger budget. ocr true adds recognized_text, whose box centers are points for this snapshot. ${saveNote}`, parameters: { type: "object", required: ["pid", "window"], @@ -550,6 +555,7 @@ export function desktopTools({ place, app }) { mode: { type: "string", enum: ["accessibility", "pixels"] }, ocr: { type: "boolean" }, scope: scope, + web_accessibility: webAccessibility, ...saveProperties, }, }, @@ -557,7 +563,7 @@ export function desktopTools({ place, app }) { { name: "desktop_read", op: "read", - description: `Read one window's text by pid and window through Accessibility, up to 96 KB, including content scrolled out of view; scope reads one inspected element's subtree. Read-only, with no snapshot. Windows without Accessibility text, such as canvases, return empty text: use desktop_inspect with ocr. With table true and a scope, it reads the first table there instead: columns, rows with their cells, row element IDs for desktop_click, selected, and depth/expanded for outlines; page with row_offset and max_rows (default ${TABLE_ROWS_DEFAULT}, at most ${TABLE_ROWS_MAX}).`, + description: `Read one window's text by pid and window through Accessibility, up to 96 KB, including content scrolled out of view; scope reads one inspected element's subtree. Read-only, with no snapshot, except that it turns on web accessibility as desktop_inspect does. Windows without Accessibility text, such as canvases, return empty text: use desktop_inspect with ocr. With table true and a scope, it reads the first table there instead: columns, rows with their cells, row element IDs for desktop_click, selected, and depth/expanded for outlines; page with row_offset and max_rows (default ${TABLE_ROWS_DEFAULT}, at most ${TABLE_ROWS_MAX}).`, parameters: { type: "object", required: ["pid", "window"], @@ -565,6 +571,7 @@ export function desktopTools({ place, app }) { pid: { type: "integer", minimum: 1 }, window: { type: "integer", minimum: 1 }, scope: scope, + web_accessibility: webAccessibility, table: { type: "boolean", const: true }, row_offset: { type: "integer", minimum: 0, maximum: 100_000 }, max_rows: { type: "integer", minimum: 1, maximum: TABLE_ROWS_MAX }, diff --git a/docs/app.md b/docs/app.md index fd6580d..21612ea 100644 --- a/docs/app.md +++ b/docs/app.md @@ -15,9 +15,9 @@ Anything running as the user can use those grants while the app serves, so grant - the time; - the outcome: completed, unknown, refused, or paused for a request refused because input was paused. -Identical operations in a row fold into one line with a count. Every desktop change is listed, and so are screen captures and clipboard reads. Listings and snapshot bookkeeping are not. Only the operation and its target are kept, never typed text, clipboard contents or values. The list lives in memory and is gone when the app quits. +Identical operations in a row fold into one line with a count. Every desktop change is listed, and so are screen captures and clipboard reads. Turning on an app's web accessibility for a read is listed once per launch of that app. Listings and snapshot bookkeeping are not. Only the operation and its target are kept, never typed text, clipboard contents or values. The list lives in memory and is gone when the app quits. -**Pause All Input** stops agents from changing anything until you turn it off again. While paused, the app refuses every click, key, typing, scroll, drag, menu command, app or window change, dialog action and clipboard write before it is sent. The refusal tells the agent to ask you to resume. Agents can still observe: listings, the accessibility tree, screen captures and clipboard reads keep working. The menu bar icon becomes a pause sign while input is paused. Only this menu item changes it; no client can pause or resume. Pausing isn't saved, so the app starts resumed. +**Pause All Input** stops agents from changing anything until you turn it off again. While paused, the app refuses every click, key, typing, scroll, drag, menu command, app or window change, dialog action and clipboard write before it is sent. The refusal tells the agent to ask you to resume. Agents can still observe: listings, the accessibility tree, screen captures and clipboard reads keep working. So does turning on a Chromium or Electron app's web accessibility before a read, since it only lets the app's web content be read and sends no input (see [tools](tools.md#input)). The menu bar icon becomes a pause sign while input is paused. Only this menu item changes it; no client can pause or resume. Pausing isn't saved, so the app starts resumed. ## Build diff --git a/docs/live-checks.md b/docs/live-checks.md index 2515205..210329e 100644 --- a/docs/live-checks.md +++ b/docs/live-checks.md @@ -17,6 +17,7 @@ What it touches: - It makes its own fixtures in a new `/tmp/desktop-tools-live-*` folder: a plain and a rich TextEdit document, a Finder folder of eight files and a subfolder, a PNG, and a web page with a React-style controlled input, a date input, a contenteditable, a table with `` headers and a grid of inputs. The folder is removed at the end unless `--keep-fixtures` is passed. - It opens those fixtures in TextEdit, Finder and Safari with `desktop_open`, and acts only on the new windows. At the end, even after a failure or Control-C, it closes each window it opened and quits only the apps it launched itself. A fixture document that asks to save is answered with Delete or Don't Save, on that document only. - Web checks are skipped while Safari has windows open, since a fixture page could open as a tab in one of them. +- The Chromium check (#107) runs only when Google Chrome is installed and not running. It opens the fixture page in Chrome, which may also restore the user's last Chrome session, and quits Chrome at the end. - Clipboard checks run only when the clipboard holds plain text, and that text is written back at the end. The input-source check selects another enabled keyboard layout, then the original again. - Modifier clicks (#77) and `desktop_system_key` (#89) use the real keyboard and pointer, as they always do; nothing else needs the foreground. @@ -38,5 +39,7 @@ These stay manual, because they are destructive, change system settings, need a - **#90 input methods:** typing in an input method's mode, such as Japanese Hiragana, needs that method enabled. - **#86 and #92 in other apps:** Numbers, Excel, Mail and Chrome, and a long Finder folder of 300 or more files. - **#88 in other apps:** Chrome, read-only native text views and Preview PDFs. +- **#107 web accessibility in Electron apps:** inspect a Slack, VS Code or Discord window and check that `web_accessibility` is `enabled` once per launch and the web content's fields are listed, that the app's window still moves and resizes normally, that All Activity lists **Turn on web accessibility** once, and that it still runs while input is paused. +- **#5 background paste into web content (spike, before any patch):** with a WebKit app such as Safari in the background and a textarea focused, answer: (1) is Edit > Paste `AXEnabled`, with and without the host's background key-window preparation; (2) does an Accessibility press of Edit > Paste paste into the focused textarea; (3) must the textarea get `AXFocused` first? - **#27 Codex:** adding the server to Codex's configuration. - Anything that asks for a password or an administrator's approval. diff --git a/docs/tools.md b/docs/tools.md index 5edfe74..ad629c0 100644 --- a/docs/tools.md +++ b/docs/tools.md @@ -141,6 +141,14 @@ A `desktop_click` with `modifiers` (any of `command`, `option` and `shift`), suc `desktop_inspect` reads at most 200 elements of a window, so a long web page, mail list or Finder folder is cut short and reports `truncated`. Two tools reach further without input. `desktop_read` reads the window's text with a budget of 4,000 elements, 64 levels deep and 1,000 children per element, within about 18 seconds; it returns the text in tree order, one line per visual line, and names each limit that cut it short. Pass `scope`, a `snapshot_id` and an element ID from an earlier inspection of the same window, to read only that element, such as the page's web area. `desktop_inspect` takes the same `scope`: it reads the window with that larger budget and lists only the scope element and its descendants, each with `attributes.depth`. The scope element is found again by its role, depth, bounds, label, title and identifier, so it must not have moved; listed IDs belong to the new snapshot, like any inspection. +Chromium and Electron apps, such as Chrome, Edge, Slack, VS Code and Discord, build the Accessibility tree of their web content only after an assistive app asks for it. Until then a web view lists few or no elements. So before `desktop_inspect` or `desktop_read` reads an app whose bundle ships an Electron, Chromium Embedded or Chromium framework, the host sets `AXManualAccessibility` on that app. It also does so after a read that finds a web area with at most one element under it, and then reads the window once more. It waits half a second after turning it on, so the page can build its tree. The host sets it at most once per launch of the app, and it stays on until the app quits, since VoiceOver or another assistive app may rely on it too. It never sets `AXEnhancedUserInterface`, which changes how some apps animate and place their windows. The result reports `web_accessibility`: +- **`enabled`**: this read turned it on. +- **`already_on`**: it was already on, from an earlier read or another assistive app. +- **`unsupported`**: the app doesn't accept it, and nothing changed. +- **`unknown`**: the request failed or wasn't confirmed; the read went ahead with the tree as it was. + +Pass `web_accessibility: false` to leave the app as it is; its web content may then be missing. Turning it on sends no input, so it runs while input is paused, and the app's activity list shows it once per app launch as **Turn on web accessibility**. A table read uses the tree as an earlier inspection left it. + `desktop_read` with `table: true` reads a table instead of text. Its `scope` is a table, outline or grid from an accessibility inspection, such as a Finder list, a Mail message list, a Numbers table or a web table, or an area that holds one, such as the scroll area or web area; the first table in it is read, and `tables_in_scope` counts the others. It reads Accessibility alone, with `desktop_read`'s budget and no screenshot, and returns: - `columns`: the header titles in order. They come from the table's columns, from the header buttons a scroll view draws just above the table, or from a web table's header row; `null` when there are none. Scope the scroll area rather than the table when a list's headers are missing. @@ -230,6 +238,7 @@ These are the rest of each tool's contract, beyond its description. - **Accessibility mode** (the default) can return an action snapshot with element IDs. - **Pixels mode** returns a snapshot with no element IDs and no focused control. It authorizes only point clicks, point scrolls and drags in that exact window. - **After an action,** the automatic observation uses accessibility mode. If it fails, inspect again with pixels mode. +- **In a Chromium or Electron app,** it first turns on the app's web accessibility, once per launch, unless `web_accessibility` is false. The result's `web_accessibility` says what happened; see [Input](#input). **`desktop_click`.** A single left click on a supported editable text field, by element or point, also requests keyboard focus. The result says whether focus was verified; use `desktop_select` to choose a range or caret position. Points stay bound to the captured window, even when the returned screenshot was scaled. diff --git a/native/patches/README.md b/native/patches/README.md index 43469c5..0ecc8f0 100644 --- a/native/patches/README.md +++ b/native/patches/README.md @@ -555,3 +555,9 @@ guard and patches deliberately, or remove each patch when the dependency include Revalidate real button clicks, changed-focus refusal, literal multiline insertion, clipboard restoration, partial input, and interruption before shipping the update. + +`peekaboo-web-accessibility.patch` adds an `enableWebAccessibility` operation, which sets `AXManualAccessibility` on one app so that Chromium and Electron build the Accessibility tree of their web content. Until an assistive client asks, their web views list few or no elements. It never sets `AXEnhancedUserInterface`, which changes how some apps animate and place their windows. +- **Request.** It carries the exact app generation, as a process identity. +- **Host.** On the main actor, the host checks that the process still has that start identity, or answers `notFound`. It keeps the generations it turned on, or found on, and answers those without touching the app again, so it sets the attribute at most once per launch. Otherwise it reads the attribute, with a one-second messaging timeout, and sets it only when it isn't already on. An app that rejects the attribute is `operationNotSupported`. Any other AX error is `internalError`, marked as possibly completed when the app timed out. +- **Result contract.** It needs Accessibility. It is a request-pinned, process-scoped, one-unit Accessibility-value background delivery answered with `bool`: true when this request turned it on. The outcome is `confirmedChange` when the attribute reads back on, `dispatchedUnverified` when it doesn't read back, and `confirmedNoChange` when it was already on. It is service-owned and joins the default allowlists. +- **Tests.** The wire count is now 133. The operation joins the unconditionally mutating set and the Accessibility permission partition. diff --git a/native/patches/peekaboo-web-accessibility.patch b/native/patches/peekaboo-web-accessibility.patch new file mode 100644 index 0000000..080bca6 --- /dev/null +++ b/native/patches/peekaboo-web-accessibility.patch @@ -0,0 +1,300 @@ +diff --git a/Core/PeekabooCore/Sources/PeekabooBridge/PeekabooBridgeClient+Interaction.swift b/Core/PeekabooCore/Sources/PeekabooBridge/PeekabooBridgeClient+Interaction.swift +index 21d7f6a1f..1ed6bb825 100644 +--- a/Core/PeekabooCore/Sources/PeekabooBridge/PeekabooBridgeClient+Interaction.swift ++++ b/Core/PeekabooCore/Sources/PeekabooBridge/PeekabooBridgeClient+Interaction.swift +@@ -319,6 +319,19 @@ extension PeekabooBridgeClient { + } + } + ++ /// Turns on one app generation's web Accessibility tree; true when this request turned it on. ++ public func enableWebAccessibilityWithOutcome( ++ _ identity: ApplicationProcessIdentity) async throws -> UIAutomationActionResult ++ { ++ try await self.actionResult( ++ for: .enableWebAccessibility(PeekabooBridgeEnableWebAccessibilityRequest(expectedIdentity: identity)), ++ expectedResponse: "web accessibility") ++ { response in ++ guard case let .bool(turnedOn) = response else { return nil } ++ return turnedOn ++ } ++ } ++ + public func waitForElement(target: ClickTarget, timeout: TimeInterval, snapshotId: String?) async throws + -> WaitForElementResult + { +diff --git a/Core/PeekabooCore/Sources/PeekabooBridge/PeekabooBridgeModels.swift b/Core/PeekabooCore/Sources/PeekabooBridge/PeekabooBridgeModels.swift +index cad033bc6..77a11d7ae 100644 +--- a/Core/PeekabooCore/Sources/PeekabooBridge/PeekabooBridgeModels.swift ++++ b/Core/PeekabooCore/Sources/PeekabooBridge/PeekabooBridgeModels.swift +@@ -97,6 +97,7 @@ public enum PeekabooBridgeOperation: String, Codable, Sendable, CaseIterable, Ha + case moveMouse + case systemDefinedKey + case selectInputSource ++ case enableWebAccessibility + case waitForElement + // Windows + case listWindows +diff --git a/Core/PeekabooCore/Sources/PeekabooBridge/PeekabooBridgeOperation+Policy.swift b/Core/PeekabooCore/Sources/PeekabooBridge/PeekabooBridgeOperation+Policy.swift +index 03e7dbac0..943448ed0 100644 +--- a/Core/PeekabooCore/Sources/PeekabooBridge/PeekabooBridgeOperation+Policy.swift ++++ b/Core/PeekabooCore/Sources/PeekabooBridge/PeekabooBridgeOperation+Policy.swift +@@ -79,6 +79,7 @@ extension PeekabooBridgeOperation { + .moveMouse, + .systemDefinedKey, + .selectInputSource, ++ .enableWebAccessibility, + .waitForElement, + .listWindows, + .focusWindow, +diff --git a/Core/PeekabooCore/Sources/PeekabooBridge/PeekabooBridgeOperationDescriptor.swift b/Core/PeekabooCore/Sources/PeekabooBridge/PeekabooBridgeOperationDescriptor.swift +index 1771b3693..c3a4fa9dc 100644 +--- a/Core/PeekabooCore/Sources/PeekabooBridge/PeekabooBridgeOperationDescriptor.swift ++++ b/Core/PeekabooCore/Sources/PeekabooBridge/PeekabooBridgeOperationDescriptor.swift +@@ -301,6 +301,13 @@ extension PeekabooBridgeOperationResultSemantics { + completion: .dispatchedUnverified(globalForeground), + targetPolicy: .global, + responseFamilies: [.ok]) ++ case .enableWebAccessibility: ++ descriptor( ++ ownership: .service, ++ requiredPermissions: [.accessibility], ++ completion: .dispatchedUnverified(valueBackground), ++ targetPolicy: .requestPinned, ++ responseFamilies: [.bool]) + case .selectInputSource: + descriptor( + ownership: .service, +diff --git a/Core/PeekabooCore/Sources/PeekabooBridge/PeekabooBridgeOperationReceipts.swift b/Core/PeekabooCore/Sources/PeekabooBridge/PeekabooBridgeOperationReceipts.swift +index 068b36c27..c22d7799e 100644 +--- a/Core/PeekabooCore/Sources/PeekabooBridge/PeekabooBridgeOperationReceipts.swift ++++ b/Core/PeekabooCore/Sources/PeekabooBridge/PeekabooBridgeOperationReceipts.swift +@@ -1351,6 +1351,9 @@ extension PeekabooBridgeRequest { + case let .menuCommand(payload): + [.init(processIdentifier: payload.expectedIdentity.processIdentifier, + processIdentity: payload.expectedIdentity)] ++ case let .enableWebAccessibility(payload): ++ [.init(processIdentifier: payload.expectedIdentity.processIdentifier, ++ processIdentity: payload.expectedIdentity)] + case let .clickMenuItem(payload): + payload.expectedIdentity.map { + [.init(processIdentifier: $0.processIdentifier, processIdentity: $0)] +diff --git a/Core/PeekabooCore/Sources/PeekabooBridge/PeekabooBridgeOperationResultSemantics.swift b/Core/PeekabooCore/Sources/PeekabooBridge/PeekabooBridgeOperationResultSemantics.swift +index 2d8bc42c2..66927f5ea 100644 +--- a/Core/PeekabooCore/Sources/PeekabooBridge/PeekabooBridgeOperationResultSemantics.swift ++++ b/Core/PeekabooCore/Sources/PeekabooBridge/PeekabooBridgeOperationResultSemantics.swift +@@ -324,6 +324,8 @@ extension PeekabooBridgeOperationResultSemantics { + } + case let .menuCommand(payload): + .process(payload.expectedIdentity) ++ case let .enableWebAccessibility(payload): ++ .process(payload.expectedIdentity) + case let .clickMenuItem(payload): + payload.expectedIdentity.map(DesktopOperationScope.process) ?? .global + case let .clickMenuItemByName(payload): +@@ -495,6 +497,7 @@ extension PeekabooBridgeOperationResultSemantics { + .moveMouse, + .systemDefinedKey, + .selectInputSource, ++ .enableWebAccessibility, + .waitForElement, + .listWindows, + .listWindowMutationInventory, +@@ -638,6 +641,8 @@ extension PeekabooBridgeOperationResultSemantics { + return [.confirmedNoChange, .dispatchedUnverified] + case .quitApplication: + return [.confirmedChange, .dispatchedUnverified, .suspectedNoop] ++ case .enableWebAccessibility: ++ return [.confirmedChange, .confirmedNoChange, .dispatchedUnverified] + case .hideDock, .showDock: + return [.confirmedChange, .confirmedNoChange, .dispatchedUnverified, .suspectedNoop] + case .unhideApplication, .dialogClickButton, .backgroundDialogClickButton, +@@ -948,6 +953,8 @@ extension PeekabooBridgeOperationResultSemantics { + return [rule(axBackground, .variable), rule(nativeBackground, .variable)] + case .menuCommand: + return [rule(axBackground, .exact(1))] ++ case .enableWebAccessibility: ++ return [rule(valueBackground, .exact(1))] + case let .clickMenuItem(payload): + return [rule( + payload.deliveryMode == .background ? axBackground : axForeground, +diff --git a/Core/PeekabooCore/Sources/PeekabooBridge/PeekabooBridgePayloads.swift b/Core/PeekabooCore/Sources/PeekabooBridge/PeekabooBridgePayloads.swift +index 06809ce88..941065a43 100644 +--- a/Core/PeekabooCore/Sources/PeekabooBridge/PeekabooBridgePayloads.swift ++++ b/Core/PeekabooCore/Sources/PeekabooBridge/PeekabooBridgePayloads.swift +@@ -406,6 +406,15 @@ public struct PeekabooBridgeSelectInputSourceRequest: Codable, Sendable { + } + } + ++public struct PeekabooBridgeEnableWebAccessibilityRequest: Codable, Sendable { ++ /// The exact app generation whose web content should build its Accessibility tree. ++ public let expectedIdentity: ApplicationProcessIdentity ++ ++ public init(expectedIdentity: ApplicationProcessIdentity) { ++ self.expectedIdentity = expectedIdentity ++ } ++} ++ + public struct PeekabooBridgeWaitRequest: Codable, Sendable { + public let target: ClickTarget + public let timeout: TimeInterval +diff --git a/Core/PeekabooCore/Sources/PeekabooBridge/PeekabooBridgeRequestResponse.swift b/Core/PeekabooCore/Sources/PeekabooBridge/PeekabooBridgeRequestResponse.swift +index cfe9d9efe..faf9a766a 100644 +--- a/Core/PeekabooCore/Sources/PeekabooBridge/PeekabooBridgeRequestResponse.swift ++++ b/Core/PeekabooCore/Sources/PeekabooBridge/PeekabooBridgeRequestResponse.swift +@@ -64,6 +64,7 @@ public enum PeekabooBridgeRequest: Codable, Sendable { + case moveMouse(PeekabooBridgeMoveMouseRequest) + case systemDefinedKey(PeekabooBridgeSystemDefinedKeyRequest) + case selectInputSource(PeekabooBridgeSelectInputSourceRequest) ++ case enableWebAccessibility(PeekabooBridgeEnableWebAccessibilityRequest) + case waitForElement(PeekabooBridgeWaitRequest) + case listWindows(PeekabooBridgeWindowTargetRequest) + case listWindowMutationInventory(PeekabooBridgeWindowTargetRequest) +@@ -207,6 +208,7 @@ extension PeekabooBridgeRequest { + case .moveMouse: .moveMouse + case .systemDefinedKey: .systemDefinedKey + case .selectInputSource: .selectInputSource ++ case .enableWebAccessibility: .enableWebAccessibility + case .waitForElement: .waitForElement + case .listWindows, .listWindowMutationInventory: .listWindows + case .focusWindow: .focusWindow +diff --git a/Core/PeekabooCore/Sources/PeekabooBridge/PeekabooBridgeServer+Handlers.swift b/Core/PeekabooCore/Sources/PeekabooBridge/PeekabooBridgeServer+Handlers.swift +index 7443f0ce9..c52518256 100644 +--- a/Core/PeekabooCore/Sources/PeekabooBridge/PeekabooBridgeServer+Handlers.swift ++++ b/Core/PeekabooCore/Sources/PeekabooBridge/PeekabooBridgeServer+Handlers.swift +@@ -104,7 +104,7 @@ extension PeekabooBridgeServer { + .setValue, .selectText, .performAction, .scroll, .targetedScroll, .hotkey, .targetedHotkey, + .exactWindowTargetedHotkey, .targetedClick, + .exactWindowTargetedClick, .swipe, .drag, .moveMouse, .systemDefinedKey, .selectInputSource, +- .waitForElement: ++ .enableWebAccessibility, .waitForElement: + return try await self.handleAutomationRequest(request) + case .createExactWindowHeldPointerOwner, .beginExactWindowHeldPointer, + .releaseExactWindowHeldPointer, .revokeExactWindowHeldPointer, +@@ -358,6 +358,11 @@ extension PeekabooBridgeServer { + evidence: .deliveryAccepted, + unitCount: .one), + target: .global)) ++ case let .enableWebAccessibility(payload): ++ let changed = try WebAccessibilityActivation.enable(payload.expectedIdentity) ++ return .init( ++ response: .bool(changed.turnedOn), ++ mutation: .init(outcome: changed.outcome, target: .requestPinned)) + case let .waitForElement(payload): + let result = try await self.services.automation.waitForElement( + target: payload.target, +diff --git a/Core/PeekabooCore/Sources/PeekabooBridge/WebAccessibilityActivation.swift b/Core/PeekabooCore/Sources/PeekabooBridge/WebAccessibilityActivation.swift +new file mode 100644 +index 000000000..edc90df89 +--- /dev/null ++++ b/Core/PeekabooCore/Sources/PeekabooBridge/WebAccessibilityActivation.swift +@@ -0,0 +1,64 @@ ++import ApplicationServices ++import Foundation ++import PeekabooAutomationKit ++import PeekabooFoundation ++ ++/// Chromium and Electron build their web content's Accessibility tree only once an assistive client sets ++/// `AXManualAccessibility` on the application. `AXEnhancedUserInterface` asks too, but it changes how some apps ++/// animate and place their windows, so it is never set. ++enum WebAccessibilityActivation { ++ private static let attribute = "AXManualAccessibility" ++ private static let delivery = DesktopActionOutcome.Delivery(mechanism: .accessibilityValue, mode: .background) ++ ++ private struct Generation: Hashable { ++ let processIdentifier: pid_t ++ let processStartIdentity: UInt64 ++ } ++ ++ /// App generations this host already turned it on for, or found it on; it is set at most once per launch. ++ @MainActor private static var enabled: Set = [] ++ ++ /// Turns it on for this exact app generation. `turnedOn` is false when it was already on. ++ @MainActor ++ static func enable(_ identity: ApplicationProcessIdentity) throws -> (turnedOn: Bool, outcome: DesktopActionOutcome) { ++ let generation = Generation( ++ processIdentifier: identity.processIdentifier, processStartIdentity: identity.processStartIdentity) ++ guard SystemIdentityResolver.processStartIdentity(identity.processIdentifier) == identity.processStartIdentity ++ else { ++ throw PeekabooBridgeErrorEnvelope( ++ code: .notFound, message: "The app quit or relaunched before its web accessibility was turned on.", ++ kind: .appNotFound) ++ } ++ let unchanged = DesktopActionOutcome.confirmedNoChange() ++ if self.enabled.contains(generation) { return (false, unchanged) } ++ let application = AXUIElementCreateApplication(identity.processIdentifier) ++ // A busy renderer must not hold the host's main thread for the default six seconds. ++ AXUIElementSetMessagingTimeout(application, 1) ++ if self.isOn(application) { ++ self.enabled.insert(generation) ++ return (false, unchanged) ++ } ++ let status = AXUIElementSetAttributeValue(application, self.attribute as CFString, kCFBooleanTrue) ++ switch status { ++ case .success: ++ self.enabled.insert(generation) ++ return (true, self.isOn(application) ++ ? .confirmedChange(delivery: self.delivery, unitCount: .one) ++ : .dispatchedUnverified(delivery: self.delivery, evidence: .deliveryAccepted, unitCount: .one)) ++ case .attributeUnsupported, .notImplemented, .illegalArgument, .actionUnsupported: ++ throw PeekabooBridgeErrorEnvelope( ++ code: .operationNotSupported, message: "The app doesn't accept AXManualAccessibility (AX error \(status.rawValue)).") ++ default: ++ // The app may still apply a request that timed out. ++ throw PeekabooBridgeErrorEnvelope( ++ code: .internalError, message: "Setting AXManualAccessibility failed (AX error \(status.rawValue)).", ++ operationMayHaveCompleted: status == .cannotComplete) ++ } ++ } ++ ++ private static func isOn(_ application: AXUIElement) -> Bool { ++ var value: CFTypeRef? ++ guard AXUIElementCopyAttributeValue(application, self.attribute as CFString, &value) == .success else { return false } ++ return (value as? Bool) == true ++ } ++} +diff --git a/Core/PeekabooCore/Tests/PeekabooTests/PeekabooBridgeOperationResultSemanticsTests.swift b/Core/PeekabooCore/Tests/PeekabooTests/PeekabooBridgeOperationResultSemanticsTests.swift +index f2a2f6c3b..86398c375 100644 +--- a/Core/PeekabooCore/Tests/PeekabooTests/PeekabooBridgeOperationResultSemanticsTests.swift ++++ b/Core/PeekabooCore/Tests/PeekabooTests/PeekabooBridgeOperationResultSemanticsTests.swift +@@ -300,6 +300,7 @@ struct PeekabooBridgeOperationResultSemanticsTests { + .moveMouse, + .systemDefinedKey, + .selectInputSource, ++ .enableWebAccessibility, + .focusWindow, + .moveWindow, + .resizeWindow, +diff --git a/Core/PeekabooCore/Tests/PeekabooTests/PeekabooBridgeOperationSemanticPlanTests.swift b/Core/PeekabooCore/Tests/PeekabooTests/PeekabooBridgeOperationSemanticPlanTests.swift +index 23b798395..48c30f946 100644 +--- a/Core/PeekabooCore/Tests/PeekabooTests/PeekabooBridgeOperationSemanticPlanTests.swift ++++ b/Core/PeekabooCore/Tests/PeekabooTests/PeekabooBridgeOperationSemanticPlanTests.swift +@@ -28,7 +28,7 @@ struct PeekabooBridgeOperationSemanticPlanTests { + + // This count is intentionally reviewed whenever the wire enum grows. The exhaustive + // policy switches also make a missing classification a compile error. +- #expect(operations.count == 132) ++ #expect(operations.count == 133) + + func partition(_ values: [Value], by keyPath: KeyPath) { + let groups = values.map { expected in +diff --git a/Core/PeekabooCore/Tests/PeekabooTests/PeekabooBridgeRequestPlanTests.swift b/Core/PeekabooCore/Tests/PeekabooTests/PeekabooBridgeRequestPlanTests.swift +index b03a2f914..2876ec160 100644 +--- a/Core/PeekabooCore/Tests/PeekabooTests/PeekabooBridgeRequestPlanTests.swift ++++ b/Core/PeekabooCore/Tests/PeekabooTests/PeekabooBridgeRequestPlanTests.swift +@@ -9,7 +9,7 @@ struct PeekabooBridgeRequestPlanTests { + @Test + func `Every wire operation has one complete static descriptor`() { + let operations = PeekabooBridgeOperation.allCases +- #expect(operations.count == 132) ++ #expect(operations.count == 133) + + let descriptors = operations.map(Semantics.operationDescriptor(for:)) + #expect(descriptors.map(\.operation) == operations) +@@ -84,6 +84,7 @@ struct PeekabooBridgeRequestPlanTests { + ] + let accessibility: Set = [ + .inspectAccessibilityTree, ++ .enableWebAccessibility, + .getFocusedElement, + .type, + .typeActions, diff --git a/native/sources/client/client.swift b/native/sources/client/client.swift index 8cca1a7..3d1dc49 100644 --- a/native/sources/client/client.swift +++ b/native/sources/client/client.swift @@ -32,8 +32,8 @@ private enum Client { let operation = args[1] guard (operation == "apps" && args.count == 2) || (operation == "windows" && args.count == 3) - || (operation == "inspect" && args.count >= 5 && args.count <= 10) - || (operation == "read" && (args.count == 4 || args.count == 6)) + || (operation == "inspect" && args.count >= 5 && args.count <= 11) + || (operation == "read" && args.count >= 4 && args.count <= 7) || (operation == "elements" && args.count == 2) || (operation == "table" && args.count == 2) || (operation == "screen" && args.count >= 3 && args.count <= 5) @@ -115,10 +115,15 @@ private enum Client { } return try await nativeScreen(client, path: args[2], index: display, ocr: ocr) case "read": + var args = args + let keep = args.last == keepWebAccessibility + if keep { args.removeLast() } + guard args.count == 4 || args.count == 6 else { throw ClientError.usage } let pid = try processID(args[2]) guard let window = UInt32(args[3]), window > 0 else { throw ClientError.usage } let scope = args.count == 6 ? ScopeRequest(snapshot: args[4], element: args[5]) : nil - return try await nativeRead(client, pid: pid, window: window, scope: scope) + let web = WebAccessibilityPolicy(allowed: !keep, handshake: handshake) + return try await nativeRead(client, pid: pid, window: window, scope: scope, web: web) case "elements": return try await nativeElements(client) case "table": @@ -176,6 +181,8 @@ private enum Client { guard let window = UInt32(args[3]), window > 0 else { throw ClientError.usage } guard args[4].hasPrefix("/") else { throw ClientError.usage } var rest = args[5...] + let keep = rest.last == keepWebAccessibility + if keep { rest = rest.dropLast() } let mode = rest.first.flatMap(InspectionMode.init(rawValue:)) ?? .accessibility if rest.first.flatMap(InspectionMode.init(rawValue:)) != nil { rest = rest.dropFirst() } let ocr = rest.first == "ocr" @@ -188,7 +195,9 @@ private enum Client { } guard rest.isEmpty else { throw ClientError.usage } return try await inspect( - client, pid: pid, window: window, path: args[4], mode: mode, options: .init(ocr: ocr, scope: scope) + client, pid: pid, window: window, path: args[4], mode: mode, options: .init( + ocr: ocr, scope: scope, web: WebAccessibilityPolicy(allowed: !keep, handshake: handshake) + ) ) } } @@ -218,23 +227,39 @@ private enum Client { // The scope element comes from an earlier snapshot; read it before this observation publishes another. var source: ScopeSource? if let scope = options.scope { source = try await scopeSource(client, scope, pid: pid, window: window) } - let observation = try await client.desktopObservationWithOutcome(.init( - target: .pid(pid, window: .id(window)), - capture: .init(scale: .logical1x, focus: .background), - detection: .init( - mode: accessibility ? .accessibility : .none, - traversalBudget: source == nil - ? .init(maxDepth: 15, maxElementCount: 200, maxChildrenPerNode: 100) : wideTraversal, - requiresFreshAccessibilityTree: accessibility - ), - output: .init(path: path, saveSnapshot: true, snapshotID: pixels, includeImageData: true), - timeout: source == nil ? .init(overall: 20, detection: 15) : .init(overall: 22, detection: 18) - )) - guard let target = observation.targetIdentity, - target.processIdentity.processIdentifier == pid, - let exact = target.exactWindow, exact.identity.windowID == Int(window) - else { throw ClientError.target } - let result = observation.payload + var web = accessibility ? await options.web.beforeRead(client, pid: pid) : nil + func capture() async throws -> (DesktopTargetIdentity, UIAutomationTarget.ExactWindow, DesktopObservationResult) { + let observation = try await client.desktopObservationWithOutcome(.init( + target: .pid(pid, window: .id(window)), + capture: .init(scale: .logical1x, focus: .background), + detection: .init( + mode: accessibility ? .accessibility : .none, + traversalBudget: source == nil + ? .init(maxDepth: 15, maxElementCount: 200, maxChildrenPerNode: 100) : wideTraversal, + requiresFreshAccessibilityTree: accessibility + ), + output: .init(path: path, saveSnapshot: true, snapshotID: pixels, includeImageData: true), + timeout: source == nil ? .init(overall: 20, detection: 15) : .init(overall: 22, detection: 18) + )) + guard let target = observation.targetIdentity, + target.processIdentity.processIdentifier == pid, + let exact = target.exactWindow, exact.identity.windowID == Int(window) + else { throw ClientError.target } + return (target, exact, observation.payload) + } + var (target, exact, result) = try await capture() + if accessibility, let detection = result.elements { + let after = await options.web.afterRead( + client, pid: pid, elements: detection.elements.all, + truncated: !traversalTruncation(detection.metadata.truncationInfo).isEmpty, before: web + ) + web = after.state + if after.readAgain { + // The first snapshot is never returned, so it must not stay published. + if let published = result.files.publishedSnapshotID { try? await client.cleanSnapshot(snapshotId: published) } + (target, exact, result) = try await capture() + } + } if let pixels { guard result.files.publishedSnapshotID == pixels, result.elements == nil else { throw ClientError.pixels } // Expose the ID only after the stored canonical projection grants pointer authority for this exact capture. @@ -283,6 +308,7 @@ private enum Client { application_name: result.target.app?.name, // The traversal reads the title after target resolution, so a live title is newer there. window_title: result.elements?.metadata.windowContext?.windowTitle ?? result.target.window?.title, + web_accessibility: web, snapshot_id: result.files.publishedSnapshotID, element_count: scope == nil ? result.elements?.metadata.elementCount ?? 0 : elements.count, scope: scope, @@ -438,6 +464,7 @@ private struct Inspection: Encodable { let note: String? let application_name: String? let window_title: String? + let web_accessibility: WebAccessibility? let snapshot_id: String? let element_count: Int let scope: InspectionScope? @@ -454,6 +481,7 @@ private struct Inspection: Encodable { private struct InspectionOptions { let ocr: Bool let scope: ScopeRequest? + let web: WebAccessibilityPolicy } /// The scope element as found in this snapshot; `element` is its ID here. diff --git a/native/sources/client/read.swift b/native/sources/client/read.swift index ec35412..a4da818 100644 --- a/native/sources/client/read.swift +++ b/native/sources/client/read.swift @@ -152,6 +152,7 @@ private struct ReadScope: Encodable { private struct Reading: Encodable { let application_name: String? let window_title: String? + let web_accessibility: WebAccessibility? let scope: ReadScope? let text: String let text_bytes: Int @@ -171,25 +172,36 @@ private struct ReadReply: Encodable { /// Reads the text of one exact window, or of one element's subtree, without a snapshot or any input. func nativeRead( - _ client: PeekabooBridgeClient, pid: Int32, window: UInt32, scope: ScopeRequest? + _ client: PeekabooBridgeClient, pid: Int32, window: UInt32, scope: ScopeRequest?, web policy: WebAccessibilityPolicy ) async throws -> Data { var source: ScopeSource? if let scope { source = try await scopeSource(client, scope, pid: pid, window: window) } - let observation = try await client.desktopObservationWithOutcome(.init( - target: .pid(pid, window: .id(window)), - capture: .init(scale: .logical1x, focus: .background), - detection: .init(mode: .accessibility, traversalBudget: wideTraversal, requiresFreshAccessibilityTree: true), - output: .init(saveSnapshot: false, includeImageData: false), - timeout: .init(overall: 22, detection: 18) - )) - guard let target = observation.targetIdentity, - target.processIdentity.processIdentifier == pid, - let exact = target.exactWindow, exact.identity.windowID == Int(window) - else { throw ReadError("The native read did not confirm the requested process and window. Refresh the window list and try again.") } - let result = observation.payload - guard result.files.publishedSnapshotID == nil, let detection = result.elements else { - throw ReadError("The native read returned no Accessibility tree for this window.") + var web = await policy.beforeRead(client, pid: pid) + func observe() async throws -> (DesktopTargetIdentity, DesktopObservationResult, ElementDetectionResult) { + let observation = try await client.desktopObservationWithOutcome(.init( + target: .pid(pid, window: .id(window)), + capture: .init(scale: .logical1x, focus: .background), + detection: .init(mode: .accessibility, traversalBudget: wideTraversal, requiresFreshAccessibilityTree: true), + output: .init(saveSnapshot: false, includeImageData: false), + timeout: .init(overall: 22, detection: 18) + )) + guard let target = observation.targetIdentity, + target.processIdentity.processIdentifier == pid, + let exact = target.exactWindow, exact.identity.windowID == Int(window) + else { throw ReadError("The native read did not confirm the requested process and window. Refresh the window list and try again.") } + let result = observation.payload + guard result.files.publishedSnapshotID == nil, let detection = result.elements else { + throw ReadError("The native read returned no Accessibility tree for this window.") + } + return (target, result, detection) } + var (target, result, detection) = try await observe() + let after = await policy.afterRead( + client, pid: pid, elements: detection.elements.all, + truncated: !traversalTruncation(detection.metadata.truncationInfo).isEmpty, before: web + ) + web = after.state + if after.readAgain { (target, result, detection) = try await observe() } let elements = try preorder(detection.elements.all) var selected = elements[...] var readScope: ReadScope? @@ -209,6 +221,7 @@ func nativeRead( let data = Reading( application_name: result.target.app?.name, window_title: detection.metadata.windowContext?.windowTitle ?? result.target.window?.title, + web_accessibility: web, scope: readScope, text: reading.text, text_bytes: reading.text.utf8.count, diff --git a/native/sources/client/webaccessibility.swift b/native/sources/client/webaccessibility.swift new file mode 100644 index 0000000..a5fd796 --- /dev/null +++ b/native/sources/client/webaccessibility.swift @@ -0,0 +1,90 @@ +import AppKit +import Foundation +import PeekabooAutomationKit +import PeekabooBridge + +/// What a read did about a Chromium or Electron app's web accessibility, which those apps build only when an +/// assistive client asks; until then a web view lists few or no elements. The host sets it at most once per launch. +enum WebAccessibility: String, Encodable { + /// This read turned it on. + case enabled + /// It was already on, from an earlier read or another assistive app such as VoiceOver. + case already_on + /// The app doesn't accept it, and nothing changed. + case unsupported + /// The request failed or wasn't confirmed; the read went ahead with the tree as it was. + case unknown +} + +/// Whether reads may turn on web accessibility: the caller allows it and the host offers it. +struct WebAccessibilityPolicy { + let allowed: Bool + + /// How long a page gets to build its tree after it is turned on, before the window is read. + static let settle: Duration = .milliseconds(500) + + init(allowed: Bool, handshake: PeekabooBridgeHandshakeResponse) { + self.allowed = allowed && handshake.enabledOperations?.contains(.enableWebAccessibility) == true + } + + /// Turns it on before the first read of an app that ships Chromium, waiting for the tree when it was off. + func beforeRead(_ client: PeekabooBridgeClient, pid: Int32) async -> WebAccessibility? { + guard allowed, shipsChromium(pid) else { return nil } + let state = await enable(client, pid: pid) + if state == .enabled { try? await Task.sleep(for: Self.settle) } + return state + } + + /// After a read that found a nearly empty web area in an app not handled before the read, turns it on. + /// Returns the new state, and whether the window should be read again. + func afterRead( + _ client: PeekabooBridgeClient, pid: Int32, elements: [DetectedElement], truncated: Bool, + before: WebAccessibility? + ) async -> (state: WebAccessibility?, readAgain: Bool) { + guard allowed, before == nil, !truncated, hasSparseWebArea(elements) else { return (before, false) } + let state = await enable(client, pid: pid) + guard state == .enabled else { return (state, false) } + try? await Task.sleep(for: Self.settle) + return (state, true) + } + + private func enable(_ client: PeekabooBridgeClient, pid: Int32) async -> WebAccessibility { + guard let start = SystemIdentityResolver.processStartIdentity(pid) else { return .unknown } + do { + let result = try await client.enableWebAccessibilityWithOutcome( + ApplicationProcessIdentity(processIdentifier: pid, processStartIdentity: start) + ) + return result.payload ? .enabled : .already_on + } catch let envelope as PeekabooBridgeErrorEnvelope where envelope.code == .operationNotSupported { + return .unsupported + } catch { + return .unknown + } + } +} + +/// Whether the app's bundle ships Chromium: an Electron or Chromium Embedded framework, or a framework with +/// Chromium's crash handler among its helpers, as Chrome, Edge and other Chromium-based apps have. +private func shipsChromium(_ pid: Int32) -> Bool { + guard let bundle = NSRunningApplication(processIdentifier: pid)?.bundleURL else { return false } + let frameworks = bundle.appendingPathComponent("Contents/Frameworks") + let files = FileManager.default + guard let names = try? files.contentsOfDirectory(atPath: frameworks.path) else { return false } + return names.contains { name in + guard name.hasSuffix(".framework") else { return false } + if name == "Electron Framework.framework" || name == "Chromium Embedded Framework.framework" { return true } + let helpers = frameworks.appendingPathComponent(name).appendingPathComponent("Versions/Current/Helpers") + return (try? files.contentsOfDirectory(atPath: helpers.path))?.contains { $0.hasSuffix("crashpad_handler") } == true + } +} + +/// Whether a web area has at most one element under it, as a Chromium web view shows before its tree is on. +private func hasSparseWebArea(_ elements: [DetectedElement]) -> Bool { + guard let ordered = try? preorder(elements) else { return false } + return ordered.indices.contains { position in + ordered[position].element.attributes["role"] == "AXWebArea" && subtree(ordered, at: position).count <= 2 + } +} + +/// The last client argument of inspect or read that leaves the app's web accessibility as it is. +let keepWebAccessibility = "keep-web-accessibility" diff --git a/native/sources/runtime/activity.swift b/native/sources/runtime/activity.swift index dc31700..94115dc 100644 --- a/native/sources/runtime/activity.swift +++ b/native/sources/runtime/activity.swift @@ -73,6 +73,7 @@ private let activityNames: [PeekabooBridgeOperation: String] = [ .hotkey: "System key", .systemDefinedKey: "Media key", .selectInputSource: "Select input source", + .enableWebAccessibility: "Turn on web accessibility", .exactWindowTargetedTypeActions: "Type", .selectText: "Select text", .literalInsert: "Insert text", @@ -147,6 +148,8 @@ private let clipboardWrites: Set = [.clipboardTextWrite private let pauseExempt: Set = [ .createSnapshot, .cleanSnapshot, .ownsSnapshot, .getDetectionResult, .finishSnapshotMutation, .releaseExactWindowHeldPointer, .revokeExactWindowHeldPointer, .disconnectExactWindowHeldPointerOwner, + // It lets a Chromium or Electron app's web content be read, and sends no input. + .enableWebAccessibility, ] /// Records one finished request when it is worth listing. Runs while the client still waits for its reply, @@ -156,6 +159,7 @@ func recordActivity(_ record: PeekabooBridgeOperationRecord) { guard !activityHidden.contains(operation), let name = activityNames[operation] ?? (record.mutatesDesktop ? operation.rawValue : nil) else { return } + if operation == .enableWebAccessibility, !WebAccessibilityLaunches.shared.isFirst(record) { return } let parent = parentProcess(of: record.clientProcessIdentifier) let agent = agentName(of: record.clientProcessIdentifier) ?? parent.flatMap(processName) @@ -173,6 +177,34 @@ func recordActivity(_ record: PeekabooBridgeOperationRecord) { )) } +/// Inspections ask for web accessibility on every read of a Chromium or Electron app, and the host sets it at most +/// once per launch, so only the first request per launch is listed. +private final class WebAccessibilityLaunches: @unchecked Sendable { + static let shared = WebAccessibilityLaunches() + private let lock = NSLock() + private var listed: Set = [] + + /// Whether this is the first request that may have changed this launch of the app; refusals changed nothing. + func isFirst(_ record: PeekabooBridgeOperationRecord) -> Bool { + guard record.outcome != .refused, let process = record.targetProcessIdentifier, + let start = processStart(process) + else { return false } + lock.lock() + defer { lock.unlock() } + return listed.insert("\(process) \(start)").inserted + } +} + +/// When the process started, which tells two launches with the same pid apart. +private func processStart(_ process: pid_t) -> String? { + var info = kinfo_proc() + var size = MemoryLayout.stride + var mib = [CTL_KERN, KERN_PROC, KERN_PROC_PID, process] + guard sysctl(&mib, 4, &info, &size, nil, 0) == 0, size > 0 else { return nil } + let start = info.kp_proc.p_un.__p_starttime + return "\(start.tv_sec).\(start.tv_usec)" +} + private func parentProcess(of process: pid_t) -> pid_t? { var info = kinfo_proc() var size = MemoryLayout.stride diff --git a/native/sources/runtime/runtime.swift b/native/sources/runtime/runtime.swift index 03573eb..0998e72 100644 --- a/native/sources/runtime/runtime.swift +++ b/native/sources/runtime/runtime.swift @@ -87,7 +87,7 @@ private final class Desktop { .createSnapshot, .cleanSnapshot, .ownsSnapshot, .getDetectionResult, .beginSnapshotMutation, .finishSnapshotMutation, .targetedClick, .exactWindowTargetedClick, .foregroundModifierClick, .setValue, .performAction, .exactWindowTargetedHotkey, .hotkey, .exactWindowTargetedTypeActions, - .selectText, .literalInsert, .attachmentInsert, .clipboardTextRead, .clipboardImageRead, .clipboardImageExport, .clipboardFilesRead, .clipboardTextWrite, .clipboardImageWrite, .clipboardFilesWrite, .targetedScroll, .exactWindowDrag, .drag, .moveMouse, .systemDefinedKey, .selectInputSource, + .selectText, .literalInsert, .attachmentInsert, .clipboardTextRead, .clipboardImageRead, .clipboardImageExport, .clipboardFilesRead, .clipboardTextWrite, .clipboardImageWrite, .clipboardFilesWrite, .targetedScroll, .exactWindowDrag, .drag, .moveMouse, .systemDefinedKey, .selectInputSource, .enableWebAccessibility, .createExactWindowHeldPointerOwner, .beginExactWindowHeldPointer, .releaseExactWindowHeldPointer, .revokeExactWindowHeldPointer, .disconnectExactWindowHeldPointerOwner, .launchApplicationWithOptions, .activateApplication, .quitApplication, .hideApplication, .backgroundCloseWindow, diff --git a/scripts/live-check.mjs b/scripts/live-check.mjs index 6afa533..378857f 100644 --- a/scripts/live-check.mjs +++ b/scripts/live-check.mjs @@ -31,6 +31,7 @@ const only = new Set((values.only ?? []).flatMap((v) => v.split(",")).map((v) => const TEXTEDIT = "com.apple.TextEdit"; const FINDER = "com.apple.finder"; const SAFARI = "com.apple.Safari"; +const CHROME = "com.google.Chrome"; const run = randomBytes(3).toString("hex"); // ---------- MCP client ---------- @@ -357,6 +358,15 @@ class Session { }); } + chrome() { + return this.once("chrome", async () => { + if (!existsSync("/Applications/Google Chrome.app")) skip("Google Chrome is not installed in /Applications"); + // A running Chrome may already have web accessibility on, and holds the user's windows. + if (await this.runningTarget(CHROME)) skip("Chrome is running; quit it to run the Chromium checks"); + return this.openWindow(this.fixtures.page, CHROME, `Live page ${run}`, "Chrome fixture page"); + }); + } + async inspect(win, extra = {}) { const result = await this.mcp.call("desktop_inspect", { pid: win.pid, window: win.window, ...extra }); const snap = result.data.snapshot_id; @@ -431,6 +441,31 @@ const selectedNames = (table) => (table?.rows ?? []).filter((r) => r.selected).m const checks = []; const check = (pr, title, run, notes) => checks.push({ pr: String(pr), title, run, notes }); +// 0.11.0 +check(107, "desktop_inspect turns on a fresh Chrome's web accessibility once, and lists the page's fields", async (s, m) => { + if (!m.tools.get("desktop_inspect")?.inputSchema?.properties?.web_accessibility) skip("this build's desktop_inspect has no web_accessibility (#107 not in it)"); + const win = await s.chrome(); + const first = await s.inspect(win); + const field = find(first.els, "AXTextField", /Controlled field/); + const again = await s.inspect(win); + const kept = await s.inspect(win, { web_accessibility: false }); + return expect( + first.result.data.web_accessibility === "enabled" && field + && again.result.data.web_accessibility === "already_on" && kept.result.data.web_accessibility === undefined, + `first ${first.result.data.web_accessibility} with ${first.els.length} elements, field ${field ? "found" : "missing"}; again ${again.result.data.web_accessibility}; opted out ${kept.result.data.web_accessibility}`, + ); +}, "Launches Chrome on the fixture page and quits it after; it may restore the user's last Chrome session meanwhile."); + +check(107, "desktop_read leaves Safari's web accessibility alone and reports nothing", async (s, m) => { + if (!m.tools.get("desktop_read")?.inputSchema?.properties?.web_accessibility) skip("this build's desktop_read has no web_accessibility (#107 not in it)"); + const win = await s.web(); + const read = await s.read(win); + return expect( + read.data.web_accessibility === undefined && (read.data.text ?? "").includes(`Live page ${run}`), + `web_accessibility ${read.data.web_accessibility}; text ${brief(read.data.text, 80)}`, + ); +}); + // 0.10.0 check(99, "desktop_insert pastes a PNG into a TextEdit rich document and restores the clipboard", async (s, m) => { const insert = m.tools.get("desktop_insert"); diff --git a/src/build.ts b/src/build.ts index ef5a8c1..0876f07 100644 --- a/src/build.ts +++ b/src/build.ts @@ -68,6 +68,7 @@ const patches = [ "peekaboo-operation-record.patch", "peekaboo-input-source.patch", "peekaboo-attachment-insert.patch", + "peekaboo-web-accessibility.patch", ].map((name) => join(native, "patches", name)); function spawn(command: string[], env?: NodeJS.ProcessEnv) { diff --git a/src/client.ts b/src/client.ts index d1d3448..dff9e85 100644 --- a/src/client.ts +++ b/src/client.ts @@ -209,6 +209,15 @@ function bounded(data: Record, field: string, limit = MAX_TEXT) return text; } +/** The native argument that leaves a Chromium or Electron app's web accessibility as it is. */ +const KEEP_WEB_ACCESSIBILITY = "keep-web-accessibility"; + +function validateWebAccessibility(request: { web_accessibility?: unknown }) { + if (request.web_accessibility !== undefined && typeof request.web_accessibility !== "boolean") { + throw new Error("Choose web_accessibility true or false."); + } +} + function validateScope(scope: unknown) { if ( !scope || typeof scope !== "object" || Object.keys(scope).length !== 2 @@ -374,6 +383,7 @@ async function inspect( validateScope(request.scope); if (mode !== "accessibility") throw new Error("A scope needs accessibility mode."); } + validateWebAccessibility(request); const savePath = await validateSave(request); const directory = await mkdtemp(join(tmpdir(), "desktop-tools-")); try { @@ -381,6 +391,7 @@ async function inspect( const args = ["inspect", pid, window, path, mode]; if (request.ocr) args.push("ocr"); if (request.scope) args.push("scope", request.scope.snapshot, request.scope.element); + if (request.web_accessibility === false) args.push(KEEP_WEB_ACCESSIBILITY); const data = await native(args, signal, { target: request }); if (request.scope && (data.scope as { from_element?: unknown } | undefined)?.from_element !== request.scope.element) { throw new Error("Native inspection did not honor the requested scope."); @@ -473,6 +484,8 @@ async function read( validateScope(request.scope); args.push(request.scope.snapshot, request.scope.element); } + validateWebAccessibility(request); + if (request.web_accessibility === false) args.push(KEEP_WEB_ACCESSIBILITY); if (request.table !== undefined || request.row_offset !== undefined || request.max_rows !== undefined) { if (request.table !== true || !request.scope) { throw new Error("A table read needs table true and scope: the table, or the area that holds it, from an accessibility desktop_inspect."); diff --git a/src/protocol.ts b/src/protocol.ts index 4286051..023f209 100644 --- a/src/protocol.ts +++ b/src/protocol.ts @@ -111,12 +111,16 @@ export type DesktopRequest = mode?: "accessibility" | "pixels"; ocr?: boolean; scope?: DesktopScope; + /** false leaves a Chromium or Electron app's web accessibility as it is. */ + web_accessibility?: boolean; } & DesktopCaptureSave | { op: "read"; pid: number; window: number; scope?: DesktopScope; + /** false leaves a Chromium or Electron app's web accessibility as it is. */ + web_accessibility?: boolean; /** Reads the scope as a table of rows and cells instead of text. */ table?: boolean; row_offset?: number; diff --git a/src/tools.ts b/src/tools.ts index 35660eb..b2f7065 100644 --- a/src/tools.ts +++ b/src/tools.ts @@ -175,6 +175,12 @@ const saveProperties: Record = { }, }, }; +const webAccessibility: JsonSchema = { + type: "boolean", + description: "false leaves a Chromium or Electron app's web accessibility as it is, so its web content may be missing.", +}; +const webAccessibilityNote = + "in a Chromium or Electron app, or a window whose web area is nearly empty, it first turns on the app's web accessibility so web content appears, once per app launch, and reports web_accessibility; it stays on until the app quits. Pass web_accessibility false to leave the app as it is."; const saveNote = "save_path, an absolute .png path in an existing folder, keeps the full-resolution capture; region, in global points, saves only part of it; an existing file needs overwrite true."; @@ -588,7 +594,7 @@ export function desktopTools({ place, app }: { place: string; app: string }): De name: "desktop_inspect", op: "inspect", description: - `Read one window's Accessibility elements and a screenshot by pid and window, returning the single-use snapshot_id that input tools take. Read-only; never activates or focuses. mode pixels, for windows Accessibility can't read, gives a snapshot for point clicks, scrolls and drags only. A large window lists its first 200 elements and reports truncated: pass scope (a container's snapshot and element from an earlier inspection) to list its subtree with a much larger budget. ocr true adds recognized_text, whose box centers are points for this snapshot. ${saveNote}`, + `Read one window's Accessibility elements and a screenshot by pid and window, returning the single-use snapshot_id that input tools take. Read-only and never activates or focuses, except that ${webAccessibilityNote} mode pixels, for windows Accessibility can't read, gives a snapshot for point clicks, scrolls and drags only. A large window lists its first 200 elements and reports truncated: pass scope (a container's snapshot and element from an earlier inspection) to list its subtree with a much larger budget. ocr true adds recognized_text, whose box centers are points for this snapshot. ${saveNote}`, parameters: { type: "object", required: ["pid", "window"], @@ -598,6 +604,7 @@ export function desktopTools({ place, app }: { place: string; app: string }): De mode: { type: "string", enum: ["accessibility", "pixels"] }, ocr: { type: "boolean" }, scope: scope, + web_accessibility: webAccessibility, ...saveProperties, }, }, @@ -606,7 +613,7 @@ export function desktopTools({ place, app }: { place: string; app: string }): De name: "desktop_read", op: "read", description: - `Read one window's text by pid and window through Accessibility, up to 96 KB, including content scrolled out of view; scope reads one inspected element's subtree. Read-only, with no snapshot. Windows without Accessibility text, such as canvases, return empty text: use desktop_inspect with ocr. With table true and a scope, it reads the first table there instead: columns, rows with their cells, row element IDs for desktop_click, selected, and depth/expanded for outlines; page with row_offset and max_rows (default ${TABLE_ROWS_DEFAULT}, at most ${TABLE_ROWS_MAX}).`, + `Read one window's text by pid and window through Accessibility, up to 96 KB, including content scrolled out of view; scope reads one inspected element's subtree. Read-only, with no snapshot, except that it turns on web accessibility as desktop_inspect does. Windows without Accessibility text, such as canvases, return empty text: use desktop_inspect with ocr. With table true and a scope, it reads the first table there instead: columns, rows with their cells, row element IDs for desktop_click, selected, and depth/expanded for outlines; page with row_offset and max_rows (default ${TABLE_ROWS_DEFAULT}, at most ${TABLE_ROWS_MAX}).`, parameters: { type: "object", required: ["pid", "window"], @@ -614,6 +621,7 @@ export function desktopTools({ place, app }: { place: string; app: string }): De pid: { type: "integer", minimum: 1 }, window: { type: "integer", minimum: 1 }, scope: scope, + web_accessibility: webAccessibility, table: { type: "boolean", const: true }, row_offset: { type: "integer", minimum: 0, maximum: 100_000 }, max_rows: { type: "integer", minimum: 1, maximum: TABLE_ROWS_MAX }, diff --git a/test/webaccessibility.test.mjs b/test/webaccessibility.test.mjs new file mode 100644 index 0000000..f8d81eb --- /dev/null +++ b/test/webaccessibility.test.mjs @@ -0,0 +1,28 @@ +import assert from "node:assert/strict"; +import { test } from "node:test"; +import { createDesktop } from "../dist/client.js"; +import { desktopTools } from "../dist/tools.js"; + +const desktop = createDesktop({ client: "/nonexistent/desktop-client", socket: "/nonexistent/desktop.sock", name: "test" }); +const signal = new AbortController().signal; +// The client refuses every request off macOS before validating it. +const macOS = { skip: process.platform !== "darwin" && "the client validates only on macOS" }; + +test("inspect and read offer web_accessibility as a boolean opt-out", () => { + const tools = new Map(desktopTools({ place: "this Mac", app: "Desktop Tools" }).map((tool) => [tool.name, tool])); + for (const name of ["desktop_inspect", "desktop_read"]) { + const property = tools.get(name).parameters.properties.web_accessibility; + assert.equal(property.type, "boolean", name); + assert.match(property.description, /false leaves/); + } + assert.match(tools.get("desktop_inspect").description, /web_accessibility false/); +}); + +test("web_accessibility must be true or false", macOS, async () => { + for (const op of ["inspect", "read"]) { + await assert.rejects( + desktop({ op, pid: 1, window: 2, web_accessibility: "no" }, signal), + /Choose web_accessibility true or false/, + ); + } +});