feat: turn on Chromium and Electron web accessibility before inspect and read, once per app launch

dist/build.js

@@ -67,6 +67,7 @@ const patches = [
     "peekaboo-operation-record.patch",
     "peekaboo-input-source.patch",
     "peekaboo-attachment-insert.patch",
+    "peekaboo-web-accessibility.patch",
 ].map((name) => join(native, "patches", name));
 function spawn(command, env) {
     return spawnSync(command[0], command.slice(1), { encoding: "utf8", env: env ?? process.env });

dist/client.js

@@ -129,6 +129,13 @@ function bounded(data, field, limit = MAX_TEXT) {
     }
     return text;
 }
+/** The native argument that leaves a Chromium or Electron app's web accessibility as it is. */
+const KEEP_WEB_ACCESSIBILITY = "keep-web-accessibility";
+function validateWebAccessibility(request) {
+    if (request.web_accessibility !== undefined && typeof request.web_accessibility !== "boolean") {
+        throw new Error("Choose web_accessibility true or false.");
+    }
+}
 function validateScope(scope) {
     if (!scope || typeof scope !== "object" || Object.keys(scope).length !== 2
         || !("snapshot" in scope) || !("element" in scope)
@@ -275,6 +282,7 @@ async function inspect(native, request, signal) {
         if (mode !== "accessibility")
             throw new Error("A scope needs accessibility mode.");
     }
+    validateWebAccessibility(request);
     const savePath = await validateSave(request);
     const directory = await mkdtemp(join(tmpdir(), "desktop-tools-"));
     try {
@@ -284,6 +292,8 @@ async function inspect(native, request, signal) {
             args.push("ocr");
         if (request.scope)
             args.push("scope", request.scope.snapshot, request.scope.element);
+        if (request.web_accessibility === false)
+            args.push(KEEP_WEB_ACCESSIBILITY);
         const data = await native(args, signal, { target: request });
         if (request.scope && data.scope?.from_element !== request.scope.element) {
             throw new Error("Native inspection did not honor the requested scope.");
@@ -370,6 +380,9 @@ async function read(native, request, signal) {
         validateScope(request.scope);
         args.push(request.scope.snapshot, request.scope.element);
     }
+    validateWebAccessibility(request);
+    if (request.web_accessibility === false)
+        args.push(KEEP_WEB_ACCESSIBILITY);
     if (request.table !== undefined || request.row_offset !== undefined || request.max_rows !== undefined) {
         if (request.table !== true || !request.scope) {
             throw new Error("A table read needs table true and scope: the table, or the area that holds it, from an accessibility desktop_inspect.");

dist/protocol.d.ts

@@ -181,11 +181,15 @@ export type DesktopRequest = DesktopManagement | DesktopLaunch | DesktopOpen | D
     mode?: "accessibility" | "pixels";
     ocr?: boolean;
     scope?: DesktopScope;
+    /** false leaves a Chromium or Electron app's web accessibility as it is. */
+    web_accessibility?: boolean;
 } & DesktopCaptureSave) | {
     op: "read";
     pid: number;
     window: number;
     scope?: DesktopScope;
+    /** false leaves a Chromium or Electron app's web accessibility as it is. */
+    web_accessibility?: boolean;
     /** Reads the scope as a table of rows and cells instead of text. */
     table?: boolean;
     row_offset?: number;

dist/tools.js

@@ -159,6 +159,11 @@ const saveProperties = {
         },
     },
 };
+const webAccessibility = {
+    type: "boolean",
+    description: "false leaves a Chromium or Electron app's web accessibility as it is, so its web content may be missing.",
+};
+const webAccessibilityNote = "in a Chromium or Electron app, or a window whose web area is nearly empty, it first turns on the app's web accessibility so web content appears, once per app launch, and reports web_accessibility; it stays on until the app quits. Pass web_accessibility false to leave the app as it is.";
 const saveNote = "save_path, an absolute .png path in an existing folder, keeps the full-resolution capture; region, in global points, saves only part of it; an existing file needs overwrite true.";
 export function desktopTools({ place, app }) {
     return [
@@ -540,7 +545,7 @@ export function desktopTools({ place, app }) {
         {
             name: "desktop_inspect",
             op: "inspect",
-            description: `Read one window's Accessibility elements and a screenshot by pid and window, returning the single-use snapshot_id that input tools take. Read-only; never activates or focuses. mode pixels, for windows Accessibility can't read, gives a snapshot for point clicks, scrolls and drags only. A large window lists its first 200 elements and reports truncated: pass scope (a container's snapshot and element from an earlier inspection) to list its subtree with a much larger budget. ocr true adds recognized_text, whose box centers are points for this snapshot. ${saveNote}`,
+            description: `Read one window's Accessibility elements and a screenshot by pid and window, returning the single-use snapshot_id that input tools take. Read-only and never activates or focuses, except that ${webAccessibilityNote} mode pixels, for windows Accessibility can't read, gives a snapshot for point clicks, scrolls and drags only. A large window lists its first 200 elements and reports truncated: pass scope (a container's snapshot and element from an earlier inspection) to list its subtree with a much larger budget. ocr true adds recognized_text, whose box centers are points for this snapshot. ${saveNote}`,
             parameters: {
                 type: "object",
                 required: ["pid", "window"],
@@ -550,21 +555,23 @@ export function desktopTools({ place, app }) {
                     mode: { type: "string", enum: ["accessibility", "pixels"] },
                     ocr: { type: "boolean" },
                     scope: scope,
+                    web_accessibility: webAccessibility,
                     ...saveProperties,
                 },
             },
         },
         {
             name: "desktop_read",
             op: "read",
-            description: `Read one window's text by pid and window through Accessibility, up to 96 KB, including content scrolled out of view; scope reads one inspected element's subtree. Read-only, with no snapshot. Windows without Accessibility text, such as canvases, return empty text: use desktop_inspect with ocr. With table true and a scope, it reads the first table there instead: columns, rows with their cells, row element IDs for desktop_click, selected, and depth/expanded for outlines; page with row_offset and max_rows (default ${TABLE_ROWS_DEFAULT}, at most ${TABLE_ROWS_MAX}).`,
+            description: `Read one window's text by pid and window through Accessibility, up to 96 KB, including content scrolled out of view; scope reads one inspected element's subtree. Read-only, with no snapshot, except that it turns on web accessibility as desktop_inspect does. Windows without Accessibility text, such as canvases, return empty text: use desktop_inspect with ocr. With table true and a scope, it reads the first table there instead: columns, rows with their cells, row element IDs for desktop_click, selected, and depth/expanded for outlines; page with row_offset and max_rows (default ${TABLE_ROWS_DEFAULT}, at most ${TABLE_ROWS_MAX}).`,
             parameters: {
                 type: "object",
                 required: ["pid", "window"],
                 properties: {
                     pid: { type: "integer", minimum: 1 },
                     window: { type: "integer", minimum: 1 },
                     scope: scope,
+                    web_accessibility: webAccessibility,
                     table: { type: "boolean", const: true },
                     row_offset: { type: "integer", minimum: 0, maximum: 100_000 },
                     max_rows: { type: "integer", minimum: 1, maximum: TABLE_ROWS_MAX },

docs/app.md

@@ -15,9 +15,9 @@ Anything running as the user can use those grants while the app serves, so grant
 - the time;
 - the outcome: completed, unknown, refused, or paused for a request refused because input was paused.
 
-Identical operations in a row fold into one line with a count. Every desktop change is listed, and so are screen captures and clipboard reads. Listings and snapshot bookkeeping are not. Only the operation and its target are kept, never typed text, clipboard contents or values. The list lives in memory and is gone when the app quits.
+Identical operations in a row fold into one line with a count. Every desktop change is listed, and so are screen captures and clipboard reads. Turning on an app's web accessibility for a read is listed once per launch of that app. Listings and snapshot bookkeeping are not. Only the operation and its target are kept, never typed text, clipboard contents or values. The list lives in memory and is gone when the app quits.
 
-**Pause All Input** stops agents from changing anything until you turn it off again. While paused, the app refuses every click, key, typing, scroll, drag, menu command, app or window change, dialog action and clipboard write before it is sent. The refusal tells the agent to ask you to resume. Agents can still observe: listings, the accessibility tree, screen captures and clipboard reads keep working. The menu bar icon becomes a pause sign while input is paused. Only this menu item changes it; no client can pause or resume. Pausing isn't saved, so the app starts resumed.
+**Pause All Input** stops agents from changing anything until you turn it off again. While paused, the app refuses every click, key, typing, scroll, drag, menu command, app or window change, dialog action and clipboard write before it is sent. The refusal tells the agent to ask you to resume. Agents can still observe: listings, the accessibility tree, screen captures and clipboard reads keep working. So does turning on a Chromium or Electron app's web accessibility before a read, since it only lets the app's web content be read and sends no input (see [tools](tools.md#input)). The menu bar icon becomes a pause sign while input is paused. Only this menu item changes it; no client can pause or resume. Pausing isn't saved, so the app starts resumed.
 
 ## Build
 

docs/live-checks.md

@@ -17,6 +17,7 @@ What it touches:
 - It makes its own fixtures in a new `/tmp/desktop-tools-live-*` folder: a plain and a rich TextEdit document, a Finder folder of eight files and a subfolder, a PNG, and a web page with a React-style controlled input, a date input, a contenteditable, a table with `<th>` headers and a grid of inputs. The folder is removed at the end unless `--keep-fixtures` is passed.
 - It opens those fixtures in TextEdit, Finder and Safari with `desktop_open`, and acts only on the new windows. At the end, even after a failure or Control-C, it closes each window it opened and quits only the apps it launched itself. A fixture document that asks to save is answered with Delete or Don't Save, on that document only.
 - Web checks are skipped while Safari has windows open, since a fixture page could open as a tab in one of them.
+- The Chromium check (#107) runs only when Google Chrome is installed and not running. It opens the fixture page in Chrome, which may also restore the user's last Chrome session, and quits Chrome at the end.
 - Clipboard checks run only when the clipboard holds plain text, and that text is written back at the end. The input-source check selects another enabled keyboard layout, then the original again.
 - Modifier clicks (#77) and `desktop_system_key` (#89) use the real keyboard and pointer, as they always do; nothing else needs the foreground.
 
@@ -38,5 +39,7 @@ These stay manual, because they are destructive, change system settings, need a
 - **#90 input methods:** typing in an input method's mode, such as Japanese Hiragana, needs that method enabled.
 - **#86 and #92 in other apps:** Numbers, Excel, Mail and Chrome, and a long Finder folder of 300 or more files.
 - **#88 in other apps:** Chrome, read-only native text views and Preview PDFs.
+- **#107 web accessibility in Electron apps:** inspect a Slack, VS Code or Discord window and check that `web_accessibility` is `enabled` once per launch and the web content's fields are listed, that the app's window still moves and resizes normally, that All Activity lists **Turn on web accessibility** once, and that it still runs while input is paused.
+- **#5 background paste into web content (spike, before any patch):** with a WebKit app such as Safari in the background and a textarea focused, answer: (1) is Edit > Paste `AXEnabled`, with and without the host's background key-window preparation; (2) does an Accessibility press of Edit > Paste paste into the focused textarea; (3) must the textarea get `AXFocused` first?
 - **#27 Codex:** adding the server to Codex's configuration.
 - Anything that asks for a password or an administrator's approval.

docs/tools.md

@@ -141,6 +141,14 @@ A `desktop_click` with `modifiers` (any of `command`, `option` and `shift`), suc
 
 `desktop_inspect` reads at most 200 elements of a window, so a long web page, mail list or Finder folder is cut short and reports `truncated`. Two tools reach further without input. `desktop_read` reads the window's text with a budget of 4,000 elements, 64 levels deep and 1,000 children per element, within about 18 seconds; it returns the text in tree order, one line per visual line, and names each limit that cut it short. Pass `scope`, a `snapshot_id` and an element ID from an earlier inspection of the same window, to read only that element, such as the page's web area. `desktop_inspect` takes the same `scope`: it reads the window with that larger budget and lists only the scope element and its descendants, each with `attributes.depth`. The scope element is found again by its role, depth, bounds, label, title and identifier, so it must not have moved; listed IDs belong to the new snapshot, like any inspection.
 
+Chromium and Electron apps, such as Chrome, Edge, Slack, VS Code and Discord, build the Accessibility tree of their web content only after an assistive app asks for it. Until then a web view lists few or no elements. So before `desktop_inspect` or `desktop_read` reads an app whose bundle ships an Electron, Chromium Embedded or Chromium framework, the host sets `AXManualAccessibility` on that app. It also does so after a read that finds a web area with at most one element under it, and then reads the window once more. It waits half a second after turning it on, so the page can build its tree. The host sets it at most once per launch of the app, and it stays on until the app quits, since VoiceOver or another assistive app may rely on it too. It never sets `AXEnhancedUserInterface`, which changes how some apps animate and place their windows. The result reports `web_accessibility`:
+- **`enabled`**: this read turned it on.
+- **`already_on`**: it was already on, from an earlier read or another assistive app.
+- **`unsupported`**: the app doesn't accept it, and nothing changed.
+- **`unknown`**: the request failed or wasn't confirmed; the read went ahead with the tree as it was.
+
+Pass `web_accessibility: false` to leave the app as it is; its web content may then be missing. Turning it on sends no input, so it runs while input is paused, and the app's activity list shows it once per app launch as **Turn on web accessibility**. A table read uses the tree as an earlier inspection left it.
+
 `desktop_read` with `table: true` reads a table instead of text. Its `scope` is a table, outline or grid from an accessibility inspection, such as a Finder list, a Mail message list, a Numbers table or a web table, or an area that holds one, such as the scroll area or web area; the first table in it is read, and `tables_in_scope` counts the others. It reads Accessibility alone, with `desktop_read`'s budget and no screenshot, and returns:
 
 - `columns`: the header titles in order. They come from the table's columns, from the header buttons a scroll view draws just above the table, or from a web table's header row; `null` when there are none. Scope the scroll area rather than the table when a list's headers are missing.
@@ -230,6 +238,7 @@ These are the rest of each tool's contract, beyond its description.
 - **Accessibility mode** (the default) can return an action snapshot with element IDs.
 - **Pixels mode** returns a snapshot with no element IDs and no focused control. It authorizes only point clicks, point scrolls and drags in that exact window.
 - **After an action,** the automatic observation uses accessibility mode. If it fails, inspect again with pixels mode.
+- **In a Chromium or Electron app,** it first turns on the app's web accessibility, once per launch, unless `web_accessibility` is false. The result's `web_accessibility` says what happened; see [Input](#input).
 
 **`desktop_click`.** A single left click on a supported editable text field, by element or point, also requests keyboard focus. The result says whether focus was verified; use `desktop_select` to choose a range or caret position. Points stay bound to the captured window, even when the returned screenshot was scaled.
 

native/patches/README.md

@@ -555,3 +555,9 @@ guard and patches deliberately, or remove each patch when the dependency include
 Revalidate real button clicks, changed-focus refusal, literal multiline insertion, clipboard
 restoration, partial input, and interruption before shipping
 the update.
+
+`peekaboo-web-accessibility.patch` adds an `enableWebAccessibility` operation, which sets `AXManualAccessibility` on one app so that Chromium and Electron build the Accessibility tree of their web content. Until an assistive client asks, their web views list few or no elements. It never sets `AXEnhancedUserInterface`, which changes how some apps animate and place their windows.
+- **Request.** It carries the exact app generation, as a process identity.
+- **Host.** On the main actor, the host checks that the process still has that start identity, or answers `notFound`. It keeps the generations it turned on, or found on, and answers those without touching the app again, so it sets the attribute at most once per launch. Otherwise it reads the attribute, with a one-second messaging timeout, and sets it only when it isn't already on. An app that rejects the attribute is `operationNotSupported`. Any other AX error is `internalError`, marked as possibly completed when the app timed out.
+- **Result contract.** It needs Accessibility. It is a request-pinned, process-scoped, one-unit Accessibility-value background delivery answered with `bool`: true when this request turned it on. The outcome is `confirmedChange` when the attribute reads back on, `dispatchedUnverified` when it doesn't read back, and `confirmedNoChange` when it was already on. It is service-owned and joins the default allowlists.
+- **Tests.** The wire count is now 133. The operation joins the unconditionally mutating set and the Accessibility permission partition.