feat: system-wide shortcuts with desktop_system_key

dist/client.js

@@ -355,6 +355,16 @@ function validateApplication(application) {
         || !/^[A-Za-z0-9.-]+$/.test(application.bundle_id))
         throw new Error("Use an exact application bundle ID for launch.");
 }
+function validateKey(request) {
+    if (!isDesktopKey(request.key))
+        throw new Error("Choose a named key or one printable character.");
+    const modifiers = request.modifiers;
+    if (modifiers !== undefined && (!Array.isArray(modifiers) || modifiers.length > 4
+        || new Set(modifiers).size !== modifiers.length
+        || modifiers.some((modifier) => !DESKTOP_MODIFIERS.includes(modifier)))) {
+        throw new Error("Use each of command, control, option, and shift at most once.");
+    }
+}
 function validateAction(request) {
     if (request.op === "launch") {
         if (Object.keys(request).some((key) => !["op", "application"].includes(key))) {
@@ -445,6 +455,12 @@ function validateAction(request) {
         }
         return;
     }
+    if (request.op === "system-key") {
+        if (Object.keys(request).some((key) => !["op", "key", "modifiers"].includes(key))) {
+            throw new Error("A system key accepts only key and modifiers.");
+        }
+        return validateKey(request);
+    }
     if (typeof request.snapshot !== "string" || !request.snapshot || request.snapshot.length > 256) {
         throw new Error("Use the snapshot_id from a fresh desktop_inspect result.");
     }
@@ -511,17 +527,8 @@ function validateAction(request) {
         }
         return;
     }
-    if (request.op === "key") {
-        if (!isDesktopKey(request.key))
-            throw new Error("Choose a named key or one printable character.");
-        const modifiers = request.modifiers;
-        if (modifiers !== undefined && (!Array.isArray(modifiers) || modifiers.length > 4
-            || new Set(modifiers).size !== modifiers.length
-            || modifiers.some((modifier) => !DESKTOP_MODIFIERS.includes(modifier)))) {
-            throw new Error("Use each of command, control, option, and shift at most once.");
-        }
-        return;
-    }
+    if (request.op === "key")
+        return validateKey(request);
     if (request.op === "insert") {
         if (typeof request.text !== "string" || !request.text || request.text.length > 8192) {
             throw new Error("Inserted text must contain 1 to 8192 UTF-16 code units.");
@@ -727,9 +734,11 @@ async function act(native, request, signal) {
                             ? "launch"
                             : request.op === "open"
                                 ? "open"
-                                : isDesktopManagement(request)
-                                    ? "management"
-                                    : "action";
+                                : request.op === "system-key"
+                                    ? "system-key"
+                                    : isDesktopManagement(request)
+                                        ? "management"
+                                        : "action";
         data = await native([operation], signal, {
             input: JSON.stringify(input),
             onDispatch() {
@@ -1030,6 +1039,7 @@ async function execute(native, request, abort) {
             "click",
             "type",
             "key",
+            "system-key",
             "insert",
             "select",
             "element-action",

dist/protocol.d.ts

@@ -183,6 +183,10 @@ export type DesktopRequest = DesktopManagement | DesktopLaunch | DesktopOpen | D
     snapshot: string;
     key: DesktopKey;
     modifiers?: DesktopModifier[];
+} | {
+    op: "system-key";
+    key: DesktopKey;
+    modifiers?: DesktopModifier[];
 };
 export type DesktopPoint = {
     x: number;
@@ -207,7 +211,7 @@ export type DesktopModifier = (typeof DESKTOP_MODIFIERS)[number];
 export type DesktopSelection = (typeof DESKTOP_SELECTIONS)[number];
 export type DesktopElementAction = (typeof DESKTOP_ELEMENT_ACTIONS)[number];
 export type DesktopAction = Extract<DesktopRequest, {
-    op: "click" | "type" | "key" | "insert" | "select" | "element-action" | "set-value" | "scroll" | "drag" | "hover" | "press" | "activate" | "launch" | "open" | "quit" | "close" | "focus" | "minimize" | "restore" | "move" | "resize" | "clipboard-write" | "menu" | "dialog-answer" | "hide" | "unhide" | "maximize" | "menubar-click" | "dock";
+    op: "click" | "type" | "key" | "system-key" | "insert" | "select" | "element-action" | "set-value" | "scroll" | "drag" | "hover" | "press" | "activate" | "launch" | "open" | "quit" | "close" | "focus" | "minimize" | "restore" | "move" | "resize" | "clipboard-write" | "menu" | "dialog-answer" | "hide" | "unhide" | "maximize" | "menubar-click" | "dock";
 }>;
 export type DesktopOutcome = "completed" | "refused" | "unknown";
 export type DesktopImage = {

dist/protocol.js

@@ -73,6 +73,7 @@ export const DESKTOP_ELEMENT_ACTIONS = [
 ];
 export function isDesktopAction(request) {
     return request.op === "click" || request.op === "type" || request.op === "key"
+        || request.op === "system-key"
         || request.op === "insert" || request.op === "select" || request.op === "scroll"
         || request.op === "drag" || request.op === "hover" || request.op === "press"
         || request.op === "clipboard-write" || request.op === "launch"

dist/tools.js

@@ -77,6 +77,30 @@ const point = {
         y: { type: "number", minimum: 0, exclusiveMaximum: 1 },
     },
 };
+const key = {
+    anyOf: [
+        ...DESKTOP_KEYS.map((key) => ({ type: "string", const: key })),
+        {
+            type: "string",
+            minLength: 1,
+            maxLength: 2,
+            description: "One character, such as , . / ; ' [ ] or a, that a key types without Shift or Option.",
+        },
+    ],
+};
+const modifiers = {
+    type: "array",
+    items: {
+        anyOf: [
+            { type: "string", const: "command" },
+            { type: "string", const: "control" },
+            { type: "string", const: "option" },
+            { type: "string", const: "shift" },
+        ],
+    },
+    maxItems: 4,
+    uniqueItems: true,
+};
 /**
  * The desktop tools every host offers agents. `place` names where they act, as in "this Mac";
  * `app` names the signed app that holds the macOS permissions and runs the native service.
@@ -644,39 +668,27 @@ To drag into another window or app, such as a file from Finder into an app, insp
         {
             name: "desktop_key",
             op: "key",
-            description: `Press and release one key or shortcut in the exact window and focused control bound by a fresh desktop_inspect result. key is a named key or the one character a key types, such as , / [ or a; the key is found on the current keyboard layout. Optional modifiers are command, control, option, and shift; each may appear once. A character that needs Shift or Option, such as ? or an uppercase letter, is refused with the unmodified key and modifier to send instead, so the chord always holds exactly the modifiers given. enter means Return, backspace deletes backward, and delete deletes forward; help is the Insert key on PC keyboards. Command-W, Command-Q, Command-H and Command-M are refused because their effect can't be verified; use desktop_close, desktop_quit, desktop_hide or desktop_minimize. Pass snapshot_id as snapshot. Unsupported shortcuts and stale targets are refused, without global input or activation. The snapshot is single-use; inspect again after the action and never blindly repeat interrupted input. No keys remain held across calls.`,
+            description: `Press and release one key or shortcut in the exact window bound by a fresh desktop_inspect result. When the snapshot reports a focused control, the key is bound to that control (receiver observed_control). Otherwise, as in lists, outlines, tables, canvases and web areas, it goes to whatever has keyboard focus in that window (receiver window_focus); before each event the app's focused element must still be inside the window and the window must be the app's key window, or the key is refused. key is a named key or the one character a key types, such as , / [ or a; the key is found on the current keyboard layout. Optional modifiers are command, control, option, and shift; each may appear once. A character that needs Shift or Option, such as ? or an uppercase letter, is refused with the unmodified key and modifier to send instead, so the chord always holds exactly the modifiers given. enter means Return, backspace deletes backward, and delete deletes forward; help is the Insert key on PC keyboards. Command-W, Command-Q, Command-H and Command-M are refused because their effect can't be verified; use desktop_close, desktop_quit, desktop_hide or desktop_minimize. Pass snapshot_id as snapshot. Unsupported shortcuts and stale targets are refused, without global input or activation. The snapshot is single-use; inspect again after the action and never blindly repeat interrupted input. No keys remain held across calls.`,
             parameters: {
                 type: "object",
                 required: ["snapshot", "key"],
                 properties: {
                     snapshot: id,
-                    key: {
-                        anyOf: [
-                            ...DESKTOP_KEYS.map((key) => ({ type: "string", const: key })),
-                            {
-                                type: "string",
-                                minLength: 1,
-                                maxLength: 2,
-                                description: "One character, such as , . / ; ' [ ] or a, that a key types without Shift or Option.",
-                            },
-                        ],
-                    },
-                    modifiers: {
-                        type: "array",
-                        items: {
-                            anyOf: [
-                                { type: "string", const: "command" },
-                                { type: "string", const: "control" },
-                                { type: "string", const: "option" },
-                                { type: "string", const: "shift" },
-                            ],
-                        },
-                        maxItems: 4,
-                        uniqueItems: true,
-                    },
+                    key,
+                    modifiers,
                 },
             },
         },
+        {
+            name: "desktop_system_key",
+            op: "system-key",
+            description: `Press and release one system-wide shortcut on ${place}, such as Command-Space for Spotlight, Command-Tab to switch apps, Control-Up for Mission Control, Command-Shift-5 for the screenshot toolbar, or Control-Space to change input source. Use desktop_key for keys meant for a window. This needs no snapshot and has no target: the keys go to whatever is frontmost, as if typed on the keyboard, and they usually move focus, activate apps or open system UI. key and modifiers work as in desktop_key, including refusing characters that need Shift or Option. Any chord with Command and W, Q, H or M is refused, because it would close, quit, hide or minimize whatever is frontmost, or log out; use desktop_close, desktop_quit, desktop_hide or desktop_minimize. The result is unknown even when the keys were delivered, because nothing reads back what they did: observe with desktop_screen, desktop_apps or desktop_windows, close what opened (often desktop_system_key escape), and never blindly repeat the shortcut, since many shortcuts toggle. No keys remain held across calls.`,
+            parameters: {
+                type: "object",
+                required: ["key"],
+                properties: { key, modifiers },
+            },
+        },
     ];
 }
 /** The request for one call; exclusive argument forms that schemas cannot state are checked here. */

docs/tools.md

@@ -58,6 +58,7 @@ A status item is pressed through Accessibility in its owner process without movi
 | `desktop_element_action` | Perform one Accessibility action an element lists, such as `AXShowMenu`, `AXIncrement`, `AXDecrement`, `AXPick` or `AXCancel` |
 | `desktop_insert` | Insert text at the caret or over the selection with one temporary paste, then restore the clipboard |
 | `desktop_key` | Press one key with optional Command, Control, Option and Shift: any character key, such as `,` or `/`, or a named key |
+| `desktop_system_key` | Press one system-wide shortcut, such as Command-Space or Control-Up, on whatever is frontmost |
 | `desktop_menu` | Run one menu command by its exact title path |
 | `desktop_dialog_answer` | Press one dialog button by its exact title, or replace one dialog text field's value |
 | `desktop_clipboard_write` | Replace the clipboard with text, an image, or file references |
@@ -66,13 +67,17 @@ Pointer input is delivered to the exact window without moving the physical point
 
 When the host turns on the window highlight, the window an inspection or action targets gets an orange outline and a label such as "Claude is using TextEdit". It shows from the moment the request reaches the host, moves to a new window when the target changes, and fades about four seconds after the last request. The outline follows the window as it moves or resizes and stays directly above it in the window order, so windows in front of the target also cover the outline. It hides while the window is minimized, off screen or on another Space. Like the cursor, it never takes clicks or focus and is left out of captures. Requests without one exact window, such as `desktop_screen`, menus or the clipboard, don't show it.
 
+`desktop_key` goes to the control the snapshot reports as focused. When there is none, as in Finder lists, outlines, tables, canvases and many web areas, it goes to whatever has keyboard focus in that window; the result's `receiver` is `observed_control` or `window_focus`. Either way the key is sent in the background, and is refused unless the app's focused element is still inside the window, and the window is still the app's key window, before each event.
+
 `desktop_key` takes a character key as the character it types, and finds that key on the current keyboard layout, so `,` is the comma key on any layout. A character that needs Shift or Option, such as `?` or `A`, is refused with the key and modifier to send instead (`/` with `shift` on a US layout); the key press never holds a modifier that wasn't asked for. The named keys are `enter`, `tab`, `escape`, `space`, `backspace`, `delete` (forward delete), `up`, `down`, `left`, `right`, `home`, `end`, `pageup`, `pagedown`, `help` (Insert on PC keyboards), `f1` to `f20`, and the keypad keys `keypad0` to `keypad9`, `keypaddecimal`, `keypadplus`, `keypadminus`, `keypadmultiply`, `keypaddivide`, `keypadequals`, `keypadenter` and `keypadclear`. Dead keys, such as `´` on some layouts, can't be pressed alone; use `desktop_insert` for text. Command-W, Command-Q, Command-H and Command-M are refused before anything is sent; `desktop_close`, `desktop_quit`, `desktop_hide` and `desktop_minimize` do the same and report the result.
 
 `desktop_inspect` lists the actions each element offers in `attributes.actions`. A pop-up button has no settable value: click it, then pick the item from the next observation, which puts an open menu's items first.
 
+`desktop_system_key` is for shortcuts that belong to macOS rather than a window: Spotlight (Command-Space), the app switcher (Command-Tab), Mission Control (Control-Up), the screenshot tools (Command-Shift-4 and 5) and the next input source (Control-Space). It takes the same keys and modifiers as `desktop_key`, but no snapshot. The keys are posted as ordinary keyboard input, so they reach whatever is frontmost, and they often move focus, activate an app or open system UI. Nothing reads back what a shortcut did, so the result is `unknown` even after delivery; observe with `desktop_screen`, `desktop_apps` or `desktop_windows`. Any chord with Command and W, Q, H or M is refused before anything is sent, including Command-Shift-Q (log out).
+
 ## Not supported
 
-- **Keys outside text fields.** `desktop_key` and `desktop_insert` need a focused text control, so Escape, arrows, Return and shortcuts in lists, the Finder or canvases are refused ([#13](https://github.com/githubnext/desktop-tools/issues/13)).
+- **Insertion outside text fields.** `desktop_insert` needs an observed focused text control with a readable selection.
 - **Confirmed drops between windows.** A drag into another window or app is always reported as `unknown`, because nothing can confirm the drop. Drags are one straight line, with no modifier keys.
 - **Choosing paths in file panels.** `desktop_dialog_answer` can type a file name into a save panel's field, but it can't navigate an open or save panel to a folder.
 - **Automatic retries.** By design, nothing is retried. After a `refused` or `unknown` result, observe again before deciding whether to act.