Skip to content

Commit c99645c

Browse files
committed
feat: save a screen or window capture as a full-resolution PNG, optionally one region
1 parent ceb45af commit c99645c

12 files changed

Lines changed: 508 additions & 14 deletions

‎dist/capture-file.d.ts‎

Lines changed: 34 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,34 @@
1+
/** A rectangle in global top-left desktop points, as window and display bounds use. */
2+
export type DesktopRegion = {
3+
x: number;
4+
y: number;
5+
width: number;
6+
height: number;
7+
};
8+
export type CaptureSave = {
9+
save_path?: unknown;
10+
overwrite?: unknown;
11+
region?: unknown;
12+
};
13+
export type SavedCapture = {
14+
path: string;
15+
width: number;
16+
height: number;
17+
region: DesktopRegion | null;
18+
};
19+
type Run = (file: string, args: string[], signal: AbortSignal) => Promise<string>;
20+
type Dimensions = (path: string, signal: AbortSignal) => Promise<{
21+
width: number;
22+
height: number;
23+
}>;
24+
/** Checks save_path, overwrite and region before anything is captured. */
25+
export declare function validateSave(request: CaptureSave): Promise<string | undefined>;
26+
/**
27+
* Writes the full-resolution capture, or the part of it under region, as a PNG at path. area is the capture's
28+
* extent in global desktop points; the capture's own pixels map onto it, so nothing is scaled.
29+
*/
30+
export declare function saveCapture(capture: string, scratch: string, path: string, request: CaptureSave, area: DesktopRegion | undefined, tools: {
31+
run: Run;
32+
dimensions: Dimensions;
33+
}, signal: AbortSignal): Promise<SavedCapture>;
34+
export {};

‎dist/capture-file.js‎

Lines changed: 99 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,99 @@
1+
import { constants } from "node:fs";
2+
import { copyFile, lstat, open, stat } from "node:fs/promises";
3+
import { dirname, isAbsolute, normalize } from "node:path";
4+
/** Checks save_path, overwrite and region before anything is captured. */
5+
export async function validateSave(request) {
6+
const { save_path: path, overwrite, region } = request;
7+
if (overwrite !== undefined && typeof overwrite !== "boolean")
8+
throw new Error("Choose overwrite true or false.");
9+
if (path === undefined) {
10+
if (overwrite !== undefined || region !== undefined) {
11+
throw new Error("overwrite and region apply to the saved file; pass save_path too.");
12+
}
13+
return undefined;
14+
}
15+
if (typeof path !== "string" || !isAbsolute(path) || normalize(path) !== path || path.length > 1024
16+
|| path.includes("\0") || !/\.png$/i.test(path)) {
17+
throw new Error("save_path must be a normalized absolute path ending in .png.");
18+
}
19+
if (region !== undefined)
20+
regionOf(region);
21+
const parent = await stat(dirname(path)).catch(() => undefined);
22+
if (!parent?.isDirectory())
23+
throw new Error("save_path's folder must already exist.");
24+
const existing = await lstat(path).catch(() => undefined);
25+
if (existing) {
26+
if (!existing.isFile())
27+
throw new Error("save_path names something other than a regular file; nothing was written.");
28+
if (overwrite !== true) {
29+
throw new Error("save_path already exists; pass overwrite true to replace it. Nothing was captured.");
30+
}
31+
}
32+
return path;
33+
}
34+
function regionOf(value) {
35+
const region = value;
36+
if (!region || typeof region !== "object" || Object.keys(region).length !== 4
37+
|| ![region.x, region.y, region.width, region.height].every((part) => typeof part === "number" && Number.isFinite(part))
38+
|| region.width <= 0 || region.height <= 0) {
39+
throw new Error("region must be x, y, width and height in global desktop points, with positive size.");
40+
}
41+
return region;
42+
}
43+
/**
44+
* Writes the full-resolution capture, or the part of it under region, as a PNG at path. area is the capture's
45+
* extent in global desktop points; the capture's own pixels map onto it, so nothing is scaled.
46+
*/
47+
export async function saveCapture(capture, scratch, path, request, area, tools, signal) {
48+
const size = await tools.dimensions(capture, signal);
49+
let source = capture;
50+
let region = null;
51+
if (request.region !== undefined) {
52+
region = regionOf(request.region);
53+
if (!area || area.width <= 0 || area.height <= 0) {
54+
throw new Error("This capture reported no desktop bounds, so a region can't be located in it.");
55+
}
56+
// Half a point of slack absorbs rounding in reported bounds; the crop still stays inside the image.
57+
const slack = 0.5;
58+
if (region.x < area.x - slack || region.y < area.y - slack
59+
|| region.x + region.width > area.x + area.width + slack
60+
|| region.y + region.height > area.y + area.height + slack) {
61+
throw new Error(`region must lie inside the captured area, x ${area.x}, y ${area.y}, width ${area.width}, height ${area.height} in desktop points.`);
62+
}
63+
const scaleX = size.width / area.width;
64+
const scaleY = size.height / area.height;
65+
const left = Math.max(0, Math.round((region.x - area.x) * scaleX));
66+
const top = Math.max(0, Math.round((region.y - area.y) * scaleY));
67+
const width = Math.min(size.width - left, Math.round(region.width * scaleX));
68+
const height = Math.min(size.height - top, Math.round(region.height * scaleY));
69+
if (width < 1 || height < 1)
70+
throw new Error("region covers less than one pixel of the capture.");
71+
source = `${scratch}/region.png`;
72+
await tools.run("/usr/bin/sips", [
73+
"-s", "format", "png",
74+
"--cropOffset", String(top), String(left),
75+
"-c", String(height), String(width),
76+
capture, "--out", source,
77+
], signal);
78+
}
79+
else if (!(await isPng(capture))) {
80+
source = `${scratch}/full.png`;
81+
await tools.run("/usr/bin/sips", ["-s", "format", "png", capture, "--out", source], signal);
82+
}
83+
const saved = await tools.dimensions(source, signal);
84+
signal.throwIfAborted();
85+
// Exclusive unless overwrite was asked for, so a file created since validation is never replaced.
86+
await copyFile(source, path, request.overwrite === true ? 0 : constants.COPYFILE_EXCL);
87+
return { path, width: saved.width, height: saved.height, region };
88+
}
89+
async function isPng(path) {
90+
const file = await open(path, "r");
91+
try {
92+
const head = Buffer.alloc(8);
93+
await file.read(head, 0, 8, 0);
94+
return head.equals(Buffer.from([0x89, 0x50, 0x4e, 0x47, 0x0d, 0x0a, 0x1a, 0x0a]));
95+
}
96+
finally {
97+
await file.close();
98+
}
99+
}

‎dist/client.js‎

Lines changed: 27 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -6,6 +6,7 @@ import { join } from "node:path";
66
import { promisify } from "node:util";
77
import { failureAvailability, windowAvailability } from "./availability.js";
88
import { canonicalDate } from "./date-value.js";
9+
import { saveCapture, validateSave } from "./capture-file.js";
910
import { DESKTOP_BUTTONS, DESKTOP_CLICKS, DESKTOP_DIRECTIONS, DESKTOP_ELEMENT_ACTIONS, DESKTOP_MODIFIERS, DESKTOP_SELECTIONS, isDesktopAction, isDesktopKey, isDesktopManagement, } from "./protocol.js";
1011
import { NOTIFICATIONS_MS, notificationAction, readNotifications, validateNotificationAction, } from "./notifications.js";
1112
import { waitFor } from "./wait.js";
@@ -185,6 +186,22 @@ async function screenshot(input, output, signal) {
185186
}
186187
throw new Error("The screenshot could not be resized within the result limit.");
187188
}
189+
const saveTools = { run: (file, args, signal) => run(file, args, signal), dimensions };
190+
/** The window capture's extent in global desktop points, from its coordinate context. */
191+
function capturedArea(data) {
192+
const bounds = data.coordinate_context?.logicalBounds;
193+
if (!Array.isArray(bounds) || bounds.length !== 2)
194+
return undefined;
195+
const [[x, y], [width, height]] = bounds;
196+
return [x, y, width, height].every(Number.isFinite) ? { x, y, width, height } : undefined;
197+
}
198+
/** The captured display's bounds in global desktop points. */
199+
function displayArea(data) {
200+
const bounds = data.display?.bounds;
201+
return bounds && [bounds.x, bounds.y, bounds.width, bounds.height].every(Number.isFinite)
202+
? { x: bounds.x, y: bounds.y, width: bounds.width, height: bounds.height }
203+
: undefined;
204+
}
188205
async function dimensions(path, signal) {
189206
const info = await run("/usr/bin/sips", ["-g", "pixelWidth", "-g", "pixelHeight", path], signal);
190207
const width = Number(/pixelWidth:\s*(\d+)/.exec(info)?.[1]);
@@ -255,6 +272,7 @@ async function inspect(native, request, signal) {
255272
if (mode !== "accessibility")
256273
throw new Error("A scope needs accessibility mode.");
257274
}
275+
const savePath = await validateSave(request);
258276
const directory = await mkdtemp(join(tmpdir(), "desktop-tools-"));
259277
try {
260278
const path = join(directory, "capture.png");
@@ -268,10 +286,14 @@ async function inspect(native, request, signal) {
268286
throw new Error("Native inspection did not honor the requested scope.");
269287
}
270288
const limit = (request.scope ? MAX_SCOPED_TEXT : MAX_TEXT) + recognizedLimit(data, request.ocr);
289+
const saved = savePath
290+
? await saveCapture(path, directory, savePath, request, capturedArea(data), saveTools, signal)
291+
: undefined;
271292
const { image, width, height } = await screenshot(path, join(directory, "image.jpg"), signal);
272293
const { screenshot_raw: _raw, screenshot_annotated: _annotated, ...observation } = data;
273294
const text = bounded({
274295
...observation,
296+
...(saved ? { saved } : {}),
275297
image: {
276298
width,
277299
height,
@@ -309,6 +331,7 @@ async function screen(native, request, signal) {
309331
}
310332
if (request.ocr !== undefined && typeof request.ocr !== "boolean")
311333
throw new Error("Choose ocr true or false.");
334+
const savePath = await validateSave(request);
312335
const directory = await mkdtemp(join(tmpdir(), "desktop-tools-"));
313336
try {
314337
const path = join(directory, "capture.png");
@@ -319,9 +342,13 @@ async function screen(native, request, signal) {
319342
args.push("ocr");
320343
const data = await native(args, signal);
321344
const limit = MAX_TEXT + recognizedLimit(data, request.ocr);
345+
const saved = savePath
346+
? await saveCapture(path, directory, savePath, request, displayArea(data), saveTools, signal)
347+
: undefined;
322348
const { image, width, height } = await screenshot(path, join(directory, "image.jpg"), signal);
323349
const text = bounded({
324350
...data,
351+
...(saved ? { saved } : {}),
325352
image: {
326353
width,
327354
height,

‎dist/protocol.d.ts‎

Lines changed: 15 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -105,6 +105,17 @@ export type DesktopWait = {
105105
text?: string;
106106
timeout_ms?: number;
107107
};
108+
/** Writes the full-resolution capture, or the part under region in global desktop points, as a PNG. */
109+
export type DesktopCaptureSave = {
110+
save_path?: string;
111+
overwrite?: boolean;
112+
region?: {
113+
x: number;
114+
y: number;
115+
width: number;
116+
height: number;
117+
};
118+
};
108119
export type DesktopRequest = DesktopManagement | DesktopLaunch | DesktopOpen | DesktopDialogAnswer | {
109120
op: "dialog";
110121
target: DesktopAppTarget | DesktopWindowTarget;
@@ -150,14 +161,14 @@ export type DesktopRequest = DesktopManagement | DesktopLaunch | DesktopOpen | D
150161
} | {
151162
op: "switch-space";
152163
space_id: number;
153-
} | {
164+
} | ({
154165
op: "inspect";
155166
pid: number;
156167
window: number;
157168
mode?: "accessibility" | "pixels";
158169
ocr?: boolean;
159170
scope?: DesktopScope;
160-
} | {
171+
} & DesktopCaptureSave) | {
161172
op: "read";
162173
pid: number;
163174
window: number;
@@ -168,11 +179,11 @@ export type DesktopRequest = DesktopManagement | DesktopLaunch | DesktopOpen | D
168179
op: "notification-action";
169180
id: string;
170181
action: string;
171-
} | {
182+
} | ({
172183
op: "screen";
173184
display?: number;
174185
ocr?: boolean;
175-
} | {
186+
} & DesktopCaptureSave) | {
176187
op: "click";
177188
snapshot: string;
178189
element?: string;

‎dist/tools.d.ts‎

Lines changed: 0 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -9,10 +9,6 @@ export type DesktopTool = {
99
/** JSON Schema for the tool's arguments; its properties are the request's fields besides `op`. */
1010
parameters: JsonSchema;
1111
};
12-
/**
13-
* The desktop tools every host offers agents. `place` names where they act, as in "this Mac";
14-
* `app` names the signed app that holds the macOS permissions and runs the native service.
15-
*/
1612
export declare function desktopTools({ place, app }: {
1713
place: string;
1814
app: string;

0 commit comments

Comments
 (0)