From 885ffa7cc538ee0a1a97a1c59c3a468e7a6b8e4d Mon Sep 17 00:00:00 2001 From: nikhil Date: Wed, 7 Oct 2026 22:12:39 +0000 Subject: [PATCH 1/8] Expose desktop cursor position and pointer modifier keys --- README.md | 21 +++ src/services/computer-action.ts | 34 ++-- src/types/computer-action.ts | 22 ++- src/types/index.ts | 2 + tests/integration/computer-action.test.ts | 184 ++++++++++++++++++++++ tests/unit/public-types-contract.test.ts | 26 ++- 6 files changed, 277 insertions(+), 12 deletions(-) create mode 100644 tests/integration/computer-action.test.ts diff --git a/README.md b/README.md index dba5978..10ea00d 100644 --- a/README.md +++ b/README.md @@ -513,6 +513,27 @@ resources. Invalid or conflicting launch sources fail before a network request. Public handle classes are available from `@hyperbrowser/sdk/sandbox`; request and response types remain under `@hyperbrowser/sdk/types`. +### Computer action cursor and modifier keys + +`cursorPosition()` returns physical desktop coordinates in `result.data`. +Click, drag and scroll accept an optional trailing `keys` array of xdotool key +names held during the action. Existing positional arguments are unchanged. +Omit click coordinates, or pass `undefined` for scroll coordinates, to use the +current cursor. + +```typescript +const session = await client.sessions.create(); +try { + const result = await client.computerAction.cursorPosition(session); + if (result.data && "x" in result.data) console.log(result.data.x, result.data.y); + await client.computerAction.click(session, 100, 200, "left", 1, false, ["Control_L"]); + await client.computerAction.drag(session, [{ x: 100, y: 200 }, { x: 200, y: 300 }], false, ["Shift_L"]); + await client.computerAction.scroll(session, undefined, undefined, 0, 2, false, ["Control_L"]); +} finally { + await client.sessions.stop(session.id); +} +``` + ### Development checks The supported Node baseline is 20.20.2. Run `yarn build`, `yarn typecheck`, diff --git a/src/services/computer-action.ts b/src/services/computer-action.ts index 7fc3674..769f617 100644 --- a/src/services/computer-action.ts +++ b/src/services/computer-action.ts @@ -49,11 +49,12 @@ export class ComputerActionService extends BaseService { async click( session: SessionDetail | string, - x: number, - y: number, + x?: number, + y?: number, button: ComputerActionMouseButton = "left", numClicks: number = 1, - returnScreenshot: boolean = false + returnScreenshot: boolean = false, + keys?: string[] ): Promise { return this.executeRequest(session, { action: ComputerAction.CLICK, @@ -61,6 +62,7 @@ export class ComputerActionService extends BaseService { y, button, numClicks, + keys, returnScreenshot, }); } @@ -83,6 +85,16 @@ export class ComputerActionService extends BaseService { }); } + async cursorPosition( + session: SessionDetail | string, + returnScreenshot: boolean = false + ): Promise { + return this.executeRequest(session, { + action: ComputerAction.CURSOR_POSITION, + returnScreenshot, + }); + } + async pressKeys( session: SessionDetail | string, keys: string[], @@ -98,11 +110,13 @@ export class ComputerActionService extends BaseService { async drag( session: SessionDetail | string, path: Coordinate[], - returnScreenshot: boolean = false + returnScreenshot: boolean = false, + keys?: string[] ): Promise { return this.executeRequest(session, { action: ComputerAction.DRAG, path, + keys, returnScreenshot, }); } @@ -123,11 +137,12 @@ export class ComputerActionService extends BaseService { async scroll( session: SessionDetail | string, - x: number, - y: number, - scrollX: number, - scrollY: number, - returnScreenshot: boolean = false + x?: number, + y?: number, + scrollX: number = 0, + scrollY: number = 0, + returnScreenshot: boolean = false, + keys?: string[] ): Promise { return this.executeRequest(session, { action: ComputerAction.SCROLL, @@ -135,6 +150,7 @@ export class ComputerActionService extends BaseService { y, scrollX, scrollY, + keys, returnScreenshot, }); } diff --git a/src/types/computer-action.ts b/src/types/computer-action.ts index a04b822..6f71cb3 100644 --- a/src/types/computer-action.ts +++ b/src/types/computer-action.ts @@ -10,6 +10,7 @@ export enum ComputerAction { PRESS_KEYS = "press_keys", MOVE_MOUSE = "move_mouse", SCREENSHOT = "screenshot", + CURSOR_POSITION = "cursor_position", SCROLL = "scroll", TYPE_TEXT = "type_text", GET_CLIPBOARD_TEXT = "get_clipboard_text", @@ -36,6 +37,7 @@ export interface ClickActionParams { y?: number; button?: ComputerActionMouseButton; numClicks?: number; + keys?: string[]; returnScreenshot?: boolean; } @@ -45,6 +47,7 @@ export interface ClickActionParams { export interface DragActionParams { action: ComputerAction.DRAG; path: Coordinate[]; + keys?: string[]; returnScreenshot?: boolean; } @@ -74,13 +77,20 @@ export interface ScreenshotActionParams { action: ComputerAction.SCREENSHOT; } +/** Parameters for reading the desktop cursor position. */ +export interface CursorPositionActionParams { + action: ComputerAction.CURSOR_POSITION; + returnScreenshot?: boolean; +} + /** * Parameters for scroll action */ export interface ScrollActionParams { action: ComputerAction.SCROLL; - x: number; - y: number; + x?: number; + y?: number; + keys?: string[]; scrollX: number; scrollY: number; returnScreenshot?: boolean; @@ -151,6 +161,7 @@ export type ComputerActionParams = | PressKeysActionParams | MoveMouseActionParams | ScreenshotActionParams + | CursorPositionActionParams | ScrollActionParams | TypeTextActionParams | HoldKeyActionParams @@ -181,7 +192,14 @@ export interface ComputerActionResponseDataListWindows { windows: ComputerActionWindow[]; } +/** Desktop cursor coordinates in physical screen pixels. */ +export interface ComputerActionResponseDataCursorPosition { + x: number; + y: number; +} + export type ComputerActionResponseData = + | ComputerActionResponseDataCursorPosition | ComputerActionResponseDataClipboardText | ComputerActionResponseDataListWindows; diff --git a/src/types/index.ts b/src/types/index.ts index c121918..4890b26 100644 --- a/src/types/index.ts +++ b/src/types/index.ts @@ -328,6 +328,7 @@ export { PressKeysActionParams, MoveMouseActionParams, ScreenshotActionParams, + CursorPositionActionParams, ScrollActionParams, TypeTextActionParams, ComputerActionParams, @@ -343,6 +344,7 @@ export { ListWindowsActionParams, ComputerActionWindow, ComputerActionResponseDataListWindows, + ComputerActionResponseDataCursorPosition, } from "./computer-action"; export { FetchParams, FetchResponse, FetchResponseData, FetchStatus } from "./web/fetch"; export { diff --git a/tests/integration/computer-action.test.ts b/tests/integration/computer-action.test.ts new file mode 100644 index 0000000..2bfefe2 --- /dev/null +++ b/tests/integration/computer-action.test.ts @@ -0,0 +1,184 @@ +import http, { Server } from "node:http"; +import { AddressInfo } from "node:net"; +import { afterEach, describe, expect, test } from "vitest"; +import { HyperbrowserClient, HyperbrowserError } from "../../src/client"; +import type { SessionDetail } from "../../src/types"; + +const servers: Server[] = []; +afterEach(async () => { + await Promise.all( + servers.splice(0).map( + (server) => + new Promise((resolve, reject) => { + server.close((error) => (error ? reject(error) : resolve())); + }) + ) + ); +}); + +async function fixture(success = true) { + const requests: { method?: string; url?: string; apiKey?: string; body?: unknown }[] = []; + let session: SessionDetail; + const server = http.createServer(async (request, response) => { + const chunks: Buffer[] = []; + for await (const chunk of request) chunks.push(Buffer.from(chunk)); + const body = chunks.length ? JSON.parse(Buffer.concat(chunks).toString("utf8")) : undefined; + requests.push({ + method: request.method, + url: request.url, + apiKey: request.headers["x-api-key"]?.toString(), + body, + }); + response.setHeader("content-type", "application/json"); + if (request.method === "GET" && request.url === "/api/session/session-id") { + response.end(JSON.stringify(session)); + } else if (request.method === "POST" && request.url === "/action") { + response.end( + JSON.stringify({ + success, + ...(body.action === "cursor_position" ? { data: { x: 123, y: 456 } } : {}), + ...(!success ? { error: "action failed" } : {}), + }) + ); + } else { + response.writeHead(404); + response.end(JSON.stringify({ error: "not found" })); + } + }); + servers.push(server); + await new Promise((resolve, reject) => { + server.once("error", reject); + server.listen(0, "127.0.0.1", resolve); + }); + const baseUrl = `http://127.0.0.1:${(server.address() as AddressInfo).port}`; + session = { + id: "session-id", + teamId: "team-id", + status: "active", + createdAt: "2026-01-01T00:00:00Z", + updatedAt: "2026-01-01T00:00:00Z", + sessionUrl: baseUrl, + token: "test-token", + wsEndpoint: "wss://session.example", + computerActionEndpoint: `${baseUrl}/action`, + creditsUsed: null, + creditBreakdown: { + creditsUsed: null, + browserTimeCreditsUsed: null, + proxyDataCreditsUsed: null, + }, + }; + const client = new HyperbrowserClient({ apiKey: "test-key", baseUrl }); + return { actions: client.computerAction, requests, session }; +} + +describe("computer action primitives", () => { + test.each([false, true])( + "reads cursor coordinates with borrowed session, by ID=%s", + async (byId) => { + const { actions, session, requests } = await fixture(); + const result = await actions.cursorPosition(byId ? session.id : session, true); + expect(result).toEqual({ success: true, data: { x: 123, y: 456 } }); + expect(requests.at(-1)).toEqual({ + method: "POST", + url: "/action", + apiKey: "test-key", + body: { action: "cursor_position", returnScreenshot: true }, + }); + expect(requests).toHaveLength(byId ? 2 : 1); + } + ); + + test.each(["click", "drag", "scroll"] as const)( + "sends %s modifier chords and preserves existing arguments", + async (action) => { + const { actions, session, requests } = await fixture(); + const keys = ["Control_L", "Shift_L"]; + if (action === "click") await actions.click(session, 10, 20, "right", 2, true, keys); + if (action === "drag") + await actions.drag( + session, + [ + { x: 10, y: 20 }, + { x: 30, y: 40 }, + ], + true, + keys + ); + if (action === "scroll") await actions.scroll(session, 10, 20, 0, -2, true, keys); + const expected = { + click: { + action, + x: 10, + y: 20, + button: "right", + numClicks: 2, + returnScreenshot: true, + keys, + }, + drag: { + action, + path: [ + { x: 10, y: 20 }, + { x: 30, y: 40 }, + ], + returnScreenshot: true, + keys, + }, + scroll: { action, x: 10, y: 20, scrollX: 0, scrollY: -2, returnScreenshot: true, keys }, + }; + expect(requests.at(-1)?.body).toEqual(expected[action]); + expect(keys).toEqual(["Control_L", "Shift_L"]); + } + ); + + test.each([undefined, []])( + "keeps omitted and empty modifier lists distinct: %s", + async (keys) => { + const { actions, session, requests } = await fixture(); + await actions.click(session, 10, 20, "left", 1, false, keys); + await actions.drag(session, [{ x: 10, y: 20 }], false, keys); + await actions.scroll(session, 10, 20, 0, 1, false, keys); + for (const request of requests) { + const body = request.body as Record; + if (keys === undefined) expect(body).not.toHaveProperty("keys"); + else expect(body.keys).toEqual([]); + } + } + ); + + test("uses current cursor when click or scroll coordinates are omitted", async () => { + const { actions, session, requests } = await fixture(); + await actions.click(session); + await actions.scroll(session, undefined, undefined, 0, 2, false, ["Control_L"]); + expect(requests[0].body).toEqual({ + action: "click", + button: "left", + numClicks: 1, + returnScreenshot: false, + }); + expect(requests[1].body).toEqual({ + action: "scroll", + scrollX: 0, + scrollY: 2, + returnScreenshot: false, + keys: ["Control_L"], + }); + }); + + test("reports missing computer endpoint without issuing a request", async () => { + const { actions, session, requests } = await fixture(); + await expect( + actions.cursorPosition({ ...session, computerActionEndpoint: undefined }) + ).rejects.toBeInstanceOf(HyperbrowserError); + expect(requests).toHaveLength(0); + }); + + test("preserves unsuccessful action responses", async () => { + const { actions, session } = await fixture(false); + expect(await actions.cursorPosition(session)).toMatchObject({ + success: false, + error: "action failed", + }); + }); +}); diff --git a/tests/unit/public-types-contract.test.ts b/tests/unit/public-types-contract.test.ts index 3cfc02e..c567875 100644 --- a/tests/unit/public-types-contract.test.ts +++ b/tests/unit/public-types-contract.test.ts @@ -1,5 +1,10 @@ import { describe, expect, expectTypeOf, test } from "vitest"; import type { + ClickActionParams, + DragActionParams, + ScrollActionParams, + CursorPositionActionParams, + ComputerActionResponseDataCursorPosition, CompleteSandboxImageBuildParams, CreateSandboxImageBuildParams, Sandbox, @@ -15,8 +20,25 @@ import type { SessionStatus, VolumeListResponse, } from "../../src/types"; +import { ComputerAction } from "../../src/types"; describe("public type compatibility", () => { + test("exports cursor-position requests and typed coordinates with optional pointer modifiers", () => { + const request: CursorPositionActionParams = { action: ComputerAction.CURSOR_POSITION }; + const response: ComputerActionResponseDataCursorPosition = { x: 10, y: 20 }; + const scroll: ScrollActionParams = { + action: ComputerAction.SCROLL, + scrollX: 0, + scrollY: 1, + keys: ["Shift_L"], + }; + expect(request.action).toBe("cursor_position"); + expect(response).toEqual({ x: 10, y: 20 }); + expect(scroll).not.toHaveProperty("x"); + expectTypeOf().toEqualTypeOf(); + expectTypeOf().toEqualTypeOf(); + expectTypeOf().toEqualTypeOf(); + }); test("keeps newly available response data optional", () => { const imageResponse: SandboxImageListResponse = { images: [] }; const snapshotResponse: SandboxSnapshotListResponse = { snapshots: [] }; @@ -52,7 +74,9 @@ describe("public type compatibility", () => { expectTypeOf().toEqualTypeOf< SandboxImageBuildInputFormat | undefined >(); - expectTypeOf().toEqualTypeOf(); + expectTypeOf().toEqualTypeOf< + number | undefined + >(); expectTypeOf().toEqualTypeOf(); expectTypeOf().toEqualTypeOf(); expectTypeOf().toEqualTypeOf< From 42f558f25caccae5083d8cc243bb1a7c0ceacb12 Mon Sep 17 00:00:00 2001 From: nikhil Date: Wed, 7 Oct 2026 23:16:22 +0000 Subject: [PATCH 2/8] Bump Node SDK minor version to 1.1.0 --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index 18f3733..cbf9770 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "@hyperbrowser/sdk", - "version": "1.0.3", + "version": "1.1.0", "description": "Node SDK for Hyperbrowser API", "author": "", "repository": { From 659f3f9fd44d5b21da447f7d52c202dc7e23c46e Mon Sep 17 00:00:00 2001 From: nikhil Date: Thu, 8 Oct 2026 00:23:00 +0000 Subject: [PATCH 3/8] Preserve existing computer-action typing contracts --- README.md | 10 +-- src/services/computer-action.ts | 63 ++++++++++++++---- src/types/computer-action.ts | 19 ++++-- src/types/index.ts | 2 + tests/integration/computer-action.test.ts | 34 +++++++++- tests/unit/public-types-contract.test.ts | 81 ++++++++++++++++++++++- 6 files changed, 181 insertions(+), 28 deletions(-) diff --git a/README.md b/README.md index 10ea00d..1633db4 100644 --- a/README.md +++ b/README.md @@ -518,17 +518,19 @@ response types remain under `@hyperbrowser/sdk/types`. `cursorPosition()` returns physical desktop coordinates in `result.data`. Click, drag and scroll accept an optional trailing `keys` array of xdotool key names held during the action. Existing positional arguments are unchanged. -Omit click coordinates, or pass `undefined` for scroll coordinates, to use the -current cursor. +Use `clickAtCursor()` or `scrollAtCursor()` to act at the current pointer. +`click()` and `scroll()` still require coordinates. `cursorPosition()` returns a separate +`CursorPositionActionResponse`; the existing action enum, request union and +response-data union remain unchanged. ```typescript const session = await client.sessions.create(); try { const result = await client.computerAction.cursorPosition(session); - if (result.data && "x" in result.data) console.log(result.data.x, result.data.y); + if (result.data) console.log(result.data.x, result.data.y); await client.computerAction.click(session, 100, 200, "left", 1, false, ["Control_L"]); await client.computerAction.drag(session, [{ x: 100, y: 200 }, { x: 200, y: 300 }], false, ["Shift_L"]); - await client.computerAction.scroll(session, undefined, undefined, 0, 2, false, ["Control_L"]); + await client.computerAction.scrollAtCursor(session, 0, 2, false, ["Control_L"]); } finally { await client.sessions.stop(session.id); } diff --git a/src/services/computer-action.ts b/src/services/computer-action.ts index 769f617..4121d5f 100644 --- a/src/services/computer-action.ts +++ b/src/services/computer-action.ts @@ -3,6 +3,9 @@ import { ComputerAction, ComputerActionParams, ComputerActionResponse, + CursorPositionActionParams, + CursorPositionActionResponse, + ScrollAtCursorActionParams, Coordinate, ComputerActionMouseButton, } from "../types/computer-action"; @@ -10,10 +13,12 @@ import { BaseService } from "./base"; import { HyperbrowserError } from "../client"; export class ComputerActionService extends BaseService { - private async executeRequest( + private async executeRequest< + Response extends ComputerActionResponse | CursorPositionActionResponse = ComputerActionResponse, + >( session: SessionDetail | string, - params: ComputerActionParams - ): Promise { + params: ComputerActionParams | CursorPositionActionParams | ScrollAtCursorActionParams + ): Promise { try { let sessionDetail: SessionDetail; @@ -30,7 +35,7 @@ export class ComputerActionService extends BaseService { ); } - return await this.request( + return await this.request( sessionDetail.computerActionEndpoint, { method: "POST", @@ -49,8 +54,8 @@ export class ComputerActionService extends BaseService { async click( session: SessionDetail | string, - x?: number, - y?: number, + x: number, + y: number, button: ComputerActionMouseButton = "left", numClicks: number = 1, returnScreenshot: boolean = false, @@ -67,6 +72,22 @@ export class ComputerActionService extends BaseService { }); } + async clickAtCursor( + session: SessionDetail | string, + button: ComputerActionMouseButton = "left", + numClicks: number = 1, + returnScreenshot: boolean = false, + keys?: string[] + ): Promise { + return this.executeRequest(session, { + action: ComputerAction.CLICK, + button, + numClicks, + returnScreenshot, + keys, + }); + } + async typeText( session: SessionDetail | string, text: string, @@ -88,9 +109,9 @@ export class ComputerActionService extends BaseService { async cursorPosition( session: SessionDetail | string, returnScreenshot: boolean = false - ): Promise { - return this.executeRequest(session, { - action: ComputerAction.CURSOR_POSITION, + ): Promise { + return this.executeRequest(session, { + action: "cursor_position", returnScreenshot, }); } @@ -137,10 +158,10 @@ export class ComputerActionService extends BaseService { async scroll( session: SessionDetail | string, - x?: number, - y?: number, - scrollX: number = 0, - scrollY: number = 0, + x: number, + y: number, + scrollX: number, + scrollY: number, returnScreenshot: boolean = false, keys?: string[] ): Promise { @@ -155,6 +176,22 @@ export class ComputerActionService extends BaseService { }); } + async scrollAtCursor( + session: SessionDetail | string, + scrollX: number = 0, + scrollY: number = 0, + returnScreenshot: boolean = false, + keys?: string[] + ): Promise { + return this.executeRequest(session, { + action: ComputerAction.SCROLL, + scrollX, + scrollY, + returnScreenshot, + keys, + }); + } + async holdKey( session: SessionDetail | string, key: string, diff --git a/src/types/computer-action.ts b/src/types/computer-action.ts index 6f71cb3..134e2ab 100644 --- a/src/types/computer-action.ts +++ b/src/types/computer-action.ts @@ -10,7 +10,6 @@ export enum ComputerAction { PRESS_KEYS = "press_keys", MOVE_MOUSE = "move_mouse", SCREENSHOT = "screenshot", - CURSOR_POSITION = "cursor_position", SCROLL = "scroll", TYPE_TEXT = "type_text", GET_CLIPBOARD_TEXT = "get_clipboard_text", @@ -79,7 +78,7 @@ export interface ScreenshotActionParams { /** Parameters for reading the desktop cursor position. */ export interface CursorPositionActionParams { - action: ComputerAction.CURSOR_POSITION; + action: "cursor_position"; returnScreenshot?: boolean; } @@ -88,14 +87,17 @@ export interface CursorPositionActionParams { */ export interface ScrollActionParams { action: ComputerAction.SCROLL; - x?: number; - y?: number; + x: number; + y: number; keys?: string[]; scrollX: number; scrollY: number; returnScreenshot?: boolean; } +/** Parameters for scrolling without moving the desktop cursor. */ +export type ScrollAtCursorActionParams = Omit; + /** * Parameters for type text action */ @@ -153,7 +155,7 @@ export interface ListWindowsActionParams { } /** - * Union type for all computer action parameters + * Existing action parameters; cursor reads and cursor-relative scrolling use dedicated helpers. */ export type ComputerActionParams = | ClickActionParams @@ -161,7 +163,6 @@ export type ComputerActionParams = | PressKeysActionParams | MoveMouseActionParams | ScreenshotActionParams - | CursorPositionActionParams | ScrollActionParams | TypeTextActionParams | HoldKeyActionParams @@ -199,7 +200,6 @@ export interface ComputerActionResponseDataCursorPosition { } export type ComputerActionResponseData = - | ComputerActionResponseDataCursorPosition | ComputerActionResponseDataClipboardText | ComputerActionResponseDataListWindows; @@ -213,3 +213,8 @@ export interface ComputerActionResponse { error?: string; message?: string; } + +/** Response from reading the desktop cursor position. */ +export interface CursorPositionActionResponse extends Omit { + data?: ComputerActionResponseDataCursorPosition; +} diff --git a/src/types/index.ts b/src/types/index.ts index 4890b26..dba85f7 100644 --- a/src/types/index.ts +++ b/src/types/index.ts @@ -329,6 +329,8 @@ export { MoveMouseActionParams, ScreenshotActionParams, CursorPositionActionParams, + CursorPositionActionResponse, + ScrollAtCursorActionParams, ScrollActionParams, TypeTextActionParams, ComputerActionParams, diff --git a/tests/integration/computer-action.test.ts b/tests/integration/computer-action.test.ts index 2bfefe2..dde98d5 100644 --- a/tests/integration/computer-action.test.ts +++ b/tests/integration/computer-action.test.ts @@ -149,8 +149,8 @@ describe("computer action primitives", () => { test("uses current cursor when click or scroll coordinates are omitted", async () => { const { actions, session, requests } = await fixture(); - await actions.click(session); - await actions.scroll(session, undefined, undefined, 0, 2, false, ["Control_L"]); + await actions.clickAtCursor(session); + await actions.scrollAtCursor(session, 0, 2, false, ["Control_L"]); expect(requests[0].body).toEqual({ action: "click", button: "left", @@ -174,6 +174,23 @@ describe("computer action primitives", () => { expect(requests).toHaveLength(0); }); + test.each([undefined, [], ["Shift_L"]])( + "scrolls at cursor with screenshot, session lookup and modifiers: %s", + async (keys) => { + const { actions, session, requests } = await fixture(); + const result = await actions.scrollAtCursor(session.id, -1, 2, true, keys); + expect(result.success).toBe(true); + expect(requests[0].method).toBe("GET"); + expect(requests[1].body).toEqual({ + action: "scroll", + scrollX: -1, + scrollY: 2, + returnScreenshot: true, + ...(keys === undefined ? {} : { keys }), + }); + } + ); + test("preserves unsuccessful action responses", async () => { const { actions, session } = await fixture(false); expect(await actions.cursorPosition(session)).toMatchObject({ @@ -181,4 +198,17 @@ describe("computer action primitives", () => { error: "action failed", }); }); + + test("clicks at cursor with buttons, counts, screenshot and modifiers", async () => { + const { actions, session, requests } = await fixture(); + await actions.clickAtCursor(session.id, "right", 2, true, ["Control_L"]); + expect(requests[0].method).toBe("GET"); + expect(requests[1].body).toEqual({ + action: "click", + button: "right", + numClicks: 2, + returnScreenshot: true, + keys: ["Control_L"], + }); + }); }); diff --git a/tests/unit/public-types-contract.test.ts b/tests/unit/public-types-contract.test.ts index c567875..e4e0667 100644 --- a/tests/unit/public-types-contract.test.ts +++ b/tests/unit/public-types-contract.test.ts @@ -3,7 +3,12 @@ import type { ClickActionParams, DragActionParams, ScrollActionParams, + ScrollAtCursorActionParams, + ComputerActionParams, + ComputerActionResponseData, + ComputerActionResponse, CursorPositionActionParams, + CursorPositionActionResponse, ComputerActionResponseDataCursorPosition, CompleteSandboxImageBuildParams, CreateSandboxImageBuildParams, @@ -21,12 +26,41 @@ import type { VolumeListResponse, } from "../../src/types"; import { ComputerAction } from "../../src/types"; +import { ComputerActionService } from "../../src/services/computer-action"; + +function existingResponseConsumer(data: ComputerActionResponseData): string { + if ("windows" in data) return data.activeWindowId; + return data.clipboardText ?? ""; +} + +function existingActionConsumer(params: ComputerActionParams): string { + switch (params.action) { + case ComputerAction.CLICK: + case ComputerAction.DRAG: + case ComputerAction.HOLD_KEY: + case ComputerAction.MOUSE_DOWN: + case ComputerAction.MOUSE_UP: + case ComputerAction.PRESS_KEYS: + case ComputerAction.MOVE_MOUSE: + case ComputerAction.SCREENSHOT: + case ComputerAction.SCROLL: + case ComputerAction.TYPE_TEXT: + case ComputerAction.GET_CLIPBOARD_TEXT: + case ComputerAction.PUT_SELECTION_TEXT: + case ComputerAction.LIST_WINDOWS: + return params.action; + default: { + const exhaustive: never = params; + return exhaustive; + } + } +} describe("public type compatibility", () => { test("exports cursor-position requests and typed coordinates with optional pointer modifiers", () => { - const request: CursorPositionActionParams = { action: ComputerAction.CURSOR_POSITION }; + const request: CursorPositionActionParams = { action: "cursor_position" }; const response: ComputerActionResponseDataCursorPosition = { x: 10, y: 20 }; - const scroll: ScrollActionParams = { + const scroll: ScrollAtCursorActionParams = { action: ComputerAction.SCROLL, scrollX: 0, scrollY: 1, @@ -39,6 +73,49 @@ describe("public type compatibility", () => { expectTypeOf().toEqualTypeOf(); expectTypeOf().toEqualTypeOf(); }); + + test("preserves existing coordinate, response and exhaustive action consumers", () => { + const actionNames: Record = { + click: "click", + drag: "drag", + hold_key: "hold_key", + mouse_down: "mouse_down", + mouse_up: "mouse_up", + press_keys: "press_keys", + move_mouse: "move_mouse", + screenshot: "screenshot", + scroll: "scroll", + type_text: "type_text", + get_clipboard_text: "get_clipboard_text", + put_selection_text: "put_selection_text", + list_windows: "list_windows", + }; + const scroll: ScrollActionParams = { + action: ComputerAction.SCROLL, + x: 10, + y: 20, + scrollX: 0, + scrollY: 1, + }; + const x: number = scroll.x; + const y: number = scroll.y; + expect(x + y).toBe(30); + expect(actionNames[scroll.action]).toBe("scroll"); + expect(existingActionConsumer(scroll)).toBe("scroll"); + expect(existingResponseConsumer({ clipboardText: "hello" })).toBe("hello"); + expectTypeOf().toEqualTypeOf(); + expectTypeOf().toEqualTypeOf(); + expectTypeOf[1]>().toEqualTypeOf(); + expectTypeOf[2]>().toEqualTypeOf(); + expectTypeOf[1]>().toEqualTypeOf(); + expectTypeOf[2]>().toEqualTypeOf(); + expectTypeOf>().toEqualTypeOf< + Promise + >(); + expectTypeOf>().toEqualTypeOf< + Promise + >(); + }); test("keeps newly available response data optional", () => { const imageResponse: SandboxImageListResponse = { images: [] }; const snapshotResponse: SandboxSnapshotListResponse = { snapshots: [] }; From d2d7df84fbd9d816206f2bc002428024a755c741 Mon Sep 17 00:00:00 2001 From: nikhil Date: Thu, 8 Oct 2026 01:45:24 +0000 Subject: [PATCH 4/8] Revert "Preserve existing computer-action typing contracts" This reverts commit 659f3f9fd44d5b21da447f7d52c202dc7e23c46e. --- README.md | 10 ++- src/services/computer-action.ts | 63 ++++-------------- src/types/computer-action.ts | 19 ++---- src/types/index.ts | 2 - tests/integration/computer-action.test.ts | 34 +--------- tests/unit/public-types-contract.test.ts | 81 +---------------------- 6 files changed, 28 insertions(+), 181 deletions(-) diff --git a/README.md b/README.md index 1633db4..10ea00d 100644 --- a/README.md +++ b/README.md @@ -518,19 +518,17 @@ response types remain under `@hyperbrowser/sdk/types`. `cursorPosition()` returns physical desktop coordinates in `result.data`. Click, drag and scroll accept an optional trailing `keys` array of xdotool key names held during the action. Existing positional arguments are unchanged. -Use `clickAtCursor()` or `scrollAtCursor()` to act at the current pointer. -`click()` and `scroll()` still require coordinates. `cursorPosition()` returns a separate -`CursorPositionActionResponse`; the existing action enum, request union and -response-data union remain unchanged. +Omit click coordinates, or pass `undefined` for scroll coordinates, to use the +current cursor. ```typescript const session = await client.sessions.create(); try { const result = await client.computerAction.cursorPosition(session); - if (result.data) console.log(result.data.x, result.data.y); + if (result.data && "x" in result.data) console.log(result.data.x, result.data.y); await client.computerAction.click(session, 100, 200, "left", 1, false, ["Control_L"]); await client.computerAction.drag(session, [{ x: 100, y: 200 }, { x: 200, y: 300 }], false, ["Shift_L"]); - await client.computerAction.scrollAtCursor(session, 0, 2, false, ["Control_L"]); + await client.computerAction.scroll(session, undefined, undefined, 0, 2, false, ["Control_L"]); } finally { await client.sessions.stop(session.id); } diff --git a/src/services/computer-action.ts b/src/services/computer-action.ts index 4121d5f..769f617 100644 --- a/src/services/computer-action.ts +++ b/src/services/computer-action.ts @@ -3,9 +3,6 @@ import { ComputerAction, ComputerActionParams, ComputerActionResponse, - CursorPositionActionParams, - CursorPositionActionResponse, - ScrollAtCursorActionParams, Coordinate, ComputerActionMouseButton, } from "../types/computer-action"; @@ -13,12 +10,10 @@ import { BaseService } from "./base"; import { HyperbrowserError } from "../client"; export class ComputerActionService extends BaseService { - private async executeRequest< - Response extends ComputerActionResponse | CursorPositionActionResponse = ComputerActionResponse, - >( + private async executeRequest( session: SessionDetail | string, - params: ComputerActionParams | CursorPositionActionParams | ScrollAtCursorActionParams - ): Promise { + params: ComputerActionParams + ): Promise { try { let sessionDetail: SessionDetail; @@ -35,7 +30,7 @@ export class ComputerActionService extends BaseService { ); } - return await this.request( + return await this.request( sessionDetail.computerActionEndpoint, { method: "POST", @@ -54,8 +49,8 @@ export class ComputerActionService extends BaseService { async click( session: SessionDetail | string, - x: number, - y: number, + x?: number, + y?: number, button: ComputerActionMouseButton = "left", numClicks: number = 1, returnScreenshot: boolean = false, @@ -72,22 +67,6 @@ export class ComputerActionService extends BaseService { }); } - async clickAtCursor( - session: SessionDetail | string, - button: ComputerActionMouseButton = "left", - numClicks: number = 1, - returnScreenshot: boolean = false, - keys?: string[] - ): Promise { - return this.executeRequest(session, { - action: ComputerAction.CLICK, - button, - numClicks, - returnScreenshot, - keys, - }); - } - async typeText( session: SessionDetail | string, text: string, @@ -109,9 +88,9 @@ export class ComputerActionService extends BaseService { async cursorPosition( session: SessionDetail | string, returnScreenshot: boolean = false - ): Promise { - return this.executeRequest(session, { - action: "cursor_position", + ): Promise { + return this.executeRequest(session, { + action: ComputerAction.CURSOR_POSITION, returnScreenshot, }); } @@ -158,10 +137,10 @@ export class ComputerActionService extends BaseService { async scroll( session: SessionDetail | string, - x: number, - y: number, - scrollX: number, - scrollY: number, + x?: number, + y?: number, + scrollX: number = 0, + scrollY: number = 0, returnScreenshot: boolean = false, keys?: string[] ): Promise { @@ -176,22 +155,6 @@ export class ComputerActionService extends BaseService { }); } - async scrollAtCursor( - session: SessionDetail | string, - scrollX: number = 0, - scrollY: number = 0, - returnScreenshot: boolean = false, - keys?: string[] - ): Promise { - return this.executeRequest(session, { - action: ComputerAction.SCROLL, - scrollX, - scrollY, - returnScreenshot, - keys, - }); - } - async holdKey( session: SessionDetail | string, key: string, diff --git a/src/types/computer-action.ts b/src/types/computer-action.ts index 134e2ab..6f71cb3 100644 --- a/src/types/computer-action.ts +++ b/src/types/computer-action.ts @@ -10,6 +10,7 @@ export enum ComputerAction { PRESS_KEYS = "press_keys", MOVE_MOUSE = "move_mouse", SCREENSHOT = "screenshot", + CURSOR_POSITION = "cursor_position", SCROLL = "scroll", TYPE_TEXT = "type_text", GET_CLIPBOARD_TEXT = "get_clipboard_text", @@ -78,7 +79,7 @@ export interface ScreenshotActionParams { /** Parameters for reading the desktop cursor position. */ export interface CursorPositionActionParams { - action: "cursor_position"; + action: ComputerAction.CURSOR_POSITION; returnScreenshot?: boolean; } @@ -87,17 +88,14 @@ export interface CursorPositionActionParams { */ export interface ScrollActionParams { action: ComputerAction.SCROLL; - x: number; - y: number; + x?: number; + y?: number; keys?: string[]; scrollX: number; scrollY: number; returnScreenshot?: boolean; } -/** Parameters for scrolling without moving the desktop cursor. */ -export type ScrollAtCursorActionParams = Omit; - /** * Parameters for type text action */ @@ -155,7 +153,7 @@ export interface ListWindowsActionParams { } /** - * Existing action parameters; cursor reads and cursor-relative scrolling use dedicated helpers. + * Union type for all computer action parameters */ export type ComputerActionParams = | ClickActionParams @@ -163,6 +161,7 @@ export type ComputerActionParams = | PressKeysActionParams | MoveMouseActionParams | ScreenshotActionParams + | CursorPositionActionParams | ScrollActionParams | TypeTextActionParams | HoldKeyActionParams @@ -200,6 +199,7 @@ export interface ComputerActionResponseDataCursorPosition { } export type ComputerActionResponseData = + | ComputerActionResponseDataCursorPosition | ComputerActionResponseDataClipboardText | ComputerActionResponseDataListWindows; @@ -213,8 +213,3 @@ export interface ComputerActionResponse { error?: string; message?: string; } - -/** Response from reading the desktop cursor position. */ -export interface CursorPositionActionResponse extends Omit { - data?: ComputerActionResponseDataCursorPosition; -} diff --git a/src/types/index.ts b/src/types/index.ts index dba85f7..4890b26 100644 --- a/src/types/index.ts +++ b/src/types/index.ts @@ -329,8 +329,6 @@ export { MoveMouseActionParams, ScreenshotActionParams, CursorPositionActionParams, - CursorPositionActionResponse, - ScrollAtCursorActionParams, ScrollActionParams, TypeTextActionParams, ComputerActionParams, diff --git a/tests/integration/computer-action.test.ts b/tests/integration/computer-action.test.ts index dde98d5..2bfefe2 100644 --- a/tests/integration/computer-action.test.ts +++ b/tests/integration/computer-action.test.ts @@ -149,8 +149,8 @@ describe("computer action primitives", () => { test("uses current cursor when click or scroll coordinates are omitted", async () => { const { actions, session, requests } = await fixture(); - await actions.clickAtCursor(session); - await actions.scrollAtCursor(session, 0, 2, false, ["Control_L"]); + await actions.click(session); + await actions.scroll(session, undefined, undefined, 0, 2, false, ["Control_L"]); expect(requests[0].body).toEqual({ action: "click", button: "left", @@ -174,23 +174,6 @@ describe("computer action primitives", () => { expect(requests).toHaveLength(0); }); - test.each([undefined, [], ["Shift_L"]])( - "scrolls at cursor with screenshot, session lookup and modifiers: %s", - async (keys) => { - const { actions, session, requests } = await fixture(); - const result = await actions.scrollAtCursor(session.id, -1, 2, true, keys); - expect(result.success).toBe(true); - expect(requests[0].method).toBe("GET"); - expect(requests[1].body).toEqual({ - action: "scroll", - scrollX: -1, - scrollY: 2, - returnScreenshot: true, - ...(keys === undefined ? {} : { keys }), - }); - } - ); - test("preserves unsuccessful action responses", async () => { const { actions, session } = await fixture(false); expect(await actions.cursorPosition(session)).toMatchObject({ @@ -198,17 +181,4 @@ describe("computer action primitives", () => { error: "action failed", }); }); - - test("clicks at cursor with buttons, counts, screenshot and modifiers", async () => { - const { actions, session, requests } = await fixture(); - await actions.clickAtCursor(session.id, "right", 2, true, ["Control_L"]); - expect(requests[0].method).toBe("GET"); - expect(requests[1].body).toEqual({ - action: "click", - button: "right", - numClicks: 2, - returnScreenshot: true, - keys: ["Control_L"], - }); - }); }); diff --git a/tests/unit/public-types-contract.test.ts b/tests/unit/public-types-contract.test.ts index e4e0667..c567875 100644 --- a/tests/unit/public-types-contract.test.ts +++ b/tests/unit/public-types-contract.test.ts @@ -3,12 +3,7 @@ import type { ClickActionParams, DragActionParams, ScrollActionParams, - ScrollAtCursorActionParams, - ComputerActionParams, - ComputerActionResponseData, - ComputerActionResponse, CursorPositionActionParams, - CursorPositionActionResponse, ComputerActionResponseDataCursorPosition, CompleteSandboxImageBuildParams, CreateSandboxImageBuildParams, @@ -26,41 +21,12 @@ import type { VolumeListResponse, } from "../../src/types"; import { ComputerAction } from "../../src/types"; -import { ComputerActionService } from "../../src/services/computer-action"; - -function existingResponseConsumer(data: ComputerActionResponseData): string { - if ("windows" in data) return data.activeWindowId; - return data.clipboardText ?? ""; -} - -function existingActionConsumer(params: ComputerActionParams): string { - switch (params.action) { - case ComputerAction.CLICK: - case ComputerAction.DRAG: - case ComputerAction.HOLD_KEY: - case ComputerAction.MOUSE_DOWN: - case ComputerAction.MOUSE_UP: - case ComputerAction.PRESS_KEYS: - case ComputerAction.MOVE_MOUSE: - case ComputerAction.SCREENSHOT: - case ComputerAction.SCROLL: - case ComputerAction.TYPE_TEXT: - case ComputerAction.GET_CLIPBOARD_TEXT: - case ComputerAction.PUT_SELECTION_TEXT: - case ComputerAction.LIST_WINDOWS: - return params.action; - default: { - const exhaustive: never = params; - return exhaustive; - } - } -} describe("public type compatibility", () => { test("exports cursor-position requests and typed coordinates with optional pointer modifiers", () => { - const request: CursorPositionActionParams = { action: "cursor_position" }; + const request: CursorPositionActionParams = { action: ComputerAction.CURSOR_POSITION }; const response: ComputerActionResponseDataCursorPosition = { x: 10, y: 20 }; - const scroll: ScrollAtCursorActionParams = { + const scroll: ScrollActionParams = { action: ComputerAction.SCROLL, scrollX: 0, scrollY: 1, @@ -73,49 +39,6 @@ describe("public type compatibility", () => { expectTypeOf().toEqualTypeOf(); expectTypeOf().toEqualTypeOf(); }); - - test("preserves existing coordinate, response and exhaustive action consumers", () => { - const actionNames: Record = { - click: "click", - drag: "drag", - hold_key: "hold_key", - mouse_down: "mouse_down", - mouse_up: "mouse_up", - press_keys: "press_keys", - move_mouse: "move_mouse", - screenshot: "screenshot", - scroll: "scroll", - type_text: "type_text", - get_clipboard_text: "get_clipboard_text", - put_selection_text: "put_selection_text", - list_windows: "list_windows", - }; - const scroll: ScrollActionParams = { - action: ComputerAction.SCROLL, - x: 10, - y: 20, - scrollX: 0, - scrollY: 1, - }; - const x: number = scroll.x; - const y: number = scroll.y; - expect(x + y).toBe(30); - expect(actionNames[scroll.action]).toBe("scroll"); - expect(existingActionConsumer(scroll)).toBe("scroll"); - expect(existingResponseConsumer({ clipboardText: "hello" })).toBe("hello"); - expectTypeOf().toEqualTypeOf(); - expectTypeOf().toEqualTypeOf(); - expectTypeOf[1]>().toEqualTypeOf(); - expectTypeOf[2]>().toEqualTypeOf(); - expectTypeOf[1]>().toEqualTypeOf(); - expectTypeOf[2]>().toEqualTypeOf(); - expectTypeOf>().toEqualTypeOf< - Promise - >(); - expectTypeOf>().toEqualTypeOf< - Promise - >(); - }); test("keeps newly available response data optional", () => { const imageResponse: SandboxImageListResponse = { images: [] }; const snapshotResponse: SandboxSnapshotListResponse = { snapshots: [] }; From a9d062ea9c9a5c6b9d93e75b1cdb9cd57a2ae826 Mon Sep 17 00:00:00 2001 From: nikhil Date: Thu, 8 Oct 2026 03:20:43 +0000 Subject: [PATCH 5/8] Restore cursor helpers and dedicated cursor response typing --- README.md | 10 ++- src/services/computer-action.ts | 58 +++++++++++--- src/types/computer-action.ts | 12 ++- src/types/index.ts | 2 + tests/integration/computer-action.test.ts | 56 +++++++++++++- tests/unit/public-types-contract.test.ts | 92 ++++++++++++++++++++++- 6 files changed, 207 insertions(+), 23 deletions(-) diff --git a/README.md b/README.md index 10ea00d..a4cf4af 100644 --- a/README.md +++ b/README.md @@ -518,17 +518,19 @@ response types remain under `@hyperbrowser/sdk/types`. `cursorPosition()` returns physical desktop coordinates in `result.data`. Click, drag and scroll accept an optional trailing `keys` array of xdotool key names held during the action. Existing positional arguments are unchanged. -Omit click coordinates, or pass `undefined` for scroll coordinates, to use the -current cursor. +Use `clickAtCursor()` or `scrollAtCursor()` to act at the current pointer. +`click()` and `scroll()` still require coordinates. `cursorPosition()` returns a dedicated +`CursorPositionActionResponse`, so its coordinates are accessible after checking +that `data` exists. The general action enum and request/response unions include cursor position. ```typescript const session = await client.sessions.create(); try { const result = await client.computerAction.cursorPosition(session); - if (result.data && "x" in result.data) console.log(result.data.x, result.data.y); + if (result.data) console.log(result.data.x, result.data.y); await client.computerAction.click(session, 100, 200, "left", 1, false, ["Control_L"]); await client.computerAction.drag(session, [{ x: 100, y: 200 }, { x: 200, y: 300 }], false, ["Shift_L"]); - await client.computerAction.scroll(session, undefined, undefined, 0, 2, false, ["Control_L"]); + await client.computerAction.scrollAtCursor(session, 0, 2, false, ["Control_L"]); } finally { await client.sessions.stop(session.id); } diff --git a/src/services/computer-action.ts b/src/services/computer-action.ts index 769f617..318b081 100644 --- a/src/services/computer-action.ts +++ b/src/services/computer-action.ts @@ -3,6 +3,8 @@ import { ComputerAction, ComputerActionParams, ComputerActionResponse, + CursorPositionActionResponse, + ScrollAtCursorActionParams, Coordinate, ComputerActionMouseButton, } from "../types/computer-action"; @@ -10,10 +12,10 @@ import { BaseService } from "./base"; import { HyperbrowserError } from "../client"; export class ComputerActionService extends BaseService { - private async executeRequest( + private async executeRequest( session: SessionDetail | string, - params: ComputerActionParams - ): Promise { + params: ComputerActionParams | ScrollAtCursorActionParams + ): Promise { try { let sessionDetail: SessionDetail; @@ -30,7 +32,7 @@ export class ComputerActionService extends BaseService { ); } - return await this.request( + return await this.request( sessionDetail.computerActionEndpoint, { method: "POST", @@ -49,8 +51,8 @@ export class ComputerActionService extends BaseService { async click( session: SessionDetail | string, - x?: number, - y?: number, + x: number, + y: number, button: ComputerActionMouseButton = "left", numClicks: number = 1, returnScreenshot: boolean = false, @@ -67,6 +69,22 @@ export class ComputerActionService extends BaseService { }); } + async clickAtCursor( + session: SessionDetail | string, + button: ComputerActionMouseButton = "left", + numClicks: number = 1, + returnScreenshot: boolean = false, + keys?: string[] + ): Promise { + return this.executeRequest(session, { + action: ComputerAction.CLICK, + button, + numClicks, + returnScreenshot, + keys, + }); + } + async typeText( session: SessionDetail | string, text: string, @@ -88,8 +106,8 @@ export class ComputerActionService extends BaseService { async cursorPosition( session: SessionDetail | string, returnScreenshot: boolean = false - ): Promise { - return this.executeRequest(session, { + ): Promise { + return this.executeRequest(session, { action: ComputerAction.CURSOR_POSITION, returnScreenshot, }); @@ -137,10 +155,10 @@ export class ComputerActionService extends BaseService { async scroll( session: SessionDetail | string, - x?: number, - y?: number, - scrollX: number = 0, - scrollY: number = 0, + x: number, + y: number, + scrollX: number, + scrollY: number, returnScreenshot: boolean = false, keys?: string[] ): Promise { @@ -155,6 +173,22 @@ export class ComputerActionService extends BaseService { }); } + async scrollAtCursor( + session: SessionDetail | string, + scrollX: number = 0, + scrollY: number = 0, + returnScreenshot: boolean = false, + keys?: string[] + ): Promise { + return this.executeRequest(session, { + action: ComputerAction.SCROLL, + scrollX, + scrollY, + returnScreenshot, + keys, + }); + } + async holdKey( session: SessionDetail | string, key: string, diff --git a/src/types/computer-action.ts b/src/types/computer-action.ts index 6f71cb3..51b31eb 100644 --- a/src/types/computer-action.ts +++ b/src/types/computer-action.ts @@ -88,14 +88,17 @@ export interface CursorPositionActionParams { */ export interface ScrollActionParams { action: ComputerAction.SCROLL; - x?: number; - y?: number; + x: number; + y: number; keys?: string[]; scrollX: number; scrollY: number; returnScreenshot?: boolean; } +/** Parameters for scrolling without moving the desktop cursor. */ +export type ScrollAtCursorActionParams = Omit; + /** * Parameters for type text action */ @@ -213,3 +216,8 @@ export interface ComputerActionResponse { error?: string; message?: string; } + +/** Response from reading the desktop cursor position. */ +export interface CursorPositionActionResponse extends Omit { + data?: ComputerActionResponseDataCursorPosition; +} diff --git a/src/types/index.ts b/src/types/index.ts index 4890b26..dba85f7 100644 --- a/src/types/index.ts +++ b/src/types/index.ts @@ -329,6 +329,8 @@ export { MoveMouseActionParams, ScreenshotActionParams, CursorPositionActionParams, + CursorPositionActionResponse, + ScrollAtCursorActionParams, ScrollActionParams, TypeTextActionParams, ComputerActionParams, diff --git a/tests/integration/computer-action.test.ts b/tests/integration/computer-action.test.ts index 2bfefe2..b7bdbe9 100644 --- a/tests/integration/computer-action.test.ts +++ b/tests/integration/computer-action.test.ts @@ -36,7 +36,7 @@ async function fixture(success = true) { response.end( JSON.stringify({ success, - ...(body.action === "cursor_position" ? { data: { x: 123, y: 456 } } : {}), + ...(success && body.action === "cursor_position" ? { data: { x: 123, y: 456 } } : {}), ...(!success ? { error: "action failed" } : {}), }) ); @@ -79,6 +79,11 @@ describe("computer action primitives", () => { const { actions, session, requests } = await fixture(); const result = await actions.cursorPosition(byId ? session.id : session, true); expect(result).toEqual({ success: true, data: { x: 123, y: 456 } }); + if (result.data) { + const x: number = result.data.x; + const y: number = result.data.y; + expect([x, y]).toEqual([123, 456]); + } expect(requests.at(-1)).toEqual({ method: "POST", url: "/action", @@ -149,8 +154,8 @@ describe("computer action primitives", () => { test("uses current cursor when click or scroll coordinates are omitted", async () => { const { actions, session, requests } = await fixture(); - await actions.click(session); - await actions.scroll(session, undefined, undefined, 0, 2, false, ["Control_L"]); + await actions.clickAtCursor(session); + await actions.scrollAtCursor(session, 0, 2, false, ["Control_L"]); expect(requests[0].body).toEqual({ action: "click", button: "left", @@ -174,11 +179,54 @@ describe("computer action primitives", () => { expect(requests).toHaveLength(0); }); + test("defaults cursor-relative scroll deltas without moving the pointer", async () => { + const { actions, session, requests } = await fixture(); + await actions.scrollAtCursor(session); + expect(requests[0].body).toEqual({ + action: "scroll", + scrollX: 0, + scrollY: 0, + returnScreenshot: false, + }); + }); + + test.each([undefined, [], ["Shift_L"]])( + "scrolls at cursor with screenshot, session lookup and modifiers: %s", + async (keys) => { + const { actions, session, requests } = await fixture(); + const result = await actions.scrollAtCursor(session.id, -1, 2, true, keys); + expect(result.success).toBe(true); + expect(requests[0].method).toBe("GET"); + expect(requests[1].body).toEqual({ + action: "scroll", + scrollX: -1, + scrollY: 2, + returnScreenshot: true, + ...(keys === undefined ? {} : { keys }), + }); + } + ); + test("preserves unsuccessful action responses", async () => { const { actions, session } = await fixture(false); - expect(await actions.cursorPosition(session)).toMatchObject({ + const result = await actions.cursorPosition(session); + expect(result).toMatchObject({ success: false, error: "action failed", }); + expect(result.data).toBeUndefined(); + }); + + test("clicks at cursor with buttons, counts, screenshot and modifiers", async () => { + const { actions, session, requests } = await fixture(); + await actions.clickAtCursor(session.id, "right", 2, true, ["Control_L"]); + expect(requests[0].method).toBe("GET"); + expect(requests[1].body).toEqual({ + action: "click", + button: "right", + numClicks: 2, + returnScreenshot: true, + keys: ["Control_L"], + }); }); }); diff --git a/tests/unit/public-types-contract.test.ts b/tests/unit/public-types-contract.test.ts index c567875..2c7f5b2 100644 --- a/tests/unit/public-types-contract.test.ts +++ b/tests/unit/public-types-contract.test.ts @@ -3,7 +3,12 @@ import type { ClickActionParams, DragActionParams, ScrollActionParams, + ScrollAtCursorActionParams, + ComputerActionParams, + ComputerActionResponseData, + ComputerActionResponse, CursorPositionActionParams, + CursorPositionActionResponse, ComputerActionResponseDataCursorPosition, CompleteSandboxImageBuildParams, CreateSandboxImageBuildParams, @@ -21,12 +26,43 @@ import type { VolumeListResponse, } from "../../src/types"; import { ComputerAction } from "../../src/types"; +import { ComputerActionService } from "../../src/services/computer-action"; + +function sharedResponseConsumer(data: ComputerActionResponseData): string { + if ("x" in data) return String(data.x); + if ("windows" in data) return data.activeWindowId; + return data.clipboardText ?? ""; +} + +function actionConsumer(params: ComputerActionParams): string { + switch (params.action) { + case ComputerAction.CLICK: + case ComputerAction.DRAG: + case ComputerAction.HOLD_KEY: + case ComputerAction.MOUSE_DOWN: + case ComputerAction.MOUSE_UP: + case ComputerAction.PRESS_KEYS: + case ComputerAction.MOVE_MOUSE: + case ComputerAction.SCREENSHOT: + case ComputerAction.CURSOR_POSITION: + case ComputerAction.SCROLL: + case ComputerAction.TYPE_TEXT: + case ComputerAction.GET_CLIPBOARD_TEXT: + case ComputerAction.PUT_SELECTION_TEXT: + case ComputerAction.LIST_WINDOWS: + return params.action; + default: { + const exhaustive: never = params; + return exhaustive; + } + } +} describe("public type compatibility", () => { test("exports cursor-position requests and typed coordinates with optional pointer modifiers", () => { const request: CursorPositionActionParams = { action: ComputerAction.CURSOR_POSITION }; const response: ComputerActionResponseDataCursorPosition = { x: 10, y: 20 }; - const scroll: ScrollActionParams = { + const scroll: ScrollAtCursorActionParams = { action: ComputerAction.SCROLL, scrollX: 0, scrollY: 1, @@ -39,6 +75,60 @@ describe("public type compatibility", () => { expectTypeOf().toEqualTypeOf(); expectTypeOf().toEqualTypeOf(); }); + + test("preserves coordinate contracts and exposes narrow cursor responses", () => { + const actionNames: Record = { + click: "click", + drag: "drag", + hold_key: "hold_key", + mouse_down: "mouse_down", + mouse_up: "mouse_up", + press_keys: "press_keys", + move_mouse: "move_mouse", + screenshot: "screenshot", + cursor_position: "cursor_position", + scroll: "scroll", + type_text: "type_text", + get_clipboard_text: "get_clipboard_text", + put_selection_text: "put_selection_text", + list_windows: "list_windows", + }; + const scroll: ScrollActionParams = { + action: ComputerAction.SCROLL, + x: 10, + y: 20, + scrollX: 0, + scrollY: 1, + }; + const x: number = scroll.x; + const y: number = scroll.y; + expect(x + y).toBe(30); + expect(actionNames[scroll.action]).toBe("scroll"); + expect(actionConsumer(scroll)).toBe("scroll"); + expect(actionConsumer({ action: ComputerAction.CURSOR_POSITION })).toBe("cursor_position"); + expect(sharedResponseConsumer({ clipboardText: "hello" })).toBe("hello"); + expect(sharedResponseConsumer({ x: 10, y: 20 })).toBe("10"); + const cursor: CursorPositionActionResponse = { success: true, data: { x: 10, y: 20 } }; + if (cursor.data) { + const x: number = cursor.data.x; + const y: number = cursor.data.y; + expect(x + y).toBe(30); + } + expectTypeOf().toEqualTypeOf(); + expectTypeOf().toEqualTypeOf(); + expectTypeOf[1]>().toEqualTypeOf(); + expectTypeOf[2]>().toEqualTypeOf(); + expectTypeOf[1]>().toEqualTypeOf(); + expectTypeOf[2]>().toEqualTypeOf(); + expectTypeOf[3]>().toEqualTypeOf(); + expectTypeOf[4]>().toEqualTypeOf(); + expectTypeOf>().toEqualTypeOf< + Promise + >(); + expectTypeOf>().toEqualTypeOf< + Promise + >(); + }); test("keeps newly available response data optional", () => { const imageResponse: SandboxImageListResponse = { images: [] }; const snapshotResponse: SandboxSnapshotListResponse = { snapshots: [] }; From 3a2e1b0bc7c912f9c393605bc5b7e0077180e4b1 Mon Sep 17 00:00:00 2001 From: nikhil Date: Thu, 8 Oct 2026 03:36:35 +0000 Subject: [PATCH 6/8] Include cursor-relative scrolling in computer action params --- src/services/computer-action.ts | 3 +-- src/types/computer-action.ts | 1 + tests/unit/public-types-contract.test.ts | 2 ++ 3 files changed, 4 insertions(+), 2 deletions(-) diff --git a/src/services/computer-action.ts b/src/services/computer-action.ts index 318b081..bd46070 100644 --- a/src/services/computer-action.ts +++ b/src/services/computer-action.ts @@ -4,7 +4,6 @@ import { ComputerActionParams, ComputerActionResponse, CursorPositionActionResponse, - ScrollAtCursorActionParams, Coordinate, ComputerActionMouseButton, } from "../types/computer-action"; @@ -14,7 +13,7 @@ import { HyperbrowserError } from "../client"; export class ComputerActionService extends BaseService { private async executeRequest( session: SessionDetail | string, - params: ComputerActionParams | ScrollAtCursorActionParams + params: ComputerActionParams ): Promise { try { let sessionDetail: SessionDetail; diff --git a/src/types/computer-action.ts b/src/types/computer-action.ts index 51b31eb..bb357d3 100644 --- a/src/types/computer-action.ts +++ b/src/types/computer-action.ts @@ -166,6 +166,7 @@ export type ComputerActionParams = | ScreenshotActionParams | CursorPositionActionParams | ScrollActionParams + | ScrollAtCursorActionParams | TypeTextActionParams | HoldKeyActionParams | MouseDownActionParams diff --git a/tests/unit/public-types-contract.test.ts b/tests/unit/public-types-contract.test.ts index 2c7f5b2..3a0ac3d 100644 --- a/tests/unit/public-types-contract.test.ts +++ b/tests/unit/public-types-contract.test.ts @@ -71,6 +71,8 @@ describe("public type compatibility", () => { expect(request.action).toBe("cursor_position"); expect(response).toEqual({ x: 10, y: 20 }); expect(scroll).not.toHaveProperty("x"); + expect(actionConsumer(scroll)).toBe("scroll"); + expectTypeOf().toExtend(); expectTypeOf().toEqualTypeOf(); expectTypeOf().toEqualTypeOf(); expectTypeOf().toEqualTypeOf(); From faefa1b268d16d2fd45f26906d68234017250e5a Mon Sep 17 00:00:00 2001 From: nikhil Date: Thu, 8 Oct 2026 04:04:48 +0000 Subject: [PATCH 7/8] Use shared response for all computer actions --- README.md | 9 +++++---- src/services/computer-action.ts | 11 +++++------ src/types/computer-action.ts | 5 ----- src/types/index.ts | 1 - tests/integration/computer-action.test.ts | 2 +- tests/unit/public-types-contract.test.ts | 9 ++++----- 6 files changed, 15 insertions(+), 22 deletions(-) diff --git a/README.md b/README.md index a4cf4af..34949d4 100644 --- a/README.md +++ b/README.md @@ -519,15 +519,16 @@ response types remain under `@hyperbrowser/sdk/types`. Click, drag and scroll accept an optional trailing `keys` array of xdotool key names held during the action. Existing positional arguments are unchanged. Use `clickAtCursor()` or `scrollAtCursor()` to act at the current pointer. -`click()` and `scroll()` still require coordinates. `cursorPosition()` returns a dedicated -`CursorPositionActionResponse`, so its coordinates are accessible after checking -that `data` exists. The general action enum and request/response unions include cursor position. +`click()` and `scroll()` still require coordinates. All actions, including +`cursorPosition()`, return `ComputerActionResponse`. Check that `data` contains +cursor coordinates before accessing them. The general action enum and +request/response unions include cursor position. ```typescript const session = await client.sessions.create(); try { const result = await client.computerAction.cursorPosition(session); - if (result.data) console.log(result.data.x, result.data.y); + if (result.data && "x" in result.data) console.log(result.data.x, result.data.y); await client.computerAction.click(session, 100, 200, "left", 1, false, ["Control_L"]); await client.computerAction.drag(session, [{ x: 100, y: 200 }, { x: 200, y: 300 }], false, ["Shift_L"]); await client.computerAction.scrollAtCursor(session, 0, 2, false, ["Control_L"]); diff --git a/src/services/computer-action.ts b/src/services/computer-action.ts index bd46070..823c29f 100644 --- a/src/services/computer-action.ts +++ b/src/services/computer-action.ts @@ -3,7 +3,6 @@ import { ComputerAction, ComputerActionParams, ComputerActionResponse, - CursorPositionActionResponse, Coordinate, ComputerActionMouseButton, } from "../types/computer-action"; @@ -11,10 +10,10 @@ import { BaseService } from "./base"; import { HyperbrowserError } from "../client"; export class ComputerActionService extends BaseService { - private async executeRequest( + private async executeRequest( session: SessionDetail | string, params: ComputerActionParams - ): Promise { + ): Promise { try { let sessionDetail: SessionDetail; @@ -31,7 +30,7 @@ export class ComputerActionService extends BaseService { ); } - return await this.request( + return await this.request( sessionDetail.computerActionEndpoint, { method: "POST", @@ -105,8 +104,8 @@ export class ComputerActionService extends BaseService { async cursorPosition( session: SessionDetail | string, returnScreenshot: boolean = false - ): Promise { - return this.executeRequest(session, { + ): Promise { + return this.executeRequest(session, { action: ComputerAction.CURSOR_POSITION, returnScreenshot, }); diff --git a/src/types/computer-action.ts b/src/types/computer-action.ts index bb357d3..7a4665a 100644 --- a/src/types/computer-action.ts +++ b/src/types/computer-action.ts @@ -217,8 +217,3 @@ export interface ComputerActionResponse { error?: string; message?: string; } - -/** Response from reading the desktop cursor position. */ -export interface CursorPositionActionResponse extends Omit { - data?: ComputerActionResponseDataCursorPosition; -} diff --git a/src/types/index.ts b/src/types/index.ts index dba85f7..3110e32 100644 --- a/src/types/index.ts +++ b/src/types/index.ts @@ -329,7 +329,6 @@ export { MoveMouseActionParams, ScreenshotActionParams, CursorPositionActionParams, - CursorPositionActionResponse, ScrollAtCursorActionParams, ScrollActionParams, TypeTextActionParams, diff --git a/tests/integration/computer-action.test.ts b/tests/integration/computer-action.test.ts index b7bdbe9..7b8d417 100644 --- a/tests/integration/computer-action.test.ts +++ b/tests/integration/computer-action.test.ts @@ -79,7 +79,7 @@ describe("computer action primitives", () => { const { actions, session, requests } = await fixture(); const result = await actions.cursorPosition(byId ? session.id : session, true); expect(result).toEqual({ success: true, data: { x: 123, y: 456 } }); - if (result.data) { + if (result.data && "x" in result.data) { const x: number = result.data.x; const y: number = result.data.y; expect([x, y]).toEqual([123, 456]); diff --git a/tests/unit/public-types-contract.test.ts b/tests/unit/public-types-contract.test.ts index 3a0ac3d..45ae9bf 100644 --- a/tests/unit/public-types-contract.test.ts +++ b/tests/unit/public-types-contract.test.ts @@ -8,7 +8,6 @@ import type { ComputerActionResponseData, ComputerActionResponse, CursorPositionActionParams, - CursorPositionActionResponse, ComputerActionResponseDataCursorPosition, CompleteSandboxImageBuildParams, CreateSandboxImageBuildParams, @@ -78,7 +77,7 @@ describe("public type compatibility", () => { expectTypeOf().toEqualTypeOf(); }); - test("preserves coordinate contracts and exposes narrow cursor responses", () => { + test("preserves coordinate contracts and uses shared cursor responses", () => { const actionNames: Record = { click: "click", drag: "drag", @@ -110,8 +109,8 @@ describe("public type compatibility", () => { expect(actionConsumer({ action: ComputerAction.CURSOR_POSITION })).toBe("cursor_position"); expect(sharedResponseConsumer({ clipboardText: "hello" })).toBe("hello"); expect(sharedResponseConsumer({ x: 10, y: 20 })).toBe("10"); - const cursor: CursorPositionActionResponse = { success: true, data: { x: 10, y: 20 } }; - if (cursor.data) { + const cursor: ComputerActionResponse = { success: true, data: { x: 10, y: 20 } }; + if (cursor.data && "x" in cursor.data) { const x: number = cursor.data.x; const y: number = cursor.data.y; expect(x + y).toBe(30); @@ -128,7 +127,7 @@ describe("public type compatibility", () => { Promise >(); expectTypeOf>().toEqualTypeOf< - Promise + Promise >(); }); test("keeps newly available response data optional", () => { From 53897b143a3d32c5c45528fa5b774c216cbdda4d Mon Sep 17 00:00:00 2001 From: nikhil Date: Thu, 8 Oct 2026 05:47:36 +0000 Subject: [PATCH 8/8] Remove standalone computer-action README section --- README.md | 24 ------------------------ 1 file changed, 24 deletions(-) diff --git a/README.md b/README.md index 34949d4..dba5978 100644 --- a/README.md +++ b/README.md @@ -513,30 +513,6 @@ resources. Invalid or conflicting launch sources fail before a network request. Public handle classes are available from `@hyperbrowser/sdk/sandbox`; request and response types remain under `@hyperbrowser/sdk/types`. -### Computer action cursor and modifier keys - -`cursorPosition()` returns physical desktop coordinates in `result.data`. -Click, drag and scroll accept an optional trailing `keys` array of xdotool key -names held during the action. Existing positional arguments are unchanged. -Use `clickAtCursor()` or `scrollAtCursor()` to act at the current pointer. -`click()` and `scroll()` still require coordinates. All actions, including -`cursorPosition()`, return `ComputerActionResponse`. Check that `data` contains -cursor coordinates before accessing them. The general action enum and -request/response unions include cursor position. - -```typescript -const session = await client.sessions.create(); -try { - const result = await client.computerAction.cursorPosition(session); - if (result.data && "x" in result.data) console.log(result.data.x, result.data.y); - await client.computerAction.click(session, 100, 200, "left", 1, false, ["Control_L"]); - await client.computerAction.drag(session, [{ x: 100, y: 200 }, { x: 200, y: 300 }], false, ["Shift_L"]); - await client.computerAction.scrollAtCursor(session, 0, 2, false, ["Control_L"]); -} finally { - await client.sessions.stop(session.id); -} -``` - ### Development checks The supported Node baseline is 20.20.2. Run `yarn build`, `yarn typecheck`,