diff --git a/package.json b/package.json index 18f3733..cbf9770 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "@hyperbrowser/sdk", - "version": "1.0.3", + "version": "1.1.0", "description": "Node SDK for Hyperbrowser API", "author": "", "repository": { diff --git a/src/services/computer-action.ts b/src/services/computer-action.ts index 7fc3674..823c29f 100644 --- a/src/services/computer-action.ts +++ b/src/services/computer-action.ts @@ -53,7 +53,8 @@ export class ComputerActionService extends BaseService { y: number, button: ComputerActionMouseButton = "left", numClicks: number = 1, - returnScreenshot: boolean = false + returnScreenshot: boolean = false, + keys?: string[] ): Promise { return this.executeRequest(session, { action: ComputerAction.CLICK, @@ -61,10 +62,27 @@ export class ComputerActionService extends BaseService { y, button, numClicks, + keys, returnScreenshot, }); } + async clickAtCursor( + session: SessionDetail | string, + button: ComputerActionMouseButton = "left", + numClicks: number = 1, + returnScreenshot: boolean = false, + keys?: string[] + ): Promise { + return this.executeRequest(session, { + action: ComputerAction.CLICK, + button, + numClicks, + returnScreenshot, + keys, + }); + } + async typeText( session: SessionDetail | string, text: string, @@ -83,6 +101,16 @@ export class ComputerActionService extends BaseService { }); } + async cursorPosition( + session: SessionDetail | string, + returnScreenshot: boolean = false + ): Promise { + return this.executeRequest(session, { + action: ComputerAction.CURSOR_POSITION, + returnScreenshot, + }); + } + async pressKeys( session: SessionDetail | string, keys: string[], @@ -98,11 +126,13 @@ export class ComputerActionService extends BaseService { async drag( session: SessionDetail | string, path: Coordinate[], - returnScreenshot: boolean = false + returnScreenshot: boolean = false, + keys?: string[] ): Promise { return this.executeRequest(session, { action: ComputerAction.DRAG, path, + keys, returnScreenshot, }); } @@ -127,7 +157,8 @@ export class ComputerActionService extends BaseService { y: number, scrollX: number, scrollY: number, - returnScreenshot: boolean = false + returnScreenshot: boolean = false, + keys?: string[] ): Promise { return this.executeRequest(session, { action: ComputerAction.SCROLL, @@ -135,10 +166,27 @@ export class ComputerActionService extends BaseService { y, scrollX, scrollY, + keys, returnScreenshot, }); } + async scrollAtCursor( + session: SessionDetail | string, + scrollX: number = 0, + scrollY: number = 0, + returnScreenshot: boolean = false, + keys?: string[] + ): Promise { + return this.executeRequest(session, { + action: ComputerAction.SCROLL, + scrollX, + scrollY, + returnScreenshot, + keys, + }); + } + async holdKey( session: SessionDetail | string, key: string, diff --git a/src/types/computer-action.ts b/src/types/computer-action.ts index a04b822..7a4665a 100644 --- a/src/types/computer-action.ts +++ b/src/types/computer-action.ts @@ -10,6 +10,7 @@ export enum ComputerAction { PRESS_KEYS = "press_keys", MOVE_MOUSE = "move_mouse", SCREENSHOT = "screenshot", + CURSOR_POSITION = "cursor_position", SCROLL = "scroll", TYPE_TEXT = "type_text", GET_CLIPBOARD_TEXT = "get_clipboard_text", @@ -36,6 +37,7 @@ export interface ClickActionParams { y?: number; button?: ComputerActionMouseButton; numClicks?: number; + keys?: string[]; returnScreenshot?: boolean; } @@ -45,6 +47,7 @@ export interface ClickActionParams { export interface DragActionParams { action: ComputerAction.DRAG; path: Coordinate[]; + keys?: string[]; returnScreenshot?: boolean; } @@ -74,6 +77,12 @@ export interface ScreenshotActionParams { action: ComputerAction.SCREENSHOT; } +/** Parameters for reading the desktop cursor position. */ +export interface CursorPositionActionParams { + action: ComputerAction.CURSOR_POSITION; + returnScreenshot?: boolean; +} + /** * Parameters for scroll action */ @@ -81,11 +90,15 @@ export interface ScrollActionParams { action: ComputerAction.SCROLL; x: number; y: number; + keys?: string[]; scrollX: number; scrollY: number; returnScreenshot?: boolean; } +/** Parameters for scrolling without moving the desktop cursor. */ +export type ScrollAtCursorActionParams = Omit; + /** * Parameters for type text action */ @@ -151,7 +164,9 @@ export type ComputerActionParams = | PressKeysActionParams | MoveMouseActionParams | ScreenshotActionParams + | CursorPositionActionParams | ScrollActionParams + | ScrollAtCursorActionParams | TypeTextActionParams | HoldKeyActionParams | MouseDownActionParams @@ -181,7 +196,14 @@ export interface ComputerActionResponseDataListWindows { windows: ComputerActionWindow[]; } +/** Desktop cursor coordinates in physical screen pixels. */ +export interface ComputerActionResponseDataCursorPosition { + x: number; + y: number; +} + export type ComputerActionResponseData = + | ComputerActionResponseDataCursorPosition | ComputerActionResponseDataClipboardText | ComputerActionResponseDataListWindows; diff --git a/src/types/index.ts b/src/types/index.ts index c121918..3110e32 100644 --- a/src/types/index.ts +++ b/src/types/index.ts @@ -328,6 +328,8 @@ export { PressKeysActionParams, MoveMouseActionParams, ScreenshotActionParams, + CursorPositionActionParams, + ScrollAtCursorActionParams, ScrollActionParams, TypeTextActionParams, ComputerActionParams, @@ -343,6 +345,7 @@ export { ListWindowsActionParams, ComputerActionWindow, ComputerActionResponseDataListWindows, + ComputerActionResponseDataCursorPosition, } from "./computer-action"; export { FetchParams, FetchResponse, FetchResponseData, FetchStatus } from "./web/fetch"; export { diff --git a/tests/integration/computer-action.test.ts b/tests/integration/computer-action.test.ts new file mode 100644 index 0000000..7b8d417 --- /dev/null +++ b/tests/integration/computer-action.test.ts @@ -0,0 +1,232 @@ +import http, { Server } from "node:http"; +import { AddressInfo } from "node:net"; +import { afterEach, describe, expect, test } from "vitest"; +import { HyperbrowserClient, HyperbrowserError } from "../../src/client"; +import type { SessionDetail } from "../../src/types"; + +const servers: Server[] = []; +afterEach(async () => { + await Promise.all( + servers.splice(0).map( + (server) => + new Promise((resolve, reject) => { + server.close((error) => (error ? reject(error) : resolve())); + }) + ) + ); +}); + +async function fixture(success = true) { + const requests: { method?: string; url?: string; apiKey?: string; body?: unknown }[] = []; + let session: SessionDetail; + const server = http.createServer(async (request, response) => { + const chunks: Buffer[] = []; + for await (const chunk of request) chunks.push(Buffer.from(chunk)); + const body = chunks.length ? JSON.parse(Buffer.concat(chunks).toString("utf8")) : undefined; + requests.push({ + method: request.method, + url: request.url, + apiKey: request.headers["x-api-key"]?.toString(), + body, + }); + response.setHeader("content-type", "application/json"); + if (request.method === "GET" && request.url === "/api/session/session-id") { + response.end(JSON.stringify(session)); + } else if (request.method === "POST" && request.url === "/action") { + response.end( + JSON.stringify({ + success, + ...(success && body.action === "cursor_position" ? { data: { x: 123, y: 456 } } : {}), + ...(!success ? { error: "action failed" } : {}), + }) + ); + } else { + response.writeHead(404); + response.end(JSON.stringify({ error: "not found" })); + } + }); + servers.push(server); + await new Promise((resolve, reject) => { + server.once("error", reject); + server.listen(0, "127.0.0.1", resolve); + }); + const baseUrl = `http://127.0.0.1:${(server.address() as AddressInfo).port}`; + session = { + id: "session-id", + teamId: "team-id", + status: "active", + createdAt: "2026-01-01T00:00:00Z", + updatedAt: "2026-01-01T00:00:00Z", + sessionUrl: baseUrl, + token: "test-token", + wsEndpoint: "wss://session.example", + computerActionEndpoint: `${baseUrl}/action`, + creditsUsed: null, + creditBreakdown: { + creditsUsed: null, + browserTimeCreditsUsed: null, + proxyDataCreditsUsed: null, + }, + }; + const client = new HyperbrowserClient({ apiKey: "test-key", baseUrl }); + return { actions: client.computerAction, requests, session }; +} + +describe("computer action primitives", () => { + test.each([false, true])( + "reads cursor coordinates with borrowed session, by ID=%s", + async (byId) => { + const { actions, session, requests } = await fixture(); + const result = await actions.cursorPosition(byId ? session.id : session, true); + expect(result).toEqual({ success: true, data: { x: 123, y: 456 } }); + if (result.data && "x" in result.data) { + const x: number = result.data.x; + const y: number = result.data.y; + expect([x, y]).toEqual([123, 456]); + } + expect(requests.at(-1)).toEqual({ + method: "POST", + url: "/action", + apiKey: "test-key", + body: { action: "cursor_position", returnScreenshot: true }, + }); + expect(requests).toHaveLength(byId ? 2 : 1); + } + ); + + test.each(["click", "drag", "scroll"] as const)( + "sends %s modifier chords and preserves existing arguments", + async (action) => { + const { actions, session, requests } = await fixture(); + const keys = ["Control_L", "Shift_L"]; + if (action === "click") await actions.click(session, 10, 20, "right", 2, true, keys); + if (action === "drag") + await actions.drag( + session, + [ + { x: 10, y: 20 }, + { x: 30, y: 40 }, + ], + true, + keys + ); + if (action === "scroll") await actions.scroll(session, 10, 20, 0, -2, true, keys); + const expected = { + click: { + action, + x: 10, + y: 20, + button: "right", + numClicks: 2, + returnScreenshot: true, + keys, + }, + drag: { + action, + path: [ + { x: 10, y: 20 }, + { x: 30, y: 40 }, + ], + returnScreenshot: true, + keys, + }, + scroll: { action, x: 10, y: 20, scrollX: 0, scrollY: -2, returnScreenshot: true, keys }, + }; + expect(requests.at(-1)?.body).toEqual(expected[action]); + expect(keys).toEqual(["Control_L", "Shift_L"]); + } + ); + + test.each([undefined, []])( + "keeps omitted and empty modifier lists distinct: %s", + async (keys) => { + const { actions, session, requests } = await fixture(); + await actions.click(session, 10, 20, "left", 1, false, keys); + await actions.drag(session, [{ x: 10, y: 20 }], false, keys); + await actions.scroll(session, 10, 20, 0, 1, false, keys); + for (const request of requests) { + const body = request.body as Record; + if (keys === undefined) expect(body).not.toHaveProperty("keys"); + else expect(body.keys).toEqual([]); + } + } + ); + + test("uses current cursor when click or scroll coordinates are omitted", async () => { + const { actions, session, requests } = await fixture(); + await actions.clickAtCursor(session); + await actions.scrollAtCursor(session, 0, 2, false, ["Control_L"]); + expect(requests[0].body).toEqual({ + action: "click", + button: "left", + numClicks: 1, + returnScreenshot: false, + }); + expect(requests[1].body).toEqual({ + action: "scroll", + scrollX: 0, + scrollY: 2, + returnScreenshot: false, + keys: ["Control_L"], + }); + }); + + test("reports missing computer endpoint without issuing a request", async () => { + const { actions, session, requests } = await fixture(); + await expect( + actions.cursorPosition({ ...session, computerActionEndpoint: undefined }) + ).rejects.toBeInstanceOf(HyperbrowserError); + expect(requests).toHaveLength(0); + }); + + test("defaults cursor-relative scroll deltas without moving the pointer", async () => { + const { actions, session, requests } = await fixture(); + await actions.scrollAtCursor(session); + expect(requests[0].body).toEqual({ + action: "scroll", + scrollX: 0, + scrollY: 0, + returnScreenshot: false, + }); + }); + + test.each([undefined, [], ["Shift_L"]])( + "scrolls at cursor with screenshot, session lookup and modifiers: %s", + async (keys) => { + const { actions, session, requests } = await fixture(); + const result = await actions.scrollAtCursor(session.id, -1, 2, true, keys); + expect(result.success).toBe(true); + expect(requests[0].method).toBe("GET"); + expect(requests[1].body).toEqual({ + action: "scroll", + scrollX: -1, + scrollY: 2, + returnScreenshot: true, + ...(keys === undefined ? {} : { keys }), + }); + } + ); + + test("preserves unsuccessful action responses", async () => { + const { actions, session } = await fixture(false); + const result = await actions.cursorPosition(session); + expect(result).toMatchObject({ + success: false, + error: "action failed", + }); + expect(result.data).toBeUndefined(); + }); + + test("clicks at cursor with buttons, counts, screenshot and modifiers", async () => { + const { actions, session, requests } = await fixture(); + await actions.clickAtCursor(session.id, "right", 2, true, ["Control_L"]); + expect(requests[0].method).toBe("GET"); + expect(requests[1].body).toEqual({ + action: "click", + button: "right", + numClicks: 2, + returnScreenshot: true, + keys: ["Control_L"], + }); + }); +}); diff --git a/tests/unit/public-types-contract.test.ts b/tests/unit/public-types-contract.test.ts index 3cfc02e..45ae9bf 100644 --- a/tests/unit/public-types-contract.test.ts +++ b/tests/unit/public-types-contract.test.ts @@ -1,5 +1,14 @@ import { describe, expect, expectTypeOf, test } from "vitest"; import type { + ClickActionParams, + DragActionParams, + ScrollActionParams, + ScrollAtCursorActionParams, + ComputerActionParams, + ComputerActionResponseData, + ComputerActionResponse, + CursorPositionActionParams, + ComputerActionResponseDataCursorPosition, CompleteSandboxImageBuildParams, CreateSandboxImageBuildParams, Sandbox, @@ -15,8 +24,112 @@ import type { SessionStatus, VolumeListResponse, } from "../../src/types"; +import { ComputerAction } from "../../src/types"; +import { ComputerActionService } from "../../src/services/computer-action"; + +function sharedResponseConsumer(data: ComputerActionResponseData): string { + if ("x" in data) return String(data.x); + if ("windows" in data) return data.activeWindowId; + return data.clipboardText ?? ""; +} + +function actionConsumer(params: ComputerActionParams): string { + switch (params.action) { + case ComputerAction.CLICK: + case ComputerAction.DRAG: + case ComputerAction.HOLD_KEY: + case ComputerAction.MOUSE_DOWN: + case ComputerAction.MOUSE_UP: + case ComputerAction.PRESS_KEYS: + case ComputerAction.MOVE_MOUSE: + case ComputerAction.SCREENSHOT: + case ComputerAction.CURSOR_POSITION: + case ComputerAction.SCROLL: + case ComputerAction.TYPE_TEXT: + case ComputerAction.GET_CLIPBOARD_TEXT: + case ComputerAction.PUT_SELECTION_TEXT: + case ComputerAction.LIST_WINDOWS: + return params.action; + default: { + const exhaustive: never = params; + return exhaustive; + } + } +} describe("public type compatibility", () => { + test("exports cursor-position requests and typed coordinates with optional pointer modifiers", () => { + const request: CursorPositionActionParams = { action: ComputerAction.CURSOR_POSITION }; + const response: ComputerActionResponseDataCursorPosition = { x: 10, y: 20 }; + const scroll: ScrollAtCursorActionParams = { + action: ComputerAction.SCROLL, + scrollX: 0, + scrollY: 1, + keys: ["Shift_L"], + }; + expect(request.action).toBe("cursor_position"); + expect(response).toEqual({ x: 10, y: 20 }); + expect(scroll).not.toHaveProperty("x"); + expect(actionConsumer(scroll)).toBe("scroll"); + expectTypeOf().toExtend(); + expectTypeOf().toEqualTypeOf(); + expectTypeOf().toEqualTypeOf(); + expectTypeOf().toEqualTypeOf(); + }); + + test("preserves coordinate contracts and uses shared cursor responses", () => { + const actionNames: Record = { + click: "click", + drag: "drag", + hold_key: "hold_key", + mouse_down: "mouse_down", + mouse_up: "mouse_up", + press_keys: "press_keys", + move_mouse: "move_mouse", + screenshot: "screenshot", + cursor_position: "cursor_position", + scroll: "scroll", + type_text: "type_text", + get_clipboard_text: "get_clipboard_text", + put_selection_text: "put_selection_text", + list_windows: "list_windows", + }; + const scroll: ScrollActionParams = { + action: ComputerAction.SCROLL, + x: 10, + y: 20, + scrollX: 0, + scrollY: 1, + }; + const x: number = scroll.x; + const y: number = scroll.y; + expect(x + y).toBe(30); + expect(actionNames[scroll.action]).toBe("scroll"); + expect(actionConsumer(scroll)).toBe("scroll"); + expect(actionConsumer({ action: ComputerAction.CURSOR_POSITION })).toBe("cursor_position"); + expect(sharedResponseConsumer({ clipboardText: "hello" })).toBe("hello"); + expect(sharedResponseConsumer({ x: 10, y: 20 })).toBe("10"); + const cursor: ComputerActionResponse = { success: true, data: { x: 10, y: 20 } }; + if (cursor.data && "x" in cursor.data) { + const x: number = cursor.data.x; + const y: number = cursor.data.y; + expect(x + y).toBe(30); + } + expectTypeOf().toEqualTypeOf(); + expectTypeOf().toEqualTypeOf(); + expectTypeOf[1]>().toEqualTypeOf(); + expectTypeOf[2]>().toEqualTypeOf(); + expectTypeOf[1]>().toEqualTypeOf(); + expectTypeOf[2]>().toEqualTypeOf(); + expectTypeOf[3]>().toEqualTypeOf(); + expectTypeOf[4]>().toEqualTypeOf(); + expectTypeOf>().toEqualTypeOf< + Promise + >(); + expectTypeOf>().toEqualTypeOf< + Promise + >(); + }); test("keeps newly available response data optional", () => { const imageResponse: SandboxImageListResponse = { images: [] }; const snapshotResponse: SandboxSnapshotListResponse = { snapshots: [] }; @@ -52,7 +165,9 @@ describe("public type compatibility", () => { expectTypeOf().toEqualTypeOf< SandboxImageBuildInputFormat | undefined >(); - expectTypeOf().toEqualTypeOf(); + expectTypeOf().toEqualTypeOf< + number | undefined + >(); expectTypeOf().toEqualTypeOf(); expectTypeOf().toEqualTypeOf(); expectTypeOf().toEqualTypeOf<