Expose cursor position and modifier keys for computer actions

package.json

@@ -1,6 +1,6 @@
 {
   "name": "@hyperbrowser/sdk",
-  "version": "1.0.3",
+  "version": "1.1.0",
   "description": "Node SDK for Hyperbrowser API",
   "author": "",
   "repository": {

src/services/computer-action.ts

@@ -53,18 +53,36 @@ export class ComputerActionService extends BaseService {
     y: number,
     button: ComputerActionMouseButton = "left",
     numClicks: number = 1,
-    returnScreenshot: boolean = false
+    returnScreenshot: boolean = false,
+    keys?: string[]
   ): Promise<ComputerActionResponse> {
     return this.executeRequest(session, {
       action: ComputerAction.CLICK,
       x,
       y,
       button,
       numClicks,
+      keys,
       returnScreenshot,
     });
   }
 
+  async clickAtCursor(
+    session: SessionDetail | string,
+    button: ComputerActionMouseButton = "left",
+    numClicks: number = 1,
+    returnScreenshot: boolean = false,
+    keys?: string[]
+  ): Promise<ComputerActionResponse> {
+    return this.executeRequest(session, {
+      action: ComputerAction.CLICK,
+      button,
+      numClicks,
+      returnScreenshot,
+      keys,
+    });
+  }
+
   async typeText(
     session: SessionDetail | string,
     text: string,
@@ -83,6 +101,16 @@ export class ComputerActionService extends BaseService {
     });
   }
 
+  async cursorPosition(
+    session: SessionDetail | string,
+    returnScreenshot: boolean = false
+  ): Promise<ComputerActionResponse> {
+    return this.executeRequest(session, {
+      action: ComputerAction.CURSOR_POSITION,
+      returnScreenshot,
+    });
+  }
+
   async pressKeys(
     session: SessionDetail | string,
     keys: string[],
@@ -98,11 +126,13 @@ export class ComputerActionService extends BaseService {
   async drag(
     session: SessionDetail | string,
     path: Coordinate[],
-    returnScreenshot: boolean = false
+    returnScreenshot: boolean = false,
+    keys?: string[]
   ): Promise<ComputerActionResponse> {
     return this.executeRequest(session, {
       action: ComputerAction.DRAG,
       path,
+      keys,
       returnScreenshot,
     });
   }
@@ -127,18 +157,36 @@ export class ComputerActionService extends BaseService {
     y: number,
     scrollX: number,
     scrollY: number,
-    returnScreenshot: boolean = false
+    returnScreenshot: boolean = false,
+    keys?: string[]
   ): Promise<ComputerActionResponse> {
     return this.executeRequest(session, {
       action: ComputerAction.SCROLL,
       x,
       y,
       scrollX,
       scrollY,
+      keys,
       returnScreenshot,
     });
   }
 
+  async scrollAtCursor(
+    session: SessionDetail | string,
+    scrollX: number = 0,
+    scrollY: number = 0,
+    returnScreenshot: boolean = false,
+    keys?: string[]
+  ): Promise<ComputerActionResponse> {
+    return this.executeRequest(session, {
+      action: ComputerAction.SCROLL,
+      scrollX,
+      scrollY,
+      returnScreenshot,
+      keys,
+    });
+  }
+
   async holdKey(
     session: SessionDetail | string,
     key: string,

src/types/computer-action.ts

@@ -10,6 +10,7 @@ export enum ComputerAction {
   PRESS_KEYS = "press_keys",
   MOVE_MOUSE = "move_mouse",
   SCREENSHOT = "screenshot",
+  CURSOR_POSITION = "cursor_position",
   SCROLL = "scroll",
   TYPE_TEXT = "type_text",
   GET_CLIPBOARD_TEXT = "get_clipboard_text",
@@ -36,6 +37,7 @@ export interface ClickActionParams {
   y?: number;
   button?: ComputerActionMouseButton;
   numClicks?: number;
+  keys?: string[];
   returnScreenshot?: boolean;
 }
 
@@ -45,6 +47,7 @@ export interface ClickActionParams {
 export interface DragActionParams {
   action: ComputerAction.DRAG;
   path: Coordinate[];
+  keys?: string[];
   returnScreenshot?: boolean;
 }
 
@@ -74,18 +77,28 @@ export interface ScreenshotActionParams {
   action: ComputerAction.SCREENSHOT;
 }
 
+/** Parameters for reading the desktop cursor position. */
+export interface CursorPositionActionParams {
+  action: ComputerAction.CURSOR_POSITION;
+  returnScreenshot?: boolean;
+}
+
 /**
  * Parameters for scroll action
  */
 export interface ScrollActionParams {
   action: ComputerAction.SCROLL;
   x: number;
   y: number;
+  keys?: string[];
   scrollX: number;
   scrollY: number;
   returnScreenshot?: boolean;
 }
 
+/** Parameters for scrolling without moving the desktop cursor. */
+export type ScrollAtCursorActionParams = Omit<ScrollActionParams, "x" | "y">;
+
 /**
  * Parameters for type text action
  */
@@ -151,7 +164,9 @@ export type ComputerActionParams =
   | PressKeysActionParams
   | MoveMouseActionParams
   | ScreenshotActionParams
+  | CursorPositionActionParams
   | ScrollActionParams
+  | ScrollAtCursorActionParams
   | TypeTextActionParams
   | HoldKeyActionParams
   | MouseDownActionParams
@@ -181,7 +196,14 @@ export interface ComputerActionResponseDataListWindows {
   windows: ComputerActionWindow[];
 }
 
+/** Desktop cursor coordinates in physical screen pixels. */
+export interface ComputerActionResponseDataCursorPosition {
+  x: number;
+  y: number;
+}
+
 export type ComputerActionResponseData =
+  | ComputerActionResponseDataCursorPosition
   | ComputerActionResponseDataClipboardText
   | ComputerActionResponseDataListWindows;
 

src/types/index.ts

@@ -328,6 +328,8 @@ export {
   PressKeysActionParams,
   MoveMouseActionParams,
   ScreenshotActionParams,
+  CursorPositionActionParams,
+  ScrollAtCursorActionParams,
   ScrollActionParams,
   TypeTextActionParams,
   ComputerActionParams,
@@ -343,6 +345,7 @@ export {
   ListWindowsActionParams,
   ComputerActionWindow,
   ComputerActionResponseDataListWindows,
+  ComputerActionResponseDataCursorPosition,
 } from "./computer-action";
 export { FetchParams, FetchResponse, FetchResponseData, FetchStatus } from "./web/fetch";
 export {