Skip to content

Expose cursor position and modifier keys for computer actions #136

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
NikhilShahi merged 8 commits into from
Oct 8, 2026
Merged

Expose cursor position and modifier keys for computer actions #136

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
8 commits
Select commit Hold shift + click to select a range
775f441
Expose desktop cursor position and pointer modifier keys
NikhilShahi Oct 7, 2026
0f4dd6c
Bump Python SDK minor version to 1.10.0
NikhilShahi Oct 7, 2026
9e92e34
Preserve existing computer-action typing contracts
NikhilShahi Oct 8, 2026
411057c
Revert "Preserve existing computer-action typing contracts"
NikhilShahi Oct 8, 2026
79ee36a
Restore cursor helpers and dedicated cursor response typing
NikhilShahi Oct 8, 2026
870d717
Include cursor-relative scrolling in computer action params
NikhilShahi Oct 8, 2026
39a4534
Use shared response for all computer actions
NikhilShahi Oct 8, 2026
7591ce8
Remove standalone computer-action README section
NikhilShahi Oct 8, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
45 changes: 39 additions & 6 deletions hyperbrowser/client/managers/async_manager/computer_action.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -12,6 +12,8 @@
PressKeysActionParams,
MoveMouseActionParams,
ScreenshotActionParams,
CursorPositionActionParams,
ScrollAtCursorActionParams,
ScrollActionParams,
TypeTextActionParams,
Coordinate,
Expand All @@ -38,6 +40,7 @@
ComputerAction.MOVE_MOUSE.value: MoveMouseActionParams,
ComputerAction.PRESS_KEYS.value: PressKeysActionParams,
ComputerAction.SCREENSHOT.value: ScreenshotActionParams,
ComputerAction.CURSOR_POSITION.value: CursorPositionActionParams,
ComputerAction.SCROLL.value: ScrollActionParams,
ComputerAction.TYPE_TEXT.value: TypeTextActionParams,
ComputerAction.GET_CLIPBOARD_TEXT.value: GetClipboardTextActionParams,
Expand All @@ -47,6 +50,9 @@


def _action_param_model(params):
if isinstance(params, ScrollAtCursorActionParams):
return ScrollAtCursorActionParams

for model in _ACTION_PARAM_MODELS.values():
if isinstance(params, model):
return model
Expand All @@ -56,6 +62,8 @@ def _action_param_model(params):
if isinstance(action, ComputerAction):
action = action.value
model = _ACTION_PARAM_MODELS.get(action)
if model is ScrollActionParams and "x" not in params and "y" not in params:
return ScrollAtCursorActionParams
if model is not None:
return model

Expand All @@ -71,18 +79,13 @@ async def _execute_request(
session: Union[SessionDetail, str],
params: Union[ComputerActionParamsDict, ComputerActionParams],
) -> ComputerActionResponse:
payload = dump_request(params, _action_param_model(params), name="params")
if isinstance(session, str):
session = await self._client.sessions.get(session)

if not session.computer_action_endpoint:
raise ValueError("Computer action endpoint not available for this session")

payload = dump_request(
params,
_action_param_model(params),
name="params",
)

response = await self._client.transport.post(
session.computer_action_endpoint,
data=payload,
Expand All @@ -97,12 +100,14 @@ async def click(
button: ComputerActionMouseButton = "left",
num_clicks: int = 1,
return_screenshot: bool = False,
keys: Optional[List[str]] = None,
) -> ComputerActionResponse:
params = ClickActionParams(
x=x,
y=y,
button=button,
num_clicks=num_clicks,
keys=keys,
return_screenshot=return_screenshot,
)
return await self._execute_request(session, params)
Expand All @@ -123,6 +128,14 @@ async def screenshot(
params = ScreenshotActionParams()
return await self._execute_request(session, params)

async def cursor_position(
self,
session: Union[SessionDetail, str],
return_screenshot: bool = False,
) -> ComputerActionResponse:
params = CursorPositionActionParams(return_screenshot=return_screenshot)
return await self._execute_request(session, params)

async def press_keys(
self,
session: Union[SessionDetail, str],
Expand Down Expand Up @@ -169,13 +182,15 @@ async def drag(
session: Union[SessionDetail, str],
path: List[Union[CoordinateDict, Coordinate]],
return_screenshot: bool = False,
keys: Optional[List[str]] = None,
) -> ComputerActionResponse:
params = DragActionParams(
path=[
coerce_request(coordinate, Coordinate, name="coordinate")
for coordinate in path
],
return_screenshot=return_screenshot,
keys=keys,
)
return await self._execute_request(session, params)

Expand All @@ -197,12 +212,30 @@ async def scroll(
scroll_x: int,
scroll_y: int,
return_screenshot: bool = False,
keys: Optional[List[str]] = None,
) -> ComputerActionResponse:
params = ScrollActionParams(
x=x,
y=y,
scroll_x=scroll_x,
scroll_y=scroll_y,
keys=keys,
return_screenshot=return_screenshot,
)
return await self._execute_request(session, params)

async def scroll_at_cursor(
self,
session: Union[SessionDetail, str],
scroll_x: int = 0,
scroll_y: int = 0,
return_screenshot: bool = False,
keys: Optional[List[str]] = None,
) -> ComputerActionResponse:
params = ScrollAtCursorActionParams(
scroll_x=scroll_x,
scroll_y=scroll_y,
keys=keys,
return_screenshot=return_screenshot,
)
return await self._execute_request(session, params)
Expand Down
45 changes: 39 additions & 6 deletions hyperbrowser/client/managers/sync_manager/computer_action.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -12,6 +12,8 @@
PressKeysActionParams,
MoveMouseActionParams,
ScreenshotActionParams,
CursorPositionActionParams,
ScrollAtCursorActionParams,
ScrollActionParams,
TypeTextActionParams,
Coordinate,
Expand All @@ -38,6 +40,7 @@
ComputerAction.MOVE_MOUSE.value: MoveMouseActionParams,
ComputerAction.PRESS_KEYS.value: PressKeysActionParams,
ComputerAction.SCREENSHOT.value: ScreenshotActionParams,
ComputerAction.CURSOR_POSITION.value: CursorPositionActionParams,
ComputerAction.SCROLL.value: ScrollActionParams,
ComputerAction.TYPE_TEXT.value: TypeTextActionParams,
ComputerAction.GET_CLIPBOARD_TEXT.value: GetClipboardTextActionParams,
Expand All @@ -47,6 +50,9 @@


def _action_param_model(params):
if isinstance(params, ScrollAtCursorActionParams):
return ScrollAtCursorActionParams

for model in _ACTION_PARAM_MODELS.values():
if isinstance(params, model):
return model
Expand All @@ -56,6 +62,8 @@ def _action_param_model(params):
if isinstance(action, ComputerAction):
action = action.value
model = _ACTION_PARAM_MODELS.get(action)
if model is ScrollActionParams and "x" not in params and "y" not in params:
return ScrollAtCursorActionParams
if model is not None:
return model

Expand All @@ -71,18 +79,13 @@ def _execute_request(
session: Union[SessionDetail, str],
params: Union[ComputerActionParamsDict, ComputerActionParams],
) -> ComputerActionResponse:
payload = dump_request(params, _action_param_model(params), name="params")
if isinstance(session, str):
session = self._client.sessions.get(session)

if not session.computer_action_endpoint:
raise ValueError("Computer action endpoint not available for this session")

payload = dump_request(
params,
_action_param_model(params),
name="params",
)

response = self._client.transport.post(
session.computer_action_endpoint,
data=payload,
Expand All @@ -97,12 +100,14 @@ def click(
button: ComputerActionMouseButton = "left",
num_clicks: int = 1,
return_screenshot: bool = False,
keys: Optional[List[str]] = None,
) -> ComputerActionResponse:
params = ClickActionParams(
x=x,
y=y,
button=button,
num_clicks=num_clicks,
keys=keys,
return_screenshot=return_screenshot,
)
return self._execute_request(session, params)
Expand All @@ -123,6 +128,14 @@ def screenshot(
params = ScreenshotActionParams()
return self._execute_request(session, params)

def cursor_position(
self,
session: Union[SessionDetail, str],
return_screenshot: bool = False,
) -> ComputerActionResponse:
params = CursorPositionActionParams(return_screenshot=return_screenshot)
return self._execute_request(session, params)

def press_keys(
self,
session: Union[SessionDetail, str],
Expand Down Expand Up @@ -169,13 +182,15 @@ def drag(
session: Union[SessionDetail, str],
path: List[Union[CoordinateDict, Coordinate]],
return_screenshot: bool = False,
keys: Optional[List[str]] = None,
) -> ComputerActionResponse:
params = DragActionParams(
path=[
coerce_request(coordinate, Coordinate, name="coordinate")
for coordinate in path
],
return_screenshot=return_screenshot,
keys=keys,
)
return self._execute_request(session, params)

Expand All @@ -197,12 +212,30 @@ def scroll(
scroll_x: int,
scroll_y: int,
return_screenshot: bool = False,
keys: Optional[List[str]] = None,
) -> ComputerActionResponse:
params = ScrollActionParams(
x=x,
y=y,
scroll_x=scroll_x,
scroll_y=scroll_y,
keys=keys,
return_screenshot=return_screenshot,
)
return self._execute_request(session, params)

def scroll_at_cursor(
self,
session: Union[SessionDetail, str],
scroll_x: int = 0,
scroll_y: int = 0,
return_screenshot: bool = False,
keys: Optional[List[str]] = None,
) -> ComputerActionResponse:
params = ScrollAtCursorActionParams(
scroll_x=scroll_x,
scroll_y=scroll_y,
keys=keys,
return_screenshot=return_screenshot,
)
return self._execute_request(session, params)
Expand Down
6 changes: 6 additions & 0 deletions hyperbrowser/models/__init__.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -274,6 +274,8 @@
MoveMouseActionParams,
PressKeysActionParams,
ScreenshotActionParams,
CursorPositionActionParams,
ScrollAtCursorActionParams,
ScrollActionParams,
TypeTextActionParams,
ComputerActionMouseButton,
Expand All @@ -283,6 +285,7 @@
ComputerActionWindow,
ComputerActionResponseDataClipboardText,
ComputerActionResponseDataListWindows,
ComputerActionResponseDataCursorPosition,
ComputerActionResponseData,
)
from .session import (
Expand Down Expand Up @@ -779,6 +782,8 @@
"MoveMouseActionParams",
"PressKeysActionParams",
"ScreenshotActionParams",
"CursorPositionActionParams",
"ScrollAtCursorActionParams",
"ScrollActionParams",
"TypeTextActionParams",
"ComputerActionMouseButton",
Expand All @@ -788,6 +793,7 @@
"ComputerActionWindow",
"ComputerActionResponseDataClipboardText",
"ComputerActionResponseDataListWindows",
"ComputerActionResponseDataCursorPosition",
"ComputerActionResponseData",
# web
"StartBatchFetchJobParams",
Expand Down
39 changes: 39 additions & 0 deletions hyperbrowser/models/computer_action.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -14,6 +14,7 @@ class ComputerAction(str, Enum):
MOVE_MOUSE = "move_mouse"
PRESS_KEYS = "press_keys"
SCREENSHOT = "screenshot"
CURSOR_POSITION = "cursor_position"
SCROLL = "scroll"
TYPE_TEXT = "type_text"
GET_CLIPBOARD_TEXT = "get_clipboard_text"
Expand Down Expand Up @@ -43,6 +44,7 @@ class ClickActionParams(BaseModel):
y: Optional[int] = Field(default=None)
button: ComputerActionMouseButton = Field(default="left")
num_clicks: int = Field(serialization_alias="numClicks", default=1)
keys: Optional[List[str]] = None
return_screenshot: bool = Field(
serialization_alias="returnScreenshot", default=False
)
Expand All @@ -55,6 +57,7 @@ class DragActionParams(BaseModel):

action: Literal[ComputerAction.DRAG] = ComputerAction.DRAG
path: List[Coordinate]
keys: Optional[List[str]] = None
return_screenshot: bool = Field(
serialization_alias="returnScreenshot", default=False
)
Expand Down Expand Up @@ -130,6 +133,17 @@ class ScreenshotActionParams(BaseModel):
action: Literal[ComputerAction.SCREENSHOT] = ComputerAction.SCREENSHOT


class CursorPositionActionParams(BaseModel):
"""Parameters for reading the desktop cursor position."""

model_config = ConfigDict(use_enum_values=True)

action: Literal[ComputerAction.CURSOR_POSITION] = ComputerAction.CURSOR_POSITION
return_screenshot: bool = Field(
serialization_alias="returnScreenshot", default=False
)


class ScrollActionParams(BaseModel):
"""Parameters for scroll action."""

Expand All @@ -138,8 +152,23 @@ class ScrollActionParams(BaseModel):
action: Literal[ComputerAction.SCROLL] = ComputerAction.SCROLL
x: int
y: int
keys: Optional[List[str]] = None
scroll_x: int = Field(serialization_alias="scrollX")
scroll_y: int = Field(serialization_alias="scrollY")
return_screenshot: bool = Field(
serialization_alias="returnScreenshot", default=False
)


class ScrollAtCursorActionParams(BaseModel):
"""Parameters for scrolling without moving the desktop cursor."""

model_config = ConfigDict(use_enum_values=True)

action: Literal[ComputerAction.SCROLL] = ComputerAction.SCROLL
scroll_x: int = Field(serialization_alias="scrollX")
scroll_y: int = Field(serialization_alias="scrollY")
keys: Optional[List[str]] = None
return_screenshot: bool = Field(
serialization_alias="returnScreenshot", default=False
)
Expand Down Expand Up @@ -201,7 +230,9 @@ class ListWindowsActionParams(BaseModel):
PressKeysActionParams,
MoveMouseActionParams,
ScreenshotActionParams,
CursorPositionActionParams,
ScrollActionParams,
ScrollAtCursorActionParams,
TypeTextActionParams,
HoldKeyActionParams,
MouseDownActionParams,
Expand Down Expand Up @@ -237,7 +268,15 @@ class ComputerActionResponseDataListWindows(BaseModel):
windows: List[ComputerActionWindow] = Field(default_factory=list)


class ComputerActionResponseDataCursorPosition(BaseModel):
"""Desktop cursor coordinates in physical screen pixels."""

x: int
y: int


ComputerActionResponseData = Union[
ComputerActionResponseDataCursorPosition,
ComputerActionResponseDataListWindows,
ComputerActionResponseDataClipboardText,
]
Expand Down
Loading