Computer Controls

Control the computer’s mouse, keyboard, and screen

Use OS-level controls to move and click the mouse, type and press keys, scroll, drag, and capture screenshots from a running browser session.

Computer control actions are only available for browsers created after October 15, 2025. Older browser instances do not support these features.

Click the mouse

Simulate mouse clicks at specific coordinates. You can select the button, click type (down, up, click), number of clicks, and optional modifier keys to hold.

Typescript/Javascript

import Kernel from '@onkernel/sdk';

const kernel = new Kernel();
const kernelBrowser = await kernel.browsers.create();

// Basic left click at (100, 200)
await kernel.browsers.computer.clickMouse(kernelBrowser.session_id, {
  x: 100,
  y: 200,
});

// Double right-click while holding Shift
await kernel.browsers.computer.clickMouse(kernelBrowser.session_id, {
  x: 100,
  y: 200,
  button: 'right',
  click_type: 'click',
  num_clicks: 2,
  hold_keys: ['Shift'],
});

from kernel import Kernel

kernel = Kernel()
kernel_browser = kernel.browsers.create()

# Basic left click at (100, 200)
kernel.browsers.computer.click_mouse(
    id=kernel_browser.session_id,
    x=100,
    y=200,
)

# Double right-click while holding Shift
kernel.browsers.computer.click_mouse(
    id=kernel_browser.session_id,
    x=100,
    y=200,
    button="right",
    click_type="click",
    num_clicks=2,
    hold_keys=["Shift"],
)

# Click the mouse at coordinates (100, 200)
kernel browsers computer click-mouse <session id> --x 100 --y 200

# Double-click the right mouse button
kernel browsers computer click-mouse <session id> --x 100 --y 200 --num-clicks 2 --button right

Move the mouse

Move the cursor to specific screen coordinates. Optionally hold modifier keys during the move.

Typescript/Javascript

import Kernel from '@onkernel/sdk';

const kernel = new Kernel();
const kernelBrowser = await kernel.browsers.create();

await kernel.browsers.computer.moveMouse(kernelBrowser.session_id, {
  x: 500,
  y: 300,
  hold_keys: ['Alt'],
});

from kernel import Kernel

kernel = Kernel()
kernel_browser = kernel.browsers.create()

kernel.browsers.computer.move_mouse(
    id=kernel_browser.session_id,
    x=500,
    y=300,
    hold_keys=["Alt"],
)

# Move the mouse to coordinates (500, 300)
kernel browsers computer move-mouse <session id> --x 500 --y 300

Take screenshots

Capture a full-screen PNG or a specific region.

Typescript/Javascript

import fs from 'fs';
import { Buffer } from 'buffer';
import Kernel from '@onkernel/sdk';

const kernel = new Kernel();
const kernelBrowser = await kernel.browsers.create();

// Full screenshot
{
  const response = await kernel.browsers.computer.captureScreenshot(kernelBrowser.session_id);
  const blob = await response.blob();
  const buffer = Buffer.from(await blob.arrayBuffer());
  fs.writeFileSync('screenshot.png', buffer);
}

// Region screenshot
{
  const response = await kernel.browsers.computer.captureScreenshot(kernelBrowser.session_id, {
    region: { x: 0, y: 0, width: 800, height: 600 },
  });
  const blob = await response.blob();
  const buffer = Buffer.from(await blob.arrayBuffer());
  fs.writeFileSync('region.png', buffer);
}

from kernel import Kernel

kernel = Kernel()
kernel_browser = kernel.browsers.create()

# Full screenshot
with open('screenshot.png', 'wb') as f:
    image_data = kernel.browsers.computer.capture_screenshot(id=kernel_browser.session_id)
    f.write(image_data.read())

# Region screenshot
with open('region.png', 'wb') as f:
    image_data = kernel.browsers.computer.capture_screenshot(
        id=kernel_browser.session_id,
        region={"x": 0, "y": 0, "width": 800, "height": 600},
    )
    f.write(image_data.read())

# Take a full screenshot
kernel browsers computer screenshot <session id> --to screenshot.png

# Take a screenshot of a specific region
kernel browsers computer screenshot <session id> --to region.png --x 0 --y 0 --width 800 --height 600

Type text

Type literal text, optionally with a delay in milliseconds between keystrokes.

Typescript/Javascript

import Kernel from '@onkernel/sdk';

const kernel = new Kernel();
const kernelBrowser = await kernel.browsers.create();

await kernel.browsers.computer.typeText(kernelBrowser.session_id, {
  text: 'Hello, World!',
});

await kernel.browsers.computer.typeText(kernelBrowser.session_id, {
  text: 'Slow typing...',
  delay: 100,
});

from kernel import Kernel

kernel = Kernel()
kernel_browser = kernel.browsers.create()

kernel.browsers.computer.type_text(
    id=kernel_browser.session_id,
    text="Hello, World!",
)

kernel.browsers.computer.type_text(
    id=kernel_browser.session_id,
    text="Slow typing...",
    delay=100,
)

# Type text in the browser
kernel browsers computer type <session id> --text "Hello, World!"

# Type text with a 100ms delay between keystrokes
kernel browsers computer type <session id> --text "Slow typing..." --delay 100

Press keys

Press one or more key symbols (including combinations like “Ctrl+t” or “Ctrl+Shift+Tab”). Optionally hold modifiers and/or set a duration to hold keys down.

Typescript/Javascript

import Kernel from '@onkernel/sdk';

const kernel = new Kernel();
const kernelBrowser = await kernel.browsers.create();

// Tap a key combination
await kernel.browsers.computer.pressKey(kernelBrowser.session_id, {
  keys: ['Ctrl+t'],
});

// Hold keys for 250ms while also holding Alt
await kernel.browsers.computer.pressKey(kernelBrowser.session_id, {
  keys: ['Ctrl+Shift+Tab'],
  duration: 250,
  hold_keys: ['Alt'],
});

from kernel import Kernel

kernel = Kernel()
kernel_browser = kernel.browsers.create()

# Tap a key combination
kernel.browsers.computer.press_key(
    id=kernel_browser.session_id,
    keys=["Ctrl+t"],
)

# Hold keys for 250ms while also holding Alt
kernel.browsers.computer.press_key(
    id=kernel_browser.session_id,
    keys=["Ctrl+Shift+Tab"],
    duration=250,
    hold_keys=["Alt"],
)

# Press one or more keys (repeatable --key)
kernel browsers computer press-key <session id> --key Ctrl+t

# Hold for a duration and add optional modifiers
kernel browsers computer press-key <session id> --key Ctrl+Shift+Tab --duration 250 --hold-key Alt

Scroll

Scroll the mouse wheel at a specific position. Positive delta_y scrolls down; negative scrolls up. Positive delta_x scrolls right; negative scrolls left.

Typescript/Javascript

import Kernel from '@onkernel/sdk';

const kernel = new Kernel();
const kernelBrowser = await kernel.browsers.create();

await kernel.browsers.computer.scroll(kernelBrowser.session_id, {
  x: 300,
  y: 400,
  delta_x: 0,
  delta_y: 120,
});

from kernel import Kernel

kernel = Kernel()
kernel_browser = kernel.browsers.create()

kernel.browsers.computer.scroll(
    id=kernel_browser.session_id,
    x=300,
    y=400,
    delta_x=0,
    delta_y=120,
)

# Scroll at a position
kernel browsers computer scroll <session id> --x 300 --y 400 --delta-y 120

Drag the mouse

Drag by pressing a button, moving along a path of points, then releasing. You can control delay before starting, the granularity and speed of the drag via steps_per_segment and step_delay_ms, and optionally hold modifier keys.

Typescript/Javascript

import Kernel from '@onkernel/sdk';

const kernel = new Kernel();
const kernelBrowser = await kernel.browsers.create();

await kernel.browsers.computer.dragMouse(kernelBrowser.session_id, {
  path: [
    [100, 200],
    [150, 220],
    [200, 260],
  ],
  button: 'left',
  delay: 0,
  steps_per_segment: 10,
  step_delay_ms: 50,
  hold_keys: ['Shift'],
});

from kernel import Kernel

kernel = Kernel()
kernel_browser = kernel.browsers.create()

kernel.browsers.computer.drag_mouse(
    id=kernel_browser.session_id,
    path=[[100, 200], [150, 220], [200, 260]],
    button="left",
    delay=0,
    steps_per_segment=10,
    step_delay_ms=50,
    hold_keys=["Shift"],
)

# Drag the mouse along a path
kernel browsers computer drag-mouse <session id> \
  --point 100,200 \
  --point 150,220 \
  --point 200,260 \
  --button left \
  --delay 0