feat(simulation): render driven UI screenshots and videos (#35739)

bun.lock

@@ -873,6 +873,7 @@
       "name": "@opencode-ai/simulation",
       "version": "1.17.13",
       "dependencies": {
+        "@napi-rs/canvas": "1.0.2",
         "@opencode-ai/core": "workspace:*",
         "@opencode-ai/llm": "workspace:*",
         "@opentui/core": "catalog:",
@@ -1983,6 +1984,30 @@
 
     "@msgpackr-extract/msgpackr-extract-win32-x64": ["@msgpackr-extract/msgpackr-extract-win32-x64@3.0.4", "", { "os": "win32", "cpu": "x64" }, "sha512-CmCXPQrkbwExx3j946/PtHWHbYJiCRBRDl4BlkRQcJB/YOwQxJRTpoo7aTsortjgoJ1x7opzTSxn7C+ASSLVjQ=="],
 
+    "@napi-rs/canvas": ["@napi-rs/canvas@1.0.2", "", { "optionalDependencies": { "@napi-rs/canvas-android-arm64": "1.0.2", "@napi-rs/canvas-darwin-arm64": "1.0.2", "@napi-rs/canvas-darwin-x64": "1.0.2", "@napi-rs/canvas-linux-arm-gnueabihf": "1.0.2", "@napi-rs/canvas-linux-arm64-gnu": "1.0.2", "@napi-rs/canvas-linux-arm64-musl": "1.0.2", "@napi-rs/canvas-linux-riscv64-gnu": "1.0.2", "@napi-rs/canvas-linux-x64-gnu": "1.0.2", "@napi-rs/canvas-linux-x64-musl": "1.0.2", "@napi-rs/canvas-win32-arm64-msvc": "1.0.2", "@napi-rs/canvas-win32-x64-msvc": "1.0.2" } }, "sha512-EYEqlMYaCbpZDz+IgDH5xp9MTd3ui4dmGqbQYryhMLnSRxrhHKq5KQWHHKxFUcEP4Hp8/BWgvqXocX4j7iSbOQ=="],
+
+    "@napi-rs/canvas-android-arm64": ["@napi-rs/canvas-android-arm64@1.0.2", "", { "os": "android", "cpu": "arm64" }, "sha512-IMXKVQod0ol4vt3gmClUfXz4JAgHYESGPCUqmH3lQxBoL0K/2greJaQE1HVBVxWWFKfLc4OLZVdxg7kXVyXv+g=="],
+
+    "@napi-rs/canvas-darwin-arm64": ["@napi-rs/canvas-darwin-arm64@1.0.2", "", { "os": "darwin", "cpu": "arm64" }, "sha512-Sc8tPi6cF+5lqOzCCKFALJHhDiRwyMzTPYm3bbhdXsOunU0lQO5f05ucyOzN2r55I23Hg5bsjH63uSCvWp3EgQ=="],
+
+    "@napi-rs/canvas-darwin-x64": ["@napi-rs/canvas-darwin-x64@1.0.2", "", { "os": "darwin", "cpu": "x64" }, "sha512-niDXZ9LhKB1zLrUdYB64RHQFDGz9rr0eGx061qtJJU3U20EMMIx28ADF5fVYbhtOgkWQrBjFicfaye1yM0U62A=="],
+
+    "@napi-rs/canvas-linux-arm-gnueabihf": ["@napi-rs/canvas-linux-arm-gnueabihf@1.0.2", "", { "os": "linux", "cpu": "arm" }, "sha512-sgatQL9JxGRH/Amzcvu0P3t8Am3duou74CisfuJ41Dwt8cWy723z/9KZ8LlgmxfypEwEZxSTNFJtU8d281lmhQ=="],
+
+    "@napi-rs/canvas-linux-arm64-gnu": ["@napi-rs/canvas-linux-arm64-gnu@1.0.2", "", { "os": "linux", "cpu": "arm64" }, "sha512-dgKuX0peF3xwY6ZF5QxGS4wbfDqpoFAJYXiLSp+guZKARQUKMkRqZSDrXKj7nfrec3UCMzC0PFCPte0ES98AiA=="],
+
+    "@napi-rs/canvas-linux-arm64-musl": ["@napi-rs/canvas-linux-arm64-musl@1.0.2", "", { "os": "linux", "cpu": "arm64" }, "sha512-qwROoDIC9upfvDoRLuPn2aNg9CGW1x0Ygr4k2Or+8paA9d0qBLwk87U+g8KQpoOviKoPoiwl97kvBYuYD7qZoA=="],
+
+    "@napi-rs/canvas-linux-riscv64-gnu": ["@napi-rs/canvas-linux-riscv64-gnu@1.0.2", "", { "os": "linux", "cpu": "none" }, "sha512-fXRjnPihdnbO6qy1QQOgxAonb68A0TCEG7rj1x7v7rxNElsE8EVIKIEUTvyDtU+sthYSbX+8e7g3oZiLGnOmxw=="],
+
+    "@napi-rs/canvas-linux-x64-gnu": ["@napi-rs/canvas-linux-x64-gnu@1.0.2", "", { "os": "linux", "cpu": "x64" }, "sha512-nPR97DXhbWIAy7yazF3jc06kEPMqYMLmPzFOVNlwKPfIoSChnI+x7dc0hTLaihz3jxrjL6j4BbA7earxfx4X3g=="],
+
+    "@napi-rs/canvas-linux-x64-musl": ["@napi-rs/canvas-linux-x64-musl@1.0.2", "", { "os": "linux", "cpu": "x64" }, "sha512-l7zZY5+jL5qnBZtDz7CoBtY6p7EkHu422g/0zWwrOrzIwWyWxZFRfZZORY1UG7YApymPLx+UbOkN206xXn/c1Q=="],
+
+    "@napi-rs/canvas-win32-arm64-msvc": ["@napi-rs/canvas-win32-arm64-msvc@1.0.2", "", { "os": "win32", "cpu": "arm64" }, "sha512-yE0koHCFF4PIbMc2o2SEALhnipz7WBISh5glLvQiomtIoCcW0np3H4Lw93ceJAfJttTTeIIWFbwH84F7EVzjMQ=="],
+
+    "@napi-rs/canvas-win32-x64-msvc": ["@napi-rs/canvas-win32-x64-msvc@1.0.2", "", { "os": "win32", "cpu": "x64" }, "sha512-okU8/t2foV6C31n0GtvEMbfD5rOFc70+/6xUNME9Guld29sgSOIGUEDScAWFlcP3k5TYQRl9TNkwJEEjh15w8A=="],
+
     "@napi-rs/wasm-runtime": ["@napi-rs/wasm-runtime@1.1.4", "", { "dependencies": { "@tybys/wasm-util": "^0.10.1" }, "peerDependencies": { "@emnapi/core": "^1.7.1", "@emnapi/runtime": "^1.7.1" } }, "sha512-3NQNNgA1YSlJb/kMH1ildASP9HW7/7kYnRI2szWJaofaS1hWmbGI4H+d3+22aGzXXN9IJ+n+GiFVcGipJP18ow=="],
 
     "@noble/hashes": ["@noble/hashes@2.2.0", "", {}, "sha512-IYqDGiTXab6FniAgnSdZwgWbomxpy9FtYvLKs7wCUs2a8RkITG+DFGO1DM9cr+E3/RgADRpFjrKVaJ1z6sjtEg=="],

packages/simulation/package.json

@@ -16,6 +16,7 @@
     "typecheck": "tsgo --noEmit"
   },
   "dependencies": {
+    "@napi-rs/canvas": "1.0.2",
     "@opencode-ai/core": "workspace:*",
     "@opencode-ai/llm": "workspace:*",
     "@opentui/core": "catalog:",

packages/simulation/src/backend/control.ts

@@ -1,7 +1,6 @@
 import { Effect } from "effect"
 import { SimulationProtocol } from "../protocol"
 import { SimulationLLMExchange } from "./llm-exchange"
-import { SimulationNetwork } from "./network"
 
 /**
  * Backend-hosted simulation control WebSocket.
@@ -19,7 +18,6 @@ import { SimulationNetwork } from "./network"
  * - `llm.finish`  { id, reason? } finish an exchange
  * - `llm.disconnect` { id } abruptly terminate an exchange without a finish
  * - `llm.pending`                 list open exchanges
- * - `network.log`                 simulated network request log
  */
 
 type ControlSocket = Bun.ServerWebSocket<{ unsubscribe?: () => void }>
@@ -58,8 +56,6 @@ async function handle(socket: ControlSocket, request: SimulationProtocol.Backend
     }
     case "llm.pending":
       return { exchanges: SimulationLLMExchange.pending() }
-    case "network.log":
-      return { entries: SimulationNetwork.log() }
   }
 }
 

packages/simulation/src/frontend/actions.ts

@@ -1,8 +1,11 @@
-import type { CliRenderer, Renderable } from "@opentui/core"
+import { mkdtemp } from "node:fs/promises"
+import { tmpdir } from "node:os"
+import { join } from "node:path"
+import type { CapturedFrame, CliRenderer, Renderable } from "@opentui/core"
 import { createMockKeys, createMockMouse, type MockInput, type MockMouse } from "@opentui/core/testing"
 import type { SimulationProtocol } from "../protocol"
 import { SimulationRenderer } from "./renderer"
-import { SimulationTrace } from "./trace"
+import { SimulationPng } from "./png"
 
 export type Action = SimulationProtocol.Frontend.Action
 export type Element = SimulationProtocol.Frontend.Element
@@ -91,57 +94,95 @@ export function elements(renderer: CliRenderer): Element[] {
     .filter((element) => element.focusable || element.clickable || element.editor)
 }
 
-export function actions(renderer: CliRenderer, options: { text?: string } = {}): Action[] {
-  const items = elements(renderer)
-  return [
-    ...(renderer.currentFocusedEditor
-      ? ([{ type: "typeText", text: options.text ?? "hello" }, { type: "pressEnter" }] satisfies Action[])
-      : []),
-    ...items.filter((item) => item.focusable && !item.focused).map((item) => ({ type: "focus" as const, target: item.num })),
-    ...items
-      .filter((item) => item.clickable)
-      .map((item) => ({
-        type: "click" as const,
-        target: item.num,
-        x: Math.floor(item.x + item.width / 2),
-        y: Math.floor(item.y + item.height / 2),
-      })),
-    { type: "pressArrow", direction: "down" },
-    { type: "pressArrow", direction: "up" },
-  ]
-}
-
 export function state(harness: Harness) {
   return {
-    screen: harness.screen(),
     focused: {
       renderable: harness.renderer.currentFocusedRenderable?.num,
       editor: Boolean(harness.renderer.currentFocusedEditor),
     },
     elements: elements(harness.renderer),
-    actions: actions(harness.renderer),
   }
 }
 
+export async function screenshot(harness: Harness) {
+  await harness.renderOnce()
+  const image = SimulationPng.screenshot(harness.renderer)
+  const path = join(await mkdtemp(join(tmpdir(), "opencode-drive-")), "screenshot.png")
+  await Bun.write(path, image.data)
+  return path
+}
+
+export function frame(harness: Harness): CapturedFrame {
+  const buffer = harness.renderer.currentRenderBuffer
+  return {
+    cols: buffer.width,
+    rows: buffer.height,
+    cursor: [0, 0],
+    lines: buffer.getSpanLines().map((line) => ({
+      spans: line.spans.map((span) => ({
+        text: span.text,
+        fg: span.fg,
+        bg: span.bg,
+        attributes: span.attributes,
+        width: span.width,
+      })),
+    })),
+  }
+}
+
+export async function video(frames: CapturedFrame[]) {
+  const directory = await mkdtemp(join(tmpdir(), "opencode-drive-recording-"))
+  await Promise.all(
+    frames.map((frame, index) =>
+      Bun.write(
+        join(directory, `frame-${index.toString().padStart(6, "0")}.png`),
+        SimulationPng.screenshotFrame(frame).data,
+      ),
+    ),
+  )
+  const path = join(directory, "recording.mp4")
+  const process = Bun.spawn(
+    [
+      "ffmpeg",
+      "-loglevel",
+      "error",
+      "-framerate",
+      "10",
+      "-i",
+      join(directory, "frame-%06d.png"),
+      "-c:v",
+      "libx264",
+      "-pix_fmt",
+      "yuv420p",
+      "-movflags",
+      "+faststart",
+      "-y",
+      path,
+    ],
+    { stderr: "pipe" },
+  )
+  if ((await process.exited) !== 0) throw new Error(`ffmpeg failed: ${await new Response(process.stderr).text()}`)
+  return path
+}
+
 export async function execute(harness: Harness, action: Action) {
-  SimulationTrace.add("ui.action", { action })
   switch (action.type) {
-    case "typeText":
+    case "ui.type":
       await harness.mockInput.typeText(action.text)
       break
-    case "pressKey":
+    case "ui.press":
       harness.mockInput.pressKey(action.key, action.modifiers)
       break
-    case "pressEnter":
+    case "ui.enter":
       harness.mockInput.pressEnter()
       break
-    case "pressArrow":
+    case "ui.arrow":
       harness.mockInput.pressArrow(action.direction)
       break
-    case "focus":
+    case "ui.focus":
       all(harness.renderer.root).find((item) => item.num === action.target)?.focus()
       break
-    case "click":
+    case "ui.click":
       await harness.mockMouse.click(action.x, action.y)
       break
   }

packages/simulation/src/frontend/png.ts

@@ -0,0 +1,84 @@
+import { fileURLToPath } from "node:url"
+import { GlobalFonts, createCanvas } from "@napi-rs/canvas"
+import { TextAttributes, type CapturedFrame, type CliRenderer, type RGBA } from "@opentui/core"
+
+const CellWidth = 10
+const CellHeight = 20
+const FontSize = 16
+const FontFamily = "OpenCode Screenshot"
+
+GlobalFonts.registerFromPath(
+  fileURLToPath(new URL("../../../ui/src/assets/fonts/JetBrainsMonoNerdFontMono-Regular.woff2", import.meta.url)),
+  FontFamily,
+)
+
+export function screenshot(renderer: CliRenderer) {
+  return screenshotFrame({
+    cols: renderer.currentRenderBuffer.width,
+    rows: renderer.currentRenderBuffer.height,
+    cursor: [0, 0],
+    lines: renderer.currentRenderBuffer.getSpanLines(),
+  })
+}
+
+export function screenshotFrame(frame: CapturedFrame) {
+  const canvas = createCanvas(frame.cols * CellWidth, frame.rows * CellHeight)
+  const context = canvas.getContext("2d")
+  context.fillStyle = "#080808"
+  context.fillRect(0, 0, canvas.width, canvas.height)
+  context.textBaseline = "top"
+
+  frame.lines.forEach((line, row) => {
+    let column = 0
+    line.spans.forEach((span) => {
+      const attributes = span.attributes & 0xff
+      const inverse = Boolean(attributes & TextAttributes.INVERSE)
+      const hidden = Boolean(attributes & TextAttributes.HIDDEN)
+      const foreground = inverse ? span.bg : span.fg
+      const background = inverse ? span.fg : span.bg
+      const chars = [...span.text]
+      let remaining = span.width
+
+      chars.forEach((char, index) => {
+        const cells = Math.max(1, remaining - (chars.length - index - 1))
+        if (background.a) {
+          context.fillStyle = color(background)
+          context.fillRect(column * CellWidth, row * CellHeight, cells * CellWidth, CellHeight)
+        }
+        if (!hidden && char.codePointAt(0) !== 0x0a00) {
+          context.fillStyle = color(foreground, attributes & TextAttributes.DIM ? 0.55 : 1)
+          context.font = `${attributes & TextAttributes.ITALIC ? "italic " : ""}${attributes & TextAttributes.BOLD ? "bold " : ""}${FontSize}px "${FontFamily}"`
+          context.fillText(char, column * CellWidth, row * CellHeight + 1)
+          if (attributes & TextAttributes.UNDERLINE) {
+            context.fillRect(column * CellWidth, row * CellHeight + 17, cells * CellWidth, 1)
+          }
+          if (attributes & TextAttributes.STRIKETHROUGH) {
+            context.fillRect(column * CellWidth, row * CellHeight + 10, cells * CellWidth, 1)
+          }
+        }
+        column += cells
+        remaining -= cells
+      })
+      while (remaining-- > 0) {
+        if (background.a) {
+          context.fillStyle = color(background)
+          context.fillRect(column * CellWidth, row * CellHeight, CellWidth, CellHeight)
+        }
+        column++
+      }
+    })
+  })
+
+  return {
+    width: canvas.width,
+    height: canvas.height,
+    data: canvas.toBuffer("image/png"),
+  }
+}
+
+function color(value: RGBA, opacity = 1) {
+  const [red, green, blue, alpha] = value.toInts()
+  return `rgba(${red}, ${green}, ${blue}, ${(alpha / 255) * opacity})`
+}
+
+export * as SimulationPng from "./png"

packages/simulation/src/frontend/server.ts

@@ -1,6 +1,6 @@
+import type { CapturedFrame } from "@opentui/core"
 import { SimulationProtocol } from "../protocol"
 import { SimulationActions, type Harness } from "./actions"
-import { SimulationTrace } from "./trace"
 
 export interface Server {
   readonly url: string
@@ -11,47 +11,90 @@ function parseRequest(input: string | Buffer) {
   return SimulationProtocol.Frontend.decodeRequest(JSON.parse(typeof input === "string" ? input : input.toString()))
 }
 
-async function handle(harness: Harness, request: SimulationProtocol.Frontend.Request, headless: boolean) {
+interface Recording {
+  readonly frames: CapturedFrame[]
+  readonly timer: ReturnType<typeof setInterval>
+  pending: Promise<void>
+}
+
+async function handle(
+  harness: Harness,
+  request: SimulationProtocol.Frontend.Request,
+  recording: { current?: Recording },
+  headless: boolean,
+) {
   switch (request.method) {
+    case "ui.screenshot":
+      return SimulationActions.screenshot(harness)
     case "ui.state": {
+      return SimulationActions.state(harness)
+    }
+    case "ui.start-record": {
+      if (recording.current) throw new Error("UI recording is already active")
+      const frames = [SimulationActions.frame(harness)]
+      const current: Recording = {
+        frames,
+        timer: setInterval(() => {
+          current.pending = current.pending.then(async () => {
+            if (headless) await harness.renderOnce()
+            frames.push(SimulationActions.frame(harness))
+          })
+        }, 100),
+        pending: Promise.resolve(),
+      }
+      recording.current = current
+      return { recording: true }
+    }
+    case "ui.end-record": {
+      if (!recording.current) throw new Error("UI recording is not active")
+      const current = recording.current
+      clearInterval(current.timer)
+      await current.pending
       if (headless) await harness.renderOnce()
-      const result = SimulationActions.state(harness)
-      SimulationTrace.add("ui.state", { elements: result.elements.length, actions: result.actions.length })
-      return result
+      current.frames.push(SimulationActions.frame(harness))
+      recording.current = undefined
+      return SimulationActions.video(current.frames)
     }
-    case "ui.action":
-      return SimulationActions.execute(harness, request.params.action)
-    case "trace.list":
-      return { records: SimulationTrace.list() }
-    case "trace.clear":
-      SimulationTrace.clear()
-      return { cleared: true }
-    case "trace.export":
-      return SimulationTrace.exportTrace()
+    case "ui.type":
+      return SimulationActions.execute(harness, { type: "ui.type", text: request.params.text })
+    case "ui.enter":
+      return SimulationActions.execute(harness, { type: "ui.enter" })
+    case "ui.press":
+      return SimulationActions.execute(harness, {
+        type: "ui.press",
+        key: request.params.key,
+        modifiers: request.params.modifiers,
+      })
+    case "ui.arrow":
+      return SimulationActions.execute(harness, { type: "ui.arrow", direction: request.params.direction })
+    case "ui.focus":
+      return SimulationActions.execute(harness, { type: "ui.focus", target: request.params.target })
+    case "ui.click":
+      return SimulationActions.execute(harness, {
+        type: "ui.click",
+        target: request.params.target,
+        x: request.params.x,
+        y: request.params.y,
+      })
   }
 }
 
 export function start(harness: Harness, endpoint: string, headless: boolean): Server {
   const url = new URL(endpoint)
-  const server = Bun.serve<{ readonly drive: true }>({
+  const recording: { current?: Recording } = {}
+  const server = Bun.serve<{ readonly drive: true; readonly headless: boolean }>({
     hostname: url.hostname,
     port: Number(url.port),
     fetch(request, server) {
-      if (server.upgrade(request, { data: { drive: true } })) return undefined
+      if (server.upgrade(request, { data: { drive: true, headless } })) return undefined
       return new Response("opencode drive ui websocket", { status: 426 })
     },
     websocket: {
-      open() {
-        SimulationTrace.add("control.connect")
-      },
-      close() {
-        SimulationTrace.add("control.disconnect")
-      },
       async message(socket, message) {
         let request: SimulationProtocol.Frontend.Request | undefined
         try {
           request = parseRequest(message)
-          const result = await handle(harness, request, headless)
+          const result = await handle(harness, request, recording, headless)
           const next = SimulationProtocol.JsonRpc.success(request.id, result)
           if (next) socket.send(JSON.stringify(next))
         } catch (error) {
@@ -60,11 +103,10 @@ export function start(harness: Harness, endpoint: string, headless: boolean): Se
       },
     },
   })
-  SimulationTrace.add("control.start", { url: endpoint })
   return {
     url: endpoint,
     stop: () => {
-      SimulationTrace.add("control.stop", { url: endpoint })
+      if (recording.current) clearInterval(recording.current.timer)
       server.stop(true)
     },
   }

packages/simulation/src/frontend/trace.ts

@@ -1,37 +0,0 @@
-export type TraceRecord = {
-  readonly id: number
-  readonly time: string
-  readonly type: string
-  readonly data?: unknown
-}
-
-const records: TraceRecord[] = []
-let nextID = 0
-
-export function add(type: string, data?: unknown) {
-  const record = {
-    id: ++nextID,
-    time: new Date().toISOString(),
-    type,
-    ...(data === undefined ? {} : { data }),
-  } satisfies TraceRecord
-  records.push(record)
-  return record
-}
-
-export function list() {
-  return [...records]
-}
-
-export function clear() {
-  records.length = 0
-  nextID = 0
-}
-
-export function exportTrace() {
-  return {
-    records: list(),
-  }
-}
-
-export * as SimulationTrace from "./trace"

packages/simulation/src/protocol/index.ts

@@ -59,12 +59,12 @@ export namespace Frontend {
   export interface KeyModifiers extends Schema.Schema.Type<typeof KeyModifiers> {}
 
   export const Action = Schema.Union([
-    Schema.Struct({ type: Schema.Literal("typeText"), text: Schema.String }),
-    Schema.Struct({ type: Schema.Literal("pressKey"), key: Schema.String, modifiers: Schema.optional(KeyModifiers) }),
-    Schema.Struct({ type: Schema.Literal("pressEnter") }),
-    Schema.Struct({ type: Schema.Literal("pressArrow"), direction: Schema.Literals(["up", "down", "left", "right"]) }),
-    Schema.Struct({ type: Schema.Literal("focus"), target: Schema.Number }),
-    Schema.Struct({ type: Schema.Literal("click"), target: Schema.Number, x: Schema.Number, y: Schema.Number }),
+    Schema.Struct({ type: Schema.Literal("ui.type"), text: Schema.String }),
+    Schema.Struct({ type: Schema.Literal("ui.press"), key: Schema.String, modifiers: Schema.optional(KeyModifiers) }),
+    Schema.Struct({ type: Schema.Literal("ui.enter") }),
+    Schema.Struct({ type: Schema.Literal("ui.arrow"), direction: Schema.Literals(["up", "down", "left", "right"]) }),
+    Schema.Struct({ type: Schema.Literal("ui.focus"), target: Schema.Number }),
+    Schema.Struct({ type: Schema.Literal("ui.click"), target: Schema.Number, x: Schema.Number, y: Schema.Number }),
   ])
   export type Action = Schema.Schema.Type<typeof Action>
 
@@ -83,39 +83,58 @@ export namespace Frontend {
   export interface Element extends Schema.Schema.Type<typeof Element> {}
 
   export const State = Schema.Struct({
-    screen: Schema.String,
     focused: Schema.Struct({
       renderable: Schema.optional(Schema.Number),
       editor: Schema.Boolean,
     }),
     elements: Schema.Array(Element),
-    actions: Schema.Array(Action),
   })
   export interface State extends Schema.Schema.Type<typeof State> {}
 
-  export const ActionParams = Schema.Struct({ action: Action })
-  export interface ActionParams extends Schema.Schema.Type<typeof ActionParams> {}
+  export const Screenshot = Schema.String
+  export type Screenshot = Schema.Schema.Type<typeof Screenshot>
+
+  export const StartRecord = Schema.Struct({ recording: Schema.Literal(true) })
+  export interface StartRecord extends Schema.Schema.Type<typeof StartRecord> {}
+
+  export const EndRecord = Schema.String
+  export type EndRecord = Schema.Schema.Type<typeof EndRecord>
+
+  export const TypeParams = Schema.Struct({ text: Schema.String })
+  export interface TypeParams extends Schema.Schema.Type<typeof TypeParams> {}
+
+  export const PressParams = Schema.Struct({ key: Schema.String, modifiers: Schema.optional(KeyModifiers) })
+  export interface PressParams extends Schema.Schema.Type<typeof PressParams> {}
+
+  export const ArrowParams = Schema.Struct({ direction: Schema.Literals(["up", "down", "left", "right"]) })
+  export interface ArrowParams extends Schema.Schema.Type<typeof ArrowParams> {}
+
+  export const FocusParams = Schema.Struct({ target: Schema.Number })
+  export interface FocusParams extends Schema.Schema.Type<typeof FocusParams> {}
+
+  export const ClickParams = Schema.Struct({ target: Schema.Number, x: Schema.Number, y: Schema.Number })
+  export interface ClickParams extends Schema.Schema.Type<typeof ClickParams> {}
 
   export const Request = Schema.Union([
-    Schema.Struct({ ...JsonRpc.RequestFields, method: Schema.Literal("ui.action"), params: ActionParams }),
+    Schema.Struct({ ...JsonRpc.RequestFields, method: Schema.Literal("ui.type"), params: TypeParams }),
+    Schema.Struct({ ...JsonRpc.RequestFields, method: Schema.Literal("ui.press"), params: PressParams }),
+    Schema.Struct({ ...JsonRpc.RequestFields, method: Schema.Literal("ui.arrow"), params: ArrowParams }),
+    Schema.Struct({ ...JsonRpc.RequestFields, method: Schema.Literal("ui.focus"), params: FocusParams }),
+    Schema.Struct({ ...JsonRpc.RequestFields, method: Schema.Literal("ui.click"), params: ClickParams }),
     Schema.Struct({
       ...JsonRpc.RequestFields,
-      method: Schema.Literals(["ui.state", "trace.list", "trace.clear", "trace.export"]),
+      method: Schema.Literals([
+        "ui.enter",
+        "ui.screenshot",
+        "ui.state",
+        "ui.start-record",
+        "ui.end-record",
+      ]),
     }),
   ])
   export type Request = Schema.Schema.Type<typeof Request>
   export const decodeRequest = Schema.decodeUnknownSync(Request)
 
-  export const TraceRecord = Schema.Struct({
-    id: Schema.Number,
-    time: Schema.String,
-    type: Schema.String,
-    data: Schema.optional(Schema.Json),
-  })
-  export interface TraceRecord extends Schema.Schema.Type<typeof TraceRecord> {}
-
-  export const TraceList = Schema.Struct({ records: Schema.Array(TraceRecord) })
-  export interface TraceList extends Schema.Schema.Type<typeof TraceList> {}
 }
 
 export namespace Backend {
@@ -148,7 +167,7 @@ export namespace Backend {
     Schema.Struct({ ...JsonRpc.RequestFields, method: Schema.Literal("llm.disconnect"), params: DisconnectParams }),
     Schema.Struct({
       ...JsonRpc.RequestFields,
-      method: Schema.Literals(["llm.attach", "llm.pending", "network.log"]),
+      method: Schema.Literals(["llm.attach", "llm.pending"]),
     }),
   ])
   export type Request = Schema.Schema.Type<typeof Request>