Jelajahi Sumber

fix(core): make V2 reads media-aware and binary-safe (#31038)

Kit Langton 2 bulan lalu
induk
melakukan
83dca45dd5

+ 1 - 0
bun.lock

@@ -276,6 +276,7 @@
         "@opentelemetry/exporter-trace-otlp-http": "0.214.0",
         "@opentelemetry/sdk-trace-base": "2.6.1",
         "@parcel/watcher": "2.5.1",
+        "@silvia-odwyer/photon-node": "0.3.4",
         "ai-gateway-provider": "3.1.2",
         "bun-pty": "0.4.8",
         "cross-spawn": "catalog:",

+ 1 - 0
packages/core/package.json

@@ -91,6 +91,7 @@
     "@opentelemetry/exporter-trace-otlp-http": "0.214.0",
     "@opentelemetry/sdk-trace-base": "2.6.1",
     "@parcel/watcher": "2.5.1",
+    "@silvia-odwyer/photon-node": "0.3.4",
     "@openrouter/ai-sdk-provider": "2.9.0",
     "ai-gateway-provider": "3.1.2",
     "bun-pty": "0.4.8",

+ 244 - 4
packages/core/src/filesystem.ts

@@ -23,9 +23,89 @@ export type ReadInput = typeof ReadInput.Type
 
 export const MAX_READ_LINES = 2_000
 export const MAX_READ_BYTES = 50 * 1024
+export const READ_SAMPLE_BYTES = 4 * 1024
+export const MAX_MEDIA_INGEST_BYTES = 20 * 1024 * 1024
 const MAX_LINE_LENGTH = 2_000
 const MAX_LINE_SUFFIX = `... (line truncated to ${MAX_LINE_LENGTH} chars)`
 
+export class ReadLimitError extends Error {
+  constructor(
+    readonly resource: string,
+    readonly maximumBytes: number,
+  ) {
+    super(`File exceeds ${maximumBytes} byte read limit: ${resource}`)
+    this.name = "ReadLimitError"
+  }
+}
+
+export class BinaryFileError extends Error {
+  constructor(readonly resource: string) {
+    super(`Cannot read binary file: ${resource}`)
+    this.name = "BinaryFileError"
+  }
+}
+
+const BINARY_EXTENSIONS = new Set([
+  ".zip",
+  ".tar",
+  ".gz",
+  ".exe",
+  ".dll",
+  ".so",
+  ".class",
+  ".jar",
+  ".war",
+  ".7z",
+  ".doc",
+  ".docx",
+  ".xls",
+  ".xlsx",
+  ".ppt",
+  ".pptx",
+  ".odt",
+  ".ods",
+  ".odp",
+  ".bin",
+  ".dat",
+  ".obj",
+  ".o",
+  ".a",
+  ".lib",
+  ".wasm",
+  ".pyc",
+  ".pyo",
+])
+
+export const isBinary = (resource: string, bytes: Uint8Array) => {
+  if (BINARY_EXTENSIONS.has(path.extname(resource).toLowerCase())) return true
+  if (bytes.length === 0) return false
+  let nonPrintable = 0
+  for (const byte of bytes) {
+    if (byte === 0) return true
+    if (byte < 9 || (byte > 13 && byte < 32)) nonPrintable++
+  }
+  return nonPrintable / bytes.length > 0.3
+}
+
+const startsWith = (bytes: Uint8Array, prefix: number[]) => prefix.every((value, index) => bytes[index] === value)
+const supportedImageMime = (bytes: Uint8Array) => {
+  if (startsWith(bytes, [0x89, 0x50, 0x4e, 0x47, 0x0d, 0x0a, 0x1a, 0x0a])) return "image/png"
+  if (startsWith(bytes, [0xff, 0xd8, 0xff])) return "image/jpeg"
+  if (startsWith(bytes, [0x47, 0x49, 0x46, 0x38])) return "image/gif"
+  if (startsWith(bytes, [0x52, 0x49, 0x46, 0x46]) && startsWith(bytes.subarray(8), [0x57, 0x45, 0x42, 0x50]))
+    return "image/webp"
+}
+
+export class MediaIngestLimitError extends Error {
+  constructor(
+    readonly resource: string,
+    readonly maximumBytes: number,
+  ) {
+    super(`Media exceeds ${maximumBytes} byte ingestion limit: ${resource}`)
+    this.name = "MediaIngestLimitError"
+  }
+}
+
 export class TextContent extends Schema.Class<TextContent>("FileSystem.TextContent")({
   type: Schema.Literal("text"),
   content: Schema.String,
@@ -158,7 +238,9 @@ export interface Interface {
   readonly resolveReadPath: (input: ReadInput) => Effect.Effect<ReadPathTarget>
   readonly resolveRead: (input: ReadInput) => Effect.Effect<ReadTarget>
   readonly readResolved: (target: ReadTarget, maximumBytes?: number) => Effect.Effect<Content>
+  readonly readSampleResolved: (target: ReadTarget, maximumBytes: number) => Effect.Effect<Uint8Array>
   readonly readTextPageResolved: (target: ReadTarget, page?: TextPageInput) => Effect.Effect<TextPage>
+  readonly readToolResolved: (target: ReadTarget, page?: TextPageInput) => Effect.Effect<Content | TextPage>
   readonly list: (input?: ListInput) => Effect.Effect<Entry[]>
   /** Select a contained canonical read root without asserting leaf policy. */
   readonly resolveRoot: (input?: ListInput) => Effect.Effect<RootTarget>
@@ -330,15 +412,29 @@ export const layer = Layer.effect(
           if (info.type !== "File") return yield* Effect.die(new Error("Path is not a file"))
           if (info.dev !== target.dev || Option.getOrUndefined(info.ino) !== target.ino)
             return yield* Effect.die(new Error("File changed after permission approval"))
-          if (info.size > maximumBytes)
-            return yield* Effect.die(new Error(`File exceeds ${maximumBytes} byte read limit`))
+          if (info.size > maximumBytes) return yield* Effect.die(new ReadLimitError(target.resource, maximumBytes))
           const bytes = yield* file.readAlloc(maximumBytes + 1).pipe(Effect.orDie)
           if (bytes._tag === "Some" && bytes.value.length > maximumBytes)
-            return yield* Effect.die(new Error(`File exceeds ${maximumBytes} byte read limit`))
+            return yield* Effect.die(new ReadLimitError(target.resource, maximumBytes))
           return yield* content(target, bytes._tag === "Some" ? bytes.value : new Uint8Array())
         }),
       )
     })
+    const readSampleResolved = Effect.fn("FileSystem.readSampleResolved")(function* (
+      target: ReadTarget,
+      maximumBytes: number,
+    ) {
+      return yield* Effect.scoped(
+        Effect.gen(function* () {
+          const file = yield* fs.open(target.real, { flag: "r" }).pipe(Effect.orDie)
+          const info = yield* file.stat.pipe(Effect.orDie)
+          if (info.type !== "File") return yield* Effect.die(new Error("Path is not a file"))
+          if (info.dev !== target.dev || Option.getOrUndefined(info.ino) !== target.ino)
+            return yield* Effect.die(new Error("File changed after permission approval"))
+          return Option.getOrElse(yield* file.readAlloc(maximumBytes).pipe(Effect.orDie), () => new Uint8Array())
+        }),
+      )
+    })
     const readTextPageResolved = Effect.fn("FileSystem.readTextPageResolved")(function* (
       target: ReadTarget,
       page: TextPageInput = {},
@@ -391,7 +487,7 @@ export const layer = Layer.effect(
           while (!done) {
             const chunk = yield* file.readAlloc(64 * 1024).pipe(Effect.orDie)
             if (Option.isNone(chunk)) break
-            if (chunk.value.includes(0)) return yield* Effect.die(new Error("Cannot page binary file"))
+            if (chunk.value.includes(0)) return yield* Effect.die(new BinaryFileError(target.resource))
             let text = decoder.decode(chunk.value, { stream: true })
             while (true) {
               const index = text.indexOf("\n")
@@ -433,6 +529,148 @@ export const layer = Layer.effect(
         }),
       )
     })
+    const readToolResolved = Effect.fn("FileSystem.readToolResolved")(function* (
+      target: ReadTarget,
+      page: TextPageInput = {},
+    ) {
+      return yield* Effect.scoped(
+        Effect.gen(function* () {
+          const file = yield* fs.open(target.real, { flag: "r" }).pipe(Effect.orDie)
+          const info = yield* file.stat.pipe(Effect.orDie)
+          if (info.type !== "File") return yield* Effect.die(new Error("Path is not a file"))
+          if (info.dev !== target.dev || Option.getOrUndefined(info.ino) !== target.ino)
+            return yield* Effect.die(new Error("File changed after permission approval"))
+
+          const first = Option.getOrElse(
+            yield* file.readAlloc(Math.min(64 * 1024, Number(info.size) || READ_SAMPLE_BYTES)).pipe(Effect.orDie),
+            () => new Uint8Array(),
+          )
+          const mime = supportedImageMime(first)
+          if (mime) {
+            if (info.size > MAX_MEDIA_INGEST_BYTES)
+              return yield* Effect.die(new MediaIngestLimitError(target.resource, MAX_MEDIA_INGEST_BYTES))
+            const chunks = [first]
+            let total = first.length
+            while (total <= MAX_MEDIA_INGEST_BYTES) {
+              const chunk = yield* file
+                .readAlloc(Math.min(64 * 1024, MAX_MEDIA_INGEST_BYTES + 1 - total))
+                .pipe(Effect.orDie)
+              if (Option.isNone(chunk)) break
+              chunks.push(chunk.value)
+              total += chunk.value.length
+            }
+            if (total > MAX_MEDIA_INGEST_BYTES)
+              return yield* Effect.die(new MediaIngestLimitError(target.resource, MAX_MEDIA_INGEST_BYTES))
+            return new BinaryContent({
+              type: "binary",
+              content: Buffer.concat(
+                chunks.map((chunk) => Buffer.from(chunk)),
+                total,
+              ).toString("base64"),
+              encoding: "base64",
+              mime,
+            })
+          }
+          if (startsWith(first, [0x25, 0x50, 0x44, 0x46]) || isBinary(target.resource, first))
+            return yield* Effect.die(new BinaryFileError(target.resource))
+
+          const paged = info.size > MAX_READ_BYTES || page.offset !== undefined || page.limit !== undefined
+          if (!paged) {
+            const decoder = new TextDecoder("utf-8", { fatal: true })
+            const text = [yield* Effect.sync(() => decoder.decode(first, { stream: true }))]
+            while (true) {
+              const chunk = yield* file.readAlloc(64 * 1024).pipe(Effect.orDie)
+              if (Option.isNone(chunk)) break
+              if (chunk.value.includes(0)) return yield* Effect.die(new BinaryFileError(target.resource))
+              text.push(yield* Effect.sync(() => decoder.decode(chunk.value, { stream: true })))
+            }
+            text.push(yield* Effect.sync(() => decoder.decode()))
+            return new TextContent({ type: "text", content: text.join(""), mime: FSUtil.mimeType(target.real) })
+          }
+
+          const offset = page.offset ?? 1
+          const limit = Math.min(page.limit ?? MAX_READ_LINES, MAX_READ_LINES)
+          const lines: string[] = []
+          const decoder = new TextDecoder("utf-8", { fatal: true })
+          let pending = ""
+          let discard = false
+          let line = 1
+          let bytes = 0
+          let found = false
+          let truncated = false
+          let next: number | undefined
+
+          const append = (input: string) => {
+            if (line < offset) {
+              line++
+              return
+            }
+            if (lines.length >= limit || bytes >= MAX_READ_BYTES) {
+              truncated = true
+              next ??= line
+              line++
+              return
+            }
+            found = true
+            const text = input.length > MAX_LINE_LENGTH ? input.slice(0, MAX_LINE_LENGTH) + MAX_LINE_SUFFIX : input
+            const size = Buffer.byteLength(text, "utf-8") + (lines.length > 0 ? 1 : 0)
+            if (bytes + size > MAX_READ_BYTES) {
+              truncated = true
+              next ??= line
+              line++
+              return
+            }
+            lines.push(text)
+            bytes += size
+            line++
+          }
+
+          const consume = (chunk: Uint8Array) => {
+            if (chunk.includes(0)) throw new BinaryFileError(target.resource)
+            let text = decoder.decode(chunk, { stream: true })
+            while (true) {
+              const index = text.indexOf("\n")
+              if (index === -1) {
+                if (!discard) {
+                  pending += text
+                  if (pending.length > MAX_LINE_LENGTH) {
+                    pending = pending.slice(0, MAX_LINE_LENGTH + 1)
+                    discard = true
+                  }
+                }
+                break
+              }
+              const current = pending + (discard ? "" : text.slice(0, index))
+              pending = ""
+              discard = false
+              text = text.slice(index + 1)
+              append(current.endsWith("\r") ? current.slice(0, -1) : current)
+            }
+          }
+
+          yield* Effect.sync(() => consume(first))
+          while (true) {
+            const chunk = yield* file.readAlloc(64 * 1024).pipe(Effect.orDie)
+            if (Option.isNone(chunk)) break
+            yield* Effect.sync(() => consume(chunk.value))
+          }
+          const tail = yield* Effect.sync(() => decoder.decode())
+          if (!discard) pending += tail
+          if (pending) append(pending.endsWith("\r") ? pending.slice(0, -1) : pending)
+          if (!found && offset !== 1) return yield* Effect.die(new Error(`Offset ${offset} is out of range`))
+
+          const text = lines.join("\n")
+          return new TextPage({
+            type: "text-page",
+            content: text,
+            mime: FSUtil.mimeType(target.real),
+            offset,
+            truncated,
+            ...(next === undefined ? {} : { next }),
+          })
+        }),
+      )
+    })
     const resolveList = Effect.fn("FileSystem.resolveList")(function* (input: ListInput = {}) {
       const directory = yield* resolve(input.path, input.reference)
       const info = yield* fs.stat(directory.real).pipe(Effect.orDie)
@@ -528,7 +766,9 @@ export const layer = Layer.effect(
       resolveReadPath,
       resolveRead,
       readResolved,
+      readSampleResolved,
       readTextPageResolved,
+      readToolResolved,
       list: Effect.fn("FileSystem.list")(function* (input) {
         return yield* listResolved(yield* resolveList(input))
       }),

+ 8 - 1
packages/core/src/session/compaction.ts

@@ -81,6 +81,13 @@ const estimate = (value: unknown) => Token.estimate(JSON.stringify(value))
 const truncate = (value: string) =>
   value.length <= TOOL_OUTPUT_MAX_CHARS ? value : `${value.slice(0, TOOL_OUTPUT_MAX_CHARS)}\n[truncated]`
 
+export const serializeToolContent = (content: SessionMessage.ToolStateCompleted["content"]) =>
+  content
+    .map((item) =>
+      item.type === "text" ? item.text : `[Attached ${item.mime}${item.name === undefined ? "" : `: ${item.name}`}]`,
+    )
+    .join("\n")
+
 const serialize = (message: SessionMessage.Message) => {
   if (message.type === "user") {
     const files = message.files?.map((file) => `[Attached ${file.mime}: ${file.name ?? file.uri}]`) ?? []
@@ -95,7 +102,7 @@ const serialize = (message: SessionMessage.Message) => {
         if (part.state.status === "completed")
           return [
             `[Assistant tool call]: ${part.name}(${input})`,
-            `[Tool result]: ${truncate(JSON.stringify(part.state.content))}`,
+            `[Tool result]: ${truncate(serializeToolContent(part.state.content))}`,
           ]
         if (part.state.status === "error")
           return [`[Assistant tool call]: ${part.name}(${input})`, `[Tool error]: ${part.state.error.message}`]

+ 1 - 1
packages/core/src/session/runner/publish-llm-event.ts

@@ -350,7 +350,7 @@ export const createLLMEventPublisher = (events: EventV2.Interface, input: Input)
           callID: event.id,
           ...result,
           outputPaths,
-          result: event.result,
+          ...(provider.executed ? { result: event.result } : {}),
           provider,
         })
         return

+ 169 - 14
packages/core/src/tool/read.ts

@@ -1,12 +1,46 @@
 export * as ReadTool from "./read"
 
 import { Tool, ToolFailure } from "@opencode-ai/llm"
+// @ts-ignore Bun's static file import is embedded by `bun build --compile`; some consumers also declare *.wasm.
+import photonWasm from "@silvia-odwyer/photon-node/photon_rs_bg.wasm" with { type: "file" }
 import { Cause, Effect, Layer, Schema } from "effect"
+import path from "node:path"
+import { fileURLToPath } from "node:url"
+import { Config } from "../config"
 import { FileSystem } from "../filesystem"
 import { PermissionV2 } from "../permission"
 import { ToolRegistry } from "./registry"
 
 export const name = "read"
+const SUPPORTED_IMAGE_MIMES = new Set(["image/jpeg", "image/png", "image/gif", "image/webp"])
+const MAX_IMAGE_BASE64_BYTES = 5 * 1024 * 1024
+const MAX_IMAGE_WIDTH = 2_000
+const MAX_IMAGE_HEIGHT = 2_000
+const JPEG_QUALITIES = [80, 85, 70, 55, 40]
+
+class ImageDecodeError extends Error {
+  constructor(readonly resource: string) {
+    super(`Image could not be decoded: ${resource}`)
+    this.name = "ImageDecodeError"
+  }
+}
+
+class ImageSizeError extends Error {
+  constructor(
+    readonly resource: string,
+    readonly width: number,
+    readonly height: number,
+    readonly bytes: number,
+    readonly maxWidth: number,
+    readonly maxHeight: number,
+    readonly maxBytes: number,
+  ) {
+    super(
+      `Image ${resource} is ${width}x${height} with base64 size ${bytes}, exceeding configured limits ${maxWidth}x${maxHeight}/${maxBytes} bytes`,
+    )
+    this.name = "ImageSizeError"
+  }
+}
 const LocationInput = Schema.Struct({
   ...FileSystem.ReadInput.fields,
   offset: FileSystem.ListPageInput.fields.offset.annotate({
@@ -21,15 +55,38 @@ const Success = Schema.Union([FileSystem.Content, FileSystem.TextPage, FileSyste
 
 const definition = Tool.make({
   description:
-    "Read a text or binary file, page through a large UTF-8 text file by line offset, or list a directory page relative to the current location. Absolute paths are accepted only for managed tool-output files.",
+    "Read a text file or supported image, page through a large UTF-8 text file by line offset, or list a directory page relative to the current location. Absolute paths are accepted only for managed tool-output files.",
   parameters: Input,
   success: Success,
+  toStructuredOutput: (output) =>
+    "type" in output && output.type === "binary" && SUPPORTED_IMAGE_MIMES.has(output.mime)
+      ? { type: "media", mime: output.mime }
+      : output,
+  toModelOutput: ({ parameters, output }) => {
+    if (!("type" in output) || output.type !== "binary" || !SUPPORTED_IMAGE_MIMES.has(output.mime)) return []
+    return [
+      { type: "text", text: "Image read successfully" },
+      {
+        type: "file",
+        source: { type: "data", data: output.content },
+        mime: output.mime,
+        name: parameters.path,
+      },
+    ]
+  },
 })
 
 export const layer = Layer.effectDiscard(
   Effect.gen(function* () {
     const registry = yield* ToolRegistry.Service
     const filesystem = yield* FileSystem.Service
+    const config = yield* Config.Service
+    const loadPhoton = yield* Effect.cached(
+      Effect.sync(() => {
+        ;(globalThis as typeof globalThis & { __OPENCODE_PHOTON_WASM_PATH?: string }).__OPENCODE_PHOTON_WASM_PATH =
+          path.isAbsolute(photonWasm) ? photonWasm : fileURLToPath(new URL(photonWasm, import.meta.url))
+      }).pipe(Effect.andThen(() => Effect.promise(() => import("@silvia-odwyer/photon-node")))),
+    )
 
     yield* registry.contribute((editor) =>
       editor.set(name, {
@@ -60,21 +117,118 @@ export const layer = Layer.effectDiscard(
             const final = yield* filesystem.resolveReadPath(input)
             if (final.type !== "file" || final.target.resource !== target.resource || final.target.real !== target.real)
               return yield* Effect.die(new Error("File changed after permission approval"))
-            if (
-              final.target.size > FileSystem.MAX_READ_BYTES ||
-              input.offset !== undefined ||
-              input.limit !== undefined
-            )
-              return yield* filesystem.readTextPageResolved(final.target, { offset: input.offset, limit: input.limit })
-            return yield* filesystem.readResolved(final.target, FileSystem.MAX_READ_BYTES)
+            const content = yield* filesystem.readToolResolved(final.target, {
+              offset: input.offset,
+              limit: input.limit,
+            })
+            if (content.type === "binary" && SUPPORTED_IMAGE_MIMES.has(content.mime)) {
+              const mime = content.mime
+              const base64 = content.content
+              const image = Object.assign(
+                {},
+                ...(yield* config.entries()).flatMap((entry) =>
+                  entry.type === "document" && entry.info.attachments?.image ? [entry.info.attachments.image] : [],
+                ),
+              )
+              const limits = {
+                autoResize: image.auto_resize ?? true,
+                maxWidth: image.max_width ?? MAX_IMAGE_WIDTH,
+                maxHeight: image.max_height ?? MAX_IMAGE_HEIGHT,
+                maxBase64Bytes: image.max_base64_bytes ?? MAX_IMAGE_BASE64_BYTES,
+              }
+              const photon = yield* loadPhoton
+              const decoded = yield* Effect.try({
+                try: () => photon.PhotonImage.new_from_byteslice(Buffer.from(base64, "base64")),
+                catch: () => new ImageDecodeError(final.target.resource),
+              })
+              try {
+                const width = decoded.get_width()
+                const height = decoded.get_height()
+                const bytes = Buffer.byteLength(base64, "utf-8")
+                if (width <= limits.maxWidth && height <= limits.maxHeight && bytes <= limits.maxBase64Bytes)
+                  return new FileSystem.BinaryContent({ type: "binary", content: base64, encoding: "base64", mime })
+                if (!limits.autoResize)
+                  return yield* Effect.die(
+                    new ImageSizeError(
+                      final.target.resource,
+                      width,
+                      height,
+                      bytes,
+                      limits.maxWidth,
+                      limits.maxHeight,
+                      limits.maxBase64Bytes,
+                    ),
+                  )
+                const scale = Math.min(1, limits.maxWidth / width, limits.maxHeight / height)
+                const sizes = Array.from({ length: 32 }).reduce<Array<{ width: number; height: number }>>((acc) => {
+                  const previous = acc.at(-1) ?? {
+                    width: Math.max(1, Math.round(width * scale)),
+                    height: Math.max(1, Math.round(height * scale)),
+                  }
+                  const next =
+                    acc.length === 0
+                      ? previous
+                      : {
+                          width: previous.width === 1 ? 1 : Math.max(1, Math.floor(previous.width * 0.75)),
+                          height: previous.height === 1 ? 1 : Math.max(1, Math.floor(previous.height * 0.75)),
+                        }
+                  return acc.some((item) => item.width === next.width && item.height === next.height)
+                    ? acc
+                    : [...acc, next]
+                }, [])
+                for (const size of sizes) {
+                  const resized = photon.resize(decoded, size.width, size.height, photon.SamplingFilter.Lanczos3)
+                  try {
+                    const candidate = [
+                      { content: Buffer.from(resized.get_bytes()).toString("base64"), mime: "image/png" },
+                      ...JPEG_QUALITIES.map((quality) => ({
+                        content: Buffer.from(resized.get_bytes_jpeg(quality)).toString("base64"),
+                        mime: "image/jpeg",
+                      })),
+                    ].find((item) => Buffer.byteLength(item.content, "utf-8") <= limits.maxBase64Bytes)
+                    if (candidate)
+                      return new FileSystem.BinaryContent({
+                        type: "binary",
+                        content: candidate.content,
+                        encoding: "base64",
+                        mime: candidate.mime,
+                      })
+                  } finally {
+                    resized.free()
+                  }
+                }
+                return yield* Effect.die(
+                  new ImageSizeError(
+                    final.target.resource,
+                    width,
+                    height,
+                    bytes,
+                    limits.maxWidth,
+                    limits.maxHeight,
+                    limits.maxBase64Bytes,
+                  ),
+                )
+              } finally {
+                decoded.free()
+              }
+            }
+            if (content.type === "binary")
+              return yield* Effect.die(new FileSystem.BinaryFileError(final.target.resource))
+            return content
           }).pipe(
             Effect.catchCause((cause) =>
-              Effect.fail(
-                new ToolFailure({
-                  message: `Unable to read ${input.path}`,
-                  error: Cause.squash(cause),
-                }),
-              ),
+              Effect.gen(function* () {
+                const error = Cause.squash(cause)
+                const message =
+                  error instanceof FileSystem.BinaryFileError ||
+                  error instanceof FileSystem.ReadLimitError ||
+                  error instanceof FileSystem.MediaIngestLimitError ||
+                  error instanceof ImageDecodeError ||
+                  error instanceof ImageSizeError
+                    ? error.message
+                    : `Unable to read ${input.path}`
+                return yield* new ToolFailure({ message, error })
+              }),
             ),
           )
         },
@@ -85,5 +239,6 @@ export const layer = Layer.effectDiscard(
 export const locationLayer = layer.pipe(
   Layer.provideMerge(ToolRegistry.defaultLayer),
   Layer.provideMerge(FileSystem.locationLayer),
+  Layer.provideMerge(Config.locationLayer),
   Layer.provideMerge(PermissionV2.locationLayer),
 )

+ 165 - 1
packages/core/test/location-filesystem.test.ts

@@ -2,7 +2,7 @@ import fs from "fs/promises"
 import path from "path"
 import { fileURLToPath } from "url"
 import { describe, expect, test } from "bun:test"
-import { Effect, Exit, Layer, Schema } from "effect"
+import { Effect, Exit, Fiber, Layer, Schema } from "effect"
 import { FSUtil } from "@opencode-ai/core/fs-util"
 import { Location } from "@opencode-ai/core/location"
 import { FileSystem } from "@opencode-ai/core/filesystem"
@@ -97,6 +97,24 @@ describe("FileSystem", () => {
     ),
   )
 
+  it.live("revalidates file identity before sampled classification", () =>
+    withTmp((directory) =>
+      Effect.gen(function* () {
+        const file = path.join(directory, "image.png")
+        yield* Effect.promise(() => fs.writeFile(file, Buffer.from([0x89, 0x50, 0x4e, 0x47])))
+        const service = yield* FileSystem.Service
+        const target = yield* service.resolveRead({ path: RelativePath.make("image.png") })
+
+        yield* Effect.promise(() => fs.rename(file, path.join(directory, "original.png")))
+        yield* Effect.promise(() => fs.writeFile(file, Buffer.from([0xff, 0xd8, 0xff])))
+
+        expect(
+          Exit.isFailure(yield* service.readSampleResolved(target, FileSystem.READ_SAMPLE_BYTES).pipe(Effect.exit)),
+        ).toBe(true)
+      }).pipe(provide(directory)),
+    ),
+  )
+
   it.live("pages large UTF-8 text files by line with continuation", () =>
     withTmp((directory) =>
       Effect.gen(function* () {
@@ -132,6 +150,152 @@ describe("FileSystem", () => {
     ),
   )
 
+  it.live("rejects paged text when a late NUL appears after the requested page", () =>
+    withTmp((directory) =>
+      Effect.gen(function* () {
+        const file = path.join(directory, "late-binary.txt")
+        yield* Effect.promise(() =>
+          fs.writeFile(
+            file,
+            Buffer.concat([Buffer.from("first\nsecond\n"), Buffer.alloc(80_000, 0x61), Buffer.from([0])]),
+          ),
+        )
+        const service = yield* FileSystem.Service
+        const target = yield* service.resolveRead({ path: RelativePath.make("late-binary.txt") })
+        expect(Exit.isFailure(yield* service.readToolResolved(target, { limit: 1 }).pipe(Effect.exit))).toBe(true)
+      }).pipe(provide(directory)),
+    ),
+  )
+
+  it.live("rejects paged text when invalid UTF-8 appears near EOF", () =>
+    withTmp((directory) =>
+      Effect.gen(function* () {
+        const file = path.join(directory, "invalid-utf8.txt")
+        yield* Effect.promise(() =>
+          fs.writeFile(
+            file,
+            Buffer.concat([Buffer.from("first\nsecond\n"), Buffer.alloc(80_000, 0x61), Buffer.from([0xc3, 0x28])]),
+          ),
+        )
+        const service = yield* FileSystem.Service
+        const target = yield* service.resolveRead({ path: RelativePath.make("invalid-utf8.txt") })
+        expect(Exit.isFailure(yield* service.readToolResolved(target, { limit: 1 }).pipe(Effect.exit))).toBe(true)
+      }).pipe(provide(directory)),
+    ),
+  )
+
+  it.live("rejects PDFs for direct, large, and paged reads", () =>
+    withTmp((directory) =>
+      Effect.gen(function* () {
+        const small = path.join(directory, "small.pdf")
+        const large = path.join(directory, "large.pdf")
+        yield* Effect.promise(() => fs.writeFile(small, "%PDF-1.7\nsmall"))
+        yield* Effect.promise(() =>
+          fs.writeFile(large, Buffer.concat([Buffer.from("%PDF-1.7\n"), Buffer.alloc(80_000)])),
+        )
+        const service = yield* FileSystem.Service
+        const smallTarget = yield* service.resolveRead({ path: RelativePath.make("small.pdf") })
+        const largeTarget = yield* service.resolveRead({ path: RelativePath.make("large.pdf") })
+        expect(Exit.isFailure(yield* service.readToolResolved(smallTarget).pipe(Effect.exit))).toBe(true)
+        expect(Exit.isFailure(yield* service.readToolResolved(largeTarget).pipe(Effect.exit))).toBe(true)
+        expect(Exit.isFailure(yield* service.readToolResolved(largeTarget, { limit: 1 }).pipe(Effect.exit))).toBe(true)
+      }).pipe(provide(directory)),
+    ),
+  )
+
+  it.live("rejects signature-bearing media beyond the ingestion cap before loading", () =>
+    withTmp((directory) =>
+      Effect.gen(function* () {
+        const file = path.join(directory, "huge.png")
+        yield* Effect.promise(async () => {
+          const handle = await fs.open(file, "w")
+          try {
+            await handle.write(Buffer.from([0x89, 0x50, 0x4e, 0x47, 0x0d, 0x0a, 0x1a, 0x0a]), 0, 8, 0)
+            await handle.truncate(FileSystem.MAX_MEDIA_INGEST_BYTES + 1)
+          } finally {
+            await handle.close()
+          }
+        })
+        const service = yield* FileSystem.Service
+        const target = yield* service.resolveRead({ path: RelativePath.make("huge.png") })
+        const exit = yield* service.readToolResolved(target).pipe(Effect.exit)
+        expect(Exit.isFailure(exit)).toBe(true)
+        if (Exit.isFailure(exit)) expect(String(exit.cause)).toContain("Media exceeds")
+      }).pipe(provide(directory)),
+    ),
+  )
+
+  it.live("never mixes a sampled image with replacement-path content", () =>
+    withTmp((directory) =>
+      Effect.gen(function* () {
+        const file = path.join(directory, "race.png")
+        const moved = path.join(directory, "original.png")
+        const original = Buffer.concat([
+          Buffer.from([0x89, 0x50, 0x4e, 0x47, 0x0d, 0x0a, 0x1a, 0x0a]),
+          Buffer.alloc(4 * 1024 * 1024, 0x11),
+        ])
+        const replacement = Buffer.concat([Buffer.from([0xff, 0xd8, 0xff]), Buffer.alloc(1024, 0x22)])
+        yield* Effect.promise(() => fs.writeFile(file, original))
+        const service = yield* FileSystem.Service
+        const target = yield* service.resolveRead({ path: RelativePath.make("race.png") })
+        const reading = yield* service.readToolResolved(target).pipe(Effect.forkChild)
+        yield* Effect.promise(async () => {
+          await fs.rename(file, moved)
+          await fs.writeFile(file, replacement)
+        })
+        const exit = yield* Fiber.join(reading).pipe(Effect.exit)
+        if (Exit.isSuccess(exit)) {
+          expect(exit.value).toMatchObject({ type: "binary", mime: "image/png" })
+          if (exit.value.type === "binary") expect(exit.value.content).toBe(original.toString("base64"))
+        }
+      }).pipe(provide(directory)),
+    ),
+  )
+
+  it.live("closes validated descriptors after successful and failed reads", () =>
+    withTmp((directory) => {
+      let active = 0
+      const filesystem = Layer.effect(
+        FSUtil.Service,
+        Effect.gen(function* () {
+          const service = yield* FSUtil.Service
+          return FSUtil.Service.of({
+            ...service,
+            open: (target, options) =>
+              Effect.acquireRelease(
+                service.open(target, options).pipe(Effect.tap(() => Effect.sync(() => active++))),
+                () => Effect.sync(() => active--),
+              ),
+          })
+        }),
+      ).pipe(Layer.provide(FSUtil.defaultLayer))
+      return Effect.gen(function* () {
+        const text = path.join(directory, "text.txt")
+        const binary = path.join(directory, "binary.pdf")
+        yield* Effect.promise(() => fs.writeFile(text, "hello"))
+        yield* Effect.promise(() => fs.writeFile(binary, "%PDF-1.7"))
+        const service = yield* FileSystem.Service
+        const before =
+          process.platform === "win32"
+            ? undefined
+            : yield* Effect.promise(() => fs.readdir("/dev/fd").then((entries) => entries.length))
+        for (let index = 0; index < 50; index++) {
+          yield* service.readToolResolved(yield* service.resolveRead({ path: RelativePath.make("text.txt") }))
+          yield* service
+            .readToolResolved(yield* service.resolveRead({ path: RelativePath.make("binary.pdf") }))
+            .pipe(Effect.exit)
+        }
+        expect(active).toBe(0)
+        if (before !== undefined) {
+          const after = yield* Effect.promise(() => fs.readdir("/dev/fd").then((entries) => entries.length))
+          expect(after).toBeLessThanOrEqual(before + 2)
+        }
+        yield* Effect.promise(() => fs.rename(text, text + ".moved"))
+        yield* Effect.promise(() => fs.rename(binary, binary + ".moved"))
+      }).pipe(provide(directory, inertReferences, filesystem))
+    }),
+  )
+
   it.live("lists direct children with relative paths and resolved URIs", () =>
     withTmp((directory) =>
       Effect.gen(function* () {

+ 18 - 0
packages/core/test/session-compaction.test.ts

@@ -0,0 +1,18 @@
+import { expect, test } from "bun:test"
+import { SessionCompaction } from "@opencode-ai/core/session/compaction"
+
+test("compaction describes tool media without embedding base64", () => {
+  const base64 = "iVBORw0KGgoAAAANSUhEUgAAAAEAAAAB"
+  const serialized = SessionCompaction.serializeToolContent([
+    { type: "text", text: "Image read successfully" },
+    {
+      type: "file",
+      source: { type: "data", data: base64 },
+      mime: "image/png",
+      name: "pixel.png",
+    },
+  ])
+
+  expect(serialized).toBe("Image read successfully\n[Attached image/png: pixel.png]")
+  expect(serialized).not.toContain(base64)
+})

+ 2 - 2
packages/core/test/session-runner-message.test.ts

@@ -109,7 +109,7 @@ Recent work
     ])
   })
 
-  test("expands assistant tool calls and settled outcomes into canonical tool messages", () => {
+  test("replays durable tool media into canonical tool messages without structured base64", () => {
     const messages = toLLMMessages(
       [
         new SessionMessage.Assistant({
@@ -140,7 +140,7 @@ Recent work
                 status: "running",
                 input: { path: "README.md" },
                 content: [],
-                structured: {},
+                structured: { type: "media", mime: "image/png" },
               }),
               time: { created },
             }),

+ 127 - 0
packages/core/test/session-runner-tool-events.test.ts

@@ -0,0 +1,127 @@
+import { expect, test } from "bun:test"
+import { Effect, Schema, Stream } from "effect"
+import { LLMEvent } from "@opencode-ai/llm"
+import { EventV2 } from "@opencode-ai/core/event"
+import { SessionEvent } from "@opencode-ai/core/session/event"
+import { SessionMessage } from "@opencode-ai/core/session/message"
+import { SessionV2 } from "@opencode-ai/core/session"
+import { ModelV2 } from "@opencode-ai/core/model"
+import { ProviderV2 } from "@opencode-ai/core/provider"
+import { createLLMEventPublisher } from "@opencode-ai/core/session/runner/publish-llm-event"
+
+const sessionID = SessionV2.ID.make("ses_tool_event_test")
+const base64 = "iVBORw0KGgoAAAANSUhEUgAAAAEAAAAB"
+
+const capture = () => {
+  const published: Array<{ readonly type: string; readonly data: unknown }> = []
+  const events = EventV2.Service.of({
+    publish: (definition, data) =>
+      Effect.sync(() => {
+        const event = { id: EventV2.ID.create(), type: definition.type, data } as EventV2.Payload<typeof definition>
+        published.push({
+          type: definition.sync ? EventV2.versionedType(definition.type, definition.sync.version) : definition.type,
+          data,
+        })
+        return event
+      }),
+    subscribe: () => Stream.empty,
+    all: () => Stream.empty,
+    aggregateEvents: () => Stream.empty,
+    sync: () => Effect.succeed(Effect.void),
+    listen: () => Effect.succeed(Effect.void),
+    beforeCommit: () => Effect.void,
+    project: () => Effect.void,
+    replay: () => Effect.void,
+    replayAll: () => Effect.succeed(undefined),
+    remove: () => Effect.void,
+    claim: () => Effect.void,
+  })
+  return {
+    published,
+    publisher: createLLMEventPublisher(events, {
+      sessionID,
+      agent: "build",
+      model: {
+        id: ModelV2.ID.make("model"),
+        providerID: ProviderV2.ID.make("provider"),
+      },
+    }),
+  }
+}
+
+const call = LLMEvent.toolCall({ id: "call-image", name: "read", input: { path: "pixel.png" } })
+const result = LLMEvent.toolResult({
+  id: "call-image",
+  name: "read",
+  result: {
+    type: "content",
+    value: [
+      { type: "text", text: "Image read successfully" },
+      { type: "media", mediaType: "image/png", data: base64, filename: "pixel.png" },
+    ],
+  },
+  output: {
+    structured: { type: "media", mime: "image/png" },
+    content: [
+      { type: "text", text: "Image read successfully" },
+      { type: "file", source: { type: "data", data: base64 }, mime: "image/png", name: "pixel.png" },
+    ],
+  },
+})
+
+test("local tool success serializes media base64 once and reconstructs from structured content", async () => {
+  const { published, publisher } = capture()
+  await Effect.runPromise(publisher.publish(call))
+  await Effect.runPromise(publisher.publish(result))
+
+  const success = published.find((event) => event.type === "session.next.tool.success.1")
+  expect(success).toBeDefined()
+  const serialized = JSON.stringify(success)
+  expect(serialized.split(base64)).toHaveLength(2)
+  expect(success?.data).not.toHaveProperty("result")
+
+  expect(success?.data).toMatchObject({
+    content: [
+      { type: "text", text: "Image read successfully" },
+      { type: "file", source: { type: "data", data: base64 }, mime: "image/png" },
+    ],
+  })
+})
+
+test("provider-executed success retains its compatibility result", async () => {
+  const { published, publisher } = capture()
+  await Effect.runPromise(publisher.publish(LLMEvent.toolCall({ ...call, providerExecuted: true })))
+  await Effect.runPromise(publisher.publish(LLMEvent.toolResult({ ...result, providerExecuted: true })))
+  const success = published.find((event) => event.type === "session.next.tool.success.1")
+  expect(success?.data).toHaveProperty("result")
+})
+
+test("binary failure emits no success event", async () => {
+  const { published, publisher } = capture()
+  await Effect.runPromise(publisher.publish(call))
+  await Effect.runPromise(
+    publisher.publish(
+      LLMEvent.toolResult({
+        id: call.id,
+        name: call.name,
+        result: { type: "error", value: "Cannot read binary file" },
+      }),
+    ),
+  )
+  expect(published.some((event) => event.type === "session.next.tool.success.1")).toBe(false)
+  expect(published.some((event) => event.type === "session.next.tool.failed.1")).toBe(true)
+})
+
+test("old success event data containing result still decodes", () => {
+  const decoded = Schema.decodeUnknownSync(SessionEvent.Tool.Success.data)({
+    sessionID,
+    timestamp: Date.now(),
+    assistantMessageID: SessionMessage.ID.create(),
+    callID: "call-old",
+    structured: { type: "media", mime: "image/png" },
+    content: [{ type: "file", source: { type: "data", data: base64 }, mime: "image/png" }],
+    result: { type: "content", value: [{ type: "media", mediaType: "image/png", data: base64 }] },
+    provider: { executed: false },
+  })
+  expect(decoded.result).toMatchObject({ type: "content" })
+})

+ 2 - 0
packages/core/test/tool-glob.test.ts

@@ -39,7 +39,9 @@ const filesystem = Layer.succeed(
     resolveReadPath: () => Effect.die("unused"),
     resolveRead: () => Effect.die("unused"),
     readResolved: () => Effect.die("unused"),
+    readSampleResolved: () => Effect.die("unused"),
     readTextPageResolved: () => Effect.die("unused"),
+    readToolResolved: () => Effect.die("unused"),
     list: () => Effect.die("unused"),
     resolveRoot: (input = {}) =>
       Effect.sync(() => {

+ 2 - 0
packages/core/test/tool-grep.test.ts

@@ -34,7 +34,9 @@ const filesystem = Layer.succeed(
     resolveReadPath: () => Effect.die("unused"),
     resolveRead: () => Effect.die("unused"),
     readResolved: () => Effect.die("unused"),
+    readSampleResolved: () => Effect.die("unused"),
     readTextPageResolved: () => Effect.die("unused"),
+    readToolResolved: () => Effect.die("unused"),
     list: () => Effect.die("unused"),
     resolveRoot: (input = {}) =>
       Effect.succeed(

+ 417 - 16
packages/core/test/tool-read.test.ts

@@ -1,5 +1,7 @@
 import { describe, expect } from "bun:test"
 import { Effect, Layer } from "effect"
+import { Config } from "@opencode-ai/core/config"
+import { ConfigAttachments } from "@opencode-ai/core/config/attachments"
 import { FileSystem } from "@opencode-ai/core/filesystem"
 import { PermissionV2 } from "@opencode-ai/core/permission"
 import { SessionV2 } from "@opencode-ai/core/session"
@@ -10,6 +12,7 @@ import { testEffect } from "./lib/effect"
 
 const assertions: PermissionV2.AssertInput[] = []
 const reads: FileSystem.ReadInput[] = []
+const samples: number[] = []
 const textPageInputs: FileSystem.TextPageInput[] = []
 const pages: FileSystem.ListTarget[] = []
 const pageInputs: Pick<FileSystem.ListPageInput, "offset" | "limit">[] = []
@@ -20,6 +23,14 @@ let listReal = "/project/src"
 let size = 5
 let real = "/project/README.md"
 let afterApproval = () => {}
+let readContent: FileSystem.Content = new FileSystem.TextContent({
+  type: "text",
+  content: "hello",
+  mime: "text/plain",
+})
+let sample = new TextEncoder().encode("hello")
+let readFailure: unknown
+let configEntries: Config.Entry[] = []
 const filesystem = Layer.succeed(
   FileSystem.Service,
   FileSystem.Service.of({
@@ -30,7 +41,7 @@ const filesystem = Layer.succeed(
             type: "file" as const,
             target: new FileSystem.ReadTarget({
               real,
-              resource: input.reference === undefined ? "README.md" : `${input.reference}:README.md`,
+              resource: input.reference === undefined ? input.path : `${input.reference}:${input.path}`,
               size,
               dev: 1,
             }),
@@ -56,7 +67,7 @@ const filesystem = Layer.succeed(
             ? Effect.succeed(
                 new FileSystem.ReadTarget({
                   real,
-                  resource: input.reference === undefined ? "README.md" : `${input.reference}:README.md`,
+                  resource: input.reference === undefined ? input.path : `${input.reference}:${input.path}`,
                   size,
                   dev: 1,
                 }),
@@ -65,22 +76,59 @@ const filesystem = Layer.succeed(
         ),
       ),
     readResolved: () =>
+      readFailure === undefined
+        ? Effect.sync(() => {
+            reads.push({ path: RelativePath.make("README.md") })
+            return readContent
+          })
+        : Effect.die(readFailure),
+    readSampleResolved: (_target, maximumBytes) =>
       Effect.sync(() => {
-        reads.push({ path: RelativePath.make("README.md") })
-        return new FileSystem.TextContent({ type: "text", content: "hello", mime: "text/plain" })
+        samples.push(maximumBytes)
+        return sample.slice(0, maximumBytes)
       }),
     readTextPageResolved: (_target, page = {}) =>
-      Effect.sync(() => {
-        textPageInputs.push(page)
-        return new FileSystem.TextPage({
-          type: "text-page",
-          content: "hello",
-          mime: "text/plain",
-          offset: page.offset ?? 1,
-          truncated: true,
-          next: (page.offset ?? 1) + 1,
+      readFailure === undefined
+        ? Effect.sync(() => {
+            textPageInputs.push(page)
+            return new FileSystem.TextPage({
+              type: "text-page",
+              content: "hello",
+              mime: "text/plain",
+              offset: page.offset ?? 1,
+              truncated: true,
+              next: (page.offset ?? 1) + 1,
+            })
+          })
+        : Effect.die(readFailure),
+    readToolResolved: (_target, page = {}) => {
+      samples.push(FileSystem.READ_SAMPLE_BYTES)
+      if (readFailure !== undefined) return Effect.die(readFailure)
+      if (sample[0] === 0x89 && sample[1] === 0x50 && sample[2] === 0x4e && sample[3] === 0x47)
+        return Effect.succeed(
+          readContent.type === "binary"
+            ? new FileSystem.BinaryContent({ ...readContent, mime: "image/png" })
+            : readContent,
+        )
+      if (FileSystem.isBinary(real.split("/").at(-1) ?? real, sample))
+        return Effect.die(new FileSystem.BinaryFileError(real.split("/").at(-1) ?? real))
+      if (size > FileSystem.MAX_READ_BYTES || page.offset !== undefined || page.limit !== undefined)
+        return Effect.sync(() => {
+          textPageInputs.push(page)
+          return new FileSystem.TextPage({
+            type: "text-page",
+            content: "hello",
+            mime: "text/plain",
+            offset: page.offset ?? 1,
+            truncated: true,
+            next: (page.offset ?? 1) + 1,
+          })
         })
-      }),
+      return Effect.sync(() => {
+        reads.push({ path: RelativePath.make("README.md") })
+        return readContent
+      })
+    },
     resolveRoot: () => Effect.die("unused"),
     revalidateRoot: Effect.succeed,
     list: () => Effect.die("unused"),
@@ -126,8 +174,14 @@ const permission = Layer.succeed(
   }),
 )
 const registry = ToolRegistry.defaultLayer.pipe(Layer.provide(permission))
-const read = ReadTool.layer.pipe(Layer.provide(registry), Layer.provide(filesystem), Layer.provide(permission))
-const it = testEffect(Layer.mergeAll(registry, filesystem, permission, read))
+const config = Layer.succeed(Config.Service, Config.Service.of({ entries: () => Effect.succeed(configEntries) }))
+const read = ReadTool.layer.pipe(
+  Layer.provide(registry),
+  Layer.provide(filesystem),
+  Layer.provide(permission),
+  Layer.provide(config),
+)
+const it = testEffect(Layer.mergeAll(registry, filesystem, permission, config, read))
 const sessionID = SessionV2.ID.make("ses_read_tool_test")
 
 describe("ReadTool", () => {
@@ -141,6 +195,10 @@ describe("ReadTool", () => {
       size = 5
       real = "/project/README.md"
       afterApproval = () => {}
+      readContent = new FileSystem.TextContent({ type: "text", content: "hello", mime: "text/plain" })
+      sample = new TextEncoder().encode("hello")
+      readFailure = undefined
+      configEntries = []
       resolvedInput = undefined
       const registry = yield* ToolRegistry.Service
 
@@ -156,6 +214,273 @@ describe("ReadTool", () => {
     }),
   )
 
+  it.effect("returns a small PNG as native media instead of durable base64 text", () =>
+    Effect.gen(function* () {
+      const png = "iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAQAAAC1HAwCAAAAC0lEQVR42mNk+A8AAQUBAScY42YAAAAASUVORK5CYII="
+      reads.length = 0
+      samples.length = 0
+      allow = true
+      resolveFailure = undefined
+      listResolveFailure = new Error("not a directory")
+      size = Buffer.from(png, "base64").length
+      real = "/project/pixel.png"
+      afterApproval = () => {}
+      sample = Buffer.from(png, "base64")
+      readContent = new FileSystem.BinaryContent({
+        type: "binary",
+        content: png,
+        encoding: "base64",
+        mime: "image/png",
+      })
+      readFailure = undefined
+      configEntries = []
+      const registry = yield* ToolRegistry.Service
+
+      expect(
+        yield* registry.execute({
+          sessionID,
+          call: { type: "tool-call", id: "call-image", name: "read", input: { path: "pixel.png" } },
+        }),
+      ).toEqual({
+        type: "content",
+        value: [
+          { type: "text", text: "Image read successfully" },
+          { type: "media", mediaType: "image/png", data: png, filename: "pixel.png" },
+        ],
+      })
+      expect(samples).toEqual([FileSystem.READ_SAMPLE_BYTES])
+      expect(reads).toHaveLength(0)
+
+      const settled = yield* registry.settle({
+        sessionID,
+        call: { type: "tool-call", id: "call-image-settle", name: "read", input: { path: "pixel.png" } },
+      })
+      expect(settled.output?.structured).toEqual({ type: "media", mime: "image/png" })
+      expect(JSON.stringify(settled.output?.structured)).not.toContain(png)
+      expect(settled.output?.content).toMatchObject([
+        { type: "text", text: "Image read successfully" },
+        { type: "file", mime: "image/png", source: { type: "data", data: png } },
+      ])
+    }),
+  )
+
+  it.effect("rejects invalid or truncated image data after signature classification", () =>
+    Effect.gen(function* () {
+      allow = true
+      resolveFailure = undefined
+      listResolveFailure = new Error("not a directory")
+      size = 8
+      real = "/project/truncated.png"
+      afterApproval = () => {}
+      sample = new Uint8Array([0x89, 0x50, 0x4e, 0x47, 0x0d, 0x0a, 0x1a, 0x0a])
+      readContent = new FileSystem.BinaryContent({
+        type: "binary",
+        content: Buffer.from(sample).toString("base64"),
+        encoding: "base64",
+        mime: "image/png",
+      })
+      readFailure = undefined
+      configEntries = []
+      const registry = yield* ToolRegistry.Service
+
+      expect(
+        yield* registry.execute({
+          sessionID,
+          call: { type: "tool-call", id: "call-truncated-image", name: "read", input: { path: "truncated.png" } },
+        }),
+      ).toEqual({ type: "error", value: "Image could not be decoded: truncated.png" })
+    }),
+  )
+
+  it.effect("rejects oversized images when resizing is disabled", () =>
+    Effect.gen(function* () {
+      const photon = yield* Effect.promise(() => import("@silvia-odwyer/photon-node"))
+      const source = new photon.PhotonImage(new Uint8Array(Array.from({ length: 16 * 4 }, () => 255)), 16, 1)
+      const base64 = Buffer.from(source.get_bytes()).toString("base64")
+      source.free()
+      allow = true
+      resolveFailure = undefined
+      listResolveFailure = new Error("not a directory")
+      size = Buffer.from(base64, "base64").length
+      real = "/project/wide.png"
+      afterApproval = () => {}
+      sample = Buffer.from(base64, "base64")
+      readContent = new FileSystem.BinaryContent({
+        type: "binary",
+        content: base64,
+        encoding: "base64",
+        mime: "image/png",
+      })
+      readFailure = undefined
+      configEntries = [
+        new Config.Document({
+          type: "document",
+          info: new Config.Info({
+            attachments: new ConfigAttachments.Info({
+              image: new ConfigAttachments.Image({ auto_resize: false, max_width: 4 }),
+            }),
+          }),
+        }),
+      ]
+      const registry = yield* ToolRegistry.Service
+      const result = yield* registry.execute({
+        sessionID,
+        call: { type: "tool-call", id: "call-wide-image", name: "read", input: { path: "wide.png" } },
+      })
+
+      expect(result.type).toBe("error")
+      if (result.type === "error") expect(result.value).toContain("exceeding configured limits 4x2000")
+    }),
+  )
+
+  it.effect("resizes images to configured dimensions before returning media", () =>
+    Effect.gen(function* () {
+      const photon = yield* Effect.promise(() => import("@silvia-odwyer/photon-node"))
+      const source = new photon.PhotonImage(new Uint8Array(Array.from({ length: 16 * 4 }, () => 255)), 16, 1)
+      const base64 = Buffer.from(source.get_bytes()).toString("base64")
+      source.free()
+      allow = true
+      resolveFailure = undefined
+      listResolveFailure = new Error("not a directory")
+      size = Buffer.from(base64, "base64").length
+      real = "/project/wide.png"
+      afterApproval = () => {}
+      sample = Buffer.from(base64, "base64")
+      readContent = new FileSystem.BinaryContent({
+        type: "binary",
+        content: base64,
+        encoding: "base64",
+        mime: "image/png",
+      })
+      readFailure = undefined
+      configEntries = [
+        new Config.Document({
+          type: "document",
+          info: new Config.Info({
+            attachments: new ConfigAttachments.Info({ image: new ConfigAttachments.Image({ max_width: 4 }) }),
+          }),
+        }),
+      ]
+      const registry = yield* ToolRegistry.Service
+      const result = yield* registry.execute({
+        sessionID,
+        call: { type: "tool-call", id: "call-resize-image", name: "read", input: { path: "wide.png" } },
+      })
+
+      expect(result.type).toBe("content")
+      if (result.type !== "content") return
+      const media = result.value[1]
+      expect(media?.type).toBe("media")
+      if (media?.type !== "media") return
+      const resized = photon.PhotonImage.new_from_byteslice(Buffer.from(media.data, "base64"))
+      expect(resized.get_width()).toBeLessThanOrEqual(4)
+      expect(resized.get_height()).toBeLessThanOrEqual(2_000)
+      resized.free()
+    }),
+  )
+
+  it.effect("enforces max base64 bytes after resize attempts", () =>
+    Effect.gen(function* () {
+      const png = "iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAQAAAC1HAwCAAAAC0lEQVR42mNk+A8AAQUBAScY42YAAAAASUVORK5CYII="
+      allow = true
+      resolveFailure = undefined
+      listResolveFailure = new Error("not a directory")
+      size = Buffer.from(png, "base64").length
+      real = "/project/pixel.png"
+      afterApproval = () => {}
+      sample = Buffer.from(png, "base64")
+      readContent = new FileSystem.BinaryContent({
+        type: "binary",
+        content: png,
+        encoding: "base64",
+        mime: "image/png",
+      })
+      readFailure = undefined
+      configEntries = [
+        new Config.Document({
+          type: "document",
+          info: new Config.Info({
+            attachments: new ConfigAttachments.Info({
+              image: new ConfigAttachments.Image({ max_base64_bytes: 1 }),
+            }),
+          }),
+        }),
+      ]
+      const registry = yield* ToolRegistry.Service
+      const result = yield* registry.execute({
+        sessionID,
+        call: { type: "tool-call", id: "call-max-bytes", name: "read", input: { path: "pixel.png" } },
+      })
+
+      expect(result.type).toBe("error")
+      if (result.type === "error") expect(result.value).toContain("/1 bytes")
+    }),
+  )
+
+  it.effect("classifies supported image contents before a misleading binary extension", () =>
+    Effect.gen(function* () {
+      const png = "iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAQAAAC1HAwCAAAAC0lEQVR42mNk+A8AAQUBAScY42YAAAAASUVORK5CYII="
+      allow = true
+      resolveFailure = undefined
+      listResolveFailure = new Error("not a directory")
+      size = Buffer.from(png, "base64").length
+      real = "/project/pixel.bin"
+      afterApproval = () => {}
+      sample = Buffer.from(png, "base64")
+      readContent = new FileSystem.BinaryContent({
+        type: "binary",
+        content: png,
+        encoding: "base64",
+        mime: "application/octet-stream",
+      })
+      readFailure = undefined
+      configEntries = []
+      const registry = yield* ToolRegistry.Service
+
+      expect(
+        yield* registry.execute({
+          sessionID,
+          call: { type: "tool-call", id: "call-disguised-image", name: "read", input: { path: "pixel.bin" } },
+        }),
+      ).toMatchObject({
+        type: "content",
+        value: [{ type: "text" }, { type: "media", mediaType: "image/png", filename: "pixel.bin" }],
+      })
+    }),
+  )
+
+  it.effect("rejects unsupported binary before direct reads or paging", () =>
+    Effect.gen(function* () {
+      reads.length = 0
+      textPageInputs.length = 0
+      samples.length = 0
+      allow = true
+      resolveFailure = undefined
+      listResolveFailure = new Error("not a directory")
+      size = FileSystem.MAX_READ_BYTES + 1
+      real = "/project/archive.dat"
+      afterApproval = () => {}
+      sample = new Uint8Array([0, 1, 2, 3])
+      readFailure = undefined
+      const registry = yield* ToolRegistry.Service
+
+      expect(
+        yield* registry.execute({
+          sessionID,
+          call: {
+            type: "tool-call",
+            id: "call-binary",
+            name: "read",
+            input: { path: "archive.dat", offset: 2, limit: 1 },
+          },
+        }),
+      ).toEqual({ type: "error", value: "Cannot read binary file: archive.dat" })
+      expect(samples).toEqual([FileSystem.READ_SAMPLE_BYTES])
+      expect(reads).toEqual([])
+      expect(textPageInputs).toEqual([])
+    }),
+  )
+
   it.effect("does not read when permission is denied", () =>
     Effect.gen(function* () {
       assertions.length = 0
@@ -301,6 +626,8 @@ describe("ReadTool", () => {
       size = FileSystem.MAX_READ_BYTES + 1
       real = "/project/large.txt"
       afterApproval = () => {}
+      sample = new TextEncoder().encode("hello")
+      readFailure = undefined
       const registry = yield* ToolRegistry.Service
 
       expect(
@@ -321,6 +648,78 @@ describe("ReadTool", () => {
     }),
   )
 
+  it.effect("preserves safe read limit errors", () =>
+    Effect.gen(function* () {
+      allow = true
+      resolveFailure = undefined
+      listResolveFailure = new Error("not a directory")
+      size = 5
+      real = "/project/changed.txt"
+      afterApproval = () => {}
+      sample = new TextEncoder().encode("hello")
+      readFailure = new FileSystem.ReadLimitError("changed.txt", FileSystem.MAX_READ_BYTES)
+      const registry = yield* ToolRegistry.Service
+
+      expect(
+        yield* registry.execute({
+          sessionID,
+          call: { type: "tool-call", id: "call-read-limit", name: "read", input: { path: "changed.txt" } },
+        }),
+      ).toEqual({
+        type: "error",
+        value: `File exceeds ${FileSystem.MAX_READ_BYTES} byte read limit: changed.txt`,
+      })
+    }),
+  )
+
+  it.effect("preserves binary errors discovered after the sample", () =>
+    Effect.gen(function* () {
+      allow = true
+      resolveFailure = undefined
+      listResolveFailure = new Error("not a directory")
+      size = FileSystem.MAX_READ_BYTES + 1
+      real = "/project/late-binary"
+      afterApproval = () => {}
+      sample = new TextEncoder().encode("text prefix")
+      readFailure = new FileSystem.BinaryFileError("late-binary")
+      const registry = yield* ToolRegistry.Service
+
+      expect(
+        yield* registry.execute({
+          sessionID,
+          call: { type: "tool-call", id: "call-late-binary", name: "read", input: { path: "late-binary" } },
+        }),
+      ).toEqual({ type: "error", value: "Cannot read binary file: late-binary" })
+    }),
+  )
+
+  it.effect("rejects unsupported binary discovered by a direct read", () =>
+    Effect.gen(function* () {
+      allow = true
+      resolveFailure = undefined
+      listResolveFailure = new Error("not a directory")
+      size = 5
+      real = "/project/late-binary"
+      afterApproval = () => {}
+      sample = new TextEncoder().encode("text prefix")
+      readFailure = undefined
+      readContent = new FileSystem.BinaryContent({
+        type: "binary",
+        content: "AAECAw==",
+        encoding: "base64",
+        mime: "application/octet-stream",
+      })
+      const registry = yield* ToolRegistry.Service
+
+      expect(
+        yield* registry.execute({
+          sessionID,
+          call: { type: "tool-call", id: "call-direct-binary", name: "read", input: { path: "late-binary" } },
+        }),
+      ).toEqual({ type: "error", value: "Cannot read binary file: late-binary" })
+    }),
+  )
+
   it.effect("does not read when the file changes after permission approval", () =>
     Effect.gen(function* () {
       assertions.length = 0
@@ -330,6 +729,8 @@ describe("ReadTool", () => {
       listResolveFailure = new Error("not a directory")
       size = 5
       real = "/project/README.md"
+      sample = new TextEncoder().encode("hello")
+      readFailure = undefined
       afterApproval = () => {
         real = "/outside/README.md"
       }

+ 20 - 14
packages/llm/src/protocols/anthropic-messages.ts

@@ -303,14 +303,17 @@ const lowerServerToolResult = Effect.fn("AnthropicMessages.lowerServerToolResult
 })
 
 const lowerImage = Effect.fn("AnthropicMessages.lowerImage")(function* (part: MediaPart) {
-  if (!part.mediaType.startsWith("image/"))
-    return yield* invalid(`Anthropic Messages user media content only supports images`)
+  const media = yield* ProviderShared.validateMedia(
+    "Anthropic Messages",
+    part,
+    new Set<string>(ProviderShared.IMAGE_MIMES),
+  )
   return {
     type: "image" as const,
     source: {
       type: "base64" as const,
-      media_type: part.mediaType,
-      data: ProviderShared.mediaBase64(part),
+      media_type: media.mime,
+      data: media.base64,
     },
   } satisfies AnthropicImageBlock
 })
@@ -321,16 +324,19 @@ const lowerToolResultContentItem = Effect.fn("AnthropicMessages.lowerToolResultC
   item: ToolResultContentPart,
 ) {
   if (item.type === "text") return { type: "text" as const, text: item.text } satisfies AnthropicTextBlock
-  if (item.mediaType.startsWith("image/"))
-    return {
-      type: "image" as const,
-      source: {
-        type: "base64" as const,
-        media_type: item.mediaType,
-        data: ProviderShared.mediaBase64(item),
-      },
-    } satisfies AnthropicImageBlock
-  return yield* invalid(`Anthropic Messages tool-result media content only supports images, got ${item.mediaType}`)
+  const media = yield* ProviderShared.validateMedia(
+    "Anthropic Messages",
+    item,
+    new Set<string>(ProviderShared.IMAGE_MIMES),
+  )
+  return {
+    type: "image" as const,
+    source: {
+      type: "base64" as const,
+      media_type: media.mime,
+      data: media.base64,
+    },
+  } satisfies AnthropicImageBlock
 })
 
 const lowerToolResultContent = Effect.fn("AnthropicMessages.lowerToolResultContent")(function* (part: ToolResultPart) {

+ 28 - 6
packages/llm/src/protocols/gemini.ts

@@ -14,12 +14,14 @@ import {
   type TextPart,
   type ToolCallPart,
   type ToolDefinition,
+  type ToolResultContentPart,
 } from "../schema"
 import { JsonObject, optionalArray, ProviderShared } from "./shared"
 import { GeminiToolSchema } from "./utils/gemini-tool-schema"
 import { Lifecycle } from "./utils/lifecycle"
 
 const ADAPTER = "gemini"
+const IMAGE_MIMES = new Set<string>(ProviderShared.IMAGE_MIMES)
 export const DEFAULT_BASE_URL = "https://generativelanguage.googleapis.com/v1beta"
 
 // =============================================================================
@@ -140,8 +142,6 @@ interface ParserState {
   readonly reasoningSignature?: string
 }
 
-const mediaData = ProviderShared.mediaBytes
-
 // =============================================================================
 // Tool Schema Conversion
 // =============================================================================
@@ -180,8 +180,11 @@ const lowerToolConfig = (toolChoice: NonNullable<LLMRequest["toolChoice"]>) =>
     tool: (name) => ({ functionCallingConfig: { mode: "ANY" as const, allowedFunctionNames: [name] } }),
   })
 
-const lowerUserPart = (part: TextPart | MediaPart) =>
-  part.type === "text" ? { text: part.text } : { inlineData: { mimeType: part.mediaType, data: mediaData(part) } }
+const lowerUserPart = Effect.fn("Gemini.lowerUserPart")(function* (part: TextPart | MediaPart) {
+  if (part.type === "text") return { text: part.text }
+  const media = yield* ProviderShared.validateMedia("Gemini", part, IMAGE_MIMES)
+  return { inlineData: { mimeType: media.mime, data: media.base64 } }
+})
 
 const googleMetadata = (metadata: Record<string, unknown>): ProviderMetadata => ({ google: metadata })
 
@@ -215,7 +218,7 @@ const lowerMessages = Effect.fn("Gemini.lowerMessages")(function* (request: LLMR
       for (const part of message.content) {
         if (!ProviderShared.supportsContent(part, ["text", "media"]))
           return yield* ProviderShared.unsupportedContent("Gemini", "user", ["text", "media"])
-        parts.push(lowerUserPart(part))
+        parts.push(yield* lowerUserPart(part))
       }
       contents.push({ role: "user", parts })
       continue
@@ -247,15 +250,34 @@ const lowerMessages = Effect.fn("Gemini.lowerMessages")(function* (request: LLMR
     for (const part of message.content) {
       if (!ProviderShared.supportsContent(part, ["tool-result"]))
         return yield* ProviderShared.unsupportedContent("Gemini", "tool", ["tool-result"])
+      if (part.result.type !== "content") {
+        parts.push({
+          functionResponse: {
+            name: part.name,
+            response: {
+              name: part.name,
+              content: ProviderShared.toolResultText(part),
+            },
+          },
+        })
+        continue
+      }
+      const content: ReadonlyArray<ToolResultContentPart> = part.result.value
+      const text = content.filter((item) => item.type === "text").map((item) => item.text)
       parts.push({
         functionResponse: {
           name: part.name,
           response: {
             name: part.name,
-            content: ProviderShared.toolResultText(part),
+            content: text.join("\n"),
           },
         },
       })
+      for (const item of content) {
+        if (item.type === "text") continue
+        const media = yield* ProviderShared.validateMedia("Gemini", item, IMAGE_MIMES)
+        parts.push({ inlineData: { mimeType: media.mime, data: media.base64 } })
+      }
     }
     contents.push({ role: "user", parts })
   }

+ 72 - 10
packages/llm/src/protocols/openai-chat.ts

@@ -9,10 +9,12 @@ import {
   Usage,
   type FinishReason,
   type LLMRequest,
+  type MediaPart,
   type ReasoningPart,
   type TextPart,
   type ToolCallPart,
   type ToolDefinition,
+  type ToolResultContentPart,
 } from "../schema"
 import { isRecord, JsonObject, optionalArray, optionalNull, ProviderShared } from "./shared"
 import { OpenAIOptions } from "./utils/openai-options"
@@ -20,6 +22,7 @@ import { Lifecycle } from "./utils/lifecycle"
 import { ToolStream } from "./utils/tool-stream"
 
 const ADAPTER = "openai-chat"
+const IMAGE_MIMES = new Set<string>(ProviderShared.IMAGE_MIMES)
 export const DEFAULT_BASE_URL = "https://api.openai.com/v1"
 export const PATH = "/chat/completions"
 
@@ -51,9 +54,20 @@ const OpenAIChatAssistantToolCall = Schema.Struct({
 })
 type OpenAIChatAssistantToolCall = Schema.Schema.Type<typeof OpenAIChatAssistantToolCall>
 
+const OpenAIChatUserContent = Schema.Union([
+  Schema.Struct({ type: Schema.Literal("text"), text: Schema.String }),
+  Schema.Struct({
+    type: Schema.Literal("image_url"),
+    image_url: Schema.Struct({ url: Schema.String }),
+  }),
+])
+
 const OpenAIChatMessage = Schema.Union([
   Schema.Struct({ role: Schema.Literal("system"), content: Schema.String }),
-  Schema.Struct({ role: Schema.Literal("user"), content: Schema.String }),
+  Schema.Struct({
+    role: Schema.Literal("user"),
+    content: Schema.Union([Schema.String, Schema.Array(OpenAIChatUserContent)]),
+  }),
   Schema.Struct({
     role: Schema.Literal("assistant"),
     content: Schema.NullOr(Schema.String),
@@ -186,17 +200,32 @@ const lowerToolCall = (part: ToolCallPart): OpenAIChatAssistantToolCall => ({
   },
 })
 
+const lowerMedia = Effect.fn("OpenAIChat.lowerMedia")(function* (
+  part: Extract<MediaPart | ToolResultContentPart, { type: "media" }>,
+) {
+  const media = yield* ProviderShared.validateMedia("OpenAI Chat", part, IMAGE_MIMES)
+  return { type: "image_url" as const, image_url: { url: media.dataUrl } }
+})
+
 const openAICompatibleReasoningContent = (native: unknown) =>
   isRecord(native) && typeof native.reasoning_content === "string" ? native.reasoning_content : undefined
 
 const lowerUserMessage = Effect.fn("OpenAIChat.lowerUserMessage")(function* (message: OpenAIChatRequestMessage) {
-  const content: TextPart[] = []
+  const content: Array<Schema.Schema.Type<typeof OpenAIChatUserContent>> = []
   for (const part of message.content) {
-    if (!ProviderShared.supportsContent(part, ["text"]))
-      return yield* ProviderShared.unsupportedContent("OpenAI Chat", "user", ["text"])
-    content.push(part)
+    if (part.type === "text") {
+      content.push({ type: "text", text: part.text })
+      continue
+    }
+    if (part.type === "media") {
+      content.push(yield* lowerMedia(part))
+      continue
+    }
+    return yield* ProviderShared.unsupportedContent("OpenAI Chat", "user", ["text", "media"])
   }
-  return { role: "user" as const, content: ProviderShared.joinText(content) }
+  if (content.every((part) => part.type === "text"))
+    return { role: "user" as const, content: content.map((part) => part.text).join("") }
+  return { role: "user" as const, content }
 })
 
 const lowerAssistantMessage = Effect.fn("OpenAIChat.lowerAssistantMessage")(function* (
@@ -234,35 +263,68 @@ const lowerAssistantMessage = Effect.fn("OpenAIChat.lowerAssistantMessage")(func
 
 const lowerToolMessages = Effect.fn("OpenAIChat.lowerToolMessages")(function* (message: OpenAIChatRequestMessage) {
   const messages: OpenAIChatMessage[] = []
+  const images: Array<Schema.Schema.Type<typeof OpenAIChatUserContent>> = []
   for (const part of message.content) {
     if (!ProviderShared.supportsContent(part, ["tool-result"]))
       return yield* ProviderShared.unsupportedContent("OpenAI Chat", "tool", ["tool-result"])
-    messages.push({ role: "tool", tool_call_id: part.id, content: ProviderShared.toolResultText(part) })
+    if (part.result.type !== "content") {
+      messages.push({ role: "tool", tool_call_id: part.id, content: ProviderShared.toolResultText(part) })
+      continue
+    }
+    const content: ReadonlyArray<ToolResultContentPart> = part.result.value
+    const text = content.filter((item) => item.type === "text").map((item) => item.text)
+    messages.push({ role: "tool", tool_call_id: part.id, content: text.join("\n") })
+    const media = content.filter(
+      (item): item is Extract<ToolResultContentPart, { type: "media" }> => item.type === "media",
+    )
+    images.push(...(yield* Effect.forEach(media, lowerMedia)))
   }
-  return messages
+  return { messages, images }
 })
 
 const lowerMessage = Effect.fn("OpenAIChat.lowerMessage")(function* (message: OpenAIChatRequestMessage) {
   if (message.role === "user") return [yield* lowerUserMessage(message)]
   if (message.role === "assistant") return [yield* lowerAssistantMessage(message)]
-  return yield* lowerToolMessages(message)
+  return (yield* lowerToolMessages(message)).messages
 })
 
 const lowerMessages = Effect.fn("OpenAIChat.lowerMessages")(function* (request: LLMRequest) {
   const system: OpenAIChatMessage[] =
     request.system.length === 0 ? [] : [{ role: "system", content: ProviderShared.joinText(request.system) }]
   const messages = [...system]
+  const pendingImages: Array<Schema.Schema.Type<typeof OpenAIChatUserContent>> = []
+  const flushImages = () => {
+    if (pendingImages.length === 0) return
+    messages.push({ role: "user", content: pendingImages.splice(0) })
+  }
   for (const message of request.messages) {
     if (message.role === "system") {
       const part = yield* ProviderShared.wrappedSystemUpdate("OpenAI Chat", message)
+      if (pendingImages.length > 0) {
+        messages.push({ role: "user", content: [...pendingImages.splice(0), { type: "text", text: part.text }] })
+        continue
+      }
       const previous = messages.at(-1)
-      if (previous?.role === "user")
+      if (previous?.role === "user" && typeof previous.content === "string")
         messages[messages.length - 1] = { role: "user", content: `${previous.content}\n${part.text}` }
+      else if (previous?.role === "user" && Array.isArray(previous.content))
+        messages[messages.length - 1] = {
+          role: "user",
+          content: [...previous.content, { type: "text", text: part.text }],
+        }
       else messages.push({ role: "user", content: part.text })
       continue
     }
+    if (message.role === "tool") {
+      const lowered = yield* lowerToolMessages(message)
+      messages.push(...lowered.messages)
+      pendingImages.push(...lowered.images)
+      continue
+    }
+    flushImages()
     messages.push(...(yield* lowerMessage(message)))
   }
+  flushImages()
   return messages
 })
 

+ 13 - 9
packages/llm/src/protocols/openai-responses.ts

@@ -304,10 +304,14 @@ const lowerUserContent = Effect.fn("OpenAIResponses.lowerUserContent")(function*
   part: LLMRequest["messages"][number]["content"][number],
 ) {
   if (part.type === "text") return { type: "input_text" as const, text: part.text }
-  if (part.type === "media" && part.mediaType.startsWith("image/")) {
-    return { type: "input_image" as const, image_url: ProviderShared.mediaDataUrl(part) }
+  if (part.type === "media") {
+    const media = yield* ProviderShared.validateMedia(
+      "OpenAI Responses",
+      part,
+      new Set<string>(ProviderShared.IMAGE_MIMES),
+    )
+    return { type: "input_image" as const, image_url: media.dataUrl }
   }
-  if (part.type === "media") return yield* invalid("OpenAI Responses user media content only supports images")
   return yield* ProviderShared.unsupportedContent("OpenAI Responses", "user", ["text", "media"])
 })
 
@@ -317,12 +321,12 @@ const lowerToolResultContentItem = Effect.fn("OpenAIResponses.lowerToolResultCon
   item: ToolResultContentPart,
 ) {
   if (item.type === "text") return { type: "input_text" as const, text: item.text }
-  if (item.mediaType.startsWith("image/"))
-    return {
-      type: "input_image" as const,
-      image_url: ProviderShared.mediaDataUrl(item),
-    }
-  return yield* invalid(`OpenAI Responses tool-result media content only supports images, got ${item.mediaType}`)
+  const media = yield* ProviderShared.validateMedia(
+    "OpenAI Responses",
+    item,
+    new Set<string>(ProviderShared.IMAGE_MIMES),
+  )
+  return { type: "input_image" as const, image_url: media.dataUrl }
 })
 
 const lowerToolResultOutput = Effect.fn("OpenAIResponses.lowerToolResultOutput")(function* (part: ToolResultPart) {

+ 43 - 15
packages/llm/src/protocols/shared.ts

@@ -186,24 +186,52 @@ export const wrappedSystemUpdate = Effect.fn("ProviderShared.wrappedSystemUpdate
 export const parseToolInput = (route: string, name: string, raw: string) =>
   parseJson(route, raw || "{}", `Invalid JSON input for ${route} tool call ${name}`)
 
-/**
- * Encode a `MediaPart`'s raw bytes for inclusion in a JSON request body.
- * `data: string` is assumed to already be base64 (matches caller convention
- * across Gemini / Bedrock); `data: Uint8Array` is base64-encoded here. Used
- * by every route that supports image / document inputs.
- */
-export const mediaBytes = (part: MediaPart) =>
-  typeof part.data === "string" ? part.data : Buffer.from(part.data).toString("base64")
+export const IMAGE_MIMES = ["image/png", "image/jpeg", "image/gif", "image/webp"] as const
+export const MAX_MEDIA_ENCODED_BYTES = 8 * 1024 * 1024
+export const MAX_MEDIA_DECODED_BYTES = 6 * 1024 * 1024
+
+const base64Pattern = /^(?:[A-Za-z0-9+/]{4})*(?:[A-Za-z0-9+/]{2}==|[A-Za-z0-9+/]{3}=)?$/
 
-export const mediaBase64 = (part: MediaPart) => {
-  if (typeof part.data !== "string" || !part.data.startsWith("data:")) return mediaBytes(part)
-  return part.data.slice(part.data.indexOf(",") + 1)
+export interface ValidatedMedia {
+  readonly mime: string
+  readonly base64: string
+  readonly dataUrl: string
+  readonly bytes: Uint8Array
 }
 
-export const mediaDataUrl = (part: MediaPart) =>
-  typeof part.data === "string" && part.data.startsWith("data:")
-    ? part.data
-    : `data:${part.mediaType};base64,${mediaBytes(part)}`
+export const validateMedia = Effect.fn("ProviderShared.validateMedia")(function* (
+  route: string,
+  part: MediaPart,
+  supportedMimes: ReadonlySet<string>,
+) {
+  const mime = part.mediaType.toLowerCase()
+  if (!supportedMimes.has(mime)) return yield* invalidRequest(`${route} does not support media type ${part.mediaType}`)
+
+  let base64: string
+  if (typeof part.data !== "string") {
+    if (part.data.byteLength > MAX_MEDIA_DECODED_BYTES)
+      return yield* invalidRequest(`${route} media exceeds the ${MAX_MEDIA_DECODED_BYTES} byte decoded limit`)
+    base64 = Buffer.from(part.data).toString("base64")
+  } else if (part.data.startsWith("data:")) {
+    const match = /^data:([^;,]+);base64,([A-Za-z0-9+/]*={0,2})$/s.exec(part.data)
+    if (!match) return yield* invalidRequest(`${route} media data URL must contain valid base64`)
+    if (match[1]!.toLowerCase() !== mime)
+      return yield* invalidRequest(`${route} media type ${part.mediaType} does not match data URL type ${match[1]}`)
+    base64 = match[2]!
+  } else {
+    base64 = part.data
+  }
+
+  if (Buffer.byteLength(base64, "utf8") > MAX_MEDIA_ENCODED_BYTES)
+    return yield* invalidRequest(`${route} media exceeds the ${MAX_MEDIA_ENCODED_BYTES} byte encoded limit`)
+  if (!base64 || base64.length % 4 !== 0 || !base64Pattern.test(base64))
+    return yield* invalidRequest(`${route} media must contain valid base64`)
+  const bytes = Buffer.from(base64, "base64")
+  if (bytes.byteLength > MAX_MEDIA_DECODED_BYTES)
+    return yield* invalidRequest(`${route} media exceeds the ${MAX_MEDIA_DECODED_BYTES} byte decoded limit`)
+  if (bytes.toString("base64") !== base64) return yield* invalidRequest(`${route} media must contain canonical base64`)
+  return { mime, base64, dataUrl: `data:${mime};base64,${base64}`, bytes } satisfies ValidatedMedia
+})
 
 export const trimBaseUrl = (value: string) => value.replace(/\/+$/, "")
 

+ 22 - 12
packages/llm/src/protocols/utils/bedrock-media.ts

@@ -49,15 +49,11 @@ const DOCUMENT_FORMATS = {
   "text/markdown": "md",
 } as const satisfies Record<string, DocumentFormat>
 
-const imageBlock = (part: MediaPart, format: ImageFormat): ImageBlock => ({
-  image: { format, source: { bytes: ProviderShared.mediaBytes(part) } },
-})
-
-const documentBlock = (part: MediaPart, format: DocumentFormat): DocumentBlock => ({
+const documentBlock = (part: MediaPart, format: DocumentFormat, bytes: string): DocumentBlock => ({
   document: {
     format,
     name: part.filename ?? `document.${format}`,
-    source: { bytes: ProviderShared.mediaBytes(part) },
+    source: { bytes },
   },
 })
 
@@ -66,15 +62,29 @@ const documentBlock = (part: MediaPart, format: DocumentFormat): DocumentBlock =
 // document block. Image MIME types not in `IMAGE_FORMATS` (e.g. `image/svg+xml`)
 // get an image-specific error so the caller knows it's a format-support issue,
 // not a kind-detection issue.
-export const lower = (part: MediaPart) => {
+export const lower = Effect.fn("BedrockMedia.lower")(function* (part: MediaPart) {
   const mime = part.mediaType.toLowerCase()
   const imageFormat = IMAGE_FORMATS[mime as keyof typeof IMAGE_FORMATS]
-  if (imageFormat) return Effect.succeed(imageBlock(part, imageFormat))
+  if (imageFormat) {
+    const media = yield* ProviderShared.validateMedia(
+      "Bedrock Converse",
+      part,
+      new Set<string>(Object.keys(IMAGE_FORMATS)),
+    )
+    return { image: { format: imageFormat, source: { bytes: media.base64 } } } satisfies ImageBlock
+  }
   if (mime.startsWith("image/"))
-    return ProviderShared.invalidRequest(`Bedrock Converse does not support image media type ${part.mediaType}`)
+    return yield* ProviderShared.invalidRequest(`Bedrock Converse does not support image media type ${part.mediaType}`)
   const documentFormat = DOCUMENT_FORMATS[mime as keyof typeof DOCUMENT_FORMATS]
-  if (documentFormat) return Effect.succeed(documentBlock(part, documentFormat))
-  return ProviderShared.invalidRequest(`Bedrock Converse does not support media type ${part.mediaType}`)
-}
+  if (documentFormat) {
+    const media = yield* ProviderShared.validateMedia(
+      "Bedrock Converse",
+      part,
+      new Set<string>(Object.keys(DOCUMENT_FORMATS)),
+    )
+    return documentBlock(part, documentFormat, media.base64)
+  }
+  return yield* ProviderShared.invalidRequest(`Bedrock Converse does not support media type ${part.mediaType}`)
+})
 
 export * as BedrockMedia from "./bedrock-media"

+ 16 - 4
packages/llm/src/tool.ts

@@ -51,6 +51,7 @@ export interface Tool<Parameters extends ToolSchema<any>, Success extends ToolSc
   readonly success: Success
   readonly execute?: ToolExecute<Parameters, Success>
   readonly toModelOutput?: ToolToModelOutput<Parameters, Success>
+  readonly toStructuredOutput?: (output: Success["Encoded"]) => unknown
   /** @internal */
   readonly _decode: (input: unknown) => Effect.Effect<Schema.Schema.Type<Parameters>, Schema.SchemaError>
   /** @internal */
@@ -86,6 +87,7 @@ type TypedToolConfig = {
   readonly success: ToolSchema<any>
   readonly execute?: ToolExecute<ToolSchema<any>, ToolSchema<any>>
   readonly toModelOutput?: ToolToModelOutput<ToolSchema<any>, ToolSchema<any>>
+  readonly toStructuredOutput?: (output: unknown) => unknown
 }
 
 type DynamicToolConfig = {
@@ -94,6 +96,7 @@ type DynamicToolConfig = {
   readonly outputSchema?: JsonSchema.JsonSchema
   readonly execute?: (params: unknown, context?: ToolExecuteContext) => Effect.Effect<unknown, ToolFailure>
   readonly toModelOutput?: (input: ToolModelOutputInput<unknown, unknown>) => ReadonlyArray<ToolContent>
+  readonly toStructuredOutput?: (output: unknown) => unknown
 }
 
 /**
@@ -133,6 +136,7 @@ export function make<Parameters extends ToolSchema<any>, Success extends ToolSch
   readonly success: Success
   readonly execute: ToolExecute<Parameters, Success>
   readonly toModelOutput?: ToolToModelOutput<Parameters, Success>
+  readonly toStructuredOutput?: (output: Success["Encoded"]) => unknown
 }): ExecutableTool<Parameters, Success>
 export function make<Parameters extends ToolSchema<any>, Success extends ToolSchema<any>>(config: {
   readonly description: string
@@ -140,6 +144,7 @@ export function make<Parameters extends ToolSchema<any>, Success extends ToolSch
   readonly success: Success
   readonly execute?: undefined
   readonly toModelOutput?: ToolToModelOutput<Parameters, Success>
+  readonly toStructuredOutput?: (output: Success["Encoded"]) => unknown
 }): Tool<Parameters, Success>
 export function make(config: {
   readonly description: string
@@ -147,6 +152,7 @@ export function make(config: {
   readonly outputSchema?: JsonSchema.JsonSchema
   readonly execute: (params: unknown, context?: ToolExecuteContext) => Effect.Effect<unknown, ToolFailure>
   readonly toModelOutput?: (input: ToolModelOutputInput<unknown, unknown>) => ReadonlyArray<ToolContent>
+  readonly toStructuredOutput?: (output: unknown) => unknown
 }): AnyExecutableTool
 export function make(config: {
   readonly description: string
@@ -154,6 +160,7 @@ export function make(config: {
   readonly outputSchema?: JsonSchema.JsonSchema
   readonly execute?: undefined
   readonly toModelOutput?: (input: ToolModelOutputInput<unknown, unknown>) => ReadonlyArray<ToolContent>
+  readonly toStructuredOutput?: (output: unknown) => unknown
 }): AnyTool
 export function make(config: TypedToolConfig | DynamicToolConfig): AnyTool {
   if ("jsonSchema" in config) {
@@ -163,10 +170,12 @@ export function make(config: TypedToolConfig | DynamicToolConfig): AnyTool {
       success: Schema.Unknown as ToolSchema<unknown>,
       execute: config.execute,
       toModelOutput: config.toModelOutput,
+      toStructuredOutput: config.toStructuredOutput,
       _decode: Effect.succeed,
       _encode: Effect.succeed,
-      _project: (parameters, callID, output) => project(config.toModelOutput, parameters, callID, output),
-      _legacyResult: config.toModelOutput === undefined,
+      _project: (parameters, callID, output) =>
+        project(config.toModelOutput, config.toStructuredOutput, parameters, callID, output),
+      _legacyResult: config.toModelOutput === undefined && config.toStructuredOutput === undefined,
       _definition: new ToolDefinition({
         name: "",
         description: config.description,
@@ -181,9 +190,11 @@ export function make(config: TypedToolConfig | DynamicToolConfig): AnyTool {
     success: config.success,
     execute: config.execute,
     toModelOutput: config.toModelOutput,
+    toStructuredOutput: config.toStructuredOutput,
     _decode: Schema.decodeUnknownEffect(config.parameters),
     _encode: Schema.encodeEffect(config.success),
-    _project: (parameters, callID, output) => project(config.toModelOutput, parameters, callID, output),
+    _project: (parameters, callID, output) =>
+      project(config.toModelOutput, config.toStructuredOutput, parameters, callID, output),
     _legacyResult: false,
     _definition: new ToolDefinition({
       name: "",
@@ -226,12 +237,13 @@ const toJsonSchema = (schema: Schema.Top): JsonSchema.JsonSchema => {
 
 const project = (
   toModelOutput: ((input: ToolModelOutputInput<any, any>) => ReadonlyArray<ToolContent>) | undefined,
+  toStructuredOutput: ((output: unknown) => unknown) | undefined,
   parameters: unknown,
   callID: ToolCallPart["id"],
   output: unknown,
 ): ToolOutputType =>
   ToolOutput.make(
-    output,
+    toStructuredOutput?.(output) ?? output,
     toModelOutput?.({ callID, parameters, output }) ??
       (typeof output === "string" ? [toolText({ type: "text", text: output })] : []),
   )

+ 4 - 4
packages/llm/test/provider/bedrock-converse.test.ts

@@ -509,8 +509,8 @@ describe("Bedrock Converse route", () => {
           model,
           messages: [
             Message.user([
-              { type: "media", mediaType: "application/pdf", data: "PDFDATA", filename: "report.pdf" },
-              { type: "media", mediaType: "text/csv", data: "CSVDATA" },
+              { type: "media", mediaType: "application/pdf", data: "UERGREFUQQ==", filename: "report.pdf" },
+              { type: "media", mediaType: "text/csv", data: "Q1NWREFUQQ==" },
             ]),
           ],
         }),
@@ -522,9 +522,9 @@ describe("Bedrock Converse route", () => {
             role: "user",
             content: [
               // Filename round-trips when supplied.
-              { document: { format: "pdf", name: "report.pdf", source: { bytes: "PDFDATA" } } },
+              { document: { format: "pdf", name: "report.pdf", source: { bytes: "UERGREFUQQ==" } } },
               // Falls back to a stable placeholder when filename is missing.
-              { document: { format: "csv", name: "document.csv", source: { bytes: "CSVDATA" } } },
+              { document: { format: "csv", name: "document.csv", source: { bytes: "Q1NWREFUQQ==" } } },
             ],
           },
         ],

+ 105 - 0
packages/llm/test/provider/gemini.test.ts

@@ -3,6 +3,7 @@ import { Effect } from "effect"
 import { LLM, LLMError, Message, ToolCallPart, Usage } from "../../src"
 import { Auth, LLMClient } from "../../src/route"
 import * as Gemini from "../../src/protocols/gemini"
+import { ProviderShared } from "../../src/protocols/shared"
 import { it } from "../lib/effect"
 import { fixedResponse } from "../lib/http"
 import { sseEvents, sseRaw } from "../lib/sse"
@@ -109,6 +110,110 @@ describe("Gemini route", () => {
     }),
   )
 
+  it.effect("continues image tool results as inline vision input without base64 text", () =>
+    Effect.gen(function* () {
+      const prepared = yield* LLMClient.prepare<Gemini.GeminiBody>(
+        LLM.request({
+          model,
+          messages: [
+            Message.assistant([ToolCallPart.make({ id: "call_image", name: "read", input: { path: "pixel.png" } })]),
+            Message.tool({
+              id: "call_image",
+              name: "read",
+              result: {
+                type: "content",
+                value: [
+                  { type: "text", text: "Image read successfully" },
+                  { type: "media", mediaType: "image/png", data: "AAECAw==", filename: "pixel.png" },
+                ],
+              },
+            }),
+          ],
+        }),
+      )
+
+      expect(prepared.body.contents).toEqual([
+        { role: "model", parts: [{ functionCall: { name: "read", args: { path: "pixel.png" } } }] },
+        {
+          role: "user",
+          parts: [
+            {
+              functionResponse: {
+                name: "read",
+                response: { name: "read", content: "Image read successfully" },
+              },
+            },
+            { inlineData: { mimeType: "image/png", data: "AAECAw==" } },
+          ],
+        },
+      ])
+      expect(JSON.stringify(prepared.body.contents)).not.toContain('"content":"AAECAw=="')
+    }),
+  )
+
+  it.effect("strips matching data URLs to raw base64 inlineData", () =>
+    Effect.gen(function* () {
+      const prepared = yield* LLMClient.prepare<Gemini.GeminiBody>(
+        LLM.request({
+          model,
+          messages: [
+            Message.user({ type: "media", mediaType: "image/png", data: "data:image/png;base64,AAEC" }),
+            Message.tool({
+              id: "call_image",
+              name: "read",
+              result: {
+                type: "content",
+                value: [{ type: "media", mediaType: "image/jpeg", data: "data:image/jpeg;base64,/9j/" }],
+              },
+            }),
+          ],
+        }),
+      )
+      expect(prepared.body.contents).toEqual([
+        { role: "user", parts: [{ inlineData: { mimeType: "image/png", data: "AAEC" } }] },
+        {
+          role: "user",
+          parts: [
+            { functionResponse: { name: "read", response: { name: "read", content: "" } } },
+            { inlineData: { mimeType: "image/jpeg", data: "/9j/" } },
+          ],
+        },
+      ])
+    }),
+  )
+
+  for (const [name, media] of [
+    ["mismatched data URL MIME", { mediaType: "image/png", data: "data:image/jpeg;base64,/9j/" }],
+    ["malformed base64", { mediaType: "image/png", data: "%%%=" }],
+    ["unsupported SVG", { mediaType: "image/svg+xml", data: "PHN2Zz4=" }],
+  ] as const)
+    it.effect(`rejects ${name}`, () =>
+      Effect.gen(function* () {
+        const error = yield* LLMClient.prepare(
+          LLM.request({ model, messages: [Message.user({ type: "media", ...media })] }),
+        ).pipe(Effect.flip)
+        expect(error.message).toMatch(/does not support|does not match|valid base64/)
+      }),
+    )
+
+  it.effect("rejects oversized image input", () =>
+    Effect.gen(function* () {
+      const error = yield* LLMClient.prepare(
+        LLM.request({
+          model,
+          messages: [
+            Message.user({
+              type: "media",
+              mediaType: "image/png",
+              data: "A".repeat(ProviderShared.MAX_MEDIA_ENCODED_BYTES + 4),
+            }),
+          ],
+        }),
+      ).pipe(Effect.flip)
+      expect(error.message).toContain("encoded limit")
+    }),
+  )
+
   it.effect("omits tools when tool choice is none", () =>
     Effect.gen(function* () {
       const prepared = yield* LLMClient.prepare(

+ 7 - 2
packages/llm/test/provider/golden.recorded.test.ts

@@ -73,7 +73,7 @@ describeRecordedGoldenScenarios([
     prefix: "openai-chat",
     model: openAIChat,
     requires: ["OPENAI_API_KEY"],
-    scenarios: ["text", "tool-call", "tool-loop"],
+    scenarios: ["text", "tool-call", "tool-loop", { id: "image-tool-result", maxTokens: 40 }],
   },
   {
     name: "OpenAI Responses gpt-5.5",
@@ -123,7 +123,12 @@ describeRecordedGoldenScenarios([
     prefix: "gemini",
     model: gemini,
     requires: ["GOOGLE_GENERATIVE_AI_API_KEY"],
-    scenarios: [{ id: "text", maxTokens: 80 }, "tool-call", { id: "image", maxTokens: 160 }],
+    scenarios: [
+      { id: "text", maxTokens: 80 },
+      "tool-call",
+      { id: "image", maxTokens: 160 },
+      { id: "image-tool-result", maxTokens: 40 },
+    ],
   },
   {
     name: "xAI Grok 3 Mini",

+ 196 - 4
packages/llm/test/provider/openai-chat.test.ts

@@ -5,6 +5,7 @@ import { LLM, LLMError, Message, Model, ToolCallPart, Usage } from "../../src"
 import * as Azure from "../../src/providers/azure"
 import * as OpenAI from "../../src/providers/openai"
 import * as OpenAIChat from "../../src/protocols/openai-chat"
+import { ProviderShared } from "../../src/protocols/shared"
 import { Auth, LLMClient } from "../../src/route"
 import { it } from "../lib/effect"
 import { dynamicResponse, fixedResponse, truncatedStream } from "../lib/http"
@@ -223,17 +224,208 @@ describe("OpenAI Chat route", () => {
     }),
   )
 
-  it.effect("rejects unsupported user media content", () =>
+  it.effect("continues image tool results as vision input without base64 text", () =>
+    Effect.gen(function* () {
+      const prepared = yield* LLMClient.prepare<OpenAIChat.OpenAIChatBody>(
+        LLM.request({
+          model,
+          messages: [
+            Message.assistant([ToolCallPart.make({ id: "call_image", name: "read", input: { path: "pixel.png" } })]),
+            Message.tool({
+              id: "call_image",
+              name: "read",
+              result: {
+                type: "content",
+                value: [
+                  { type: "text", text: "Image read successfully" },
+                  { type: "media", mediaType: "image/png", data: "AAECAw==", filename: "pixel.png" },
+                ],
+              },
+            }),
+          ],
+        }),
+      )
+
+      expect(prepared.body.messages).toEqual([
+        {
+          role: "assistant",
+          content: null,
+          tool_calls: [
+            {
+              id: "call_image",
+              type: "function",
+              function: { name: "read", arguments: encodeJson({ path: "pixel.png" }) },
+            },
+          ],
+        },
+        { role: "tool", tool_call_id: "call_image", content: "Image read successfully" },
+        {
+          role: "user",
+          content: [{ type: "image_url", image_url: { url: "data:image/png;base64,AAECAw==" } }],
+        },
+      ])
+      expect(JSON.stringify(prepared.body.messages)).not.toContain('"content":"AAECAw=="')
+    }),
+  )
+
+  it.effect("orders parallel tool responses before one aggregated vision message", () =>
+    Effect.gen(function* () {
+      const prepared = yield* LLMClient.prepare<OpenAIChat.OpenAIChatBody>(
+        LLM.request({
+          model,
+          messages: [
+            Message.assistant([
+              ToolCallPart.make({ id: "call_1", name: "read", input: {} }),
+              ToolCallPart.make({ id: "call_2", name: "read", input: {} }),
+            ]),
+            Message.make({
+              role: "tool",
+              content: [
+                {
+                  type: "tool-result",
+                  id: "call_1",
+                  name: "read",
+                  result: { type: "content", value: [{ type: "media", mediaType: "image/png", data: "AAEC" }] },
+                },
+                {
+                  type: "tool-result",
+                  id: "call_2",
+                  name: "read",
+                  result: { type: "content", value: [{ type: "media", mediaType: "image/jpeg", data: "/9j/" }] },
+                },
+              ],
+            }),
+          ],
+        }),
+      )
+      expect(prepared.body.messages.slice(1)).toEqual([
+        { role: "tool", tool_call_id: "call_1", content: "" },
+        { role: "tool", tool_call_id: "call_2", content: "" },
+        {
+          role: "user",
+          content: [
+            { type: "image_url", image_url: { url: "data:image/png;base64,AAEC" } },
+            { type: "image_url", image_url: { url: "data:image/jpeg;base64,/9j/" } },
+          ],
+        },
+      ])
+    }),
+  )
+
+  it.effect("aggregates consecutive tool images with a following system update", () =>
+    Effect.gen(function* () {
+      const prepared = yield* LLMClient.prepare<OpenAIChat.OpenAIChatBody>(
+        LLM.request({
+          model,
+          messages: [
+            Message.tool({
+              id: "call_1",
+              name: "read",
+              result: { type: "content", value: [{ type: "media", mediaType: "image/png", data: "AAEC" }] },
+            }),
+            Message.tool({
+              id: "call_2",
+              name: "read",
+              result: { type: "content", value: [{ type: "media", mediaType: "image/webp", data: "UklG" }] },
+            }),
+            Message.system("Inspect both images."),
+          ],
+        }),
+      )
+      expect(prepared.body.messages).toEqual([
+        { role: "tool", tool_call_id: "call_1", content: "" },
+        { role: "tool", tool_call_id: "call_2", content: "" },
+        {
+          role: "user",
+          content: [
+            { type: "image_url", image_url: { url: "data:image/png;base64,AAEC" } },
+            { type: "image_url", image_url: { url: "data:image/webp;base64,UklG" } },
+            { type: "text", text: "<system-update>\nInspect both images.\n</system-update>" },
+          ],
+        },
+      ])
+    }),
+  )
+
+  it.effect("appends system updates without replacing multipart user content", () =>
+    Effect.gen(function* () {
+      const prepared = yield* LLMClient.prepare<OpenAIChat.OpenAIChatBody>(
+        LLM.request({
+          model,
+          messages: [
+            Message.user({ type: "media", mediaType: "image/png", data: "AAEC" }),
+            Message.system("Keep the image."),
+          ],
+        }),
+      )
+      expect(prepared.body.messages).toEqual([
+        {
+          role: "user",
+          content: [
+            { type: "image_url", image_url: { url: "data:image/png;base64,AAEC" } },
+            { type: "text", text: "<system-update>\nKeep the image.\n</system-update>" },
+          ],
+        },
+      ])
+    }),
+  )
+
+  for (const [name, media] of [
+    ["mismatched data URL MIME", { mediaType: "image/png", data: "data:image/jpeg;base64,/9j/" }],
+    ["malformed base64", { mediaType: "image/png", data: "not-base64" }],
+    ["unsupported SVG", { mediaType: "image/svg+xml", data: "PHN2Zz4=" }],
+  ] as const)
+    it.effect(`rejects ${name}`, () =>
+      Effect.gen(function* () {
+        const error = yield* LLMClient.prepare(
+          LLM.request({ model, messages: [Message.user({ type: "media", ...media })] }),
+        ).pipe(Effect.flip)
+        expect(error.message).toMatch(/does not support|does not match|valid base64/)
+      }),
+    )
+
+  it.effect("rejects oversized image input", () =>
     Effect.gen(function* () {
       const error = yield* LLMClient.prepare(
         LLM.request({
-          id: "req_media",
           model,
-          messages: [Message.user({ type: "media", mediaType: "image/png", data: "AAECAw==" })],
+          messages: [
+            Message.user({
+              type: "media",
+              mediaType: "image/png",
+              data: "A".repeat(ProviderShared.MAX_MEDIA_ENCODED_BYTES + 4),
+            }),
+          ],
         }),
       ).pipe(Effect.flip)
+      expect(error.message).toContain("encoded limit")
+    }),
+  )
+
+  it.effect("prepares raw and data URL image media as vision input", () =>
+    Effect.gen(function* () {
+      const prepared = yield* LLMClient.prepare<OpenAIChat.OpenAIChatBody>(
+        LLM.request({
+          id: "req_media",
+          model,
+          messages: [
+            Message.user([
+              { type: "media", mediaType: "image/png", data: "AAECAw==" },
+              { type: "media", mediaType: "image/jpeg", data: "data:image/jpeg;base64,/9j/" },
+            ]),
+          ],
+        }),
+      )
 
-      expect(error.message).toContain("OpenAI Chat user messages only support text content for now")
+      expect(prepared.body.messages).toEqual([
+        {
+          role: "user",
+          content: [
+            { type: "image_url", image_url: { url: "data:image/png;base64,AAECAw==" } },
+            { type: "image_url", image_url: { url: "data:image/jpeg;base64,/9j/" } },
+          ],
+        },
+      ])
     }),
   )
 

+ 1 - 1
packages/llm/test/provider/openai-responses.test.ts

@@ -1254,7 +1254,7 @@ describe("OpenAI Responses route", () => {
         }),
       ).pipe(Effect.flip)
 
-      expect(error.message).toContain("OpenAI Responses user media content only supports images")
+      expect(error.message).toContain("OpenAI Responses does not support media type application/pdf")
     }),
   )
 

+ 25 - 0
packages/llm/test/tool-runtime.test.ts

@@ -208,6 +208,31 @@ describe("LLMClient tools", () => {
     }),
   )
 
+  it.effect("can retain model media while redacting duplicated structured payloads", () =>
+    Effect.gen(function* () {
+      const image = Tool.make({
+        description: "Return an image.",
+        parameters: Schema.Struct({}),
+        success: Schema.Struct({ mime: Schema.String, data: Schema.String }),
+        execute: () => Effect.succeed({ mime: "image/png", data: "AAECAw==" }),
+        toStructuredOutput: (output) => ({ mime: output.mime }),
+        toModelOutput: ({ output }) => [
+          { type: "file", source: { type: "data", data: output.data }, mime: output.mime },
+        ],
+      })
+
+      const dispatched = yield* ToolRuntime.dispatch(
+        { image },
+        LLMEvent.toolCall({ id: "call_image", name: "image", input: {} }),
+      )
+
+      expect(dispatched.output).toEqual({
+        structured: { mime: "image/png" },
+        content: [{ type: "file", source: { type: "data", data: "AAECAw==" }, mime: "image/png" }],
+      })
+    }),
+  )
+
   it.effect("models canonical tool files with explicit data, url, and file sources", () =>
     Effect.sync(() => {
       const decode = Schema.decodeUnknownSync(ToolContent)