fix(core): preserve tool-result images for AI SDK providers (#42849)

Co-authored-by: Aiden Cline <rekram1-node@users.noreply.github.com>
Co-authored-by: Aiden Cline <63023139+rekram1-node@users.noreply.github.com>

packages/core/src/aisdk.ts

@@ -435,7 +435,22 @@ function prompt(request: LLMRequest): LanguageModelV3Prompt {
     .map((part) => part.text)
     .filter(Boolean)
     .join("\n\n")
-  const messages = request.messages.flatMap(message)
+  const pending: UserContent = []
+  const messages = request.messages.flatMap((input, index) => {
+    if (input.role !== "tool") return message(input)
+    const lowered = toolMessage(input)
+    pending.push(...lowered.media)
+    if (request.messages[index + 1]?.role === "tool" || pending.length === 0) return lowered.messages
+    const media = [...pending]
+    pending.length = 0
+    return [
+      ...lowered.messages,
+      {
+        role: "user" as const,
+        content: [{ type: "text" as const, text: "Attached media from tool result:" }, ...media],
+      },
+    ]
+  })
   if (!system.length) return messages
   return [{ role: "system", content: system }, ...messages]
 }
@@ -448,10 +463,33 @@ function message(input: LLMRequest["messages"][number]): LanguageModelV3Message[
       return [{ role: "user", content: input.content.flatMap(userPart) }]
     case "assistant":
       return [{ role: "assistant", content: input.content.flatMap(assistantPart) }]
-    case "tool": {
-      const content = input.content.flatMap(toolResultPart)
-      return content.length ? [{ role: "tool", content }] : []
-    }
+    case "tool":
+      return toolMessage(input).messages
+  }
+}
+
+function toolMessage(input: LLMRequest["messages"][number]) {
+  const media: UserContent = []
+  const content = input.content.flatMap((part) => {
+    if (part.type !== "tool-result" || part.result.type !== "content") return toolResultPart(part)
+    const value = part.result.value.filter((item) => {
+      if (item.type !== "file") return true
+      if (!item.mime.startsWith("image/") && item.mime !== "application/pdf") return true
+      const data = /^data:[^;,]+(?:;[^,]*)*;base64,(.*)$/s.exec(item.uri)?.[1] ?? item.uri
+      media.push({ type: "file", mediaType: item.mime, data, filename: item.name })
+      return false
+    })
+    return toolResultPart({
+      ...part,
+      result:
+        value.length === 0
+          ? { type: "text", value: "Media attached in the following user message." }
+          : { ...part.result, value },
+    })
+  })
+  return {
+    messages: content.length ? ([{ role: "tool", content }] satisfies LanguageModelV3Message[]) : [],
+    media,
   }
 }
 

packages/core/test/aisdk.test.ts

@@ -1,5 +1,6 @@
 import { APICallError } from "@ai-sdk/provider"
 import type { LanguageModelV3, LanguageModelV3StreamPart } from "@ai-sdk/provider"
+import { createMistral } from "@ai-sdk/mistral"
 import { AISDK } from "@opencode-ai/core/aisdk"
 import { SessionRunnerRetry } from "@opencode-ai/core/session/runner/retry"
 import { toSessionError } from "@opencode-ai/core/session/to-session-error"
@@ -277,69 +278,90 @@ it.effect("projects replay metadata onto AI SDK prompt parts", () =>
   }),
 )
 
-it.effect("preserves tool result content in AI SDK prompts", () =>
+it.effect("moves a tool image through the real Mistral provider as a user message", () =>
   Effect.gen(function* () {
     const aisdk = yield* AISDK.Service
+    let body: { messages?: unknown[] } | undefined
+    const mockFetch = Object.assign(
+      async (_input: Parameters<typeof fetch>[0], init?: RequestInit) => {
+        body = JSON.parse(String(init?.body))
+        const chunks = [
+          {
+            id: "response-1",
+            created: 0,
+            model: "pixtral-large-latest",
+            choices: [{ index: 0, delta: { content: [{ type: "text", text: "I see it." }] } }],
+          },
+          {
+            id: "response-1",
+            created: 0,
+            model: "pixtral-large-latest",
+            choices: [{ index: 0, delta: {}, finish_reason: "stop" }],
+            usage: { prompt_tokens: 1, completion_tokens: 1, total_tokens: 2 },
+          },
+        ]
+        return new Response(chunks.map((chunk) => `data: ${JSON.stringify(chunk)}\n\n`).join(""), {
+          headers: { "Content-Type": "text/event-stream" },
+        })
+      },
+      { preconnect: fetch.preconnect },
+    )
     yield* aisdk.hook.sdk((event) => {
-      event.sdk = { languageModel: () => ({ provider: event.model.providerID }) }
+      event.sdk = createMistral({ apiKey: "test", fetch: mockFetch })
     })
 
-    const resolved = yield* aisdk.model(model("test-ai-sdk"))
-    const prepared = yield* compileRequest(
+    const resolved = yield* aisdk.model({
+      ...model("@ai-sdk/mistral"),
+      modelID: Model.ID.make("pixtral-large-latest"),
+    })
+    yield* LLMClient.generate(
       LLM.request({
         model: resolved,
         messages: [
+          Message.user("Inspect the screenshot."),
+          Message.assistant({ type: "tool-call", id: "call_1", name: "screenshot", input: {} }),
           Message.tool({
+            type: "tool-result",
             id: "call_1",
-            name: "read",
+            name: "screenshot",
             result: {
               type: "content",
               value: [
-                { type: "text", text: "attachments" },
-                { type: "file", uri: "data:image/png;base64,AAAA", mime: "image/png", name: "pixel.png" },
-                {
-                  type: "file",
-                  uri: "data:application/pdf;charset=utf-8;base64,JVBERg==",
-                  mime: "application/pdf",
-                  name: "document.pdf",
-                },
-                { type: "file", uri: "data:audio/mpeg;base64,SUQz", mime: "audio/mpeg", name: "clip.mp3" },
-                { type: "file", uri: "https://example.com/pixel.png", mime: "image/png" },
-                { type: "file", uri: "https://example.com/document.pdf", mime: "application/pdf" },
+                { type: "text", text: "Screenshot captured" },
+                { type: "file", uri: "data:image/png;base64,AAAA", mime: "image/png", name: "screen.png" },
               ],
             },
           }),
         ],
       }),
-    )
+    ).pipe(Effect.provide(client))
 
-    expect(prepared.body.prompt).toEqual([
+    expect(body?.messages).toEqual([
+      { role: "user", content: [{ type: "text", text: "Inspect the screenshot." }] },
       {
-        role: "tool",
-        content: [
+        role: "assistant",
+        content: "",
+        tool_calls: [
           {
-            type: "tool-result",
-            toolCallId: "call_1",
-            toolName: "read",
-            output: {
-              type: "content",
-              value: [
-                { type: "text", text: "attachments" },
-                { type: "image-data", data: "AAAA", mediaType: "image/png" },
-                {
-                  type: "file-data",
-                  data: "JVBERg==",
-                  mediaType: "application/pdf",
-                  filename: "document.pdf",
-                },
-                { type: "file-data", data: "SUQz", mediaType: "audio/mpeg", filename: "clip.mp3" },
-                { type: "image-url", url: "https://example.com/pixel.png" },
-                { type: "file-url", url: "https://example.com/document.pdf" },
-              ],
-            },
+            id: "call_1",
+            type: "function",
+            function: { name: "screenshot", arguments: "{}" },
           },
         ],
       },
+      {
+        role: "tool",
+        name: "screenshot",
+        tool_call_id: "call_1",
+        content: '[{"type":"text","text":"Screenshot captured"}]',
+      },
+      {
+        role: "user",
+        content: [
+          { type: "text", text: "Attached media from tool result:" },
+          { type: "image_url", image_url: "data:image/png;base64,AAAA" },
+        ],
+      },
     ])
   }),
 )