fix(ai): pass media through protocol lowering (#42921)

Co-authored-by: Aiden Cline <rekram1-node@users.noreply.github.com>

packages/ai/src/protocols/anthropic-messages.ts

@@ -30,7 +30,6 @@ import { ToolSchemaProjection } from "./utils/tool-schema.js"
 import { ToolStream } from "./utils/tool-stream.js"
 
 const ADAPTER = "anthropic-messages"
-const MEDIA_MIMES = new Set<string>([...ProviderShared.IMAGE_MIMES, ...ProviderShared.PDF_MIMES])
 export const DEFAULT_BASE_URL = "https://api.anthropic.com/v1"
 export const PATH = "/messages"
 
@@ -400,7 +399,7 @@ const lowerServerToolResult = Effect.fn("AnthropicMessages.lowerServerToolResult
 })
 
 const lowerMedia = Effect.fn("AnthropicMessages.lowerMedia")(function* (part: MediaPart) {
-  const media = yield* ProviderShared.validateMedia("Anthropic Messages", part, MEDIA_MIMES)
+  const media = ProviderShared.normalizeMedia(part)
   if (media.mime === "application/pdf")
     return {
       type: "document" as const,
@@ -410,6 +409,8 @@ const lowerMedia = Effect.fn("AnthropicMessages.lowerMedia")(function* (part: Me
         data: media.base64,
       },
     } satisfies AnthropicDocumentBlock
+  if (!media.mime.startsWith("image/"))
+    return yield* invalid(`Anthropic Messages does not support media type ${part.mediaType}`)
   return {
     type: "image" as const,
     source: {

packages/ai/src/protocols/gemini.ts

@@ -24,7 +24,6 @@ import { Lifecycle } from "./utils/lifecycle.js"
 import { ToolSchemaProjection } from "./utils/tool-schema.js"
 
 const ADAPTER = "gemini"
-const MEDIA_MIMES = new Set<string>(ProviderShared.MEDIA_MIMES)
 // Google documents this sentinel for replaying Gemini 3 function calls after their original signature was lost.
 const SKIP_THOUGHT_SIGNATURE_VALIDATOR = "skip_thought_signature_validator"
 export const DEFAULT_BASE_URL = "https://generativelanguage.googleapis.com/v1beta"
@@ -248,7 +247,7 @@ const lowerToolConfig = (toolChoice: NonNullable<LLMRequest["toolChoice"]>) =>
 
 const lowerUserPart = Effect.fn("Gemini.lowerUserPart")(function* (part: TextPart | MediaPart) {
   if (part.type === "text") return { text: part.text }
-  const media = yield* ProviderShared.validateMedia("Gemini", part, MEDIA_MIMES)
+  const media = ProviderShared.normalizeMedia(part)
   return { inlineData: { mimeType: media.mime, data: media.base64 } }
 })
 
@@ -353,7 +352,7 @@ const lowerMessages = Effect.fn("Gemini.lowerMessages")(function* (request: LLMR
       const media: GeminiInlineDataPart[] = []
       for (const item of content) {
         if (item.type === "text") continue
-        const value = yield* ProviderShared.validateToolFile("Gemini", item, MEDIA_MIMES)
+        const value = ProviderShared.normalizeToolFile(item)
         media.push({ inlineData: { mimeType: value.mime, data: value.base64 } })
       }
       parts.push({

packages/ai/src/protocols/open-responses.ts

@@ -26,7 +26,6 @@ import { ToolStream } from "./utils/tool-stream.js"
 
 const ADAPTER = "open-responses"
 const NAME = "Open Responses"
-const MEDIA_MIMES = new Set<string>([...ProviderShared.IMAGE_MIMES, ...ProviderShared.PDF_MIMES])
 export const PATH = "/responses"
 
 // =============================================================================
@@ -285,7 +284,7 @@ export interface Extension {
   readonly name: string
   readonly lowerMedia?: (input: {
     readonly part: MediaPart
-    readonly media: ProviderShared.ValidatedMedia
+    readonly media: ProviderShared.NormalizedMedia
     readonly request: LLMRequest
   }) => MediaInput | undefined
   readonly messagePhase?: (value: unknown) => MessagePhase | null | undefined
@@ -380,13 +379,13 @@ const lowerMedia = Effect.fn("OpenResponses.lowerMedia")(function* (
   request: LLMRequest,
   extension: Extension,
 ) {
-  const media = yield* ProviderShared.validateMedia(extension.name, part, MEDIA_MIMES)
+  const media = ProviderShared.normalizeMedia(part)
   const extended = extension.lowerMedia?.({ part, media, request })
   if (extended) return extended
-  if (media.mime === "application/pdf") {
+  if (!media.mime.startsWith("image/")) {
     return {
       type: "input_file" as const,
-      filename: part.filename ?? "document.pdf",
+      filename: part.filename ?? (media.mime === "application/pdf" ? "document.pdf" : "file"),
       file_data: media.dataUrl,
     }
   }

packages/ai/src/protocols/openai-chat.ts

@@ -28,7 +28,6 @@ import { ToolSchemaProjection } from "./utils/tool-schema.js"
 import { ToolStream } from "./utils/tool-stream.js"
 
 const ADAPTER = "openai-chat"
-const IMAGE_MIMES = new Set<string>(ProviderShared.IMAGE_MIMES)
 const RESERVED_REASONING_FIELDS = new Set(["role", "content", "tool_calls"])
 export const DEFAULT_BASE_URL = "https://api.openai.com/v1"
 export const PATH = "/chat/completions"
@@ -284,7 +283,9 @@ const lowerToolCall = (part: ToolCallPart): OpenAIChatAssistantToolCall => ({
 })
 
 const lowerMedia = Effect.fn("OpenAIChat.lowerMedia")(function* (part: MediaPart) {
-  const media = yield* ProviderShared.validateMedia("OpenAI Chat", part, IMAGE_MIMES)
+  const media = ProviderShared.normalizeMedia(part)
+  if (!media.mime.startsWith("image/"))
+    return yield* ProviderShared.invalidRequest(`OpenAI Chat does not support media type ${part.mediaType}`)
   return { type: "image_url" as const, image_url: { url: media.dataUrl } }
 })
 

packages/ai/src/protocols/shared.ts

@@ -155,59 +155,24 @@ export const wrappedSystemUpdate = Effect.fn("ProviderShared.wrappedSystemUpdate
 export const parseToolInput = (route: string, name: string, raw: string) =>
   parseJson(route, raw || "{}", `Invalid JSON input for ${route} tool call ${name}`)
 
-export const IMAGE_MIMES = ["image/png", "image/jpeg", "image/gif", "image/webp"] as const
-export const VIDEO_MIMES = ["video/mp4", "video/webm", "video/quicktime"] as const
-export const AUDIO_MIMES = ["audio/wav", "audio/mp3", "audio/aiff", "audio/aac", "audio/ogg", "audio/flac"] as const
-export const PDF_MIMES = ["application/pdf"] as const
-export const MEDIA_MIMES = [...IMAGE_MIMES, ...VIDEO_MIMES, ...AUDIO_MIMES, ...PDF_MIMES] as const
-export const MAX_MEDIA_ENCODED_BYTES = 28 * 1024 * 1024
-export const MAX_MEDIA_DECODED_BYTES = 20 * 1024 * 1024
-
-const base64Pattern = /^(?:[A-Za-z0-9+/]{4})*(?:[A-Za-z0-9+/]{2}==|[A-Za-z0-9+/]{3}=)?$/
-
-export interface ValidatedMedia {
+export interface NormalizedMedia {
   readonly mime: string
   readonly base64: string
   readonly dataUrl: string
-  readonly bytes: Uint8Array
 }
 
-export const validateMedia = Effect.fn("ProviderShared.validateMedia")(function* (
-  route: string,
-  part: MediaPart,
-  supportedMimes: ReadonlySet<string>,
-) {
+export const normalizeMedia = (part: MediaPart): NormalizedMedia => {
   const mime = part.mediaType.toLowerCase()
-  if (!supportedMimes.has(mime)) return yield* invalidRequest(`${route} does not support media type ${part.mediaType}`)
-
-  let base64: string
   if (typeof part.data !== "string") {
-    if (part.data.byteLength > MAX_MEDIA_DECODED_BYTES)
-      return yield* invalidRequest(`${route} media exceeds the ${MAX_MEDIA_DECODED_BYTES} byte decoded limit`)
-    base64 = Buffer.from(part.data).toString("base64")
-  } else if (part.data.startsWith("data:")) {
-    const match = /^data:([^;,]+);base64,([A-Za-z0-9+/]*={0,2})$/s.exec(part.data)
-    if (!match) return yield* invalidRequest(`${route} media data URL must contain valid base64`)
-    if (match[1]!.toLowerCase() !== mime)
-      return yield* invalidRequest(`${route} media type ${part.mediaType} does not match data URL type ${match[1]}`)
-    base64 = match[2]!
-  } else {
-    base64 = part.data
+    const base64 = Buffer.from(part.data).toString("base64")
+    return { mime, base64, dataUrl: `data:${mime};base64,${base64}` }
   }
+  if (!part.data.startsWith("data:")) return { mime, base64: part.data, dataUrl: `data:${mime};base64,${part.data}` }
+  return { mime, base64: part.data.slice(part.data.indexOf(",") + 1), dataUrl: part.data }
+}
 
-  if (Buffer.byteLength(base64, "utf8") > MAX_MEDIA_ENCODED_BYTES)
-    return yield* invalidRequest(`${route} media exceeds the ${MAX_MEDIA_ENCODED_BYTES} byte encoded limit`)
-  if (!base64 || base64.length % 4 !== 0 || !base64Pattern.test(base64))
-    return yield* invalidRequest(`${route} media must contain valid base64`)
-  const bytes = Buffer.from(base64, "base64")
-  if (bytes.byteLength > MAX_MEDIA_DECODED_BYTES)
-    return yield* invalidRequest(`${route} media exceeds the ${MAX_MEDIA_DECODED_BYTES} byte decoded limit`)
-  if (bytes.toString("base64") !== base64) return yield* invalidRequest(`${route} media must contain canonical base64`)
-  return { mime, base64, dataUrl: `data:${mime};base64,${base64}`, bytes } satisfies ValidatedMedia
-})
-
-export const validateToolFile = (route: string, part: Tool.FileContent, supportedMimes: ReadonlySet<string>) =>
-  validateMedia(route, { type: "media", mediaType: part.mime, data: part.uri, filename: part.name }, supportedMimes)
+export const normalizeToolFile = (part: Tool.FileContent) =>
+  normalizeMedia({ type: "media", mediaType: part.mime, data: part.uri, filename: part.name })
 
 export const trimBaseUrl = (value: string) => value.replace(/\/+$/, "")
 

packages/ai/src/protocols/utils/bedrock-media.ts

@@ -66,11 +66,7 @@ export const lower = Effect.fn("BedrockMedia.lower")(function* (part: MediaPart)
   const mime = part.mediaType.toLowerCase()
   const imageFormat = IMAGE_FORMATS[mime as keyof typeof IMAGE_FORMATS]
   if (imageFormat) {
-    const media = yield* ProviderShared.validateMedia(
-      "Bedrock Converse",
-      part,
-      new Set<string>(Object.keys(IMAGE_FORMATS)),
-    )
+    const media = ProviderShared.normalizeMedia(part)
     return { image: { format: imageFormat, source: { bytes: media.base64 } } } satisfies ImageBlock
   }
   if (mime.startsWith("image/"))
@@ -79,11 +75,7 @@ export const lower = Effect.fn("BedrockMedia.lower")(function* (part: MediaPart)
   if (documentFormat) {
     if (!part.filename)
       return yield* ProviderShared.invalidRequest("Bedrock Converse document media requires a filename")
-    const media = yield* ProviderShared.validateMedia(
-      "Bedrock Converse",
-      part,
-      new Set<string>(Object.keys(DOCUMENT_FORMATS)),
-    )
+    const media = ProviderShared.normalizeMedia(part)
     return documentBlock(part.filename, documentFormat, media.base64)
   }
   return yield* ProviderShared.invalidRequest(`Bedrock Converse does not support media type ${part.mediaType}`)

packages/ai/test/provider/anthropic-messages.test.ts

@@ -399,7 +399,7 @@ describe("Anthropic Messages route", () => {
     }),
   )
 
-  it.effect("rejects unsupported media in tool-result content with a clear error", () =>
+  it.effect("rejects tool-result media that cannot be lowered", () =>
     Effect.gen(function* () {
       const error = yield* compileRequest(
         LLM.request({
@@ -418,8 +418,7 @@ describe("Anthropic Messages route", () => {
         }),
       ).pipe(Effect.flip)
 
-      expect(error.message).toContain("Anthropic Messages")
-      expect(error.message).toContain("audio/mpeg")
+      expect(error.message).toContain("Anthropic Messages does not support media type audio/mpeg")
     }),
   )
 

packages/ai/test/provider/gemini.test.ts

@@ -4,7 +4,6 @@ import { LLM, AIError, LLMRequest, Message, ToolCallPart, ToolDefinition, Usage
 import { Auth, LLMClient } from "../../src/route.js"
 import { compileRequest } from "../../src/route/client.js"
 import * as Gemini from "../../src/protocols/gemini.js"
-import { ProviderShared } from "../../src/protocols/shared.js"
 import { it } from "../lib/effect.js"
 import { fixedResponse } from "../lib/http.js"
 import { sseEvents, sseRaw } from "../lib/sse.js"
@@ -291,35 +290,30 @@ describe("Gemini route", () => {
     }),
   )
 
-  for (const [name, media] of [
-    ["mismatched data URL MIME", { mediaType: "image/png", data: "data:image/jpeg;base64,/9j/" }],
-    ["malformed base64", { mediaType: "image/png", data: "%%%=" }],
-    ["unsupported SVG", { mediaType: "image/svg+xml", data: "PHN2Zz4=" }],
-  ] as const)
-    it.effect(`rejects ${name}`, () =>
-      Effect.gen(function* () {
-        const error = yield* compileRequest(
-          LLM.request({ model, messages: [Message.user({ type: "media", ...media })] }),
-        ).pipe(Effect.flip)
-        expect(error.message).toMatch(/does not support|does not match|valid base64/)
-      }),
-    )
-
-  it.effect("rejects oversized image input", () =>
+  it.effect("passes encoded media through without local validation", () =>
     Effect.gen(function* () {
-      const error = yield* compileRequest(
+      const prepared = yield* compileRequest(
         LLM.request({
           model,
           messages: [
-            Message.user({
-              type: "media",
-              mediaType: "image/png",
-              data: "A".repeat(ProviderShared.MAX_MEDIA_ENCODED_BYTES + 4),
-            }),
+            Message.user([
+              { type: "media", mediaType: "image/png", data: "%%%=" },
+              { type: "media", mediaType: "image/png", data: "data:image/jpeg;base64,/9j/" },
+              { type: "media", mediaType: "image/svg+xml", data: "PHN2Zz4=" },
+            ]),
           ],
         }),
-      ).pipe(Effect.flip)
-      expect(error.message).toContain("encoded limit")
+      )
+      expect(prepared.body.contents).toEqual([
+        {
+          role: "user",
+          parts: [
+            { inlineData: { mimeType: "image/png", data: "%%%=" } },
+            { inlineData: { mimeType: "image/png", data: "/9j/" } },
+            { inlineData: { mimeType: "image/svg+xml", data: "PHN2Zz4=" } },
+          ],
+        },
+      ])
     }),
   )
 

packages/ai/test/provider/openai-chat.test.ts

@@ -527,35 +527,42 @@ describe("OpenAI Chat route", () => {
     }),
   )
 
-  for (const [name, media] of [
-    ["mismatched data URL MIME", { mediaType: "image/png", data: "data:image/jpeg;base64,/9j/" }],
-    ["malformed base64", { mediaType: "image/png", data: "not-base64" }],
-    ["unsupported SVG", { mediaType: "image/svg+xml", data: "PHN2Zz4=" }],
-  ] as const)
-    it.effect(`rejects ${name}`, () =>
-      Effect.gen(function* () {
-        const error = yield* compileRequest(
-          LLM.request({ model, messages: [Message.user({ type: "media", ...media })] }),
-        ).pipe(Effect.flip)
-        expect(error.message).toMatch(/does not support|does not match|valid base64/)
-      }),
-    )
-
-  it.effect("rejects oversized image input", () =>
+  it.effect("passes encoded image media through without local validation", () =>
     Effect.gen(function* () {
-      const error = yield* compileRequest(
+      const prepared = yield* compileRequest(
         LLM.request({
           model,
           messages: [
-            Message.user({
-              type: "media",
-              mediaType: "image/png",
-              data: "A".repeat(ProviderShared.MAX_MEDIA_ENCODED_BYTES + 4),
-            }),
+            Message.user([
+              { type: "media", mediaType: "image/png", data: "not-base64" },
+              { type: "media", mediaType: "image/png", data: "data:image/jpeg;base64,/9j/" },
+              { type: "media", mediaType: "image/svg+xml", data: "PHN2Zz4=" },
+            ]),
+          ],
+        }),
+      )
+      expect(prepared.body.messages).toEqual([
+        {
+          role: "user",
+          content: [
+            { type: "image_url", image_url: { url: "data:image/png;base64,not-base64" } },
+            { type: "image_url", image_url: { url: "data:image/jpeg;base64,/9j/" } },
+            { type: "image_url", image_url: { url: "data:image/svg+xml;base64,PHN2Zz4=" } },
           ],
+        },
+      ])
+    }),
+  )
+
+  it.effect("rejects non-image media that cannot be lowered", () =>
+    Effect.gen(function* () {
+      const error = yield* compileRequest(
+        LLM.request({
+          model,
+          messages: [Message.user({ type: "media", mediaType: "audio/mpeg", data: "AAECAw==" })],
         }),
       ).pipe(Effect.flip)
-      expect(error.message).toContain("encoded limit")
+      expect(error.message).toContain("OpenAI Chat does not support media type audio/mpeg")
     }),
   )
 

packages/ai/test/provider/openai-responses.test.ts

@@ -1149,6 +1149,32 @@ describe("OpenAI Responses route", () => {
     }),
   )
 
+  it.effect("passes large PDF tool-result content through", () =>
+    Effect.gen(function* () {
+      const base64 = "A".repeat(8_125_844)
+      const dataUrl = `data:application/pdf;base64,${base64}`
+      const prepared = yield* compileRequest(
+        LLM.request({
+          id: "req_tool_result_large_pdf",
+          model,
+          messages: [
+            Message.assistant([ToolCallPart.make({ id: "call_1", name: "read", input: {} })]),
+            Message.tool({
+              id: "call_1",
+              name: "read",
+              resultType: "content",
+              result: [{ type: "file", uri: dataUrl, mime: "application/pdf", name: "report.pdf" }],
+            }),
+          ],
+        }),
+      )
+
+      expect(expectToolOutput(prepared.body).output).toEqual([
+        { type: "input_file", filename: "report.pdf", file_data: dataUrl },
+      ])
+    }),
+  )
+
   it.effect("uses xAI inline file encoding for PDF tool results", () =>
     Effect.gen(function* () {
       const prepared = yield* compileRequest(
@@ -1184,9 +1210,9 @@ describe("OpenAI Responses route", () => {
     }),
   )
 
-  it.effect("rejects unsupported media in tool-result content with a clear error", () =>
+  it.effect("passes non-image tool-result content through as an input file", () =>
     Effect.gen(function* () {
-      const error = yield* compileRequest(
+      const prepared = yield* compileRequest(
         LLM.request({
           id: "req_tool_result_unsupported_media",
           model,
@@ -1200,10 +1226,11 @@ describe("OpenAI Responses route", () => {
             }),
           ],
         }),
-      ).pipe(Effect.flip)
+      )
 
-      expect(error.message).toContain("OpenAI Responses")
-      expect(error.message).toContain("audio/mpeg")
+      expect(expectToolOutput(prepared.body).output).toEqual([
+        { type: "input_file", filename: "file", file_data: "data:audio/mpeg;base64,AAECAw==" },
+      ])
     }),
   )
 
@@ -2394,17 +2421,28 @@ describe("OpenAI Responses route", () => {
     }),
   )
 
-  it.effect("rejects unsupported user media content", () =>
+  it.effect("passes non-image user media through as an input file", () =>
     Effect.gen(function* () {
-      const error = yield* compileRequest(
+      const prepared = yield* compileRequest(
         LLM.request({
           id: "req_media",
           model,
           messages: [Message.user({ type: "media", mediaType: "application/x-tar", data: "AAECAw==" })],
         }),
-      ).pipe(Effect.flip)
+      )
 
-      expect(error.message).toContain("OpenAI Responses does not support media type application/x-tar")
+      expect(prepared.body.input).toEqual([
+        {
+          role: "user",
+          content: [
+            {
+              type: "input_file",
+              filename: "file",
+              file_data: "data:application/x-tar;base64,AAECAw==",
+            },
+          ],
+        },
+      ])
     }),
   )