fix(ai): classify malformed Responses tool calls (#40549)

Co-authored-by: Aiden Cline <rekram1-node@users.noreply.github.com>

packages/ai/src/protocols/open-responses.ts

@@ -264,6 +264,7 @@ export interface ParserState {
   readonly providerMetadataKey: string
   readonly tools: ToolStream.State<string>
   readonly hasFunctionCall: boolean
+  readonly hasFunctionCallError: boolean
   readonly lifecycle: Lifecycle.State
   readonly messageItems: ReadonlySet<string>
   readonly messagePhase: (value: unknown) => MessagePhase | null | undefined
@@ -608,15 +609,16 @@ const mapUsage = (usage: OpenResponsesUsage | null | undefined, providerMetadata
   })
 }
 
-const mapFinishReason = (event: Event, hasFunctionCall: boolean): FinishReason => {
+const mapFinishReason = (event: Event, hasFunctionCall: boolean, hasFunctionCallError: boolean): FinishReason => {
   const reason = event.response?.incomplete_details?.reason
+  if (reason === "max_output_tokens") return "length"
+  if (reason === "content_filter") return "content-filter"
+  if (hasFunctionCallError) return "error"
   if (reason === undefined || reason === null) {
     if (hasFunctionCall) return "tool-calls"
     if (event.type === "response.incomplete") return "unknown"
     return "stop"
   }
-  if (reason === "max_output_tokens") return "length"
-  if (reason === "content_filter") return "content-filter"
   return hasFunctionCall ? "tool-calls" : "unknown"
 }
 
@@ -896,9 +898,8 @@ const onOutputItemDone = Effect.fn("OpenResponses.onOutputItemDone")(function* (
       {
         ...state,
         lifecycle,
-        hasFunctionCall:
-          resultEvents.some((event) => LLMEvent.is.toolCall(event) || LLMEvent.is.toolInputError(event)) ||
-          state.hasFunctionCall,
+        hasFunctionCall: resultEvents.some(LLMEvent.is.toolCall) || state.hasFunctionCall,
+        hasFunctionCallError: resultEvents.some(LLMEvent.is.toolInputError) || state.hasFunctionCallError,
         tools: result.tools,
       },
       events,
@@ -938,7 +939,7 @@ const onResponseFinish = (state: ParserState, event: Event): StepResult => {
   const events: LLMEvent[] = []
   const lifecycle = Lifecycle.finish(state.lifecycle, events, {
     reason: {
-      normalized: mapFinishReason(event, state.hasFunctionCall),
+      normalized: mapFinishReason(event, state.hasFunctionCall, state.hasFunctionCallError),
       raw: event.response?.incomplete_details?.reason,
     },
     usage: mapUsage(event.response?.usage, state.providerMetadataKey),
@@ -1031,6 +1032,7 @@ export const initial = (request: LLMRequest, extension: Extension = BASE): Parse
   name: extension.name,
   providerMetadataKey: request.model.route.providerMetadataKey ?? "openresponses",
   hasFunctionCall: false,
+  hasFunctionCallError: false,
   tools: ToolStream.empty<string>(),
   lifecycle: Lifecycle.initial(),
   messageItems: new Set<string>(),

packages/ai/test/provider/openai-responses.test.ts

@@ -1599,7 +1599,7 @@ describe("OpenAI Responses route", () => {
         name: "lookup",
         raw: '{"query":"partial',
       })
-      expect(response.finishReason.normalized).toBe("tool-calls")
+      expect(response.finishReason.normalized).toBe("error")
       expect(response.events.some(LLMEvent.is.toolCall)).toBeFalse()
     }),
   )
@@ -1626,7 +1626,7 @@ describe("OpenAI Responses route", () => {
         name: "lookup",
         raw: '{"query":"partial',
       })
-      expect(response.finishReason.normalized).toBe("tool-calls")
+      expect(response.finishReason.normalized).toBe("error")
     }),
   )