fix(aikit): keep images from error tool results

packages/aikit/src/llm/registry.ts

@@ -83,13 +83,18 @@ function defaultMethodForModel(model: Model.Info): Model.APIMethodEnum {
 	return Model.APIMethodEnum.languageModel;
 }
 
+/** The endpoint method a model is called through: the request's override, the catalog's, or the default. */
+export function resolveMethod(model: Model.Info, methodOverride?: Model.APIMethodEnum): Model.APIMethodEnum {
+	return methodOverride ?? model.api?.method ?? defaultMethodForModel(model);
+}
+
 export function resolveLanguageModel(
 	provider: unknown,
 	model: Model.Info,
 	methodOverride?: Model.APIMethodEnum,
 ): LanguageModel {
 	const modelId = model.api?.id ?? model.id;
-	const method = methodOverride ?? model.api?.method ?? defaultMethodForModel(model);
+	const method = resolveMethod(model, methodOverride);
 	const callable = provider as ProviderWithMethods;
 
 	if (method === Model.APIMethodEnum.responses && callable.responses) return callable.responses(modelId);

packages/aikit/src/llm/stream.ts

@@ -403,7 +403,7 @@ export const stream: Protocol.StreamFunction<Model.KnownProviderEnum, typeof Opt
 			const plan = Thinking.resolvePlan(model, context, runtimeOptions);
 			const languageModel = await resolveAISDKLanguageModel(model, runtimeOptions, plan.budget);
 			const tools = convertTools(context.tools, model);
-			const messages = convertMessages(context, model);
+			const messages = convertMessages(context, model, runtimeOptions.method);
 			const providerOptions = resolveProviderOptions(model, runtimeOptions, plan) as Parameters<
 				typeof streamText<ToolSet>
 			>[0]["providerOptions"];

packages/aikit/src/llm/transform.ts

@@ -16,7 +16,9 @@ import { resolveOpenAICodexToolConstraint } from "../providers/openai-codex/code
 import { shortHash } from "../utils/hash.ts";
 import { parseStreamingJson } from "../utils/jsonparse.ts";
 import { sanitizeSurrogates } from "../utils/sanitize.ts";
+import { packageForModel, resolveMethod } from "./registry.ts";
 import { resolveStrictParameters } from "./strict.ts";
+import { isAnthropicMessagesModel } from "./thinking.ts";
 
 type ToolCallPart = Extract<Message.AssistantMessage["parts"][number], { type: "toolCall" }>;
 type TerminalToolCall = Exclude<ToolCallPart, Message.ToolCallPendingPart | Message.ToolCallRunningPart>;
@@ -82,14 +84,78 @@ function sanitizeRecord(value: Record<string, unknown>): Record<string, unknown>
 	return sanitizeValue(value) as Record<string, unknown>;
 }
 
-function toolResultOutput(toolCall: TerminalToolCall) {
-	const text = toolCall.result.content
+const ERROR_TOOL_IMAGE_PLACEHOLDER = "(image attached after the tool results)";
+const MOVED_TOOL_IMAGE_PLACEHOLDER = "(see attached image)";
+const CHAT_TOOL_IMAGES_CAPTION = "Attached image(s) from tool result:";
+const GEMINI_TOOL_IMAGES_CAPTION = "Tool result image:";
+
+/**
+ * Where a tool result's images go when the wire cannot carry them inside the result. Each case
+ * sends them in a user message after the turn's tool results:
+ * - `anthropicError`: Anthropic rejects images in error tool results.
+ * - `chat`: Chat Completions adapters serialize tool result images as JSON text.
+ * - `gemini`: before Gemini 3 (and for non-Gemini models), an image-only result loses its function response.
+ */
+type MovedImages = "anthropicError" | "chat" | "gemini";
+
+const COPILOT = "@codeworksh/ai-sdk-github-copilot";
+
+function isAnthropicWire(model: Model.Info, method: Model.APIMethodEnum): boolean {
+	if (packageForModel(model) === COPILOT) return method === Model.APIMethodEnum.messages;
+	return isAnthropicMessagesModel(model);
+}
+
+function isChatCompletions(model: Model.Info, method: Model.APIMethodEnum): boolean {
+	const npm = packageForModel(model);
+	// The compatible adapter falls back to its chat model for every other method.
+	if (npm === "@ai-sdk/openai-compatible") return true;
+	if (npm === COPILOT) return method !== Model.APIMethodEnum.responses && method !== Model.APIMethodEnum.messages;
+	return npm === "@ai-sdk/openai" && method === Model.APIMethodEnum.chat;
+}
+
+/** The Google adapter's own split: anything but Gemini 3 and later gets the legacy function response. */
+function isLegacyGoogleToolResults(model: Model.Info): boolean {
+	if (model.protocol !== Model.KnownProviderEnum.google && model.protocol !== Model.KnownProviderEnum.googleVertex) {
+		return false;
+	}
+	const id = model.api?.id ?? model.id;
+	const gemini = /(^|[/.])gemini-/i.test(id);
+	const legacy =
+		/(^|[/.])gemini-[12](?:[.-]|$)/i.test(id) ||
+		/(^|[/.])gemini-pro(?:-vision)?$/i.test(id) ||
+		/(^|[/.])gemini-robotics-er-1\.5(?:[.-]|$)/i.test(id);
+	return !gemini || legacy;
+}
+
+function movedImages(
+	toolCall: TerminalToolCall,
+	model: Model.Info,
+	method: Model.APIMethodEnum,
+): MovedImages | undefined {
+	if (!toolCall.result.content.some((content) => content.type === "image")) return;
+	if (isAnthropicWire(model, method)) return toolCall.result.isError ? "anthropicError" : undefined;
+	if (isChatCompletions(model, method)) return "chat";
+	if (isLegacyGoogleToolResults(model)) return "gemini";
+}
+
+function toolResultOutput(toolCall: TerminalToolCall, moved: MovedImages | undefined) {
+	const content =
+		moved === "anthropicError"
+			? Message.omitImages(toolCall.result.content, ERROR_TOOL_IMAGE_PLACEHOLDER)
+			: moved
+				? toolCall.result.content.filter((content) => content.type === "text")
+				: toolCall.result.content;
+	const joined = content
 		.filter((content) => content.type === "text")
 		.map((content) => sanitizeSurrogates(content.text))
 		.join("\n");
-	const images = toolCall.result.content.filter((content) => content.type === "image");
+	const text =
+		moved && moved !== "anthropicError" && joined.trim().length === 0 ? MOVED_TOOL_IMAGE_PLACEHOLDER : joined;
+	const images = content.filter((content) => content.type === "image");
 
-	if (toolCall.result.isError) {
+	// The AI SDK's content output has no error flag, so other providers keep the images and lose
+	// the flag: Google drops its `error` key, OpenAI-family wires have no flag to lose.
+	if (toolCall.result.isError && images.length === 0) {
 		return {
 			type: "error-text" as const,
 			value: text || "tool returned an error",
@@ -210,11 +276,16 @@ export function googleThoughtSignature(metadata: unknown): string | undefined {
 	}
 }
 
-function assistantMessages(message: Message.AssistantMessage, model: Model.Info): ModelMessage[] {
+function assistantMessages(
+	message: Message.AssistantMessage,
+	model: Model.Info,
+	method: Model.APIMethodEnum,
+): ModelMessage[] {
 	if (message.stopReason === "error" || message.stopReason === "aborted") return [];
 
 	const assistantContent: Exclude<AssistantModelMessage["content"], string> = [];
 	const toolResults: ToolModelMessage["content"] = [];
+	const movedImageContent: Exclude<UserModelMessage["content"], string> = [];
 	const googleReplay =
 		(model.protocol === "google" || model.protocol === "google-vertex") &&
 		message.protocol === model.protocol &&
@@ -322,6 +393,7 @@ function assistantMessages(message: Message.AssistantMessage, model: Model.Info)
 		});
 
 		const terminal = part.status === "pending" || part.status === "running" ? undefined : part;
+		const moved = terminal ? movedImages(terminal, model, method) : undefined;
 		toolResults.push({
 			type: "tool-result",
 			toolCallId: part.callID,
@@ -330,12 +402,30 @@ function assistantMessages(message: Message.AssistantMessage, model: Model.Info)
 				? { providerOptions: { "openai-codex": { addedToolNames: terminal.addedToolNames } } }
 				: {}),
 			output: terminal
-				? toolResultOutput(terminal)
+				? toolResultOutput(terminal, moved)
 				: {
 						type: "error-text",
 						value: "no result provided",
 					},
 		});
+		if (terminal && moved) {
+			const caption =
+				moved === "anthropicError"
+					? `Images from the error result of tool call ${part.callID} (${part.name}):`
+					: moved === "gemini"
+						? GEMINI_TOOL_IMAGES_CAPTION
+						: movedImageContent.length === 0
+							? CHAT_TOOL_IMAGES_CAPTION
+							: undefined;
+			movedImageContent.push(
+				...(caption === undefined ? [] : [{ type: "text" as const, text: caption }]),
+				...terminal.result.content.flatMap((content) =>
+					content.type === "image"
+						? [{ type: "file" as const, data: content.data, mediaType: content.mimeType }]
+						: [],
+				),
+			);
+		}
 	}
 
 	const result: ModelMessage[] = [];
@@ -345,10 +435,19 @@ function assistantMessages(message: Message.AssistantMessage, model: Model.Info)
 	if (toolResults.length > 0) {
 		result.push({ role: "tool", content: toolResults });
 	}
+	// @ai-sdk/anthropic merges this into the tool results' user message, after the tool_result blocks.
+	if (movedImageContent.length > 0) {
+		result.push({ role: "user", content: movedImageContent });
+	}
 	return result;
 }
 
-export function convertMessages(context: Message.Context, model: Model.Info): ModelMessage[] {
+export function convertMessages(
+	context: Message.Context,
+	model: Model.Info,
+	methodOverride?: Model.APIMethodEnum,
+): ModelMessage[] {
+	const method = resolveMethod(model, methodOverride);
 	const messages: ModelMessage[] = [];
 	const transformedMessages = Message.transformMessages(context.messages, model, toolCallIdNormalizer(model));
 
@@ -357,7 +456,7 @@ export function convertMessages(context: Message.Context, model: Model.Info): Mo
 			messages.push(...userContent(msg.parts));
 			continue;
 		}
-		messages.push(...assistantMessages(msg, model));
+		messages.push(...assistantMessages(msg, model, method));
 	}
 
 	return messages;

packages/aikit/src/message/message.ts

@@ -323,7 +323,7 @@ const USER_IMAGE_PLACEHOLDER = "(image omitted: model does not support images)";
 const TOOL_IMAGE_PLACEHOLDER = "(tool image omitted: model does not support images)";
 
 // Consecutive images collapse into a single placeholder.
-function omitImages(content: (TextContent | ImageContent)[], placeholder: string): TextContent[] {
+export function omitImages(content: (TextContent | ImageContent)[], placeholder: string): TextContent[] {
 	const result: TextContent[] = [];
 	for (const [index, block] of content.entries()) {
 		if (block.type === "text") result.push(block);

packages/aikit/test/e2e/__artifacts__/error-tool-image.anthropic.json

@@ -0,0 +1,25 @@
+[
+	{
+		"role": "user",
+		"content": [
+			{
+				"type": "tool_result",
+				"tool_use_id": "<call-id-0>",
+				"content": "Screenshot assertion failed: the page did not match.\n(image attached after the tool results)",
+				"is_error": true
+			},
+			{
+				"type": "text",
+				"text": "Images from the error result of tool call <call-id-0> (check_screenshot):"
+			},
+			{
+				"type": "image",
+				"source": {
+					"type": "base64",
+					"media_type": "image/png",
+					"data": "<red-circle.png>"
+				}
+			}
+		]
+	}
+]

packages/aikit/test/e2e/__artifacts__/error-tool-image.google-gemini-2.json

@@ -0,0 +1,31 @@
+[
+	{
+		"role": "user",
+		"parts": [
+			{
+				"functionResponse": {
+					"id": "<call-id-0>",
+					"name": "check_screenshot",
+					"response": {
+						"name": "check_screenshot",
+						"error": "Screenshot assertion failed: the page did not match."
+					}
+				}
+			}
+		]
+	},
+	{
+		"role": "user",
+		"parts": [
+			{
+				"text": "Tool result image:"
+			},
+			{
+				"inlineData": {
+					"mimeType": "image/png",
+					"data": "<red-circle.png>"
+				}
+			}
+		]
+	}
+]

packages/aikit/test/e2e/__artifacts__/error-tool-image.google.json

@@ -0,0 +1,25 @@
+[
+	{
+		"role": "user",
+		"parts": [
+			{
+				"functionResponse": {
+					"id": "<call-id-0>",
+					"name": "check_screenshot",
+					"response": {
+						"name": "check_screenshot",
+						"content": "Screenshot assertion failed: the page did not match."
+					},
+					"parts": [
+						{
+							"inlineData": {
+								"mimeType": "image/png",
+								"data": "<red-circle.png>"
+							}
+						}
+					]
+				}
+			}
+		]
+	}
+]

packages/aikit/test/e2e/__artifacts__/error-tool-image.openai-chat.json

@@ -0,0 +1,22 @@
+[
+	{
+		"role": "tool",
+		"tool_call_id": "<call-id-0>",
+		"content": "{\"error\":\"Screenshot assertion failed: the page did not match.\"}"
+	},
+	{
+		"role": "user",
+		"content": [
+			{
+				"type": "text",
+				"text": "Attached image(s) from tool result:"
+			},
+			{
+				"type": "image_url",
+				"image_url": {
+					"url": "data:image/png;base64,<red-circle.png>"
+				}
+			}
+		]
+	}
+]

packages/aikit/test/e2e/__artifacts__/error-tool-image.openai-codex.json

@@ -0,0 +1,17 @@
+[
+	{
+		"type": "function_call_output",
+		"call_id": "<call-id-0>",
+		"output": [
+			{
+				"type": "input_text",
+				"text": "Screenshot assertion failed: the page did not match."
+			},
+			{
+				"type": "input_image",
+				"image_url": "data:image/png;base64,<red-circle.png>",
+				"detail": "auto"
+			}
+		]
+	}
+]

packages/aikit/test/e2e/__artifacts__/error-tool-image.openai.json

@@ -0,0 +1,16 @@
+[
+	{
+		"type": "function_call_output",
+		"call_id": "<call-id-0>",
+		"output": [
+			{
+				"type": "input_text",
+				"text": "Screenshot assertion failed: the page did not match."
+			},
+			{
+				"type": "input_image",
+				"image_url": "data:image/png;base64,<red-circle.png>"
+			}
+		]
+	}
+]

packages/aikit/test/e2e/__artifacts__/error-tool-image.openrouter.json

@@ -0,0 +1,19 @@
+[
+	{
+		"role": "tool",
+		"tool_call_id": "<call-id-0>",
+		"content": [
+			{
+				"type": "text",
+				"text": "Screenshot assertion failed: the page did not match."
+			},
+			{
+				"type": "image_url",
+				"image_url": {
+					"url": "data:image/png;base64,<red-circle.png>"
+				}
+			}
+		],
+		"name": "check_screenshot"
+	}
+]