diff --git a/packages/aikit/src/llm/registry.ts b/packages/aikit/src/llm/registry.ts index 0667e673..32c2d76c 100644 --- a/packages/aikit/src/llm/registry.ts +++ b/packages/aikit/src/llm/registry.ts @@ -83,13 +83,18 @@ function defaultMethodForModel(model: Model.Info): Model.APIMethodEnum { return Model.APIMethodEnum.languageModel; } +/** The endpoint method a model is called through: the request's override, the catalog's, or the default. */ +export function resolveMethod(model: Model.Info, methodOverride?: Model.APIMethodEnum): Model.APIMethodEnum { + return methodOverride ?? model.api?.method ?? defaultMethodForModel(model); +} + export function resolveLanguageModel( provider: unknown, model: Model.Info, methodOverride?: Model.APIMethodEnum, ): LanguageModel { const modelId = model.api?.id ?? model.id; - const method = methodOverride ?? model.api?.method ?? defaultMethodForModel(model); + const method = resolveMethod(model, methodOverride); const callable = provider as ProviderWithMethods; if (method === Model.APIMethodEnum.responses && callable.responses) return callable.responses(modelId); diff --git a/packages/aikit/src/llm/stream.ts b/packages/aikit/src/llm/stream.ts index 2ac4d97f..aa157d7f 100644 --- a/packages/aikit/src/llm/stream.ts +++ b/packages/aikit/src/llm/stream.ts @@ -403,7 +403,7 @@ export const stream: Protocol.StreamFunction >[0]["providerOptions"]; diff --git a/packages/aikit/src/llm/transform.ts b/packages/aikit/src/llm/transform.ts index 46f4c68d..8f96cb1d 100644 --- a/packages/aikit/src/llm/transform.ts +++ b/packages/aikit/src/llm/transform.ts @@ -16,7 +16,9 @@ import { resolveOpenAICodexToolConstraint } from "../providers/openai-codex/code import { shortHash } from "../utils/hash.ts"; import { parseStreamingJson } from "../utils/jsonparse.ts"; import { sanitizeSurrogates } from "../utils/sanitize.ts"; +import { packageForModel, resolveMethod } from "./registry.ts"; import { resolveStrictParameters } from "./strict.ts"; +import { isAnthropicMessagesModel } from "./thinking.ts"; type ToolCallPart = Extract; type TerminalToolCall = Exclude; @@ -82,14 +84,78 @@ function sanitizeRecord(value: Record): Record return sanitizeValue(value) as Record; } -function toolResultOutput(toolCall: TerminalToolCall) { - const text = toolCall.result.content +const ERROR_TOOL_IMAGE_PLACEHOLDER = "(image attached after the tool results)"; +const MOVED_TOOL_IMAGE_PLACEHOLDER = "(see attached image)"; +const CHAT_TOOL_IMAGES_CAPTION = "Attached image(s) from tool result:"; +const GEMINI_TOOL_IMAGES_CAPTION = "Tool result image:"; + +/** + * Where a tool result's images go when the wire cannot carry them inside the result. Each case + * sends them in a user message after the turn's tool results: + * - `anthropicError`: Anthropic rejects images in error tool results. + * - `chat`: Chat Completions adapters serialize tool result images as JSON text. + * - `gemini`: before Gemini 3 (and for non-Gemini models), an image-only result loses its function response. + */ +type MovedImages = "anthropicError" | "chat" | "gemini"; + +const COPILOT = "@codeworksh/ai-sdk-github-copilot"; + +function isAnthropicWire(model: Model.Info, method: Model.APIMethodEnum): boolean { + if (packageForModel(model) === COPILOT) return method === Model.APIMethodEnum.messages; + return isAnthropicMessagesModel(model); +} + +function isChatCompletions(model: Model.Info, method: Model.APIMethodEnum): boolean { + const npm = packageForModel(model); + // The compatible adapter falls back to its chat model for every other method. + if (npm === "@ai-sdk/openai-compatible") return true; + if (npm === COPILOT) return method !== Model.APIMethodEnum.responses && method !== Model.APIMethodEnum.messages; + return npm === "@ai-sdk/openai" && method === Model.APIMethodEnum.chat; +} + +/** The Google adapter's own split: anything but Gemini 3 and later gets the legacy function response. */ +function isLegacyGoogleToolResults(model: Model.Info): boolean { + if (model.protocol !== Model.KnownProviderEnum.google && model.protocol !== Model.KnownProviderEnum.googleVertex) { + return false; + } + const id = model.api?.id ?? model.id; + const gemini = /(^|[/.])gemini-/i.test(id); + const legacy = + /(^|[/.])gemini-[12](?:[.-]|$)/i.test(id) || + /(^|[/.])gemini-pro(?:-vision)?$/i.test(id) || + /(^|[/.])gemini-robotics-er-1\.5(?:[.-]|$)/i.test(id); + return !gemini || legacy; +} + +function movedImages( + toolCall: TerminalToolCall, + model: Model.Info, + method: Model.APIMethodEnum, +): MovedImages | undefined { + if (!toolCall.result.content.some((content) => content.type === "image")) return; + if (isAnthropicWire(model, method)) return toolCall.result.isError ? "anthropicError" : undefined; + if (isChatCompletions(model, method)) return "chat"; + if (isLegacyGoogleToolResults(model)) return "gemini"; +} + +function toolResultOutput(toolCall: TerminalToolCall, moved: MovedImages | undefined) { + const content = + moved === "anthropicError" + ? Message.omitImages(toolCall.result.content, ERROR_TOOL_IMAGE_PLACEHOLDER) + : moved + ? toolCall.result.content.filter((content) => content.type === "text") + : toolCall.result.content; + const joined = content .filter((content) => content.type === "text") .map((content) => sanitizeSurrogates(content.text)) .join("\n"); - const images = toolCall.result.content.filter((content) => content.type === "image"); + const text = + moved && moved !== "anthropicError" && joined.trim().length === 0 ? MOVED_TOOL_IMAGE_PLACEHOLDER : joined; + const images = content.filter((content) => content.type === "image"); - if (toolCall.result.isError) { + // The AI SDK's content output has no error flag, so other providers keep the images and lose + // the flag: Google drops its `error` key, OpenAI-family wires have no flag to lose. + if (toolCall.result.isError && images.length === 0) { return { type: "error-text" as const, value: text || "tool returned an error", @@ -210,11 +276,16 @@ export function googleThoughtSignature(metadata: unknown): string | undefined { } } -function assistantMessages(message: Message.AssistantMessage, model: Model.Info): ModelMessage[] { +function assistantMessages( + message: Message.AssistantMessage, + model: Model.Info, + method: Model.APIMethodEnum, +): ModelMessage[] { if (message.stopReason === "error" || message.stopReason === "aborted") return []; const assistantContent: Exclude = []; const toolResults: ToolModelMessage["content"] = []; + const movedImageContent: Exclude = []; const googleReplay = (model.protocol === "google" || model.protocol === "google-vertex") && message.protocol === model.protocol && @@ -322,6 +393,7 @@ function assistantMessages(message: Message.AssistantMessage, model: Model.Info) }); const terminal = part.status === "pending" || part.status === "running" ? undefined : part; + const moved = terminal ? movedImages(terminal, model, method) : undefined; toolResults.push({ type: "tool-result", toolCallId: part.callID, @@ -330,12 +402,30 @@ function assistantMessages(message: Message.AssistantMessage, model: Model.Info) ? { providerOptions: { "openai-codex": { addedToolNames: terminal.addedToolNames } } } : {}), output: terminal - ? toolResultOutput(terminal) + ? toolResultOutput(terminal, moved) : { type: "error-text", value: "no result provided", }, }); + if (terminal && moved) { + const caption = + moved === "anthropicError" + ? `Images from the error result of tool call ${part.callID} (${part.name}):` + : moved === "gemini" + ? GEMINI_TOOL_IMAGES_CAPTION + : movedImageContent.length === 0 + ? CHAT_TOOL_IMAGES_CAPTION + : undefined; + movedImageContent.push( + ...(caption === undefined ? [] : [{ type: "text" as const, text: caption }]), + ...terminal.result.content.flatMap((content) => + content.type === "image" + ? [{ type: "file" as const, data: content.data, mediaType: content.mimeType }] + : [], + ), + ); + } } const result: ModelMessage[] = []; @@ -345,10 +435,19 @@ function assistantMessages(message: Message.AssistantMessage, model: Model.Info) if (toolResults.length > 0) { result.push({ role: "tool", content: toolResults }); } + // @ai-sdk/anthropic merges this into the tool results' user message, after the tool_result blocks. + if (movedImageContent.length > 0) { + result.push({ role: "user", content: movedImageContent }); + } return result; } -export function convertMessages(context: Message.Context, model: Model.Info): ModelMessage[] { +export function convertMessages( + context: Message.Context, + model: Model.Info, + methodOverride?: Model.APIMethodEnum, +): ModelMessage[] { + const method = resolveMethod(model, methodOverride); const messages: ModelMessage[] = []; const transformedMessages = Message.transformMessages(context.messages, model, toolCallIdNormalizer(model)); @@ -357,7 +456,7 @@ export function convertMessages(context: Message.Context, model: Model.Info): Mo messages.push(...userContent(msg.parts)); continue; } - messages.push(...assistantMessages(msg, model)); + messages.push(...assistantMessages(msg, model, method)); } return messages; diff --git a/packages/aikit/src/message/message.ts b/packages/aikit/src/message/message.ts index 9c6aca02..e5ab7bb6 100644 --- a/packages/aikit/src/message/message.ts +++ b/packages/aikit/src/message/message.ts @@ -323,7 +323,7 @@ const USER_IMAGE_PLACEHOLDER = "(image omitted: model does not support images)"; const TOOL_IMAGE_PLACEHOLDER = "(tool image omitted: model does not support images)"; // Consecutive images collapse into a single placeholder. -function omitImages(content: (TextContent | ImageContent)[], placeholder: string): TextContent[] { +export function omitImages(content: (TextContent | ImageContent)[], placeholder: string): TextContent[] { const result: TextContent[] = []; for (const [index, block] of content.entries()) { if (block.type === "text") result.push(block); diff --git a/packages/aikit/test/e2e/__artifacts__/error-tool-image.anthropic.json b/packages/aikit/test/e2e/__artifacts__/error-tool-image.anthropic.json new file mode 100644 index 00000000..e140bcb9 --- /dev/null +++ b/packages/aikit/test/e2e/__artifacts__/error-tool-image.anthropic.json @@ -0,0 +1,25 @@ +[ + { + "role": "user", + "content": [ + { + "type": "tool_result", + "tool_use_id": "", + "content": "Screenshot assertion failed: the page did not match.\n(image attached after the tool results)", + "is_error": true + }, + { + "type": "text", + "text": "Images from the error result of tool call (check_screenshot):" + }, + { + "type": "image", + "source": { + "type": "base64", + "media_type": "image/png", + "data": "" + } + } + ] + } +] \ No newline at end of file diff --git a/packages/aikit/test/e2e/__artifacts__/error-tool-image.google-gemini-2.json b/packages/aikit/test/e2e/__artifacts__/error-tool-image.google-gemini-2.json new file mode 100644 index 00000000..fa4b98c0 --- /dev/null +++ b/packages/aikit/test/e2e/__artifacts__/error-tool-image.google-gemini-2.json @@ -0,0 +1,31 @@ +[ + { + "role": "user", + "parts": [ + { + "functionResponse": { + "id": "", + "name": "check_screenshot", + "response": { + "name": "check_screenshot", + "error": "Screenshot assertion failed: the page did not match." + } + } + } + ] + }, + { + "role": "user", + "parts": [ + { + "text": "Tool result image:" + }, + { + "inlineData": { + "mimeType": "image/png", + "data": "" + } + } + ] + } +] \ No newline at end of file diff --git a/packages/aikit/test/e2e/__artifacts__/error-tool-image.google.json b/packages/aikit/test/e2e/__artifacts__/error-tool-image.google.json new file mode 100644 index 00000000..a59a7baf --- /dev/null +++ b/packages/aikit/test/e2e/__artifacts__/error-tool-image.google.json @@ -0,0 +1,25 @@ +[ + { + "role": "user", + "parts": [ + { + "functionResponse": { + "id": "", + "name": "check_screenshot", + "response": { + "name": "check_screenshot", + "content": "Screenshot assertion failed: the page did not match." + }, + "parts": [ + { + "inlineData": { + "mimeType": "image/png", + "data": "" + } + } + ] + } + } + ] + } +] \ No newline at end of file diff --git a/packages/aikit/test/e2e/__artifacts__/error-tool-image.openai-chat.json b/packages/aikit/test/e2e/__artifacts__/error-tool-image.openai-chat.json new file mode 100644 index 00000000..b1a9c8a6 --- /dev/null +++ b/packages/aikit/test/e2e/__artifacts__/error-tool-image.openai-chat.json @@ -0,0 +1,22 @@ +[ + { + "role": "tool", + "tool_call_id": "", + "content": "{\"error\":\"Screenshot assertion failed: the page did not match.\"}" + }, + { + "role": "user", + "content": [ + { + "type": "text", + "text": "Attached image(s) from tool result:" + }, + { + "type": "image_url", + "image_url": { + "url": "data:image/png;base64," + } + } + ] + } +] \ No newline at end of file diff --git a/packages/aikit/test/e2e/__artifacts__/error-tool-image.openai-codex.json b/packages/aikit/test/e2e/__artifacts__/error-tool-image.openai-codex.json new file mode 100644 index 00000000..23b2e22e --- /dev/null +++ b/packages/aikit/test/e2e/__artifacts__/error-tool-image.openai-codex.json @@ -0,0 +1,17 @@ +[ + { + "type": "function_call_output", + "call_id": "", + "output": [ + { + "type": "input_text", + "text": "Screenshot assertion failed: the page did not match." + }, + { + "type": "input_image", + "image_url": "data:image/png;base64,", + "detail": "auto" + } + ] + } +] \ No newline at end of file diff --git a/packages/aikit/test/e2e/__artifacts__/error-tool-image.openai.json b/packages/aikit/test/e2e/__artifacts__/error-tool-image.openai.json new file mode 100644 index 00000000..70152a9f --- /dev/null +++ b/packages/aikit/test/e2e/__artifacts__/error-tool-image.openai.json @@ -0,0 +1,16 @@ +[ + { + "type": "function_call_output", + "call_id": "", + "output": [ + { + "type": "input_text", + "text": "Screenshot assertion failed: the page did not match." + }, + { + "type": "input_image", + "image_url": "data:image/png;base64," + } + ] + } +] \ No newline at end of file diff --git a/packages/aikit/test/e2e/__artifacts__/error-tool-image.openrouter.json b/packages/aikit/test/e2e/__artifacts__/error-tool-image.openrouter.json new file mode 100644 index 00000000..4a0d9877 --- /dev/null +++ b/packages/aikit/test/e2e/__artifacts__/error-tool-image.openrouter.json @@ -0,0 +1,19 @@ +[ + { + "role": "tool", + "tool_call_id": "", + "content": [ + { + "type": "text", + "text": "Screenshot assertion failed: the page did not match." + }, + { + "type": "image_url", + "image_url": { + "url": "data:image/png;base64," + } + } + ], + "name": "check_screenshot" + } +] \ No newline at end of file diff --git a/packages/aikit/test/e2e/image.tool.error.e2e.test.ts b/packages/aikit/test/e2e/image.tool.error.e2e.test.ts new file mode 100644 index 00000000..b2f7eaa3 --- /dev/null +++ b/packages/aikit/test/e2e/image.tool.error.e2e.test.ts @@ -0,0 +1,190 @@ +import { readFileSync } from "node:fs"; +import { zstdDecompressSync } from "node:zlib"; +import { fileURLToPath } from "node:url"; +import Type from "typebox"; +import { describe, expect, it } from "vite-plus/test"; +import * as Message from "../../src/message/message.ts"; +import * as Model from "../../src/model/model.ts"; +import { complete } from "../../src/stream.ts"; +import { + ANTHROPIC_E2E_MODELS, + anthropicOptions, + describeIfAnthropic, + describeIfGoogle, + describeIfOpenAI, + describeIfOpenAICodex, + describeIfOpenRouter, + getAnthropicModel, + getGoogleModel, + getOpenAICodexModel, + getOpenAIModel, + getOpenRouterModel, + getText, + googleOptions, + openaiCodexOptions, + openaiOptions, + openrouterOptions, + type StreamableModel, +} from "../utils/llm.ts"; + +const image = readFileSync(fileURLToPath(new URL("../data/red-circle.png", import.meta.url))).toString("base64"); + +function failToolCall(message: Message.AssistantMessage, callID: string): Message.AssistantMessage { + return { + ...message, + parts: message.parts.map((part) => { + if (part.type !== "toolCall" || part.callID !== callID) return part; + return { + ...part, + status: "error", + result: { + content: [ + { type: "text", text: "Screenshot assertion failed: the page did not match." }, + { type: "image", data: image, mimeType: "image/png" }, + ], + isError: true, + }, + time: { ...part.time, end: Date.now() }, + } satisfies Message.ToolCallErrorPart; + }), + }; +} + +/** Records each request body the provider SDK sends (Codex zstd-compresses its own), then forwards it. */ +function recordRequests() { + const bodies: Array> = []; + const record: typeof fetch = (input, init) => { + const body = init?.body; + if (typeof body === "string") bodies.push(JSON.parse(body)); + else if (body instanceof Uint8Array) bodies.push(JSON.parse(zstdDecompressSync(body).toString())); + return fetch(input, init); + }; + return { bodies, fetch: record }; +} + +/** The provider payload after the assistant's tool call: what the model reads for the failed result. */ +type Tail = (body: Record) => unknown; +const after = + (key: string, isCall: (item: unknown) => boolean): Tail => + (body) => { + const items = body[key] as unknown[]; + return items.slice(items.findLastIndex(isCall) + 1); + }; +const has = (pattern: string) => (item: unknown) => JSON.stringify(item).includes(pattern); +const anthropicTail = after("messages", has('"type":"tool_use"')); +const responsesTail = after("input", has('"type":"function_call"')); +const chatTail = after("messages", has('"tool_calls"')); +const googleTail = after("contents", has('"functionCall"')); + +/* + * A failed tool call that still returns a screenshot: the model must see the + * image whether the wire carries it in the error result or after it. + */ +async function handleErrorToolWithImageResult(model: StreamableModel, options: object, name: string, tail: Tail) { + expect(model.input).toContain("image"); + const context: Message.Context = { + systemPrompt: "You are a helpful assistant that uses tools when asked.", + messages: [ + Message.createUserMessage({ + role: "user", + parts: [ + { + type: "text", + text: "Call the check_screenshot tool once. It may fail; either way, describe the shape and color in its screenshot.", + }, + ], + time: { created: Date.now() }, + }), + ], + tools: [ + Message.defineTool({ + name: "check_screenshot", + description: "Takes a screenshot and checks it against the expected page", + parameters: Type.Object({}), + }), + ], + }; + + const first = await complete(model, context, options as never); + expect(first.stopReason, first.errorMessage).toBe("toolUse"); + const toolCall = first.parts.find((part): part is Message.ToolCall => part.type === "toolCall"); + if (!toolCall) throw new Error("expected tool call"); + context.messages.push(failToolCall(first, toolCall.callID)); + + const requests = recordRequests(); + const second = await complete(model, context, { + ...options, + factoryOptions: { ...(options as { factoryOptions?: object }).factoryOptions, fetch: requests.fetch }, + } as never); + const body = requests.bodies.at(-1); + if (!body) throw new Error("expected a recorded request"); + const ids = toolCall.callID.split("|").filter((id) => id.length > 0); + const normalized = ids.reduce( + (json, id, index) => json.replaceAll(id, ``), + JSON.stringify(tail(body), null, "\t").replaceAll(image, ""), + ); + await expect(normalized).toMatchFileSnapshot(`./__artifacts__/error-tool-image.${name}.json`); + + expect(second.stopReason, second.errorMessage).toBe("stop"); + const text = getText(second).toLowerCase(); + expect(text).toContain("red"); + expect(text).toMatch(/circle|dot|disc|disk|round/); +} + +describe("Error tool results with images", () => { + describeIfAnthropic("Anthropic provider", () => { + it(ANTHROPIC_E2E_MODELS[0], { retry: 2, timeout: 60_000 }, async () => { + const model = await getAnthropicModel(ANTHROPIC_E2E_MODELS[0]); + await handleErrorToolWithImageResult(model, anthropicOptions({ maxTokens: 256 }), "anthropic", anthropicTail); + }); + }); + + describeIfOpenAI("OpenAI provider", () => { + it("gpt-6-luna", { retry: 2, timeout: 60_000 }, async () => { + const model = await getOpenAIModel("gpt-6-luna"); + await handleErrorToolWithImageResult(model, openaiOptions({ maxTokens: 1_024 }), "openai", responsesTail); + }); + }); + + describeIfOpenAI("OpenAI Chat Completions", () => { + it("gpt-6-luna", { retry: 2, timeout: 60_000 }, async () => { + const model = await getOpenAIModel("gpt-6-luna"); + // gpt-6-luna takes no function tools with a reasoning effort on Chat Completions. + const { reasoning: _, ...options } = openaiOptions({ maxTokens: 1_024, method: Model.APIMethodEnum.chat }); + await handleErrorToolWithImageResult(model, options, "openai-chat", chatTail); + }); + }); + + describeIfOpenAICodex("OpenAI Codex provider", () => { + it("gpt-6-luna", { retry: 2, timeout: 120_000 }, async () => { + const model = await getOpenAICodexModel("gpt-6-luna"); + await handleErrorToolWithImageResult(model, openaiCodexOptions(), "openai-codex", responsesTail); + }); + }); + + describeIfOpenRouter("OpenRouter provider", () => { + it("meta/muse-spark-1.3-contributor", { retry: 2, timeout: 60_000 }, async () => { + const model = await getOpenRouterModel("meta/muse-spark-1.3-contributor"); + await handleErrorToolWithImageResult(model, openrouterOptions(), "openrouter", chatTail); + }); + }); + + describeIfGoogle("Google provider (Gemini before 3)", () => { + it("gemini-2.5-flash", { retry: 2, timeout: 90_000 }, async () => { + const model = await getGoogleModel("gemini-2.5-flash"); + await handleErrorToolWithImageResult( + model, + googleOptions({ maxTokens: 4_096 }), + "google-gemini-2", + googleTail, + ); + }); + }); + + describeIfGoogle("Google provider", () => { + it("gemini-3.5-flash", { retry: 2, timeout: 90_000 }, async () => { + const model = await getGoogleModel("gemini-3.5-flash"); + await handleErrorToolWithImageResult(model, googleOptions({ maxTokens: 4_096 }), "google", googleTail); + }); + }); +}); diff --git a/packages/aikit/test/e2e/image.tool.result.e2e.test.ts b/packages/aikit/test/e2e/image.tool.result.e2e.test.ts index c35e8e3a..cee6564d 100644 --- a/packages/aikit/test/e2e/image.tool.result.e2e.test.ts +++ b/packages/aikit/test/e2e/image.tool.result.e2e.test.ts @@ -2,7 +2,7 @@ import { readFileSync } from "node:fs"; import { fileURLToPath } from "node:url"; import Type from "typebox"; import { describe, expect, it } from "vite-plus/test"; -import type { AnthropicOptions, OpenAICodexOptions, OpenAIOptions } from "../../src/llm/options.ts"; +import type { AnthropicOptions, GoogleOptions, OpenAICodexOptions, OpenAIOptions } from "../../src/llm/options.ts"; import * as Message from "../../src/message/message.ts"; import * as Model from "../../src/model/model.ts"; import { convertMessages } from "../../src/llm/transform.ts"; @@ -12,15 +12,18 @@ import { ANTHROPIC_E2E_MODELS, anthropicOptions, describeIfAnthropic, + describeIfGoogle, describeIfOpenAI, describeIfOpenAICodex, describeIfOpenRouter, OPENROUTER_E2E_MODELS, getAnthropicModel, + getGoogleModel, getOpenAIModel, getOpenAICodexModel, getOpenRouterModel, getText, + googleOptions, OPENAI_CODEX_E2E_MODELS, OPENAI_E2E_MODELS, openaiCodexOptions, @@ -32,8 +35,9 @@ type SupportedModel = | Model.TModel | Model.TModel | Model.TModel - | Model.TModel; -type SupportedOptions = AnthropicOptions | OpenAICodexOptions | OpenAIOptions; + | Model.TModel + | Model.TModel; +type SupportedOptions = AnthropicOptions | GoogleOptions | OpenAICodexOptions | OpenAIOptions; function getImageBase64(): string { const imagePath = fileURLToPath(new URL("../data/red-circle.png", import.meta.url)); @@ -147,7 +151,7 @@ async function handleToolWithTextAndImageResult(model: SupportedModel, options: parts: [ { type: "text", - text: "Use the get_circle_with_description tool and tell me the shape color plus the image metadata.", + text: "Use the get_circle_with_description tool and tell me what you learned. Also say what color the shape is.", }, ], time: { created: Date.now() }, @@ -257,6 +261,37 @@ describe("Tool Results with Images", () => { }); }); + // Chat Completions cannot carry tool result images, so they follow the tool results. + describeIfOpenAI("OpenAI Chat Completions (gpt-6-luna)", () => { + // gpt-6-luna takes no function tools with a reasoning effort on Chat Completions. + const { reasoning: _, ...options } = openaiOptions({ maxTokens: 256, method: Model.APIMethodEnum.chat }); + + it("should handle tool result with only image", { retry: 3, timeout: 30000 }, async () => { + const model = await getOpenAIModel("gpt-6-luna"); + await handleToolWithImageResult(model, options); + }); + + it("should handle tool result with text and image", { retry: 3, timeout: 30000 }, async () => { + const model = await getOpenAIModel("gpt-6-luna"); + await handleToolWithTextAndImageResult(model, options); + }); + }); + + // Gemini before 3 takes the images after the function responses; Gemini 3 takes them inside. + describeIfGoogle.each(["gemini-2.5-flash", "gemini-3.5-flash"])("Google provider (%s)", (modelId) => { + const options = googleOptions({ maxTokens: 4_096 }); + + it("should handle tool result with only image", { retry: 3, timeout: 60_000 }, async () => { + const model = await getGoogleModel(modelId); + await handleToolWithImageResult(model, options); + }); + + it("should handle tool result with text and image", { retry: 3, timeout: 60_000 }, async () => { + const model = await getGoogleModel(modelId); + await handleToolWithTextAndImageResult(model, options); + }); + }); + describeIfAnthropic.each(ANTHROPIC_E2E_MODELS)("Anthropic provider (%s)", (modelId) => { const options = anthropicOptions({ maxTokens: 256, temperature: 0 }); diff --git a/packages/aikit/test/transform.test.ts b/packages/aikit/test/transform.test.ts index ce140704..86a903fd 100644 --- a/packages/aikit/test/transform.test.ts +++ b/packages/aikit/test/transform.test.ts @@ -1,3 +1,4 @@ +import type { ModelMessage } from "ai"; import Type from "typebox"; import { describe, expect, it } from "vite-plus/test"; import { @@ -12,12 +13,15 @@ import * as Message from "../src/message/message.ts"; import * as Model from "../src/model/model.ts"; import { openAICodexTools } from "../src/providers/openai-codex/index.ts"; import { shortHash } from "../src/utils/hash.ts"; +import { stream } from "../src/llm/stream.ts"; +import { fakeAnthropic, textReply } from "./utils/anthropic.ts"; import { makeAssistantMessage, makeLanguageModelUsage, makeCompletedToolCall, makeModel, makePendingToolCall, + makeUserMessage, } from "./utils/fixtures.ts"; const PNG = "aGVsbG8="; @@ -682,8 +686,9 @@ describe("convertMessages", () => { }); }); - it("uses only text from error results that also include images", () => { - const assistant = makeAssistantMessage(imageModel, { + it("keeps error result images in place on the OpenAI Responses wire", () => { + const openai = makeOpenAIModel({ npm: "@ai-sdk/openai", input: ["text", "image"] }); + const assistant = makeAssistantMessage(openai, { stopReason: "toolUse", parts: [ makeErrorToolCall("call-1", [ @@ -692,11 +697,25 @@ describe("convertMessages", () => { ]), ], }); - expect( - convertMessages({ messages: [assistant] }, imageModel).find((message) => message.role === "tool"), - ).toMatchObject({ - content: [{ output: { type: "error-text", value: "failed" } }], - }); + expect(convertMessages({ messages: [assistant] }, openai).slice(1)).toEqual([ + { + role: "tool", + content: [ + { + type: "tool-result", + toolCallId: "call-1", + toolName: "search", + output: { + type: "content", + value: [ + { type: "text", text: "failed" }, + { type: "file", data: { type: "data", data: PNG }, mediaType: "image/png" }, + ], + }, + }, + ], + }, + ]); }); it("sanitizes unpaired surrogates in tool arguments and results", () => { @@ -719,3 +738,322 @@ describe("convertMessages", () => { }); }); }); + +describe("Anthropic error tool results with images", () => { + const claude = makeModel({ npm: "@ai-sdk/anthropic", input: ["text", "image"] }); + const image = (data: string) => ({ type: "image", source: { type: "base64", media_type: "image/png", data } }); + + async function wireMessages(parts: Message.AssistantMessage["parts"]) { + const server = fakeAnthropic(textReply); + const context: Message.Context = { + messages: [ + makeUserMessage("Show the screenshots"), + makeAssistantMessage(claude, { stopReason: "toolUse", parts }), + ], + }; + await stream(claude, context, { apiKey: "test-key", factoryOptions: { fetch: server.fetch } }).result(); + return (server.body() as { messages: unknown[] }).messages; + } + + it("moves the images after the tool results of the turn", async () => { + const messages = await wireMessages([ + makeErrorToolCall( + "call-1", + [ + { type: "text", text: "Script failed" }, + { type: "image", data: "AAAA", mimeType: "image/png" }, + { type: "image", data: "CCCC", mimeType: "image/png" }, + { type: "text", text: "Script error" }, + ], + "codemode", + ), + makeCompletedToolCall("call-2", "read", [{ type: "image", data: "BBBB", mimeType: "image/png" }]), + ]); + + expect(messages[2]).toEqual({ + role: "user", + content: [ + { + type: "tool_result", + tool_use_id: "call-1", + content: "Script failed\n(image attached after the tool results)\nScript error", + is_error: true, + }, + { type: "tool_result", tool_use_id: "call-2", content: [image("BBBB")] }, + { type: "text", text: "Images from the error result of tool call call-1 (codemode):" }, + image("AAAA"), + image("CCCC"), + ], + }); + }); + + it("sends the images of every error result after all tool results, in call order", async () => { + const messages = await wireMessages([ + makeErrorToolCall("call-1", [{ type: "image", data: "AAAA", mimeType: "image/png" }], "screenshot"), + makeCompletedToolCall("call-2", "search", [{ type: "text", text: "found" }]), + makeErrorToolCall( + "call-3", + [ + { type: "text", text: "Render failed" }, + { type: "image", data: "BBBB", mimeType: "image/png" }, + ], + "render", + ), + ]); + + expect(messages[2]).toEqual({ + role: "user", + content: [ + { + type: "tool_result", + tool_use_id: "call-1", + content: "(image attached after the tool results)", + is_error: true, + }, + { type: "tool_result", tool_use_id: "call-2", content: "found" }, + { + type: "tool_result", + tool_use_id: "call-3", + content: "Render failed\n(image attached after the tool results)", + is_error: true, + }, + { type: "text", text: "Images from the error result of tool call call-1 (screenshot):" }, + image("AAAA"), + { type: "text", text: "Images from the error result of tool call call-3 (render):" }, + image("BBBB"), + ], + }); + }); + + it("keeps error tool results without images unchanged", async () => { + const messages = await wireMessages([ + makeErrorToolCall("call-1", [{ type: "text", text: "Script failed" }], "codemode"), + ]); + + expect(messages[2]).toEqual({ + role: "user", + content: [{ type: "tool_result", tool_use_id: "call-1", content: "Script failed", is_error: true }], + }); + }); + + it("omits error result images for a text-only model without splitting", async () => { + const assistant = makeAssistantMessage(sameModel, { + stopReason: "toolUse", + parts: [ + makeErrorToolCall("call-1", [ + { type: "text", text: "failed" }, + { type: "image", data: PNG, mimeType: "image/png" }, + ]), + ], + }); + expect(convertMessages({ messages: [assistant] }, sameModel).slice(1)).toMatchObject([ + { + role: "tool", + content: [ + { + output: { + type: "error-text", + value: "failed\n(tool image omitted: model does not support images)", + }, + }, + ], + }, + ]); + }); +}); + +describe("tool result images on wires that cannot carry them", () => { + const turn = (model: Model.Info) => + makeAssistantMessage(model, { + stopReason: "toolUse", + parts: [ + makeCompletedToolCall("call-1", "screenshot", [{ type: "image", data: "AAAA", mimeType: "image/png" }]), + makeErrorToolCall( + "call-2", + [ + { type: "text", text: "Render failed" }, + { type: "image", data: "BBBB", mimeType: "image/png" }, + ], + "render", + ), + ], + }); + const file = (data: string) => ({ type: "file", data, mediaType: "image/png" }); + const outputs = (messages: ModelMessage[]) => + messages.flatMap((message) => + message.role === "tool" ? message.content.map((part) => ("output" in part ? part.output : part)) : [], + ); + + it("sends Chat Completions tool results as text and their images in one user message", () => { + const chat = makeModel({ + npm: "@ai-sdk/openai-compatible", + protocol: "openai-compatible", + input: ["text", "image"], + }); + const messages = convertMessages({ messages: [turn(chat)] }, chat); + + expect(outputs(messages)).toEqual([ + { type: "text", value: "(see attached image)" }, + { type: "error-text", value: "Render failed" }, + ]); + expect(messages.at(-1)).toEqual({ + role: "user", + content: [{ type: "text", text: "Attached image(s) from tool result:" }, file("AAAA"), file("BBBB")], + }); + }); + + it("uses Chat Completions for OpenAI only when the request picks the chat method", () => { + const openai = makeOpenAIModel({ npm: "@ai-sdk/openai", input: ["text", "image"] }); + + expect(convertMessages({ messages: [turn(openai)] }, openai).at(-1)?.role).toBe("tool"); + expect(convertMessages({ messages: [turn(openai)] }, openai, Model.APIMethodEnum.chat).at(-1)).toMatchObject({ + role: "user", + content: [{ type: "text", text: "Attached image(s) from tool result:" }, file("AAAA"), file("BBBB")], + }); + }); + + it("sends each tool result's images after the results for Gemini before 3", () => { + const gemini = makeModel({ + id: "gemini-2.5-flash", + npm: "@ai-sdk/google", + protocol: "google", + input: ["text", "image"], + }); + const messages = convertMessages({ messages: [turn(gemini)] }, gemini); + + expect(outputs(messages)).toEqual([ + { type: "text", value: "(see attached image)" }, + { type: "error-text", value: "Render failed" }, + ]); + expect(messages.at(-1)).toEqual({ + role: "user", + content: [ + { type: "text", text: "Tool result image:" }, + file("AAAA"), + { type: "text", text: "Tool result image:" }, + file("BBBB"), + ], + }); + }); + + it("keeps images inside the function response from Gemini 3", () => { + const gemini = makeModel({ + id: "gemini-3.5-flash", + npm: "@ai-sdk/google", + protocol: "google", + input: ["text", "image"], + }); + const messages = convertMessages({ messages: [turn(gemini)] }, gemini); + + expect(messages.at(-1)?.role).toBe("tool"); + expect(outputs(messages).map((output) => (output as { type: string }).type)).toEqual(["content", "content"]); + }); + + it("follows the Copilot endpoint the request resolves to", () => { + const copilot = (method: Model.APIMethodEnum) => + makeModel({ + npm: "@codeworksh/ai-sdk-github-copilot", + protocol: "github-copilot", + input: ["text", "image"], + api: { method }, + }); + const messagesEntry = copilot(Model.APIMethodEnum.messages); + const chatEntry = copilot(Model.APIMethodEnum.chat); + const last = (model: Model.Info, method?: Model.APIMethodEnum) => + convertMessages({ messages: [turn(model)] }, model, method).at(-1); + + expect(last(messagesEntry)).toMatchObject({ + role: "user", + content: [{ type: "text", text: "Images from the error result of tool call call-2 (render):" }, file("BBBB")], + }); + expect(last(messagesEntry, Model.APIMethodEnum.chat)).toMatchObject({ + role: "user", + content: [{ type: "text", text: "Attached image(s) from tool result:" }, file("AAAA"), file("BBBB")], + }); + expect(last(chatEntry, Model.APIMethodEnum.messages)).toMatchObject({ + role: "user", + content: [{ type: "text", text: "Images from the error result of tool call call-2 (render):" }, file("BBBB")], + }); + expect(last(chatEntry, Model.APIMethodEnum.responses)?.role).toBe("tool"); + }); + + it("matches the Google adapter's legacy models, on Gemini and Vertex", () => { + const legacy = [ + "gemini-2.5-flash", + "gemini-1.5-pro", + "gemini-pro-vision", + "gemini-robotics-er-1.5", + "gemma-3-27b-it", + ]; + for (const id of legacy) { + for (const [protocol, npm] of [ + ["google", "@ai-sdk/google"], + ["google-vertex", "@ai-sdk/google-vertex"], + ] as const) { + const model = makeModel({ id, protocol, npm, input: ["text", "image"] }); + expect(convertMessages({ messages: [turn(model)] }, model).at(-1)?.role, `${protocol}/${id}`).toBe("user"); + } + } + const current = makeModel({ + id: "gemini-3.5-flash", + protocol: "google-vertex", + npm: "@ai-sdk/google-vertex", + input: ["text", "image"], + }); + expect(convertMessages({ messages: [turn(current)] }, current).at(-1)?.role).toBe("tool"); + }); + + it("keeps every round's images after its own tool results", () => { + const chat = makeModel({ + npm: "@ai-sdk/openai-compatible", + protocol: "openai-compatible", + input: ["text", "image"], + }); + const round = (callID: string, data: string) => + makeAssistantMessage(chat, { + stopReason: "toolUse", + parts: [makeCompletedToolCall(callID, "screenshot", [{ type: "image", data, mimeType: "image/png" }])], + }); + const messages = convertMessages( + { + messages: [ + makeUserMessage("go"), + round("call-1", "AAAA"), + round("call-2", "BBBB"), + makeUserMessage("next"), + ], + }, + chat, + ); + + expect(messages.map((message) => message.role)).toEqual([ + "user", + "assistant", + "tool", + "user", + "assistant", + "tool", + "user", + "user", + ]); + expect(messages[3]).toEqual({ + role: "user", + content: [{ type: "text", text: "Attached image(s) from tool result:" }, file("AAAA")], + }); + expect(messages[6]).toEqual({ + role: "user", + content: [{ type: "text", text: "Attached image(s) from tool result:" }, file("BBBB")], + }); + }); + + it("moves nothing for a text-only model, whose images are already placeholders", () => { + const chat = makeModel({ npm: "@ai-sdk/openai-compatible", protocol: "openai-compatible", input: ["text"] }); + const messages = convertMessages({ messages: [turn(chat)] }, chat); + + expect(messages.at(-1)?.role).toBe("tool"); + expect(outputs(messages)).toEqual([ + { type: "text", value: "(tool image omitted: model does not support images)" }, + { type: "error-text", value: "Render failed\n(tool image omitted: model does not support images)" }, + ]); + }); +});