From 46495bc9997e11fb6cd003ca77e37ae9e8fb16a3 Mon Sep 17 00:00:00 2001 From: Sanchit Rk <0xsanchit@gmail.com> Date: Mon, 26 Oct 2026 16:44:00 +0530 Subject: [PATCH 1/4] feat(harness): tell text-only models when read omits an image Tool calls now carry the input modalities of the model they answer (`ToolCallContext.modelInput`). On a model without image input, `read` adds a note that the image will be omitted, next to the placeholder the transform puts in its place. A session test reads an image on a text-only model, switches to a vision model and snapshots both requests. Co-Authored-By: Claude Opus 5.5 --- .../harness/src/plugin/builtin/tool/read.ts | 22 +++- packages/harness/src/runner/loop.ts | 1 + packages/harness/src/tool/executor.ts | 8 +- .../test/__artifacts__/tools.read.vision.json | 73 +++++++++++ .../harness/test/tools.read.vision.test.ts | 116 ++++++++++++++++++ packages/plugin/src/tool.ts | 2 + 6 files changed, 216 insertions(+), 6 deletions(-) create mode 100644 packages/harness/test/__artifacts__/tools.read.vision.json create mode 100644 packages/harness/test/tools.read.vision.test.ts diff --git a/packages/harness/src/plugin/builtin/tool/read.ts b/packages/harness/src/plugin/builtin/tool/read.ts index 55573849..d7b6dd98 100644 --- a/packages/harness/src/plugin/builtin/tool/read.ts +++ b/packages/harness/src/plugin/builtin/tool/read.ts @@ -90,14 +90,26 @@ const fsFailure = (path: string) => (error: SandboxFileSystem.FileSystemError) = : `Could not read file: ${path}`, ); -const readImage = (fs: SandboxFileSystem.Interface, resolved: string, mimeType: string) => +const NON_VISION_IMAGE_NOTE = "[Current model does not support images. The image will be omitted from this request.]"; + +const readImage = ( + fs: SandboxFileSystem.Interface, + resolved: string, + mimeType: string, + modelInput: Tool.ToolCallContext["modelInput"], +) => Effect.gen(function* () { const bytes = yield* fs.readFileBuffer(resolved); const inline: Image.Inline = yield* Image.inline(bytes, mimeType); - if (!inline.ok) return { type: "image" as const, note: `Read image file [${mimeType}]\n${inline.message}` }; + const nonVision = modelInput !== undefined && !modelInput.includes("image") ? [NON_VISION_IMAGE_NOTE] : []; + if (!inline.ok) + return { + type: "image" as const, + note: [`Read image file [${mimeType}]`, inline.message, ...nonVision].join("\n"), + }; return { type: "image" as const, - note: [`Read image file [${inline.mimeType}]`, ...inline.hints].join("\n"), + note: [`Read image file [${inline.mimeType}]`, ...inline.hints, ...nonVision].join("\n"), image: { data: inline.data, mimeType: inline.mimeType }, }; }); @@ -177,7 +189,7 @@ export const readHandler: Tool.Handler< typeof ReadSuccess, typeof ReadFailed, SandboxIO.FileSystem -> = (params) => +> = (params, ctx) => Effect.gen(function* () { const fs = yield* SandboxIO.FileSystem; const resolved = yield* ToolPath.resolveRead(fs, params.path); @@ -195,7 +207,7 @@ export const readHandler: Tool.Handler< ? Effect.succeed(head.subarray(offset, offset + length)) : fs.readBytes(resolved, offset, length), }); - if (mimeType !== undefined) return yield* readImage(fs, resolved, mimeType); + if (mimeType !== undefined) return yield* readImage(fs, resolved, mimeType, ctx.modelInput); if (head.includes(0)) return yield* fail(`Unsupported read ${params.path}: binary file.`); return yield* readText(fs, resolved, params); }).pipe(Effect.catchTag("SandboxFileSystemError", fsFailure(params.path))); diff --git a/packages/harness/src/runner/loop.ts b/packages/harness/src/runner/loop.ts index 4ec49536..25a7fd01 100644 --- a/packages/harness/src/runner/loop.ts +++ b/packages/harness/src/runner/loop.ts @@ -208,6 +208,7 @@ export const layer = (options: Options = {}) => .handle(call, { sessionId: snapshot.sessionId, messageId, + modelInput: snapshot.resolvedModel.input, onProgress: (progress) => events .publish(EventList.ToolProgress, { diff --git a/packages/harness/src/tool/executor.ts b/packages/harness/src/tool/executor.ts index 21a2c843..489e3186 100644 --- a/packages/harness/src/tool/executor.ts +++ b/packages/harness/src/tool/executor.ts @@ -58,6 +58,7 @@ export interface ProgressEvent { export interface HandleOptions { readonly sessionId?: ToolBefore["sessionId"]; readonly messageId?: ToolBefore["messageId"]; + readonly modelInput?: ToolCallContext["modelInput"]; /** Sliding-queue capacity for best-effort progress. Bounded default (64); tunable. */ readonly progressBuffer?: number; /** @@ -335,7 +336,12 @@ export const make = (tools: ReadonlyArray): E }); } - const ctx: ToolCallContext = { callID: call.callID, toolName: call.name, rawArgs: call.arguments }; + const ctx: ToolCallContext = { + callID: call.callID, + toolName: call.name, + rawArgs: call.arguments, + ...(options?.modelInput === undefined ? {} : { modelInput: options.modelInput }), + }; const hasHooks = hooks.beforeToolCall !== undefined || hooks.afterToolCall !== undefined; if (hasHooks && (options?.sessionId === undefined || options.messageId === undefined)) { diff --git a/packages/harness/test/__artifacts__/tools.read.vision.json b/packages/harness/test/__artifacts__/tools.read.vision.json new file mode 100644 index 00000000..d6616305 --- /dev/null +++ b/packages/harness/test/__artifacts__/tools.read.vision.json @@ -0,0 +1,73 @@ +[ + { + "model": "text-only", + "messages": [ + { + "role": "user", + "content": [ + "read the image" + ] + } + ] + }, + { + "model": "text-only", + "messages": [ + { + "role": "user", + "content": [ + "read the image" + ] + }, + { + "role": "assistant", + "parts": [ + { + "toolCall": "read", + "result": [ + "Read image file [image/png]\n[Current model does not support images. The image will be omitted from this request.]", + "(tool image omitted: model does not support images)" + ] + } + ] + } + ] + }, + { + "model": "vision", + "messages": [ + { + "role": "user", + "content": [ + "read the image" + ] + }, + { + "role": "assistant", + "parts": [ + { + "toolCall": "read", + "result": [ + "Read image file [image/png]\n[Current model does not support images. The image will be omitted from this request.]", + { + "image": "image/png" + } + ] + } + ] + }, + { + "role": "assistant", + "parts": [ + "response 2" + ] + }, + { + "role": "user", + "content": [ + "what do you see now?" + ] + } + ] + } +] \ No newline at end of file diff --git a/packages/harness/test/tools.read.vision.test.ts b/packages/harness/test/tools.read.vision.test.ts new file mode 100644 index 00000000..e269e388 --- /dev/null +++ b/packages/harness/test/tools.read.vision.test.ts @@ -0,0 +1,116 @@ +import "./utils/env.ts"; +import { createAssistantMessageEventStream, Message, type Model } from "@codeworksh/aikit"; +import { Effect } from "effect"; +import { copyFile, writeFile } from "node:fs/promises"; +import { join } from "node:path"; +import { fileURLToPath } from "node:url"; +import { describe, expect, it } from "vite-plus/test"; +import { Harness } from "../src/effect/harness.ts"; +import { Session } from "../src/effect/session.ts"; +import type { LLM } from "../src/runner/llm.ts"; +import { assistant } from "./fixtures/llm.ts"; +import { withSettings } from "./fixtures/settings.ts"; + +const image = fileURLToPath(new URL("../../aikit/test/data/red-circle.png", import.meta.url)); + +const model = (id: string, input: Model.Info["input"]): Model.Info => ({ + id, + name: id, + provider: { id: "scripted", name: "scripted", source: "custom", env: [] }, + baseUrl: "http://scripted.test/v1", + reasoning: false, + input, + cost: { input: 0, output: 0, cacheRead: 0, cacheWrite: 0 }, + contextWindow: 128_000, + maxTokens: 4096, + protocol: "openai", +}); + +const textOnly = model("text-only", ["text"]); +const vision = model("vision", ["text", "image"]); + +/** What the model would receive: the history after aikit's per-model transform, images reduced to their type. */ +const view = (input: LLM.Input) => + Message.transformMessages(input.context.messages, input.resolvedModel).map((message) => { + const content = (parts: ReadonlyArray) => + parts.map((part) => (part.type === "text" ? part.text : { image: part.mimeType })); + if (message.role === "user") return { role: "user", content: content(message.parts) }; + return { + role: "assistant", + parts: message.parts.map((part) => + part.type === "toolCall" && "result" in part + ? { toolCall: part.name, result: content(part.result.content) } + : part.type === "text" + ? part.text + : part.type, + ), + }; + }); + +describe("read on a text-only model, then a model switch", () => { + it("tells a text-only model the image is omitted and sends it once the model sees images", () => + withSettings(async ({ root, custom }) => { + await copyFile(image, join(root, "red-circle.png")); + const select = (selected: Model.Info) => + writeFile( + join(custom, "settings.jsonc"), + JSON.stringify({ model: { provider: "scripted", id: selected.id }, models: [textOnly, vision] }), + ); + const requests: Array> = []; + const open: LLM.Open = (input) => + Effect.sync(() => { + requests.push({ model: input.resolvedModel.id, messages: view(input) }); + const first = requests.length === 1; + const message = assistant( + input, + requests.length, + first + ? { + stopReason: "toolUse", + parts: [ + { + type: "toolCall", + callID: "read_1", + name: "read", + arguments: { path: "red-circle.png" }, + status: "pending", + time: { start: 1, end: 1 }, + }, + ], + } + : {}, + ); + const stream = createAssistantMessageEventStream(); + stream.push({ type: "start", partial: message }); + stream.push({ type: "done", reason: first ? "toolUse" : "stop", message }); + return stream; + }); + + await Effect.runPromise( + Effect.gen(function* () { + yield* Effect.promise(() => select(textOnly)); + const handle = yield* Session.create({ directory: root }); + yield* handle.run("read the image"); + yield* Effect.promise(() => select(vision)); + yield* handle.run("what do you see now?"); + yield* Effect.promise(() => + expect(JSON.stringify(requests, null, "\t")).toMatchFileSnapshot( + "./__artifacts__/tools.read.vision.json", + ), + ); + }).pipe( + Effect.provide( + Harness.layer({ + home: join(root, "home"), + hostCwd: root, + database: ":memory:", + userConfigDir: custom, + llm: open, + }), + ), + Effect.scoped, + Effect.timeout("10 seconds"), + ), + ); + })); +}); diff --git a/packages/plugin/src/tool.ts b/packages/plugin/src/tool.ts index a1b4f437..6a578da4 100644 --- a/packages/plugin/src/tool.ts +++ b/packages/plugin/src/tool.ts @@ -28,6 +28,8 @@ export const ToolCallContext = Schema.Struct({ callID: Schema.String, toolName: Schema.String, rawArgs: Schema.Record(Schema.String, Schema.Unknown), + /** Input modalities of the model the call answers, e.g. `["text", "image"]`. */ + modelInput: Schema.optional(Schema.Array(Schema.Literals(["text", "image"]))), }); export type ToolCallContext = typeof ToolCallContext.Type; From 6b3dd96e8acb3a75b3c732f8bc9084ecfa2edd44 Mon Sep 17 00:00:00 2001 From: Sanchit Rk <0xsanchit@gmail.com> Date: Mon, 26 Oct 2026 16:52:33 +0530 Subject: [PATCH 2/4] fix(aikit): keep images from error tool results Error tool results dropped their images for every provider. Anthropic-wire models now get a text-only error result with a placeholder, followed by a caption and the images after the turn's tool results. Other providers keep the images in place. Co-Authored-By: Claude Opus 5.5 --- packages/aikit/src/llm/transform.ts | 43 +++++- packages/aikit/src/message/message.ts | 2 +- .../error-tool-image.anthropic.json | 30 ++++ .../error-tool-image.google.json | 29 ++++ .../error-tool-image.openai-codex.json | 29 ++++ .../error-tool-image.openai.json | 29 ++++ .../error-tool-image.openrouter.json | 29 ++++ .../test/e2e/image.tool.error.e2e.test.ts | 136 ++++++++++++++++++ packages/aikit/test/transform.test.ts | 117 ++++++++++++++- 9 files changed, 431 insertions(+), 13 deletions(-) create mode 100644 packages/aikit/test/e2e/__artifacts__/error-tool-image.anthropic.json create mode 100644 packages/aikit/test/e2e/__artifacts__/error-tool-image.google.json create mode 100644 packages/aikit/test/e2e/__artifacts__/error-tool-image.openai-codex.json create mode 100644 packages/aikit/test/e2e/__artifacts__/error-tool-image.openai.json create mode 100644 packages/aikit/test/e2e/__artifacts__/error-tool-image.openrouter.json create mode 100644 packages/aikit/test/e2e/image.tool.error.e2e.test.ts diff --git a/packages/aikit/src/llm/transform.ts b/packages/aikit/src/llm/transform.ts index 46f4c68d..bcf507c1 100644 --- a/packages/aikit/src/llm/transform.ts +++ b/packages/aikit/src/llm/transform.ts @@ -17,6 +17,7 @@ import { shortHash } from "../utils/hash.ts"; import { parseStreamingJson } from "../utils/jsonparse.ts"; import { sanitizeSurrogates } from "../utils/sanitize.ts"; import { resolveStrictParameters } from "./strict.ts"; +import { isAnthropicMessagesModel } from "./thinking.ts"; type ToolCallPart = Extract; type TerminalToolCall = Exclude; @@ -82,14 +83,31 @@ function sanitizeRecord(value: Record): Record return sanitizeValue(value) as Record; } -function toolResultOutput(toolCall: TerminalToolCall) { - const text = toolCall.result.content +const ERROR_TOOL_IMAGE_PLACEHOLDER = "(image attached after the tool results)"; + +// Anthropic rejects images in error tool results, so they follow the turn's tool results instead. +function splitsErrorImages(toolCall: TerminalToolCall, model: Model.Info): boolean { + return ( + toolCall.result.isError && + isAnthropicMessagesModel(model) && + toolCall.result.content.some((content) => content.type === "image") + ); +} + +function toolResultOutput(toolCall: TerminalToolCall, model: Model.Info) { + const split = splitsErrorImages(toolCall, model); + const content = split + ? Message.omitImages(toolCall.result.content, ERROR_TOOL_IMAGE_PLACEHOLDER) + : toolCall.result.content; + const text = content .filter((content) => content.type === "text") .map((content) => sanitizeSurrogates(content.text)) .join("\n"); - const images = toolCall.result.content.filter((content) => content.type === "image"); + const images = content.filter((content) => content.type === "image"); - if (toolCall.result.isError) { + // The AI SDK's content output has no error flag, so other providers keep the images and lose + // the flag: Google drops its `error` key, OpenAI-family wires have no flag to lose. + if (toolCall.result.isError && images.length === 0) { return { type: "error-text" as const, value: text || "tool returned an error", @@ -215,6 +233,7 @@ function assistantMessages(message: Message.AssistantMessage, model: Model.Info) const assistantContent: Exclude = []; const toolResults: ToolModelMessage["content"] = []; + const errorImages: Exclude = []; const googleReplay = (model.protocol === "google" || model.protocol === "google-vertex") && message.protocol === model.protocol && @@ -330,12 +349,22 @@ function assistantMessages(message: Message.AssistantMessage, model: Model.Info) ? { providerOptions: { "openai-codex": { addedToolNames: terminal.addedToolNames } } } : {}), output: terminal - ? toolResultOutput(terminal) + ? toolResultOutput(terminal, model) : { type: "error-text", value: "no result provided", }, }); + if (terminal && splitsErrorImages(terminal, model)) { + errorImages.push( + { type: "text", text: `Images from the error result of tool call ${part.callID} (${part.name}):` }, + ...terminal.result.content.flatMap((content) => + content.type === "image" + ? [{ type: "file" as const, data: content.data, mediaType: content.mimeType }] + : [], + ), + ); + } } const result: ModelMessage[] = []; @@ -345,6 +374,10 @@ function assistantMessages(message: Message.AssistantMessage, model: Model.Info) if (toolResults.length > 0) { result.push({ role: "tool", content: toolResults }); } + // @ai-sdk/anthropic merges this into the tool results' user message, after the tool_result blocks. + if (errorImages.length > 0) { + result.push({ role: "user", content: errorImages }); + } return result; } diff --git a/packages/aikit/src/message/message.ts b/packages/aikit/src/message/message.ts index 9c6aca02..e5ab7bb6 100644 --- a/packages/aikit/src/message/message.ts +++ b/packages/aikit/src/message/message.ts @@ -323,7 +323,7 @@ const USER_IMAGE_PLACEHOLDER = "(image omitted: model does not support images)"; const TOOL_IMAGE_PLACEHOLDER = "(tool image omitted: model does not support images)"; // Consecutive images collapse into a single placeholder. -function omitImages(content: (TextContent | ImageContent)[], placeholder: string): TextContent[] { +export function omitImages(content: (TextContent | ImageContent)[], placeholder: string): TextContent[] { const result: TextContent[] = []; for (const [index, block] of content.entries()) { if (block.type === "text") result.push(block); diff --git a/packages/aikit/test/e2e/__artifacts__/error-tool-image.anthropic.json b/packages/aikit/test/e2e/__artifacts__/error-tool-image.anthropic.json new file mode 100644 index 00000000..0e87bd98 --- /dev/null +++ b/packages/aikit/test/e2e/__artifacts__/error-tool-image.anthropic.json @@ -0,0 +1,30 @@ +[ + { + "role": "tool", + "content": [ + { + "type": "tool-result", + "toolCallId": "", + "toolName": "check_screenshot", + "output": { + "type": "error-text", + "value": "Screenshot assertion failed: the page did not match.\n(image attached after the tool results)" + } + } + ] + }, + { + "role": "user", + "content": [ + { + "type": "text", + "text": "Images from the error result of tool call (check_screenshot):" + }, + { + "type": "file", + "data": "", + "mediaType": "image/png" + } + ] + } +] \ No newline at end of file diff --git a/packages/aikit/test/e2e/__artifacts__/error-tool-image.google.json b/packages/aikit/test/e2e/__artifacts__/error-tool-image.google.json new file mode 100644 index 00000000..03238ad4 --- /dev/null +++ b/packages/aikit/test/e2e/__artifacts__/error-tool-image.google.json @@ -0,0 +1,29 @@ +[ + { + "role": "tool", + "content": [ + { + "type": "tool-result", + "toolCallId": "", + "toolName": "check_screenshot", + "output": { + "type": "content", + "value": [ + { + "type": "text", + "text": "Screenshot assertion failed: the page did not match." + }, + { + "type": "file", + "data": { + "type": "data", + "data": "" + }, + "mediaType": "image/png" + } + ] + } + } + ] + } +] \ No newline at end of file diff --git a/packages/aikit/test/e2e/__artifacts__/error-tool-image.openai-codex.json b/packages/aikit/test/e2e/__artifacts__/error-tool-image.openai-codex.json new file mode 100644 index 00000000..03238ad4 --- /dev/null +++ b/packages/aikit/test/e2e/__artifacts__/error-tool-image.openai-codex.json @@ -0,0 +1,29 @@ +[ + { + "role": "tool", + "content": [ + { + "type": "tool-result", + "toolCallId": "", + "toolName": "check_screenshot", + "output": { + "type": "content", + "value": [ + { + "type": "text", + "text": "Screenshot assertion failed: the page did not match." + }, + { + "type": "file", + "data": { + "type": "data", + "data": "" + }, + "mediaType": "image/png" + } + ] + } + } + ] + } +] \ No newline at end of file diff --git a/packages/aikit/test/e2e/__artifacts__/error-tool-image.openai.json b/packages/aikit/test/e2e/__artifacts__/error-tool-image.openai.json new file mode 100644 index 00000000..03238ad4 --- /dev/null +++ b/packages/aikit/test/e2e/__artifacts__/error-tool-image.openai.json @@ -0,0 +1,29 @@ +[ + { + "role": "tool", + "content": [ + { + "type": "tool-result", + "toolCallId": "", + "toolName": "check_screenshot", + "output": { + "type": "content", + "value": [ + { + "type": "text", + "text": "Screenshot assertion failed: the page did not match." + }, + { + "type": "file", + "data": { + "type": "data", + "data": "" + }, + "mediaType": "image/png" + } + ] + } + } + ] + } +] \ No newline at end of file diff --git a/packages/aikit/test/e2e/__artifacts__/error-tool-image.openrouter.json b/packages/aikit/test/e2e/__artifacts__/error-tool-image.openrouter.json new file mode 100644 index 00000000..03238ad4 --- /dev/null +++ b/packages/aikit/test/e2e/__artifacts__/error-tool-image.openrouter.json @@ -0,0 +1,29 @@ +[ + { + "role": "tool", + "content": [ + { + "type": "tool-result", + "toolCallId": "", + "toolName": "check_screenshot", + "output": { + "type": "content", + "value": [ + { + "type": "text", + "text": "Screenshot assertion failed: the page did not match." + }, + { + "type": "file", + "data": { + "type": "data", + "data": "" + }, + "mediaType": "image/png" + } + ] + } + } + ] + } +] \ No newline at end of file diff --git a/packages/aikit/test/e2e/image.tool.error.e2e.test.ts b/packages/aikit/test/e2e/image.tool.error.e2e.test.ts new file mode 100644 index 00000000..9fe15005 --- /dev/null +++ b/packages/aikit/test/e2e/image.tool.error.e2e.test.ts @@ -0,0 +1,136 @@ +import { readFileSync } from "node:fs"; +import { fileURLToPath } from "node:url"; +import Type from "typebox"; +import { describe, expect, it } from "vite-plus/test"; +import { convertMessages } from "../../src/llm/transform.ts"; +import * as Message from "../../src/message/message.ts"; +import { complete } from "../../src/stream.ts"; +import { + ANTHROPIC_E2E_MODELS, + anthropicOptions, + describeIfAnthropic, + describeIfGoogle, + describeIfOpenAI, + describeIfOpenAICodex, + describeIfOpenRouter, + getAnthropicModel, + getGoogleModel, + getOpenAICodexModel, + getOpenAIModel, + getOpenRouterModel, + getText, + googleOptions, + openaiCodexOptions, + openaiOptions, + openrouterOptions, + type StreamableModel, +} from "../utils/llm.ts"; + +const image = readFileSync(fileURLToPath(new URL("../data/red-circle.png", import.meta.url))).toString("base64"); + +function failToolCall(message: Message.AssistantMessage, callID: string): Message.AssistantMessage { + return { + ...message, + parts: message.parts.map((part) => { + if (part.type !== "toolCall" || part.callID !== callID) return part; + return { + ...part, + status: "error", + result: { + content: [ + { type: "text", text: "Screenshot assertion failed: the page did not match." }, + { type: "image", data: image, mimeType: "image/png" }, + ], + isError: true, + }, + time: { ...part.time, end: Date.now() }, + } satisfies Message.ToolCallErrorPart; + }), + }; +} + +/* + * A failed tool call that still returns a screenshot: the model must see the + * image whether the wire carries it in the error result or after it. + */ +async function handleErrorToolWithImageResult(model: StreamableModel, options: object, name: string) { + expect(model.input).toContain("image"); + const context: Message.Context = { + systemPrompt: "You are a helpful assistant that uses tools when asked.", + messages: [ + Message.createUserMessage({ + role: "user", + parts: [ + { + type: "text", + text: "Call the check_screenshot tool once. It may fail; either way, describe the shape and color in its screenshot.", + }, + ], + time: { created: Date.now() }, + }), + ], + tools: [ + Message.defineTool({ + name: "check_screenshot", + description: "Takes a screenshot and checks it against the expected page", + parameters: Type.Object({}), + }), + ], + }; + + const first = await complete(model, context, options as never); + expect(first.stopReason, first.errorMessage).toBe("toolUse"); + const toolCall = first.parts.find((part): part is Message.ToolCall => part.type === "toolCall"); + if (!toolCall) throw new Error("expected tool call"); + context.messages.push(failToolCall(first, toolCall.callID)); + + const wire = convertMessages(context, model); + const results = wire.slice(wire.findIndex((message) => message.role === "tool")); + const normalized = JSON.stringify(results, null, "\t") + .replaceAll(image, "") + .replaceAll(toolCall.callID, ""); + await expect(normalized).toMatchFileSnapshot(`./__artifacts__/error-tool-image.${name}.json`); + + const second = await complete(model, context, options as never); + expect(second.stopReason, second.errorMessage).toBe("stop"); + const text = getText(second).toLowerCase(); + expect(text).toContain("red"); + expect(text).toMatch(/circle|dot|disc|disk|round/); +} + +describe("Error tool results with images", () => { + describeIfAnthropic("Anthropic provider", () => { + it(ANTHROPIC_E2E_MODELS[0], { retry: 2, timeout: 60_000 }, async () => { + const model = await getAnthropicModel(ANTHROPIC_E2E_MODELS[0]); + await handleErrorToolWithImageResult(model, anthropicOptions({ maxTokens: 256 }), "anthropic"); + }); + }); + + describeIfOpenAI("OpenAI provider", () => { + it("gpt-6-luna", { retry: 2, timeout: 60_000 }, async () => { + const model = await getOpenAIModel("gpt-6-luna"); + await handleErrorToolWithImageResult(model, openaiOptions({ maxTokens: 1_024 }), "openai"); + }); + }); + + describeIfOpenAICodex("OpenAI Codex provider", () => { + it("gpt-6-luna", { retry: 2, timeout: 120_000 }, async () => { + const model = await getOpenAICodexModel("gpt-6-luna"); + await handleErrorToolWithImageResult(model, openaiCodexOptions(), "openai-codex"); + }); + }); + + describeIfOpenRouter("OpenRouter provider", () => { + it("meta/muse-spark-1.3-contributor", { retry: 2, timeout: 60_000 }, async () => { + const model = await getOpenRouterModel("meta/muse-spark-1.3-contributor"); + await handleErrorToolWithImageResult(model, openrouterOptions(), "openrouter"); + }); + }); + + describeIfGoogle("Google provider", () => { + it("gemini-3.5-flash", { retry: 2, timeout: 90_000 }, async () => { + const model = await getGoogleModel("gemini-3.5-flash"); + await handleErrorToolWithImageResult(model, googleOptions({ maxTokens: 4_096 }), "google"); + }); + }); +}); diff --git a/packages/aikit/test/transform.test.ts b/packages/aikit/test/transform.test.ts index ce140704..cb212aa1 100644 --- a/packages/aikit/test/transform.test.ts +++ b/packages/aikit/test/transform.test.ts @@ -12,12 +12,15 @@ import * as Message from "../src/message/message.ts"; import * as Model from "../src/model/model.ts"; import { openAICodexTools } from "../src/providers/openai-codex/index.ts"; import { shortHash } from "../src/utils/hash.ts"; +import { stream } from "../src/llm/stream.ts"; +import { fakeAnthropic, textReply } from "./utils/anthropic.ts"; import { makeAssistantMessage, makeLanguageModelUsage, makeCompletedToolCall, makeModel, makePendingToolCall, + makeUserMessage, } from "./utils/fixtures.ts"; const PNG = "aGVsbG8="; @@ -682,8 +685,9 @@ describe("convertMessages", () => { }); }); - it("uses only text from error results that also include images", () => { - const assistant = makeAssistantMessage(imageModel, { + it("keeps error result images in place for providers outside the Anthropic wire", () => { + const openai = makeOpenAIModel({ input: ["text", "image"] }); + const assistant = makeAssistantMessage(openai, { stopReason: "toolUse", parts: [ makeErrorToolCall("call-1", [ @@ -692,11 +696,25 @@ describe("convertMessages", () => { ]), ], }); - expect( - convertMessages({ messages: [assistant] }, imageModel).find((message) => message.role === "tool"), - ).toMatchObject({ - content: [{ output: { type: "error-text", value: "failed" } }], - }); + expect(convertMessages({ messages: [assistant] }, openai).slice(1)).toEqual([ + { + role: "tool", + content: [ + { + type: "tool-result", + toolCallId: "call-1", + toolName: "search", + output: { + type: "content", + value: [ + { type: "text", text: "failed" }, + { type: "file", data: { type: "data", data: PNG }, mediaType: "image/png" }, + ], + }, + }, + ], + }, + ]); }); it("sanitizes unpaired surrogates in tool arguments and results", () => { @@ -719,3 +737,88 @@ describe("convertMessages", () => { }); }); }); + +describe("Anthropic error tool results with images", () => { + const claude = makeModel({ npm: "@ai-sdk/anthropic", input: ["text", "image"] }); + const image = (data: string) => ({ type: "image", source: { type: "base64", media_type: "image/png", data } }); + + async function wireMessages(parts: Message.AssistantMessage["parts"]) { + const server = fakeAnthropic(textReply); + const context: Message.Context = { + messages: [ + makeUserMessage("Show the screenshots"), + makeAssistantMessage(claude, { stopReason: "toolUse", parts }), + ], + }; + await stream(claude, context, { apiKey: "test-key", factoryOptions: { fetch: server.fetch } }).result(); + return (server.body() as { messages: unknown[] }).messages; + } + + it("moves the images after the tool results of the turn", async () => { + const messages = await wireMessages([ + makeErrorToolCall( + "call-1", + [ + { type: "text", text: "Script failed" }, + { type: "image", data: "AAAA", mimeType: "image/png" }, + { type: "image", data: "CCCC", mimeType: "image/png" }, + { type: "text", text: "Script error" }, + ], + "codemode", + ), + makeCompletedToolCall("call-2", "read", [{ type: "image", data: "BBBB", mimeType: "image/png" }]), + ]); + + expect(messages[2]).toEqual({ + role: "user", + content: [ + { + type: "tool_result", + tool_use_id: "call-1", + content: "Script failed\n(image attached after the tool results)\nScript error", + is_error: true, + }, + { type: "tool_result", tool_use_id: "call-2", content: [image("BBBB")] }, + { type: "text", text: "Images from the error result of tool call call-1 (codemode):" }, + image("AAAA"), + image("CCCC"), + ], + }); + }); + + it("keeps error tool results without images unchanged", async () => { + const messages = await wireMessages([ + makeErrorToolCall("call-1", [{ type: "text", text: "Script failed" }], "codemode"), + ]); + + expect(messages[2]).toEqual({ + role: "user", + content: [{ type: "tool_result", tool_use_id: "call-1", content: "Script failed", is_error: true }], + }); + }); + + it("omits error result images for a text-only model without splitting", async () => { + const assistant = makeAssistantMessage(sameModel, { + stopReason: "toolUse", + parts: [ + makeErrorToolCall("call-1", [ + { type: "text", text: "failed" }, + { type: "image", data: PNG, mimeType: "image/png" }, + ]), + ], + }); + expect(convertMessages({ messages: [assistant] }, sameModel).slice(1)).toMatchObject([ + { + role: "tool", + content: [ + { + output: { + type: "error-text", + value: "failed\n(tool image omitted: model does not support images)", + }, + }, + ], + }, + ]); + }); +}); From aafc57c443f734549ee07ff0c917efd18cb0bb3e Mon Sep 17 00:00:00 2001 From: Sanchit Rk <0xsanchit@gmail.com> Date: Mon, 26 Oct 2026 17:52:35 +0530 Subject: [PATCH 3/4] test(aikit): check error tool result images in each provider's request The live test now records the request body each provider SDK sends and snapshots the part after the tool call, so it catches a wire that drops or misplaces the image. A unit case covers two failed calls in one turn. Co-Authored-By: Claude Opus 5.5 --- .../error-tool-image.anthropic.json | 31 +++++----- .../error-tool-image.google.json | 30 +++++----- .../error-tool-image.openai-codex.json | 32 ++++------ .../error-tool-image.openai.json | 31 +++------- .../error-tool-image.openrouter.json | 30 ++++------ .../test/e2e/image.tool.error.e2e.test.ts | 58 ++++++++++++++----- packages/aikit/test/transform.test.ts | 38 ++++++++++++ 7 files changed, 138 insertions(+), 112 deletions(-) diff --git a/packages/aikit/test/e2e/__artifacts__/error-tool-image.anthropic.json b/packages/aikit/test/e2e/__artifacts__/error-tool-image.anthropic.json index 0e87bd98..e140bcb9 100644 --- a/packages/aikit/test/e2e/__artifacts__/error-tool-image.anthropic.json +++ b/packages/aikit/test/e2e/__artifacts__/error-tool-image.anthropic.json @@ -1,29 +1,24 @@ [ - { - "role": "tool", - "content": [ - { - "type": "tool-result", - "toolCallId": "", - "toolName": "check_screenshot", - "output": { - "type": "error-text", - "value": "Screenshot assertion failed: the page did not match.\n(image attached after the tool results)" - } - } - ] - }, { "role": "user", "content": [ + { + "type": "tool_result", + "tool_use_id": "", + "content": "Screenshot assertion failed: the page did not match.\n(image attached after the tool results)", + "is_error": true + }, { "type": "text", - "text": "Images from the error result of tool call (check_screenshot):" + "text": "Images from the error result of tool call (check_screenshot):" }, { - "type": "file", - "data": "", - "mediaType": "image/png" + "type": "image", + "source": { + "type": "base64", + "media_type": "image/png", + "data": "" + } } ] } diff --git a/packages/aikit/test/e2e/__artifacts__/error-tool-image.google.json b/packages/aikit/test/e2e/__artifacts__/error-tool-image.google.json index 03238ad4..a59a7baf 100644 --- a/packages/aikit/test/e2e/__artifacts__/error-tool-image.google.json +++ b/packages/aikit/test/e2e/__artifacts__/error-tool-image.google.json @@ -1,25 +1,21 @@ [ { - "role": "tool", - "content": [ + "role": "user", + "parts": [ { - "type": "tool-result", - "toolCallId": "", - "toolName": "check_screenshot", - "output": { - "type": "content", - "value": [ + "functionResponse": { + "id": "", + "name": "check_screenshot", + "response": { + "name": "check_screenshot", + "content": "Screenshot assertion failed: the page did not match." + }, + "parts": [ { - "type": "text", - "text": "Screenshot assertion failed: the page did not match." - }, - { - "type": "file", - "data": { - "type": "data", + "inlineData": { + "mimeType": "image/png", "data": "" - }, - "mediaType": "image/png" + } } ] } diff --git a/packages/aikit/test/e2e/__artifacts__/error-tool-image.openai-codex.json b/packages/aikit/test/e2e/__artifacts__/error-tool-image.openai-codex.json index 03238ad4..23b2e22e 100644 --- a/packages/aikit/test/e2e/__artifacts__/error-tool-image.openai-codex.json +++ b/packages/aikit/test/e2e/__artifacts__/error-tool-image.openai-codex.json @@ -1,28 +1,16 @@ [ { - "role": "tool", - "content": [ + "type": "function_call_output", + "call_id": "", + "output": [ { - "type": "tool-result", - "toolCallId": "", - "toolName": "check_screenshot", - "output": { - "type": "content", - "value": [ - { - "type": "text", - "text": "Screenshot assertion failed: the page did not match." - }, - { - "type": "file", - "data": { - "type": "data", - "data": "" - }, - "mediaType": "image/png" - } - ] - } + "type": "input_text", + "text": "Screenshot assertion failed: the page did not match." + }, + { + "type": "input_image", + "image_url": "data:image/png;base64,", + "detail": "auto" } ] } diff --git a/packages/aikit/test/e2e/__artifacts__/error-tool-image.openai.json b/packages/aikit/test/e2e/__artifacts__/error-tool-image.openai.json index 03238ad4..70152a9f 100644 --- a/packages/aikit/test/e2e/__artifacts__/error-tool-image.openai.json +++ b/packages/aikit/test/e2e/__artifacts__/error-tool-image.openai.json @@ -1,28 +1,15 @@ [ { - "role": "tool", - "content": [ + "type": "function_call_output", + "call_id": "", + "output": [ { - "type": "tool-result", - "toolCallId": "", - "toolName": "check_screenshot", - "output": { - "type": "content", - "value": [ - { - "type": "text", - "text": "Screenshot assertion failed: the page did not match." - }, - { - "type": "file", - "data": { - "type": "data", - "data": "" - }, - "mediaType": "image/png" - } - ] - } + "type": "input_text", + "text": "Screenshot assertion failed: the page did not match." + }, + { + "type": "input_image", + "image_url": "data:image/png;base64," } ] } diff --git a/packages/aikit/test/e2e/__artifacts__/error-tool-image.openrouter.json b/packages/aikit/test/e2e/__artifacts__/error-tool-image.openrouter.json index 03238ad4..4a0d9877 100644 --- a/packages/aikit/test/e2e/__artifacts__/error-tool-image.openrouter.json +++ b/packages/aikit/test/e2e/__artifacts__/error-tool-image.openrouter.json @@ -1,29 +1,19 @@ [ { "role": "tool", + "tool_call_id": "", "content": [ { - "type": "tool-result", - "toolCallId": "", - "toolName": "check_screenshot", - "output": { - "type": "content", - "value": [ - { - "type": "text", - "text": "Screenshot assertion failed: the page did not match." - }, - { - "type": "file", - "data": { - "type": "data", - "data": "" - }, - "mediaType": "image/png" - } - ] + "type": "text", + "text": "Screenshot assertion failed: the page did not match." + }, + { + "type": "image_url", + "image_url": { + "url": "data:image/png;base64," } } - ] + ], + "name": "check_screenshot" } ] \ No newline at end of file diff --git a/packages/aikit/test/e2e/image.tool.error.e2e.test.ts b/packages/aikit/test/e2e/image.tool.error.e2e.test.ts index 9fe15005..f617a451 100644 --- a/packages/aikit/test/e2e/image.tool.error.e2e.test.ts +++ b/packages/aikit/test/e2e/image.tool.error.e2e.test.ts @@ -1,8 +1,8 @@ import { readFileSync } from "node:fs"; +import { zstdDecompressSync } from "node:zlib"; import { fileURLToPath } from "node:url"; import Type from "typebox"; import { describe, expect, it } from "vite-plus/test"; -import { convertMessages } from "../../src/llm/transform.ts"; import * as Message from "../../src/message/message.ts"; import { complete } from "../../src/stream.ts"; import { @@ -49,11 +49,37 @@ function failToolCall(message: Message.AssistantMessage, callID: string): Messag }; } +/** Records each request body the provider SDK sends (Codex zstd-compresses its own), then forwards it. */ +function recordRequests() { + const bodies: Array> = []; + const record: typeof fetch = (input, init) => { + const body = init?.body; + if (typeof body === "string") bodies.push(JSON.parse(body)); + else if (body instanceof Uint8Array) bodies.push(JSON.parse(zstdDecompressSync(body).toString())); + return fetch(input, init); + }; + return { bodies, fetch: record }; +} + +/** The provider payload after the assistant's tool call: what the model reads for the failed result. */ +type Tail = (body: Record) => unknown; +const after = + (key: string, isCall: (item: unknown) => boolean): Tail => + (body) => { + const items = body[key] as unknown[]; + return items.slice(items.findLastIndex(isCall) + 1); + }; +const has = (pattern: string) => (item: unknown) => JSON.stringify(item).includes(pattern); +const anthropicTail = after("messages", has('"type":"tool_use"')); +const responsesTail = after("input", has('"type":"function_call"')); +const chatTail = after("messages", has('"tool_calls"')); +const googleTail = after("contents", has('"functionCall"')); + /* * A failed tool call that still returns a screenshot: the model must see the * image whether the wire carries it in the error result or after it. */ -async function handleErrorToolWithImageResult(model: StreamableModel, options: object, name: string) { +async function handleErrorToolWithImageResult(model: StreamableModel, options: object, name: string, tail: Tail) { expect(model.input).toContain("image"); const context: Message.Context = { systemPrompt: "You are a helpful assistant that uses tools when asked.", @@ -84,14 +110,20 @@ async function handleErrorToolWithImageResult(model: StreamableModel, options: o if (!toolCall) throw new Error("expected tool call"); context.messages.push(failToolCall(first, toolCall.callID)); - const wire = convertMessages(context, model); - const results = wire.slice(wire.findIndex((message) => message.role === "tool")); - const normalized = JSON.stringify(results, null, "\t") - .replaceAll(image, "") - .replaceAll(toolCall.callID, ""); + const requests = recordRequests(); + const second = await complete(model, context, { + ...options, + factoryOptions: { ...(options as { factoryOptions?: object }).factoryOptions, fetch: requests.fetch }, + } as never); + const body = requests.bodies.at(-1); + if (!body) throw new Error("expected a recorded request"); + const ids = toolCall.callID.split("|").filter((id) => id.length > 0); + const normalized = ids.reduce( + (json, id, index) => json.replaceAll(id, ``), + JSON.stringify(tail(body), null, "\t").replaceAll(image, ""), + ); await expect(normalized).toMatchFileSnapshot(`./__artifacts__/error-tool-image.${name}.json`); - const second = await complete(model, context, options as never); expect(second.stopReason, second.errorMessage).toBe("stop"); const text = getText(second).toLowerCase(); expect(text).toContain("red"); @@ -102,35 +134,35 @@ describe("Error tool results with images", () => { describeIfAnthropic("Anthropic provider", () => { it(ANTHROPIC_E2E_MODELS[0], { retry: 2, timeout: 60_000 }, async () => { const model = await getAnthropicModel(ANTHROPIC_E2E_MODELS[0]); - await handleErrorToolWithImageResult(model, anthropicOptions({ maxTokens: 256 }), "anthropic"); + await handleErrorToolWithImageResult(model, anthropicOptions({ maxTokens: 256 }), "anthropic", anthropicTail); }); }); describeIfOpenAI("OpenAI provider", () => { it("gpt-6-luna", { retry: 2, timeout: 60_000 }, async () => { const model = await getOpenAIModel("gpt-6-luna"); - await handleErrorToolWithImageResult(model, openaiOptions({ maxTokens: 1_024 }), "openai"); + await handleErrorToolWithImageResult(model, openaiOptions({ maxTokens: 1_024 }), "openai", responsesTail); }); }); describeIfOpenAICodex("OpenAI Codex provider", () => { it("gpt-6-luna", { retry: 2, timeout: 120_000 }, async () => { const model = await getOpenAICodexModel("gpt-6-luna"); - await handleErrorToolWithImageResult(model, openaiCodexOptions(), "openai-codex"); + await handleErrorToolWithImageResult(model, openaiCodexOptions(), "openai-codex", responsesTail); }); }); describeIfOpenRouter("OpenRouter provider", () => { it("meta/muse-spark-1.3-contributor", { retry: 2, timeout: 60_000 }, async () => { const model = await getOpenRouterModel("meta/muse-spark-1.3-contributor"); - await handleErrorToolWithImageResult(model, openrouterOptions(), "openrouter"); + await handleErrorToolWithImageResult(model, openrouterOptions(), "openrouter", chatTail); }); }); describeIfGoogle("Google provider", () => { it("gemini-3.5-flash", { retry: 2, timeout: 90_000 }, async () => { const model = await getGoogleModel("gemini-3.5-flash"); - await handleErrorToolWithImageResult(model, googleOptions({ maxTokens: 4_096 }), "google"); + await handleErrorToolWithImageResult(model, googleOptions({ maxTokens: 4_096 }), "google", googleTail); }); }); }); diff --git a/packages/aikit/test/transform.test.ts b/packages/aikit/test/transform.test.ts index cb212aa1..82e67e8d 100644 --- a/packages/aikit/test/transform.test.ts +++ b/packages/aikit/test/transform.test.ts @@ -786,6 +786,44 @@ describe("Anthropic error tool results with images", () => { }); }); + it("sends the images of every error result after all tool results, in call order", async () => { + const messages = await wireMessages([ + makeErrorToolCall("call-1", [{ type: "image", data: "AAAA", mimeType: "image/png" }], "screenshot"), + makeCompletedToolCall("call-2", "search", [{ type: "text", text: "found" }]), + makeErrorToolCall( + "call-3", + [ + { type: "text", text: "Render failed" }, + { type: "image", data: "BBBB", mimeType: "image/png" }, + ], + "render", + ), + ]); + + expect(messages[2]).toEqual({ + role: "user", + content: [ + { + type: "tool_result", + tool_use_id: "call-1", + content: "(image attached after the tool results)", + is_error: true, + }, + { type: "tool_result", tool_use_id: "call-2", content: "found" }, + { + type: "tool_result", + tool_use_id: "call-3", + content: "Render failed\n(image attached after the tool results)", + is_error: true, + }, + { type: "text", text: "Images from the error result of tool call call-1 (screenshot):" }, + image("AAAA"), + { type: "text", text: "Images from the error result of tool call call-3 (render):" }, + image("BBBB"), + ], + }); + }); + it("keeps error tool results without images unchanged", async () => { const messages = await wireMessages([ makeErrorToolCall("call-1", [{ type: "text", text: "Script failed" }], "codemode"), From 3587269391e3de09924401500670ac3801278f84 Mon Sep 17 00:00:00 2001 From: Sanchit Rk <0xsanchit@gmail.com> Date: Mon, 26 Oct 2026 18:17:48 +0530 Subject: [PATCH 4/4] fix(aikit): move tool result images after the results where the wire cannot carry them Chat Completions adapters serialize a tool result's images as JSON text, and the Google adapter's legacy shape (before Gemini 3) drops the function response of an image-only result. On those wires a tool result with images now goes out as text, with "(see attached image)" when it has none, and its images follow the turn's tool results in a user message, as Anthropic error results already did. The wire follows the method the request resolves to, so Copilot and OpenAI method overrides are honoured. The live image tests now cover OpenAI over Chat Completions and Gemini 2.5, and the text-and-image prompt asks for what the model learned and the shape's color. Co-Authored-By: Claude Opus 5.5 --- packages/aikit/src/llm/registry.ts | 7 +- packages/aikit/src/llm/stream.ts | 2 +- packages/aikit/src/llm/transform.ts | 112 ++++++++-- .../error-tool-image.google-gemini-2.json | 31 +++ .../error-tool-image.openai-chat.json | 22 ++ .../test/e2e/image.tool.error.e2e.test.ts | 22 ++ .../test/e2e/image.tool.result.e2e.test.ts | 43 +++- packages/aikit/test/transform.test.ts | 201 +++++++++++++++++- 8 files changed, 409 insertions(+), 31 deletions(-) create mode 100644 packages/aikit/test/e2e/__artifacts__/error-tool-image.google-gemini-2.json create mode 100644 packages/aikit/test/e2e/__artifacts__/error-tool-image.openai-chat.json diff --git a/packages/aikit/src/llm/registry.ts b/packages/aikit/src/llm/registry.ts index 0667e673..32c2d76c 100644 --- a/packages/aikit/src/llm/registry.ts +++ b/packages/aikit/src/llm/registry.ts @@ -83,13 +83,18 @@ function defaultMethodForModel(model: Model.Info): Model.APIMethodEnum { return Model.APIMethodEnum.languageModel; } +/** The endpoint method a model is called through: the request's override, the catalog's, or the default. */ +export function resolveMethod(model: Model.Info, methodOverride?: Model.APIMethodEnum): Model.APIMethodEnum { + return methodOverride ?? model.api?.method ?? defaultMethodForModel(model); +} + export function resolveLanguageModel( provider: unknown, model: Model.Info, methodOverride?: Model.APIMethodEnum, ): LanguageModel { const modelId = model.api?.id ?? model.id; - const method = methodOverride ?? model.api?.method ?? defaultMethodForModel(model); + const method = resolveMethod(model, methodOverride); const callable = provider as ProviderWithMethods; if (method === Model.APIMethodEnum.responses && callable.responses) return callable.responses(modelId); diff --git a/packages/aikit/src/llm/stream.ts b/packages/aikit/src/llm/stream.ts index 2ac4d97f..aa157d7f 100644 --- a/packages/aikit/src/llm/stream.ts +++ b/packages/aikit/src/llm/stream.ts @@ -403,7 +403,7 @@ export const stream: Protocol.StreamFunction >[0]["providerOptions"]; diff --git a/packages/aikit/src/llm/transform.ts b/packages/aikit/src/llm/transform.ts index bcf507c1..8f96cb1d 100644 --- a/packages/aikit/src/llm/transform.ts +++ b/packages/aikit/src/llm/transform.ts @@ -16,6 +16,7 @@ import { resolveOpenAICodexToolConstraint } from "../providers/openai-codex/code import { shortHash } from "../utils/hash.ts"; import { parseStreamingJson } from "../utils/jsonparse.ts"; import { sanitizeSurrogates } from "../utils/sanitize.ts"; +import { packageForModel, resolveMethod } from "./registry.ts"; import { resolveStrictParameters } from "./strict.ts"; import { isAnthropicMessagesModel } from "./thinking.ts"; @@ -84,25 +85,72 @@ function sanitizeRecord(value: Record): Record } const ERROR_TOOL_IMAGE_PLACEHOLDER = "(image attached after the tool results)"; +const MOVED_TOOL_IMAGE_PLACEHOLDER = "(see attached image)"; +const CHAT_TOOL_IMAGES_CAPTION = "Attached image(s) from tool result:"; +const GEMINI_TOOL_IMAGES_CAPTION = "Tool result image:"; -// Anthropic rejects images in error tool results, so they follow the turn's tool results instead. -function splitsErrorImages(toolCall: TerminalToolCall, model: Model.Info): boolean { - return ( - toolCall.result.isError && - isAnthropicMessagesModel(model) && - toolCall.result.content.some((content) => content.type === "image") - ); +/** + * Where a tool result's images go when the wire cannot carry them inside the result. Each case + * sends them in a user message after the turn's tool results: + * - `anthropicError`: Anthropic rejects images in error tool results. + * - `chat`: Chat Completions adapters serialize tool result images as JSON text. + * - `gemini`: before Gemini 3 (and for non-Gemini models), an image-only result loses its function response. + */ +type MovedImages = "anthropicError" | "chat" | "gemini"; + +const COPILOT = "@codeworksh/ai-sdk-github-copilot"; + +function isAnthropicWire(model: Model.Info, method: Model.APIMethodEnum): boolean { + if (packageForModel(model) === COPILOT) return method === Model.APIMethodEnum.messages; + return isAnthropicMessagesModel(model); +} + +function isChatCompletions(model: Model.Info, method: Model.APIMethodEnum): boolean { + const npm = packageForModel(model); + // The compatible adapter falls back to its chat model for every other method. + if (npm === "@ai-sdk/openai-compatible") return true; + if (npm === COPILOT) return method !== Model.APIMethodEnum.responses && method !== Model.APIMethodEnum.messages; + return npm === "@ai-sdk/openai" && method === Model.APIMethodEnum.chat; +} + +/** The Google adapter's own split: anything but Gemini 3 and later gets the legacy function response. */ +function isLegacyGoogleToolResults(model: Model.Info): boolean { + if (model.protocol !== Model.KnownProviderEnum.google && model.protocol !== Model.KnownProviderEnum.googleVertex) { + return false; + } + const id = model.api?.id ?? model.id; + const gemini = /(^|[/.])gemini-/i.test(id); + const legacy = + /(^|[/.])gemini-[12](?:[.-]|$)/i.test(id) || + /(^|[/.])gemini-pro(?:-vision)?$/i.test(id) || + /(^|[/.])gemini-robotics-er-1\.5(?:[.-]|$)/i.test(id); + return !gemini || legacy; } -function toolResultOutput(toolCall: TerminalToolCall, model: Model.Info) { - const split = splitsErrorImages(toolCall, model); - const content = split - ? Message.omitImages(toolCall.result.content, ERROR_TOOL_IMAGE_PLACEHOLDER) - : toolCall.result.content; - const text = content +function movedImages( + toolCall: TerminalToolCall, + model: Model.Info, + method: Model.APIMethodEnum, +): MovedImages | undefined { + if (!toolCall.result.content.some((content) => content.type === "image")) return; + if (isAnthropicWire(model, method)) return toolCall.result.isError ? "anthropicError" : undefined; + if (isChatCompletions(model, method)) return "chat"; + if (isLegacyGoogleToolResults(model)) return "gemini"; +} + +function toolResultOutput(toolCall: TerminalToolCall, moved: MovedImages | undefined) { + const content = + moved === "anthropicError" + ? Message.omitImages(toolCall.result.content, ERROR_TOOL_IMAGE_PLACEHOLDER) + : moved + ? toolCall.result.content.filter((content) => content.type === "text") + : toolCall.result.content; + const joined = content .filter((content) => content.type === "text") .map((content) => sanitizeSurrogates(content.text)) .join("\n"); + const text = + moved && moved !== "anthropicError" && joined.trim().length === 0 ? MOVED_TOOL_IMAGE_PLACEHOLDER : joined; const images = content.filter((content) => content.type === "image"); // The AI SDK's content output has no error flag, so other providers keep the images and lose @@ -228,12 +276,16 @@ export function googleThoughtSignature(metadata: unknown): string | undefined { } } -function assistantMessages(message: Message.AssistantMessage, model: Model.Info): ModelMessage[] { +function assistantMessages( + message: Message.AssistantMessage, + model: Model.Info, + method: Model.APIMethodEnum, +): ModelMessage[] { if (message.stopReason === "error" || message.stopReason === "aborted") return []; const assistantContent: Exclude = []; const toolResults: ToolModelMessage["content"] = []; - const errorImages: Exclude = []; + const movedImageContent: Exclude = []; const googleReplay = (model.protocol === "google" || model.protocol === "google-vertex") && message.protocol === model.protocol && @@ -341,6 +393,7 @@ function assistantMessages(message: Message.AssistantMessage, model: Model.Info) }); const terminal = part.status === "pending" || part.status === "running" ? undefined : part; + const moved = terminal ? movedImages(terminal, model, method) : undefined; toolResults.push({ type: "tool-result", toolCallId: part.callID, @@ -349,15 +402,23 @@ function assistantMessages(message: Message.AssistantMessage, model: Model.Info) ? { providerOptions: { "openai-codex": { addedToolNames: terminal.addedToolNames } } } : {}), output: terminal - ? toolResultOutput(terminal, model) + ? toolResultOutput(terminal, moved) : { type: "error-text", value: "no result provided", }, }); - if (terminal && splitsErrorImages(terminal, model)) { - errorImages.push( - { type: "text", text: `Images from the error result of tool call ${part.callID} (${part.name}):` }, + if (terminal && moved) { + const caption = + moved === "anthropicError" + ? `Images from the error result of tool call ${part.callID} (${part.name}):` + : moved === "gemini" + ? GEMINI_TOOL_IMAGES_CAPTION + : movedImageContent.length === 0 + ? CHAT_TOOL_IMAGES_CAPTION + : undefined; + movedImageContent.push( + ...(caption === undefined ? [] : [{ type: "text" as const, text: caption }]), ...terminal.result.content.flatMap((content) => content.type === "image" ? [{ type: "file" as const, data: content.data, mediaType: content.mimeType }] @@ -375,13 +436,18 @@ function assistantMessages(message: Message.AssistantMessage, model: Model.Info) result.push({ role: "tool", content: toolResults }); } // @ai-sdk/anthropic merges this into the tool results' user message, after the tool_result blocks. - if (errorImages.length > 0) { - result.push({ role: "user", content: errorImages }); + if (movedImageContent.length > 0) { + result.push({ role: "user", content: movedImageContent }); } return result; } -export function convertMessages(context: Message.Context, model: Model.Info): ModelMessage[] { +export function convertMessages( + context: Message.Context, + model: Model.Info, + methodOverride?: Model.APIMethodEnum, +): ModelMessage[] { + const method = resolveMethod(model, methodOverride); const messages: ModelMessage[] = []; const transformedMessages = Message.transformMessages(context.messages, model, toolCallIdNormalizer(model)); @@ -390,7 +456,7 @@ export function convertMessages(context: Message.Context, model: Model.Info): Mo messages.push(...userContent(msg.parts)); continue; } - messages.push(...assistantMessages(msg, model)); + messages.push(...assistantMessages(msg, model, method)); } return messages; diff --git a/packages/aikit/test/e2e/__artifacts__/error-tool-image.google-gemini-2.json b/packages/aikit/test/e2e/__artifacts__/error-tool-image.google-gemini-2.json new file mode 100644 index 00000000..fa4b98c0 --- /dev/null +++ b/packages/aikit/test/e2e/__artifacts__/error-tool-image.google-gemini-2.json @@ -0,0 +1,31 @@ +[ + { + "role": "user", + "parts": [ + { + "functionResponse": { + "id": "", + "name": "check_screenshot", + "response": { + "name": "check_screenshot", + "error": "Screenshot assertion failed: the page did not match." + } + } + } + ] + }, + { + "role": "user", + "parts": [ + { + "text": "Tool result image:" + }, + { + "inlineData": { + "mimeType": "image/png", + "data": "" + } + } + ] + } +] \ No newline at end of file diff --git a/packages/aikit/test/e2e/__artifacts__/error-tool-image.openai-chat.json b/packages/aikit/test/e2e/__artifacts__/error-tool-image.openai-chat.json new file mode 100644 index 00000000..b1a9c8a6 --- /dev/null +++ b/packages/aikit/test/e2e/__artifacts__/error-tool-image.openai-chat.json @@ -0,0 +1,22 @@ +[ + { + "role": "tool", + "tool_call_id": "", + "content": "{\"error\":\"Screenshot assertion failed: the page did not match.\"}" + }, + { + "role": "user", + "content": [ + { + "type": "text", + "text": "Attached image(s) from tool result:" + }, + { + "type": "image_url", + "image_url": { + "url": "data:image/png;base64," + } + } + ] + } +] \ No newline at end of file diff --git a/packages/aikit/test/e2e/image.tool.error.e2e.test.ts b/packages/aikit/test/e2e/image.tool.error.e2e.test.ts index f617a451..b2f7eaa3 100644 --- a/packages/aikit/test/e2e/image.tool.error.e2e.test.ts +++ b/packages/aikit/test/e2e/image.tool.error.e2e.test.ts @@ -4,6 +4,7 @@ import { fileURLToPath } from "node:url"; import Type from "typebox"; import { describe, expect, it } from "vite-plus/test"; import * as Message from "../../src/message/message.ts"; +import * as Model from "../../src/model/model.ts"; import { complete } from "../../src/stream.ts"; import { ANTHROPIC_E2E_MODELS, @@ -145,6 +146,15 @@ describe("Error tool results with images", () => { }); }); + describeIfOpenAI("OpenAI Chat Completions", () => { + it("gpt-6-luna", { retry: 2, timeout: 60_000 }, async () => { + const model = await getOpenAIModel("gpt-6-luna"); + // gpt-6-luna takes no function tools with a reasoning effort on Chat Completions. + const { reasoning: _, ...options } = openaiOptions({ maxTokens: 1_024, method: Model.APIMethodEnum.chat }); + await handleErrorToolWithImageResult(model, options, "openai-chat", chatTail); + }); + }); + describeIfOpenAICodex("OpenAI Codex provider", () => { it("gpt-6-luna", { retry: 2, timeout: 120_000 }, async () => { const model = await getOpenAICodexModel("gpt-6-luna"); @@ -159,6 +169,18 @@ describe("Error tool results with images", () => { }); }); + describeIfGoogle("Google provider (Gemini before 3)", () => { + it("gemini-2.5-flash", { retry: 2, timeout: 90_000 }, async () => { + const model = await getGoogleModel("gemini-2.5-flash"); + await handleErrorToolWithImageResult( + model, + googleOptions({ maxTokens: 4_096 }), + "google-gemini-2", + googleTail, + ); + }); + }); + describeIfGoogle("Google provider", () => { it("gemini-3.5-flash", { retry: 2, timeout: 90_000 }, async () => { const model = await getGoogleModel("gemini-3.5-flash"); diff --git a/packages/aikit/test/e2e/image.tool.result.e2e.test.ts b/packages/aikit/test/e2e/image.tool.result.e2e.test.ts index c35e8e3a..cee6564d 100644 --- a/packages/aikit/test/e2e/image.tool.result.e2e.test.ts +++ b/packages/aikit/test/e2e/image.tool.result.e2e.test.ts @@ -2,7 +2,7 @@ import { readFileSync } from "node:fs"; import { fileURLToPath } from "node:url"; import Type from "typebox"; import { describe, expect, it } from "vite-plus/test"; -import type { AnthropicOptions, OpenAICodexOptions, OpenAIOptions } from "../../src/llm/options.ts"; +import type { AnthropicOptions, GoogleOptions, OpenAICodexOptions, OpenAIOptions } from "../../src/llm/options.ts"; import * as Message from "../../src/message/message.ts"; import * as Model from "../../src/model/model.ts"; import { convertMessages } from "../../src/llm/transform.ts"; @@ -12,15 +12,18 @@ import { ANTHROPIC_E2E_MODELS, anthropicOptions, describeIfAnthropic, + describeIfGoogle, describeIfOpenAI, describeIfOpenAICodex, describeIfOpenRouter, OPENROUTER_E2E_MODELS, getAnthropicModel, + getGoogleModel, getOpenAIModel, getOpenAICodexModel, getOpenRouterModel, getText, + googleOptions, OPENAI_CODEX_E2E_MODELS, OPENAI_E2E_MODELS, openaiCodexOptions, @@ -32,8 +35,9 @@ type SupportedModel = | Model.TModel | Model.TModel | Model.TModel - | Model.TModel; -type SupportedOptions = AnthropicOptions | OpenAICodexOptions | OpenAIOptions; + | Model.TModel + | Model.TModel; +type SupportedOptions = AnthropicOptions | GoogleOptions | OpenAICodexOptions | OpenAIOptions; function getImageBase64(): string { const imagePath = fileURLToPath(new URL("../data/red-circle.png", import.meta.url)); @@ -147,7 +151,7 @@ async function handleToolWithTextAndImageResult(model: SupportedModel, options: parts: [ { type: "text", - text: "Use the get_circle_with_description tool and tell me the shape color plus the image metadata.", + text: "Use the get_circle_with_description tool and tell me what you learned. Also say what color the shape is.", }, ], time: { created: Date.now() }, @@ -257,6 +261,37 @@ describe("Tool Results with Images", () => { }); }); + // Chat Completions cannot carry tool result images, so they follow the tool results. + describeIfOpenAI("OpenAI Chat Completions (gpt-6-luna)", () => { + // gpt-6-luna takes no function tools with a reasoning effort on Chat Completions. + const { reasoning: _, ...options } = openaiOptions({ maxTokens: 256, method: Model.APIMethodEnum.chat }); + + it("should handle tool result with only image", { retry: 3, timeout: 30000 }, async () => { + const model = await getOpenAIModel("gpt-6-luna"); + await handleToolWithImageResult(model, options); + }); + + it("should handle tool result with text and image", { retry: 3, timeout: 30000 }, async () => { + const model = await getOpenAIModel("gpt-6-luna"); + await handleToolWithTextAndImageResult(model, options); + }); + }); + + // Gemini before 3 takes the images after the function responses; Gemini 3 takes them inside. + describeIfGoogle.each(["gemini-2.5-flash", "gemini-3.5-flash"])("Google provider (%s)", (modelId) => { + const options = googleOptions({ maxTokens: 4_096 }); + + it("should handle tool result with only image", { retry: 3, timeout: 60_000 }, async () => { + const model = await getGoogleModel(modelId); + await handleToolWithImageResult(model, options); + }); + + it("should handle tool result with text and image", { retry: 3, timeout: 60_000 }, async () => { + const model = await getGoogleModel(modelId); + await handleToolWithTextAndImageResult(model, options); + }); + }); + describeIfAnthropic.each(ANTHROPIC_E2E_MODELS)("Anthropic provider (%s)", (modelId) => { const options = anthropicOptions({ maxTokens: 256, temperature: 0 }); diff --git a/packages/aikit/test/transform.test.ts b/packages/aikit/test/transform.test.ts index 82e67e8d..86a903fd 100644 --- a/packages/aikit/test/transform.test.ts +++ b/packages/aikit/test/transform.test.ts @@ -1,3 +1,4 @@ +import type { ModelMessage } from "ai"; import Type from "typebox"; import { describe, expect, it } from "vite-plus/test"; import { @@ -685,8 +686,8 @@ describe("convertMessages", () => { }); }); - it("keeps error result images in place for providers outside the Anthropic wire", () => { - const openai = makeOpenAIModel({ input: ["text", "image"] }); + it("keeps error result images in place on the OpenAI Responses wire", () => { + const openai = makeOpenAIModel({ npm: "@ai-sdk/openai", input: ["text", "image"] }); const assistant = makeAssistantMessage(openai, { stopReason: "toolUse", parts: [ @@ -860,3 +861,199 @@ describe("Anthropic error tool results with images", () => { ]); }); }); + +describe("tool result images on wires that cannot carry them", () => { + const turn = (model: Model.Info) => + makeAssistantMessage(model, { + stopReason: "toolUse", + parts: [ + makeCompletedToolCall("call-1", "screenshot", [{ type: "image", data: "AAAA", mimeType: "image/png" }]), + makeErrorToolCall( + "call-2", + [ + { type: "text", text: "Render failed" }, + { type: "image", data: "BBBB", mimeType: "image/png" }, + ], + "render", + ), + ], + }); + const file = (data: string) => ({ type: "file", data, mediaType: "image/png" }); + const outputs = (messages: ModelMessage[]) => + messages.flatMap((message) => + message.role === "tool" ? message.content.map((part) => ("output" in part ? part.output : part)) : [], + ); + + it("sends Chat Completions tool results as text and their images in one user message", () => { + const chat = makeModel({ + npm: "@ai-sdk/openai-compatible", + protocol: "openai-compatible", + input: ["text", "image"], + }); + const messages = convertMessages({ messages: [turn(chat)] }, chat); + + expect(outputs(messages)).toEqual([ + { type: "text", value: "(see attached image)" }, + { type: "error-text", value: "Render failed" }, + ]); + expect(messages.at(-1)).toEqual({ + role: "user", + content: [{ type: "text", text: "Attached image(s) from tool result:" }, file("AAAA"), file("BBBB")], + }); + }); + + it("uses Chat Completions for OpenAI only when the request picks the chat method", () => { + const openai = makeOpenAIModel({ npm: "@ai-sdk/openai", input: ["text", "image"] }); + + expect(convertMessages({ messages: [turn(openai)] }, openai).at(-1)?.role).toBe("tool"); + expect(convertMessages({ messages: [turn(openai)] }, openai, Model.APIMethodEnum.chat).at(-1)).toMatchObject({ + role: "user", + content: [{ type: "text", text: "Attached image(s) from tool result:" }, file("AAAA"), file("BBBB")], + }); + }); + + it("sends each tool result's images after the results for Gemini before 3", () => { + const gemini = makeModel({ + id: "gemini-2.5-flash", + npm: "@ai-sdk/google", + protocol: "google", + input: ["text", "image"], + }); + const messages = convertMessages({ messages: [turn(gemini)] }, gemini); + + expect(outputs(messages)).toEqual([ + { type: "text", value: "(see attached image)" }, + { type: "error-text", value: "Render failed" }, + ]); + expect(messages.at(-1)).toEqual({ + role: "user", + content: [ + { type: "text", text: "Tool result image:" }, + file("AAAA"), + { type: "text", text: "Tool result image:" }, + file("BBBB"), + ], + }); + }); + + it("keeps images inside the function response from Gemini 3", () => { + const gemini = makeModel({ + id: "gemini-3.5-flash", + npm: "@ai-sdk/google", + protocol: "google", + input: ["text", "image"], + }); + const messages = convertMessages({ messages: [turn(gemini)] }, gemini); + + expect(messages.at(-1)?.role).toBe("tool"); + expect(outputs(messages).map((output) => (output as { type: string }).type)).toEqual(["content", "content"]); + }); + + it("follows the Copilot endpoint the request resolves to", () => { + const copilot = (method: Model.APIMethodEnum) => + makeModel({ + npm: "@codeworksh/ai-sdk-github-copilot", + protocol: "github-copilot", + input: ["text", "image"], + api: { method }, + }); + const messagesEntry = copilot(Model.APIMethodEnum.messages); + const chatEntry = copilot(Model.APIMethodEnum.chat); + const last = (model: Model.Info, method?: Model.APIMethodEnum) => + convertMessages({ messages: [turn(model)] }, model, method).at(-1); + + expect(last(messagesEntry)).toMatchObject({ + role: "user", + content: [{ type: "text", text: "Images from the error result of tool call call-2 (render):" }, file("BBBB")], + }); + expect(last(messagesEntry, Model.APIMethodEnum.chat)).toMatchObject({ + role: "user", + content: [{ type: "text", text: "Attached image(s) from tool result:" }, file("AAAA"), file("BBBB")], + }); + expect(last(chatEntry, Model.APIMethodEnum.messages)).toMatchObject({ + role: "user", + content: [{ type: "text", text: "Images from the error result of tool call call-2 (render):" }, file("BBBB")], + }); + expect(last(chatEntry, Model.APIMethodEnum.responses)?.role).toBe("tool"); + }); + + it("matches the Google adapter's legacy models, on Gemini and Vertex", () => { + const legacy = [ + "gemini-2.5-flash", + "gemini-1.5-pro", + "gemini-pro-vision", + "gemini-robotics-er-1.5", + "gemma-3-27b-it", + ]; + for (const id of legacy) { + for (const [protocol, npm] of [ + ["google", "@ai-sdk/google"], + ["google-vertex", "@ai-sdk/google-vertex"], + ] as const) { + const model = makeModel({ id, protocol, npm, input: ["text", "image"] }); + expect(convertMessages({ messages: [turn(model)] }, model).at(-1)?.role, `${protocol}/${id}`).toBe("user"); + } + } + const current = makeModel({ + id: "gemini-3.5-flash", + protocol: "google-vertex", + npm: "@ai-sdk/google-vertex", + input: ["text", "image"], + }); + expect(convertMessages({ messages: [turn(current)] }, current).at(-1)?.role).toBe("tool"); + }); + + it("keeps every round's images after its own tool results", () => { + const chat = makeModel({ + npm: "@ai-sdk/openai-compatible", + protocol: "openai-compatible", + input: ["text", "image"], + }); + const round = (callID: string, data: string) => + makeAssistantMessage(chat, { + stopReason: "toolUse", + parts: [makeCompletedToolCall(callID, "screenshot", [{ type: "image", data, mimeType: "image/png" }])], + }); + const messages = convertMessages( + { + messages: [ + makeUserMessage("go"), + round("call-1", "AAAA"), + round("call-2", "BBBB"), + makeUserMessage("next"), + ], + }, + chat, + ); + + expect(messages.map((message) => message.role)).toEqual([ + "user", + "assistant", + "tool", + "user", + "assistant", + "tool", + "user", + "user", + ]); + expect(messages[3]).toEqual({ + role: "user", + content: [{ type: "text", text: "Attached image(s) from tool result:" }, file("AAAA")], + }); + expect(messages[6]).toEqual({ + role: "user", + content: [{ type: "text", text: "Attached image(s) from tool result:" }, file("BBBB")], + }); + }); + + it("moves nothing for a text-only model, whose images are already placeholders", () => { + const chat = makeModel({ npm: "@ai-sdk/openai-compatible", protocol: "openai-compatible", input: ["text"] }); + const messages = convertMessages({ messages: [turn(chat)] }, chat); + + expect(messages.at(-1)?.role).toBe("tool"); + expect(outputs(messages)).toEqual([ + { type: "text", value: "(tool image omitted: model does not support images)" }, + { type: "error-text", value: "Render failed\n(tool image omitted: model does not support images)" }, + ]); + }); +});