From 46495bc9997e11fb6cd003ca77e37ae9e8fb16a3 Mon Sep 17 00:00:00 2001 From: Sanchit Rk <0xsanchit@gmail.com> Date: Mon, 26 Oct 2026 16:44:00 +0530 Subject: [PATCH 1/3] feat(harness): tell text-only models when read omits an image Tool calls now carry the input modalities of the model they answer (`ToolCallContext.modelInput`). On a model without image input, `read` adds a note that the image will be omitted, next to the placeholder the transform puts in its place. A session test reads an image on a text-only model, switches to a vision model and snapshots both requests. Co-Authored-By: Claude Opus 5.5 --- .../harness/src/plugin/builtin/tool/read.ts | 22 +++- packages/harness/src/runner/loop.ts | 1 + packages/harness/src/tool/executor.ts | 8 +- .../test/__artifacts__/tools.read.vision.json | 73 +++++++++++ .../harness/test/tools.read.vision.test.ts | 116 ++++++++++++++++++ packages/plugin/src/tool.ts | 2 + 6 files changed, 216 insertions(+), 6 deletions(-) create mode 100644 packages/harness/test/__artifacts__/tools.read.vision.json create mode 100644 packages/harness/test/tools.read.vision.test.ts diff --git a/packages/harness/src/plugin/builtin/tool/read.ts b/packages/harness/src/plugin/builtin/tool/read.ts index 55573849..d7b6dd98 100644 --- a/packages/harness/src/plugin/builtin/tool/read.ts +++ b/packages/harness/src/plugin/builtin/tool/read.ts @@ -90,14 +90,26 @@ const fsFailure = (path: string) => (error: SandboxFileSystem.FileSystemError) = : `Could not read file: ${path}`, ); -const readImage = (fs: SandboxFileSystem.Interface, resolved: string, mimeType: string) => +const NON_VISION_IMAGE_NOTE = "[Current model does not support images. The image will be omitted from this request.]"; + +const readImage = ( + fs: SandboxFileSystem.Interface, + resolved: string, + mimeType: string, + modelInput: Tool.ToolCallContext["modelInput"], +) => Effect.gen(function* () { const bytes = yield* fs.readFileBuffer(resolved); const inline: Image.Inline = yield* Image.inline(bytes, mimeType); - if (!inline.ok) return { type: "image" as const, note: `Read image file [${mimeType}]\n${inline.message}` }; + const nonVision = modelInput !== undefined && !modelInput.includes("image") ? [NON_VISION_IMAGE_NOTE] : []; + if (!inline.ok) + return { + type: "image" as const, + note: [`Read image file [${mimeType}]`, inline.message, ...nonVision].join("\n"), + }; return { type: "image" as const, - note: [`Read image file [${inline.mimeType}]`, ...inline.hints].join("\n"), + note: [`Read image file [${inline.mimeType}]`, ...inline.hints, ...nonVision].join("\n"), image: { data: inline.data, mimeType: inline.mimeType }, }; }); @@ -177,7 +189,7 @@ export const readHandler: Tool.Handler< typeof ReadSuccess, typeof ReadFailed, SandboxIO.FileSystem -> = (params) => +> = (params, ctx) => Effect.gen(function* () { const fs = yield* SandboxIO.FileSystem; const resolved = yield* ToolPath.resolveRead(fs, params.path); @@ -195,7 +207,7 @@ export const readHandler: Tool.Handler< ? Effect.succeed(head.subarray(offset, offset + length)) : fs.readBytes(resolved, offset, length), }); - if (mimeType !== undefined) return yield* readImage(fs, resolved, mimeType); + if (mimeType !== undefined) return yield* readImage(fs, resolved, mimeType, ctx.modelInput); if (head.includes(0)) return yield* fail(`Unsupported read ${params.path}: binary file.`); return yield* readText(fs, resolved, params); }).pipe(Effect.catchTag("SandboxFileSystemError", fsFailure(params.path))); diff --git a/packages/harness/src/runner/loop.ts b/packages/harness/src/runner/loop.ts index 4ec49536..25a7fd01 100644 --- a/packages/harness/src/runner/loop.ts +++ b/packages/harness/src/runner/loop.ts @@ -208,6 +208,7 @@ export const layer = (options: Options = {}) => .handle(call, { sessionId: snapshot.sessionId, messageId, + modelInput: snapshot.resolvedModel.input, onProgress: (progress) => events .publish(EventList.ToolProgress, { diff --git a/packages/harness/src/tool/executor.ts b/packages/harness/src/tool/executor.ts index 21a2c843..489e3186 100644 --- a/packages/harness/src/tool/executor.ts +++ b/packages/harness/src/tool/executor.ts @@ -58,6 +58,7 @@ export interface ProgressEvent { export interface HandleOptions { readonly sessionId?: ToolBefore["sessionId"]; readonly messageId?: ToolBefore["messageId"]; + readonly modelInput?: ToolCallContext["modelInput"]; /** Sliding-queue capacity for best-effort progress. Bounded default (64); tunable. */ readonly progressBuffer?: number; /** @@ -335,7 +336,12 @@ export const make = (tools: ReadonlyArray): E }); } - const ctx: ToolCallContext = { callID: call.callID, toolName: call.name, rawArgs: call.arguments }; + const ctx: ToolCallContext = { + callID: call.callID, + toolName: call.name, + rawArgs: call.arguments, + ...(options?.modelInput === undefined ? {} : { modelInput: options.modelInput }), + }; const hasHooks = hooks.beforeToolCall !== undefined || hooks.afterToolCall !== undefined; if (hasHooks && (options?.sessionId === undefined || options.messageId === undefined)) { diff --git a/packages/harness/test/__artifacts__/tools.read.vision.json b/packages/harness/test/__artifacts__/tools.read.vision.json new file mode 100644 index 00000000..d6616305 --- /dev/null +++ b/packages/harness/test/__artifacts__/tools.read.vision.json @@ -0,0 +1,73 @@ +[ + { + "model": "text-only", + "messages": [ + { + "role": "user", + "content": [ + "read the image" + ] + } + ] + }, + { + "model": "text-only", + "messages": [ + { + "role": "user", + "content": [ + "read the image" + ] + }, + { + "role": "assistant", + "parts": [ + { + "toolCall": "read", + "result": [ + "Read image file [image/png]\n[Current model does not support images. The image will be omitted from this request.]", + "(tool image omitted: model does not support images)" + ] + } + ] + } + ] + }, + { + "model": "vision", + "messages": [ + { + "role": "user", + "content": [ + "read the image" + ] + }, + { + "role": "assistant", + "parts": [ + { + "toolCall": "read", + "result": [ + "Read image file [image/png]\n[Current model does not support images. The image will be omitted from this request.]", + { + "image": "image/png" + } + ] + } + ] + }, + { + "role": "assistant", + "parts": [ + "response 2" + ] + }, + { + "role": "user", + "content": [ + "what do you see now?" + ] + } + ] + } +] \ No newline at end of file diff --git a/packages/harness/test/tools.read.vision.test.ts b/packages/harness/test/tools.read.vision.test.ts new file mode 100644 index 00000000..e269e388 --- /dev/null +++ b/packages/harness/test/tools.read.vision.test.ts @@ -0,0 +1,116 @@ +import "./utils/env.ts"; +import { createAssistantMessageEventStream, Message, type Model } from "@codeworksh/aikit"; +import { Effect } from "effect"; +import { copyFile, writeFile } from "node:fs/promises"; +import { join } from "node:path"; +import { fileURLToPath } from "node:url"; +import { describe, expect, it } from "vite-plus/test"; +import { Harness } from "../src/effect/harness.ts"; +import { Session } from "../src/effect/session.ts"; +import type { LLM } from "../src/runner/llm.ts"; +import { assistant } from "./fixtures/llm.ts"; +import { withSettings } from "./fixtures/settings.ts"; + +const image = fileURLToPath(new URL("../../aikit/test/data/red-circle.png", import.meta.url)); + +const model = (id: string, input: Model.Info["input"]): Model.Info => ({ + id, + name: id, + provider: { id: "scripted", name: "scripted", source: "custom", env: [] }, + baseUrl: "http://scripted.test/v1", + reasoning: false, + input, + cost: { input: 0, output: 0, cacheRead: 0, cacheWrite: 0 }, + contextWindow: 128_000, + maxTokens: 4096, + protocol: "openai", +}); + +const textOnly = model("text-only", ["text"]); +const vision = model("vision", ["text", "image"]); + +/** What the model would receive: the history after aikit's per-model transform, images reduced to their type. */ +const view = (input: LLM.Input) => + Message.transformMessages(input.context.messages, input.resolvedModel).map((message) => { + const content = (parts: ReadonlyArray) => + parts.map((part) => (part.type === "text" ? part.text : { image: part.mimeType })); + if (message.role === "user") return { role: "user", content: content(message.parts) }; + return { + role: "assistant", + parts: message.parts.map((part) => + part.type === "toolCall" && "result" in part + ? { toolCall: part.name, result: content(part.result.content) } + : part.type === "text" + ? part.text + : part.type, + ), + }; + }); + +describe("read on a text-only model, then a model switch", () => { + it("tells a text-only model the image is omitted and sends it once the model sees images", () => + withSettings(async ({ root, custom }) => { + await copyFile(image, join(root, "red-circle.png")); + const select = (selected: Model.Info) => + writeFile( + join(custom, "settings.jsonc"), + JSON.stringify({ model: { provider: "scripted", id: selected.id }, models: [textOnly, vision] }), + ); + const requests: Array> = []; + const open: LLM.Open = (input) => + Effect.sync(() => { + requests.push({ model: input.resolvedModel.id, messages: view(input) }); + const first = requests.length === 1; + const message = assistant( + input, + requests.length, + first + ? { + stopReason: "toolUse", + parts: [ + { + type: "toolCall", + callID: "read_1", + name: "read", + arguments: { path: "red-circle.png" }, + status: "pending", + time: { start: 1, end: 1 }, + }, + ], + } + : {}, + ); + const stream = createAssistantMessageEventStream(); + stream.push({ type: "start", partial: message }); + stream.push({ type: "done", reason: first ? "toolUse" : "stop", message }); + return stream; + }); + + await Effect.runPromise( + Effect.gen(function* () { + yield* Effect.promise(() => select(textOnly)); + const handle = yield* Session.create({ directory: root }); + yield* handle.run("read the image"); + yield* Effect.promise(() => select(vision)); + yield* handle.run("what do you see now?"); + yield* Effect.promise(() => + expect(JSON.stringify(requests, null, "\t")).toMatchFileSnapshot( + "./__artifacts__/tools.read.vision.json", + ), + ); + }).pipe( + Effect.provide( + Harness.layer({ + home: join(root, "home"), + hostCwd: root, + database: ":memory:", + userConfigDir: custom, + llm: open, + }), + ), + Effect.scoped, + Effect.timeout("10 seconds"), + ), + ); + })); +}); diff --git a/packages/plugin/src/tool.ts b/packages/plugin/src/tool.ts index a1b4f437..6a578da4 100644 --- a/packages/plugin/src/tool.ts +++ b/packages/plugin/src/tool.ts @@ -28,6 +28,8 @@ export const ToolCallContext = Schema.Struct({ callID: Schema.String, toolName: Schema.String, rawArgs: Schema.Record(Schema.String, Schema.Unknown), + /** Input modalities of the model the call answers, e.g. `["text", "image"]`. */ + modelInput: Schema.optional(Schema.Array(Schema.Literals(["text", "image"]))), }); export type ToolCallContext = typeof ToolCallContext.Type; From d42513486006373d01360a024b21eb99f08715bc Mon Sep 17 00:00:00 2001 From: Sanchit Rk <0xsanchit@gmail.com> Date: Mon, 26 Oct 2026 17:45:38 +0530 Subject: [PATCH 2/3] test(harness): cover the read note when an image cannot be inlined Co-Authored-By: Claude Opus 5.5 --- .../test/__artifacts__/tools.read.vision.json | 12 ++++++++ .../harness/test/tools.read.vision.test.ts | 28 ++++++++++++++----- 2 files changed, 33 insertions(+), 7 deletions(-) diff --git a/packages/harness/test/__artifacts__/tools.read.vision.json b/packages/harness/test/__artifacts__/tools.read.vision.json index d6616305..36094334 100644 --- a/packages/harness/test/__artifacts__/tools.read.vision.json +++ b/packages/harness/test/__artifacts__/tools.read.vision.json @@ -28,6 +28,12 @@ "Read image file [image/png]\n[Current model does not support images. The image will be omitted from this request.]", "(tool image omitted: model does not support images)" ] + }, + { + "toolCall": "read", + "result": [ + "Read image file [image/bmp]\n[Image omitted: could not be converted to a supported inline image format.]\n[Current model does not support images. The image will be omitted from this request.]" + ] } ] } @@ -53,6 +59,12 @@ "image": "image/png" } ] + }, + { + "toolCall": "read", + "result": [ + "Read image file [image/bmp]\n[Image omitted: could not be converted to a supported inline image format.]\n[Current model does not support images. The image will be omitted from this request.]" + ] } ] }, diff --git a/packages/harness/test/tools.read.vision.test.ts b/packages/harness/test/tools.read.vision.test.ts index e269e388..10a82322 100644 --- a/packages/harness/test/tools.read.vision.test.ts +++ b/packages/harness/test/tools.read.vision.test.ts @@ -26,6 +26,19 @@ const model = (id: string, input: Model.Info["input"]): Model.Info => ({ protocol: "openai", }); +/** A BMP header with no pixel data: detected as an image, but it cannot be converted to inline. */ +const truncatedBmp = () => { + const header = Buffer.alloc(54); + header.write("BM", 0, "ascii"); + header.writeUInt32LE(54, 10); + header.writeUInt32LE(40, 14); + header.writeInt32LE(2, 18); + header.writeInt32LE(2, 22); + header.writeUInt16LE(1, 26); + header.writeUInt16LE(24, 28); + return header; +}; + const textOnly = model("text-only", ["text"]); const vision = model("vision", ["text", "image"]); @@ -48,9 +61,10 @@ const view = (input: LLM.Input) => }); describe("read on a text-only model, then a model switch", () => { - it("tells a text-only model the image is omitted and sends it once the model sees images", () => + it("tells a text-only model each image is omitted and sends them once the model sees images", () => withSettings(async ({ root, custom }) => { await copyFile(image, join(root, "red-circle.png")); + await writeFile(join(root, "broken.bmp"), truncatedBmp()); const select = (selected: Model.Info) => writeFile( join(custom, "settings.jsonc"), @@ -68,14 +82,14 @@ describe("read on a text-only model, then a model switch", () => { ? { stopReason: "toolUse", parts: [ - { - type: "toolCall", - callID: "read_1", + ...["red-circle.png", "broken.bmp"].map((path, index) => ({ + type: "toolCall" as const, + callID: `read_${index + 1}`, name: "read", - arguments: { path: "red-circle.png" }, - status: "pending", + arguments: { path }, + status: "pending" as const, time: { start: 1, end: 1 }, - }, + })), ], } : {}, From 229e6a002b373854f0009031abe26f7967956028 Mon Sep 17 00:00:00 2001 From: Sanchit Rk <0xsanchit@gmail.com> Date: Mon, 26 Oct 2026 18:17:26 +0530 Subject: [PATCH 3/3] test(harness): build the read calls without a redundant spread Co-Authored-By: Claude Opus 5.5 --- .../harness/test/tools.read.vision.test.ts | 18 ++++++++---------- 1 file changed, 8 insertions(+), 10 deletions(-) diff --git a/packages/harness/test/tools.read.vision.test.ts b/packages/harness/test/tools.read.vision.test.ts index 10a82322..97d50e4a 100644 --- a/packages/harness/test/tools.read.vision.test.ts +++ b/packages/harness/test/tools.read.vision.test.ts @@ -81,16 +81,14 @@ describe("read on a text-only model, then a model switch", () => { first ? { stopReason: "toolUse", - parts: [ - ...["red-circle.png", "broken.bmp"].map((path, index) => ({ - type: "toolCall" as const, - callID: `read_${index + 1}`, - name: "read", - arguments: { path }, - status: "pending" as const, - time: { start: 1, end: 1 }, - })), - ], + parts: ["red-circle.png", "broken.bmp"].map((path, index) => ({ + type: "toolCall" as const, + callID: `read_${index + 1}`, + name: "read", + arguments: { path }, + status: "pending" as const, + time: { start: 1, end: 1 }, + })), } : {}, );