feat(harness): tell text-only models when read omits an image

packages/harness/src/plugin/builtin/tool/read.ts

@@ -90,14 +90,26 @@ const fsFailure = (path: string) => (error: SandboxFileSystem.FileSystemError) =
 				: `Could not read file: ${path}`,
 	);
 
-const readImage = (fs: SandboxFileSystem.Interface, resolved: string, mimeType: string) =>
+const NON_VISION_IMAGE_NOTE = "[Current model does not support images. The image will be omitted from this request.]";
+
+const readImage = (
+	fs: SandboxFileSystem.Interface,
+	resolved: string,
+	mimeType: string,
+	modelInput: Tool.ToolCallContext["modelInput"],
+) =>
 	Effect.gen(function* () {
 		const bytes = yield* fs.readFileBuffer(resolved);
 		const inline: Image.Inline = yield* Image.inline(bytes, mimeType);
-		if (!inline.ok) return { type: "image" as const, note: `Read image file [${mimeType}]\n${inline.message}` };
+		const nonVision = modelInput !== undefined && !modelInput.includes("image") ? [NON_VISION_IMAGE_NOTE] : [];
+		if (!inline.ok)
+			return {
+				type: "image" as const,
+				note: [`Read image file [${mimeType}]`, inline.message, ...nonVision].join("\n"),
+			};
 		return {
 			type: "image" as const,
-			note: [`Read image file [${inline.mimeType}]`, ...inline.hints].join("\n"),
+			note: [`Read image file [${inline.mimeType}]`, ...inline.hints, ...nonVision].join("\n"),
 			image: { data: inline.data, mimeType: inline.mimeType },
 		};
 	});
@@ -177,7 +189,7 @@ export const readHandler: Tool.Handler<
 	typeof ReadSuccess,
 	typeof ReadFailed,
 	SandboxIO.FileSystem
-> = (params) =>
+> = (params, ctx) =>
 	Effect.gen(function* () {
 		const fs = yield* SandboxIO.FileSystem;
 		const resolved = yield* ToolPath.resolveRead(fs, params.path);
@@ -195,7 +207,7 @@ export const readHandler: Tool.Handler<
 					? Effect.succeed(head.subarray(offset, offset + length))
 					: fs.readBytes(resolved, offset, length),
 		});
-		if (mimeType !== undefined) return yield* readImage(fs, resolved, mimeType);
+		if (mimeType !== undefined) return yield* readImage(fs, resolved, mimeType, ctx.modelInput);
 		if (head.includes(0)) return yield* fail(`Unsupported read ${params.path}: binary file.`);
 		return yield* readText(fs, resolved, params);
 	}).pipe(Effect.catchTag("SandboxFileSystemError", fsFailure(params.path)));

packages/harness/src/runner/loop.ts

@@ -208,6 +208,7 @@ export const layer = (options: Options = {}) =>
 								.handle(call, {
 									sessionId: snapshot.sessionId,
 									messageId,
+									modelInput: snapshot.resolvedModel.input,
 									onProgress: (progress) =>
 										events
 											.publish(EventList.ToolProgress, {

packages/harness/src/tool/executor.ts

@@ -58,6 +58,7 @@ export interface ProgressEvent {
 export interface HandleOptions<RProgress = never, EProgress = never> {
 	readonly sessionId?: ToolBefore["sessionId"];
 	readonly messageId?: ToolBefore["messageId"];
+	readonly modelInput?: ToolCallContext["modelInput"];
 	/** Sliding-queue capacity for best-effort progress. Bounded default (64); tunable. */
 	readonly progressBuffer?: number;
 	/**
@@ -335,7 +336,12 @@ export const make = (tools: ReadonlyArray<RegisteredTool | ToolRegistration>): E
 				});
 			}
 
-			const ctx: ToolCallContext = { callID: call.callID, toolName: call.name, rawArgs: call.arguments };
+			const ctx: ToolCallContext = {
+				callID: call.callID,
+				toolName: call.name,
+				rawArgs: call.arguments,
+				...(options?.modelInput === undefined ? {} : { modelInput: options.modelInput }),
+			};
 
 			const hasHooks = hooks.beforeToolCall !== undefined || hooks.afterToolCall !== undefined;
 			if (hasHooks && (options?.sessionId === undefined || options.messageId === undefined)) {

packages/harness/test/__artifacts__/tools.read.vision.json

@@ -0,0 +1,85 @@
+[
+	{
+		"model": "text-only",
+		"messages": [
+			{
+				"role": "user",
+				"content": [
+					"read the image"
+				]
+			}
+		]
+	},
+	{
+		"model": "text-only",
+		"messages": [
+			{
+				"role": "user",
+				"content": [
+					"read the image"
+				]
+			},
+			{
+				"role": "assistant",
+				"parts": [
+					{
+						"toolCall": "read",
+						"result": [
+							"Read image file [image/png]\n[Current model does not support images. The image will be omitted from this request.]",
+							"(tool image omitted: model does not support images)"
+						]
+					},
+					{
+						"toolCall": "read",
+						"result": [
+							"Read image file [image/bmp]\n[Image omitted: could not be converted to a supported inline image format.]\n[Current model does not support images. The image will be omitted from this request.]"
+						]
+					}
+				]
+			}
+		]
+	},
+	{
+		"model": "vision",
+		"messages": [
+			{
+				"role": "user",
+				"content": [
+					"read the image"
+				]
+			},
+			{
+				"role": "assistant",
+				"parts": [
+					{
+						"toolCall": "read",
+						"result": [
+							"Read image file [image/png]\n[Current model does not support images. The image will be omitted from this request.]",
+							{
+								"image": "image/png"
+							}
+						]
+					},
+					{
+						"toolCall": "read",
+						"result": [
+							"Read image file [image/bmp]\n[Image omitted: could not be converted to a supported inline image format.]\n[Current model does not support images. The image will be omitted from this request.]"
+						]
+					}
+				]
+			},
+			{
+				"role": "assistant",
+				"parts": [
+					"response 2"
+				]
+			},
+			{
+				"role": "user",
+				"content": [
+					"what do you see now?"
+				]
+			}
+		]
+	}
+]

packages/harness/test/tools.read.vision.test.ts

@@ -0,0 +1,128 @@
+import "./utils/env.ts";
+import { createAssistantMessageEventStream, Message, type Model } from "@codeworksh/aikit";
+import { Effect } from "effect";
+import { copyFile, writeFile } from "node:fs/promises";
+import { join } from "node:path";
+import { fileURLToPath } from "node:url";
+import { describe, expect, it } from "vite-plus/test";
+import { Harness } from "../src/effect/harness.ts";
+import { Session } from "../src/effect/session.ts";
+import type { LLM } from "../src/runner/llm.ts";
+import { assistant } from "./fixtures/llm.ts";
+import { withSettings } from "./fixtures/settings.ts";
+
+const image = fileURLToPath(new URL("../../aikit/test/data/red-circle.png", import.meta.url));
+
+const model = (id: string, input: Model.Info["input"]): Model.Info => ({
+	id,
+	name: id,
+	provider: { id: "scripted", name: "scripted", source: "custom", env: [] },
+	baseUrl: "http://scripted.test/v1",
+	reasoning: false,
+	input,
+	cost: { input: 0, output: 0, cacheRead: 0, cacheWrite: 0 },
+	contextWindow: 128_000,
+	maxTokens: 4096,
+	protocol: "openai",
+});
+
+/** A BMP header with no pixel data: detected as an image, but it cannot be converted to inline. */
+const truncatedBmp = () => {
+	const header = Buffer.alloc(54);
+	header.write("BM", 0, "ascii");
+	header.writeUInt32LE(54, 10);
+	header.writeUInt32LE(40, 14);
+	header.writeInt32LE(2, 18);
+	header.writeInt32LE(2, 22);
+	header.writeUInt16LE(1, 26);
+	header.writeUInt16LE(24, 28);
+	return header;
+};
+
+const textOnly = model("text-only", ["text"]);
+const vision = model("vision", ["text", "image"]);
+
+/** What the model would receive: the history after aikit's per-model transform, images reduced to their type. */
+const view = (input: LLM.Input) =>
+	Message.transformMessages(input.context.messages, input.resolvedModel).map((message) => {
+		const content = (parts: ReadonlyArray<Message.TextContent | Message.ImageContent>) =>
+			parts.map((part) => (part.type === "text" ? part.text : { image: part.mimeType }));
+		if (message.role === "user") return { role: "user", content: content(message.parts) };
+		return {
+			role: "assistant",
+			parts: message.parts.map((part) =>
+				part.type === "toolCall" && "result" in part
+					? { toolCall: part.name, result: content(part.result.content) }
+					: part.type === "text"
+						? part.text
+						: part.type,
+			),
+		};
+	});
+
+describe("read on a text-only model, then a model switch", () => {
+	it("tells a text-only model each image is omitted and sends them once the model sees images", () =>
+		withSettings(async ({ root, custom }) => {
+			await copyFile(image, join(root, "red-circle.png"));
+			await writeFile(join(root, "broken.bmp"), truncatedBmp());
+			const select = (selected: Model.Info) =>
+				writeFile(
+					join(custom, "settings.jsonc"),
+					JSON.stringify({ model: { provider: "scripted", id: selected.id }, models: [textOnly, vision] }),
+				);
+			const requests: Array<Record<string, unknown>> = [];
+			const open: LLM.Open = (input) =>
+				Effect.sync(() => {
+					requests.push({ model: input.resolvedModel.id, messages: view(input) });
+					const first = requests.length === 1;
+					const message = assistant(
+						input,
+						requests.length,
+						first
+							? {
+									stopReason: "toolUse",
+									parts: ["red-circle.png", "broken.bmp"].map((path, index) => ({
+										type: "toolCall" as const,
+										callID: `read_${index + 1}`,
+										name: "read",
+										arguments: { path },
+										status: "pending" as const,
+										time: { start: 1, end: 1 },
+									})),
+								}
+							: {},
+					);
+					const stream = createAssistantMessageEventStream();
+					stream.push({ type: "start", partial: message });
+					stream.push({ type: "done", reason: first ? "toolUse" : "stop", message });
+					return stream;
+				});
+
+			await Effect.runPromise(
+				Effect.gen(function* () {
+					yield* Effect.promise(() => select(textOnly));
+					const handle = yield* Session.create({ directory: root });
+					yield* handle.run("read the image");
+					yield* Effect.promise(() => select(vision));
+					yield* handle.run("what do you see now?");
+					yield* Effect.promise(() =>
+						expect(JSON.stringify(requests, null, "\t")).toMatchFileSnapshot(
+							"./__artifacts__/tools.read.vision.json",
+						),
+					);
+				}).pipe(
+					Effect.provide(
+						Harness.layer({
+							home: join(root, "home"),
+							hostCwd: root,
+							database: ":memory:",
+							userConfigDir: custom,
+							llm: open,
+						}),
+					),
+					Effect.scoped,
+					Effect.timeout("10 seconds"),
+				),
+			);
+		}));
+});

packages/plugin/src/tool.ts

@@ -28,6 +28,8 @@ export const ToolCallContext = Schema.Struct({
 	callID: Schema.String,
 	toolName: Schema.String,
 	rawArgs: Schema.Record(Schema.String, Schema.Unknown),
+	/** Input modalities of the model the call answers, e.g. `["text", "image"]`. */
+	modelInput: Schema.optional(Schema.Array(Schema.Literals(["text", "image"]))),
 });
 export type ToolCallContext = typeof ToolCallContext.Type;