diff --git a/packages/harness/src/plugin/builtin/tool/read.ts b/packages/harness/src/plugin/builtin/tool/read.ts index 55573849..d7b6dd98 100644 --- a/packages/harness/src/plugin/builtin/tool/read.ts +++ b/packages/harness/src/plugin/builtin/tool/read.ts @@ -90,14 +90,26 @@ const fsFailure = (path: string) => (error: SandboxFileSystem.FileSystemError) = : `Could not read file: ${path}`, ); -const readImage = (fs: SandboxFileSystem.Interface, resolved: string, mimeType: string) => +const NON_VISION_IMAGE_NOTE = "[Current model does not support images. The image will be omitted from this request.]"; + +const readImage = ( + fs: SandboxFileSystem.Interface, + resolved: string, + mimeType: string, + modelInput: Tool.ToolCallContext["modelInput"], +) => Effect.gen(function* () { const bytes = yield* fs.readFileBuffer(resolved); const inline: Image.Inline = yield* Image.inline(bytes, mimeType); - if (!inline.ok) return { type: "image" as const, note: `Read image file [${mimeType}]\n${inline.message}` }; + const nonVision = modelInput !== undefined && !modelInput.includes("image") ? [NON_VISION_IMAGE_NOTE] : []; + if (!inline.ok) + return { + type: "image" as const, + note: [`Read image file [${mimeType}]`, inline.message, ...nonVision].join("\n"), + }; return { type: "image" as const, - note: [`Read image file [${inline.mimeType}]`, ...inline.hints].join("\n"), + note: [`Read image file [${inline.mimeType}]`, ...inline.hints, ...nonVision].join("\n"), image: { data: inline.data, mimeType: inline.mimeType }, }; }); @@ -177,7 +189,7 @@ export const readHandler: Tool.Handler< typeof ReadSuccess, typeof ReadFailed, SandboxIO.FileSystem -> = (params) => +> = (params, ctx) => Effect.gen(function* () { const fs = yield* SandboxIO.FileSystem; const resolved = yield* ToolPath.resolveRead(fs, params.path); @@ -195,7 +207,7 @@ export const readHandler: Tool.Handler< ? Effect.succeed(head.subarray(offset, offset + length)) : fs.readBytes(resolved, offset, length), }); - if (mimeType !== undefined) return yield* readImage(fs, resolved, mimeType); + if (mimeType !== undefined) return yield* readImage(fs, resolved, mimeType, ctx.modelInput); if (head.includes(0)) return yield* fail(`Unsupported read ${params.path}: binary file.`); return yield* readText(fs, resolved, params); }).pipe(Effect.catchTag("SandboxFileSystemError", fsFailure(params.path))); diff --git a/packages/harness/src/runner/loop.ts b/packages/harness/src/runner/loop.ts index 4ec49536..25a7fd01 100644 --- a/packages/harness/src/runner/loop.ts +++ b/packages/harness/src/runner/loop.ts @@ -208,6 +208,7 @@ export const layer = (options: Options = {}) => .handle(call, { sessionId: snapshot.sessionId, messageId, + modelInput: snapshot.resolvedModel.input, onProgress: (progress) => events .publish(EventList.ToolProgress, { diff --git a/packages/harness/src/tool/executor.ts b/packages/harness/src/tool/executor.ts index 21a2c843..489e3186 100644 --- a/packages/harness/src/tool/executor.ts +++ b/packages/harness/src/tool/executor.ts @@ -58,6 +58,7 @@ export interface ProgressEvent { export interface HandleOptions { readonly sessionId?: ToolBefore["sessionId"]; readonly messageId?: ToolBefore["messageId"]; + readonly modelInput?: ToolCallContext["modelInput"]; /** Sliding-queue capacity for best-effort progress. Bounded default (64); tunable. */ readonly progressBuffer?: number; /** @@ -335,7 +336,12 @@ export const make = (tools: ReadonlyArray): E }); } - const ctx: ToolCallContext = { callID: call.callID, toolName: call.name, rawArgs: call.arguments }; + const ctx: ToolCallContext = { + callID: call.callID, + toolName: call.name, + rawArgs: call.arguments, + ...(options?.modelInput === undefined ? {} : { modelInput: options.modelInput }), + }; const hasHooks = hooks.beforeToolCall !== undefined || hooks.afterToolCall !== undefined; if (hasHooks && (options?.sessionId === undefined || options.messageId === undefined)) { diff --git a/packages/harness/test/__artifacts__/tools.read.vision.json b/packages/harness/test/__artifacts__/tools.read.vision.json new file mode 100644 index 00000000..36094334 --- /dev/null +++ b/packages/harness/test/__artifacts__/tools.read.vision.json @@ -0,0 +1,85 @@ +[ + { + "model": "text-only", + "messages": [ + { + "role": "user", + "content": [ + "read the image" + ] + } + ] + }, + { + "model": "text-only", + "messages": [ + { + "role": "user", + "content": [ + "read the image" + ] + }, + { + "role": "assistant", + "parts": [ + { + "toolCall": "read", + "result": [ + "Read image file [image/png]\n[Current model does not support images. The image will be omitted from this request.]", + "(tool image omitted: model does not support images)" + ] + }, + { + "toolCall": "read", + "result": [ + "Read image file [image/bmp]\n[Image omitted: could not be converted to a supported inline image format.]\n[Current model does not support images. The image will be omitted from this request.]" + ] + } + ] + } + ] + }, + { + "model": "vision", + "messages": [ + { + "role": "user", + "content": [ + "read the image" + ] + }, + { + "role": "assistant", + "parts": [ + { + "toolCall": "read", + "result": [ + "Read image file [image/png]\n[Current model does not support images. The image will be omitted from this request.]", + { + "image": "image/png" + } + ] + }, + { + "toolCall": "read", + "result": [ + "Read image file [image/bmp]\n[Image omitted: could not be converted to a supported inline image format.]\n[Current model does not support images. The image will be omitted from this request.]" + ] + } + ] + }, + { + "role": "assistant", + "parts": [ + "response 2" + ] + }, + { + "role": "user", + "content": [ + "what do you see now?" + ] + } + ] + } +] \ No newline at end of file diff --git a/packages/harness/test/tools.read.vision.test.ts b/packages/harness/test/tools.read.vision.test.ts new file mode 100644 index 00000000..97d50e4a --- /dev/null +++ b/packages/harness/test/tools.read.vision.test.ts @@ -0,0 +1,128 @@ +import "./utils/env.ts"; +import { createAssistantMessageEventStream, Message, type Model } from "@codeworksh/aikit"; +import { Effect } from "effect"; +import { copyFile, writeFile } from "node:fs/promises"; +import { join } from "node:path"; +import { fileURLToPath } from "node:url"; +import { describe, expect, it } from "vite-plus/test"; +import { Harness } from "../src/effect/harness.ts"; +import { Session } from "../src/effect/session.ts"; +import type { LLM } from "../src/runner/llm.ts"; +import { assistant } from "./fixtures/llm.ts"; +import { withSettings } from "./fixtures/settings.ts"; + +const image = fileURLToPath(new URL("../../aikit/test/data/red-circle.png", import.meta.url)); + +const model = (id: string, input: Model.Info["input"]): Model.Info => ({ + id, + name: id, + provider: { id: "scripted", name: "scripted", source: "custom", env: [] }, + baseUrl: "http://scripted.test/v1", + reasoning: false, + input, + cost: { input: 0, output: 0, cacheRead: 0, cacheWrite: 0 }, + contextWindow: 128_000, + maxTokens: 4096, + protocol: "openai", +}); + +/** A BMP header with no pixel data: detected as an image, but it cannot be converted to inline. */ +const truncatedBmp = () => { + const header = Buffer.alloc(54); + header.write("BM", 0, "ascii"); + header.writeUInt32LE(54, 10); + header.writeUInt32LE(40, 14); + header.writeInt32LE(2, 18); + header.writeInt32LE(2, 22); + header.writeUInt16LE(1, 26); + header.writeUInt16LE(24, 28); + return header; +}; + +const textOnly = model("text-only", ["text"]); +const vision = model("vision", ["text", "image"]); + +/** What the model would receive: the history after aikit's per-model transform, images reduced to their type. */ +const view = (input: LLM.Input) => + Message.transformMessages(input.context.messages, input.resolvedModel).map((message) => { + const content = (parts: ReadonlyArray) => + parts.map((part) => (part.type === "text" ? part.text : { image: part.mimeType })); + if (message.role === "user") return { role: "user", content: content(message.parts) }; + return { + role: "assistant", + parts: message.parts.map((part) => + part.type === "toolCall" && "result" in part + ? { toolCall: part.name, result: content(part.result.content) } + : part.type === "text" + ? part.text + : part.type, + ), + }; + }); + +describe("read on a text-only model, then a model switch", () => { + it("tells a text-only model each image is omitted and sends them once the model sees images", () => + withSettings(async ({ root, custom }) => { + await copyFile(image, join(root, "red-circle.png")); + await writeFile(join(root, "broken.bmp"), truncatedBmp()); + const select = (selected: Model.Info) => + writeFile( + join(custom, "settings.jsonc"), + JSON.stringify({ model: { provider: "scripted", id: selected.id }, models: [textOnly, vision] }), + ); + const requests: Array> = []; + const open: LLM.Open = (input) => + Effect.sync(() => { + requests.push({ model: input.resolvedModel.id, messages: view(input) }); + const first = requests.length === 1; + const message = assistant( + input, + requests.length, + first + ? { + stopReason: "toolUse", + parts: ["red-circle.png", "broken.bmp"].map((path, index) => ({ + type: "toolCall" as const, + callID: `read_${index + 1}`, + name: "read", + arguments: { path }, + status: "pending" as const, + time: { start: 1, end: 1 }, + })), + } + : {}, + ); + const stream = createAssistantMessageEventStream(); + stream.push({ type: "start", partial: message }); + stream.push({ type: "done", reason: first ? "toolUse" : "stop", message }); + return stream; + }); + + await Effect.runPromise( + Effect.gen(function* () { + yield* Effect.promise(() => select(textOnly)); + const handle = yield* Session.create({ directory: root }); + yield* handle.run("read the image"); + yield* Effect.promise(() => select(vision)); + yield* handle.run("what do you see now?"); + yield* Effect.promise(() => + expect(JSON.stringify(requests, null, "\t")).toMatchFileSnapshot( + "./__artifacts__/tools.read.vision.json", + ), + ); + }).pipe( + Effect.provide( + Harness.layer({ + home: join(root, "home"), + hostCwd: root, + database: ":memory:", + userConfigDir: custom, + llm: open, + }), + ), + Effect.scoped, + Effect.timeout("10 seconds"), + ), + ); + })); +}); diff --git a/packages/plugin/src/tool.ts b/packages/plugin/src/tool.ts index a1b4f437..6a578da4 100644 --- a/packages/plugin/src/tool.ts +++ b/packages/plugin/src/tool.ts @@ -28,6 +28,8 @@ export const ToolCallContext = Schema.Struct({ callID: Schema.String, toolName: Schema.String, rawArgs: Schema.Record(Schema.String, Schema.Unknown), + /** Input modalities of the model the call answers, e.g. `["text", "image"]`. */ + modelInput: Schema.optional(Schema.Array(Schema.Literals(["text", "image"]))), }); export type ToolCallContext = typeof ToolCallContext.Type;