From 46495bc9997e11fb6cd003ca77e37ae9e8fb16a3 Mon Sep 17 00:00:00 2001 From: Sanchit Rk <0xsanchit@gmail.com> Date: Mon, 26 Oct 2026 16:44:00 +0530 Subject: [PATCH 1/3] feat(harness): tell text-only models when read omits an image Tool calls now carry the input modalities of the model they answer (`ToolCallContext.modelInput`). On a model without image input, `read` adds a note that the image will be omitted, next to the placeholder the transform puts in its place. A session test reads an image on a text-only model, switches to a vision model and snapshots both requests. Co-Authored-By: Claude Opus 5.5 --- .../harness/src/plugin/builtin/tool/read.ts | 22 +++- packages/harness/src/runner/loop.ts | 1 + packages/harness/src/tool/executor.ts | 8 +- .../test/__artifacts__/tools.read.vision.json | 73 +++++++++++ .../harness/test/tools.read.vision.test.ts | 116 ++++++++++++++++++ packages/plugin/src/tool.ts | 2 + 6 files changed, 216 insertions(+), 6 deletions(-) create mode 100644 packages/harness/test/__artifacts__/tools.read.vision.json create mode 100644 packages/harness/test/tools.read.vision.test.ts diff --git a/packages/harness/src/plugin/builtin/tool/read.ts b/packages/harness/src/plugin/builtin/tool/read.ts index 55573849..d7b6dd98 100644 --- a/packages/harness/src/plugin/builtin/tool/read.ts +++ b/packages/harness/src/plugin/builtin/tool/read.ts @@ -90,14 +90,26 @@ const fsFailure = (path: string) => (error: SandboxFileSystem.FileSystemError) = : `Could not read file: ${path}`, ); -const readImage = (fs: SandboxFileSystem.Interface, resolved: string, mimeType: string) => +const NON_VISION_IMAGE_NOTE = "[Current model does not support images. The image will be omitted from this request.]"; + +const readImage = ( + fs: SandboxFileSystem.Interface, + resolved: string, + mimeType: string, + modelInput: Tool.ToolCallContext["modelInput"], +) => Effect.gen(function* () { const bytes = yield* fs.readFileBuffer(resolved); const inline: Image.Inline = yield* Image.inline(bytes, mimeType); - if (!inline.ok) return { type: "image" as const, note: `Read image file [${mimeType}]\n${inline.message}` }; + const nonVision = modelInput !== undefined && !modelInput.includes("image") ? [NON_VISION_IMAGE_NOTE] : []; + if (!inline.ok) + return { + type: "image" as const, + note: [`Read image file [${mimeType}]`, inline.message, ...nonVision].join("\n"), + }; return { type: "image" as const, - note: [`Read image file [${inline.mimeType}]`, ...inline.hints].join("\n"), + note: [`Read image file [${inline.mimeType}]`, ...inline.hints, ...nonVision].join("\n"), image: { data: inline.data, mimeType: inline.mimeType }, }; }); @@ -177,7 +189,7 @@ export const readHandler: Tool.Handler< typeof ReadSuccess, typeof ReadFailed, SandboxIO.FileSystem -> = (params) => +> = (params, ctx) => Effect.gen(function* () { const fs = yield* SandboxIO.FileSystem; const resolved = yield* ToolPath.resolveRead(fs, params.path); @@ -195,7 +207,7 @@ export const readHandler: Tool.Handler< ? Effect.succeed(head.subarray(offset, offset + length)) : fs.readBytes(resolved, offset, length), }); - if (mimeType !== undefined) return yield* readImage(fs, resolved, mimeType); + if (mimeType !== undefined) return yield* readImage(fs, resolved, mimeType, ctx.modelInput); if (head.includes(0)) return yield* fail(`Unsupported read ${params.path}: binary file.`); return yield* readText(fs, resolved, params); }).pipe(Effect.catchTag("SandboxFileSystemError", fsFailure(params.path))); diff --git a/packages/harness/src/runner/loop.ts b/packages/harness/src/runner/loop.ts index 4ec49536..25a7fd01 100644 --- a/packages/harness/src/runner/loop.ts +++ b/packages/harness/src/runner/loop.ts @@ -208,6 +208,7 @@ export const layer = (options: Options = {}) => .handle(call, { sessionId: snapshot.sessionId, messageId, + modelInput: snapshot.resolvedModel.input, onProgress: (progress) => events .publish(EventList.ToolProgress, { diff --git a/packages/harness/src/tool/executor.ts b/packages/harness/src/tool/executor.ts index 21a2c843..489e3186 100644 --- a/packages/harness/src/tool/executor.ts +++ b/packages/harness/src/tool/executor.ts @@ -58,6 +58,7 @@ export interface ProgressEvent { export interface HandleOptions { readonly sessionId?: ToolBefore["sessionId"]; readonly messageId?: ToolBefore["messageId"]; + readonly modelInput?: ToolCallContext["modelInput"]; /** Sliding-queue capacity for best-effort progress. Bounded default (64); tunable. */ readonly progressBuffer?: number; /** @@ -335,7 +336,12 @@ export const make = (tools: ReadonlyArray): E }); } - const ctx: ToolCallContext = { callID: call.callID, toolName: call.name, rawArgs: call.arguments }; + const ctx: ToolCallContext = { + callID: call.callID, + toolName: call.name, + rawArgs: call.arguments, + ...(options?.modelInput === undefined ? {} : { modelInput: options.modelInput }), + }; const hasHooks = hooks.beforeToolCall !== undefined || hooks.afterToolCall !== undefined; if (hasHooks && (options?.sessionId === undefined || options.messageId === undefined)) { diff --git a/packages/harness/test/__artifacts__/tools.read.vision.json b/packages/harness/test/__artifacts__/tools.read.vision.json new file mode 100644 index 00000000..d6616305 --- /dev/null +++ b/packages/harness/test/__artifacts__/tools.read.vision.json @@ -0,0 +1,73 @@ +[ + { + "model": "text-only", + "messages": [ + { + "role": "user", + "content": [ + "read the image" + ] + } + ] + }, + { + "model": "text-only", + "messages": [ + { + "role": "user", + "content": [ + "read the image" + ] + }, + { + "role": "assistant", + "parts": [ + { + "toolCall": "read", + "result": [ + "Read image file [image/png]\n[Current model does not support images. The image will be omitted from this request.]", + "(tool image omitted: model does not support images)" + ] + } + ] + } + ] + }, + { + "model": "vision", + "messages": [ + { + "role": "user", + "content": [ + "read the image" + ] + }, + { + "role": "assistant", + "parts": [ + { + "toolCall": "read", + "result": [ + "Read image file [image/png]\n[Current model does not support images. The image will be omitted from this request.]", + { + "image": "image/png" + } + ] + } + ] + }, + { + "role": "assistant", + "parts": [ + "response 2" + ] + }, + { + "role": "user", + "content": [ + "what do you see now?" + ] + } + ] + } +] \ No newline at end of file diff --git a/packages/harness/test/tools.read.vision.test.ts b/packages/harness/test/tools.read.vision.test.ts new file mode 100644 index 00000000..e269e388 --- /dev/null +++ b/packages/harness/test/tools.read.vision.test.ts @@ -0,0 +1,116 @@ +import "./utils/env.ts"; +import { createAssistantMessageEventStream, Message, type Model } from "@codeworksh/aikit"; +import { Effect } from "effect"; +import { copyFile, writeFile } from "node:fs/promises"; +import { join } from "node:path"; +import { fileURLToPath } from "node:url"; +import { describe, expect, it } from "vite-plus/test"; +import { Harness } from "../src/effect/harness.ts"; +import { Session } from "../src/effect/session.ts"; +import type { LLM } from "../src/runner/llm.ts"; +import { assistant } from "./fixtures/llm.ts"; +import { withSettings } from "./fixtures/settings.ts"; + +const image = fileURLToPath(new URL("../../aikit/test/data/red-circle.png", import.meta.url)); + +const model = (id: string, input: Model.Info["input"]): Model.Info => ({ + id, + name: id, + provider: { id: "scripted", name: "scripted", source: "custom", env: [] }, + baseUrl: "http://scripted.test/v1", + reasoning: false, + input, + cost: { input: 0, output: 0, cacheRead: 0, cacheWrite: 0 }, + contextWindow: 128_000, + maxTokens: 4096, + protocol: "openai", +}); + +const textOnly = model("text-only", ["text"]); +const vision = model("vision", ["text", "image"]); + +/** What the model would receive: the history after aikit's per-model transform, images reduced to their type. */ +const view = (input: LLM.Input) => + Message.transformMessages(input.context.messages, input.resolvedModel).map((message) => { + const content = (parts: ReadonlyArray) => + parts.map((part) => (part.type === "text" ? part.text : { image: part.mimeType })); + if (message.role === "user") return { role: "user", content: content(message.parts) }; + return { + role: "assistant", + parts: message.parts.map((part) => + part.type === "toolCall" && "result" in part + ? { toolCall: part.name, result: content(part.result.content) } + : part.type === "text" + ? part.text + : part.type, + ), + }; + }); + +describe("read on a text-only model, then a model switch", () => { + it("tells a text-only model the image is omitted and sends it once the model sees images", () => + withSettings(async ({ root, custom }) => { + await copyFile(image, join(root, "red-circle.png")); + const select = (selected: Model.Info) => + writeFile( + join(custom, "settings.jsonc"), + JSON.stringify({ model: { provider: "scripted", id: selected.id }, models: [textOnly, vision] }), + ); + const requests: Array> = []; + const open: LLM.Open = (input) => + Effect.sync(() => { + requests.push({ model: input.resolvedModel.id, messages: view(input) }); + const first = requests.length === 1; + const message = assistant( + input, + requests.length, + first + ? { + stopReason: "toolUse", + parts: [ + { + type: "toolCall", + callID: "read_1", + name: "read", + arguments: { path: "red-circle.png" }, + status: "pending", + time: { start: 1, end: 1 }, + }, + ], + } + : {}, + ); + const stream = createAssistantMessageEventStream(); + stream.push({ type: "start", partial: message }); + stream.push({ type: "done", reason: first ? "toolUse" : "stop", message }); + return stream; + }); + + await Effect.runPromise( + Effect.gen(function* () { + yield* Effect.promise(() => select(textOnly)); + const handle = yield* Session.create({ directory: root }); + yield* handle.run("read the image"); + yield* Effect.promise(() => select(vision)); + yield* handle.run("what do you see now?"); + yield* Effect.promise(() => + expect(JSON.stringify(requests, null, "\t")).toMatchFileSnapshot( + "./__artifacts__/tools.read.vision.json", + ), + ); + }).pipe( + Effect.provide( + Harness.layer({ + home: join(root, "home"), + hostCwd: root, + database: ":memory:", + userConfigDir: custom, + llm: open, + }), + ), + Effect.scoped, + Effect.timeout("10 seconds"), + ), + ); + })); +}); diff --git a/packages/plugin/src/tool.ts b/packages/plugin/src/tool.ts index a1b4f437..6a578da4 100644 --- a/packages/plugin/src/tool.ts +++ b/packages/plugin/src/tool.ts @@ -28,6 +28,8 @@ export const ToolCallContext = Schema.Struct({ callID: Schema.String, toolName: Schema.String, rawArgs: Schema.Record(Schema.String, Schema.Unknown), + /** Input modalities of the model the call answers, e.g. `["text", "image"]`. */ + modelInput: Schema.optional(Schema.Array(Schema.Literals(["text", "image"]))), }); export type ToolCallContext = typeof ToolCallContext.Type; From fc6e9a871f7e1739e4018d472eae0ce44d769667 Mon Sep 17 00:00:00 2001 From: Sanchit Rk <0xsanchit@gmail.com> Date: Mon, 26 Oct 2026 17:28:34 +0530 Subject: [PATCH 2/3] chore: skip formatting for agent skills and tool config Co-Authored-By: Claude Opus 5.5 --- vite.config.ts | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/vite.config.ts b/vite.config.ts index 369f3135..83247d78 100644 --- a/vite.config.ts +++ b/vite.config.ts @@ -165,8 +165,15 @@ export default defineConfig({ }, fmt: { // A file snapshot is the exact bytes a test wrote, and the bundled model catalog the exact - // bytes `aikit modelgen --baseline` wrote; reformatting either would make it drift. - ignorePatterns: [...ignoredPaths, "**/__artifacts__/**", "packages/aikit/src/model/baseline.json"], + // bytes `aikit modelgen --baseline` wrote; reformatting either would make it drift. Agent + // skills and tool config stay as installed. + ignorePatterns: [ + ...ignoredPaths, + "**/__artifacts__/**", + "packages/aikit/src/model/baseline.json", + ".agents/**", + ".claude/**", + ], printWidth: 120, useTabs: true, tabWidth: 3, From 76f1c1a029a15bc89c478d952e49e4e7ad88a55f Mon Sep 17 00:00:00 2001 From: Sanchit Rk <0xsanchit@gmail.com> Date: Mon, 26 Oct 2026 17:28:34 +0530 Subject: [PATCH 3/3] fix(aikit): drop thinking from other models instead of sending it as text Thinking from a different provider, protocol or model is now dropped, signed, unsigned and redacted alike. Sent as text, it made the new model read the old model's reasoning as part of its own replies, and signatures only verify on the model that produced them. Text and tool calls are kept. Co-Authored-By: Claude Opus 5.5 --- packages/aikit/src/llm/transform.ts | 4 +- packages/aikit/src/message/message.ts | 12 ++-- ...ing-handoff.anthropic-to-openai-codex.json | 20 ++++++ ...ing-handoff.openai-codex-to-anthropic.json | 20 ++++++ packages/aikit/test/e2e/handoff.e2e.test.ts | 62 ++++++++++++++++ packages/aikit/test/transform.test.ts | 72 ++++++++++++++++--- 6 files changed, 173 insertions(+), 17 deletions(-) create mode 100644 packages/aikit/test/e2e/__artifacts__/thinking-handoff.anthropic-to-openai-codex.json create mode 100644 packages/aikit/test/e2e/__artifacts__/thinking-handoff.openai-codex-to-anthropic.json diff --git a/packages/aikit/src/llm/transform.ts b/packages/aikit/src/llm/transform.ts index 46f4c68d..d8875638 100644 --- a/packages/aikit/src/llm/transform.ts +++ b/packages/aikit/src/llm/transform.ts @@ -255,8 +255,8 @@ function assistantMessages(message: Message.AssistantMessage, model: Model.Info) * Copilot signs reasoning differently per endpoint family: Responses * models round-trip encoded item metadata, Messages models carry * Anthropic signatures, and the chat endpoint cannot replay thinking - * at all. A signature from the wrong family is foreign material, so - * the text survives as plain text instead of going out unsigned. + * at all. Thinking that cannot be replayed natively survives as plain + * text instead of going out unsigned. */ const method = model.api?.method; const openai = part.thinkingSignature ? decodeOpenAIReasoningSignature(part.thinkingSignature) : undefined; diff --git a/packages/aikit/src/message/message.ts b/packages/aikit/src/message/message.ts index 9c6aca02..9463d92e 100644 --- a/packages/aikit/src/message/message.ts +++ b/packages/aikit/src/message/message.ts @@ -363,13 +363,13 @@ export function transformMessages( const transformedParts = assistantMsg.parts.flatMap((block) => { if (block.type === "thinking") { - if (isSameModel && block.thinkingSignature) return block; + // Signatures only verify on the model that made them, and readable thinking sent as + // text makes the new model treat it as part of its visible reply. + if (!isSameModel) return []; + // Signed or redacted blocks replay even when empty (OpenAI encrypted reasoning). + if (block.redacted || block.thinkingSignature) return block; if (!block.thinking || block.thinking.trim() === "") return []; - if (isSameModel) return block; - return { - type: "text" as const, - text: block.thinking, - }; + return block; } if (block.type === "text") { diff --git a/packages/aikit/test/e2e/__artifacts__/thinking-handoff.anthropic-to-openai-codex.json b/packages/aikit/test/e2e/__artifacts__/thinking-handoff.anthropic-to-openai-codex.json new file mode 100644 index 00000000..659c2bbe --- /dev/null +++ b/packages/aikit/test/e2e/__artifacts__/thinking-handoff.anthropic-to-openai-codex.json @@ -0,0 +1,20 @@ +[ + { + "role": "user", + "content": [ + "text" + ] + }, + { + "role": "assistant", + "content": [ + "text" + ] + }, + { + "role": "user", + "content": [ + "text" + ] + } +] \ No newline at end of file diff --git a/packages/aikit/test/e2e/__artifacts__/thinking-handoff.openai-codex-to-anthropic.json b/packages/aikit/test/e2e/__artifacts__/thinking-handoff.openai-codex-to-anthropic.json new file mode 100644 index 00000000..659c2bbe --- /dev/null +++ b/packages/aikit/test/e2e/__artifacts__/thinking-handoff.openai-codex-to-anthropic.json @@ -0,0 +1,20 @@ +[ + { + "role": "user", + "content": [ + "text" + ] + }, + { + "role": "assistant", + "content": [ + "text" + ] + }, + { + "role": "user", + "content": [ + "text" + ] + } +] \ No newline at end of file diff --git a/packages/aikit/test/e2e/handoff.e2e.test.ts b/packages/aikit/test/e2e/handoff.e2e.test.ts index 160854df..e70b20cb 100644 --- a/packages/aikit/test/e2e/handoff.e2e.test.ts +++ b/packages/aikit/test/e2e/handoff.e2e.test.ts @@ -1,5 +1,6 @@ import Type from "typebox"; import { expect, it } from "vite-plus/test"; +import { convertMessages } from "../../src/llm/transform.ts"; import * as Message from "../../src/message/message.ts"; import { stream } from "../../src/stream.ts"; import { @@ -258,3 +259,64 @@ describeIfOpenRouter.each(OPENROUTER_E2E_MODELS)("OpenRouter handoff (%s)", (mod }, ); }); + +/* + * Thinking never crosses models, as reasoning or as text: the wire for the + * new model carries the old model's answer only, and the new model still + * picks the conversation up from it. + */ +async function expectThinkingDropped(source: Target, target: Target, name: string) { + const question = user("What is 17 multiplied by 19? Think it through, then give the number."); + const first = await stream.complete(source.model, { messages: [question] }, source.options); + expect(first.stopReason, first.errorMessage).toBe("stop"); + const thinking = first.parts.filter((part) => part.type === "thinking"); + expect(thinking.length, "expected thinking").toBeGreaterThan(0); + + const context = { + systemPrompt: "You are a helpful assistant. Answer concisely.", + messages: [question, first, user("What was the result? Number only.")], + }; + const wire = convertMessages(context, target.model); + const serialized = JSON.stringify(wire); + for (const part of thinking) { + if (part.thinking.trim()) expect(serialized).not.toContain(JSON.stringify(part.thinking.trim()).slice(1, -1)); + if (part.thinkingSignature) expect(serialized).not.toContain(part.thinkingSignature); + } + const shape = wire.map((message) => ({ + role: message.role, + content: typeof message.content === "string" ? "string" : message.content.map((part) => part.type), + })); + await expect(JSON.stringify(shape, null, "\t")).toMatchFileSnapshot(`./__artifacts__/thinking-handoff.${name}.json`); + + const response = await stream.complete(target.model, context, target.options); + expect(response.stopReason, response.errorMessage).toBe("stop"); + expect(getText(response)).toContain("323"); +} + +const THINKING_HANDOFF_ANTHROPIC_MODEL = ANTHROPIC_E2E_MODELS[0]; + +describeIfAnthropic("Anthropic <-> OpenAI Codex thinking handoff", () => { + it.runIf(process.env.OPENAI_CODEX_API_KEY)( + "drops Anthropic thinking on OpenAI Codex", + { retry: 2, timeout: 180_000 }, + async () => { + await expectThinkingDropped( + await anthropic(THINKING_HANDOFF_ANTHROPIC_MODEL), + await codex(OPENAI_CODEX_E2E_MODELS[0]), + "anthropic-to-openai-codex", + ); + }, + ); + + it.runIf(process.env.OPENAI_CODEX_API_KEY)( + "drops OpenAI Codex thinking on Anthropic", + { retry: 2, timeout: 180_000 }, + async () => { + await expectThinkingDropped( + await codex(OPENAI_CODEX_E2E_MODELS[0]), + await anthropic(THINKING_HANDOFF_ANTHROPIC_MODEL), + "openai-codex-to-anthropic", + ); + }, + ); +}); diff --git a/packages/aikit/test/transform.test.ts b/packages/aikit/test/transform.test.ts index ce140704..46cfba77 100644 --- a/packages/aikit/test/transform.test.ts +++ b/packages/aikit/test/transform.test.ts @@ -18,6 +18,7 @@ import { makeCompletedToolCall, makeModel, makePendingToolCall, + makeUserMessage, } from "./utils/fixtures.ts"; const PNG = "aGVsbG8="; @@ -147,27 +148,80 @@ describe("Message.transformMessages", () => { ]); }); - it("downgrades thinking to text when handing off to another model", () => { + it("keeps redacted thinking parts without a signature for the same model", () => { const assistant = makeAssistantMessage(sameModel, { - parts: [{ type: "thinking", thinking: "reasoning here", thinkingSignature: "sig-1" }], + parts: [{ type: "thinking", thinking: "", redacted: true }], }); expect(expectSingleAssistant(Message.transformMessages([assistant], sameModel)).parts).toEqual([ - { type: "thinking", thinking: "reasoning here", thinkingSignature: "sig-1" }, - ]); - - expect(expectSingleAssistant(Message.transformMessages([assistant], otherModel)).parts).toEqual([ - { type: "text", text: "reasoning here" }, + { type: "thinking", thinking: "", redacted: true }, ]); }); - it("drops empty thinking parts when handing off to another model", () => { + it("drops signed, unsigned and redacted thinking when handing off to another model", () => { const assistant = makeAssistantMessage(sameModel, { - parts: [{ type: "thinking", thinking: " ", thinkingSignature: "sig-1", redacted: true }], + parts: [ + { type: "thinking", thinking: "signed reasoning", thinkingSignature: "sig-1" }, + { type: "thinking", thinking: "unsigned reasoning" }, + { type: "thinking", thinking: "", thinkingSignature: "sig-2", redacted: true }, + ], }); + expect(expectSingleAssistant(Message.transformMessages([assistant], sameModel)).parts).toEqual( + assistant.parts, + ); expect(expectSingleAssistant(Message.transformMessages([assistant], otherModel)).parts).toEqual([]); }); + + it("does not send another model's readable thinking as text", () => { + const assistant = makeAssistantMessage(otherModel, { + stopReason: "toolUse", + parts: [ + { type: "thinking", thinking: "Candidates: proxy, IPv6, TLS.", thinkingSignature: "reasoning_content" }, + { type: "text", text: "Let me check the TLS versions." }, + makeCompletedToolCall("call-1"), + ], + }); + + expect(expectSingleAssistant(Message.transformMessages([assistant], sameModel)).parts).toEqual([ + { type: "text", text: "Let me check the TLS versions." }, + makeCompletedToolCall("call-1"), + ]); + }); + + it("keeps thinking only from the target model when models alternate", () => { + const modelB = makeModel({ id: "model-b" }); + const turn = (model: Model.Info, text: string) => + makeAssistantMessage(model, { + parts: [ + { type: "thinking", thinking: `${text} thinking`, thinkingSignature: "reasoning_content" }, + { type: "text", text }, + ], + }); + const messages = [ + makeUserMessage("1"), + turn(sameModel, "a1"), + makeUserMessage("2"), + turn(modelB, "b"), + makeUserMessage("3"), + turn(sameModel, "a2"), + ]; + + const parts = Message.transformMessages(messages, sameModel) + .filter((message) => message.role === "assistant") + .map((message) => (message as Message.AssistantMessage).parts); + expect(parts).toEqual([ + [ + { type: "thinking", thinking: "a1 thinking", thinkingSignature: "reasoning_content" }, + { type: "text", text: "a1" }, + ], + [{ type: "text", text: "b" }], + [ + { type: "thinking", thinking: "a2 thinking", thinkingSignature: "reasoning_content" }, + { type: "text", text: "a2" }, + ], + ]); + }); }); describe("text parts", () => {