diff --git a/packages/aikit/src/llm/transform.ts b/packages/aikit/src/llm/transform.ts index 8f96cb1d..c51c3fd7 100644 --- a/packages/aikit/src/llm/transform.ts +++ b/packages/aikit/src/llm/transform.ts @@ -326,8 +326,8 @@ function assistantMessages( * Copilot signs reasoning differently per endpoint family: Responses * models round-trip encoded item metadata, Messages models carry * Anthropic signatures, and the chat endpoint cannot replay thinking - * at all. A signature from the wrong family is foreign material, so - * the text survives as plain text instead of going out unsigned. + * at all. Thinking that cannot be replayed natively survives as plain + * text instead of going out unsigned. */ const method = model.api?.method; const openai = part.thinkingSignature ? decodeOpenAIReasoningSignature(part.thinkingSignature) : undefined; diff --git a/packages/aikit/src/message/message.ts b/packages/aikit/src/message/message.ts index e5ab7bb6..e16d2a77 100644 --- a/packages/aikit/src/message/message.ts +++ b/packages/aikit/src/message/message.ts @@ -363,13 +363,13 @@ export function transformMessages( const transformedParts = assistantMsg.parts.flatMap((block) => { if (block.type === "thinking") { - if (isSameModel && block.thinkingSignature) return block; + // Signatures only verify on the model that made them, and readable thinking sent as + // text makes the new model treat it as part of its visible reply. + if (!isSameModel) return []; + // Signed or redacted blocks replay even when empty (OpenAI encrypted reasoning). + if (block.redacted || block.thinkingSignature) return block; if (!block.thinking || block.thinking.trim() === "") return []; - if (isSameModel) return block; - return { - type: "text" as const, - text: block.thinking, - }; + return block; } if (block.type === "text") { diff --git a/packages/aikit/test/e2e/__artifacts__/thinking-handoff.anthropic-to-openai-codex.json b/packages/aikit/test/e2e/__artifacts__/thinking-handoff.anthropic-to-openai-codex.json new file mode 100644 index 00000000..659c2bbe --- /dev/null +++ b/packages/aikit/test/e2e/__artifacts__/thinking-handoff.anthropic-to-openai-codex.json @@ -0,0 +1,20 @@ +[ + { + "role": "user", + "content": [ + "text" + ] + }, + { + "role": "assistant", + "content": [ + "text" + ] + }, + { + "role": "user", + "content": [ + "text" + ] + } +] \ No newline at end of file diff --git a/packages/aikit/test/e2e/__artifacts__/thinking-handoff.openai-codex-to-anthropic.json b/packages/aikit/test/e2e/__artifacts__/thinking-handoff.openai-codex-to-anthropic.json new file mode 100644 index 00000000..659c2bbe --- /dev/null +++ b/packages/aikit/test/e2e/__artifacts__/thinking-handoff.openai-codex-to-anthropic.json @@ -0,0 +1,20 @@ +[ + { + "role": "user", + "content": [ + "text" + ] + }, + { + "role": "assistant", + "content": [ + "text" + ] + }, + { + "role": "user", + "content": [ + "text" + ] + } +] \ No newline at end of file diff --git a/packages/aikit/test/e2e/handoff.e2e.test.ts b/packages/aikit/test/e2e/handoff.e2e.test.ts index 160854df..e70b20cb 100644 --- a/packages/aikit/test/e2e/handoff.e2e.test.ts +++ b/packages/aikit/test/e2e/handoff.e2e.test.ts @@ -1,5 +1,6 @@ import Type from "typebox"; import { expect, it } from "vite-plus/test"; +import { convertMessages } from "../../src/llm/transform.ts"; import * as Message from "../../src/message/message.ts"; import { stream } from "../../src/stream.ts"; import { @@ -258,3 +259,64 @@ describeIfOpenRouter.each(OPENROUTER_E2E_MODELS)("OpenRouter handoff (%s)", (mod }, ); }); + +/* + * Thinking never crosses models, as reasoning or as text: the wire for the + * new model carries the old model's answer only, and the new model still + * picks the conversation up from it. + */ +async function expectThinkingDropped(source: Target, target: Target, name: string) { + const question = user("What is 17 multiplied by 19? Think it through, then give the number."); + const first = await stream.complete(source.model, { messages: [question] }, source.options); + expect(first.stopReason, first.errorMessage).toBe("stop"); + const thinking = first.parts.filter((part) => part.type === "thinking"); + expect(thinking.length, "expected thinking").toBeGreaterThan(0); + + const context = { + systemPrompt: "You are a helpful assistant. Answer concisely.", + messages: [question, first, user("What was the result? Number only.")], + }; + const wire = convertMessages(context, target.model); + const serialized = JSON.stringify(wire); + for (const part of thinking) { + if (part.thinking.trim()) expect(serialized).not.toContain(JSON.stringify(part.thinking.trim()).slice(1, -1)); + if (part.thinkingSignature) expect(serialized).not.toContain(part.thinkingSignature); + } + const shape = wire.map((message) => ({ + role: message.role, + content: typeof message.content === "string" ? "string" : message.content.map((part) => part.type), + })); + await expect(JSON.stringify(shape, null, "\t")).toMatchFileSnapshot(`./__artifacts__/thinking-handoff.${name}.json`); + + const response = await stream.complete(target.model, context, target.options); + expect(response.stopReason, response.errorMessage).toBe("stop"); + expect(getText(response)).toContain("323"); +} + +const THINKING_HANDOFF_ANTHROPIC_MODEL = ANTHROPIC_E2E_MODELS[0]; + +describeIfAnthropic("Anthropic <-> OpenAI Codex thinking handoff", () => { + it.runIf(process.env.OPENAI_CODEX_API_KEY)( + "drops Anthropic thinking on OpenAI Codex", + { retry: 2, timeout: 180_000 }, + async () => { + await expectThinkingDropped( + await anthropic(THINKING_HANDOFF_ANTHROPIC_MODEL), + await codex(OPENAI_CODEX_E2E_MODELS[0]), + "anthropic-to-openai-codex", + ); + }, + ); + + it.runIf(process.env.OPENAI_CODEX_API_KEY)( + "drops OpenAI Codex thinking on Anthropic", + { retry: 2, timeout: 180_000 }, + async () => { + await expectThinkingDropped( + await codex(OPENAI_CODEX_E2E_MODELS[0]), + await anthropic(THINKING_HANDOFF_ANTHROPIC_MODEL), + "openai-codex-to-anthropic", + ); + }, + ); +}); diff --git a/packages/aikit/test/transform.test.ts b/packages/aikit/test/transform.test.ts index 86a903fd..435b7b97 100644 --- a/packages/aikit/test/transform.test.ts +++ b/packages/aikit/test/transform.test.ts @@ -151,27 +151,80 @@ describe("Message.transformMessages", () => { ]); }); - it("downgrades thinking to text when handing off to another model", () => { + it("keeps redacted thinking parts without a signature for the same model", () => { const assistant = makeAssistantMessage(sameModel, { - parts: [{ type: "thinking", thinking: "reasoning here", thinkingSignature: "sig-1" }], + parts: [{ type: "thinking", thinking: "", redacted: true }], }); expect(expectSingleAssistant(Message.transformMessages([assistant], sameModel)).parts).toEqual([ - { type: "thinking", thinking: "reasoning here", thinkingSignature: "sig-1" }, - ]); - - expect(expectSingleAssistant(Message.transformMessages([assistant], otherModel)).parts).toEqual([ - { type: "text", text: "reasoning here" }, + { type: "thinking", thinking: "", redacted: true }, ]); }); - it("drops empty thinking parts when handing off to another model", () => { + it("drops signed, unsigned and redacted thinking when handing off to another model", () => { const assistant = makeAssistantMessage(sameModel, { - parts: [{ type: "thinking", thinking: " ", thinkingSignature: "sig-1", redacted: true }], + parts: [ + { type: "thinking", thinking: "signed reasoning", thinkingSignature: "sig-1" }, + { type: "thinking", thinking: "unsigned reasoning" }, + { type: "thinking", thinking: "", thinkingSignature: "sig-2", redacted: true }, + ], }); + expect(expectSingleAssistant(Message.transformMessages([assistant], sameModel)).parts).toEqual( + assistant.parts, + ); expect(expectSingleAssistant(Message.transformMessages([assistant], otherModel)).parts).toEqual([]); }); + + it("does not send another model's readable thinking as text", () => { + const assistant = makeAssistantMessage(otherModel, { + stopReason: "toolUse", + parts: [ + { type: "thinking", thinking: "Candidates: proxy, IPv6, TLS.", thinkingSignature: "reasoning_content" }, + { type: "text", text: "Let me check the TLS versions." }, + makeCompletedToolCall("call-1"), + ], + }); + + expect(expectSingleAssistant(Message.transformMessages([assistant], sameModel)).parts).toEqual([ + { type: "text", text: "Let me check the TLS versions." }, + makeCompletedToolCall("call-1"), + ]); + }); + + it("keeps thinking only from the target model when models alternate", () => { + const modelB = makeModel({ id: "model-b" }); + const turn = (model: Model.Info, text: string) => + makeAssistantMessage(model, { + parts: [ + { type: "thinking", thinking: `${text} thinking`, thinkingSignature: "reasoning_content" }, + { type: "text", text }, + ], + }); + const messages = [ + makeUserMessage("1"), + turn(sameModel, "a1"), + makeUserMessage("2"), + turn(modelB, "b"), + makeUserMessage("3"), + turn(sameModel, "a2"), + ]; + + const parts = Message.transformMessages(messages, sameModel) + .filter((message) => message.role === "assistant") + .map((message) => (message as Message.AssistantMessage).parts); + expect(parts).toEqual([ + [ + { type: "thinking", thinking: "a1 thinking", thinkingSignature: "reasoning_content" }, + { type: "text", text: "a1" }, + ], + [{ type: "text", text: "b" }], + [ + { type: "thinking", thinking: "a2 thinking", thinkingSignature: "reasoning_content" }, + { type: "text", text: "a2" }, + ], + ]); + }); }); describe("text parts", () => { diff --git a/vite.config.ts b/vite.config.ts index 369f3135..83247d78 100644 --- a/vite.config.ts +++ b/vite.config.ts @@ -165,8 +165,15 @@ export default defineConfig({ }, fmt: { // A file snapshot is the exact bytes a test wrote, and the bundled model catalog the exact - // bytes `aikit modelgen --baseline` wrote; reformatting either would make it drift. - ignorePatterns: [...ignoredPaths, "**/__artifacts__/**", "packages/aikit/src/model/baseline.json"], + // bytes `aikit modelgen --baseline` wrote; reformatting either would make it drift. Agent + // skills and tool config stay as installed. + ignorePatterns: [ + ...ignoredPaths, + "**/__artifacts__/**", + "packages/aikit/src/model/baseline.json", + ".agents/**", + ".claude/**", + ], printWidth: 120, useTabs: true, tabWidth: 3,