Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 2 additions & 2 deletions packages/aikit/src/llm/transform.ts
Original file line number Diff line number Diff line change
Expand Up @@ -326,8 +326,8 @@ function assistantMessages(
* Copilot signs reasoning differently per endpoint family: Responses
* models round-trip encoded item metadata, Messages models carry
* Anthropic signatures, and the chat endpoint cannot replay thinking
* at all. A signature from the wrong family is foreign material, so
* the text survives as plain text instead of going out unsigned.
* at all. Thinking that cannot be replayed natively survives as plain
* text instead of going out unsigned.
*/
const method = model.api?.method;
const openai = part.thinkingSignature ? decodeOpenAIReasoningSignature(part.thinkingSignature) : undefined;
Expand Down
12 changes: 6 additions & 6 deletions packages/aikit/src/message/message.ts
Original file line number Diff line number Diff line change
Expand Up @@ -363,13 +363,13 @@ export function transformMessages<TProtocol extends Model.KnownProviderEnum>(

const transformedParts = assistantMsg.parts.flatMap((block) => {
if (block.type === "thinking") {
if (isSameModel && block.thinkingSignature) return block;
// Signatures only verify on the model that made them, and readable thinking sent as
// text makes the new model treat it as part of its visible reply.
if (!isSameModel) return [];
// Signed or redacted blocks replay even when empty (OpenAI encrypted reasoning).
if (block.redacted || block.thinkingSignature) return block;
if (!block.thinking || block.thinking.trim() === "") return [];
if (isSameModel) return block;
return {
type: "text" as const,
text: block.thinking,
};
return block;
}

if (block.type === "text") {
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,20 @@
[
{
"role": "user",
"content": [
"text"
]
},
{
"role": "assistant",
"content": [
"text"
]
},
{
"role": "user",
"content": [
"text"
]
}
]
Original file line number Diff line number Diff line change
@@ -0,0 +1,20 @@
[
{
"role": "user",
"content": [
"text"
]
},
{
"role": "assistant",
"content": [
"text"
]
},
{
"role": "user",
"content": [
"text"
]
}
]
62 changes: 62 additions & 0 deletions packages/aikit/test/e2e/handoff.e2e.test.ts
Original file line number Diff line number Diff line change
@@ -1,5 +1,6 @@
import Type from "typebox";
import { expect, it } from "vite-plus/test";
import { convertMessages } from "../../src/llm/transform.ts";
import * as Message from "../../src/message/message.ts";
import { stream } from "../../src/stream.ts";
import {
Expand Down Expand Up @@ -258,3 +259,64 @@ describeIfOpenRouter.each(OPENROUTER_E2E_MODELS)("OpenRouter handoff (%s)", (mod
},
);
});

/*
* Thinking never crosses models, as reasoning or as text: the wire for the
* new model carries the old model's answer only, and the new model still
* picks the conversation up from it.
*/
async function expectThinkingDropped(source: Target, target: Target, name: string) {
const question = user("What is 17 multiplied by 19? Think it through, then give the number.");
const first = await stream.complete(source.model, { messages: [question] }, source.options);
expect(first.stopReason, first.errorMessage).toBe("stop");
const thinking = first.parts.filter((part) => part.type === "thinking");
expect(thinking.length, "expected thinking").toBeGreaterThan(0);

const context = {
systemPrompt: "You are a helpful assistant. Answer concisely.",
messages: [question, first, user("What was the result? Number only.")],
};
const wire = convertMessages(context, target.model);
const serialized = JSON.stringify(wire);
for (const part of thinking) {
if (part.thinking.trim()) expect(serialized).not.toContain(JSON.stringify(part.thinking.trim()).slice(1, -1));
if (part.thinkingSignature) expect(serialized).not.toContain(part.thinkingSignature);
}
const shape = wire.map((message) => ({
role: message.role,
content: typeof message.content === "string" ? "string" : message.content.map((part) => part.type),
}));
await expect(JSON.stringify(shape, null, "\t")).toMatchFileSnapshot(`./__artifacts__/thinking-handoff.${name}.json`);

const response = await stream.complete(target.model, context, target.options);
expect(response.stopReason, response.errorMessage).toBe("stop");
expect(getText(response)).toContain("323");
}

const THINKING_HANDOFF_ANTHROPIC_MODEL = ANTHROPIC_E2E_MODELS[0];

describeIfAnthropic("Anthropic <-> OpenAI Codex thinking handoff", () => {
it.runIf(process.env.OPENAI_CODEX_API_KEY)(
"drops Anthropic thinking on OpenAI Codex",
{ retry: 2, timeout: 180_000 },
async () => {
await expectThinkingDropped(
await anthropic(THINKING_HANDOFF_ANTHROPIC_MODEL),
await codex(OPENAI_CODEX_E2E_MODELS[0]),
"anthropic-to-openai-codex",
);
},
);

it.runIf(process.env.OPENAI_CODEX_API_KEY)(
"drops OpenAI Codex thinking on Anthropic",
{ retry: 2, timeout: 180_000 },
async () => {
await expectThinkingDropped(
await codex(OPENAI_CODEX_E2E_MODELS[0]),
await anthropic(THINKING_HANDOFF_ANTHROPIC_MODEL),
"openai-codex-to-anthropic",
);
},
);
});
71 changes: 62 additions & 9 deletions packages/aikit/test/transform.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -151,27 +151,80 @@ describe("Message.transformMessages", () => {
]);
});

it("downgrades thinking to text when handing off to another model", () => {
it("keeps redacted thinking parts without a signature for the same model", () => {
const assistant = makeAssistantMessage(sameModel, {
parts: [{ type: "thinking", thinking: "reasoning here", thinkingSignature: "sig-1" }],
parts: [{ type: "thinking", thinking: "", redacted: true }],
});

expect(expectSingleAssistant(Message.transformMessages([assistant], sameModel)).parts).toEqual([
{ type: "thinking", thinking: "reasoning here", thinkingSignature: "sig-1" },
]);

expect(expectSingleAssistant(Message.transformMessages([assistant], otherModel)).parts).toEqual([
{ type: "text", text: "reasoning here" },
{ type: "thinking", thinking: "", redacted: true },
]);
});

it("drops empty thinking parts when handing off to another model", () => {
it("drops signed, unsigned and redacted thinking when handing off to another model", () => {
const assistant = makeAssistantMessage(sameModel, {
parts: [{ type: "thinking", thinking: " ", thinkingSignature: "sig-1", redacted: true }],
parts: [
{ type: "thinking", thinking: "signed reasoning", thinkingSignature: "sig-1" },
{ type: "thinking", thinking: "unsigned reasoning" },
{ type: "thinking", thinking: "", thinkingSignature: "sig-2", redacted: true },
],
});

expect(expectSingleAssistant(Message.transformMessages([assistant], sameModel)).parts).toEqual(
assistant.parts,
);
expect(expectSingleAssistant(Message.transformMessages([assistant], otherModel)).parts).toEqual([]);
});

it("does not send another model's readable thinking as text", () => {
const assistant = makeAssistantMessage(otherModel, {
stopReason: "toolUse",
parts: [
{ type: "thinking", thinking: "Candidates: proxy, IPv6, TLS.", thinkingSignature: "reasoning_content" },
{ type: "text", text: "Let me check the TLS versions." },
makeCompletedToolCall("call-1"),
],
});

expect(expectSingleAssistant(Message.transformMessages([assistant], sameModel)).parts).toEqual([
{ type: "text", text: "Let me check the TLS versions." },
makeCompletedToolCall("call-1"),
]);
});

it("keeps thinking only from the target model when models alternate", () => {
const modelB = makeModel({ id: "model-b" });
const turn = (model: Model.Info, text: string) =>
makeAssistantMessage(model, {
parts: [
{ type: "thinking", thinking: `${text} thinking`, thinkingSignature: "reasoning_content" },
{ type: "text", text },
],
});
const messages = [
makeUserMessage("1"),
turn(sameModel, "a1"),
makeUserMessage("2"),
turn(modelB, "b"),
makeUserMessage("3"),
turn(sameModel, "a2"),
];

const parts = Message.transformMessages(messages, sameModel)
.filter((message) => message.role === "assistant")
.map((message) => (message as Message.AssistantMessage).parts);
expect(parts).toEqual([
[
{ type: "thinking", thinking: "a1 thinking", thinkingSignature: "reasoning_content" },
{ type: "text", text: "a1" },
],
[{ type: "text", text: "b" }],
[
{ type: "thinking", thinking: "a2 thinking", thinkingSignature: "reasoning_content" },
{ type: "text", text: "a2" },
],
]);
});
});

describe("text parts", () => {
Expand Down
11 changes: 9 additions & 2 deletions vite.config.ts
Original file line number Diff line number Diff line change
Expand Up @@ -165,8 +165,15 @@ export default defineConfig({
},
fmt: {
// A file snapshot is the exact bytes a test wrote, and the bundled model catalog the exact
// bytes `aikit modelgen --baseline` wrote; reformatting either would make it drift.
ignorePatterns: [...ignoredPaths, "**/__artifacts__/**", "packages/aikit/src/model/baseline.json"],
// bytes `aikit modelgen --baseline` wrote; reformatting either would make it drift. Agent
// skills and tool config stay as installed.
ignorePatterns: [
...ignoredPaths,
"**/__artifacts__/**",
"packages/aikit/src/model/baseline.json",
".agents/**",
".claude/**",
],
printWidth: 120,
useTabs: true,
tabWidth: 3,
Expand Down
Loading