feat: agent.addToolOutput for client-side tools

CHANGELOG.md

@@ -11,6 +11,8 @@
   `output-error` (#162, #441)
 - Store a step's reasoning once, in content, so long reasoning fits the document
   size limit (#418, #423)
+- `agent.addToolOutput` saves a client-side tool's result next to its call, and
+  `createTool` accepts a tool without `execute` (#231)
 
 ## 0.7.9
 

docs/tool-approval.mdx

@@ -218,6 +218,65 @@ The `ToolUIPart` states relevant to approval are:
 | `output-denied`      | Tool was denied                                      |
 | `output-error`       | Tool execution failed                                |
 
+## Client-side tools
+
+A tool without `execute` runs in the browser. Generation stops at its call, and
+the tool part waits in the `input-available` state until the client answers it
+with `addToolOutput`. Like an approval, the output is saved next to its call,
+and a second answer is rejected. Naming the tool keeps a client from answering a
+call to any other tool. A client tool cannot set `needsApproval`: ask the user
+in the client before answering.
+
+```ts
+const getLocation = createTool({
+  description: "Get the user's location from their browser",
+  inputSchema: z.object({ precise: z.boolean() }),
+});
+
+export const submitToolOutput = mutation({
+  args: { threadId: v.string(), toolCallId: v.string(), output: v.any() },
+  handler: async (ctx, { threadId, toolCallId, output }) => {
+    // Check the user can access the thread.
+    const { messageId } = await agent.addToolOutput(ctx, {
+      threadId,
+      tool: "getLocation",
+      toolCallId,
+      output,
+    });
+    // If no call in this step is still unanswered, schedule
+    // generateResponse with promptMessageId: messageId, as submitApproval
+    // does in the example files below.
+  },
+});
+```
+
+The output is sent to the model as text (a string) or JSON; a tool's
+`toModelOutput` is not applied. Pass `state: "output-error"` and `errorText`
+instead of `output` to report a failure to the model.
+
+On the client, answer each call once its message is saved. A call seen only in
+stream deltas is not saved yet, and `addToolOutput` rejects it:
+
+```tsx
+const submitToolOutput = useMutation(api.chat.submitToolOutput);
+const answered = useRef(new Set<string>());
+useEffect(() => {
+  for (const message of messages) {
+    if (message.status !== "success") continue;
+    for (const part of message.parts) {
+      if (part.type !== "tool-getLocation") continue;
+      if (part.state !== "input-available") continue;
+      const { toolCallId } = part;
+      if (answered.current.has(toolCallId)) continue;
+      answered.current.add(toolCallId);
+      readLocation(part.input)
+        .then((output) => submitToolOutput({ threadId, toolCallId, output }))
+        .catch(() => answered.current.delete(toolCallId));
+    }
+  }
+}, [messages, threadId, submitToolOutput]);
+```
+
 ## Example files
 
 For a complete working example, see:

src/approvals.ts

@@ -1,80 +1,157 @@
-import type { Message, MessageDoc } from "./validators.js";
+import type { Infer } from "convex/values";
+import type { Message, MessageDoc, vToolResultOutput } from "./validators.js";
 
 export type ToolCallApprovalDecision = {
   approvalId: string;
   approved: boolean;
   reason?: string;
 };
 
-// The window a batch is resolved against. Approvals are near the end of a
-// thread, so one bounded page finds them.
+export type ToolCallOutput = {
+  toolCallId: string;
+  toolName: string;
+  output: Infer<typeof vToolResultOutput>;
+};
+
+// The window a batch is resolved against. Approvals and client tool calls are
+// near the end of a thread, so one bounded page finds them.
 export const APPROVAL_LOOKUP_MESSAGES = 100;
 
-export function planToolCallApprovals(
+export function planToolCallResponses(
   messages: MessageDoc[],
-  decisions: ToolCallApprovalDecision[],
+  {
+    decisions,
+    outputs,
+  }: { decisions: ToolCallApprovalDecision[]; outputs: ToolCallOutput[] },
   threadId: string,
 ): { requestMessageId: string; message: Extract<Message, { role: "tool" }> } {
-  if (decisions.length === 0) {
+  if (decisions.length + outputs.length === 0) {
     throw new Error("An approval batch must contain at least one decision");
   }
-  const pending = new Set(decisions.map(({ approvalId }) => approvalId));
-  if (pending.size !== decisions.length) {
+  const pendingApprovals = new Set(
+    decisions.map(({ approvalId }) => approvalId),
+  );
+  if (pendingApprovals.size !== decisions.length) {
     throw new Error("Duplicate approval ids in batch");
   }
+  const pendingCalls = new Set(outputs.map(({ toolCallId }) => toolCallId));
+  if (pendingCalls.size !== outputs.length) {
+    throw new Error("Duplicate tool call ids in batch");
+  }
   let requestMessageId: string | undefined;
+  const claim = (doc: MessageDoc) => {
+    if (requestMessageId && requestMessageId !== doc._id) {
+      throw new Error(
+        "Approval decisions in one batch must belong to the same request message",
+      );
+    }
+    requestMessageId = doc._id;
+  };
   const providerExecuted = new Set<string>();
+  const answeredApprovals = new Set<string>();
+  const answeredCalls = new Set<string>();
 
+  // Newest first, so a call's answers are seen before the call itself.
   for (const doc of messages) {
     if (!Array.isArray(doc.message?.content)) continue;
-    const providerCalls = new Set(
-      doc.message.content.flatMap((part) =>
-        part.type === "tool-call" && part.providerExecuted
-          ? [part.toolCallId]
-          : [],
-      ),
-    );
-    for (const part of doc.message.content) {
-      if (!("approvalId" in part) || !pending.has(part.approvalId)) continue;
+    const content = doc.message.content;
+    for (const part of content) {
       if (part.type === "tool-approval-response") {
-        throw new Error(`Approval ${part.approvalId} was already handled`);
+        answeredApprovals.add(part.approvalId);
+      } else if (part.type === "tool-result") {
+        answeredCalls.add(part.toolCallId);
       }
+    }
+    if (doc.message.role !== "assistant") continue;
+    const providerCalls = new Set<string>();
+    const approvalRequests = new Map<string, string>();
+    for (const part of content) {
+      if (part.type === "tool-call" && part.providerExecuted) {
+        providerCalls.add(part.toolCallId);
+      } else if (part.type === "tool-approval-request") {
+        approvalRequests.set(part.toolCallId, part.approvalId);
+      }
+    }
+    for (const part of content) {
       if (
         part.type === "tool-approval-request" &&
-        doc.message.role === "assistant"
+        pendingApprovals.has(part.approvalId)
       ) {
-        if (requestMessageId && requestMessageId !== doc._id) {
-          throw new Error(
-            "Approval decisions in one batch must belong to the same request message",
-          );
+        if (
+          answeredApprovals.has(part.approvalId) ||
+          answeredCalls.has(part.toolCallId)
+        ) {
+          throw new Error(`Approval ${part.approvalId} was already handled`);
         }
-        requestMessageId = doc._id;
+        claim(doc);
         if (providerCalls.has(part.toolCallId)) {
           providerExecuted.add(part.approvalId);
         }
-        pending.delete(part.approvalId);
+        pendingApprovals.delete(part.approvalId);
+      } else if (
+        part.type === "tool-call" &&
+        pendingCalls.has(part.toolCallId)
+      ) {
+        const approvalId = approvalRequests.get(part.toolCallId);
+        if (
+          answeredCalls.has(part.toolCallId) ||
+          (approvalId && answeredApprovals.has(approvalId))
+        ) {
+          throw new Error(`Tool call ${part.toolCallId} was already handled`);
+        }
+        if (approvalId) {
+          throw new Error(`Tool call ${part.toolCallId} is awaiting approval`);
+        }
+        if (part.providerExecuted) {
+          throw new Error(`Tool call ${part.toolCallId} is provider-executed`);
+        }
+        const { toolName } = outputs.find(
+          ({ toolCallId }) => toolCallId === part.toolCallId,
+        )!;
+        if (part.toolName !== toolName) {
+          throw new Error(
+            `Tool call ${part.toolCallId} is a call to ${part.toolName}, not ${toolName}`,
+          );
+        }
+        claim(doc);
+        pendingCalls.delete(part.toolCallId);
       }
     }
-    if (pending.size === 0) break;
+    if (pendingApprovals.size === 0 && pendingCalls.size === 0) break;
   }
 
-  if (pending.size > 0) {
+  if (pendingApprovals.size > 0) {
+    throw new Error(
+      `Approval request ${pendingApprovals.values().next().value} was not found in the last ${APPROVAL_LOOKUP_MESSAGES} messages of thread ${threadId}`,
+    );
+  }
+  if (pendingCalls.size > 0) {
     throw new Error(
-      `Approval request ${pending.values().next().value} was not found in the last ${APPROVAL_LOOKUP_MESSAGES} messages of thread ${threadId}`,
+      `Tool call ${pendingCalls.values().next().value} was not found in the last ${APPROVAL_LOOKUP_MESSAGES} messages of thread ${threadId}`,
     );
   }
 
   return {
     requestMessageId: requestMessageId!,
     message: {
       role: "tool",
-      content: decisions.map(({ approvalId, approved, reason }) => ({
-        type: "tool-approval-response",
-        approvalId,
-        approved,
-        reason,
-        ...(providerExecuted.has(approvalId) ? { providerExecuted: true } : {}),
-      })),
+      content: [
+        ...decisions.map(({ approvalId, approved, reason }) => ({
+          type: "tool-approval-response" as const,
+          approvalId,
+          approved,
+          reason,
+          ...(providerExecuted.has(approvalId)
+            ? { providerExecuted: true }
+            : {}),
+        })),
+        ...outputs.map(({ toolCallId, toolName, output }) => ({
+          type: "tool-result" as const,
+          toolCallId,
+          toolName,
+          output,
+        })),
+      ],
     },
   };
 }