From ac1dfa92e6100ffa5bbf95cda3e3272bc2703375 Mon Sep 17 00:00:00 2001 From: Matt Carey Date: Wed, 30 Sep 2026 16:20:17 +0100 Subject: [PATCH 1/6] computer: Export named functions from trusted modules A trusted module used to be one call(method, args, context) handler, and caller code reached it through a single generic export: call("batch", requests). Every host module had to dispatch on a method string by hand, and code in the isolate could not import the functions it wanted by name. A trusted module is now an object of host functions. Each function becomes a named export, so { "ws:container": { exec } } lets caller code write import { exec } from "ws:container". Each function receives the arguments as an array and a { signal, deadline } context. Importing a name the module does not export fails when the module graph links, and the bridge only dispatches to functions the module owns, so inherited members such as toString are not reachable. The backend now checks specifiers and export names at construction rather than on the first execution. Export names must be JavaScript identifier names other than default and then. The rlm example moves ws:model to a named batch function. --- .changeset/named-trusted-module-functions.md | 7 ++ docs/17_isolate_javascript.md | 35 ++++++- examples/rlm/README.md | 6 +- examples/rlm/worker/capability.test.ts | 58 ++++++------ examples/rlm/worker/capability.ts | 20 ++-- examples/rlm/worker/prompts.ts | 4 +- examples/rlm/worker/rlm-interfaces.txt | 4 +- examples/rlm/worker/step-settings.ts | 2 +- examples/rlm/worker/structured-rlm.test.ts | 2 +- .../worker-javascript/module-graph.ts | 87 ++++++++++++++---- .../worker-javascript.test.ts | 91 ++++++++++++++----- .../worker-javascript/worker-javascript.ts | 16 +++- packages/computer/src/index.ts | 2 + packages/computer/src/runtime/bridge.test.ts | 24 ++--- packages/computer/src/runtime/bridge.ts | 34 ++++--- packages/computer/src/runtime/types.ts | 36 ++++++-- .../computer/tests/script-runner-worker.ts | 42 ++++++--- packages/computer/tests/script-runner.test.ts | 62 ++++++++++--- 18 files changed, 385 insertions(+), 147 deletions(-) create mode 100644 .changeset/named-trusted-module-functions.md diff --git a/.changeset/named-trusted-module-functions.md b/.changeset/named-trusted-module-functions.md new file mode 100644 index 00000000..5cb2ebae --- /dev/null +++ b/.changeset/named-trusted-module-functions.md @@ -0,0 +1,7 @@ +--- +"@cloudflare/computer": minor +--- + +Trusted modules now export named functions. Pass `trustedModules: { "ws:container": { exec } }` and caller code writes `import { exec } from "ws:container"`. Each host function receives `(args, { signal, deadline })`. + +This replaces the single `call(method, args, context)` handler and the generic `call` export. Move each method into its own function: `{ call(method, args) { if (method === "batch") ... } }` becomes `{ batch(args, context) { ... } }`, and `call("batch", requests)` in caller code becomes `batch(requests)`. The backend now rejects a bad specifier or export name at construction instead of at the first execution. diff --git a/docs/17_isolate_javascript.md b/docs/17_isolate_javascript.md index 2db84ecf..47e77adf 100644 --- a/docs/17_isolate_javascript.md +++ b/docs/17_isolate_javascript.md @@ -193,4 +193,37 @@ Standard output and standard error stream live. The Dynamic Worker hands the rea ## Trusted integrations -A host can configure additional reserved capability modules through `WorkerJavaScriptBackend.trustedModules`; these modules are fixed when the backend is constructed and cannot be supplied or replaced by caller source. +A host can add its own reserved modules through `WorkerJavaScriptBackend.trustedModules`. Each module is an object of host functions, and each function becomes a named export in the isolate: + +```ts +new WorkerJavaScriptBackend({ + loader: env.LOADER, + trustedModules: { + "ws:container": { + async exec(args, { signal }) { + const command = parseCommand(args); + const handle = await workspace.runtime.exec(command, { backend: "container" }); + signal.addEventListener("abort", () => void handle.kill()); + const result = await handle.result(); + return { exitCode: result.exitCode }; + }, + }, + }, +}); +``` + +Caller source imports the functions by name: + +```js +import { exec } from "ws:container"; + +export default async function () { + return exec("npm test"); +} +``` + +Each host function receives the arguments the isolate passed, as an array of JSON-compatible values, and a `{ signal, deadline }` context. The arguments come from caller code, so parse them before use. The return value must be JSON-compatible and fits within the same capability byte limits as every other host call. + +The backend checks trusted modules when it is constructed. A specifier must be a simple `ws:*` name that does not shadow `ws:git` or `ws:artifacts`. A module must export at least one function, and every export name must be a JavaScript identifier name other than `default` or `then`. A reserved word such as `delete` is allowed, and caller code renames it on import: `import { delete as remove } from "ws:files"`. Importing a name the module does not export fails when the module graph links, before any code runs. + +These modules are fixed at construction, and caller source cannot supply or replace them. diff --git a/examples/rlm/README.md b/examples/rlm/README.md index 7d08e6db..fe2b04b1 100644 --- a/examples/rlm/README.md +++ b/examples/rlm/README.md @@ -83,7 +83,7 @@ A generated module follows this shape: ```js import fs from "node:fs/promises"; -import { call as callModel } from "ws:model"; +import { batch } from "ws:model"; export default async function () { const manifest = JSON.parse( @@ -96,7 +96,7 @@ export default async function () { })), ); - const mapped = await callModel("batch", requests); + const mapped = await batch(requests); const totals = validateAndSum(mapped); return { answer: largestLabel(totals) }; } @@ -150,7 +150,7 @@ The reducer is exact relative to its inputs. Model classifications can still be The shortest path through the example is: 1. [`worker/rlm-agent.ts`](worker/rlm-agent.ts) wires together the model, Workspace, Worker JavaScript backend, executor tool, and `ws:model`. -2. [`worker/capability.ts`](worker/capability.ts) implements the bounded `ws:model("batch", requests)` interface. +2. [`worker/capability.ts`](worker/capability.ts) implements the bounded `batch(requests)` function behind `ws:model`. 3. [`worker/structured-rlm.ts`](worker/structured-rlm.ts) describes the map result and JavaScript reduction for each task family. 4. [`worker/agent-common.ts`](worker/agent-common.ts) writes the same corpus into each Computer Workspace. 5. [`worker/executor-tool.ts`](worker/executor-tool.ts) creates the native Computer executor tool and keeps its browser-facing result small. diff --git a/examples/rlm/worker/capability.test.ts b/examples/rlm/worker/capability.test.ts index d6bc9fcf..a54d9385 100644 --- a/examples/rlm/worker/capability.test.ts +++ b/examples/rlm/worker/capability.test.ts @@ -30,6 +30,10 @@ function successfulResult(text = "ok") { }; } +function context() { + return { signal: new AbortController().signal, deadline: Date.now() + 1_000 }; +} + async function waitForCalls(count: number): Promise { for (let attempt = 0; attempt < 50; attempt += 1) { if (generateTextMock.mock.calls.length >= count) return; @@ -44,28 +48,25 @@ beforeEach(() => { }); describe("recursive model batch capability", () => { - it("exposes only the batch method", async () => { + it("exposes only the batch function", () => { const capability = createModelCapability(model, hooks()); - await expect(capability.call("generate", [[]])).rejects.toThrow( - "Unknown model capability method", - ); - expect(generateTextMock).not.toHaveBeenCalled(); + expect(Object.keys(capability)).toEqual(["batch"]); }); it("strictly validates the external argument and request shapes", async () => { const capability = createModelCapability(model, hooks()); - await expect(capability.call("batch", [])).rejects.toThrow("exactly one argument"); - await expect(capability.call("batch", [null])).rejects.toThrow("non-empty array"); - await expect(capability.call("batch", [[]])).rejects.toThrow("non-empty array"); + await expect(capability.batch([], context())).rejects.toThrow("exactly one argument"); + await expect(capability.batch([null], context())).rejects.toThrow("non-empty array"); + await expect(capability.batch([[]], context())).rejects.toThrow("non-empty array"); await expect( - capability.call("batch", [[{ prompt: "classify", input: null, extra: true }], null]), + capability.batch([[{ prompt: "classify", input: null, extra: true }], null], context()), ).rejects.toThrow("exactly one argument"); await expect( - capability.call("batch", [[{ prompt: "classify", input: null, extra: true }]]), + capability.batch([[{ prompt: "classify", input: null, extra: true }]], context()), ).rejects.toThrow("Invalid batch request"); - await expect(capability.call("batch", [[{ prompt: "", input: null }]])).rejects.toThrow( + await expect(capability.batch([[{ prompt: "", input: null }]], context())).rejects.toThrow( "requires a prompt", ); expect(generateTextMock).not.toHaveBeenCalled(); @@ -78,7 +79,9 @@ describe("recursive model batch capability", () => { input: null, })); - await expect(capability.call("batch", [requests])).rejects.toThrow("cannot exceed 24 requests"); + await expect(capability.batch([requests], context())).rejects.toThrow( + "cannot exceed 24 requests", + ); expect(generateTextMock).not.toHaveBeenCalled(); }); @@ -88,7 +91,7 @@ describe("recursive model batch capability", () => { const capability = createModelCapability(model, { ...observed, admit }); const requests = [{ prompt: "classify", input: "evidence" }]; - await expect(capability.call("batch", [requests])).rejects.toThrow( + await expect(capability.batch([requests], context())).rejects.toThrow( "exhausted its child-model call budget", ); expect(admit).toHaveBeenCalledWith(1); @@ -99,7 +102,7 @@ describe("recursive model batch capability", () => { const capability = createModelCapability(model, hooks()); await expect( - capability.call("batch", [[{ prompt: "é".repeat(8 * 1024 + 1), input: null }]]), + capability.batch([[{ prompt: "é".repeat(8 * 1024 + 1), input: null }]], context()), ).rejects.toThrow("16384 bytes"); expect(generateTextMock).not.toHaveBeenCalled(); }); @@ -109,10 +112,10 @@ describe("recursive model batch capability", () => { const maximumBody = "x".repeat(48 * 1024 - 11); await expect( - capability.call("batch", [[{ prompt: "classify", input: { body: maximumBody } }]]), + capability.batch([[{ prompt: "classify", input: { body: maximumBody } }]], context()), ).resolves.toHaveLength(1); await expect( - capability.call("batch", [[{ prompt: "classify", input: { body: `${maximumBody}x` } }]]), + capability.batch([[{ prompt: "classify", input: { body: `${maximumBody}x` } }]], context()), ).rejects.toThrow("49152 bytes"); }); @@ -123,7 +126,7 @@ describe("recursive model batch capability", () => { input: { body: "x".repeat(48 * 1024 - 12) }, })); - await expect(capability.call("batch", [requests])).rejects.toThrow( + await expect(capability.batch([requests], context())).rejects.toThrow( "Model batch request cannot exceed", ); expect(generateTextMock).not.toHaveBeenCalled(); @@ -145,9 +148,10 @@ describe("recursive model batch capability", () => { }), ); const capability = createModelCapability(model, hooks()); - const resultPromise = capability.call("batch", [ - Array.from({ length: 8 }, (_, index) => ({ prompt: `request ${index}`, input: null })), - ]); + const resultPromise = capability.batch( + [Array.from({ length: 8 }, (_, index) => ({ prompt: `request ${index}`, input: null }))], + context(), + ); await waitForCalls(4); expect(generateTextMock).toHaveBeenCalledTimes(4); @@ -186,7 +190,7 @@ describe("recursive model batch capability", () => { const signal = new AbortController().signal; await expect( - capability.call("batch", [[{ prompt: "classify", input: { evidence: "safe" } }]], { + capability.batch([[{ prompt: "classify", input: { evidence: "safe" } }]], { signal, deadline: Date.now() + 1_000, }), @@ -225,7 +229,7 @@ describe("recursive model batch capability", () => { const capability = createModelCapability(model, observer); await expect( - capability.call("batch", [[{ prompt: "classify", input: null }]]), + capability.batch([[{ prompt: "classify", input: null }]], context()), ).resolves.toEqual([ { id: expect.any(String), @@ -255,8 +259,7 @@ describe("recursive model batch capability", () => { const observer = hooks(); const capability = createModelCapability(model, observer); const controller = new AbortController(); - const result = capability.call( - "batch", + const result = capability.batch( [Array.from({ length: 8 }, (_, index) => ({ prompt: `request ${index}`, input: null }))], { signal: controller.signal, deadline: Date.now() + 1_000 }, ); @@ -274,9 +277,10 @@ describe("recursive model batch capability", () => { const observer = hooks(); const capability = createModelCapability(model, observer); - await capability.call("batch", [ - [{ prompt: "SECRET_PROMPT", input: { evidence: "SECRET_INPUT" } }], - ]); + await capability.batch( + [[{ prompt: "SECRET_PROMPT", input: { evidence: "SECRET_INPUT" } }]], + context(), + ); const synchronizedHookData = JSON.stringify({ started: observer.started.mock.calls, diff --git a/examples/rlm/worker/capability.ts b/examples/rlm/worker/capability.ts index bc8b0e05..e27fb8c1 100644 --- a/examples/rlm/worker/capability.ts +++ b/examples/rlm/worker/capability.ts @@ -1,4 +1,4 @@ -import type { WorkspaceRuntimeValue, WorkspaceTrustedModule } from "@cloudflare/computer"; +import type { WorkspaceRuntimeValue, WorkspaceTrustedFunction } from "@cloudflare/computer"; import { generateText, type LanguageModel } from "ai"; import { z } from "zod"; @@ -73,18 +73,20 @@ interface ChildHooks { failed(metadata: FailedMetadata): void; } -export function createModelCapability( - model: LanguageModel, - hooks: ChildHooks, -): WorkspaceTrustedModule { +/** The `ws:model` trusted module: one `batch` function over bounded child requests. */ +export type ModelCapability = { + /** Run up to 24 child model requests and return one result per request. */ + readonly batch: WorkspaceTrustedFunction; +}; + +export function createModelCapability(model: LanguageModel, hooks: ChildHooks): ModelCapability { return { - async call(method, args, context) { - if (method !== "batch") throw new Error(`Unknown model capability method: ${method}`); + async batch(args, context) { const requests = parseBatchArgs(args); if (hooks.admit && !hooks.admit(requests.length)) { throw new Error("This run has exhausted its child-model call budget."); } - const signal = context?.signal; + const signal = context.signal; throwIfAborted(signal); const results: Array = new Array(requests.length); @@ -165,7 +167,7 @@ async function runChild( } } -function parseBatchArgs(args: WorkspaceRuntimeValue[]): ChildRequest[] { +function parseBatchArgs(args: readonly WorkspaceRuntimeValue[]): ChildRequest[] { if (args.length !== 1) throw new Error("Model batch requires exactly one argument."); const value = args[0]; if (!Array.isArray(value) || value.length === 0) { diff --git a/examples/rlm/worker/prompts.ts b/examples/rlm/worker/prompts.ts index 3243e9c0..ca554246 100644 --- a/examples/rlm/worker/prompts.ts +++ b/examples/rlm/worker/prompts.ts @@ -16,11 +16,11 @@ The executor command must be a complete ES module with export default async func export const RLM_SYSTEM_PROMPT = `Solve the Oolong benchmark with one comprehensive recursive Computer execution, then answer briefly. -The executor command must be a complete ES module with export default async function. Use import fs from "node:fs/promises" and import { call as callModel } from "ws:model". Read /workspace/oolong-real/manifest.json. Its contextChunks contain the long corpus. +The executor command must be a complete ES module with export default async function. Use import fs from "node:fs/promises" and import { batch } from "ws:model". Read /workspace/oolong-real/manifest.json. Its contextChunks contain the long corpus. Use Computer as an RLM. Do not make manifest-inspection, schema-discovery, or diagnostic-only calls: 1. Read the bounded corpus chunks. -2. You have one child-call budget of 24 total requests. Call callModel("batch", requests) exactly once with at most 24 focused requests. Each request is { prompt, input }; keep each input to one chunk and ask for structured, question-specific evidence. +2. You have one child-call budget of 24 total requests. Call batch(requests) exactly once with at most 24 focused requests. Each request is { prompt, input }; keep each input to one chunk and ask for structured, question-specific evidence. 3. Each child result is { index, ok, text, error }. Aggregate successful findings in JavaScript and tolerate failed workers. For first/last-event questions, retain chunk indexes and choose evidence by transcript position; never replace an earlier event with a later, more salient one. 4. Parse the child text, aggregate findings in corpus chunk order, and return { answer } from the default function. If a later execution is needed to finalize from prior findings, it must still return { answer }. diff --git a/examples/rlm/worker/rlm-interfaces.txt b/examples/rlm/worker/rlm-interfaces.txt index 7e876c9a..139ce825 100644 --- a/examples/rlm/worker/rlm-interfaces.txt +++ b/examples/rlm/worker/rlm-interfaces.txt @@ -1,7 +1,7 @@ Module interface: import fs from "node:fs/promises"; -import { call as callModel } from "ws:model"; +import { batch } from "ws:model"; export default async function () { // Put every file read, batch call, reduction, and return statement inside this function. } -No asynchronous operation or return statement may appear at module scope. Call recursive inference exactly once as callModel("batch", requests). +No asynchronous operation or return statement may appear at module scope. Call recursive inference exactly once as batch(requests). diff --git a/examples/rlm/worker/step-settings.ts b/examples/rlm/worker/step-settings.ts index 7e91f873..443bcfbb 100644 --- a/examples/rlm/worker/step-settings.ts +++ b/examples/rlm/worker/step-settings.ts @@ -5,7 +5,7 @@ export const COMPUTER_FINALIZATION_STEP = 1; const FINALIZATION_INSTRUCTION = 'Return the benchmark answer now. The next executor module must return a typed { answer } object using evidence already seen. Do not inspect more data, return diagnostics, or call ws:model again. The final module must contain no imports or file I/O. Use exactly this shape: export default async function () { return { answer: "derived answer" }; }'; const RECURSION_RETRY_INSTRUCTION = - 'The previous execution did not use recursive inference. Call ws:model("batch", requests) now with question-specific requests over the Workspace chunks, then return the child findings.'; + "The previous execution did not use recursive inference. Call batch(requests) from ws:model now with question-specific requests over the Workspace chunks, then return the child findings."; export function requiredComputerStep( stepNumber: number, diff --git a/examples/rlm/worker/structured-rlm.test.ts b/examples/rlm/worker/structured-rlm.test.ts index 4c76af39..789808c8 100644 --- a/examples/rlm/worker/structured-rlm.test.ts +++ b/examples/rlm/worker/structured-rlm.test.ts @@ -114,7 +114,7 @@ describe("structured RLM strategy", () => { expect(prompt).toContain("structured-v1"); expect(prompt).toContain("last_spell_by_episode"); - expect(prompt).toContain('callModel("batch", requests)'); + expect(prompt).toContain("batch(requests)"); expect(prompt).not.toContain("SECRET_GOLD"); }); }); diff --git a/packages/computer/src/backends/worker-javascript/module-graph.ts b/packages/computer/src/backends/worker-javascript/module-graph.ts index 8280e7b5..365c6303 100644 --- a/packages/computer/src/backends/worker-javascript/module-graph.ts +++ b/packages/computer/src/backends/worker-javascript/module-graph.ts @@ -1,7 +1,7 @@ import { parse } from "acorn"; import type { WorkspaceRuntimeCapability } from "../../runtime/capability.js"; -import type { WorkspaceRuntimeLoader } from "../../runtime/types.js"; +import type { WorkspaceRuntimeLoader, WorkspaceTrustedModule } from "../../runtime/types.js"; export type JavaScriptModuleMap = WorkspaceRuntimeLoader extends { load(code: { modules: infer Modules }): unknown; @@ -13,13 +13,65 @@ const ENTRY_BASENAME = "__workspace_entry__.js"; const RUNNER_MODULE = "workspace-runtime-runner.js"; const CAPABILITIES_MODULE = "workspace-capabilities.js"; const TRUSTED_MODULES = ["node:fs", "node:fs/promises", "ws:git", "ws:artifacts"] as const; +const TRUSTED_SPECIFIER = /^ws:[A-Za-z0-9][A-Za-z0-9._-]*$/; +const EXPORT_NAME = /^[A-Za-z_$][A-Za-z0-9_$]*$/; +// `default` would turn the function into the default export, and a +// `then` export makes the module namespace look like a promise to +// `await import(...)`. +const RESERVED_EXPORT_NAMES = new Set(["default", "then"]); + +/** Specifier of a host trusted module mapped to the function names it exports. */ +export type TrustedModuleExports = ReadonlyMap; + +/** + * Check host trusted modules once, at backend construction, and record + * the named exports each one installs. + * + * @param trustedModules - The modules passed to the backend. + * @returns The export names of each module, keyed by specifier. + * @throws When a specifier or function name is not allowed. The host + * configured the backend wrongly and no execution can use it. + */ +export function parseTrustedModuleExports( + trustedModules: Readonly>, +): TrustedModuleExports { + const parsed = new Map(); + for (const [specifier, module] of Object.entries(trustedModules)) { + if ( + !TRUSTED_SPECIFIER.test(specifier) || + TRUSTED_MODULES.some((reserved) => reserved === specifier) + ) { + throw new Error( + `Trusted module ${JSON.stringify(specifier)} must use a unique simple reserved ws:* name.`, + ); + } + const names = Object.keys(module); + if (names.length === 0) { + throw new Error(`Trusted module ${JSON.stringify(specifier)} must export a function.`); + } + for (const name of names) { + if (!EXPORT_NAME.test(name) || RESERVED_EXPORT_NAMES.has(name)) { + throw new Error( + `Trusted module ${JSON.stringify(specifier)} export ${JSON.stringify(name)} must be a JavaScript identifier other than "default" or "then".`, + ); + } + if (typeof module[name] !== "function") { + throw new Error( + `Trusted module ${JSON.stringify(specifier)} export ${JSON.stringify(name)} must be a function.`, + ); + } + } + parsed.set(specifier, names); + } + return parsed; +} export interface BuildModuleGraphOptions { source: string; cwd: string; capability: WorkspaceRuntimeCapability; configuredModules: Record; - trustedModuleNames?: string[]; + trustedModules: TrustedModuleExports; maxSourceBytes: number; maxCapabilityBytes: number; maxModules?: number; @@ -39,18 +91,10 @@ export async function buildModuleGraph(options: BuildModuleGraphOptions) { let totalBytes = new TextEncoder().encode(options.source).byteLength; const maxModules = options.maxModules ?? 128; const maxDepth = options.maxDepth ?? 32; - const trustedModuleNames = new Set(TRUSTED_MODULES); - for (const name of options.trustedModuleNames ?? []) { - if ( - !/^ws:[A-Za-z0-9][A-Za-z0-9._-]*$/.test(name) || - TRUSTED_MODULES.includes(name as (typeof TRUSTED_MODULES)[number]) - ) { - throw new Error( - `Trusted module ${JSON.stringify(name)} must use a unique simple reserved ws:* name.`, - ); - } - trustedModuleNames.add(name); - } + const trustedModuleNames = new Set([ + ...TRUSTED_MODULES, + ...options.trustedModules.keys(), + ]); async function visit(path: string, source: string, depth: number): Promise { if (depth > maxDepth) throw new Error(`Workspace JavaScript import depth exceeds ${maxDepth}.`); @@ -136,9 +180,9 @@ export async function buildModuleGraph(options: BuildModuleGraphOptions) { const toCapabilities = relativeModule(directory, CAPABILITIES_MODULE); modules[`${prefix}ws:git`] = { js: gitModule(toCapabilities) }; modules[`${prefix}ws:artifacts`] = { js: artifactsModule(toCapabilities) }; - for (const specifier of options.trustedModuleNames ?? []) { + for (const [specifier, names] of options.trustedModules) { modules[`${prefix}${specifier}`] = { - js: trustedModule(toCapabilities, specifier), + js: trustedModule(toCapabilities, specifier, names), }; } for (const [specifier, source] of Object.entries(options.configuredModules)) { @@ -304,10 +348,15 @@ function proxyModule(capabilitiesImport: string, namespace: string, methods: str `; } -function trustedModule(capabilitiesImport: string, specifier: string) { +// Exports go through `export { local as name }` rather than +// `export const name`, so a reserved word such as `delete` still works +// as an export name. +function trustedModule(capabilitiesImport: string, specifier: string, names: readonly string[]) { + const namespace = JSON.stringify(`trusted/${specifier}`); return ` - import { call as hostCall } from ${JSON.stringify(capabilitiesImport)}; - export const call = (method, ...args) => hostCall(${JSON.stringify(`trusted/${specifier}`)}, "call", [method, ...args]); + import { call } from ${JSON.stringify(capabilitiesImport)}; + ${names.map((name, index) => `const fn${index} = (...args) => call(${namespace}, ${JSON.stringify(name)}, args);`).join("\n")} + export { ${names.map((name, index) => `fn${index} as ${name}`).join(", ")} }; `; } diff --git a/packages/computer/src/backends/worker-javascript/worker-javascript.test.ts b/packages/computer/src/backends/worker-javascript/worker-javascript.test.ts index f75ef9bc..004243ab 100644 --- a/packages/computer/src/backends/worker-javascript/worker-javascript.test.ts +++ b/packages/computer/src/backends/worker-javascript/worker-javascript.test.ts @@ -764,9 +764,9 @@ describe("WorkerJavaScriptBackend", () => { maxHostCallMs: 5, trustedModules: { "ws:test": { - call(_method, _args, context) { + run(_args, context) { return new Promise((_resolve, reject) => { - context?.signal.addEventListener("abort", () => { + context.signal.addEventListener("abort", () => { aborted = true; reject(context.signal.reason); }); @@ -783,7 +783,7 @@ describe("WorkerJavaScriptBackend", () => { _input: unknown, host: { call(name: string, args: string): Promise }, ) { - await host.call("trusted/ws:test.call", JSON.stringify(["run"])); + await host.call("trusted/ws:test.run", JSON.stringify([])); }, }; }, @@ -997,28 +997,77 @@ describe("WorkerJavaScriptBackend", () => { await handle.close(); }); - it("rejects malformed host trusted-module names", async () => { - const workspace = new Workspace({ - storage: new SQLiteTestStorage(), - backends: [ + it.each([ + [ + "a specifier with a path", + { "ws:bad/path": { run: async () => null } }, + /simple reserved ws:\*/, + ], + ["a built-in specifier", { "ws:git": { run: async () => null } }, /simple reserved ws:\*/], + ["a module with no functions", { "ws:empty": {} }, /must export a function/], + [ + "a non-identifier name", + { "ws:test": { "not-a-name": async () => null } }, + /JavaScript identifier/, + ], + ["a default export", { "ws:test": { default: async () => null } }, /JavaScript identifier/], + [ + "a then export", + // biome-ignore lint/suspicious/noThenProperty: The case checks that the backend rejects a `then` export. + { "ws:test": { then: async () => null } }, + /JavaScript identifier/, + ], + ["a non-function export", { "ws:test": { run: "nope" } }, /must be a function/], + ])("rejects trusted modules with %s at construction", (_label, trustedModules, message) => { + expect( + () => new WorkerJavaScriptBackend({ loader: throwingLoader("must not load"), - trustedModules: { - "ws:bad/path": { - async call() { - return null; - }, - }, - } as never, + // SAFETY: Each case hands the constructor a shape the types forbid, to check its runtime guard. + trustedModules: trustedModules as never, }), - ], + ).toThrow(message); + }); + + it("does not dispatch inherited members of a trusted module", async () => { + const db = new Database(new SQLiteTestStorage()); + initializeSchema(db, () => 0); + const fs = new WorkspaceFilesystem(db); + await fs.mkdir("/workspace", { recursive: true }); + let response = ""; + const backend = new WorkerJavaScriptBackend({ + trustedModules: { "ws:test": { run: async () => null } }, + loader: { + load() { + return { + getEntrypoint() { + return { + async evaluate( + _input: unknown, + host: { call(name: string, args: string): Promise }, + ) { + response = await host.call("trusted/ws:test.toString", JSON.stringify([])); + }, + }; + }, + }; + }, + }, }); - await workspace.fs.mkdir("/workspace", { recursive: true }); - await expect( - workspace.runtime.exec(`import { call } from "ws:bad/path"; export default call;`, { - backend: "worker-javascript", - }), - ).rejects.toThrow(/simple reserved ws:\*/); + const handle = await backend.connect({ + db, + fs, + git: undefined as never, + artifacts: undefined as never, + }); + const execution = await handle.exec({ id: "inherited", source: "export default 1" }); + for await (const _event of execution.events) { + // Drain the run so the host call settles. + } + expect(JSON.parse(response)).toMatchObject({ + error: { message: expect.stringContaining("Unknown trusted Workspace module call") }, + }); + await handle.close?.(); }); it("rejects relative imports that collide with internal Loader modules", async () => { diff --git a/packages/computer/src/backends/worker-javascript/worker-javascript.ts b/packages/computer/src/backends/worker-javascript/worker-javascript.ts index 4bd5fcfe..ef9d14f4 100644 --- a/packages/computer/src/backends/worker-javascript/worker-javascript.ts +++ b/packages/computer/src/backends/worker-javascript/worker-javascript.ts @@ -14,7 +14,11 @@ import type { WorkspaceTrustedModule, } from "../../runtime/types.js"; import { decodeRuntimeFrames, type RuntimeFrame } from "./frames.js"; -import { buildModuleGraph } from "./module-graph.js"; +import { + buildModuleGraph, + parseTrustedModuleExports, + type TrustedModuleExports, +} from "./module-graph.js"; export interface WorkerJavaScriptBackendOptions { loader: WorkspaceRuntimeLoader; @@ -24,7 +28,11 @@ export interface WorkerJavaScriptBackendOptions { modules?: Record; /** * Host-owned capability modules installed under reserved ws:* specifiers. - * Caller source may import them, but cannot provide or replace them. + * Each function in a module becomes a named export, so + * `{ "ws:container": { exec } }` lets caller source write + * `import { exec } from "ws:container"`. Caller source may import these + * modules, but cannot provide or replace them. The constructor throws + * when a specifier or function name is not allowed. */ trustedModules?: Record<`ws:${string}`, WorkspaceTrustedModule>; defaultTimeoutMs?: number; @@ -92,6 +100,7 @@ type ResolvedWorkerJavaScriptBackendOptions = Required< > & Omit & { egress: WorkspaceEgressPolicy; + trustedModuleExports: TrustedModuleExports; }; interface WorkspaceExecutionContext { @@ -198,6 +207,7 @@ export class WorkerJavaScriptBackend implements WorkspaceModuleBackend { this.#options = { ...backendOptions, egress: resolvedEgress, + trustedModuleExports: parseTrustedModuleExports(options.trustedModules ?? {}), root: options.root ?? "/workspace", access: options.access ?? "read-write", defaultTimeoutMs, @@ -354,7 +364,7 @@ class JavaScriptBackendHandle implements WorkspaceModuleBackendHandle { cwd: input.cwd ?? this.#options.root, capability, configuredModules: this.#options.modules ?? {}, - trustedModuleNames: Object.keys(this.#options.trustedModules ?? {}), + trustedModules: this.#options.trustedModuleExports, maxSourceBytes: this.#options.maxSourceBytes, maxCapabilityBytes: this.#options.maxCapabilityBytes, }); diff --git a/packages/computer/src/index.ts b/packages/computer/src/index.ts index 587c2053..87684af7 100644 --- a/packages/computer/src/index.ts +++ b/packages/computer/src/index.ts @@ -88,6 +88,8 @@ export type { WorkspaceRuntimeResult, WorkspaceRuntimeStatus, WorkspaceRuntimeValue, + WorkspaceTrustedCallContext, + WorkspaceTrustedFunction, WorkspaceTrustedModule, } from "./runtime/types.js"; export { decodeRuntimeEvents, encodeRuntimeEvent } from "./runtime/wire.js"; diff --git a/packages/computer/src/runtime/bridge.test.ts b/packages/computer/src/runtime/bridge.test.ts index 18f330a0..0f71e5ef 100644 --- a/packages/computer/src/runtime/bridge.test.ts +++ b/packages/computer/src/runtime/bridge.test.ts @@ -4,7 +4,7 @@ import { WorkspaceRuntimeBridge } from "./bridge.js"; import type { WorkspaceRuntimeCapability } from "./capability.js"; const encoder = new TextEncoder(); -const args = JSON.stringify(["run", "value"]); +const args = JSON.stringify(["value"]); function bridge(limits: { maxCalls?: number; @@ -15,7 +15,7 @@ function bridge(limits: { ...limits, trustedModules: { "ws:test": { - async call() { + async run() { return "ok"; }, }, @@ -30,9 +30,9 @@ async function message(response: Promise) { describe("WorkspaceRuntimeBridge cumulative limits", () => { it("accepts the configured call count and rejects the next call", async () => { const target = bridge({ maxCalls: 2 }); - await expect(message(target.call("trusted/ws:test.call", args))).resolves.toBeUndefined(); - await expect(message(target.call("trusted/ws:test.call", args))).resolves.toBeUndefined(); - await expect(message(target.call("trusted/ws:test.call", args))).resolves.toContain( + await expect(message(target.call("trusted/ws:test.run", args))).resolves.toBeUndefined(); + await expect(message(target.call("trusted/ws:test.run", args))).resolves.toBeUndefined(); + await expect(message(target.call("trusted/ws:test.run", args))).resolves.toContain( "exceeds 2 capability calls", ); }); @@ -40,20 +40,20 @@ describe("WorkspaceRuntimeBridge cumulative limits", () => { it("accepts requests at the cumulative byte boundary and rejects the next request", async () => { const bytes = encoder.encode(args).byteLength; const target = bridge({ maxTotalRequestBytes: bytes * 2 }); - await expect(message(target.call("trusted/ws:test.call", args))).resolves.toBeUndefined(); - await expect(message(target.call("trusted/ws:test.call", args))).resolves.toBeUndefined(); - await expect(message(target.call("trusted/ws:test.call", args))).resolves.toContain( + await expect(message(target.call("trusted/ws:test.run", args))).resolves.toBeUndefined(); + await expect(message(target.call("trusted/ws:test.run", args))).resolves.toBeUndefined(); + await expect(message(target.call("trusted/ws:test.run", args))).resolves.toContain( `requests exceed ${bytes * 2} bytes`, ); }); it("accepts responses at the cumulative byte boundary and rejects the next response", async () => { - const sample = await bridge({}).call("trusted/ws:test.call", args); + const sample = await bridge({}).call("trusted/ws:test.run", args); const bytes = encoder.encode(sample).byteLength; const target = bridge({ maxTotalResponseBytes: bytes * 2 }); - await expect(message(target.call("trusted/ws:test.call", args))).resolves.toBeUndefined(); - await expect(message(target.call("trusted/ws:test.call", args))).resolves.toBeUndefined(); - await expect(message(target.call("trusted/ws:test.call", args))).resolves.toContain( + await expect(message(target.call("trusted/ws:test.run", args))).resolves.toBeUndefined(); + await expect(message(target.call("trusted/ws:test.run", args))).resolves.toBeUndefined(); + await expect(message(target.call("trusted/ws:test.run", args))).resolves.toContain( `responses exceed ${bytes * 2} bytes`, ); }); diff --git a/packages/computer/src/runtime/bridge.ts b/packages/computer/src/runtime/bridge.ts index 552e28f0..efe1cfa7 100644 --- a/packages/computer/src/runtime/bridge.ts +++ b/packages/computer/src/runtime/bridge.ts @@ -3,7 +3,7 @@ import { RpcTarget } from "cloudflare:workers"; import type { ArtifactClient } from "../artifacts/index.js"; import type { GitClient } from "../git/index.js"; import { assertRuntimeValue, type WorkspaceRuntimeCapability } from "./capability.js"; -import type { WorkspaceTrustedModule } from "./types.js"; +import type { WorkspaceTrustedCallContext, WorkspaceTrustedModule } from "./types.js"; export class WorkspaceRuntimeBridge extends RpcTarget { readonly #capability: WorkspaceRuntimeCapability; @@ -221,21 +221,27 @@ export class WorkspaceRuntimeBridge extends RpcTarget { } } - async #callTrusted( - name: string, - args: unknown[], - context: { signal: AbortSignal; deadline: number }, - ) { - const suffix = ".call"; - const specifier = name.endsWith(suffix) ? name.slice("trusted/".length, -suffix.length) : ""; - const trusted = this.#trustedModules[specifier]; - if (!trusted) { + // `name` is `trusted/.`. Function names are + // identifiers and never contain a dot, so the last dot splits them + // from a specifier such as `ws:a.b`. Own-property checks keep + // isolate code from reaching `toString` or other inherited members. + async #callTrusted(name: string, args: unknown[], context: WorkspaceTrustedCallContext) { + const target = name.slice("trusted/".length); + const dot = target.lastIndexOf("."); + const specifier = dot === -1 ? "" : target.slice(0, dot); + const functionName = dot === -1 ? "" : target.slice(dot + 1); + const trusted = Object.hasOwn(this.#trustedModules, specifier) + ? this.#trustedModules[specifier] + : undefined; + const fn = + trusted !== undefined && Object.hasOwn(trusted, functionName) + ? trusted[functionName] + : undefined; + if (typeof fn !== "function") { throw new Error(`Unknown trusted Workspace module call ${JSON.stringify(name)}.`); } - const method = String(args[0]); - const callArgs = args.slice(1); - assertBridgeValues(callArgs); - const result = await trusted.call(method, callArgs, context); + assertBridgeValues(args); + const result = await fn(args, context); assertBridgeValues([result]); return result; } diff --git a/packages/computer/src/runtime/types.ts b/packages/computer/src/runtime/types.ts index c92a9f64..40aa6c7a 100644 --- a/packages/computer/src/runtime/types.ts +++ b/packages/computer/src/runtime/types.ts @@ -4,15 +4,37 @@ import type { ExecEncoding, ExecSyncResult, KillSignal } from "../shell.js"; export type WorkspaceRuntimeAccess = "read" | "read-write"; -export interface WorkspaceTrustedModule { - /** Dispatch a call made through a host-installed reserved ws:* module. */ - call( - method: string, - args: WorkspaceRuntimeValue[], - context?: { signal: AbortSignal; deadline: number }, - ): Promise; +/** Cancellation and timing for one call into a trusted module function. */ +export interface WorkspaceTrustedCallContext { + /** Aborts when the call passes its deadline or the execution is cancelled. */ + readonly signal: AbortSignal; + /** Epoch milliseconds after which the caller stops waiting for this call. */ + readonly deadline: number; } +/** + * One host function exported by a trusted module. + * + * `args` holds the arguments the isolate passed, decoded from the wire. + * They come from untrusted code, so parse them before use. + */ +export type WorkspaceTrustedFunction = ( + args: readonly WorkspaceRuntimeValue[], + context: WorkspaceTrustedCallContext, +) => Promise; + +/** + * A host-owned module installed under a reserved `ws:*` specifier. + * + * Each key becomes a named export in the isolate, so + * `{ exec: async (args) => ... }` installed as `ws:container` lets + * code write `import { exec } from "ws:container"`. Keys must be + * JavaScript identifier names other than `default` and `then`. A + * reserved word such as `delete` works, but code must rename it on + * import: `import { delete as remove } from "ws:files"`. + */ +export type WorkspaceTrustedModule = Readonly>; + export type WorkspaceRuntimeValue = | null | boolean diff --git a/packages/computer/tests/script-runner-worker.ts b/packages/computer/tests/script-runner-worker.ts index 244b548b..6cbe7d1e 100644 --- a/packages/computer/tests/script-runner-worker.ts +++ b/packages/computer/tests/script-runner-worker.ts @@ -33,20 +33,34 @@ export class HostDO extends DurableObject { }, trustedModules: { "ws:test-host": { - async call(method, args) { - if (method === "invalid-result") return new Date() as never; - if (method === "large-error") throw new Error("x".repeat(5000)); - if (method === "slow") { - await new Promise((resolve) => setTimeout(resolve, 20)); - return null; - } - if (method === "marker") { - return { - __workspace_codec__: { version: 1, type: "bytes", data: [1] }, - keep: true, - }; - } - return { method, args }; + async echo(args) { + return { args: [...args] }; + }, + async sum(args) { + return args.reduce( + (total, value) => total + (typeof value === "number" ? value : 0), + 0, + ); + }, + async delete(args) { + return { deleted: args[0] ?? null }; + }, + async invalidResult() { + // SAFETY: The test hands the bridge a non-JSON value on purpose to check that it rejects it. + return new Date() as never; + }, + async largeError() { + throw new Error("x".repeat(5000)); + }, + async slow() { + await new Promise((resolve) => setTimeout(resolve, 20)); + return null; + }, + async marker() { + return { + __workspace_codec__: { version: 1, type: "bytes", data: [1] }, + keep: true, + }; }, }, }, diff --git a/packages/computer/tests/script-runner.test.ts b/packages/computer/tests/script-runner.test.ts index 61c8b63f..453279ae 100644 --- a/packages/computer/tests/script-runner.test.ts +++ b/packages/computer/tests/script-runner.test.ts @@ -56,7 +56,7 @@ describe("WorkspaceRuntime", () => { import fs from "node:fs/promises"; import { promises as nodeFs } from "node:fs"; import * as git from "ws:git"; - import { call } from "ws:test-host"; + import { echo, sum, delete as remove } from "ws:test-host"; export default async function main(input) { const value = double(input.value); await fs.writeFile("/workspace/runtime-result.txt", String(value)); @@ -65,7 +65,9 @@ describe("WorkspaceRuntime", () => { value, persisted: await fs.readFile("/workspace/runtime-result.txt", "utf8"), gitExitCode: initialized.exitCode, - trusted: await call("echo", input.value), + trusted: await echo(input.value, "second"), + summed: await sum(1, 2, 3), + removed: await remove("/workspace/gone.txt"), nodeFs: { isFile: (await nodeFs.stat("/workspace/runtime-result.txt")).isFile(), entries: await nodeFs.readdir("/workspace"), @@ -86,7 +88,9 @@ describe("WorkspaceRuntime", () => { value: 42, persisted: "42", gitExitCode: 0, - trusted: { method: "echo", args: [21] }, + trusted: { args: [21, "second"] }, + summed: 6, + removed: { deleted: "/workspace/gone.txt" }, nodeFs: { isFile: true, entries: expect.arrayContaining(["runtime-result.txt"]), @@ -100,14 +104,14 @@ describe("WorkspaceRuntime", () => { const response = await runtime({ source: ` import fs from "node:fs/promises"; - import { call } from "ws:test-host"; + import { marker } from "ws:test-host"; export default async () => { await fs.writeFile("/workspace/bytes.bin", new Uint8Array([0, 127, 255])); const value = await fs.readFile("/workspace/bytes.bin"); return { isBytes: value instanceof Uint8Array, bytes: Array.from(value), - marker: await call("marker"), + marker: await marker(), }; }; `, @@ -233,8 +237,8 @@ describe("WorkspaceRuntime", () => { it("bounds oversized trusted-module error responses", async () => { const response = await runtime({ source: ` - import { call } from "ws:test-host"; - export default () => call("large-error"); + import { largeError } from "ws:test-host"; + export default () => largeError(); `, cwd: "/workspace", }); @@ -261,9 +265,9 @@ describe("WorkspaceRuntime", () => { it("bounds concurrent host capability calls", async () => { const response = await runtime({ source: ` - import { call } from "ws:test-host"; + import { slow } from "ws:test-host"; export default async () => { - const settled = await Promise.allSettled([call("slow"), call("slow"), call("slow")]); + const settled = await Promise.allSettled([slow(), slow(), slow()]); return settled.map((item) => item.status); }; `, @@ -277,8 +281,8 @@ describe("WorkspaceRuntime", () => { it("rejects non-plain results from host trusted modules", async () => { const response = await runtime({ source: ` - import { call } from "ws:test-host"; - export default () => call("invalid-result"); + import { invalidResult } from "ws:test-host"; + export default () => invalidResult(); `, cwd: "/workspace", }); @@ -292,6 +296,42 @@ describe("WorkspaceRuntime", () => { }); }); + it("exposes only the functions a trusted module declares", async () => { + const response = await runtime({ + source: ` + import * as host from "ws:test-host"; + export default () => Object.keys(host).sort(); + `, + cwd: "/workspace", + }); + const text = await response.text(); + expect(response.status, text).toBe(200); + expect(JSON.parse(text).result.value).toEqual([ + "delete", + "echo", + "invalidResult", + "largeError", + "marker", + "slow", + "sum", + ]); + }); + + it("fails to link an import the trusted module does not export", async () => { + const response = await runtime({ + source: ` + import { call } from "ws:test-host"; + export default () => call("echo"); + `, + cwd: "/workspace", + }); + const text = await response.text(); + expect(response.status, text).toBe(200); + expect(JSON.parse(text), text).toMatchObject({ + result: { status: "failed", stderr: expect.stringContaining("does not provide an export") }, + }); + }); + it("does not expose unrestricted host operations through the node:fs dispatcher", async () => { const response = await runtime({ source: ` From 6fe7901a291fa428edbd8d765299a0991c66b2d8 Mon Sep 17 00:00:00 2001 From: Matt Carey Date: Wed, 30 Sep 2026 16:24:12 +0100 Subject: [PATCH 2/6] computer: Drop the exec backend argument for one backend The exec tool always offered a backend argument, even when only one backend was configured. The model saw an enum with a single value and a description written for choosing between several backends, including advice to retry a failed command somewhere else. With exactly one backend the tool now has no backend argument and always runs there, and defaultBackend becomes optional. The description talks about what that backend does rather than about picking one. A single callable backend describes command as module source and keeps input; a single shell backend drops input, which it could never accept. A backend value sent anyway is removed by the schema. With more than one backend the tool is unchanged, and defaultBackend is still required. The output still names the backend that ran. --- .changeset/exec-tool-single-backend.md | 5 + docs/09_tool_interface.md | 25 ++- packages/computer/src/tools/ai.test.ts | 105 ++++++++++- packages/computer/src/tools/exec.ts | 232 +++++++++++++++++-------- 4 files changed, 295 insertions(+), 72 deletions(-) create mode 100644 .changeset/exec-tool-single-backend.md diff --git a/.changeset/exec-tool-single-backend.md b/.changeset/exec-tool-single-backend.md new file mode 100644 index 00000000..0b63d0f6 --- /dev/null +++ b/.changeset/exec-tool-single-backend.md @@ -0,0 +1,5 @@ +--- +"@cloudflare/computer": minor +--- + +The `exec` tool has no `backend` argument when only one backend is configured. It always runs there, its description no longer talks about choosing a backend, and `defaultBackend` becomes optional. A single shell backend also drops the `input` argument it could never accept, and a single callable backend describes `command` as ES module source. With more than one backend, nothing changes and `defaultBackend` is still required. diff --git a/docs/09_tool_interface.md b/docs/09_tool_interface.md index a8629a1c..47ccfb84 100644 --- a/docs/09_tool_interface.md +++ b/docs/09_tool_interface.md @@ -55,7 +55,20 @@ export class Agent { Pass the returned AI SDK `ToolSet` to `generateText`, `streamText`, or an agent framework hook such as `getTools()`. -Pass `shell` only when the Workspace has matching backend ids: +Pass `shell` only when the Workspace has matching backend ids. With one backend, `exec` has no `backend` argument and always runs there: + +```ts +const tools = createAITools({ + workspace, + shell: { + backends: { + "worker-javascript": { description: "Isolated JavaScript with the durable workspace filesystem." }, + }, + }, +}); +``` + +With more than one, pass `defaultBackend` and the model picks a backend per call: ```ts const tools = createAITools({ @@ -246,6 +259,16 @@ The tool uses forced removal, so deleting a missing path succeeds. Set `recursiv `exec` is opt-in. It calls `workspace.runtime.exec` with the configured backend and streams bounded output. Backend descriptions are included in the model-facing tool description, so describe capabilities and startup cost in plain language. +The tool's arguments depend on how many backends you pass: + +| Backends | Arguments | Description | +| --- | --- | --- | +| One shell backend | `command`, `cwd`, `env` | Describes a shell command. | +| One callable backend | `command`, `cwd`, `env`, `input` | Describes `command` as ES module source and the return value as `result`. | +| More than one | `command`, `cwd`, `backend`, `env`, `input` | Lists every backend and the default. `defaultBackend` is required. | + +A `backend` value the model sends anyway is dropped when only one backend is configured. The output still names the backend that ran. + Wire this tool carefully: it executes arbitrary shell commands inside the configured backend. Treat its output as untrusted text when including it in later model input. Omit `shell` or use `readonly: true` when command execution is not part of the agent's job. ## `publish` diff --git a/packages/computer/src/tools/ai.test.ts b/packages/computer/src/tools/ai.test.ts index 5d283781..7493fd2b 100644 --- a/packages/computer/src/tools/ai.test.ts +++ b/packages/computer/src/tools/ai.test.ts @@ -1,5 +1,6 @@ import { SQLiteTestStorage } from "@cloudflare/dofs/testing"; import { describe, expect, it } from "vitest"; +import { z } from "zod"; import type { WorkspaceRuntimeExecHandle, WorkspaceRuntimeResult } from "../runtime/types.js"; import { Workspace } from "../workspace.js"; import { @@ -85,6 +86,17 @@ function toolDescription(tool: unknown): string { return description; } +function inputSchema(tool: unknown): z.ZodType { + const schema = (tool as { inputSchema?: unknown }).inputSchema; + if (!(schema instanceof z.ZodType)) throw new Error("tool has no zod input schema"); + return schema; +} + +function inputProperties(tool: unknown): string[] { + const json = z.toJSONSchema(inputSchema(tool)) as { properties?: Record }; + return Object.keys(json.properties ?? {}).sort(); +} + function makeWorkspace(): Workspace { return new Workspace({ storage: new SQLiteTestStorage(), now: () => 1_700_000_000_000 }); } @@ -1697,7 +1709,98 @@ describe("createAITools callable exec", () => { }, }); - expect(toolDescription(tools.exec)).toContain("callable"); + expect(toolDescription(tools.exec)).toContain("Run a JavaScript module"); + expect(toolDescription(tools.exec)).toContain("ES module source"); + expect(toolDescription(tools.exec)).toContain("JavaScript module runtime"); + }); +}); + +describe("createAITools exec with one backend", () => { + function recordingWorkspace(callable: boolean) { + const calls: Array<{ command: string; backend: string | undefined; input: unknown }> = []; + const workspace = { + runtime: { + async exec( + command: string, + options: { encoding: "utf8"; backend?: string; input?: unknown }, + ) { + calls.push({ command, backend: options.backend, input: options.input }); + return { result: async () => ({ exitCode: 0, stdout: "", stderr: "", value: 1 }) }; + }, + isCallable: (id: string) => callable && id === "worker-javascript", + }, + }; + return { calls, workspace }; + } + + it("has no backend argument and runs on the only backend without defaultBackend", async () => { + const { calls, workspace } = recordingWorkspace(true); + const tools = createAITools({ + workspace, + shell: { backends: { "worker-javascript": { description: "Isolated JavaScript." } } }, + }); + + expect(inputProperties(tools.exec)).toEqual(["command", "cwd", "env", "input"]); + const parsed = inputSchema(tools.exec).parse({ + command: "export default () => 1", + backend: "container", + }); + expect(parsed).toEqual({ command: "export default () => 1" }); + await executeTool(tools.exec, parsed); + expect(calls).toEqual([ + { command: "export default () => 1", backend: "worker-javascript", input: undefined }, + ]); + }); + + it("does not mention backends in the description", () => { + const { workspace } = recordingWorkspace(true); + const tools = createAITools({ + workspace, + shell: { backends: { "worker-javascript": { description: "Isolated JavaScript." } } }, + }); + + expect(toolDescription(tools.exec)).not.toMatch(/backend/i); + }); + + it("drops input for a single shell backend", () => { + const { workspace } = recordingWorkspace(false); + const tools = createAITools({ + workspace, + shell: { backends: { shell: { description: "Fast shell." } } }, + }); + + expect(inputProperties(tools.exec)).toEqual(["command", "cwd", "env"]); + expect(toolDescription(tools.exec)).toContain("Run a shell command"); + expect(toolDescription(tools.exec)).not.toMatch(/backend/i); + }); + + it("keeps the backend argument when more than one backend is configured", () => { + const { workspace } = recordingWorkspace(true); + const tools = createAITools({ + workspace, + shell: { + defaultBackend: "worker-javascript", + backends: { + "worker-javascript": { description: "Isolated JavaScript." }, + container: { description: "Full Linux." }, + }, + }, + }); + + expect(inputProperties(tools.exec)).toEqual(["backend", "command", "cwd", "env", "input"]); + }); + + it("requires defaultBackend when more than one backend is configured", () => { + const { workspace } = recordingWorkspace(false); + + expect(() => + createAITools({ + workspace, + shell: { + backends: { shell: { description: "Fast shell." }, container: { description: "Linux." } }, + }, + }), + ).toThrow(/defaultBackend/); }); }); diff --git a/packages/computer/src/tools/exec.ts b/packages/computer/src/tools/exec.ts index 5a58fb36..1f462630 100644 --- a/packages/computer/src/tools/exec.ts +++ b/packages/computer/src/tools/exec.ts @@ -73,8 +73,13 @@ export interface ExecBackendDescription { export interface ExecToolOptions { workspace: ExecWorkspaceLike; + // Backends the model may run on. With exactly one, the tool has no + // `backend` argument and always runs there, so the model never has + // to reason about backends. backends: Record; - defaultBackend: string; + // Backend used when the model omits `backend`. Required when more + // than one backend is configured; with one it defaults to that one. + defaultBackend?: string; // Per-snapshot display cap for each of stdout and stderr, in bytes. // Output past it is shown as a truncation marker. Defaults to 64 KiB. maxBytes?: number; @@ -110,91 +115,53 @@ export type ExecToolOutput = } | { command: string; cwd: string | null; backend: string; error: string }; -export function createExecTool(options: ExecToolOptions): Tool< - { - command: string; - cwd?: string; - backend?: string; - env?: Record; - input?: WorkspaceRuntimeValue; - }, - ExecToolOutput -> { +type ExecToolInput = { + command: string; + cwd?: string; + backend?: string; + env?: Record; + input?: WorkspaceRuntimeValue; +}; + +export function createExecTool(options: ExecToolOptions): Tool { const maxBytes = options.maxBytes ?? DEFAULT_MAX_BYTES; const streamMaxBytes = options.streamMaxBytes ?? DEFAULT_STREAM_MAX_BYTES; const now = options.now ?? Date.now; const backendIds = Object.keys(options.backends); - if (backendIds.length === 0) { + const [onlyBackend, ...otherBackends] = backendIds; + if (onlyBackend === undefined) { throw new Error("createExecTool: pass at least one backend in `backends`"); } - if (!backendIds.includes(options.defaultBackend)) { + const single = otherBackends.length === 0; + const defaultBackend = options.defaultBackend ?? (single ? onlyBackend : undefined); + if (defaultBackend === undefined) { + throw new Error( + "createExecTool: pass `defaultBackend` when more than one backend is configured", + ); + } + if (!backendIds.includes(defaultBackend)) { throw new Error( - `createExecTool: defaultBackend ${JSON.stringify(options.defaultBackend)} is not one of ${backendIds.map((id) => JSON.stringify(id)).join(", ")}`, + `createExecTool: defaultBackend ${JSON.stringify(defaultBackend)} is not one of ${backendIds.map((id) => JSON.stringify(id)).join(", ")}`, ); } const isCallable = options.workspace.runtime.isCallable?.bind(options.workspace.runtime); const callableBackendIds = new Set(backendIds.filter((id) => isCallable?.(id) === true)); - const backendGuidance = backendIds - .map((id) => { - const suffix = callableBackendIds.has(id) ? " (callable)" : ""; - return `- ${JSON.stringify(id)}${suffix}: ${options.backends[id].description}`; - }) - .join("\n"); - const callableGuidance = - callableBackendIds.size > 0 - ? [ - "", - `Callable backends (${[...callableBackendIds].map((id) => JSON.stringify(id)).join(", ")}) run \`command\` as module source rather than a shell command. Pass \`input\` to hand the module a structured value, and read the module's returned value back from the \`result\` field. Other backends reject \`input\`.`, - ].join("\n") - : ""; - const description = [ - "Run a shell command in the workspace. The workspace exposes multiple backends, each with different capabilities.", - "Pick the cheapest backend that can run the command; fall back to a heavier one only when the lighter backend's command set doesn't cover what you need.", - "", - "Backends:", - backendGuidance, - "", - `Default backend: ${JSON.stringify(options.defaultBackend)}. Try this first for any command you're not sure about; if it fails with a "command not found" or a similar capability error, retry on a backend whose description covers the missing tool.`, - "Use for builds, test runs, typechecks, formatters, and git plumbing. Prefer the dedicated read, write, and edit tools for file operations. Long output is truncated to keep tool replies small.", - callableGuidance, - ].join("\n"); - - const backendSchema = z - .enum(backendIds as [string, ...string[]]) - .optional() - .describe( - [ - "Which backend to run on. Omit to use the default", - `(${JSON.stringify(options.defaultBackend)}). Set explicitly when the`, - "default backend is not capable of running the command. If a command fails because the backend lacks that tool, retry on a backend whose description covers it.", - ].join(" "), - ); + const description = single + ? singleBackendDescription( + options.backends[onlyBackend].description, + callableBackendIds.has(onlyBackend), + ) + : multipleBackendDescription(options.backends, defaultBackend, callableBackendIds); + const inputSchema = single + ? singleBackendInputSchema(callableBackendIds.has(onlyBackend)) + : multipleBackendInputSchema(backendIds, defaultBackend); return tool({ description, - inputSchema: z.object({ - command: z - .string() - .describe( - "Shell command, e.g. 'npm test' or 'git diff HEAD'. For a callable backend this is the module source to run.", - ), - cwd: z.string().optional().describe("Working directory. Defaults to the workspace root."), - backend: backendSchema, - env: z - .record(z.string(), z.string()) - .optional() - .describe( - "Environment variables for this run only. Values override the backend's base environment without affecting later runs.", - ), - input: jsonValueSchema - .optional() - .describe( - "Structured value handed to a callable backend's module. Only callable backends accept it; other backends reject it.", - ), - }), + inputSchema, execute: async function* ({ command, cwd, backend, env, input }, { abortSignal }) { - const selectedBackend = backend ?? options.defaultBackend; + const selectedBackend = backend ?? defaultBackend; const base = { command, cwd: cwd ?? null, backend: selectedBackend }; if (input !== undefined && !callableBackendIds.has(selectedBackend)) { yield { ...base, error: notCallableMessage(selectedBackend) }; @@ -297,6 +264,131 @@ export function createExecTool(options: ExecToolOptions): Tool< }); } +const FILE_TOOLS_HINT = + "Prefer the dedicated read, write, and edit tools for file operations. Long output is truncated to keep tool replies small."; +const SHELL_USE_HINT = "Use for builds, test runs, typechecks, formatters, and git plumbing."; +const CALLABLE_GUIDANCE = + "`command` is ES module source, not a shell command. Pass `input` to hand the module's default export a structured value, and read its return value back from the `result` field."; + +// One backend: describe what it does and nothing about choosing one. +function singleBackendDescription(backendDescription: string, callable: boolean): string { + return callable + ? [ + "Run a JavaScript module in the workspace.", + "", + backendDescription, + "", + CALLABLE_GUIDANCE, + FILE_TOOLS_HINT, + ].join("\n") + : [ + "Run a shell command in the workspace.", + "", + backendDescription, + "", + `${SHELL_USE_HINT} ${FILE_TOOLS_HINT}`, + ].join("\n"); +} + +function multipleBackendDescription( + backends: Record, + defaultBackend: string, + callableBackendIds: ReadonlySet, +): string { + const backendGuidance = Object.entries(backends) + .map(([id, backend]) => { + const suffix = callableBackendIds.has(id) ? " (callable)" : ""; + return `- ${JSON.stringify(id)}${suffix}: ${backend.description}`; + }) + .join("\n"); + const callableGuidance = + callableBackendIds.size > 0 + ? [ + "", + `Callable backends (${[...callableBackendIds].map((id) => JSON.stringify(id)).join(", ")}) run \`command\` as module source rather than a shell command. Pass \`input\` to hand the module a structured value, and read the module's returned value back from the \`result\` field. Other backends reject \`input\`.`, + ].join("\n") + : ""; + return [ + "Run a shell command in the workspace. The workspace exposes multiple backends, each with different capabilities.", + "Pick the cheapest backend that can run the command; fall back to a heavier one only when the lighter backend's command set doesn't cover what you need.", + "", + "Backends:", + backendGuidance, + "", + `Default backend: ${JSON.stringify(defaultBackend)}. Try this first for any command you're not sure about; if it fails with a "command not found" or a similar capability error, retry on a backend whose description covers the missing tool.`, + `${SHELL_USE_HINT} ${FILE_TOOLS_HINT}`, + callableGuidance, + ].join("\n"); +} + +const cwdSchema = z + .string() + .optional() + .describe("Working directory. Defaults to the workspace root."); +const envSchema = z + .record(z.string(), z.string()) + .optional() + .describe( + "Environment variables for this run only. Values override the base environment without affecting later runs.", + ); + +// One backend: no `backend` argument, and `input` only when the +// backend can accept it. +function singleBackendInputSchema(callable: boolean): z.ZodType { + if (!callable) { + return z.object({ + command: z.string().describe("Shell command, e.g. 'npm test' or 'git diff HEAD'."), + cwd: cwdSchema, + env: envSchema, + }); + } + return z.object({ + command: z + .string() + .describe( + "ES module source to run. Its default export receives `input` and its return value comes back as `result`.", + ), + cwd: cwdSchema.describe( + "Working directory for relative imports. Defaults to the workspace root.", + ), + env: envSchema, + input: jsonValueSchema + .optional() + .describe("Structured value handed to the module's default export."), + }); +} + +function multipleBackendInputSchema( + backendIds: string[], + defaultBackend: string, +): z.ZodType { + return z.object({ + command: z + .string() + .describe( + "Shell command, e.g. 'npm test' or 'git diff HEAD'. For a callable backend this is the module source to run.", + ), + cwd: cwdSchema, + backend: z + // SAFETY: createExecTool checked that backendIds has at least one entry before calling this. + .enum(backendIds as [string, ...string[]]) + .optional() + .describe( + [ + "Which backend to run on. Omit to use the default", + `(${JSON.stringify(defaultBackend)}). Set explicitly when the`, + "default backend is not capable of running the command. If a command fails because the backend lacks that tool, retry on a backend whose description covers it.", + ].join(" "), + ), + env: envSchema, + input: jsonValueSchema + .optional() + .describe( + "Structured value handed to a callable backend's module. Only callable backends accept it; other backends reject it.", + ), + }); +} + function errorMessage(err: unknown): string { return err instanceof Error ? err.message : String(err); } From 33152a4d9957295c6b56e13b79425d76306f8368 Mon Sep 17 00:00:00 2001 From: Matt Carey Date: Wed, 30 Sep 2026 16:29:00 +0100 Subject: [PATCH 3/6] computer: Add ws:container for isolate JavaScript An agent that wants both isolated JavaScript and a full Linux container has so far needed two exec backends, and the model had to pick one per command. This lets JavaScript be the only backend the model sees, with the container as a library it can call. createContainerModule() builds a trusted module that installs as ws:container on a WorkerJavaScriptBackend. Its exec(command, options) runs through workspace.runtime.exec on the container backend, so the container shares the Workspace's files through the usual sync bracket. It returns the exit code and bounded output once the command finishes, kills the command when the execution is cancelled, and caps the command's timeout at the host call deadline. The runtime is resolved on each call so the module can be built before the Workspace that owns both backends. describeContainerModule() returns text for the JavaScript backend's tool description so the model knows the module exists. --- .changeset/ws-container-module.md | 5 + docs/17_isolate_javascript.md | 58 ++++ .../container/container-module.test.ts | 192 +++++++++++++ .../backends/container/container-module.ts | 252 ++++++++++++++++++ .../computer/src/backends/container/index.ts | 10 + .../computer/tests/script-runner-worker.ts | 44 +++ packages/computer/tests/script-runner.test.ts | 43 +++ 7 files changed, 604 insertions(+) create mode 100644 .changeset/ws-container-module.md create mode 100644 packages/computer/src/backends/container/container-module.test.ts create mode 100644 packages/computer/src/backends/container/container-module.ts diff --git a/.changeset/ws-container-module.md b/.changeset/ws-container-module.md new file mode 100644 index 00000000..b6b3ca67 --- /dev/null +++ b/.changeset/ws-container-module.md @@ -0,0 +1,5 @@ +--- +"@cloudflare/computer": minor +--- + +Add `createContainerModule()` to `@cloudflare/computer/backends/container`. Install the returned module as `ws:container` on a `WorkerJavaScriptBackend`, and JavaScript can run shell commands in the Workspace's container with `import { exec } from "ws:container"`. The container shares the Workspace's files, and a canceled execution kills the running command. `describeContainerModule()` returns text for the model that explains the module. diff --git a/docs/17_isolate_javascript.md b/docs/17_isolate_javascript.md index 47e77adf..2eeb2524 100644 --- a/docs/17_isolate_javascript.md +++ b/docs/17_isolate_javascript.md @@ -227,3 +227,61 @@ Each host function receives the arguments the isolate passed, as an array of JSO The backend checks trusted modules when it is constructed. A specifier must be a simple `ws:*` name that does not shadow `ws:git` or `ws:artifacts`. A module must export at least one function, and every export name must be a JavaScript identifier name other than `default` or `then`. A reserved word such as `delete` is allowed, and caller code renames it on import: `import { delete as remove } from "ws:files"`. Importing a name the module does not export fails when the module graph links, before any code runs. These modules are fixed at construction, and caller source cannot supply or replace them. + +### Container commands with `ws:container` + +`createContainerModule()` from `@cloudflare/computer/backends/container` builds a ready-made `ws:container` module. With it, JavaScript is the only backend the model sees, and the container is something that JavaScript can call: + +```ts +import { WorkerJavaScriptBackend } from "@cloudflare/computer/backends/worker-javascript"; +import { + CloudflareContainerBackend, + createContainerModule, + describeContainerModule, +} from "@cloudflare/computer/backends/container"; +import { createAITools } from "@cloudflare/computer/tools"; + +this.workspace = new Workspace({ + storage: ctx.storage, + backends: [ + new WorkerJavaScriptBackend({ + loader: env.LOADER, + access: "read-write", + trustedModules: { + "ws:container": createContainerModule({ runtime: () => this.workspace.runtime }), + }, + }), + new CloudflareContainerBackend({ /* ... */ }), + ], +}); + +const tools = createAITools({ + workspace: this.workspace, + shell: { + backends: { + "worker-javascript": { + description: `Isolated JavaScript with the durable workspace filesystem. ${describeContainerModule()}`, + }, + }, + }, +}); +``` + +```js +import { exec } from "ws:container"; + +export default async function () { + const { exitCode, stdout, stderr } = await exec("npm test", { cwd: "/workspace/app" }); + return { passed: exitCode === 0, stdout, stderr }; +} +``` + +`exec(command, { cwd, env, stdin, timeoutMs })` runs through `workspace.runtime.exec` on the container backend (`"container-shell"` unless you pass `backend`). The container shares the Workspace's files: writes the module made before the call are pushed to the container, and the container's changes are pulled back before `exec` returns. A non-zero exit code comes back as a value, not as an error. + +A few limits follow from `exec` being a host call: + +- Output comes back when the command finishes, not while it runs. Each stream is cut at `maxOutputBytes` (64 KiB by default), which must stay well under the backend's `maxCapabilityBytes`. +- The command's timeout is capped at the time left before the host call deadline (`maxHostCallMs`, which defaults to `maxTimeoutMs`). Raise `defaultTimeoutMs`, `maxTimeoutMs`, and `maxHostCallMs` for slow installs and builds, and remember the container's first start. +- Cancelling the execution kills the running command. + +A container command can write to the Workspace and reach the network, whatever the JavaScript backend's `access` and egress settings say. Install `ws:container` only on a read-write backend you would also trust with a shell. diff --git a/packages/computer/src/backends/container/container-module.test.ts b/packages/computer/src/backends/container/container-module.test.ts new file mode 100644 index 00000000..d818f222 --- /dev/null +++ b/packages/computer/src/backends/container/container-module.test.ts @@ -0,0 +1,192 @@ +import { describe, expect, it } from "vitest"; + +import type { WorkspaceTrustedCallContext } from "../../runtime/types.js"; +import { + type ContainerModuleExecOptions, + type ContainerModuleRuntime, + createContainerModule, + describeContainerModule, +} from "./container-module.js"; + +interface Run { + readonly command: string; + readonly options: ContainerModuleExecOptions; + killed: boolean; +} + +// An in-memory runtime that records each command and finishes it with +// the given output, or holds it open until it is killed. +function fakeRuntime(output: { + exitCode?: number; + stdout?: string; + stderr?: string; + hang?: boolean; +}): { runtime: ContainerModuleRuntime; runs: Run[] } { + const runs: Run[] = []; + const runtime: ContainerModuleRuntime = { + async exec(command, options) { + const run: Run = { command, options, killed: false }; + runs.push(run); + let stop: () => void = () => undefined; + const stopped = new Promise((resolve) => { + stop = resolve; + }); + return { + async result() { + if (output.hang) await stopped; + return { + exitCode: run.killed ? 130 : (output.exitCode ?? 0), + stdout: output.stdout ?? "", + stderr: output.stderr ?? "", + }; + }, + async kill() { + run.killed = true; + stop(); + }, + }; + }, + }; + return { runtime, runs }; +} + +function callContext(overrides: Partial = {}) { + return { + signal: new AbortController().signal, + deadline: Date.now() + 60_000, + ...overrides, + }; +} + +describe("createContainerModule", () => { + it("runs the command on the container backend and returns its output", async () => { + const { runtime, runs } = fakeRuntime({ exitCode: 3, stdout: "out", stderr: "err" }); + const container = createContainerModule({ runtime: () => runtime }); + + await expect( + container.exec( + ["npm test", { cwd: "/workspace/app", env: { CI: "1" }, stdin: "y\n" }], + callContext(), + ), + ).resolves.toEqual({ exitCode: 3, stdout: "out", stderr: "err" }); + expect(runs).toHaveLength(1); + expect(runs[0]).toMatchObject({ + command: "npm test", + options: { + backend: "container-shell", + encoding: "utf8", + cwd: "/workspace/app", + env: { CI: "1" }, + stdin: "y\n", + }, + }); + }); + + it("uses the configured backend id and omits unset options", async () => { + const { runtime, runs } = fakeRuntime({}); + const container = createContainerModule({ runtime: () => runtime, backend: "linux" }); + + await container.exec(["ls"], callContext()); + expect(Object.keys(runs[0]?.options ?? {}).sort()).toEqual([ + "backend", + "encoding", + "timeoutMs", + ]); + expect(runs[0]?.options.backend).toBe("linux"); + }); + + it("resolves the runtime on each call, so it can be built before the Workspace", async () => { + let current: ContainerModuleRuntime | undefined; + const container = createContainerModule({ + runtime: () => { + if (!current) throw new Error("Workspace not constructed yet"); + return current; + }, + }); + const { runtime, runs } = fakeRuntime({}); + current = runtime; + + await container.exec(["true"], callContext()); + expect(runs).toHaveLength(1); + }); + + it("caps the timeout at the time left before the host call deadline", async () => { + const { runtime, runs } = fakeRuntime({}); + const container = createContainerModule({ runtime: () => runtime }); + + await container.exec( + ["sleep 1", { timeoutMs: 600_000 }], + callContext({ deadline: Date.now() + 5_000 }), + ); + await container.exec(["sleep 1", { timeoutMs: 1_000 }], callContext()); + expect(runs[0]?.options.timeoutMs).toBeLessThanOrEqual(5_000); + expect(runs[1]?.options.timeoutMs).toBe(1_000); + }); + + it("kills the command when the call is aborted", async () => { + const { runtime, runs } = fakeRuntime({ hang: true }); + const container = createContainerModule({ runtime: () => runtime }); + const controller = new AbortController(); + + const pending = container.exec(["sleep 100"], callContext({ signal: controller.signal })); + await new Promise((resolve) => setTimeout(resolve, 0)); + controller.abort(new Error("cancelled")); + + await expect(pending).resolves.toMatchObject({ exitCode: 130 }); + expect(runs[0]?.killed).toBe(true); + }); + + it("does not start a command once the call is aborted", async () => { + const { runtime, runs } = fakeRuntime({}); + const container = createContainerModule({ runtime: () => runtime }); + const controller = new AbortController(); + controller.abort(new Error("cancelled")); + + await expect( + container.exec(["ls"], callContext({ signal: controller.signal })), + ).rejects.toThrow("cancelled"); + expect(runs).toHaveLength(0); + }); + + it("truncates each stream on UTF-8 boundaries", async () => { + const { runtime } = fakeRuntime({ stdout: "a🙂b", stderr: "🙂🙂" }); + const container = createContainerModule({ runtime: () => runtime, maxOutputBytes: 5 }); + + await expect(container.exec(["echo"], callContext())).resolves.toEqual({ + exitCode: 0, + stdout: "a🙂\n\n[truncated, 1 more bytes]", + stderr: "🙂\n\n[truncated, 4 more bytes]", + }); + }); + + it.each([ + ["no arguments", [], /takes a command/], + ["too many arguments", ["ls", {}, {}], /takes a command/], + ["an empty command", [" "], /non-empty string/], + ["a non-string command", [["ls"]], /non-empty string/], + ["non-object options", ["ls", "fast"], /options must be an object/], + ["an unknown option", ["ls", { shell: "zsh" }], /unknown option "shell"/], + ["a non-string cwd", ["ls", { cwd: 1 }], /cwd must be a string/], + ["a non-string env value", ["ls", { env: { A: 1 } }], /env "A" must be a string/], + ["a non-positive timeout", ["ls", { timeoutMs: 0 }], /timeoutMs must be a positive number/], + ])("rejects %s without running anything", async (_label, args, message) => { + const { runtime, runs } = fakeRuntime({}); + const container = createContainerModule({ runtime: () => runtime }); + + // SAFETY: Each case hands exec arguments that isolate code could send; the cast only widens the test table's inferred type. + await expect(container.exec(args as never, callContext())).rejects.toThrow(message); + expect(runs).toHaveLength(0); + }); + + it("rejects a bad maxOutputBytes at construction", () => { + const { runtime } = fakeRuntime({}); + expect(() => createContainerModule({ runtime: () => runtime, maxOutputBytes: 0 })).toThrow( + /maxOutputBytes/, + ); + }); + + it("describes the module under its installed specifier", () => { + expect(describeContainerModule("ws:linux")).toContain('import { exec } from "ws:linux"'); + expect(describeContainerModule()).toContain('"ws:container"'); + }); +}); diff --git a/packages/computer/src/backends/container/container-module.ts b/packages/computer/src/backends/container/container-module.ts new file mode 100644 index 00000000..87fce8c7 --- /dev/null +++ b/packages/computer/src/backends/container/container-module.ts @@ -0,0 +1,252 @@ +// A trusted module that lets isolate JavaScript run shell commands in +// the Workspace's container backend. +// +// Installed on a WorkerJavaScriptBackend as `ws:container`, it turns +// the container into a library the JavaScript backend calls, rather +// than a second backend the model has to choose between: +// +// import { exec } from "ws:container"; +// const { exitCode, stdout } = await exec("npm test", { cwd: "/workspace" }); +// +// Each call goes through `workspace.runtime.exec`, so the container +// sees the same files as the isolate: the usual sync bracket pushes +// pending Workspace writes before the command and pulls the +// container's changes after it. + +import type { + WorkspaceRuntimeValue, + WorkspaceTrustedCallContext, + WorkspaceTrustedFunction, +} from "../../runtime/types.js"; + +const DEFAULT_BACKEND = "container-shell"; +const DEFAULT_MAX_OUTPUT_BYTES = 64 * 1024; +const EXEC_OPTION_KEYS = new Set(["cwd", "env", "stdin", "timeoutMs"]); + +/** Options the container module passes to `workspace.runtime.exec`. */ +export interface ContainerModuleExecOptions { + /** Backend id the command runs on. */ + readonly backend: string; + /** Output encoding. The module always asks for text. */ + readonly encoding: "utf8"; + /** Working directory inside the container. */ + readonly cwd?: string; + /** Environment variables for this command only. */ + readonly env?: Record; + /** Text piped to the command's standard input. */ + readonly stdin?: string; + /** Wall-clock limit for the command, in milliseconds. */ + readonly timeoutMs: number; +} + +/** The part of a Workspace execution handle the container module uses. */ +export interface ContainerModuleExecHandle { + /** Wait for the command to finish and return its output. */ + result(): Promise<{ exitCode: number; stdout: string; stderr: string }>; + /** Stop the command. */ + kill(): Promise; +} + +/** The part of `workspace.runtime` the container module uses. */ +export interface ContainerModuleRuntime { + /** Start a command on a Workspace backend. */ + exec(command: string, options: ContainerModuleExecOptions): Promise; +} + +/** Options for {@link createContainerModule}. */ +export interface ContainerModuleOptions { + /** + * Returns the Workspace runtime. It is called on every command + * rather than once, so the module can be built before the + * Workspace that owns both backends: pass + * `() => this.workspace.runtime`. + */ + readonly runtime: () => ContainerModuleRuntime; + /** Id of the container backend. Defaults to `"container-shell"`. */ + readonly backend?: string; + /** + * Largest standard output and standard error returned to the + * isolate, in bytes per stream. Output past it is cut and ends with + * a truncation marker. Defaults to 64 KiB. Keep both streams well + * under the backend's `maxCapabilityBytes`. + */ + readonly maxOutputBytes?: number; +} + +/** The `ws:container` trusted module. */ +export type ContainerModule = { + /** + * Run a shell command in the container. + * + * Isolate code calls `exec(command, { cwd, env, stdin, timeoutMs })` + * and gets `{ exitCode, stdout, stderr }` back once the command + * finishes. A non-zero exit code is a normal result, not an error. + */ + readonly exec: WorkspaceTrustedFunction; +}; + +/** + * Build the `ws:container` trusted module over a Workspace's container + * backend. + * + * Install it only on a read-write JavaScript backend. A container + * command can write to the Workspace and reach the network, whatever + * the isolate's own access and egress settings are. + * + * @param options - How to reach the Workspace runtime and which backend to use. + * @returns The module to pass as `trustedModules["ws:container"]`. + * @throws When `maxOutputBytes` is not a positive integer. The host + * configured the module wrongly. + */ +export function createContainerModule(options: ContainerModuleOptions): ContainerModule { + const backend = options.backend ?? DEFAULT_BACKEND; + const maxOutputBytes = options.maxOutputBytes ?? DEFAULT_MAX_OUTPUT_BYTES; + if (!Number.isInteger(maxOutputBytes) || maxOutputBytes <= 0) { + throw new Error("createContainerModule: maxOutputBytes must be a positive integer."); + } + + return { + async exec(args, context) { + const request = parseExecArgs(args); + const timeoutMs = remainingTime(request.timeoutMs, context); + context.signal.throwIfAborted(); + + const handle = await options.runtime().exec(request.command, { + backend, + encoding: "utf8", + timeoutMs, + ...(request.cwd === undefined ? {} : { cwd: request.cwd }), + ...(request.env === undefined ? {} : { env: request.env }), + ...(request.stdin === undefined ? {} : { stdin: request.stdin }), + }); + // Cancelling the isolate execution, or passing the host call + // deadline, stops the command instead of leaving it running. + const kill = () => void handle.kill().catch(() => undefined); + if (context.signal.aborted) kill(); + else context.signal.addEventListener("abort", kill, { once: true }); + try { + const result = await handle.result(); + return { + exitCode: result.exitCode, + stdout: truncate(result.stdout, maxOutputBytes), + stderr: truncate(result.stderr, maxOutputBytes), + }; + } finally { + context.signal.removeEventListener("abort", kill); + } + }, + }; +} + +/** + * Describe `ws:container` for a model. + * + * Append the returned text to the JavaScript backend's description in + * the `exec` tool, so the model knows the module exists and when to + * reach for it. + * + * @param specifier - The specifier the module is installed under. + * @returns A short plain-text description with a usage example. + */ +export function describeContainerModule(specifier = "ws:container"): string { + return [ + `\`import { exec } from ${JSON.stringify(specifier)}\` runs a shell command in a full Linux container that shares this workspace's files.`, + "Use it for npm, node, python, package managers, native binaries, and network access. The container can take a while to start on first use.", + 'Call it as `const { exitCode, stdout, stderr } = await exec("npm test", { cwd: "/workspace" })`. Options are `cwd`, `env`, `stdin`, and `timeoutMs`.', + "Output comes back when the command finishes, and long output is truncated. A non-zero `exitCode` is returned, not thrown.", + ].join(" "); +} + +interface ExecRequest { + readonly command: string; + readonly cwd: string | undefined; + readonly env: Record | undefined; + readonly stdin: string | undefined; + readonly timeoutMs: number | undefined; +} + +// Arguments come from isolate code. A malformed call throws, and the +// bridge hands that error back to the isolate as a rejected promise. +function parseExecArgs(args: readonly WorkspaceRuntimeValue[]): ExecRequest { + if (args.length === 0 || args.length > 2) { + throw new TypeError("exec(command, options?) takes a command and an optional options object."); + } + const [command, options] = args; + if (typeof command !== "string" || command.trim().length === 0) { + throw new TypeError("exec: command must be a non-empty string."); + } + if (options === undefined || options === null) { + return { command, cwd: undefined, env: undefined, stdin: undefined, timeoutMs: undefined }; + } + if (typeof options !== "object" || Array.isArray(options)) { + throw new TypeError("exec: options must be an object."); + } + for (const key of Object.keys(options)) { + if (!EXEC_OPTION_KEYS.has(key)) { + throw new TypeError( + `exec: unknown option ${JSON.stringify(key)}. Use cwd, env, stdin, or timeoutMs.`, + ); + } + } + return { + command, + cwd: optionalString(options.cwd, "cwd"), + env: optionalEnv(options.env), + stdin: optionalString(options.stdin, "stdin"), + timeoutMs: optionalTimeout(options.timeoutMs), + }; +} + +function optionalString(value: WorkspaceRuntimeValue | undefined, name: string) { + if (value === undefined || value === null) return undefined; + if (typeof value !== "string") throw new TypeError(`exec: ${name} must be a string.`); + return value; +} + +function optionalEnv(value: WorkspaceRuntimeValue | undefined) { + if (value === undefined || value === null) return undefined; + if (typeof value !== "object" || Array.isArray(value)) { + throw new TypeError("exec: env must be an object of strings."); + } + const env: Record = {}; + for (const [key, entry] of Object.entries(value)) { + if (typeof entry !== "string") { + throw new TypeError(`exec: env ${JSON.stringify(key)} must be a string.`); + } + env[key] = entry; + } + return env; +} + +function optionalTimeout(value: WorkspaceRuntimeValue | undefined) { + if (value === undefined || value === null) return undefined; + if (typeof value !== "number" || !Number.isFinite(value) || value <= 0) { + throw new TypeError("exec: timeoutMs must be a positive number."); + } + return value; +} + +// The command must finish before the host call deadline, or the +// isolate stops waiting while the container keeps working. Cap the +// requested timeout at the time left. +function remainingTime(requested: number | undefined, context: WorkspaceTrustedCallContext) { + const remaining = context.deadline - Date.now(); + if (remaining <= 0) throw new Error("exec: the host call deadline has already passed."); + return requested === undefined ? remaining : Math.min(requested, remaining); +} + +const encoder = new TextEncoder(); + +function truncate(value: string, maxBytes: number): string { + const totalBytes = encoder.encode(value).byteLength; + if (totalBytes <= maxBytes) return value; + let usedBytes = 0; + let endOffset = 0; + for (const char of value) { + const charBytes = encoder.encode(char).byteLength; + if (usedBytes + charBytes > maxBytes) break; + usedBytes += charBytes; + endOffset += char.length; + } + return `${value.slice(0, endOffset)}\n\n[truncated, ${totalBytes - usedBytes} more bytes]`; +} diff --git a/packages/computer/src/backends/container/index.ts b/packages/computer/src/backends/container/index.ts index 5ad9453f..71ca8741 100644 --- a/packages/computer/src/backends/container/index.ts +++ b/packages/computer/src/backends/container/index.ts @@ -9,6 +9,7 @@ // // import { // CloudflareContainerBackend, +// createContainerModule, // withWorkspaceContainer, // } from "@cloudflare/computer/backends/container"; @@ -25,3 +26,12 @@ export { withWorkspaceContainer, } from "./container-host.js"; export type { ContainerLaunchSpec } from "./container-launch-record.js"; +export { + type ContainerModule, + type ContainerModuleExecHandle, + type ContainerModuleExecOptions, + type ContainerModuleOptions, + type ContainerModuleRuntime, + createContainerModule, + describeContainerModule, +} from "./container-module.js"; diff --git a/packages/computer/tests/script-runner-worker.ts b/packages/computer/tests/script-runner-worker.ts index 6cbe7d1e..238fa311 100644 --- a/packages/computer/tests/script-runner-worker.ts +++ b/packages/computer/tests/script-runner-worker.ts @@ -1,4 +1,7 @@ +import type { ShellRPC, SyncRPC } from "@cloudflare/computer-rpc"; import { DurableObject, RpcTarget, WorkerEntrypoint } from "cloudflare:workers"; +import type { WorkspaceBackend } from "../src/backend.js"; +import { createContainerModule } from "../src/backends/container/index.js"; import { WorkerJavaScriptBackend } from "../src/backends/worker-javascript/index.js"; import { createGitClient } from "../src/git/index.js"; import type { @@ -13,6 +16,43 @@ export interface Env { LOADER: WorkerLoader; } +// A command backend that stands in for the container. It echoes the +// command, working directory, one environment variable, and standard +// input, and exits with the length of the command. +function fakeContainerBackend(): WorkspaceBackend { + const encoder = new TextEncoder(); + const shell: ShellRPC = { + async exec(input) { + const id = input.id ?? crypto.randomUUID(); + const stdin = input.stdin ? new TextDecoder().decode(input.stdin) : ""; + const stdout = `ran ${input.source} in ${input.cwd ?? "?"} with ${input.env?.WHO ?? "-"} and ${stdin || "-"}\n`; + return { + id, + events: new ReadableStream({ + start(controller) { + controller.enqueue({ id, seq: 1, name: "stdout", value: encoder.encode(stdout) }); + controller.enqueue({ id, seq: 2, name: "stderr", value: encoder.encode("warn\n") }); + controller.enqueue({ id, seq: 3, name: "exit", code: input.source.length % 256 }); + controller.close(); + }, + }), + }; + }, + getExec: () => Promise.reject(new Error("not used")), + killExec: () => Promise.resolve(), + disposeExec: () => Promise.resolve(), + }; + // SAFETY: The fake backend declares sync "none", so the Workspace never calls these methods. + const sync = new Proxy({}, { get: () => () => Promise.reject(new Error("sync: none")) }) as SyncRPC; + return { + id: "container-shell", + type: "fake-container", + async connect() { + return { rpc: { sync, shell }, sync: "none", close: async () => {} }; + }, + }; +} + export class HostDO extends DurableObject { readonly #workspace: Workspace; @@ -32,6 +72,9 @@ export class HostDO extends DurableObject { "math-kit": "export const double = (value) => value * 2;", }, trustedModules: { + "ws:container": createContainerModule({ + runtime: () => this.#workspace.runtime, + }), "ws:test-host": { async echo(args) { return { args: [...args] }; @@ -65,6 +108,7 @@ export class HostDO extends DurableObject { }, }, }), + fakeContainerBackend(), ], }); } diff --git a/packages/computer/tests/script-runner.test.ts b/packages/computer/tests/script-runner.test.ts index 453279ae..7cc89196 100644 --- a/packages/computer/tests/script-runner.test.ts +++ b/packages/computer/tests/script-runner.test.ts @@ -332,6 +332,49 @@ describe("WorkspaceRuntime", () => { }); }); + it("runs container commands from isolate code through ws:container", async () => { + const response = await runtime({ + source: ` + import { exec } from "ws:container"; + export default () => + exec("npm test", { cwd: "/workspace/app", env: { WHO: "isolate" }, stdin: "y" }); + `, + cwd: "/workspace", + }); + const text = await response.text(); + expect(response.status, text).toBe(200); + expect(JSON.parse(text), text).toMatchObject({ + result: { + status: "completed", + value: { + exitCode: 8, + stdout: "ran npm test in /workspace/app with isolate and y\n", + stderr: "warn\n", + }, + }, + }); + }); + + it("rejects a malformed ws:container call inside the isolate", async () => { + const response = await runtime({ + source: ` + import { exec } from "ws:container"; + export default async () => { + try { + await exec("ls", { shell: "zsh" }); + return "ran"; + } catch (error) { + return error.message; + } + }; + `, + cwd: "/workspace", + }); + const text = await response.text(); + expect(response.status, text).toBe(200); + expect(JSON.parse(text).result.value).toContain('unknown option "shell"'); + }); + it("does not expose unrestricted host operations through the node:fs dispatcher", async () => { const response = await runtime({ source: ` From 82ad877d21607644cd9c2e96499ec71b3bb6ca77 Mon Sep 17 00:00:00 2001 From: Matt Carey Date: Wed, 30 Sep 2026 16:44:24 +0100 Subject: [PATCH 4/6] computer: Format the script runner fixture --- packages/computer/tests/script-runner-worker.ts | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/packages/computer/tests/script-runner-worker.ts b/packages/computer/tests/script-runner-worker.ts index 238fa311..d6657b6d 100644 --- a/packages/computer/tests/script-runner-worker.ts +++ b/packages/computer/tests/script-runner-worker.ts @@ -1,5 +1,5 @@ -import type { ShellRPC, SyncRPC } from "@cloudflare/computer-rpc"; import { DurableObject, RpcTarget, WorkerEntrypoint } from "cloudflare:workers"; +import type { ShellRPC, SyncRPC } from "@cloudflare/computer-rpc"; import type { WorkspaceBackend } from "../src/backend.js"; import { createContainerModule } from "../src/backends/container/index.js"; import { WorkerJavaScriptBackend } from "../src/backends/worker-javascript/index.js"; @@ -43,7 +43,10 @@ function fakeContainerBackend(): WorkspaceBackend { disposeExec: () => Promise.resolve(), }; // SAFETY: The fake backend declares sync "none", so the Workspace never calls these methods. - const sync = new Proxy({}, { get: () => () => Promise.reject(new Error("sync: none")) }) as SyncRPC; + const sync = new Proxy( + {}, + { get: () => () => Promise.reject(new Error("sync: none")) }, + ) as SyncRPC; return { id: "container-shell", type: "fake-container", From 6c91d5997bbac6135a4603a9a4f81e8a25b6bffe Mon Sep 17 00:00:00 2001 From: Matt Carey Date: Wed, 30 Sep 2026 16:42:35 +0100 Subject: [PATCH 5/6] computer: Configure all isolate modules through modules WorkerJavaScriptBackend had two ways to add imports: modules for bundled source and trustedModules for host functions. On top of those, node:fs, ws:git, and ws:artifacts were always installed, with git and artifacts special-cased in the bridge and gated by allowGitNetwork and allowArtifactNetwork. There is now one modules option. A string is bundled source, as before. A host module, built with defineModule(), runs in the Durable Object under a ws:* specifier and exports its functions by name. A factory form receives the Workspace's Git client, Artifacts client, and runtime when the backend connects, and every call gets its access level and a path resolver confined to the backend root alongside the signal and deadline. Git, Artifacts, and the container become prebuilt host modules under @cloudflare/computer/modules/*, and nothing under ws: is installed unless it is configured. Network access moves to allowNetwork on the git and artifacts modules. The container module takes the runtime from the host instead of a getter and refuses to run on a read-only backend. node:fs and node:fs/promises stay built in. --- .changeset/named-trusted-module-functions.md | 7 - .changeset/unified-modules.md | 9 + .changeset/ws-container-module.md | 5 - README.md | 4 +- docs/16_code_execution.md | 4 +- docs/17_isolate_javascript.md | 156 ++++++++-------- docs/README.md | 9 +- examples/rlm/README.md | 6 +- examples/rlm/worker/capability.test.ts | 17 +- examples/rlm/worker/capability.ts | 4 +- examples/rlm/worker/rlm-agent.ts | 3 +- packages/computer/README.md | 9 +- packages/computer/package.json | 12 ++ packages/computer/rolldown.config.ts | 3 + .../computer/src/backends/container/index.ts | 10 - .../worker-javascript/module-graph.ts | 165 +++++++++-------- .../worker-javascript.test.ts | 136 +++++++++++--- .../worker-javascript/worker-javascript.ts | 69 ++++--- packages/computer/src/index.ts | 10 +- packages/computer/src/modules/artifacts.ts | 89 +++++++++ .../container.test.ts} | 89 +++++---- .../container.ts} | 96 +++------- packages/computer/src/modules/git.ts | 138 ++++++++++++++ packages/computer/src/runtime/bridge.test.ts | 28 ++- packages/computer/src/runtime/bridge.ts | 173 +++--------------- packages/computer/src/runtime/module.ts | 32 ++++ packages/computer/src/runtime/types.ts | 69 +++++-- packages/computer/src/workspace.ts | 1 + .../computer/tests/script-runner-worker.ts | 18 +- packages/computer/tests/script-runner.test.ts | 18 +- 30 files changed, 833 insertions(+), 556 deletions(-) delete mode 100644 .changeset/named-trusted-module-functions.md create mode 100644 .changeset/unified-modules.md delete mode 100644 .changeset/ws-container-module.md create mode 100644 packages/computer/src/modules/artifacts.ts rename packages/computer/src/{backends/container/container-module.test.ts => modules/container.test.ts} (70%) rename packages/computer/src/{backends/container/container-module.ts => modules/container.ts} (72%) create mode 100644 packages/computer/src/modules/git.ts create mode 100644 packages/computer/src/runtime/module.ts diff --git a/.changeset/named-trusted-module-functions.md b/.changeset/named-trusted-module-functions.md deleted file mode 100644 index 5cb2ebae..00000000 --- a/.changeset/named-trusted-module-functions.md +++ /dev/null @@ -1,7 +0,0 @@ ---- -"@cloudflare/computer": minor ---- - -Trusted modules now export named functions. Pass `trustedModules: { "ws:container": { exec } }` and caller code writes `import { exec } from "ws:container"`. Each host function receives `(args, { signal, deadline })`. - -This replaces the single `call(method, args, context)` handler and the generic `call` export. Move each method into its own function: `{ call(method, args) { if (method === "batch") ... } }` becomes `{ batch(args, context) { ... } }`, and `call("batch", requests)` in caller code becomes `batch(requests)`. The backend now rejects a bad specifier or export name at construction instead of at the first execution. diff --git a/.changeset/unified-modules.md b/.changeset/unified-modules.md new file mode 100644 index 00000000..6fa7fb41 --- /dev/null +++ b/.changeset/unified-modules.md @@ -0,0 +1,9 @@ +--- +"@cloudflare/computer": minor +--- + +`WorkerJavaScriptBackend` takes a single `modules` option. A string value is bundled source, as before. A host module runs in the Durable Object under a `ws:*` specifier, and each of its functions becomes a named export, so `modules: { "ws:container": createContainerModule() }` lets code write `import { exec } from "ws:container"`. Build your own with `defineModule({ fn })`, or `defineModule((host) => ({ fn }))` to use the Workspace's Git client, Artifacts client, or runtime. Each function receives `(args, { signal, deadline, access, resolvePath })`. + +`ws:git` and `ws:artifacts` are no longer installed automatically. Add `createGitModule()` from `@cloudflare/computer/modules/git` and `createArtifactsModule()` from `@cloudflare/computer/modules/artifacts`. `createContainerModule()` from `@cloudflare/computer/modules/container` runs shell commands in the Workspace's container backend. The container shares the Workspace's files, a canceled execution kills the command, and it refuses to run on a read-only backend. `describeContainerModule()` returns text for the model that explains it. `node:fs` and `node:fs/promises` stay built in. + +To migrate, move `trustedModules` entries into `modules` and wrap each in `defineModule()`, replacing any `call(method, args)` handler with one function per method. Replace `allowGitNetwork: true` with `createGitModule({ allowNetwork: true })` and `allowArtifactNetwork: true` with `createArtifactsModule({ allowNetwork: true })`. diff --git a/.changeset/ws-container-module.md b/.changeset/ws-container-module.md deleted file mode 100644 index b6b3ca67..00000000 --- a/.changeset/ws-container-module.md +++ /dev/null @@ -1,5 +0,0 @@ ---- -"@cloudflare/computer": minor ---- - -Add `createContainerModule()` to `@cloudflare/computer/backends/container`. Install the returned module as `ws:container` on a `WorkerJavaScriptBackend`, and JavaScript can run shell commands in the Workspace's container with `import { exec } from "ws:container"`. The container shares the Workspace's files, and a canceled execution kills the running command. `describeContainerModule()` returns text for the model that explains the module. diff --git a/README.md b/README.md index dd124e41..cb7daa64 100644 --- a/README.md +++ b/README.md @@ -14,8 +14,8 @@ SQLite and exposes one pluggable execution surface through Workers RPC, so there is no second store or sync round trip. - **Isolate JavaScript** runs an ECMAScript module in a fresh Dynamic Worker with structured input/results, durable relative imports, - configured libraries, Workspace-backed `node:fs/promises`, and trusted `ws:git` and - `ws:artifacts` modules. + configured libraries, Workspace-backed `node:fs/promises`, and host modules such as + `ws:git`, `ws:artifacts`, and `ws:container`. A Workspace may register multiple backends under stable IDs. `workspace.runtime.exec(source, { backend })` is the single execution diff --git a/docs/16_code_execution.md b/docs/16_code_execution.md index 26371512..34c316a2 100644 --- a/docs/16_code_execution.md +++ b/docs/16_code_execution.md @@ -17,7 +17,7 @@ The selected backend defines how it interprets `source`. | --- | --- | --- | | `container-shell` | shell command | Full Linux, native binaries, installed packages, processes | | `worker-shell` | just-bash command | Fast text tools and Workspace Git without a Container | -| `worker-javascript` | ECMAScript module | Isolated structured JavaScript with trusted Workspace modules | +| `worker-javascript` | ECMAScript module | Isolated structured JavaScript with Workspace host modules | Applications may register additional command or module backends under their own IDs. Backend IDs are part of the execution contract: changing the backend may change the source language. @@ -85,4 +85,4 @@ new WorkerJavaScriptBackend({ The backend argument is never itself authorization. -See [17. Isolate JavaScript](./17_isolate_javascript.md) for module and trusted-package behavior. +See [17. Isolate JavaScript](./17_isolate_javascript.md) for module behavior. diff --git a/docs/17_isolate_javascript.md b/docs/17_isolate_javascript.md index 2eeb2524..680761eb 100644 --- a/docs/17_isolate_javascript.md +++ b/docs/17_isolate_javascript.md @@ -88,7 +88,7 @@ Completed execution records remain available for replay for sixty minutes by def Cancellation stops new host capability calls, disposes the Dynamic Worker, and waits for host calls that were already accepted. Exit 130 is published only after those calls settle. Normal completion uses the same drain rule, so an unawaited capability call cannot mutate the workspace after exit 0. -Host calls have a caller-visible deadline, controlled by `maxHostCallMs` and defaulting to `maxTimeoutMs`. Missing the deadline fails the capability call and marks the execution failed, even if caller code catches that error. Execution still waits for the accepted host operation itself before publishing a terminal event because many host APIs cannot roll back an external side effect after dispatch. Trusted modules receive an optional `{ signal, deadline }` context and must stop promptly when the signal aborts. A trusted module that ignores cancellation and never settles will keep execution in its finalizing state. `compatibilityDate` and `compatibilityFlags` control the Dynamic Worker runtime and default to the package-tested settings. +Host calls have a caller-visible deadline, controlled by `maxHostCallMs` and defaulting to `maxTimeoutMs`. Missing the deadline fails the capability call and marks the execution failed, even if caller code catches that error. Execution still waits for the accepted host operation itself before publishing a terminal event because many host APIs cannot roll back an external side effect after dispatch. Host module functions receive a `signal` in their context and must stop promptly when it aborts. A host module that ignores cancellation and never settles will keep execution in its finalizing state. `compatibilityDate` and `compatibilityFlags` control the Dynamic Worker runtime and default to the package-tested settings. ## Environment, standard input, and the `process` shim @@ -119,22 +119,37 @@ const handle = await workspace.runtime.exec( ); ``` -## Configured modules +## Modules -Bare imports are installed at backend construction, not passed on individual executions: +Caller source can import three kinds of module, and all of them are fixed when the backend is constructed: + +| Kind | Configured with | Runs in | Example | +| --- | --- | --- | --- | +| Built in | Always installed | The isolate, backed by the Workspace | `node:fs`, `node:fs/promises` | +| Source | `modules: { name: "source" }` | The isolate | a bundled library | +| Host | `modules: { "ws:name": hostModule }` | The Durable Object | `ws:git`, `ws:container`, your own | ```ts +import { defineModule } from "@cloudflare/computer"; +import { createArtifactsModule } from "@cloudflare/computer/modules/artifacts"; +import { createContainerModule } from "@cloudflare/computer/modules/container"; +import { createGitModule } from "@cloudflare/computer/modules/git"; + new WorkerJavaScriptBackend({ loader: env.LOADER, modules: { "tar-stream": TAR_STREAM_BUNDLE, + "ws:git": createGitModule(), + "ws:artifacts": createArtifactsModule(), + "ws:container": createContainerModule(), + "ws:model": defineModule({ async batch(args, context) { /* ... */ } }), }, }); ``` -Unknown bare imports fail before Worker creation. `node:fs` and `node:fs/promises` are host-installed exceptions backed by the durable Workspace. Configured modules are code, not host authority, and may not use the reserved `ws:` namespace or shadow either filesystem specifier. +An import that is not built in, configured, or a relative Workspace path fails before the Worker is created. Caller source and durable files cannot shadow a configured or built-in module. -## Trusted Workspace modules +### Built-in filesystem Filesystem access uses the familiar asynchronous Node API, but is backed by the durable Workspace rather than an isolate-local filesystem. Both forms are installed automatically: @@ -148,108 +163,76 @@ await fs.writeFile("/workspace/output.txt", text.toUpperCase()); Supported promise APIs are `readFile`, `writeFile`, `mkdir`, `rm`, `chmod`, `symlink`, `readlink`, `readdir`, `stat`, `lstat`, and `access`. `readFile` returns bytes when encoding is omitted and supports `"utf8"` / `"utf-8"` for text; other encodings are rejected. `writeFile` supports the default `"w"` flag and exclusive `"wx"`; other Node flags are rejected, and—as in Node—the parent directory must already exist. Relative symlink targets are preserved by `readlink`, while reads and writes through symlinks are rejected by the Workspace confinement boundary. Synchronous and callback-style Node filesystem APIs are intentionally unavailable because every operation crosses the isolate-to-Workspace capability boundary. -The entire `ws:` namespace remains reserved for other Workspace-maintained host capabilities. The built-in runtime installs `ws:git` and `ws:artifacts`. +Path confinement rejects lexical escapes and every symlink component before an operation. These checks are not an atomic inode-style “resolve beneath root” primitive: do not treat one isolate capability as a security boundary against a separate, more privileged principal concurrently replacing paths in the same mutable Workspace. Deployments requiring that adversarial concurrency need a future transactional DOFS primitive or separate Workspace identities. -### `ws:git` +### Source modules -```js -import { clone, diff, status, log, cli } from "ws:git"; -``` +A string value is JavaScript source installed as a bare import, such as a bundled library. It is plain code with no host access, and it cannot use the `ws:` namespace or replace `node:fs` or `node:fs/promises`. -`ws:git` is explicit host authority rather than ambient isolate networking. Clone, fetch, pull, push, `ls-remote`, and submodule commands can perform host-side requests even when the Dynamic Worker has `globalOutbound: null`, so they are denied by default. Enable them only on a trusted backend construction with `allowGitNetwork: true`; local Git operations remain available without that authority. Remote `ws:artifacts.importArtifact()` is independently denied unless backend construction sets `allowArtifactNetwork: true`. +### Host modules -### `ws:artifacts` +A host module runs in the Durable Object, and each of its functions becomes a named export in the isolate. Host modules must use a simple `ws:*` specifier. Nothing under `ws:` is installed unless you configure it. -```js -import { - create, - get, - list, - importArtifact, - deleteArtifact, -} from "ws:artifacts"; -``` - -These modules are sandbox-side shims over host RPC. Loader bindings, credentials, Durable Object storage, and unrestricted Workspace objects never enter user code. The host bridge checks the backend's fixed read/read-write authority on every mutation. Artifacts methods fail clearly when no Artifacts binding is configured. +Build your own with `defineModule()`. Pass the functions directly, or pass a factory that builds them from the Workspace's Git client, Artifacts client, and runtime. The backend calls the factory once when it connects to its Workspace: -Caller modules and durable files cannot shadow `node:fs`, `node:fs/promises`, or `ws:*`. - -Path confinement rejects lexical escapes and every symlink component before an operation. These checks are not an atomic inode-style “resolve beneath root” primitive: do not treat one isolate capability as a security boundary against a separate, more privileged principal concurrently replacing paths in the same mutable Workspace. Deployments requiring that adversarial concurrency need a future transactional DOFS primitive or separate Workspace identities. +```ts +modules: { + "ws:repo": defineModule((host) => ({ + async recent(args, context) { + const dir = await context.resolvePath(String(args[0] ?? ".")); + const commits = await host.git.log({ dir, depth: 5 }); + return commits.map((commit) => ({ oid: commit.oid, message: commit.message })); + }, + })), +} +``` -## Isolation and lifecycle +```js +import { recent } from "ws:repo"; +export default () => recent("/workspace/app"); +``` -Each execution receives a fresh Dynamic Worker with: +Each function receives the arguments the isolate passed, as an array of JSON-compatible values, and a context: -- explicit Worker Loader CPU limits; -- a host wall-clock deadline; -- `globalOutbound: null` by default; -- finite, acyclic JSON-compatible input and structured result validation; -- configurable source/module graph, input, result, stdin, stdio, file/capability request, and response byte limits (`maxSourceBytes`, `maxInputBytes`, `maxResultBytes`, `maxStdinBytes`, `maxStdioBytes`, and `maxCapabilityBytes`); -- explicit entrypoint and Worker disposal; -- host-owned cancellation; -- retained events and result rows in the Workspace database. +| Field | Meaning | +| --- | --- | +| `signal` | Aborts when the call passes its deadline or the execution is cancelled. | +| `deadline` | Epoch milliseconds after which the isolate stops waiting. | +| `access` | The backend's `"read"` or `"read-write"` access. Check it before any write. | +| `resolvePath(path, { allowMissing })` | Confines a caller path to the backend root and rejects symlinks. | -Standard output and standard error stream live. The Dynamic Worker hands the readable end of its output stream to the host through the `attachOutput` bridge call, and the host drains it frame by frame while user code is still running, appending each chunk to the execution event stream as it arrives rather than buffering the run and publishing at the end. The structured result and the exit event settle once the output stream closes, so the terminal events always follow the last output. Output remains bounded by `maxStdioBytes` across both streams. Completed writes are durable immediately. Failure or cancellation does not roll back filesystem effects already completed. +The arguments come from caller code, so parse them before use. The return value must be JSON-compatible and fits within the same capability byte limits as every other host call. A function that ignores `signal` and never settles keeps the execution in its finalizing state. -## Trusted integrations +Specifiers are checked at construction, and export names when the backend connects. A module must export at least one function, and every export name must be a JavaScript identifier name other than `default` or `then`. A reserved word such as `delete` is allowed, and caller code renames it on import: `import { delete as remove } from "ws:files"`. Importing a name the module does not export fails when the module graph links, before any code runs. -A host can add its own reserved modules through `WorkerJavaScriptBackend.trustedModules`. Each module is an object of host functions, and each function becomes a named export in the isolate: +### `ws:git` -```ts -new WorkerJavaScriptBackend({ - loader: env.LOADER, - trustedModules: { - "ws:container": { - async exec(args, { signal }) { - const command = parseCommand(args); - const handle = await workspace.runtime.exec(command, { backend: "container" }); - signal.addEventListener("abort", () => void handle.kill()); - const result = await handle.result(); - return { exitCode: result.exitCode }; - }, - }, - }, -}); +```js +import { clone, diff, status, log, cli } from "ws:git"; ``` -Caller source imports the functions by name: +`createGitModule()` from `@cloudflare/computer/modules/git` wraps the Workspace's Git client. Every `dir` and `cwd` is confined to the backend root, `clone` and `cli` need a read-write backend, and `cli` rejects `-C`, `--git-dir`, and `--work-tree`. Clone, fetch, pull, push, `ls-remote`, and submodule commands run from the host, even when the Dynamic Worker has `globalOutbound: null`, so they are denied unless you pass `createGitModule({ allowNetwork: true })`. -```js -import { exec } from "ws:container"; +### `ws:artifacts` -export default async function () { - return exec("npm test"); -} +```js +import { create, get, list, importArtifact, deleteArtifact } from "ws:artifacts"; ``` -Each host function receives the arguments the isolate passed, as an array of JSON-compatible values, and a `{ signal, deadline }` context. The arguments come from caller code, so parse them before use. The return value must be JSON-compatible and fits within the same capability byte limits as every other host call. +`createArtifactsModule()` from `@cloudflare/computer/modules/artifacts` wraps the Workspace's Artifacts client. Calls that change Artifacts need a read-write backend. `importArtifact()` fetches from a caller-chosen URL on the host, so it is denied unless you pass `createArtifactsModule({ allowNetwork: true })`. Every call fails clearly when no Artifacts binding is configured. -The backend checks trusted modules when it is constructed. A specifier must be a simple `ws:*` name that does not shadow `ws:git` or `ws:artifacts`. A module must export at least one function, and every export name must be a JavaScript identifier name other than `default` or `then`. A reserved word such as `delete` is allowed, and caller code renames it on import: `import { delete as remove } from "ws:files"`. Importing a name the module does not export fails when the module graph links, before any code runs. +### `ws:container` -These modules are fixed at construction, and caller source cannot supply or replace them. - -### Container commands with `ws:container` - -`createContainerModule()` from `@cloudflare/computer/backends/container` builds a ready-made `ws:container` module. With it, JavaScript is the only backend the model sees, and the container is something that JavaScript can call: +`createContainerModule()` from `@cloudflare/computer/modules/container` lets JavaScript run shell commands in the Workspace's container backend. With it, JavaScript is the only backend the model sees, and the container is something that JavaScript can call: ```ts -import { WorkerJavaScriptBackend } from "@cloudflare/computer/backends/worker-javascript"; -import { - CloudflareContainerBackend, - createContainerModule, - describeContainerModule, -} from "@cloudflare/computer/backends/container"; -import { createAITools } from "@cloudflare/computer/tools"; - this.workspace = new Workspace({ storage: ctx.storage, backends: [ new WorkerJavaScriptBackend({ loader: env.LOADER, access: "read-write", - trustedModules: { - "ws:container": createContainerModule({ runtime: () => this.workspace.runtime }), - }, + modules: { "ws:container": createContainerModule() }, }), new CloudflareContainerBackend({ /* ... */ }), ], @@ -284,4 +267,19 @@ A few limits follow from `exec` being a host call: - The command's timeout is capped at the time left before the host call deadline (`maxHostCallMs`, which defaults to `maxTimeoutMs`). Raise `defaultTimeoutMs`, `maxTimeoutMs`, and `maxHostCallMs` for slow installs and builds, and remember the container's first start. - Cancelling the execution kills the running command. -A container command can write to the Workspace and reach the network, whatever the JavaScript backend's `access` and egress settings say. Install `ws:container` only on a read-write backend you would also trust with a shell. +A container command can write to the Workspace and reach the network, whatever the JavaScript backend's egress settings say. `exec` refuses to run on a read-only backend. + +## Isolation and lifecycle + +Each execution receives a fresh Dynamic Worker with: + +- explicit Worker Loader CPU limits; +- a host wall-clock deadline; +- `globalOutbound: null` by default; +- finite, acyclic JSON-compatible input and structured result validation; +- configurable source/module graph, input, result, stdin, stdio, file/capability request, and response byte limits (`maxSourceBytes`, `maxInputBytes`, `maxResultBytes`, `maxStdinBytes`, `maxStdioBytes`, and `maxCapabilityBytes`); +- explicit entrypoint and Worker disposal; +- host-owned cancellation; +- retained events and result rows in the Workspace database. + +Standard output and standard error stream live. The Dynamic Worker hands the readable end of its output stream to the host through the `attachOutput` bridge call, and the host drains it frame by frame while user code is still running, appending each chunk to the execution event stream as it arrives rather than buffering the run and publishing at the end. The structured result and the exit event settle once the output stream closes, so the terminal events always follow the last output. Output remains bounded by `maxStdioBytes` across both streams. Completed writes are durable immediately. Failure or cancellation does not roll back filesystem effects already completed. diff --git a/docs/README.md b/docs/README.md index 25b76eb7..2775b03e 100644 --- a/docs/README.md +++ b/docs/README.md @@ -20,7 +20,7 @@ It provides: - R2-backed mounts for pre-filling read-only data into the workspace tree. - Durability over DO restarts for all file operations. - Pluggable execution backends selected through `workspace.runtime`: a Cloudflare Container shell, a just-bash Dynamic Worker, or an isolated ECMAScript-module Dynamic Worker. - - Isolated JavaScript with structured input/results, durable relative imports, configured libraries, durable `node:fs/promises`, trusted `ws:git` / `ws:artifacts`, and managed execution records. + - Isolated JavaScript with structured input/results, durable relative imports, configured libraries, durable `node:fs/promises`, host modules such as `ws:git` and `ws:container`, and managed execution records. - Workspace constructable without a backend, for filesystem-only use cases. - Out-of-the-box AI SDK tools for `@cloudflare/agents` through `@cloudflare/computer/tools`. @@ -45,9 +45,12 @@ The package ships several entrypoints: | `@cloudflare/computer` | The Workspace wrapper, first-class `workspace.runtime`, stub types, the R2 mount, and proxy classes. | | `@cloudflare/computer/backends/container` | `CloudflareContainerBackend` and `withWorkspaceContainer`. Pulls in the computerd / capnweb sync plumbing. | | `@cloudflare/computer/backends/worker-shell` | `WorkerShellBackend` and the bundled just-bash command runtime. | -| `@cloudflare/computer/backends/worker-javascript` | `WorkerJavaScriptBackend`, configured libraries, durable relative imports, `node:fs/promises`, and trusted `ws:git` / `ws:artifacts`. | +| `@cloudflare/computer/backends/worker-javascript` | `WorkerJavaScriptBackend`, configured libraries, durable relative imports, `node:fs/promises`, and host modules. | | `@cloudflare/computer/git` | Opt-in isomorphic-git glue for working with checkouts inside the workspace. Bundled lazily, with `pako` replaced by Workers `node:zlib`, and kept out of the default `@cloudflare/computer` graph. | | `@cloudflare/computer/artifacts` | `createArtifact`, an optionally session-scoped wrapper over the Cloudflare Artifacts Workers binding, plus its argv CLI. | +| `@cloudflare/computer/modules/container` | `createContainerModule()` for `ws:container`: run container commands from isolate JavaScript. | +| `@cloudflare/computer/modules/git` | `createGitModule()` for `ws:git`: confined Git from isolate JavaScript. | +| `@cloudflare/computer/modules/artifacts` | `createArtifactsModule()` for `ws:artifacts`: Artifacts from isolate JavaScript. | | `@cloudflare/computer/tools` | AI SDK tools for agents: read, write, edit, ls, optional exec, and optional publish. | A consumer that only uses the container backend never imports the @@ -240,7 +243,7 @@ above, then dive into the area you're working on. | [14. Assets interface](./14_assets_interface.md) | `share` a workspace file to R2 and get back a presigned URL. | | [15. Artifacts interface](./15_artifacts_interface.md) | `createArtifact` and the `artifacts` CLI, an optionally session-scoped wrapper over the Cloudflare Artifacts binding. | | [16. Execution runtime architecture](./16_code_execution.md) | One runtime entry point over command and module backends. | -| [17. Isolate JavaScript runtime](./17_isolate_javascript.md) | ECMAScript modules, durable imports, configured libraries, durable `node:fs/promises`, trusted `ws:git` / `ws:artifacts`, and managed lifecycle. | +| [17. Isolate JavaScript runtime](./17_isolate_javascript.md) | ECMAScript modules, durable imports, configured libraries, durable `node:fs/promises`, host modules, and managed lifecycle. | | [18. Runtime migration](./18_runtime_migration.md) | Breaking preview-API mappings from public shell and script-execution surfaces to `workspace.runtime`. | | [19. Performance](./19_performance.md) | Filesystem benchmarks: `fs-bench` numbers, an `npm install` comparison, and how to reproduce them. | diff --git a/examples/rlm/README.md b/examples/rlm/README.md index fe2b04b1..8a012998 100644 --- a/examples/rlm/README.md +++ b/examples/rlm/README.md @@ -66,8 +66,8 @@ const backend = new WorkerJavaScriptBackend({ root: "/workspace", access: "read", egress: { mode: "none" }, - trustedModules: { - "ws:model": modelCapability, + modules: { + "ws:model": defineModule(modelCapability), }, }); @@ -77,7 +77,7 @@ const workspace = new Workspace({ }); ``` -The important line is `trustedModules`. Generated code cannot read model credentials or call the network directly. It can only use the host-owned `ws:model` interface. +The important line is `modules`. Generated code cannot read model credentials or call the network directly. It can only use the host-owned `ws:model` interface. A generated module follows this shape: diff --git a/examples/rlm/worker/capability.test.ts b/examples/rlm/worker/capability.test.ts index a54d9385..c902cdea 100644 --- a/examples/rlm/worker/capability.test.ts +++ b/examples/rlm/worker/capability.test.ts @@ -1,3 +1,4 @@ +import type { WorkspaceModuleCallContext } from "@cloudflare/computer"; import type { LanguageModel } from "ai"; import { beforeEach, describe, expect, it, vi } from "vitest"; @@ -30,8 +31,13 @@ function successfulResult(text = "ok") { }; } -function context() { - return { signal: new AbortController().signal, deadline: Date.now() + 1_000 }; +function context(signal = new AbortController().signal): WorkspaceModuleCallContext { + return { + signal, + deadline: Date.now() + 1_000, + access: "read", + resolvePath: async (path) => path, + }; } async function waitForCalls(count: number): Promise { @@ -190,10 +196,7 @@ describe("recursive model batch capability", () => { const signal = new AbortController().signal; await expect( - capability.batch([[{ prompt: "classify", input: { evidence: "safe" } }]], { - signal, - deadline: Date.now() + 1_000, - }), + capability.batch([[{ prompt: "classify", input: { evidence: "safe" } }]], context(signal)), ).resolves.toEqual([ { id: expect.any(String), index: 0, ok: true, text: "classification", error: null }, ]); @@ -261,7 +264,7 @@ describe("recursive model batch capability", () => { const controller = new AbortController(); const result = capability.batch( [Array.from({ length: 8 }, (_, index) => ({ prompt: `request ${index}`, input: null }))], - { signal: controller.signal, deadline: Date.now() + 1_000 }, + context(controller.signal), ); await waitForCalls(4); diff --git a/examples/rlm/worker/capability.ts b/examples/rlm/worker/capability.ts index e27fb8c1..7ad79083 100644 --- a/examples/rlm/worker/capability.ts +++ b/examples/rlm/worker/capability.ts @@ -1,4 +1,4 @@ -import type { WorkspaceRuntimeValue, WorkspaceTrustedFunction } from "@cloudflare/computer"; +import type { WorkspaceModuleFunction, WorkspaceRuntimeValue } from "@cloudflare/computer"; import { generateText, type LanguageModel } from "ai"; import { z } from "zod"; @@ -76,7 +76,7 @@ interface ChildHooks { /** The `ws:model` trusted module: one `batch` function over bounded child requests. */ export type ModelCapability = { /** Run up to 24 child model requests and return one result per request. */ - readonly batch: WorkspaceTrustedFunction; + readonly batch: WorkspaceModuleFunction; }; export function createModelCapability(model: LanguageModel, hooks: ChildHooks): ModelCapability { diff --git a/examples/rlm/worker/rlm-agent.ts b/examples/rlm/worker/rlm-agent.ts index 7d0fd706..071d4b9b 100644 --- a/examples/rlm/worker/rlm-agent.ts +++ b/examples/rlm/worker/rlm-agent.ts @@ -1,6 +1,7 @@ import { AIChatAgent, type OnChatMessageOptions } from "@cloudflare/ai-chat"; import { type DurableObjectStorageLike, + defineModule, Workspace, type WorkspaceRuntimeLoader, } from "@cloudflare/computer"; @@ -107,7 +108,7 @@ export class RlmAgent extends AIChatAgent { root: WORKSPACE_ROOT, access: "read", egress: { mode: "none" }, - trustedModules: { "ws:model": modelCapability }, + modules: { "ws:model": defineModule(modelCapability) }, maxConcurrentExecutions: 1, maxConcurrentCapabilityCalls: 4, // One manifest read + 24 chunk reads + one bounded ws:model batch. diff --git a/packages/computer/README.md b/packages/computer/README.md index 14833957..3f0115b8 100644 --- a/packages/computer/README.md +++ b/packages/computer/README.md @@ -254,8 +254,8 @@ Alongside `exec`, the runtime exposes `getExec`, `killExec`, and [`examples/worker-shell`](../../examples/worker-shell). - **Worker JavaScript** evaluates a module with structured input/results, durable relative imports, configured libraries, - Workspace-backed `node:fs/promises`, and trusted `ws:git` / - `ws:artifacts` modules. It runs after `runtime.exec()` returns; the + Workspace-backed `node:fs/promises`, and host modules such as + `ws:git`, `ws:artifacts`, and `ws:container`. It runs after `runtime.exec()` returns; the run stays alive while its event stream is consumed. See [`docs/17_isolate_javascript.md`](../../docs/17_isolate_javascript.md) and [`examples/worker-javascript`](../../examples/worker-javascript). @@ -417,7 +417,10 @@ on a computerd instance. | `@cloudflare/computer` | The `Workspace` wrapper, `workspace.runtime`, stub types, the R2 mount, and proxy classes. | | `@cloudflare/computer/backends/container` | `CloudflareContainerBackend` and `withWorkspaceContainer`. Pulls in the computerd / capnweb sync plumbing. | | `@cloudflare/computer/backends/worker-shell` | `WorkerShellBackend` and the bundled just-bash runtime. | -| `@cloudflare/computer/backends/worker-javascript` | `WorkerJavaScriptBackend`, configured libraries, durable imports, `node:fs/promises`, and trusted `ws:git` / `ws:artifacts`. | +| `@cloudflare/computer/backends/worker-javascript` | `WorkerJavaScriptBackend`, configured libraries, durable imports, `node:fs/promises`, and host modules. | +| `@cloudflare/computer/modules/container` | `createContainerModule()` for `ws:container`: run container commands from isolate JavaScript. | +| `@cloudflare/computer/modules/git` | `createGitModule()` for `ws:git`: confined Git from isolate JavaScript. | +| `@cloudflare/computer/modules/artifacts` | `createArtifactsModule()` for `ws:artifacts`: Artifacts from isolate JavaScript. | | `@cloudflare/computer/tools` | AI SDK tools for agents: `read`, `ls`, `find`, `grep`, `write`, `edit`, `delete`, and optional `exec` and `publish`. | | `@cloudflare/computer/git` | Opt-in `isomorphic-git` glue for checkouts inside the workspace. | | `@cloudflare/computer/assets` | `createAssets` — share a workspace file to R2 as a presigned URL. | diff --git a/packages/computer/package.json b/packages/computer/package.json index 42898ddf..7baeb171 100644 --- a/packages/computer/package.json +++ b/packages/computer/package.json @@ -31,6 +31,18 @@ "types": "./dist/artifacts/index.d.ts", "import": "./dist/artifacts/index.js" }, + "./modules/container": { + "types": "./dist/modules/container.d.ts", + "import": "./dist/modules/container.js" + }, + "./modules/git": { + "types": "./dist/modules/git.d.ts", + "import": "./dist/modules/git.js" + }, + "./modules/artifacts": { + "types": "./dist/modules/artifacts.d.ts", + "import": "./dist/modules/artifacts.js" + }, "./tools": { "types": "./dist/tools/index.d.ts", "import": "./dist/tools/index.js" diff --git a/packages/computer/rolldown.config.ts b/packages/computer/rolldown.config.ts index 7f21309e..7179b638 100644 --- a/packages/computer/rolldown.config.ts +++ b/packages/computer/rolldown.config.ts @@ -31,6 +31,9 @@ export default defineConfig({ "artifacts/index": "src/artifacts/index.ts", "assets/index": "src/assets/index.ts", "tools/index": "src/tools/index.ts", + "modules/container": "src/modules/container.ts", + "modules/git": "src/modules/git.ts", + "modules/artifacts": "src/modules/artifacts.ts", "backends/container/index": "src/backends/container/index.ts", "backends/worker-javascript/index": "src/backends/worker-javascript/index.ts", "backends/worker-shell/index": "src/backends/worker-shell/index.ts", diff --git a/packages/computer/src/backends/container/index.ts b/packages/computer/src/backends/container/index.ts index 71ca8741..5ad9453f 100644 --- a/packages/computer/src/backends/container/index.ts +++ b/packages/computer/src/backends/container/index.ts @@ -9,7 +9,6 @@ // // import { // CloudflareContainerBackend, -// createContainerModule, // withWorkspaceContainer, // } from "@cloudflare/computer/backends/container"; @@ -26,12 +25,3 @@ export { withWorkspaceContainer, } from "./container-host.js"; export type { ContainerLaunchSpec } from "./container-launch-record.js"; -export { - type ContainerModule, - type ContainerModuleExecHandle, - type ContainerModuleExecOptions, - type ContainerModuleOptions, - type ContainerModuleRuntime, - createContainerModule, - describeContainerModule, -} from "./container-module.js"; diff --git a/packages/computer/src/backends/worker-javascript/module-graph.ts b/packages/computer/src/backends/worker-javascript/module-graph.ts index 365c6303..704c488f 100644 --- a/packages/computer/src/backends/worker-javascript/module-graph.ts +++ b/packages/computer/src/backends/worker-javascript/module-graph.ts @@ -1,7 +1,12 @@ import { parse } from "acorn"; import type { WorkspaceRuntimeCapability } from "../../runtime/capability.js"; -import type { WorkspaceRuntimeLoader, WorkspaceTrustedModule } from "../../runtime/types.js"; +import type { + WorkspaceHostModule, + WorkspaceModule, + WorkspaceModuleFunctions, + WorkspaceRuntimeLoader, +} from "../../runtime/types.js"; export type JavaScriptModuleMap = WorkspaceRuntimeLoader extends { load(code: { modules: infer Modules }): unknown; @@ -12,66 +17,103 @@ export type JavaScriptModuleMap = WorkspaceRuntimeLoader extends { const ENTRY_BASENAME = "__workspace_entry__.js"; const RUNNER_MODULE = "workspace-runtime-runner.js"; const CAPABILITIES_MODULE = "workspace-capabilities.js"; -const TRUSTED_MODULES = ["node:fs", "node:fs/promises", "ws:git", "ws:artifacts"] as const; -const TRUSTED_SPECIFIER = /^ws:[A-Za-z0-9][A-Za-z0-9._-]*$/; +// Installed in every execution and backed by the Workspace. No module +// in the `modules` option may use these names. +const BUILT_IN_MODULES = ["node:fs", "node:fs/promises"] as const; +const HOST_SPECIFIER = /^ws:[A-Za-z0-9][A-Za-z0-9._-]*$/; const EXPORT_NAME = /^[A-Za-z_$][A-Za-z0-9_$]*$/; // `default` would turn the function into the default export, and a // `then` export makes the module namespace look like a promise to // `await import(...)`. const RESERVED_EXPORT_NAMES = new Set(["default", "then"]); -/** Specifier of a host trusted module mapped to the function names it exports. */ -export type TrustedModuleExports = ReadonlyMap; +/** Host module specifiers mapped to the function names each one exports. */ +export type HostModuleExports = ReadonlyMap; + +/** The `modules` option split into bundled source and host modules. */ +export interface ParsedModules { + readonly source: Readonly>; + readonly host: ReadonlyMap; +} /** - * Check host trusted modules once, at backend construction, and record - * the named exports each one installs. + * Split the backend's `modules` option into bundled source modules and + * host modules, and check every specifier. * - * @param trustedModules - The modules passed to the backend. - * @returns The export names of each module, keyed by specifier. - * @throws When a specifier or function name is not allowed. The host - * configured the backend wrongly and no execution can use it. + * @param modules - The modules passed to the backend. + * @returns Source modules and host modules, keyed by specifier. + * @throws When a specifier is not allowed. The host configured the + * backend wrongly and no execution can use it. */ -export function parseTrustedModuleExports( - trustedModules: Readonly>, -): TrustedModuleExports { - const parsed = new Map(); - for (const [specifier, module] of Object.entries(trustedModules)) { - if ( - !TRUSTED_SPECIFIER.test(specifier) || - TRUSTED_MODULES.some((reserved) => reserved === specifier) - ) { - throw new Error( - `Trusted module ${JSON.stringify(specifier)} must use a unique simple reserved ws:* name.`, - ); +export function parseModules(modules: Readonly>): ParsedModules { + const source: Record = Object.create(null); + const host = new Map(); + for (const [specifier, module] of Object.entries(modules)) { + if (BUILT_IN_MODULES.some((name) => name === specifier)) { + throw new Error(`Module ${JSON.stringify(specifier)} is built in and cannot be replaced.`); } - const names = Object.keys(module); - if (names.length === 0) { - throw new Error(`Trusted module ${JSON.stringify(specifier)} must export a function.`); - } - for (const name of names) { - if (!EXPORT_NAME.test(name) || RESERVED_EXPORT_NAMES.has(name)) { - throw new Error( - `Trusted module ${JSON.stringify(specifier)} export ${JSON.stringify(name)} must be a JavaScript identifier other than "default" or "then".`, - ); - } - if (typeof module[name] !== "function") { + if (typeof module === "string") { + if (specifier.startsWith("ws:")) { throw new Error( - `Trusted module ${JSON.stringify(specifier)} export ${JSON.stringify(name)} must be a function.`, + `Module ${JSON.stringify(specifier)} uses the ws:* namespace, which is only for host modules.`, ); } + source[specifier] = module; + continue; + } + if (module?.kind !== "host" || typeof module.create !== "function") { + throw new Error( + `Module ${JSON.stringify(specifier)} must be source text or a host module from defineModule().`, + ); + } + if (!HOST_SPECIFIER.test(specifier)) { + throw new Error( + `Host module ${JSON.stringify(specifier)} must use a simple ws:* name, such as "ws:container".`, + ); + } + host.set(specifier, module); + } + return { source, host }; +} + +/** + * Check the functions a host module built and return their export names. + * + * @param specifier - The module's specifier, for error messages. + * @param functions - The functions the module's factory returned. + * @returns The export names, in declaration order. + * @throws When the module exports nothing, a name is not allowed, or a + * value is not a function. + */ +export function parseHostModuleExports( + specifier: string, + functions: WorkspaceModuleFunctions, +): readonly string[] { + const names = Object.keys(functions); + if (names.length === 0) { + throw new Error(`Host module ${JSON.stringify(specifier)} must export a function.`); + } + for (const name of names) { + if (!EXPORT_NAME.test(name) || RESERVED_EXPORT_NAMES.has(name)) { + throw new Error( + `Host module ${JSON.stringify(specifier)} export ${JSON.stringify(name)} must be a JavaScript identifier other than "default" or "then".`, + ); + } + if (typeof functions[name] !== "function") { + throw new Error( + `Host module ${JSON.stringify(specifier)} export ${JSON.stringify(name)} must be a function.`, + ); } - parsed.set(specifier, names); } - return parsed; + return names; } export interface BuildModuleGraphOptions { source: string; cwd: string; capability: WorkspaceRuntimeCapability; - configuredModules: Record; - trustedModules: TrustedModuleExports; + configuredModules: Readonly>; + hostModules: HostModuleExports; maxSourceBytes: number; maxCapabilityBytes: number; maxModules?: number; @@ -91,9 +133,9 @@ export async function buildModuleGraph(options: BuildModuleGraphOptions) { let totalBytes = new TextEncoder().encode(options.source).byteLength; const maxModules = options.maxModules ?? 128; const maxDepth = options.maxDepth ?? 32; - const trustedModuleNames = new Set([ - ...TRUSTED_MODULES, - ...options.trustedModules.keys(), + const importableModuleNames = new Set([ + ...BUILT_IN_MODULES, + ...options.hostModules.keys(), ]); async function visit(path: string, source: string, depth: number): Promise { @@ -107,12 +149,14 @@ export async function buildModuleGraph(options: BuildModuleGraphOptions) { } for (const specifier of imports(source)) { - if (trustedModuleNames.has(specifier)) continue; + if (importableModuleNames.has(specifier)) continue; if (specifier === CAPABILITIES_MODULE) { throw new Error(`Module ${JSON.stringify(specifier)} is reserved for Workspace internals.`); } if (specifier.startsWith("ws:")) { - throw new Error(`Unknown trusted Workspace module ${JSON.stringify(specifier)}.`); + throw new Error( + `Module ${JSON.stringify(specifier)} is not configured. Add it to the backend's modules option.`, + ); } if (specifier.startsWith(".")) { const resolved = resolveRelative(path, specifier); @@ -157,7 +201,7 @@ export async function buildModuleGraph(options: BuildModuleGraphOptions) { for (const specifier of Object.keys(options.configuredModules)) { if ( - trustedModuleNames.has(specifier) || + importableModuleNames.has(specifier) || specifier.startsWith("ws:") || specifier === ENTRY_BASENAME || specifier === RUNNER_MODULE || @@ -178,11 +222,9 @@ export async function buildModuleGraph(options: BuildModuleGraphOptions) { for (const directory of directories) { const prefix = directory ? `${directory}/` : ""; const toCapabilities = relativeModule(directory, CAPABILITIES_MODULE); - modules[`${prefix}ws:git`] = { js: gitModule(toCapabilities) }; - modules[`${prefix}ws:artifacts`] = { js: artifactsModule(toCapabilities) }; - for (const [specifier, names] of options.trustedModules) { + for (const [specifier, names] of options.hostModules) { modules[`${prefix}${specifier}`] = { - js: trustedModule(toCapabilities, specifier, names), + js: hostModule(toCapabilities, specifier, names), }; } for (const [specifier, source] of Object.entries(options.configuredModules)) { @@ -341,18 +383,11 @@ function capabilitiesModule(maxCapabilityBytes: number) { `; } -function proxyModule(capabilitiesImport: string, namespace: string, methods: string[]) { - return ` - import { call } from ${JSON.stringify(capabilitiesImport)}; - ${methods.map((method) => `export const ${method} = (...args) => call(${JSON.stringify(namespace)}, ${JSON.stringify(method)}, args);`).join("\n")} - `; -} - // Exports go through `export { local as name }` rather than // `export const name`, so a reserved word such as `delete` still works // as an export name. -function trustedModule(capabilitiesImport: string, specifier: string, names: readonly string[]) { - const namespace = JSON.stringify(`trusted/${specifier}`); +function hostModule(capabilitiesImport: string, specifier: string, names: readonly string[]) { + const namespace = JSON.stringify(`host/${specifier}`); return ` import { call } from ${JSON.stringify(capabilitiesImport)}; ${names.map((name, index) => `const fn${index} = (...args) => call(${namespace}, ${JSON.stringify(name)}, args);`).join("\n")} @@ -419,17 +454,3 @@ function nodeFsPromisesModule() { function nodeFsModule() { return `${nodeFsPromisesModule()}\nexport { default as promises } from "node:fs/promises";`; } - -function gitModule(capabilitiesImport: string) { - return proxyModule(capabilitiesImport, "git", ["clone", "diff", "status", "log", "cli"]); -} - -function artifactsModule(capabilitiesImport: string) { - return proxyModule(capabilitiesImport, "artifacts", [ - "create", - "get", - "list", - "importArtifact", - "deleteArtifact", - ]); -} diff --git a/packages/computer/src/backends/worker-javascript/worker-javascript.test.ts b/packages/computer/src/backends/worker-javascript/worker-javascript.test.ts index 004243ab..686b011c 100644 --- a/packages/computer/src/backends/worker-javascript/worker-javascript.test.ts +++ b/packages/computer/src/backends/worker-javascript/worker-javascript.test.ts @@ -2,6 +2,7 @@ import { Database, initializeSchema, WorkspaceFilesystem } from "@cloudflare/dof import { SQLiteTestStorage } from "@cloudflare/dofs/testing"; import { describe, expect, it, vi } from "vitest"; +import { defineModule } from "../../runtime/module.js"; import { Workspace } from "../../workspace.js"; import { WorkerJavaScriptBackend } from "./worker-javascript.js"; @@ -244,7 +245,13 @@ describe("WorkerJavaScriptBackend", () => { ); const fs = new WorkspaceFilesystem(db); const backend = new WorkerJavaScriptBackend({ loader: throwingLoader("unused") }); - await backend.connect({ db, fs, git: undefined as never, artifacts: undefined as never }); + await backend.connect({ + db, + fs, + git: undefined as never, + artifacts: undefined as never, + runtime: undefined as never, + }); const columns = db.all<{ name: string }>("PRAGMA table_info(workspace_runtime_executions)"); expect(columns.map((column) => column.name)).toEqual( expect.arrayContaining(["created_at", "finished_at"]), @@ -310,6 +317,7 @@ describe("WorkerJavaScriptBackend", () => { fs, git: undefined as never, artifacts: undefined as never, + runtime: undefined as never, }); const execution = handle.exec({ source: `import task from "./task.js"; export default task;`, @@ -564,6 +572,7 @@ describe("WorkerJavaScriptBackend", () => { fs, git: undefined as never, artifacts: undefined as never, + runtime: undefined as never, }); const execution = await handle.exec({ id: "successful-host-call", source: "export default 1" }); let settled = false; @@ -628,6 +637,7 @@ describe("WorkerJavaScriptBackend", () => { fs, git: undefined as never, artifacts: undefined as never, + runtime: undefined as never, }); const execution = await handle.exec({ id: "live-stream", source: "export default 1" }); const reader = execution.events.getReader(); @@ -682,6 +692,7 @@ describe("WorkerJavaScriptBackend", () => { fs, git: undefined as never, artifacts: undefined as never, + runtime: undefined as never, }); const execution = await handle.exec({ id: "kill-mid-stream", source: "export default 1" }); const reader = execution.events.getReader(); @@ -744,6 +755,7 @@ describe("WorkerJavaScriptBackend", () => { fs, git: undefined as never, artifacts: undefined as never, + runtime: undefined as never, }); const execution = await handle.exec({ id: "no-exit", source: "export default 1" }); const events = []; @@ -754,7 +766,7 @@ describe("WorkerJavaScriptBackend", () => { await handle.close(); }); - it("aborts cooperative trusted-module calls at their deadline", async () => { + it("aborts cooperative host module calls at their deadline", async () => { const db = new Database(new SQLiteTestStorage()); initializeSchema(db, () => 0); const fs = new WorkspaceFilesystem(db); @@ -762,8 +774,8 @@ describe("WorkerJavaScriptBackend", () => { let aborted = false; const backend = new WorkerJavaScriptBackend({ maxHostCallMs: 5, - trustedModules: { - "ws:test": { + modules: { + "ws:test": defineModule({ run(_args, context) { return new Promise((_resolve, reject) => { context.signal.addEventListener("abort", () => { @@ -772,7 +784,7 @@ describe("WorkerJavaScriptBackend", () => { }); }); }, - }, + }), }, loader: { load() { @@ -783,7 +795,7 @@ describe("WorkerJavaScriptBackend", () => { _input: unknown, host: { call(name: string, args: string): Promise }, ) { - await host.call("trusted/ws:test.run", JSON.stringify([])); + await host.call("host/ws:test.run", JSON.stringify([])); }, }; }, @@ -796,6 +808,7 @@ describe("WorkerJavaScriptBackend", () => { fs, git: undefined as never, artifacts: undefined as never, + runtime: undefined as never, }); const execution = await handle.exec({ id: "trusted-timeout", source: "export default 1" }); const events = []; @@ -847,6 +860,7 @@ describe("WorkerJavaScriptBackend", () => { fs, git: undefined as never, artifacts: undefined as never, + runtime: undefined as never, }); const execution = await handle.exec({ id: "cancel-host-call", source: "export default 1" }); await started; @@ -989,6 +1003,7 @@ describe("WorkerJavaScriptBackend", () => { fs, git: undefined as never, artifacts: undefined as never, + runtime: undefined as never, }); await handle.exec({ id: "subscribers", source: "export default 1" }); await handle.getExec({ id: "subscribers", after: "tail" }); @@ -999,44 +1014,96 @@ describe("WorkerJavaScriptBackend", () => { it.each([ [ - "a specifier with a path", - { "ws:bad/path": { run: async () => null } }, - /simple reserved ws:\*/, + "a host module with a path", + { "ws:bad/path": defineModule({ run: async () => null }) }, + /simple ws:\*/, ], - ["a built-in specifier", { "ws:git": { run: async () => null } }, /simple reserved ws:\*/], - ["a module with no functions", { "ws:empty": {} }, /must export a function/], [ - "a non-identifier name", - { "ws:test": { "not-a-name": async () => null } }, - /JavaScript identifier/, + "a host module outside ws:*", + { container: defineModule({ run: async () => null }) }, + /simple ws:\*/, ], - ["a default export", { "ws:test": { default: async () => null } }, /JavaScript identifier/], + ["source under ws:*", { "ws:lib": "export const x = 1;" }, /only for host modules/], + ["a replacement node:fs", { "node:fs": "export default {};" }, /built in/], [ - "a then export", - // biome-ignore lint/suspicious/noThenProperty: The case checks that the backend rejects a `then` export. - { "ws:test": { then: async () => null } }, - /JavaScript identifier/, + "a replacement node:fs host module", + { "node:fs/promises": defineModule({ run: async () => null }) }, + /built in/, ], - ["a non-function export", { "ws:test": { run: "nope" } }, /must be a function/], - ])("rejects trusted modules with %s at construction", (_label, trustedModules, message) => { + ["a plain object of functions", { "ws:test": { run: async () => null } }, /defineModule/], + ])("rejects %s at construction", (_label, modules, message) => { expect( () => new WorkerJavaScriptBackend({ loader: throwingLoader("must not load"), - // SAFETY: Each case hands the constructor a shape the types forbid, to check its runtime guard. - trustedModules: trustedModules as never, + // SAFETY: Each case hands the constructor a shape the types may forbid, to check its runtime guard. + modules: modules as never, }), ).toThrow(message); }); - it("does not dispatch inherited members of a trusted module", async () => { + it.each([ + ["no functions", {}, /must export a function/], + ["a non-identifier name", { "not-a-name": async () => null }, /JavaScript identifier/], + ["a default export", { default: async () => null }, /JavaScript identifier/], + [ + "a then export", + // biome-ignore lint/suspicious/noThenProperty: The case checks that the backend rejects a `then` export. + { then: async () => null }, + /JavaScript identifier/, + ], + ["a non-function export", { run: "nope" }, /must be a function/], + ])("rejects a host module with %s when it connects", async (_label, functions, message) => { + const db = new Database(new SQLiteTestStorage()); + initializeSchema(db, () => 0); + const backend = new WorkerJavaScriptBackend({ + loader: throwingLoader("must not load"), + // SAFETY: Each case builds functions the types may forbid, to check the connect-time guard. + modules: { "ws:test": defineModule(functions as never) }, + }); + await expect( + backend.connect({ + db, + fs: new WorkspaceFilesystem(db), + git: undefined as never, + artifacts: undefined as never, + runtime: undefined as never, + }), + ).rejects.toThrow(message); + }); + + it("builds host modules from the Workspace services when it connects", async () => { + const db = new Database(new SQLiteTestStorage()); + initializeSchema(db, () => 0); + const git = { marker: "git" }; + let seen: unknown; + const backend = new WorkerJavaScriptBackend({ + loader: throwingLoader("must not load"), + modules: { + "ws:test": defineModule((host) => { + seen = host.git; + return { run: async () => null }; + }), + }, + }); + await backend.connect({ + db, + fs: new WorkspaceFilesystem(db), + git: git as never, + artifacts: undefined as never, + runtime: undefined as never, + }); + expect(seen).toBe(git); + }); + + it("does not dispatch inherited members of a host module", async () => { const db = new Database(new SQLiteTestStorage()); initializeSchema(db, () => 0); const fs = new WorkspaceFilesystem(db); await fs.mkdir("/workspace", { recursive: true }); let response = ""; const backend = new WorkerJavaScriptBackend({ - trustedModules: { "ws:test": { run: async () => null } }, + modules: { "ws:test": defineModule({ run: async () => null }) }, loader: { load() { return { @@ -1046,7 +1113,7 @@ describe("WorkerJavaScriptBackend", () => { _input: unknown, host: { call(name: string, args: string): Promise }, ) { - response = await host.call("trusted/ws:test.toString", JSON.stringify([])); + response = await host.call("host/ws:test.toString", JSON.stringify([])); }, }; }, @@ -1059,17 +1126,31 @@ describe("WorkerJavaScriptBackend", () => { fs, git: undefined as never, artifacts: undefined as never, + runtime: undefined as never, }); const execution = await handle.exec({ id: "inherited", source: "export default 1" }); for await (const _event of execution.events) { // Drain the run so the host call settles. } expect(JSON.parse(response)).toMatchObject({ - error: { message: expect.stringContaining("Unknown trusted Workspace module call") }, + error: { message: expect.stringContaining("Unknown Workspace host module call") }, }); await handle.close?.(); }); + it("does not install ws:git or ws:artifacts unless they are configured", async () => { + const workspace = new Workspace({ + storage: new SQLiteTestStorage(), + backends: [new WorkerJavaScriptBackend({ loader: throwingLoader("must not load") })], + }); + await workspace.fs.mkdir("/workspace", { recursive: true }); + for (const specifier of ["ws:git", "ws:artifacts"]) { + await expect( + workspace.runtime.exec(`import * as m from "${specifier}"; export default () => m;`), + ).rejects.toThrow(/is not configured/); + } + }); + it("rejects relative imports that collide with internal Loader modules", async () => { const load = vi.fn(); const workspace = new Workspace({ @@ -1094,7 +1175,6 @@ describe("WorkerJavaScriptBackend", () => { loader: { load }, modules: { "__workspace_entry__.js": "export default 42", - "node:fs": "export default {};", }, }), ], diff --git a/packages/computer/src/backends/worker-javascript/worker-javascript.ts b/packages/computer/src/backends/worker-javascript/worker-javascript.ts index ef9d14f4..31187045 100644 --- a/packages/computer/src/backends/worker-javascript/worker-javascript.ts +++ b/packages/computer/src/backends/worker-javascript/worker-javascript.ts @@ -4,20 +4,23 @@ import { dynamicWorkerEgress, type WorkspaceEgressPolicy } from "../../runtime/e import type { ModuleExecutionEnvelope, ModuleExecutionInput, + WorkspaceModule, WorkspaceModuleBackend, WorkspaceModuleBackendHandle, WorkspaceModuleBackendHost, + WorkspaceModuleFunctions, WorkspaceRuntimeAccess, WorkspaceRuntimeEvent, WorkspaceRuntimeLoader, WorkspaceRuntimeValue, - WorkspaceTrustedModule, } from "../../runtime/types.js"; import { decodeRuntimeFrames, type RuntimeFrame } from "./frames.js"; import { buildModuleGraph, - parseTrustedModuleExports, - type TrustedModuleExports, + type HostModuleExports, + type ParsedModules, + parseHostModuleExports, + parseModules, } from "./module-graph.js"; export interface WorkerJavaScriptBackendOptions { @@ -25,16 +28,25 @@ export interface WorkerJavaScriptBackendOptions { id?: string; root?: string; access?: WorkspaceRuntimeAccess; - modules?: Record; /** - * Host-owned capability modules installed under reserved ws:* specifiers. - * Each function in a module becomes a named export, so - * `{ "ws:container": { exec } }` lets caller source write - * `import { exec } from "ws:container"`. Caller source may import these - * modules, but cannot provide or replace them. The constructor throws - * when a specifier or function name is not allowed. + * Modules caller source can import by specifier. + * + * A string value is JavaScript source bundled into the isolate, such as + * a library build. A host module runs in the Durable Object under a + * `ws:*` specifier, and each of its functions becomes a named export: + * + * ```ts + * modules: { + * "ws:container": createContainerModule(), + * "ws:git": createGitModule(), + * } + * ``` + * + * `node:fs` and `node:fs/promises` are always installed and cannot be + * replaced. The constructor throws when a specifier is not allowed, and + * connecting throws when a host module's export names are not allowed. */ - trustedModules?: Record<`ws:${string}`, WorkspaceTrustedModule>; + modules?: Record; defaultTimeoutMs?: number; maxTimeoutMs?: number; maxSourceBytes?: number; @@ -64,10 +76,6 @@ export interface WorkerJavaScriptBackendOptions { compatibilityFlags?: string[]; egress?: WorkspaceEgressPolicy; globalOutbound?: Fetcher | null; - /** Allow ws:git operations that can perform host-side network requests. */ - allowGitNetwork?: boolean; - /** Allow ws:artifacts imports from caller-selected remote URLs. */ - allowArtifactNetwork?: boolean; } type ResolvedWorkerJavaScriptBackendOptions = Required< @@ -98,9 +106,9 @@ type ResolvedWorkerJavaScriptBackendOptions = Required< | "compatibilityFlags" > > & - Omit & { + Omit & { egress: WorkspaceEgressPolicy; - trustedModuleExports: TrustedModuleExports; + modules: ParsedModules; }; interface WorkspaceExecutionContext { @@ -207,7 +215,7 @@ export class WorkerJavaScriptBackend implements WorkspaceModuleBackend { this.#options = { ...backendOptions, egress: resolvedEgress, - trustedModuleExports: parseTrustedModuleExports(options.trustedModules ?? {}), + modules: parseModules(options.modules ?? {}), root: options.root ?? "/workspace", access: options.access ?? "read-write", defaultTimeoutMs, @@ -242,6 +250,8 @@ export class WorkerJavaScriptBackend implements WorkspaceModuleBackend { class JavaScriptBackendHandle implements WorkspaceModuleBackendHandle { readonly #options: ResolvedWorkerJavaScriptBackendOptions; readonly #host: WorkspaceModuleBackendHost; + readonly #hostModuleFunctions: ReadonlyMap; + readonly #hostModuleExports: HostModuleExports; readonly #records = new Map(); readonly #pendingIds = new Set(); #closed = false; @@ -253,6 +263,19 @@ class JavaScriptBackendHandle implements WorkspaceModuleBackendHandle { constructor(options: ResolvedWorkerJavaScriptBackendOptions, host: WorkspaceModuleBackendHost) { this.#options = options; this.#host = host; + const functions = new Map(); + const exports = new Map(); + for (const [specifier, module] of options.modules.host) { + const built = module.create({ + git: host.git, + artifacts: host.artifacts, + runtime: host.runtime, + }); + exports.set(specifier, parseHostModuleExports(specifier, built)); + functions.set(specifier, built); + } + this.#hostModuleFunctions = functions; + this.#hostModuleExports = exports; host.db.run(` CREATE TABLE IF NOT EXISTS workspace_runtime_executions ( backend TEXT NOT NULL, @@ -363,8 +386,8 @@ class JavaScriptBackendHandle implements WorkspaceModuleBackendHandle { source: input.source, cwd: input.cwd ?? this.#options.root, capability, - configuredModules: this.#options.modules ?? {}, - trustedModules: this.#options.trustedModuleExports, + configuredModules: this.#options.modules.source, + hostModules: this.#hostModuleExports, maxSourceBytes: this.#options.maxSourceBytes, maxCapabilityBytes: this.#options.maxCapabilityBytes, }); @@ -399,11 +422,7 @@ class JavaScriptBackendHandle implements WorkspaceModuleBackendHandle { this.#records.set(id, record); try { const bridge = new WorkspaceRuntimeBridge(capability, { - git: this.#host.git, - artifacts: this.#host.artifacts, - trustedModules: this.#options.trustedModules, - allowGitNetwork: this.#options.allowGitNetwork ?? false, - allowArtifactNetwork: this.#options.allowArtifactNetwork ?? false, + hostModules: this.#hostModuleFunctions, maxPayloadBytes: this.#options.maxCapabilityBytes, maxCallDurationMs: this.#options.maxHostCallMs, maxConcurrentCalls: this.#options.maxConcurrentCapabilityCalls, diff --git a/packages/computer/src/index.ts b/packages/computer/src/index.ts index 87684af7..e2ce4c78 100644 --- a/packages/computer/src/index.ts +++ b/packages/computer/src/index.ts @@ -70,12 +70,19 @@ export { type WorkspaceServiceProxyProps, } from "./proxy.js"; export type { WorkspaceEgressPolicy } from "./runtime/egress.js"; +export { defineModule } from "./runtime/module.js"; export type { ModuleExecutionEnvelope, ModuleExecutionInput, + WorkspaceHostModule, + WorkspaceModule, WorkspaceModuleBackend, WorkspaceModuleBackendHandle, WorkspaceModuleBackendHost, + WorkspaceModuleCallContext, + WorkspaceModuleFunction, + WorkspaceModuleFunctions, + WorkspaceModuleHost, WorkspaceRegisteredBackend, WorkspaceRuntimeAccess, WorkspaceRuntimeDisposeOptions, @@ -88,9 +95,6 @@ export type { WorkspaceRuntimeResult, WorkspaceRuntimeStatus, WorkspaceRuntimeValue, - WorkspaceTrustedCallContext, - WorkspaceTrustedFunction, - WorkspaceTrustedModule, } from "./runtime/types.js"; export { decodeRuntimeEvents, encodeRuntimeEvent } from "./runtime/wire.js"; export { type RawShellValue, type ShellValue, sh, shellQuote } from "./sh.js"; diff --git a/packages/computer/src/modules/artifacts.ts b/packages/computer/src/modules/artifacts.ts new file mode 100644 index 00000000..e7944516 --- /dev/null +++ b/packages/computer/src/modules/artifacts.ts @@ -0,0 +1,89 @@ +// `ws:artifacts`: the Workspace's Artifacts client for isolate JavaScript. +// +// import { create, get, list, importArtifact, deleteArtifact } from "ws:artifacts"; +// +// Calls that change Artifacts need a read-write backend. Importing from +// a caller-chosen URL is denied unless the module is created with +// `allowNetwork: true`: the request runs from the host, so the +// isolate's own egress settings do not stop it. + +import type { ArtifactClient } from "../artifacts/index.js"; +import { defineModule } from "../runtime/module.js"; +import type { + WorkspaceHostModule, + WorkspaceModuleCallContext, + WorkspaceRuntimeValue, +} from "../runtime/types.js"; + +/** Options for {@link createArtifactsModule}. */ +export interface ArtifactsModuleOptions { + /** + * Allow `importArtifact` from a caller-chosen remote URL. Defaults to + * `false`. The request runs from the host, so the JavaScript + * backend's egress settings do not apply to it. + */ + readonly allowNetwork?: boolean; +} + +/** + * Build the `ws:artifacts` host module over the Workspace's Artifacts client. + * + * It exports `create`, `get`, `list`, `importArtifact`, and + * `deleteArtifact`, with the same arguments as the matching + * `ArtifactClient` methods. + * + * @param options - Whether remote imports are allowed. + * @returns The module to pass as `modules["ws:artifacts"]`. + */ +export function createArtifactsModule(options: ArtifactsModuleOptions = {}): WorkspaceHostModule { + const allowNetwork = options.allowNetwork ?? false; + + // SAFETY for the casts below: the isolate's arguments pass through to the Artifacts client, as they did when ws:artifacts was built in. The client checks its own inputs. + return defineModule((host) => ({ + async create([name, createOptions], context) { + requireWrite(context, "Artifacts create"); + return toRuntimeValue( + await host.artifacts.create( + String(name), + createOptions as unknown as Parameters[1], + ), + ); + }, + async get([name]) { + return toRuntimeValue(await host.artifacts.get(String(name))); + }, + async list() { + return toRuntimeValue(await host.artifacts.list()); + }, + async importArtifact([name, source, importOptions], context) { + requireWrite(context, "Artifacts import"); + if (!allowNetwork) { + throw new Error("Artifacts import requires createArtifactsModule({ allowNetwork: true })."); + } + return toRuntimeValue( + await host.artifacts.import( + String(name), + source as unknown as Parameters[1], + importOptions as unknown as Parameters[2], + ), + ); + }, + async deleteArtifact([name], context) { + requireWrite(context, "Artifacts delete"); + return host.artifacts.delete(String(name)); + }, + })); +} + +function requireWrite(context: WorkspaceModuleCallContext, operation: string) { + if (context.access !== "read-write") { + throw new Error(`${operation} requires Workspace write access.`); + } +} + +// Artifacts results are plain data, but may carry `undefined` fields +// that the bridge rejects. A JSON round trip drops them. +function toRuntimeValue(value: unknown): WorkspaceRuntimeValue { + // SAFETY: JSON.parse of a JSON.stringify result is always a JSON value. + return JSON.parse(JSON.stringify(value ?? null)) as WorkspaceRuntimeValue; +} diff --git a/packages/computer/src/backends/container/container-module.test.ts b/packages/computer/src/modules/container.test.ts similarity index 70% rename from packages/computer/src/backends/container/container-module.test.ts rename to packages/computer/src/modules/container.test.ts index d818f222..526b11b5 100644 --- a/packages/computer/src/backends/container/container-module.test.ts +++ b/packages/computer/src/modules/container.test.ts @@ -1,30 +1,38 @@ import { describe, expect, it } from "vitest"; -import type { WorkspaceTrustedCallContext } from "../../runtime/types.js"; -import { - type ContainerModuleExecOptions, - type ContainerModuleRuntime, - createContainerModule, - describeContainerModule, -} from "./container-module.js"; +import type { + WorkspaceModuleCallContext, + WorkspaceModuleFunction, + WorkspaceModuleHost, +} from "../runtime/types.js"; +import { createContainerModule, describeContainerModule } from "./container.js"; + +interface ExecOptions { + readonly backend: string; + readonly encoding: "utf8"; + readonly cwd?: string; + readonly env?: Record; + readonly stdin?: string; + readonly timeoutMs: number; +} interface Run { readonly command: string; - readonly options: ContainerModuleExecOptions; + readonly options: ExecOptions; killed: boolean; } -// An in-memory runtime that records each command and finishes it with -// the given output, or holds it open until it is killed. +// An in-memory Workspace runtime that records each command and finishes +// it with the given output, or holds it open until it is killed. function fakeRuntime(output: { exitCode?: number; stdout?: string; stderr?: string; hang?: boolean; -}): { runtime: ContainerModuleRuntime; runs: Run[] } { +}) { const runs: Run[] = []; - const runtime: ContainerModuleRuntime = { - async exec(command, options) { + const runtime = { + async exec(command: string, options: ExecOptions) { const run: Run = { command, options, killed: false }; runs.push(run); let stop: () => void = () => undefined; @@ -50,10 +58,27 @@ function fakeRuntime(output: { return { runtime, runs }; } -function callContext(overrides: Partial = {}) { +// Build the module's functions the way the backend does when it connects. +function build( + runtime: ReturnType["runtime"], + options?: Parameters[0], +): { readonly exec: WorkspaceModuleFunction } { + // SAFETY: The module only calls runtime.exec, and the fake implements the part of WorkspaceRuntime it uses. + const host = { runtime, git: undefined, artifacts: undefined } as unknown as WorkspaceModuleHost; + const functions = createContainerModule(options).create(host); + const exec = functions.exec; + if (!exec) throw new Error("ws:container must export exec"); + return { exec }; +} + +function callContext( + overrides: Partial = {}, +): WorkspaceModuleCallContext { return { signal: new AbortController().signal, deadline: Date.now() + 60_000, + access: "read-write", + resolvePath: async (path) => path, ...overrides, }; } @@ -61,7 +86,7 @@ function callContext(overrides: Partial = {}) { describe("createContainerModule", () => { it("runs the command on the container backend and returns its output", async () => { const { runtime, runs } = fakeRuntime({ exitCode: 3, stdout: "out", stderr: "err" }); - const container = createContainerModule({ runtime: () => runtime }); + const container = build(runtime); await expect( container.exec( @@ -84,7 +109,7 @@ describe("createContainerModule", () => { it("uses the configured backend id and omits unset options", async () => { const { runtime, runs } = fakeRuntime({}); - const container = createContainerModule({ runtime: () => runtime, backend: "linux" }); + const container = build(runtime, { backend: "linux" }); await container.exec(["ls"], callContext()); expect(Object.keys(runs[0]?.options ?? {}).sort()).toEqual([ @@ -95,24 +120,19 @@ describe("createContainerModule", () => { expect(runs[0]?.options.backend).toBe("linux"); }); - it("resolves the runtime on each call, so it can be built before the Workspace", async () => { - let current: ContainerModuleRuntime | undefined; - const container = createContainerModule({ - runtime: () => { - if (!current) throw new Error("Workspace not constructed yet"); - return current; - }, - }); + it("refuses to run on a read-only backend", async () => { const { runtime, runs } = fakeRuntime({}); - current = runtime; + const container = build(runtime); - await container.exec(["true"], callContext()); - expect(runs).toHaveLength(1); + await expect(container.exec(["ls"], callContext({ access: "read" }))).rejects.toThrow( + /write access/, + ); + expect(runs).toHaveLength(0); }); it("caps the timeout at the time left before the host call deadline", async () => { const { runtime, runs } = fakeRuntime({}); - const container = createContainerModule({ runtime: () => runtime }); + const container = build(runtime); await container.exec( ["sleep 1", { timeoutMs: 600_000 }], @@ -125,7 +145,7 @@ describe("createContainerModule", () => { it("kills the command when the call is aborted", async () => { const { runtime, runs } = fakeRuntime({ hang: true }); - const container = createContainerModule({ runtime: () => runtime }); + const container = build(runtime); const controller = new AbortController(); const pending = container.exec(["sleep 100"], callContext({ signal: controller.signal })); @@ -138,7 +158,7 @@ describe("createContainerModule", () => { it("does not start a command once the call is aborted", async () => { const { runtime, runs } = fakeRuntime({}); - const container = createContainerModule({ runtime: () => runtime }); + const container = build(runtime); const controller = new AbortController(); controller.abort(new Error("cancelled")); @@ -150,7 +170,7 @@ describe("createContainerModule", () => { it("truncates each stream on UTF-8 boundaries", async () => { const { runtime } = fakeRuntime({ stdout: "a🙂b", stderr: "🙂🙂" }); - const container = createContainerModule({ runtime: () => runtime, maxOutputBytes: 5 }); + const container = build(runtime, { maxOutputBytes: 5 }); await expect(container.exec(["echo"], callContext())).resolves.toEqual({ exitCode: 0, @@ -171,7 +191,7 @@ describe("createContainerModule", () => { ["a non-positive timeout", ["ls", { timeoutMs: 0 }], /timeoutMs must be a positive number/], ])("rejects %s without running anything", async (_label, args, message) => { const { runtime, runs } = fakeRuntime({}); - const container = createContainerModule({ runtime: () => runtime }); + const container = build(runtime); // SAFETY: Each case hands exec arguments that isolate code could send; the cast only widens the test table's inferred type. await expect(container.exec(args as never, callContext())).rejects.toThrow(message); @@ -179,10 +199,7 @@ describe("createContainerModule", () => { }); it("rejects a bad maxOutputBytes at construction", () => { - const { runtime } = fakeRuntime({}); - expect(() => createContainerModule({ runtime: () => runtime, maxOutputBytes: 0 })).toThrow( - /maxOutputBytes/, - ); + expect(() => createContainerModule({ maxOutputBytes: 0 })).toThrow(/maxOutputBytes/); }); it("describes the module under its installed specifier", () => { diff --git a/packages/computer/src/backends/container/container-module.ts b/packages/computer/src/modules/container.ts similarity index 72% rename from packages/computer/src/backends/container/container-module.ts rename to packages/computer/src/modules/container.ts index 87fce8c7..b812d34f 100644 --- a/packages/computer/src/backends/container/container-module.ts +++ b/packages/computer/src/modules/container.ts @@ -1,9 +1,9 @@ -// A trusted module that lets isolate JavaScript run shell commands in -// the Workspace's container backend. +// `ws:container`: lets isolate JavaScript run shell commands in the +// Workspace's container backend. // -// Installed on a WorkerJavaScriptBackend as `ws:container`, it turns -// the container into a library the JavaScript backend calls, rather -// than a second backend the model has to choose between: +// Installed on a WorkerJavaScriptBackend, it turns the container into a +// library the JavaScript backend calls, rather than a second backend +// the model has to choose between: // // import { exec } from "ws:container"; // const { exitCode, stdout } = await exec("npm test", { cwd: "/workspace" }); @@ -13,55 +13,19 @@ // pending Workspace writes before the command and pulls the // container's changes after it. +import { defineModule } from "../runtime/module.js"; import type { + WorkspaceHostModule, + WorkspaceModuleCallContext, WorkspaceRuntimeValue, - WorkspaceTrustedCallContext, - WorkspaceTrustedFunction, -} from "../../runtime/types.js"; +} from "../runtime/types.js"; const DEFAULT_BACKEND = "container-shell"; const DEFAULT_MAX_OUTPUT_BYTES = 64 * 1024; const EXEC_OPTION_KEYS = new Set(["cwd", "env", "stdin", "timeoutMs"]); -/** Options the container module passes to `workspace.runtime.exec`. */ -export interface ContainerModuleExecOptions { - /** Backend id the command runs on. */ - readonly backend: string; - /** Output encoding. The module always asks for text. */ - readonly encoding: "utf8"; - /** Working directory inside the container. */ - readonly cwd?: string; - /** Environment variables for this command only. */ - readonly env?: Record; - /** Text piped to the command's standard input. */ - readonly stdin?: string; - /** Wall-clock limit for the command, in milliseconds. */ - readonly timeoutMs: number; -} - -/** The part of a Workspace execution handle the container module uses. */ -export interface ContainerModuleExecHandle { - /** Wait for the command to finish and return its output. */ - result(): Promise<{ exitCode: number; stdout: string; stderr: string }>; - /** Stop the command. */ - kill(): Promise; -} - -/** The part of `workspace.runtime` the container module uses. */ -export interface ContainerModuleRuntime { - /** Start a command on a Workspace backend. */ - exec(command: string, options: ContainerModuleExecOptions): Promise; -} - /** Options for {@link createContainerModule}. */ export interface ContainerModuleOptions { - /** - * Returns the Workspace runtime. It is called on every command - * rather than once, so the module can be built before the - * Workspace that owns both backends: pass - * `() => this.workspace.runtime`. - */ - readonly runtime: () => ContainerModuleRuntime; /** Id of the container backend. Defaults to `"container-shell"`. */ readonly backend?: string; /** @@ -73,45 +37,41 @@ export interface ContainerModuleOptions { readonly maxOutputBytes?: number; } -/** The `ws:container` trusted module. */ -export type ContainerModule = { - /** - * Run a shell command in the container. - * - * Isolate code calls `exec(command, { cwd, env, stdin, timeoutMs })` - * and gets `{ exitCode, stdout, stderr }` back once the command - * finishes. A non-zero exit code is a normal result, not an error. - */ - readonly exec: WorkspaceTrustedFunction; -}; - /** - * Build the `ws:container` trusted module over a Workspace's container + * Build the `ws:container` host module over the Workspace's container * backend. * - * Install it only on a read-write JavaScript backend. A container - * command can write to the Workspace and reach the network, whatever - * the isolate's own access and egress settings are. + * It exports `exec(command, { cwd, env, stdin, timeoutMs })`, which + * returns `{ exitCode, stdout, stderr }` once the command finishes. A + * non-zero exit code is a normal result, not an error. Cancelling the + * execution kills the command. + * + * A container command can write to the Workspace and reach the network, + * so `exec` refuses to run on a read-only backend. Egress settings on + * the JavaScript backend do not apply to the container. * - * @param options - How to reach the Workspace runtime and which backend to use. - * @returns The module to pass as `trustedModules["ws:container"]`. + * @param options - Which backend to use and how much output to return. + * @returns The module to pass as `modules["ws:container"]`. * @throws When `maxOutputBytes` is not a positive integer. The host * configured the module wrongly. */ -export function createContainerModule(options: ContainerModuleOptions): ContainerModule { +export function createContainerModule(options: ContainerModuleOptions = {}): WorkspaceHostModule { const backend = options.backend ?? DEFAULT_BACKEND; const maxOutputBytes = options.maxOutputBytes ?? DEFAULT_MAX_OUTPUT_BYTES; if (!Number.isInteger(maxOutputBytes) || maxOutputBytes <= 0) { throw new Error("createContainerModule: maxOutputBytes must be a positive integer."); } - return { + return defineModule((host) => ({ async exec(args, context) { + if (context.access !== "read-write") { + throw new Error("ws:container exec requires Workspace write access."); + } const request = parseExecArgs(args); const timeoutMs = remainingTime(request.timeoutMs, context); context.signal.throwIfAborted(); - const handle = await options.runtime().exec(request.command, { + const handle = await host.runtime.exec(request.command, { backend, encoding: "utf8", timeoutMs, @@ -135,7 +95,7 @@ export function createContainerModule(options: ContainerModuleOptions): Containe context.signal.removeEventListener("abort", kill); } }, - }; + })); } /** @@ -229,7 +189,7 @@ function optionalTimeout(value: WorkspaceRuntimeValue | undefined) { // The command must finish before the host call deadline, or the // isolate stops waiting while the container keeps working. Cap the // requested timeout at the time left. -function remainingTime(requested: number | undefined, context: WorkspaceTrustedCallContext) { +function remainingTime(requested: number | undefined, context: WorkspaceModuleCallContext) { const remaining = context.deadline - Date.now(); if (remaining <= 0) throw new Error("exec: the host call deadline has already passed."); return requested === undefined ? remaining : Math.min(requested, remaining); diff --git a/packages/computer/src/modules/git.ts b/packages/computer/src/modules/git.ts new file mode 100644 index 00000000..614a1b5b --- /dev/null +++ b/packages/computer/src/modules/git.ts @@ -0,0 +1,138 @@ +// `ws:git`: the Workspace's Git client for isolate JavaScript. +// +// import { status, diff, log, clone, cli } from "ws:git"; +// +// Every path the isolate passes is confined to the JavaScript backend's +// root. Commands that change the repository need a read-write backend, +// and commands that reach the network are denied unless the module is +// created with `allowNetwork: true`: they run from the host, so the +// isolate's own egress settings do not stop them. + +import type { GitClient } from "../git/index.js"; +import { defineModule } from "../runtime/module.js"; +import type { + WorkspaceHostModule, + WorkspaceModuleCallContext, + WorkspaceRuntimeValue, +} from "../runtime/types.js"; + +const NETWORK_COMMANDS = new Set(["clone", "fetch", "pull", "push", "ls-remote", "submodule"]); + +/** Options for {@link createGitModule}. */ +export interface GitModuleOptions { + /** + * Allow `clone` and network `cli` commands such as `fetch` and `push`. + * Defaults to `false`. These requests run from the host, so the + * JavaScript backend's egress settings do not apply to them. + */ + readonly allowNetwork?: boolean; +} + +/** + * Build the `ws:git` host module over the Workspace's Git client. + * + * It exports `clone`, `diff`, `status`, `log`, and `cli`. Each takes the + * same options object as the matching `GitClient` method, with `dir` or + * `cwd` resolved against the backend root. + * + * @param options - Whether network commands are allowed. + * @returns The module to pass as `modules["ws:git"]`. + */ +export function createGitModule(options: GitModuleOptions = {}): WorkspaceHostModule { + const allowNetwork = options.allowNetwork ?? false; + const requireNetwork = (operation: string) => { + if (!allowNetwork) { + throw new Error(`${operation} requires createGitModule({ allowNetwork: true }).`); + } + }; + + return defineModule((host) => ({ + async clone([value], context) { + requireWrite(context, "Git clone"); + requireNetwork("Git clone"); + // SAFETY: The isolate's options object passes through to the Git client, as it did when ws:git was built in. The client checks its own options; only the path is rewritten here. + await host.git.clone( + (await withDir(value, context, true)) as unknown as Parameters[0], + ); + return null; + }, + async diff([value], context) { + // SAFETY: As for clone. + return host.git.diff((await withDir(value, context)) as Parameters[0]); + }, + async status([value], context) { + // SAFETY: As for clone. + const entries = await host.git.status( + (await withDir(value, context)) as Parameters[0], + ); + return toRuntimeValue(entries); + }, + async log([value], context) { + // SAFETY: As for clone. + const commits = await host.git.log( + (await withDir(value, context)) as Parameters[0], + ); + return toRuntimeValue(commits); + }, + async cli([value], context) { + requireWrite(context, "Git CLI"); + // SAFETY: As for clone. + const input = (value ?? {}) as unknown as Parameters[0]; + assertSafeCliArguments(input.argv); + if (input.argv?.some((argument) => NETWORK_COMMANDS.has(argument.toLowerCase()))) { + requireNetwork("Git CLI network command"); + } + const result = await host.git.cli({ + ...input, + cwd: await context.resolvePath(input.cwd ?? ".", { allowMissing: true }), + }); + return toRuntimeValue(result); + }, + })); +} + +function requireWrite(context: WorkspaceModuleCallContext, operation: string) { + if (context.access !== "read-write") { + throw new Error(`${operation} requires Workspace write access.`); + } +} + +async function withDir( + value: WorkspaceRuntimeValue | undefined, + context: WorkspaceModuleCallContext, + allowMissing = false, +): Promise> { + const options = value !== null && typeof value === "object" && !Array.isArray(value) ? value : {}; + return { + ...options, + dir: await context.resolvePath(typeof options.dir === "string" ? options.dir : ".", { + allowMissing, + }), + }; +} + +// Git path overrides would let a command escape the confined directory. +function assertSafeCliArguments(argv: string[] | undefined) { + if ( + argv?.some( + (argument) => + argument === "-C" || + argument.startsWith("-C") || + argument === "--git-dir" || + argument.startsWith("--git-dir=") || + argument === "--work-tree" || + argument.startsWith("--work-tree="), + ) + ) { + throw new Error( + "Git CLI path overrides are not available inside a confined Workspace runtime.", + ); + } +} + +// Git results are plain data, but may carry `undefined` fields that the +// bridge rejects. A JSON round trip drops them. +function toRuntimeValue(value: unknown): WorkspaceRuntimeValue { + // SAFETY: JSON.parse of a JSON.stringify result is always a JSON value. + return JSON.parse(JSON.stringify(value ?? null)) as WorkspaceRuntimeValue; +} diff --git a/packages/computer/src/runtime/bridge.test.ts b/packages/computer/src/runtime/bridge.test.ts index 0f71e5ef..94d16a4f 100644 --- a/packages/computer/src/runtime/bridge.test.ts +++ b/packages/computer/src/runtime/bridge.test.ts @@ -13,13 +13,7 @@ function bridge(limits: { }) { return new WorkspaceRuntimeBridge({} as WorkspaceRuntimeCapability, { ...limits, - trustedModules: { - "ws:test": { - async run() { - return "ok"; - }, - }, - }, + hostModules: new Map([["ws:test", { run: async () => "ok" }]]), }); } @@ -30,9 +24,9 @@ async function message(response: Promise) { describe("WorkspaceRuntimeBridge cumulative limits", () => { it("accepts the configured call count and rejects the next call", async () => { const target = bridge({ maxCalls: 2 }); - await expect(message(target.call("trusted/ws:test.run", args))).resolves.toBeUndefined(); - await expect(message(target.call("trusted/ws:test.run", args))).resolves.toBeUndefined(); - await expect(message(target.call("trusted/ws:test.run", args))).resolves.toContain( + await expect(message(target.call("host/ws:test.run", args))).resolves.toBeUndefined(); + await expect(message(target.call("host/ws:test.run", args))).resolves.toBeUndefined(); + await expect(message(target.call("host/ws:test.run", args))).resolves.toContain( "exceeds 2 capability calls", ); }); @@ -40,20 +34,20 @@ describe("WorkspaceRuntimeBridge cumulative limits", () => { it("accepts requests at the cumulative byte boundary and rejects the next request", async () => { const bytes = encoder.encode(args).byteLength; const target = bridge({ maxTotalRequestBytes: bytes * 2 }); - await expect(message(target.call("trusted/ws:test.run", args))).resolves.toBeUndefined(); - await expect(message(target.call("trusted/ws:test.run", args))).resolves.toBeUndefined(); - await expect(message(target.call("trusted/ws:test.run", args))).resolves.toContain( + await expect(message(target.call("host/ws:test.run", args))).resolves.toBeUndefined(); + await expect(message(target.call("host/ws:test.run", args))).resolves.toBeUndefined(); + await expect(message(target.call("host/ws:test.run", args))).resolves.toContain( `requests exceed ${bytes * 2} bytes`, ); }); it("accepts responses at the cumulative byte boundary and rejects the next response", async () => { - const sample = await bridge({}).call("trusted/ws:test.run", args); + const sample = await bridge({}).call("host/ws:test.run", args); const bytes = encoder.encode(sample).byteLength; const target = bridge({ maxTotalResponseBytes: bytes * 2 }); - await expect(message(target.call("trusted/ws:test.run", args))).resolves.toBeUndefined(); - await expect(message(target.call("trusted/ws:test.run", args))).resolves.toBeUndefined(); - await expect(message(target.call("trusted/ws:test.run", args))).resolves.toContain( + await expect(message(target.call("host/ws:test.run", args))).resolves.toBeUndefined(); + await expect(message(target.call("host/ws:test.run", args))).resolves.toBeUndefined(); + await expect(message(target.call("host/ws:test.run", args))).resolves.toContain( `responses exceed ${bytes * 2} bytes`, ); }); diff --git a/packages/computer/src/runtime/bridge.ts b/packages/computer/src/runtime/bridge.ts index efe1cfa7..e24aec53 100644 --- a/packages/computer/src/runtime/bridge.ts +++ b/packages/computer/src/runtime/bridge.ts @@ -1,17 +1,11 @@ import { RpcTarget } from "cloudflare:workers"; -import type { ArtifactClient } from "../artifacts/index.js"; -import type { GitClient } from "../git/index.js"; import { assertRuntimeValue, type WorkspaceRuntimeCapability } from "./capability.js"; -import type { WorkspaceTrustedCallContext, WorkspaceTrustedModule } from "./types.js"; +import type { WorkspaceModuleCallContext, WorkspaceModuleFunctions } from "./types.js"; export class WorkspaceRuntimeBridge extends RpcTarget { readonly #capability: WorkspaceRuntimeCapability; - readonly #git: GitClient | undefined; - readonly #artifacts: ArtifactClient | undefined; - readonly #trustedModules: Record; - readonly #allowGitNetwork: boolean; - readonly #allowArtifactNetwork: boolean; + readonly #hostModules: ReadonlyMap; readonly #maxPayloadBytes: number; readonly #maxCallDurationMs: number; readonly #maxConcurrentCalls: number; @@ -31,11 +25,7 @@ export class WorkspaceRuntimeBridge extends RpcTarget { constructor( capability: WorkspaceRuntimeCapability, integrations: { - git?: GitClient; - artifacts?: ArtifactClient; - trustedModules?: Record; - allowGitNetwork?: boolean; - allowArtifactNetwork?: boolean; + hostModules?: ReadonlyMap; maxPayloadBytes?: number; maxCallDurationMs?: number; maxConcurrentCalls?: number; @@ -48,11 +38,7 @@ export class WorkspaceRuntimeBridge extends RpcTarget { ) { super(); this.#capability = capability; - this.#git = integrations.git; - this.#artifacts = integrations.artifacts; - this.#trustedModules = integrations.trustedModules ?? {}; - this.#allowGitNetwork = integrations.allowGitNetwork ?? false; - this.#allowArtifactNetwork = integrations.allowArtifactNetwork ?? false; + this.#hostModules = integrations.hostModules ?? new Map(); this.#maxPayloadBytes = integrations.maxPayloadBytes ?? 1024 * 1024; this.#maxCallDurationMs = integrations.maxCallDurationMs ?? 30_000; this.#maxConcurrentCalls = integrations.maxConcurrentCalls ?? 16; @@ -114,10 +100,14 @@ export class WorkspaceRuntimeBridge extends RpcTarget { const operation = encodeCall(async () => { const encodedArgs = JSON.parse(argsJson) as unknown[]; const args = encodedArgs.map(decodeBridgeValue); - if (name.startsWith("git.")) return this.#callGit(name.slice(4), args); - if (name.startsWith("artifacts.")) return this.#callArtifacts(name.slice(10), args); - if (name.startsWith("trusted/")) { - return this.#callTrusted(name, args, { signal: abort.signal, deadline }); + if (name.startsWith("host/")) { + return this.#callHostModule(name, args, { + signal: abort.signal, + deadline, + access: this.#capability.access, + resolvePath: (path, options) => + this.#capability.resolveConfined(path, options?.allowMissing ?? false), + }); } const operation = name.startsWith("fs.") ? name.slice(3) : name; switch (operation) { @@ -221,146 +211,28 @@ export class WorkspaceRuntimeBridge extends RpcTarget { } } - // `name` is `trusted/.`. Function names are + // `name` is `host/.`. Function names are // identifiers and never contain a dot, so the last dot splits them // from a specifier such as `ws:a.b`. Own-property checks keep // isolate code from reaching `toString` or other inherited members. - async #callTrusted(name: string, args: unknown[], context: WorkspaceTrustedCallContext) { - const target = name.slice("trusted/".length); + async #callHostModule(name: string, args: unknown[], context: WorkspaceModuleCallContext) { + const target = name.slice("host/".length); const dot = target.lastIndexOf("."); const specifier = dot === -1 ? "" : target.slice(0, dot); const functionName = dot === -1 ? "" : target.slice(dot + 1); - const trusted = Object.hasOwn(this.#trustedModules, specifier) - ? this.#trustedModules[specifier] - : undefined; + const functions = this.#hostModules.get(specifier); const fn = - trusted !== undefined && Object.hasOwn(trusted, functionName) - ? trusted[functionName] + functions !== undefined && Object.hasOwn(functions, functionName) + ? functions[functionName] : undefined; if (typeof fn !== "function") { - throw new Error(`Unknown trusted Workspace module call ${JSON.stringify(name)}.`); + throw new Error(`Unknown Workspace host module call ${JSON.stringify(name)}.`); } assertBridgeValues(args); const result = await fn(args, context); assertBridgeValues([result]); return result; } - - async #callGit(name: string, args: unknown[]) { - if (!this.#git) throw new Error("Workspace Git is not configured for this execution."); - switch (name) { - case "clone": - this.#requireWrite("Git clone"); - this.#requireGitNetwork("Git clone"); - return this.#git - .clone( - (await this.#gitOptions(args[0], true)) as unknown as Parameters[0], - ) - .then(() => null); - case "diff": - return this.#git.diff( - (await this.#gitOptions(args[0])) as Parameters[0], - ); - case "status": - return this.#git.status( - (await this.#gitOptions(args[0])) as Parameters[0], - ); - case "log": - return this.#git.log((await this.#gitOptions(args[0])) as Parameters[0]); - case "cli": { - this.#requireWrite("Git CLI"); - const input = (args[0] ?? {}) as Parameters[0]; - assertSafeGitCliArguments(input.argv); - if (isGitNetworkCommand(input.argv)) this.#requireGitNetwork("Git CLI network command"); - return this.#git.cli({ - ...input, - cwd: await this.#capability.resolveConfined(input.cwd ?? ".", true), - }); - } - default: - throw new Error(`Unknown Workspace Git operation ${JSON.stringify(name)}.`); - } - } - - #callArtifacts(name: string, args: unknown[]) { - if (!this.#artifacts) - throw new Error("Workspace Artifacts are not configured for this execution."); - switch (name) { - case "create": - this.#requireWrite("Artifacts create"); - return this.#artifacts.create( - String(args[0]), - args[1] as Parameters[1], - ); - case "get": - return this.#artifacts.get(String(args[0])); - case "list": - return this.#artifacts.list(); - case "importArtifact": - this.#requireWrite("Artifacts import"); - if (!this.#allowArtifactNetwork) { - throw new Error( - "Artifacts import requires WorkerJavaScriptBackend allowArtifactNetwork: true.", - ); - } - return this.#artifacts.import( - String(args[0]), - args[1] as Parameters[1], - args[2] as Parameters[2], - ); - case "deleteArtifact": - this.#requireWrite("Artifacts delete"); - return this.#artifacts.delete(String(args[0])); - default: - throw new Error(`Unknown Workspace Artifacts operation ${JSON.stringify(name)}.`); - } - } - - async #gitOptions(value: unknown, allowMissing = false): Promise> { - const options = (value ?? {}) as Record; - return { - ...options, - dir: await this.#capability.resolveConfined( - typeof options.dir === "string" ? options.dir : ".", - allowMissing, - ), - }; - } - - #requireGitNetwork(operation: string) { - if (!this.#allowGitNetwork) { - throw new Error(`${operation} requires WorkerJavaScriptBackend allowGitNetwork: true.`); - } - } - - #requireWrite(operation: string) { - if (this.#capability.access !== "read-write") { - throw new Error(`${operation} requires Workspace write access.`); - } - } -} - -function assertSafeGitCliArguments(argv: string[] | undefined) { - if ( - argv?.some( - (argument) => - argument === "-C" || - argument.startsWith("-C") || - argument === "--git-dir" || - argument.startsWith("--git-dir=") || - argument === "--work-tree" || - argument.startsWith("--work-tree="), - ) - ) { - throw new Error( - "Git CLI path overrides are not available inside a confined Workspace runtime.", - ); - } -} - -function isGitNetworkCommand(argv: string[] | undefined) { - const networkCommands = new Set(["clone", "fetch", "pull", "push", "ls-remote", "submodule"]); - return argv?.some((argument) => networkCommands.has(argument.toLowerCase())) ?? false; } function assertBridgeValues( @@ -375,15 +247,14 @@ function assertBridgeValues( (typeof value === "number" && Number.isFinite(value)) ) return; - if (typeof value !== "object") - throw new Error("Trusted module values must be JSON-compatible."); - if (seen.has(value)) throw new Error("Trusted module values must be acyclic."); + if (typeof value !== "object") throw new Error("Host module values must be JSON-compatible."); + if (seen.has(value)) throw new Error("Host module values must be acyclic."); seen.add(value); if (Array.isArray(value)) for (const item of value) visit(item); else { const prototype = Object.getPrototypeOf(value); if (prototype !== Object.prototype && prototype !== null) { - throw new Error("Trusted module values must contain only plain objects."); + throw new Error("Host module values must contain only plain objects."); } for (const item of Object.values(value as Record)) visit(item); } diff --git a/packages/computer/src/runtime/module.ts b/packages/computer/src/runtime/module.ts new file mode 100644 index 00000000..2230b863 --- /dev/null +++ b/packages/computer/src/runtime/module.ts @@ -0,0 +1,32 @@ +import type { + WorkspaceHostModule, + WorkspaceModuleFunctions, + WorkspaceModuleHost, +} from "./types.js"; + +/** + * Define a host module for `WorkerJavaScriptBackend`'s `modules` option. + * + * Pass the functions directly when they need nothing from the + * Workspace, or pass a factory that builds them from the Workspace's + * Git client, Artifacts client, and runtime. The backend calls the + * factory once when it connects. + * + * ```ts + * modules: { + * "ws:model": defineModule({ async batch(args) { ... } }), + * "ws:repo": defineModule((host) => ({ async log() { return host.git.log(); } })), + * } + * ``` + * + * @param functions - The module's functions, or a factory that builds them. + * @returns A host module. + */ +export function defineModule( + functions: WorkspaceModuleFunctions | ((host: WorkspaceModuleHost) => WorkspaceModuleFunctions), +): WorkspaceHostModule { + return { + kind: "host", + create: typeof functions === "function" ? functions : () => functions, + }; +} diff --git a/packages/computer/src/runtime/types.ts b/packages/computer/src/runtime/types.ts index 40aa6c7a..d2ea3c03 100644 --- a/packages/computer/src/runtime/types.ts +++ b/packages/computer/src/runtime/types.ts @@ -4,36 +4,73 @@ import type { ExecEncoding, ExecSyncResult, KillSignal } from "../shell.js"; export type WorkspaceRuntimeAccess = "read" | "read-write"; -/** Cancellation and timing for one call into a trusted module function. */ -export interface WorkspaceTrustedCallContext { +/** Per-call context the backend passes to every host module function. */ +export interface WorkspaceModuleCallContext { /** Aborts when the call passes its deadline or the execution is cancelled. */ readonly signal: AbortSignal; /** Epoch milliseconds after which the caller stops waiting for this call. */ readonly deadline: number; + /** Access level of the backend running the call. */ + readonly access: WorkspaceRuntimeAccess; + /** + * Resolve a path the isolate passed against the backend's root. + * Rejects paths that escape the root or pass through a symlink. + * + * @param path - An absolute path, or one relative to the backend root. + * @param options - Set `allowMissing` when the path may not exist yet. + * @returns The confined absolute path. + */ + resolvePath(path: string, options?: { readonly allowMissing?: boolean }): Promise; } /** - * One host function exported by a trusted module. + * One host function exported by a host module. * * `args` holds the arguments the isolate passed, decoded from the wire. * They come from untrusted code, so parse them before use. */ -export type WorkspaceTrustedFunction = ( +export type WorkspaceModuleFunction = ( args: readonly WorkspaceRuntimeValue[], - context: WorkspaceTrustedCallContext, + context: WorkspaceModuleCallContext, ) => Promise; +/** Named functions a host module exports into the isolate. */ +export type WorkspaceModuleFunctions = Readonly>; + +/** Workspace services a host module can build its functions from. */ +export interface WorkspaceModuleHost { + /** The Workspace's Git client. Throws on use when Git is not configured. */ + readonly git: import("../git/index.js").GitClient; + /** The Workspace's Artifacts client. Throws on use when Artifacts is not configured. */ + readonly artifacts: import("../artifacts/index.js").ArtifactClient; + /** The Workspace runtime, for running commands on other backends. */ + readonly runtime: import("./runtime.js").WorkspaceRuntime; +} + +/** + * A module whose functions run in the Durable Object rather than in the + * isolate. Build one with `defineModule()`, or use a prebuilt one from + * `@cloudflare/computer/modules/*`. + */ +export interface WorkspaceHostModule { + readonly kind: "host"; + /** + * Build the module's functions. The backend calls this once when it + * connects to its Workspace. + */ + create(host: WorkspaceModuleHost): WorkspaceModuleFunctions; +} + /** - * A host-owned module installed under a reserved `ws:*` specifier. + * A module caller source can import. * - * Each key becomes a named export in the isolate, so - * `{ exec: async (args) => ... }` installed as `ws:container` lets - * code write `import { exec } from "ws:container"`. Keys must be - * JavaScript identifier names other than `default` and `then`. A - * reserved word such as `delete` works, but code must rename it on - * import: `import { delete as remove } from "ws:files"`. + * A string is JavaScript source bundled into the isolate. It is plain + * code with no host access. A host module runs in the Durable Object, + * must use a `ws:*` specifier, and each of its functions becomes a + * named export: `{ "ws:container": createContainerModule() }` lets code + * write `import { exec } from "ws:container"`. */ -export type WorkspaceTrustedModule = Readonly>; +export type WorkspaceModule = string | WorkspaceHostModule; export type WorkspaceRuntimeValue = | null @@ -203,7 +240,11 @@ export interface WorkspaceModuleBackendHandle { close?(): Promise; } -export type WorkspaceModuleBackendHost = import("../backend.js").WorkspaceBackendHost; +/** What a module backend receives when it connects to its Workspace. */ +export type WorkspaceModuleBackendHost = import("../backend.js").WorkspaceBackendHost & { + /** The Workspace runtime, handed to host modules. */ + readonly runtime: import("./runtime.js").WorkspaceRuntime; +}; export interface WorkspaceModuleBackend { readonly protocol: "module"; diff --git a/packages/computer/src/workspace.ts b/packages/computer/src/workspace.ts index 25f797a2..f9d841d5 100644 --- a/packages/computer/src/workspace.ts +++ b/packages/computer/src/workspace.ts @@ -998,6 +998,7 @@ export class Workspace { fs: this.#fs, git: this.#gitFactory ? this.git : DISABLED_GIT_CLIENT, artifacts: this.#artifacts, + runtime: this.runtime, }), ) .then(async (handle) => { diff --git a/packages/computer/tests/script-runner-worker.ts b/packages/computer/tests/script-runner-worker.ts index d6657b6d..4ea05f79 100644 --- a/packages/computer/tests/script-runner-worker.ts +++ b/packages/computer/tests/script-runner-worker.ts @@ -1,7 +1,6 @@ import { DurableObject, RpcTarget, WorkerEntrypoint } from "cloudflare:workers"; import type { ShellRPC, SyncRPC } from "@cloudflare/computer-rpc"; import type { WorkspaceBackend } from "../src/backend.js"; -import { createContainerModule } from "../src/backends/container/index.js"; import { WorkerJavaScriptBackend } from "../src/backends/worker-javascript/index.js"; import { createGitClient } from "../src/git/index.js"; import type { @@ -9,7 +8,10 @@ import type { WorkspaceRuntimeValue, WorkspaceStub, } from "../src/index.js"; -import { Workspace } from "../src/index.js"; +import { defineModule, Workspace } from "../src/index.js"; +import { createArtifactsModule } from "../src/modules/artifacts.js"; +import { createContainerModule } from "../src/modules/container.js"; +import { createGitModule } from "../src/modules/git.js"; export interface Env { HOST: DurableObjectNamespace; @@ -73,12 +75,10 @@ export class HostDO extends DurableObject { maxConcurrentCapabilityCalls: 2, modules: { "math-kit": "export const double = (value) => value * 2;", - }, - trustedModules: { - "ws:container": createContainerModule({ - runtime: () => this.#workspace.runtime, - }), - "ws:test-host": { + "ws:git": createGitModule(), + "ws:artifacts": createArtifactsModule(), + "ws:container": createContainerModule(), + "ws:test-host": defineModule({ async echo(args) { return { args: [...args] }; }, @@ -108,7 +108,7 @@ export class HostDO extends DurableObject { keep: true, }; }, - }, + }), }, }), fakeContainerBackend(), diff --git a/packages/computer/tests/script-runner.test.ts b/packages/computer/tests/script-runner.test.ts index 7cc89196..5fea0805 100644 --- a/packages/computer/tests/script-runner.test.ts +++ b/packages/computer/tests/script-runner.test.ts @@ -49,7 +49,7 @@ describe("WorkspaceRuntime", () => { }); }); - it("executes an ES module with configured and trusted modules", async () => { + it("executes an ES module with source and host modules", async () => { const response = await runtime({ source: ` import { double } from "math-kit"; @@ -234,7 +234,7 @@ describe("WorkspaceRuntime", () => { expect(payload.result.stdout).toContain("stdio truncated"); }); - it("bounds oversized trusted-module error responses", async () => { + it("bounds oversized host module error responses", async () => { const response = await runtime({ source: ` import { largeError } from "ws:test-host"; @@ -278,7 +278,7 @@ describe("WorkspaceRuntime", () => { expect(JSON.parse(text).result.value).toEqual(["fulfilled", "fulfilled", "rejected"]); }); - it("rejects non-plain results from host trusted modules", async () => { + it("rejects non-plain results from host modules", async () => { const response = await runtime({ source: ` import { invalidResult } from "ws:test-host"; @@ -296,7 +296,7 @@ describe("WorkspaceRuntime", () => { }); }); - it("exposes only the functions a trusted module declares", async () => { + it("exposes only the functions a host module declares", async () => { const response = await runtime({ source: ` import * as host from "ws:test-host"; @@ -317,7 +317,7 @@ describe("WorkspaceRuntime", () => { ]); }); - it("fails to link an import the trusted module does not export", async () => { + it("fails to link an import the host module does not export", async () => { const response = await runtime({ source: ` import { call } from "ws:test-host"; @@ -439,7 +439,7 @@ describe("WorkspaceRuntime", () => { }); }); - it("confines trusted Git operations to the backend root", async () => { + it("confines ws:git operations to the backend root", async () => { const response = await runtime({ source: ` import { status } from "ws:git"; @@ -488,7 +488,7 @@ describe("WorkspaceRuntime", () => { expect(JSON.parse(text), text).toMatchObject({ result: { status: "failed", - stderr: expect.stringContaining("allowArtifac"), + stderr: expect.stringContaining("createArtifactsModule"), }, }); }); @@ -506,12 +506,12 @@ describe("WorkspaceRuntime", () => { expect(JSON.parse(text), text).toMatchObject({ result: { status: "failed", - stderr: expect.stringContaining("allowGitNetwork"), + stderr: expect.stringContaining("createGitModule"), }, }); }); - it("rejects trusted Git paths that traverse a symlink", async () => { + it("rejects ws:git paths that traverse a symlink", async () => { await write("/outside/repository/README.md", "outside"); await symlink("/outside/repository", "/workspace/linked-repository"); const response = await runtime({ From 0019f0d1ce29f0c6d918ed291af14f33c0756085 Mon Sep 17 00:00:00 2001 From: Matt Carey Date: Wed, 30 Sep 2026 17:13:50 +0100 Subject: [PATCH 6/6] computer: Pass host modules as plain objects and describe them A host module no longer needs a wrapper. In modules, an object of functions is a host module and a function is a factory that builds one from the Workspace's Git client, Artifacts client, and runtime. The prebuilt modules are factories. Host functions may return any value; the bridge already checks results at runtime, and now treats undefined as null and drops undefined object fields the way JSON does, so wrappers no longer need a JSON round trip to satisfy the types. The JavaScript backend now describes its source language and every importable module, built from the same modules option it runs with. A factory contributes its description property and an object is listed by its export names. Backends expose this as description, the runtime returns it from describe(id), and the exec tool adds it to each backend's entry. The caller's own description becomes optional, so the module list the model reads cannot drift from what is installed, and describeContainerModule() goes away. The exec tool builds one input schema and offers input only when some backend accepts it. Source module name checks move from every execution to construction, and the exec tool and container module share one UTF-8 truncation helper. --- .changeset/unified-modules.md | 8 +- docs/09_tool_interface.md | 22 +- docs/17_isolate_javascript.md | 54 ++-- examples/rlm/README.md | 2 +- examples/rlm/worker/rlm-agent.ts | 3 +- packages/computer/src/backend.ts | 5 + .../worker-javascript/module-graph.ts | 87 +++--- .../worker-javascript.test.ts | 116 ++++---- .../worker-javascript/worker-javascript.ts | 36 +-- packages/computer/src/index.ts | 3 +- packages/computer/src/modules/artifacts.ts | 56 ++-- .../computer/src/modules/container.test.ts | 9 +- packages/computer/src/modules/container.ts | 62 ++-- packages/computer/src/modules/git.ts | 37 +-- packages/computer/src/runtime/bridge.ts | 11 +- packages/computer/src/runtime/module.ts | 32 --- packages/computer/src/runtime/runtime.test.ts | 6 +- packages/computer/src/runtime/runtime.ts | 13 +- packages/computer/src/runtime/types.ts | 45 +-- packages/computer/src/text-truncation.ts | 37 +++ packages/computer/src/tools/ai.test.ts | 75 ++++- packages/computer/src/tools/exec.ts | 271 +++++++----------- packages/computer/src/workspace.ts | 20 +- .../computer/tests/script-runner-worker.ts | 6 +- 24 files changed, 525 insertions(+), 491 deletions(-) delete mode 100644 packages/computer/src/runtime/module.ts create mode 100644 packages/computer/src/text-truncation.ts diff --git a/.changeset/unified-modules.md b/.changeset/unified-modules.md index 6fa7fb41..42f091ec 100644 --- a/.changeset/unified-modules.md +++ b/.changeset/unified-modules.md @@ -2,8 +2,10 @@ "@cloudflare/computer": minor --- -`WorkerJavaScriptBackend` takes a single `modules` option. A string value is bundled source, as before. A host module runs in the Durable Object under a `ws:*` specifier, and each of its functions becomes a named export, so `modules: { "ws:container": createContainerModule() }` lets code write `import { exec } from "ws:container"`. Build your own with `defineModule({ fn })`, or `defineModule((host) => ({ fn }))` to use the Workspace's Git client, Artifacts client, or runtime. Each function receives `(args, { signal, deadline, access, resolvePath })`. +`WorkerJavaScriptBackend` takes a single `modules` option. A string is bundled source, as before. An object of functions is a host module that runs in the Durable Object under a `ws:*` specifier, and each function becomes a named export: `modules: { "ws:weather": { forecast } }` lets code write `import { forecast } from "ws:weather"`. A factory, `(host) => ({ ... })`, builds a host module from the Workspace's Git client, Artifacts client, or runtime. Each function receives `(args, { signal, deadline, access, resolvePath })` and may return any JSON-compatible value. -`ws:git` and `ws:artifacts` are no longer installed automatically. Add `createGitModule()` from `@cloudflare/computer/modules/git` and `createArtifactsModule()` from `@cloudflare/computer/modules/artifacts`. `createContainerModule()` from `@cloudflare/computer/modules/container` runs shell commands in the Workspace's container backend. The container shares the Workspace's files, a canceled execution kills the command, and it refuses to run on a read-only backend. `describeContainerModule()` returns text for the model that explains it. `node:fs` and `node:fs/promises` stay built in. +`ws:git` and `ws:artifacts` are no longer installed automatically. Add `createGitModule()` from `@cloudflare/computer/modules/git` and `createArtifactsModule()` from `@cloudflare/computer/modules/artifacts`. `createContainerModule()` from `@cloudflare/computer/modules/container` runs shell commands in the Workspace's container backend. The container shares the Workspace's files, a canceled execution kills the command, and it refuses to run on a read-only backend. `node:fs` and `node:fs/promises` stay built in. -To migrate, move `trustedModules` entries into `modules` and wrap each in `defineModule()`, replacing any `call(method, args)` handler with one function per method. Replace `allowGitNetwork: true` with `createGitModule({ allowNetwork: true })` and `allowArtifactNetwork: true` with `createArtifactsModule({ allowNetwork: true })`. +The backend now describes its source language and every module for a model, and the `exec` tool shows that text. A backend's `description` in the tool options becomes optional when the backend describes itself, so the module list the model reads always matches what is installed. The tool also stops offering `input` when no configured backend accepts it. + +To migrate, move `trustedModules` entries into `modules`, replacing any `call(method, args)` handler with one function per method. Replace `allowGitNetwork: true` with `createGitModule({ allowNetwork: true })` and `allowArtifactNetwork: true` with `createArtifactsModule({ allowNetwork: true })`. diff --git a/docs/09_tool_interface.md b/docs/09_tool_interface.md index 47ccfb84..181a2ba7 100644 --- a/docs/09_tool_interface.md +++ b/docs/09_tool_interface.md @@ -60,11 +60,7 @@ Pass `shell` only when the Workspace has matching backend ids. With one backend, ```ts const tools = createAITools({ workspace, - shell: { - backends: { - "worker-javascript": { description: "Isolated JavaScript with the durable workspace filesystem." }, - }, - }, + shell: { backends: { "worker-javascript": {} } }, }); ``` @@ -257,15 +253,17 @@ The tool uses forced removal, so deleting a missing path succeeds. Set `recursiv ## `exec` -`exec` is opt-in. It calls `workspace.runtime.exec` with the configured backend and streams bounded output. Backend descriptions are included in the model-facing tool description, so describe capabilities and startup cost in plain language. +`exec` is opt-in. It calls `workspace.runtime.exec` with the configured backend and streams bounded output. -The tool's arguments depend on how many backends you pass: +Each backend's entry in the tool description joins two parts: the `description` you pass, and what the backend says about itself (`backend.description`, read through `workspace.runtime.describe(id)`). `WorkerJavaScriptBackend` describes its source language and every module code can import, so `{ "worker-javascript": {} }` is enough and the list stays in step with `modules`. A backend that does not describe itself needs a `description`. Describe capabilities and startup cost in plain language. -| Backends | Arguments | Description | -| --- | --- | --- | -| One shell backend | `command`, `cwd`, `env` | Describes a shell command. | -| One callable backend | `command`, `cwd`, `env`, `input` | Describes `command` as ES module source and the return value as `result`. | -| More than one | `command`, `cwd`, `backend`, `env`, `input` | Lists every backend and the default. `defaultBackend` is required. | +The tool offers only the arguments that can work: + +| Backends | Arguments | +| --- | --- | +| One shell backend | `command`, `cwd`, `env` | +| One callable backend | `command`, `cwd`, `env`, `input` | +| More than one | `command`, `cwd`, `backend`, `env`, plus `input` when any is callable. `defaultBackend` is required. | A `backend` value the model sends anyway is dropped when only one backend is configured. The output still names the backend that ran. diff --git a/docs/17_isolate_javascript.md b/docs/17_isolate_javascript.md index 680761eb..93555d4a 100644 --- a/docs/17_isolate_javascript.md +++ b/docs/17_isolate_javascript.md @@ -127,10 +127,9 @@ Caller source can import three kinds of module, and all of them are fixed when t | --- | --- | --- | --- | | Built in | Always installed | The isolate, backed by the Workspace | `node:fs`, `node:fs/promises` | | Source | `modules: { name: "source" }` | The isolate | a bundled library | -| Host | `modules: { "ws:name": hostModule }` | The Durable Object | `ws:git`, `ws:container`, your own | +| Host | `modules: { "ws:name": { fn } }`, or a factory | The Durable Object | `ws:git`, `ws:container`, your own | ```ts -import { defineModule } from "@cloudflare/computer"; import { createArtifactsModule } from "@cloudflare/computer/modules/artifacts"; import { createContainerModule } from "@cloudflare/computer/modules/container"; import { createGitModule } from "@cloudflare/computer/modules/git"; @@ -142,13 +141,31 @@ new WorkerJavaScriptBackend({ "ws:git": createGitModule(), "ws:artifacts": createArtifactsModule(), "ws:container": createContainerModule(), - "ws:model": defineModule({ async batch(args, context) { /* ... */ } }), + "ws:weather": { + forecast: ([city]) => lookUpForecast(String(city)), + }, }, }); ``` An import that is not built in, configured, or a relative Workspace path fails before the Worker is created. Caller source and durable files cannot shadow a configured or built-in module. +The backend describes its modules for a model in `backend.description`, and the `exec` tool shows that text, so the list the model reads always matches what is installed: + +```text +`command` is ECMAScript module source, run in an isolated JavaScript runtime. Relative imports resolve from `cwd` in the workspace. +Code has no direct network access. + +Modules code can import: +- `node:fs/promises` (also `node:fs`): the workspace's files. ... +- `tar-stream`: a bundled library. +- `ws:git`: The workspace's Git repository tools: `status({ dir })`, ... +- `ws:container`: Runs shell commands in a full Linux container that shares this workspace's files. ... +- `ws:weather`: exports `forecast`. +``` + +A factory adds its own text through a `description` property, as the prebuilt modules do. An object of functions is listed by its export names; say more about it in the `exec` tool's backend description if the model needs it. + ### Built-in filesystem Filesystem access uses the familiar asynchronous Node API, but is backed by the durable Workspace rather than an isolate-local filesystem. Both forms are installed automatically: @@ -173,17 +190,26 @@ A string value is JavaScript source installed as a bare import, such as a bundle A host module runs in the Durable Object, and each of its functions becomes a named export in the isolate. Host modules must use a simple `ws:*` specifier. Nothing under `ws:` is installed unless you configure it. -Build your own with `defineModule()`. Pass the functions directly, or pass a factory that builds them from the Workspace's Git client, Artifacts client, and runtime. The backend calls the factory once when it connects to its Workspace: +Pass an object of functions: + +```ts +modules: { + "ws:weather": { + forecast: ([city]) => lookUpForecast(String(city)), + }, +} +``` + +When the functions need the Workspace's Git client, Artifacts client, or runtime, pass a factory instead. The backend calls it once when it connects to its Workspace. This is how the prebuilt modules work: ```ts modules: { - "ws:repo": defineModule((host) => ({ + "ws:repo": (host) => ({ async recent(args, context) { const dir = await context.resolvePath(String(args[0] ?? ".")); - const commits = await host.git.log({ dir, depth: 5 }); - return commits.map((commit) => ({ oid: commit.oid, message: commit.message })); + return host.git.log({ dir, depth: 5 }); }, - })), + }), } ``` @@ -201,9 +227,9 @@ Each function receives the arguments the isolate passed, as an array of JSON-com | `access` | The backend's `"read"` or `"read-write"` access. Check it before any write. | | `resolvePath(path, { allowMissing })` | Confines a caller path to the backend root and rejects symlinks. | -The arguments come from caller code, so parse them before use. The return value must be JSON-compatible and fits within the same capability byte limits as every other host call. A function that ignores `signal` and never settles keeps the execution in its finalizing state. +The arguments come from caller code, so parse them before use. A function may return a value or a promise. The result must be JSON-compatible, and the bridge checks it at runtime: `undefined` becomes `null` and `undefined` object fields are dropped, as with `JSON.stringify`. It fits within the same capability byte limits as every other host call. A function that ignores `signal` and never settles keeps the execution in its finalizing state. -Specifiers are checked at construction, and export names when the backend connects. A module must export at least one function, and every export name must be a JavaScript identifier name other than `default` or `then`. A reserved word such as `delete` is allowed, and caller code renames it on import: `import { delete as remove } from "ws:files"`. Importing a name the module does not export fails when the module graph links, before any code runs. +Specifiers and the export names of an object are checked at construction. A factory's export names are checked when the backend connects and the factory runs. A module must export at least one function, and every export name must be a JavaScript identifier name other than `default` or `then`. A reserved word such as `delete` is allowed, and caller code renames it on import: `import { delete as remove } from "ws:files"`. Importing a name the module does not export fails when the module graph links, before any code runs. ### `ws:git` @@ -240,13 +266,7 @@ this.workspace = new Workspace({ const tools = createAITools({ workspace: this.workspace, - shell: { - backends: { - "worker-javascript": { - description: `Isolated JavaScript with the durable workspace filesystem. ${describeContainerModule()}`, - }, - }, - }, + shell: { backends: { "worker-javascript": {} } }, }); ``` diff --git a/examples/rlm/README.md b/examples/rlm/README.md index 8a012998..899533f3 100644 --- a/examples/rlm/README.md +++ b/examples/rlm/README.md @@ -67,7 +67,7 @@ const backend = new WorkerJavaScriptBackend({ access: "read", egress: { mode: "none" }, modules: { - "ws:model": defineModule(modelCapability), + "ws:model": modelCapability, }, }); diff --git a/examples/rlm/worker/rlm-agent.ts b/examples/rlm/worker/rlm-agent.ts index 071d4b9b..bd34039e 100644 --- a/examples/rlm/worker/rlm-agent.ts +++ b/examples/rlm/worker/rlm-agent.ts @@ -1,7 +1,6 @@ import { AIChatAgent, type OnChatMessageOptions } from "@cloudflare/ai-chat"; import { type DurableObjectStorageLike, - defineModule, Workspace, type WorkspaceRuntimeLoader, } from "@cloudflare/computer"; @@ -108,7 +107,7 @@ export class RlmAgent extends AIChatAgent { root: WORKSPACE_ROOT, access: "read", egress: { mode: "none" }, - modules: { "ws:model": defineModule(modelCapability) }, + modules: { "ws:model": modelCapability }, maxConcurrentExecutions: 1, maxConcurrentCapabilityCalls: 4, // One manifest read + 24 chunk reads + one bounded ws:model batch. diff --git a/packages/computer/src/backend.ts b/packages/computer/src/backend.ts index dac310eb..fd6c3fd9 100644 --- a/packages/computer/src/backend.ts +++ b/packages/computer/src/backend.ts @@ -60,6 +60,11 @@ export interface WorkspaceBackend { // too. Defaults to false when omitted. readonly callable?: boolean; + // What the backend tells a model about itself, such as the language + // it runs and what that code can use. The exec tool shows it next to + // the caller's own description. Omit when there is nothing to add. + readonly description?: string; + // Materialise a connection. Called lazily on first use, once // per backend per workspace lifetime. The Workspace caches the // resulting handle by `id`; subsequent exec / push / pull diff --git a/packages/computer/src/backends/worker-javascript/module-graph.ts b/packages/computer/src/backends/worker-javascript/module-graph.ts index 704c488f..66cdcae3 100644 --- a/packages/computer/src/backends/worker-javascript/module-graph.ts +++ b/packages/computer/src/backends/worker-javascript/module-graph.ts @@ -2,8 +2,8 @@ import { parse } from "acorn"; import type { WorkspaceRuntimeCapability } from "../../runtime/capability.js"; import type { - WorkspaceHostModule, WorkspaceModule, + WorkspaceModuleFactory, WorkspaceModuleFunctions, WorkspaceRuntimeLoader, } from "../../runtime/types.js"; @@ -27,29 +27,35 @@ const EXPORT_NAME = /^[A-Za-z_$][A-Za-z0-9_$]*$/; // `await import(...)`. const RESERVED_EXPORT_NAMES = new Set(["default", "then"]); -/** Host module specifiers mapped to the function names each one exports. */ -export type HostModuleExports = ReadonlyMap; - -/** The `modules` option split into bundled source and host modules. */ +/** The `modules` option, parsed once when the backend is constructed. */ export interface ParsedModules { readonly source: Readonly>; - readonly host: ReadonlyMap; + readonly host: ReadonlyMap; + /** One markdown bullet per importable module, for a model. */ + readonly description: string; } +const FILESYSTEM_DESCRIPTION = + "- `node:fs/promises` (also `node:fs`): the workspace's files. `readFile`, `writeFile`, `mkdir`, `rm`, `readdir`, `stat`, `lstat`, `readlink`, `symlink`, `chmod`, and `access`. Async only."; + /** - * Split the backend's `modules` option into bundled source modules and - * host modules, and check every specifier. + * Parse the backend's `modules` option: split it into bundled source + * and host module factories, check every specifier and every object's + * export names, and describe each module for a model. A factory's + * export names are checked when the backend connects and it runs. * * @param modules - The modules passed to the backend. - * @returns Source modules and host modules, keyed by specifier. - * @throws When a specifier is not allowed. The host configured the - * backend wrongly and no execution can use it. + * @returns Source modules, host module factories, and their description. + * @throws When a specifier or an object's export names are not allowed. + * The host configured the backend wrongly and no execution can use it. */ export function parseModules(modules: Readonly>): ParsedModules { const source: Record = Object.create(null); - const host = new Map(); + const host = new Map(); + const lines = [FILESYSTEM_DESCRIPTION]; for (const [specifier, module] of Object.entries(modules)) { - if (BUILT_IN_MODULES.some((name) => name === specifier)) { + const name = `\`${specifier}\``; + if (BUILT_IN_MODULES.some((builtIn) => builtIn === specifier)) { throw new Error(`Module ${JSON.stringify(specifier)} is built in and cannot be replaced.`); } if (typeof module === "string") { @@ -58,37 +64,48 @@ export function parseModules(modules: Readonly>) `Module ${JSON.stringify(specifier)} uses the ws:* namespace, which is only for host modules.`, ); } + if (specifier.includes("/") || isInternalModuleName(specifier)) { + throw new Error(`Module ${JSON.stringify(specifier)} uses a reserved module name.`); + } source[specifier] = module; + lines.push(`- ${name}: a bundled library.`); continue; } - if (module?.kind !== "host" || typeof module.create !== "function") { + if (!HOST_SPECIFIER.test(specifier)) { throw new Error( - `Module ${JSON.stringify(specifier)} must be source text or a host module from defineModule().`, + `Host module ${JSON.stringify(specifier)} must use a simple ws:* name, such as "ws:container".`, ); } - if (!HOST_SPECIFIER.test(specifier)) { + if (typeof module === "function") { + host.set(specifier, module); + lines.push(`- ${name}: ${module.description ?? "a host module."}`); + continue; + } + if (module === null || typeof module !== "object" || Array.isArray(module)) { throw new Error( - `Host module ${JSON.stringify(specifier)} must use a simple ws:* name, such as "ws:container".`, + `Module ${JSON.stringify(specifier)} must be source text, an object of functions, or a factory.`, ); } - host.set(specifier, module); + assertHostModuleExports(specifier, module); + host.set(specifier, () => module); + const exports = Object.keys(module).map((key) => `\`${key}\``); + lines.push(`- ${name}: exports ${exports.join(", ")}.`); } - return { source, host }; + return { source, host, description: lines.join("\n") }; } /** - * Check the functions a host module built and return their export names. + * Check the functions a host module exports. * * @param specifier - The module's specifier, for error messages. - * @param functions - The functions the module's factory returned. - * @returns The export names, in declaration order. + * @param functions - The module's functions. * @throws When the module exports nothing, a name is not allowed, or a * value is not a function. */ -export function parseHostModuleExports( +export function assertHostModuleExports( specifier: string, functions: WorkspaceModuleFunctions, -): readonly string[] { +): void { const names = Object.keys(functions); if (names.length === 0) { throw new Error(`Host module ${JSON.stringify(specifier)} must export a function.`); @@ -105,7 +122,6 @@ export function parseHostModuleExports( ); } } - return names; } export interface BuildModuleGraphOptions { @@ -113,7 +129,7 @@ export interface BuildModuleGraphOptions { cwd: string; capability: WorkspaceRuntimeCapability; configuredModules: Readonly>; - hostModules: HostModuleExports; + hostModules: ReadonlyMap; maxSourceBytes: number; maxCapabilityBytes: number; maxModules?: number; @@ -199,21 +215,6 @@ export async function buildModuleGraph(options: BuildModuleGraphOptions) { await visit(entryPath, options.source, 0); - for (const specifier of Object.keys(options.configuredModules)) { - if ( - importableModuleNames.has(specifier) || - specifier.startsWith("ws:") || - specifier === ENTRY_BASENAME || - specifier === RUNNER_MODULE || - specifier === CAPABILITIES_MODULE || - specifier.includes("/") - ) { - throw new Error( - `Configured module ${JSON.stringify(specifier)} uses a reserved module name.`, - ); - } - } - // node:* specifiers use protocol-style resolution and therefore need exact // module-map keys rather than the importer-directory aliases used by ws:*. modules["node:fs/promises"] = { js: nodeFsPromisesModule() }; @@ -222,9 +223,9 @@ export async function buildModuleGraph(options: BuildModuleGraphOptions) { for (const directory of directories) { const prefix = directory ? `${directory}/` : ""; const toCapabilities = relativeModule(directory, CAPABILITIES_MODULE); - for (const [specifier, names] of options.hostModules) { + for (const [specifier, functions] of options.hostModules) { modules[`${prefix}${specifier}`] = { - js: hostModule(toCapabilities, specifier, names), + js: hostModule(toCapabilities, specifier, Object.keys(functions)), }; } for (const [specifier, source] of Object.entries(options.configuredModules)) { diff --git a/packages/computer/src/backends/worker-javascript/worker-javascript.test.ts b/packages/computer/src/backends/worker-javascript/worker-javascript.test.ts index 686b011c..e75b00fb 100644 --- a/packages/computer/src/backends/worker-javascript/worker-javascript.test.ts +++ b/packages/computer/src/backends/worker-javascript/worker-javascript.test.ts @@ -2,7 +2,6 @@ import { Database, initializeSchema, WorkspaceFilesystem } from "@cloudflare/dof import { SQLiteTestStorage } from "@cloudflare/dofs/testing"; import { describe, expect, it, vi } from "vitest"; -import { defineModule } from "../../runtime/module.js"; import { Workspace } from "../../workspace.js"; import { WorkerJavaScriptBackend } from "./worker-javascript.js"; @@ -775,7 +774,7 @@ describe("WorkerJavaScriptBackend", () => { const backend = new WorkerJavaScriptBackend({ maxHostCallMs: 5, modules: { - "ws:test": defineModule({ + "ws:test": { run(_args, context) { return new Promise((_resolve, reject) => { context.signal.addEventListener("abort", () => { @@ -784,7 +783,7 @@ describe("WorkerJavaScriptBackend", () => { }); }); }, - }), + }, }, loader: { load() { @@ -1013,24 +1012,34 @@ describe("WorkerJavaScriptBackend", () => { }); it.each([ - [ - "a host module with a path", - { "ws:bad/path": defineModule({ run: async () => null }) }, - /simple ws:\*/, - ], - [ - "a host module outside ws:*", - { container: defineModule({ run: async () => null }) }, - /simple ws:\*/, - ], + ["a host module with a path", { "ws:bad/path": { run: async () => null } }, /simple ws:\*/], + ["a host module outside ws:*", { container: { run: async () => null } }, /simple ws:\*/], ["source under ws:*", { "ws:lib": "export const x = 1;" }, /only for host modules/], ["a replacement node:fs", { "node:fs": "export default {};" }, /built in/], [ "a replacement node:fs host module", - { "node:fs/promises": defineModule({ run: async () => null }) }, + { "node:fs/promises": { run: async () => null } }, /built in/, ], - ["a plain object of functions", { "ws:test": { run: async () => null } }, /defineModule/], + ["a number", { "ws:test": 42 }, /source text, an object of functions, or a factory/], + ["an object with no functions", { "ws:test": {} }, /must export a function/], + [ + "an object with a non-identifier name", + { "ws:test": { "not-a-name": async () => null } }, + /JavaScript identifier/, + ], + [ + "an object with a default export", + { "ws:test": { default: async () => null } }, + /JavaScript identifier/, + ], + [ + "an object with a then export", + // biome-ignore lint/suspicious/noThenProperty: The case checks that the backend rejects a `then` export. + { "ws:test": { then: async () => null } }, + /JavaScript identifier/, + ], + ["an object with a non-function export", { "ws:test": { run: "nope" } }, /must be a function/], ])("rejects %s at construction", (_label, modules, message) => { expect( () => @@ -1042,24 +1051,12 @@ describe("WorkerJavaScriptBackend", () => { ).toThrow(message); }); - it.each([ - ["no functions", {}, /must export a function/], - ["a non-identifier name", { "not-a-name": async () => null }, /JavaScript identifier/], - ["a default export", { default: async () => null }, /JavaScript identifier/], - [ - "a then export", - // biome-ignore lint/suspicious/noThenProperty: The case checks that the backend rejects a `then` export. - { then: async () => null }, - /JavaScript identifier/, - ], - ["a non-function export", { run: "nope" }, /must be a function/], - ])("rejects a host module with %s when it connects", async (_label, functions, message) => { + it("rejects a factory whose functions are not allowed when it connects", async () => { const db = new Database(new SQLiteTestStorage()); initializeSchema(db, () => 0); const backend = new WorkerJavaScriptBackend({ loader: throwingLoader("must not load"), - // SAFETY: Each case builds functions the types may forbid, to check the connect-time guard. - modules: { "ws:test": defineModule(functions as never) }, + modules: { "ws:test": () => ({ "not-a-name": async () => null }) }, }); await expect( backend.connect({ @@ -1069,7 +1066,29 @@ describe("WorkerJavaScriptBackend", () => { artifacts: undefined as never, runtime: undefined as never, }), - ).rejects.toThrow(message); + ).rejects.toThrow(/JavaScript identifier/); + }); + + it("describes its modules for a model", () => { + const backend = new WorkerJavaScriptBackend({ + loader: throwingLoader("must not load"), + access: "read", + modules: { + lib: "export const x = 1;", + "ws:weather": { forecast: () => null, alerts: () => null }, + "ws:described": Object.assign(() => ({ run: () => null }), { + description: "Does a thing.", + }), + "ws:plain": () => ({ run: () => null }), + }, + }); + + expect(backend.description).toContain("ECMAScript module source"); + expect(backend.description).toContain("read-only"); + expect(backend.description).toContain("- `lib`: a bundled library."); + expect(backend.description).toContain("- `ws:weather`: exports `forecast`, `alerts`."); + expect(backend.description).toContain("- `ws:described`: Does a thing."); + expect(backend.description).toContain("- `ws:plain`: a host module."); }); it("builds host modules from the Workspace services when it connects", async () => { @@ -1080,10 +1099,10 @@ describe("WorkerJavaScriptBackend", () => { const backend = new WorkerJavaScriptBackend({ loader: throwingLoader("must not load"), modules: { - "ws:test": defineModule((host) => { + "ws:test": (host) => { seen = host.git; return { run: async () => null }; - }), + }, }, }); await backend.connect({ @@ -1103,7 +1122,7 @@ describe("WorkerJavaScriptBackend", () => { await fs.mkdir("/workspace", { recursive: true }); let response = ""; const backend = new WorkerJavaScriptBackend({ - modules: { "ws:test": defineModule({ run: async () => null }) }, + modules: { "ws:test": { run: async () => null } }, loader: { load() { return { @@ -1166,23 +1185,16 @@ describe("WorkerJavaScriptBackend", () => { expect(load).not.toHaveBeenCalled(); }); - it("rejects configured module names that collide with generated modules", async () => { - const load = vi.fn(); - const workspace = new Workspace({ - storage: new SQLiteTestStorage(), - backends: [ - new WorkerJavaScriptBackend({ - loader: { load }, - modules: { - "__workspace_entry__.js": "export default 42", - }, - }), - ], - }); - await workspace.fs.mkdir("/workspace", { recursive: true }); - await expect( - workspace.runtime.exec("export default 1", { backend: "worker-javascript" }), - ).rejects.toThrow(/reserved module name/); - expect(load).not.toHaveBeenCalled(); - }); + it.each(["__workspace_entry__.js", "workspace-capabilities.js", "nested/lib"])( + "rejects a source module named %s at construction", + (specifier) => { + expect( + () => + new WorkerJavaScriptBackend({ + loader: throwingLoader("must not load"), + modules: { [specifier]: "export default 42" }, + }), + ).toThrow(/reserved module name/); + }, + ); }); diff --git a/packages/computer/src/backends/worker-javascript/worker-javascript.ts b/packages/computer/src/backends/worker-javascript/worker-javascript.ts index 31187045..5f211e6e 100644 --- a/packages/computer/src/backends/worker-javascript/worker-javascript.ts +++ b/packages/computer/src/backends/worker-javascript/worker-javascript.ts @@ -16,10 +16,9 @@ import type { } from "../../runtime/types.js"; import { decodeRuntimeFrames, type RuntimeFrame } from "./frames.js"; import { + assertHostModuleExports, buildModuleGraph, - type HostModuleExports, type ParsedModules, - parseHostModuleExports, parseModules, } from "./module-graph.js"; @@ -32,13 +31,15 @@ export interface WorkerJavaScriptBackendOptions { * Modules caller source can import by specifier. * * A string value is JavaScript source bundled into the isolate, such as - * a library build. A host module runs in the Durable Object under a - * `ws:*` specifier, and each of its functions becomes a named export: + * a library build. An object of functions, or a factory that builds + * one, is a host module: it runs in the Durable Object under a `ws:*` + * specifier, and each function becomes a named export. * * ```ts * modules: { + * "tar-stream": TAR_STREAM_BUNDLE, * "ws:container": createContainerModule(), - * "ws:git": createGitModule(), + * "ws:weather": { forecast: ([city]) => lookUpForecast(String(city)) }, * } * ``` * @@ -156,6 +157,8 @@ export class WorkerJavaScriptBackend implements WorkspaceModuleBackend { readonly protocol = "module" as const; readonly type = "worker-javascript"; readonly callable = true; + /** What this backend tells a model: the source language and every importable module. */ + readonly description: string; readonly id: string; readonly #options: ResolvedWorkerJavaScriptBackendOptions; @@ -240,6 +243,14 @@ export class WorkerJavaScriptBackend implements WorkspaceModuleBackend { compatibilityDate, compatibilityFlags: options.compatibilityFlags ?? ["nodejs_compat"], }; + this.description = [ + "`command` is ECMAScript module source, run in an isolated JavaScript runtime. Relative imports resolve from `cwd` in the workspace.", + ...(resolvedEgress.mode === "none" ? ["Code has no direct network access."] : []), + ...(this.#options.access === "read" ? ["The workspace is read-only here."] : []), + "", + "Modules code can import:", + this.#options.modules.description, + ].join("\n"); } async connect(host: WorkspaceModuleBackendHost): Promise { @@ -251,7 +262,6 @@ class JavaScriptBackendHandle implements WorkspaceModuleBackendHandle { readonly #options: ResolvedWorkerJavaScriptBackendOptions; readonly #host: WorkspaceModuleBackendHost; readonly #hostModuleFunctions: ReadonlyMap; - readonly #hostModuleExports: HostModuleExports; readonly #records = new Map(); readonly #pendingIds = new Set(); #closed = false; @@ -264,18 +274,12 @@ class JavaScriptBackendHandle implements WorkspaceModuleBackendHandle { this.#options = options; this.#host = host; const functions = new Map(); - const exports = new Map(); - for (const [specifier, module] of options.modules.host) { - const built = module.create({ - git: host.git, - artifacts: host.artifacts, - runtime: host.runtime, - }); - exports.set(specifier, parseHostModuleExports(specifier, built)); + for (const [specifier, factory] of options.modules.host) { + const built = factory({ git: host.git, artifacts: host.artifacts, runtime: host.runtime }); + assertHostModuleExports(specifier, built); functions.set(specifier, built); } this.#hostModuleFunctions = functions; - this.#hostModuleExports = exports; host.db.run(` CREATE TABLE IF NOT EXISTS workspace_runtime_executions ( backend TEXT NOT NULL, @@ -387,7 +391,7 @@ class JavaScriptBackendHandle implements WorkspaceModuleBackendHandle { cwd: input.cwd ?? this.#options.root, capability, configuredModules: this.#options.modules.source, - hostModules: this.#hostModuleExports, + hostModules: this.#hostModuleFunctions, maxSourceBytes: this.#options.maxSourceBytes, maxCapabilityBytes: this.#options.maxCapabilityBytes, }); diff --git a/packages/computer/src/index.ts b/packages/computer/src/index.ts index e2ce4c78..b98ab49a 100644 --- a/packages/computer/src/index.ts +++ b/packages/computer/src/index.ts @@ -70,16 +70,15 @@ export { type WorkspaceServiceProxyProps, } from "./proxy.js"; export type { WorkspaceEgressPolicy } from "./runtime/egress.js"; -export { defineModule } from "./runtime/module.js"; export type { ModuleExecutionEnvelope, ModuleExecutionInput, - WorkspaceHostModule, WorkspaceModule, WorkspaceModuleBackend, WorkspaceModuleBackendHandle, WorkspaceModuleBackendHost, WorkspaceModuleCallContext, + WorkspaceModuleFactory, WorkspaceModuleFunction, WorkspaceModuleFunctions, WorkspaceModuleHost, diff --git a/packages/computer/src/modules/artifacts.ts b/packages/computer/src/modules/artifacts.ts index e7944516..e577c5ef 100644 --- a/packages/computer/src/modules/artifacts.ts +++ b/packages/computer/src/modules/artifacts.ts @@ -8,11 +8,11 @@ // isolate's own egress settings do not stop it. import type { ArtifactClient } from "../artifacts/index.js"; -import { defineModule } from "../runtime/module.js"; import type { - WorkspaceHostModule, WorkspaceModuleCallContext, - WorkspaceRuntimeValue, + WorkspaceModuleFactory, + WorkspaceModuleFunctions, + WorkspaceModuleHost, } from "../runtime/types.js"; /** Options for {@link createArtifactsModule}. */ @@ -35,44 +35,45 @@ export interface ArtifactsModuleOptions { * @param options - Whether remote imports are allowed. * @returns The module to pass as `modules["ws:artifacts"]`. */ -export function createArtifactsModule(options: ArtifactsModuleOptions = {}): WorkspaceHostModule { +export function createArtifactsModule( + options: ArtifactsModuleOptions = {}, +): WorkspaceModuleFactory { const allowNetwork = options.allowNetwork ?? false; // SAFETY for the casts below: the isolate's arguments pass through to the Artifacts client, as they did when ws:artifacts was built in. The client checks its own inputs. - return defineModule((host) => ({ - async create([name, createOptions], context) { + const create = (host: WorkspaceModuleHost): WorkspaceModuleFunctions => ({ + create([name, createOptions], context) { requireWrite(context, "Artifacts create"); - return toRuntimeValue( - await host.artifacts.create( - String(name), - createOptions as unknown as Parameters[1], - ), + return host.artifacts.create( + String(name), + createOptions as unknown as Parameters[1], ); }, - async get([name]) { - return toRuntimeValue(await host.artifacts.get(String(name))); + get([name]) { + return host.artifacts.get(String(name)); }, - async list() { - return toRuntimeValue(await host.artifacts.list()); + list() { + return host.artifacts.list(); }, - async importArtifact([name, source, importOptions], context) { + importArtifact([name, source, importOptions], context) { requireWrite(context, "Artifacts import"); if (!allowNetwork) { throw new Error("Artifacts import requires createArtifactsModule({ allowNetwork: true })."); } - return toRuntimeValue( - await host.artifacts.import( - String(name), - source as unknown as Parameters[1], - importOptions as unknown as Parameters[2], - ), + return host.artifacts.import( + String(name), + source as unknown as Parameters[1], + importOptions as unknown as Parameters[2], ); }, - async deleteArtifact([name], context) { + deleteArtifact([name], context) { requireWrite(context, "Artifacts delete"); return host.artifacts.delete(String(name)); }, - })); + }); + return Object.assign(create, { + description: `Git repositories stored in Cloudflare Artifacts: \`create(name)\`, \`get(name)\`, \`list()\`, \`importArtifact(name, source)\`, and \`deleteArtifact(name)\`.${allowNetwork ? "" : " Importing from a remote URL is not allowed."}`, + }); } function requireWrite(context: WorkspaceModuleCallContext, operation: string) { @@ -80,10 +81,3 @@ function requireWrite(context: WorkspaceModuleCallContext, operation: string) { throw new Error(`${operation} requires Workspace write access.`); } } - -// Artifacts results are plain data, but may carry `undefined` fields -// that the bridge rejects. A JSON round trip drops them. -function toRuntimeValue(value: unknown): WorkspaceRuntimeValue { - // SAFETY: JSON.parse of a JSON.stringify result is always a JSON value. - return JSON.parse(JSON.stringify(value ?? null)) as WorkspaceRuntimeValue; -} diff --git a/packages/computer/src/modules/container.test.ts b/packages/computer/src/modules/container.test.ts index 526b11b5..c6fdd2a4 100644 --- a/packages/computer/src/modules/container.test.ts +++ b/packages/computer/src/modules/container.test.ts @@ -5,7 +5,7 @@ import type { WorkspaceModuleFunction, WorkspaceModuleHost, } from "../runtime/types.js"; -import { createContainerModule, describeContainerModule } from "./container.js"; +import { createContainerModule } from "./container.js"; interface ExecOptions { readonly backend: string; @@ -65,7 +65,7 @@ function build( ): { readonly exec: WorkspaceModuleFunction } { // SAFETY: The module only calls runtime.exec, and the fake implements the part of WorkspaceRuntime it uses. const host = { runtime, git: undefined, artifacts: undefined } as unknown as WorkspaceModuleHost; - const functions = createContainerModule(options).create(host); + const functions = createContainerModule(options)(host); const exec = functions.exec; if (!exec) throw new Error("ws:container must export exec"); return { exec }; @@ -202,8 +202,7 @@ describe("createContainerModule", () => { expect(() => createContainerModule({ maxOutputBytes: 0 })).toThrow(/maxOutputBytes/); }); - it("describes the module under its installed specifier", () => { - expect(describeContainerModule("ws:linux")).toContain('import { exec } from "ws:linux"'); - expect(describeContainerModule()).toContain('"ws:container"'); + it("describes itself for a model", () => { + expect(createContainerModule().description).toContain("full Linux container"); }); }); diff --git a/packages/computer/src/modules/container.ts b/packages/computer/src/modules/container.ts index b812d34f..06720b7a 100644 --- a/packages/computer/src/modules/container.ts +++ b/packages/computer/src/modules/container.ts @@ -13,12 +13,14 @@ // pending Workspace writes before the command and pulls the // container's changes after it. -import { defineModule } from "../runtime/module.js"; import type { - WorkspaceHostModule, WorkspaceModuleCallContext, + WorkspaceModuleFactory, + WorkspaceModuleFunctions, + WorkspaceModuleHost, WorkspaceRuntimeValue, } from "../runtime/types.js"; +import { truncateText } from "../text-truncation.js"; const DEFAULT_BACKEND = "container-shell"; const DEFAULT_MAX_OUTPUT_BYTES = 64 * 1024; @@ -51,18 +53,21 @@ export interface ContainerModuleOptions { * the JavaScript backend do not apply to the container. * * @param options - Which backend to use and how much output to return. - * @returns The module to pass as `modules["ws:container"]`. + * @returns The module to pass as `modules["ws:container"]`. Its + * `description` tells the model how to use it. * @throws When `maxOutputBytes` is not a positive integer. The host * configured the module wrongly. */ -export function createContainerModule(options: ContainerModuleOptions = {}): WorkspaceHostModule { +export function createContainerModule( + options: ContainerModuleOptions = {}, +): WorkspaceModuleFactory { const backend = options.backend ?? DEFAULT_BACKEND; const maxOutputBytes = options.maxOutputBytes ?? DEFAULT_MAX_OUTPUT_BYTES; if (!Number.isInteger(maxOutputBytes) || maxOutputBytes <= 0) { throw new Error("createContainerModule: maxOutputBytes must be a positive integer."); } - return defineModule((host) => ({ + const create = (host: WorkspaceModuleHost): WorkspaceModuleFunctions => ({ async exec(args, context) { if (context.access !== "read-write") { throw new Error("ws:container exec requires Workspace write access."); @@ -88,34 +93,23 @@ export function createContainerModule(options: ContainerModuleOptions = {}): Wor const result = await handle.result(); return { exitCode: result.exitCode, - stdout: truncate(result.stdout, maxOutputBytes), - stderr: truncate(result.stderr, maxOutputBytes), + stdout: truncateText(result.stdout, maxOutputBytes), + stderr: truncateText(result.stderr, maxOutputBytes), }; } finally { context.signal.removeEventListener("abort", kill); } }, - })); + }); + return Object.assign(create, { description: DESCRIPTION }); } -/** - * Describe `ws:container` for a model. - * - * Append the returned text to the JavaScript backend's description in - * the `exec` tool, so the model knows the module exists and when to - * reach for it. - * - * @param specifier - The specifier the module is installed under. - * @returns A short plain-text description with a usage example. - */ -export function describeContainerModule(specifier = "ws:container"): string { - return [ - `\`import { exec } from ${JSON.stringify(specifier)}\` runs a shell command in a full Linux container that shares this workspace's files.`, - "Use it for npm, node, python, package managers, native binaries, and network access. The container can take a while to start on first use.", - 'Call it as `const { exitCode, stdout, stderr } = await exec("npm test", { cwd: "/workspace" })`. Options are `cwd`, `env`, `stdin`, and `timeoutMs`.', - "Output comes back when the command finishes, and long output is truncated. A non-zero `exitCode` is returned, not thrown.", - ].join(" "); -} +const DESCRIPTION = [ + "Runs shell commands in a full Linux container that shares this workspace's files.", + "Use it for npm, node, python, package managers, native binaries, and network access. The container can take a while to start on first use.", + 'Call `const { exitCode, stdout, stderr } = await exec("npm test", { cwd: "/workspace" })`. Options are `cwd`, `env`, `stdin`, and `timeoutMs`.', + "Output comes back when the command finishes, and long output is truncated. A non-zero `exitCode` is returned, not thrown.", +].join(" "); interface ExecRequest { readonly command: string; @@ -194,19 +188,3 @@ function remainingTime(requested: number | undefined, context: WorkspaceModuleCa if (remaining <= 0) throw new Error("exec: the host call deadline has already passed."); return requested === undefined ? remaining : Math.min(requested, remaining); } - -const encoder = new TextEncoder(); - -function truncate(value: string, maxBytes: number): string { - const totalBytes = encoder.encode(value).byteLength; - if (totalBytes <= maxBytes) return value; - let usedBytes = 0; - let endOffset = 0; - for (const char of value) { - const charBytes = encoder.encode(char).byteLength; - if (usedBytes + charBytes > maxBytes) break; - usedBytes += charBytes; - endOffset += char.length; - } - return `${value.slice(0, endOffset)}\n\n[truncated, ${totalBytes - usedBytes} more bytes]`; -} diff --git a/packages/computer/src/modules/git.ts b/packages/computer/src/modules/git.ts index 614a1b5b..837ce108 100644 --- a/packages/computer/src/modules/git.ts +++ b/packages/computer/src/modules/git.ts @@ -9,10 +9,11 @@ // isolate's own egress settings do not stop them. import type { GitClient } from "../git/index.js"; -import { defineModule } from "../runtime/module.js"; import type { - WorkspaceHostModule, WorkspaceModuleCallContext, + WorkspaceModuleFactory, + WorkspaceModuleFunctions, + WorkspaceModuleHost, WorkspaceRuntimeValue, } from "../runtime/types.js"; @@ -38,7 +39,7 @@ export interface GitModuleOptions { * @param options - Whether network commands are allowed. * @returns The module to pass as `modules["ws:git"]`. */ -export function createGitModule(options: GitModuleOptions = {}): WorkspaceHostModule { +export function createGitModule(options: GitModuleOptions = {}): WorkspaceModuleFactory { const allowNetwork = options.allowNetwork ?? false; const requireNetwork = (operation: string) => { if (!allowNetwork) { @@ -46,15 +47,14 @@ export function createGitModule(options: GitModuleOptions = {}): WorkspaceHostMo } }; - return defineModule((host) => ({ + const create = (host: WorkspaceModuleHost): WorkspaceModuleFunctions => ({ async clone([value], context) { requireWrite(context, "Git clone"); requireNetwork("Git clone"); // SAFETY: The isolate's options object passes through to the Git client, as it did when ws:git was built in. The client checks its own options; only the path is rewritten here. - await host.git.clone( + return host.git.clone( (await withDir(value, context, true)) as unknown as Parameters[0], ); - return null; }, async diff([value], context) { // SAFETY: As for clone. @@ -62,17 +62,11 @@ export function createGitModule(options: GitModuleOptions = {}): WorkspaceHostMo }, async status([value], context) { // SAFETY: As for clone. - const entries = await host.git.status( - (await withDir(value, context)) as Parameters[0], - ); - return toRuntimeValue(entries); + return host.git.status((await withDir(value, context)) as Parameters[0]); }, async log([value], context) { // SAFETY: As for clone. - const commits = await host.git.log( - (await withDir(value, context)) as Parameters[0], - ); - return toRuntimeValue(commits); + return host.git.log((await withDir(value, context)) as Parameters[0]); }, async cli([value], context) { requireWrite(context, "Git CLI"); @@ -82,13 +76,15 @@ export function createGitModule(options: GitModuleOptions = {}): WorkspaceHostMo if (input.argv?.some((argument) => NETWORK_COMMANDS.has(argument.toLowerCase()))) { requireNetwork("Git CLI network command"); } - const result = await host.git.cli({ + return host.git.cli({ ...input, cwd: await context.resolvePath(input.cwd ?? ".", { allowMissing: true }), }); - return toRuntimeValue(result); }, - })); + }); + return Object.assign(create, { + description: `The workspace's Git repository tools: \`status({ dir })\`, \`diff({ dir })\`, \`log({ dir, depth })\`, \`clone({ url, dir })\`, and \`cli({ argv, cwd })\` for any other git subcommand.${allowNetwork ? "" : " Network commands such as clone, fetch, and push are not allowed."}`, + }); } function requireWrite(context: WorkspaceModuleCallContext, operation: string) { @@ -129,10 +125,3 @@ function assertSafeCliArguments(argv: string[] | undefined) { ); } } - -// Git results are plain data, but may carry `undefined` fields that the -// bridge rejects. A JSON round trip drops them. -function toRuntimeValue(value: unknown): WorkspaceRuntimeValue { - // SAFETY: JSON.parse of a JSON.stringify result is always a JSON value. - return JSON.parse(JSON.stringify(value ?? null)) as WorkspaceRuntimeValue; -} diff --git a/packages/computer/src/runtime/bridge.ts b/packages/computer/src/runtime/bridge.ts index e24aec53..cfe4a6e8 100644 --- a/packages/computer/src/runtime/bridge.ts +++ b/packages/computer/src/runtime/bridge.ts @@ -229,7 +229,7 @@ export class WorkspaceRuntimeBridge extends RpcTarget { throw new Error(`Unknown Workspace host module call ${JSON.stringify(name)}.`); } assertBridgeValues(args); - const result = await fn(args, context); + const result = (await fn(args, context)) ?? null; assertBridgeValues([result]); return result; } @@ -256,7 +256,10 @@ function assertBridgeValues( if (prototype !== Object.prototype && prototype !== null) { throw new Error("Host module values must contain only plain objects."); } - for (const item of Object.values(value as Record)) visit(item); + // An undefined field is absent, as in JSON. encodeBridgeValue drops it. + for (const item of Object.values(value as Record)) { + if (item !== undefined) visit(item); + } } seen.delete(value); }; @@ -275,7 +278,9 @@ function encodeBridgeValue(value: unknown): unknown { if (Array.isArray(value)) return wrap("array", { items: value.map(encodeBridgeValue) }); if (value && typeof value === "object") { return wrap("object", { - entries: Object.entries(value).map(([key, child]) => [key, encodeBridgeValue(child)]), + entries: Object.entries(value) + .filter(([, child]) => child !== undefined) + .map(([key, child]) => [key, encodeBridgeValue(child)]), }); } return value; diff --git a/packages/computer/src/runtime/module.ts b/packages/computer/src/runtime/module.ts deleted file mode 100644 index 2230b863..00000000 --- a/packages/computer/src/runtime/module.ts +++ /dev/null @@ -1,32 +0,0 @@ -import type { - WorkspaceHostModule, - WorkspaceModuleFunctions, - WorkspaceModuleHost, -} from "./types.js"; - -/** - * Define a host module for `WorkerJavaScriptBackend`'s `modules` option. - * - * Pass the functions directly when they need nothing from the - * Workspace, or pass a factory that builds them from the Workspace's - * Git client, Artifacts client, and runtime. The backend calls the - * factory once when it connects. - * - * ```ts - * modules: { - * "ws:model": defineModule({ async batch(args) { ... } }), - * "ws:repo": defineModule((host) => ({ async log() { return host.git.log(); } })), - * } - * ``` - * - * @param functions - The module's functions, or a factory that builds them. - * @returns A host module. - */ -export function defineModule( - functions: WorkspaceModuleFunctions | ((host: WorkspaceModuleHost) => WorkspaceModuleFunctions), -): WorkspaceHostModule { - return { - kind: "host", - create: typeof functions === "function" ? functions : () => functions, - }; -} diff --git a/packages/computer/src/runtime/runtime.test.ts b/packages/computer/src/runtime/runtime.test.ts index f91ca460..7cdb3a23 100644 --- a/packages/computer/src/runtime/runtime.test.ts +++ b/packages/computer/src/runtime/runtime.test.ts @@ -52,7 +52,7 @@ function replayBackend(events: WorkspaceRuntimeEvent[]): WorkspaceModuleBackendH function runtimeFor(handle: WorkspaceModuleBackendHandle): WorkspaceRuntime { return new WorkspaceRuntime({ - callableBackendIds: new Set(), + backends: new Map(), backendHandle: async () => handle, resolveBackendId: () => "backend", }); @@ -181,7 +181,7 @@ describe("WorkspaceRuntime utf8 encoding", () => { describe("WorkspaceRuntime callable gate", () => { it("rejects structured input for a non-callable backend", async () => { const runtime = new WorkspaceRuntime({ - callableBackendIds: new Set(), + backends: new Map(), backendHandle: async () => moduleHandleStub(), resolveBackendId: () => "worker-shell", }); @@ -194,7 +194,7 @@ describe("WorkspaceRuntime callable gate", () => { it("accepts structured input for a callable module backend", async () => { const handle = moduleHandleStub(); const runtime = new WorkspaceRuntime({ - callableBackendIds: new Set(["worker-javascript"]), + backends: new Map([["worker-javascript", { callable: true }]]), backendHandle: async () => handle, resolveBackendId: () => "worker-javascript", }); diff --git a/packages/computer/src/runtime/runtime.ts b/packages/computer/src/runtime/runtime.ts index ec43d5df..f74965f8 100644 --- a/packages/computer/src/runtime/runtime.ts +++ b/packages/computer/src/runtime/runtime.ts @@ -14,7 +14,8 @@ import type { } from "./types.js"; interface WorkspaceRuntimeRouterOptions { - callableBackendIds: ReadonlySet; + // What each registered backend says about itself. + backends: ReadonlyMap; backendHandle: (id: string) => Promise; resolveBackendId: (id: string | undefined) => string; } @@ -39,7 +40,15 @@ export class WorkspaceRuntime { // this to know whether a backend is callable without the caller // having to declare it a second time. isCallable(id: string): boolean { - return this.#options.callableBackendIds.has(id); + return this.#options.backends.get(id)?.callable === true; + } + + // What the named backend says about itself for a model: its source + // language and, for the JavaScript backend, the modules code can + // import. The exec tool adds it to the backend's entry so a caller + // does not have to repeat it. + describe(id: string): string | undefined { + return this.#options.backends.get(id)?.description; } exec(source: string): Promise>; diff --git a/packages/computer/src/runtime/types.ts b/packages/computer/src/runtime/types.ts index d2ea3c03..ae6e4934 100644 --- a/packages/computer/src/runtime/types.ts +++ b/packages/computer/src/runtime/types.ts @@ -27,17 +27,21 @@ export interface WorkspaceModuleCallContext { * One host function exported by a host module. * * `args` holds the arguments the isolate passed, decoded from the wire. - * They come from untrusted code, so parse them before use. + * They come from untrusted code, so parse them before use. The function + * may return a value or a promise of one. The result must be + * JSON-compatible: the bridge checks it at runtime, treats `undefined` + * as `null`, and drops `undefined` object fields, the way + * `JSON.stringify` does. */ export type WorkspaceModuleFunction = ( args: readonly WorkspaceRuntimeValue[], context: WorkspaceModuleCallContext, -) => Promise; +) => unknown; /** Named functions a host module exports into the isolate. */ export type WorkspaceModuleFunctions = Readonly>; -/** Workspace services a host module can build its functions from. */ +/** Workspace services a host module factory can build its functions from. */ export interface WorkspaceModuleHost { /** The Workspace's Git client. Throws on use when Git is not configured. */ readonly git: import("../git/index.js").GitClient; @@ -48,29 +52,34 @@ export interface WorkspaceModuleHost { } /** - * A module whose functions run in the Durable Object rather than in the - * isolate. Build one with `defineModule()`, or use a prebuilt one from - * `@cloudflare/computer/modules/*`. + * Builds a host module's functions from the Workspace's services. The + * backend calls it once when it connects to its Workspace. The + * prebuilt modules in `@cloudflare/computer/modules/*` are factories. */ -export interface WorkspaceHostModule { - readonly kind: "host"; +export interface WorkspaceModuleFactory { + (host: WorkspaceModuleHost): WorkspaceModuleFunctions; /** - * Build the module's functions. The backend calls this once when it - * connects to its Workspace. + * What the module does and how to call it, for a model. The + * JavaScript backend adds it to its own description, which the exec + * tool shows. Objects of functions are listed by their export names. */ - create(host: WorkspaceModuleHost): WorkspaceModuleFunctions; + readonly description?: string; } /** * A module caller source can import. * - * A string is JavaScript source bundled into the isolate. It is plain - * code with no host access. A host module runs in the Durable Object, - * must use a `ws:*` specifier, and each of its functions becomes a - * named export: `{ "ws:container": createContainerModule() }` lets code - * write `import { exec } from "ws:container"`. + * - A string is JavaScript source bundled into the isolate, with no host access. + * - An object of functions is a host module. Its functions run in the + * Durable Object and each becomes a named export: + * `{ "ws:weather": { forecast } }` lets code write + * `import { forecast } from "ws:weather"`. + * - A factory is a host module that needs the Workspace's Git client, + * Artifacts client, or runtime, such as `createContainerModule()`. + * + * Host modules must use a `ws:*` specifier. */ -export type WorkspaceModule = string | WorkspaceHostModule; +export type WorkspaceModule = string | WorkspaceModuleFunctions | WorkspaceModuleFactory; export type WorkspaceRuntimeValue = | null @@ -251,6 +260,8 @@ export interface WorkspaceModuleBackend { readonly id: string; readonly type: string; readonly callable?: boolean; + /** What the backend tells a model about itself. Shown by the exec tool. */ + readonly description?: string; connect(host: WorkspaceModuleBackendHost): Promise; } diff --git a/packages/computer/src/text-truncation.ts b/packages/computer/src/text-truncation.ts new file mode 100644 index 00000000..8100d812 --- /dev/null +++ b/packages/computer/src/text-truncation.ts @@ -0,0 +1,37 @@ +const encoder = new TextEncoder(); + +/** + * Cut text to at most `maxBytes` UTF-8 bytes without splitting a + * character, and say how much was left out. + * + * @param value - The text to cut. + * @param maxBytes - The largest number of UTF-8 bytes to keep. + * @returns The text unchanged when it fits, or its longest whole-character + * prefix followed by a `[truncated, N more bytes]` marker. + */ +export function truncateText(value: string, maxBytes: number): string { + const totalBytes = encoder.encode(value).byteLength; + if (totalBytes <= maxBytes) return value; + const { text, bytes } = utf8Prefix(value, maxBytes); + return `${text}\n\n[truncated, ${totalBytes - bytes} more bytes]`; +} + +/** + * The longest whole-character prefix of `value` that fits in `maxBytes` + * UTF-8 bytes. + * + * @param value - The text to cut. + * @param maxBytes - The largest number of UTF-8 bytes to keep. + * @returns The prefix and its size in bytes. + */ +export function utf8Prefix(value: string, maxBytes: number): { text: string; bytes: number } { + let bytes = 0; + let end = 0; + for (const char of value) { + const charBytes = encoder.encode(char).byteLength; + if (bytes + charBytes > maxBytes) break; + bytes += charBytes; + end += char.length; + } + return { text: value.slice(0, end), bytes }; +} diff --git a/packages/computer/src/tools/ai.test.ts b/packages/computer/src/tools/ai.test.ts index 7493fd2b..93506a95 100644 --- a/packages/computer/src/tools/ai.test.ts +++ b/packages/computer/src/tools/ai.test.ts @@ -1,6 +1,8 @@ import { SQLiteTestStorage } from "@cloudflare/dofs/testing"; import { describe, expect, it } from "vitest"; import { z } from "zod"; +import { WorkerJavaScriptBackend } from "../backends/worker-javascript/worker-javascript.js"; +import { createContainerModule } from "../modules/container.js"; import type { WorkspaceRuntimeExecHandle, WorkspaceRuntimeResult } from "../runtime/types.js"; import { Workspace } from "../workspace.js"; import { @@ -1709,10 +1711,79 @@ describe("createAITools callable exec", () => { }, }); - expect(toolDescription(tools.exec)).toContain("Run a JavaScript module"); - expect(toolDescription(tools.exec)).toContain("ES module source"); + expect(toolDescription(tools.exec)).toContain("Run code in the workspace"); + expect(toolDescription(tools.exec)).toContain("`result` field"); expect(toolDescription(tools.exec)).toContain("JavaScript module runtime"); }); + + it("adds what the backend says about itself after the caller's description", () => { + const workspace = { + runtime: { + async exec() { + throw new Error("not used"); + }, + isCallable: () => true, + describe: (id: string) => + id === "js" ? "Modules: `ws:weather` exports `forecast`." : undefined, + }, + }; + const withBoth = createAITools({ + workspace, + shell: { backends: { js: { description: "Use for data work." } } }, + }); + const withBackendOnly = createAITools({ workspace, shell: { backends: { js: {} } } }); + + expect(toolDescription(withBoth.exec)).toContain( + "Use for data work.\n\nModules: `ws:weather` exports `forecast`.", + ); + expect(toolDescription(withBackendOnly.exec)).toContain("`ws:weather` exports `forecast`"); + }); + + it("requires a description for a backend that does not describe itself", () => { + const workspace = { + runtime: { + async exec() { + throw new Error("not used"); + }, + }, + }; + + expect(() => createAITools({ workspace, shell: { backends: { shell: {} } } })).toThrow( + /does not describe itself/, + ); + }); +}); + +describe("createAITools exec against a real JavaScript backend", () => { + it("lists the backend's modules in the tool description", () => { + const workspace = new Workspace({ + storage: new SQLiteTestStorage(), + backends: [ + new WorkerJavaScriptBackend({ + loader: { load: () => ({ getEntrypoint: () => ({}) }) }, + modules: { + "tar-stream": "export default {};", + "ws:weather": { forecast: ([city]) => ({ city, sky: "clear" }) }, + "ws:container": createContainerModule(), + }, + }), + ], + }); + const tools = createAITools({ + workspace, + shell: { backends: { "worker-javascript": {} } }, + }); + const description = toolDescription(tools.exec); + + expect(description).toContain("`node:fs/promises`"); + expect(description).toContain("- `tar-stream`: a bundled library."); + expect(description).toContain("- `ws:weather`: exports `forecast`."); + expect(description).toContain( + "- `ws:container`: Runs shell commands in a full Linux container", + ); + expect(description).toContain("no direct network access"); + expect(inputProperties(tools.exec)).toEqual(["command", "cwd", "env", "input"]); + }); }); describe("createAITools exec with one backend", () => { diff --git a/packages/computer/src/tools/exec.ts b/packages/computer/src/tools/exec.ts index 1f462630..620d8e56 100644 --- a/packages/computer/src/tools/exec.ts +++ b/packages/computer/src/tools/exec.ts @@ -3,6 +3,7 @@ import { z } from "zod"; import { notCallableMessage } from "../runtime/runtime.js"; import type { WorkspaceRuntimeValue } from "../runtime/types.js"; +import { truncateText, utf8Prefix } from "../text-truncation.js"; // A finite JSON value: what a callable backend accepts as `input` and // returns as `result`. Declared as a concrete recursive schema rather @@ -64,11 +65,18 @@ export interface ExecWorkspaceLike { // are callable; the runtime derives it from each backend's // `callable` flag. Omit when no backend is callable. isCallable?(id: string): boolean; + // What a backend says about itself for a model, such as the + // language it runs and the modules that code can import. The tool + // shows it after the caller's own description. + describe?(id: string): string | undefined; }; } export interface ExecBackendDescription { - description: string; + // Guidance for the model about this backend, shown before whatever + // the backend says about itself. Required only when the backend does + // not describe itself. + description?: string; } export interface ExecToolOptions { @@ -128,34 +136,63 @@ export function createExecTool(options: ExecToolOptions): Tool JSON.stringify(id)).join(", ")}`, + `createExecTool: pass a defaultBackend that is one of ${backendIds.map((id) => JSON.stringify(id)).join(", ")}`, ); } - const isCallable = options.workspace.runtime.isCallable?.bind(options.workspace.runtime); - const callableBackendIds = new Set(backendIds.filter((id) => isCallable?.(id) === true)); - const description = single - ? singleBackendDescription( - options.backends[onlyBackend].description, - callableBackendIds.has(onlyBackend), - ) - : multipleBackendDescription(options.backends, defaultBackend, callableBackendIds); - const inputSchema = single - ? singleBackendInputSchema(callableBackendIds.has(onlyBackend)) - : multipleBackendInputSchema(backendIds, defaultBackend); + const runtime = options.workspace.runtime; + const backends = backendIds.map((id) => { + const text = [options.backends[id]?.description, runtime.describe?.(id)] + .filter((part) => part !== undefined && part !== "") + .join("\n\n"); + if (text === "") { + throw new Error( + `createExecTool: backend ${JSON.stringify(id)} does not describe itself; pass a description`, + ); + } + return { id, text, callable: runtime.isCallable?.(id) === true }; + }); + const callableBackendIds = new Set(backends.filter((b) => b.callable).map((b) => b.id)); + const description = describeTool(backends, defaultBackend); + // Offer only the fields that can work: `backend` when there is a + // choice, `input` when some backend accepts it. + const shape: Record = { + command: z.string().describe(commandHint(backends)), + cwd: z.string().optional().describe("Working directory. Defaults to the workspace root."), + env: z + .record(z.string(), z.string()) + .optional() + .describe( + "Environment variables for this run only. Values override the base environment without affecting later runs.", + ), + }; + if (!single) { + shape.backend = z + // SAFETY: createExecTool checked that backendIds has at least one entry. + .enum(backendIds as [string, ...string[]]) + .optional() + .describe( + `Which backend to run on. Omit to use the default (${JSON.stringify(defaultBackend)}). If a command fails because the backend lacks that tool, retry on a backend whose description covers it.`, + ); + } + if (callableBackendIds.size > 0) { + shape.input = jsonValueSchema + .optional() + .describe( + single + ? "Structured value handed to the module." + : "Structured value handed to a callable backend's module. Other backends reject it.", + ); + } + // SAFETY: Every field in `shape` has the type ExecToolInput gives it, and the fields left out are optional there. + const inputSchema = z.object(shape) as unknown as z.ZodType; return tool({ description, @@ -252,8 +289,8 @@ export function createExecTool(options: ExecToolOptions): Tool, - defaultBackend: string, - callableBackendIds: ReadonlySet, -): string { - const backendGuidance = Object.entries(backends) - .map(([id, backend]) => { - const suffix = callableBackendIds.has(id) ? " (callable)" : ""; - return `- ${JSON.stringify(id)}${suffix}: ${backend.description}`; - }) - .join("\n"); - const callableGuidance = - callableBackendIds.size > 0 - ? [ +// With one backend the description is about what it does. With several +// it lists them and explains how to choose. +function describeTool(backends: readonly DescribedBackend[], defaultBackend: string): string { + const [only, ...others] = backends; + if (only !== undefined && others.length === 0) { + return only.callable + ? ["Run code in the workspace.", "", only.text, "", CALLABLE_HINT, FILE_TOOLS_HINT].join("\n") + : [ + "Run a shell command in the workspace.", + "", + only.text, "", - `Callable backends (${[...callableBackendIds].map((id) => JSON.stringify(id)).join(", ")}) run \`command\` as module source rather than a shell command. Pass \`input\` to hand the module a structured value, and read the module's returned value back from the \`result\` field. Other backends reject \`input\`.`, - ].join("\n") - : ""; + `${SHELL_HINT} ${FILE_TOOLS_HINT}`, + ].join("\n"); + } + const callable = backends.filter((backend) => backend.callable).map((b) => JSON.stringify(b.id)); return [ "Run a shell command in the workspace. The workspace exposes multiple backends, each with different capabilities.", "Pick the cheapest backend that can run the command; fall back to a heavier one only when the lighter backend's command set doesn't cover what you need.", "", "Backends:", - backendGuidance, + ...backends.map( + (b) => `- ${JSON.stringify(b.id)}${b.callable ? " (callable)" : ""}: ${b.text}`, + ), "", `Default backend: ${JSON.stringify(defaultBackend)}. Try this first for any command you're not sure about; if it fails with a "command not found" or a similar capability error, retry on a backend whose description covers the missing tool.`, - `${SHELL_USE_HINT} ${FILE_TOOLS_HINT}`, - callableGuidance, + `${SHELL_HINT} ${FILE_TOOLS_HINT}`, + ...(callable.length === 0 + ? [] + : [ + "", + `Callable backends (${callable.join(", ")}) run \`command\` as module source rather than a shell command. ${CALLABLE_HINT} Other backends reject \`input\`.`, + ]), ].join("\n"); } -const cwdSchema = z - .string() - .optional() - .describe("Working directory. Defaults to the workspace root."); -const envSchema = z - .record(z.string(), z.string()) - .optional() - .describe( - "Environment variables for this run only. Values override the base environment without affecting later runs.", - ); - -// One backend: no `backend` argument, and `input` only when the -// backend can accept it. -function singleBackendInputSchema(callable: boolean): z.ZodType { - if (!callable) { - return z.object({ - command: z.string().describe("Shell command, e.g. 'npm test' or 'git diff HEAD'."), - cwd: cwdSchema, - env: envSchema, - }); +function commandHint(backends: readonly DescribedBackend[]): string { + if (backends.every((backend) => backend.callable)) return "Module source to run."; + if (backends.every((backend) => !backend.callable)) { + return "Shell command, e.g. 'npm test' or 'git diff HEAD'."; } - return z.object({ - command: z - .string() - .describe( - "ES module source to run. Its default export receives `input` and its return value comes back as `result`.", - ), - cwd: cwdSchema.describe( - "Working directory for relative imports. Defaults to the workspace root.", - ), - env: envSchema, - input: jsonValueSchema - .optional() - .describe("Structured value handed to the module's default export."), - }); -} - -function multipleBackendInputSchema( - backendIds: string[], - defaultBackend: string, -): z.ZodType { - return z.object({ - command: z - .string() - .describe( - "Shell command, e.g. 'npm test' or 'git diff HEAD'. For a callable backend this is the module source to run.", - ), - cwd: cwdSchema, - backend: z - // SAFETY: createExecTool checked that backendIds has at least one entry before calling this. - .enum(backendIds as [string, ...string[]]) - .optional() - .describe( - [ - "Which backend to run on. Omit to use the default", - `(${JSON.stringify(defaultBackend)}). Set explicitly when the`, - "default backend is not capable of running the command. If a command fails because the backend lacks that tool, retry on a backend whose description covers it.", - ].join(" "), - ), - env: envSchema, - input: jsonValueSchema - .optional() - .describe( - "Structured value handed to a callable backend's module. Only callable backends accept it; other backends reject it.", - ), - }); + return "Shell command, e.g. 'npm test' or 'git diff HEAD'. For a callable backend this is the module source to run."; } function errorMessage(err: unknown): string { @@ -421,47 +389,16 @@ class StreamBuffer { } // The chunk crosses the cap: keep the largest whole-character // prefix that fits, then stop growing the head. - let used = this.#headBytes; - let end = 0; - for (const char of chunk) { - const charBytes = encoder.encode(char).byteLength; - if (used + charBytes > this.#cap) break; - used += charBytes; - end += char.length; - } - this.#head += chunk.slice(0, end); - this.#headBytes = used; + const prefix = utf8Prefix(chunk, this.#cap - this.#headBytes); + this.#head += prefix.text; + this.#headBytes += prefix.bytes; } render(maxBytes: number): string { if (this.#totalBytes <= maxBytes && this.#totalBytes === this.#headBytes) { return this.#head; } - let used = 0; - let end = 0; - for (const char of this.#head) { - const charBytes = encoder.encode(char).byteLength; - if (used + charBytes > maxBytes) break; - used += charBytes; - end += char.length; - } - return `${this.#head.slice(0, end)}\n\n[truncated, ${this.#totalBytes - used} more bytes]`; + const shown = utf8Prefix(this.#head, maxBytes); + return `${shown.text}\n\n[truncated, ${this.#totalBytes - shown.bytes} more bytes]`; } } - -function truncate(value: string, maxBytes: number): string { - if (!value) return value; - const totalBytes = encoder.encode(value).byteLength; - if (totalBytes <= maxBytes) return value; - - let usedBytes = 0; - let endOffset = 0; - for (const char of value) { - const charBytes = encoder.encode(char).byteLength; - if (usedBytes + charBytes > maxBytes) break; - usedBytes += charBytes; - endOffset += char.length; - } - - return `${value.slice(0, endOffset)}\n\n[truncated, ${totalBytes - usedBytes} more bytes]`; -} diff --git a/packages/computer/src/workspace.ts b/packages/computer/src/workspace.ts index f9d841d5..d8d4b325 100644 --- a/packages/computer/src/workspace.ts +++ b/packages/computer/src/workspace.ts @@ -219,8 +219,7 @@ export class Workspace { readonly #backends: WorkspaceBackend[]; readonly #backendsById: Map; readonly #moduleBackendsById: Map; - readonly #registeredBackendIds: Set; - readonly #callableBackendIds: Set; + readonly #registeredBackends: Map; readonly #defaultBackendId: string | undefined; readonly #observer: WorkspaceObserver; readonly #syncLogger: SyncLogger; @@ -305,19 +304,16 @@ export class Workspace { this.#moduleBackendsById = new Map( registered.filter(isModuleBackend).map((backend) => [backend.id, backend]), ); - this.#registeredBackendIds = new Set(); - this.#callableBackendIds = new Set( - registered.filter((backend) => backend.callable === true).map((backend) => backend.id), - ); + this.#registeredBackends = new Map(); for (const backend of registered) { - if (this.#registeredBackendIds.has(backend.id)) { + if (this.#registeredBackends.has(backend.id)) { throw new Error( `Workspace: duplicate backend id ${JSON.stringify(backend.id)}. ` + "Pass an explicit `id` on each backend's constructor options to " + "distinguish them.", ); } - this.#registeredBackendIds.add(backend.id); + this.#registeredBackends.set(backend.id, backend); } this.#defaultBackendId = registered[0]?.id; this.#observer = options.observer ?? noopObserver; @@ -429,7 +425,7 @@ export class Workspace { get runtime(): WorkspaceRuntime { if (!this.#runtime) { this.#runtime = new WorkspaceRuntime({ - callableBackendIds: this.#callableBackendIds, + backends: this.#registeredBackends, backendHandle: (id) => this.#backendHandleFor(id), resolveBackendId: (id) => this.#resolveBackendId(id) ?? "", }); @@ -861,13 +857,13 @@ export class Workspace { // workspace; throws on an unknown id. Omitted ids fall through // to the first backend in the list (the default). #resolveBackendId(id: string | undefined): string | undefined { - if (this.#registeredBackendIds.size === 0) return undefined; + if (this.#registeredBackends.size === 0) return undefined; const target = id ?? this.#defaultBackendId; if (target === undefined) return undefined; - if (!this.#registeredBackendIds.has(target)) { + if (!this.#registeredBackends.has(target)) { throw new Error( `Workspace: no backend with id ${JSON.stringify(target)}. ` + - `Configured backends: ${[...this.#registeredBackendIds].map((key) => JSON.stringify(key)).join(", ") || ""}.`, + `Configured backends: ${[...this.#registeredBackends.keys()].map((key) => JSON.stringify(key)).join(", ") || ""}.`, ); } return target; diff --git a/packages/computer/tests/script-runner-worker.ts b/packages/computer/tests/script-runner-worker.ts index 4ea05f79..86fac5fc 100644 --- a/packages/computer/tests/script-runner-worker.ts +++ b/packages/computer/tests/script-runner-worker.ts @@ -8,7 +8,7 @@ import type { WorkspaceRuntimeValue, WorkspaceStub, } from "../src/index.js"; -import { defineModule, Workspace } from "../src/index.js"; +import { Workspace } from "../src/index.js"; import { createArtifactsModule } from "../src/modules/artifacts.js"; import { createContainerModule } from "../src/modules/container.js"; import { createGitModule } from "../src/modules/git.js"; @@ -78,7 +78,7 @@ export class HostDO extends DurableObject { "ws:git": createGitModule(), "ws:artifacts": createArtifactsModule(), "ws:container": createContainerModule(), - "ws:test-host": defineModule({ + "ws:test-host": { async echo(args) { return { args: [...args] }; }, @@ -108,7 +108,7 @@ export class HostDO extends DurableObject { keep: true, }; }, - }), + }, }, }), fakeContainerBackend(),