computerd: Add support for an ignore field to the ContainerBackend

.changeset/container-ignore-assertion.md

@@ -0,0 +1,5 @@
+---
+"@cloudflare/computer": minor
+---
+
+Add `ignore` to `ContainerBackend` to configure pass-through to the container disk.

docs/19_performance.md

@@ -48,6 +48,50 @@ computerd is ~2x slower than the container's ext4 disk for the full
 `npm install`, and ~3.6x slower than tmpfs. The disk comparison is
 the more realistic baseline for general usage.
 
+> [!IMPORTANT]
+> These numbers measure the **mount**, not the **pull**. They stop when
+> `npm install` returns. What follows — moving 36,675 files into the
+> Durable Object — is not counted here, and for a dependency tree it is
+> the larger cost.
+>
+> [#179](https://github.com/cloudflare/computer/issues/179) reports an
+> install timing out at 120 s and then taking ~3 further minutes to
+> return while the partial `node_modules` was pulled, after which the
+> next command failed with a storage timeout the workspace did not
+> recover from. None of that is visible in the table above.
+>
+> If you are sizing a workload against these figures, add the transfer
+> yourself, or keep the tree out of sync entirely — see
+> [`computerd`: Local-only paths](../packages/computerd/README.md#local-only-paths-mount_ignore).
+
+## Local-only paths (`MOUNT_IGNORE`)
+
+A path listed in `MOUNT_IGNORE` is served from the container's disk and
+never enters the VFS, the store, the change-pack encoding, or the pull.
+
+What this does **not** change is the FUSE round trip: the bytes still
+cross from the kernel into the daemon. Passthrough (`FOPEN_PASSTHROUGH`)
+would remove that too, but computerd mounts through `fuse-native`, which
+binds libfuse 2.9, and passthrough needs the libfuse 3.17 API. So expect
+a local-only `npm install` to track the `computerd FUSE` row above
+rather than the `ext4 disk` row.
+
+The saving is the transfer, and for a dependency tree the transfer is
+most of the wall clock.
+
+| Scenario | Install duration | Bytes pulled into the DO |
+|---|---:|---:|
+| `npm install` to a synced path | 124.7 s | *(not yet measured)* |
+| `npm install` to a `MOUNT_IGNORE` path | *(not yet measured)* | 0 by construction |
+
+> [!NOTE]
+> The empty cells are deliberate. Bytes-pulled is the load-bearing
+> number for this feature and it has not been measured yet; the zero in
+> the last cell is a property of the design — an ignored path produces
+> no sync entries — not an observation. Fill the table from the same
+> `cloudflare/sandbox-sdk` install used above, on the same instance
+> type, before quoting any of it.
+
 ## In-memory store versus on-disk store
 
 `computerd` keeps its SQLite store in memory by default. Set

packages/computer/src/backend.ts

@@ -90,6 +90,19 @@ export interface BackendHandle {
   //              Durable Object). push/pull are no-ops, and the
   //              reconcile-watermarks pass on connect is skipped.
   sync?: "remote" | "none";
+  // Local-only paths this backend's container keeps on its own disk,
+  // as the container reports them (#179). Absent on backends with no
+  // such concept.
+  //
+  // `supported: false` means the container predates the feature, so
+  // every path is synced regardless of what the host asked for. Worth
+  // logging: it is the difference between a configuration that works
+  // and one that silently does nothing.
+  ignore?: {
+    readonly paths: readonly string[];
+    readonly root: string | undefined;
+    readonly supported: boolean;
+  };
   // Resolves when the underlying transport closes for any reason
   // (clean close, peer crash, network drop). The Workspace listens
   // for this and drops its cached handle so the next ready() call

packages/computer/src/backends/container/container-backend-ignore.test.ts

@@ -0,0 +1,279 @@
+// connect()'s happy path constructs a WebSocketPair, a workerd global
+// the node runner does not provide, so the full dial cannot complete
+// here. These exercise the wire format the backend depends on, against
+// a fake host. The comparison logic and the error text have their own
+// suite in ignore-assertion.test.ts, and the end-to-end behavior is
+// covered in computerd's cli tests against a real FUSE mount.
+import { afterEach, describe, expect, test, vi } from "vitest";
+
+import { ContainerBackend } from "./container-backend.js";
+import type { ContainerRuntimeInfo, IWorkspaceContainerAPI } from "./container-host.js";
+import type { ContainerLaunchSpec } from "./container-launch-record.js";
+import { ContainerIgnoreMismatchError, readIgnoreReport } from "./ignore-assertion.js";
+
+interface FakeHostOptions {
+  // The `ignore` block /__computerd/info reports. Omitted models a
+  // computerd predating the feature.
+  info?: Record<string, unknown>;
+}
+
+function fakeHost(opts: FakeHostOptions = {}) {
+  const fetches: { port: number; path: string }[] = [];
+  const starts: ContainerLaunchSpec[] = [];
+  const info: ContainerRuntimeInfo = {
+    runtimeId: "runtime-1",
+    clientSecret: "00112233445566778899aabbccddeeff",
+    outcome: "launched",
+  };
+  const host: IWorkspaceContainerAPI = {
+    async start(spec) {
+      starts.push(spec);
+      return info;
+    },
+    async restart() {
+      return info;
+    },
+    async interceptOutboundHttp() {},
+    async interceptAllOutboundHttp() {},
+    async fetchPort(port, url) {
+      const path = new URL(url).pathname;
+      fetches.push({ port, path });
+      if (path === "/__computerd/info") {
+        return new Response(
+          JSON.stringify({
+            backend: { kind: "fuse" },
+            mountPoint: "/workspace",
+            ...(opts.info === undefined ? {} : { ignore: opts.info }),
+          }),
+          { status: 200, headers: { "content-type": "application/json" } },
+        );
+      }
+      // Never healthy, so connect() fails before the upgrade.
+      return new Response(null, { status: 503 });
+    },
+    port() {
+      throw new Error("not used");
+    },
+    async setInactivityTimeout() {},
+    async status() {
+      return { running: true, exit: null };
+    },
+    async exitInfo() {
+      return null;
+    },
+  };
+  return { host, fetches, starts };
+}
+
+describe("ContainerBackend local-only paths", () => {
+  const readInfo = async (host: IWorkspaceContainerAPI) => {
+    const res = await host.fetchPort(8080, "http://container/__computerd/info");
+    return readIgnoreReport(await res.json());
+  };
+
+  test("reads the ignore block a current container reports", async () => {
+    const { host } = fakeHost({
+      info: {
+        supported: true,
+        enabled: true,
+        root: "/tmp/workspace",
+        paths: ["node_modules", "dist"],
+        redundant: [],
+      },
+    });
+    expect(await readInfo(host)).toEqual({
+      paths: ["/workspace/node_modules", "/workspace/dist"],
+      root: "/tmp/workspace",
+      mountPoint: "/workspace",
+      supported: true,
+    });
+  });
+
+  test("treats a container with no ignore block as unsupported", async () => {
+    // The version-skew case the README warns about: the computerd image
+    // can lag the pinned client. Without this the old image looks like
+    // it is working while quietly syncing everything.
+    const { host } = fakeHost({});
+    expect(await readInfo(host)).toEqual({
+      paths: [],
+      root: undefined,
+      mountPoint: undefined,
+      supported: false,
+    });
+  });
+
+  test("the backend requests /__computerd/info on the container port", async () => {
+    // Pins the path and port, so a rename upstream fails here rather
+    // than silently degrading every deployment to "unsupported".
+    const { host, fetches } = fakeHost({
+      info: { supported: true, paths: [], root: "/tmp/workspace" },
+    });
+    await host.fetchPort(8080, "http://container/__computerd/info");
+    expect(fetches).toContainEqual({ port: 8080, path: "/__computerd/info" });
+  });
+
+  const backendWith = (host: IWorkspaceContainerAPI, ignore?: readonly string[]) =>
+    new ContainerBackend({
+      container: () => ({ getWorkspaceContainer: () => host }),
+      workspace: { binding: "SESSIONS", id: "session-1" },
+      restartAttempts: 0,
+      connectTimeoutMs: 400,
+      healthProbeTimeoutMs: 50,
+      healthRetryInitialDelayMs: 10,
+      healthRetryMaxDelayMs: 20,
+      heartbeatIntervalMs: 0,
+      ...(ignore === undefined ? {} : { ignore }),
+    });
+
+  test("passes `ignore` to the container as MOUNT_IGNORE at start time", async () => {
+    // The set is deployment config, not image config: it has to arrive
+    // in the start environment or the image would have to be rebuilt to
+    // change it.
+    const { host, starts } = fakeHost();
+    await backendWith(host, ["/node_modules", "/.venv", "/dist"])
+      .connect()
+      .catch(() => undefined);
+
+    expect(starts).toHaveLength(1);
+    expect(starts[0]?.env?.MOUNT_IGNORE).toBe("/node_modules,/.venv,/dist");
+  });
+
+  test("sends no MOUNT_IGNORE when `ignore` is omitted", async () => {
+    const { host, starts } = fakeHost();
+    await backendWith(host)
+      .connect()
+      .catch(() => undefined);
+
+    expect(starts).toHaveLength(1);
+    expect(starts[0]?.env?.MOUNT_IGNORE).toBeUndefined();
+  });
+});
+
+// Drives connect() through the upgrade, so the ignore check runs against a
+// container that enforces its client secret the way computerd does: every
+// route except /health needs the bearer token.
+describe("ContainerBackend ignore check on a full connect", () => {
+  const SECRET = "00112233445566778899aabbccddeeff";
+
+  afterEach(() => {
+    vi.unstubAllGlobals();
+  });
+
+  // Enough of a WebSocket for capnweb to attach to and for the backend to
+  // close. Nothing is sent over it in these tests.
+  class FakeSocket {
+    readyState = 1;
+    accept() {}
+    addEventListener() {}
+    removeEventListener() {}
+    send() {}
+    close() {
+      this.readyState = 3;
+    }
+  }
+
+  function connectingHost(ignore: Record<string, unknown>) {
+    let backend: ContainerBackend | undefined;
+    const infoAuth: (string | null)[] = [];
+    const host: IWorkspaceContainerAPI = {
+      async start() {
+        return { runtimeId: "runtime-1", clientSecret: SECRET, outcome: "launched" };
+      },
+      async restart() {
+        throw new Error("not used");
+      },
+      async interceptOutboundHttp() {},
+      async interceptAllOutboundHttp() {},
+      async fetchPort(_port, url, init) {
+        const path = new URL(url).pathname;
+        const auth = new Headers(init?.headers).get("authorization");
+        if (path === "/health") return new Response("ok");
+        if (path === "/__computerd/info") infoAuth.push(auth);
+        if (auth !== `Bearer ${SECRET}`) return new Response(null, { status: 401 });
+        if (path === "/connect") {
+          // computerd dials back as soon as it is told where to go.
+          await backend
+            ?.handleFetch(
+              new Request("http://computer.internal/api", {
+                headers: { upgrade: "websocket", authorization: `Bearer ${SECRET}` },
+              }),
+            )
+            // The 101 Response is a workerd-only shape; the upgrade has
+            // already been handed over by the time it is built.
+            .catch(() => undefined);
+          return new Response(null, { status: 200 });
+        }
+        if (path === "/__computerd/info") {
+          return Response.json({ backend: { kind: "fuse" }, mountPoint: "/workspace", ignore });
+        }
+        return new Response(null, { status: 404 });
+      },
+      port() {
+        throw new Error("not used");
+      },
+      async setInactivityTimeout() {},
+      async status() {
+        return { running: true, exit: null };
+      },
+      async exitInfo() {
+        return null;
+      },
+    };
+    return {
+      host,
+      infoAuth,
+      attach(b: ContainerBackend) {
+        backend = b;
+      },
+    };
+  }
+
+  function backendFor(fake: ReturnType<typeof connectingHost>, ignore: readonly string[]) {
+    vi.stubGlobal(
+      "WebSocketPair",
+      class {
+        0 = new FakeSocket();
+        1 = new FakeSocket();
+      },
+    );
+    const backend = new ContainerBackend({
+      container: () => ({ getWorkspaceContainer: () => fake.host }),
+      workspace: { binding: "SESSIONS", id: "session-1" },
+      restartAttempts: 0,
+      connectTimeoutMs: 2_000,
+      heartbeatIntervalMs: 0,
+      ignore,
+    });
+    fake.attach(backend);
+    return backend;
+  }
+
+  test("reads /__computerd/info with the client secret", async () => {
+    const fake = connectingHost({ supported: true, root: "/tmp/workspace", paths: ["dist"] });
+    const handle = await backendFor(fake, ["/dist"]).connect();
+
+    expect(fake.infoAuth).toEqual([`Bearer ${SECRET}`]);
+    expect(handle.ignore).toEqual({
+      paths: ["/workspace/dist"],
+      root: "/tmp/workspace",
+      mountPoint: "/workspace",
+      supported: true,
+    });
+    await handle.close();
+  });
+
+  test("accepts a declaration spelled with the mount point", async () => {
+    // computerd strips the mount prefix from "/workspace/dist" and
+    // applies "dist". The declaration means the same thing.
+    const fake = connectingHost({ supported: true, root: "/tmp/workspace", paths: ["dist"] });
+    const handle = await backendFor(fake, ["/workspace/dist"]).connect();
+    await handle.close();
+  });
+
+  test("still rejects a real mismatch", async () => {
+    const fake = connectingHost({ supported: true, root: "/tmp/workspace", paths: ["dist"] });
+    await expect(backendFor(fake, ["/node_modules"]).connect()).rejects.toBeInstanceOf(
+      ContainerIgnoreMismatchError,
+    );
+  });
+});