diff --git a/.env.example b/.env.example index acc971401c..915c1dc6c5 100644 --- a/.env.example +++ b/.env.example @@ -166,6 +166,11 @@ COPILOTKIT_LICENSE_TOKEN= # CPK_INTELLIGENCE_SKILLS_REVISION=exact-revision-id # See docs/automatic-learning.md for managed/self-hosted setup and permissions. +# OpenBot accepts the OpenBot Recorder extension only from its own extension id. A fork, or a build +# signed with another key, has another id: list it here, comma separated, each 32 letters from a to +# p. Any other value stops the server at boot. +# OPENBOT_RECORDER_EXTRA_EXTENSION_IDS= + # How long a Bot's stream may say nothing before this deployment gives up on the turn, in # milliseconds. A Bot is any AG-UI endpoint, which means it will be redeployed mid-answer, its own # upstream will time out, and it will sometimes accept a connection and then write nothing at all. @@ -207,6 +212,14 @@ OPENAI_API_KEY= # BOT_MODEL=openai/gpt-5.5 # # OPENAI_BASE_URL= +# Behind an OPENAI_BASE_URL that is not OpenAI or Azure, built-in Bots call the endpoint's Chat +# Completions API, which is what most compatible endpoints serve, unless their model takes tools only +# through the Responses API (gpt-5.6 and later, the -pro models, gpt-5.3-codex, under a gateway's +# `openai/` name too): those call `/responses`, which a gateway or proxy in front of OpenAI serves. +# A gateway that serves the Responses API keeps Responses-only features, such as reasoning summaries, +# for every model with `responses`. `chat` forces Chat Completions for every model; any other value is +# an error when a built-in Bot runs. +# COPILOTKIT_OPENAI_API=responses # Optional dictation, independent of every agent/chat provider. Set provider, base URL, # and model together. The endpoint must implement POST /audio/transcriptions under this @@ -367,6 +380,16 @@ COMPUTER_TOKEN= # Optional absolute app-owned data root; defaults to the platform's OpenBot user-data directory. # OPENBOT_LOCAL_COMPUTER_DIR= +# The OpenBot browser: each person's own browser, opened from OpenBot with the Recorder already in it +# and connected. Unset, the supervisor below gives it a computer of its own (on a computer in the cloud +# it reaches the open web, this OpenBot and the Intelligence gateway, nothing inside the deployment); +# the one computer every Bot shares never runs it. Set, it runs on this +# computer instead, as a window on the person's own desktop, while Bots' computers stay where they are: +# OpenBot Desktop points it at its host-side local Chrome computer, called with COMPUTER_TOKEN. The +# Recorder in it reaches OpenBot at OPENBOT_PUBLIC_URL (or BETTER_AUTH_URL), which must be https, or +# http on 127.0.0.1 or [::1] with a port. +# OPENBOT_BROWSER_URL=http://127.0.0.1:4101 + # Which Bot this computer belongs to, and therefore which profile directory it uses. # COMPUTER_BOT_ID=shared @@ -426,7 +449,9 @@ MANAGED_AGENT_TOKEN= # One computer per Bot. Unset, every Bot shares the computer at AGENT_COMPUTER_URL, suitable on a # laptop and explicit about being one shared machine. Set, the supervisor gives each Bot a container -# of its own, with its own /workspace and its own browser profile. +# of its own, with its own /workspace and its own browser profile. A person's own OpenBot browser +# runs only on a computer of its own, so it needs this (or OPENBOT_BROWSER_URL above): on the shared +# computer a Bot's shell could read the person's logins off the disk. COMPUTER_SUPERVISOR_URL= # Shared with the supervisor. Not the security boundary, its four verbs are, but it keeps anything # else that can reach the port from cycling a Bot's computer. diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index d442a197f2..fe0c0082ad 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -140,7 +140,8 @@ jobs: - run: >- bun test tests/sign-in.test.ts tests/session-cookies.test.ts tests/follows-popup.test.ts tests/demonstration-live.test.ts tests/webrtc-proxy.test.ts tests/web-bot-auth-browser.test.ts - tests/handoff-page-restore.test.ts tests/enter-text.test.ts + tests/handoff-page-restore.test.ts tests/enter-text.test.ts tests/person-browser.test.ts tests/person-browser-computer.test.ts + tests/screencast-keys-browser.test.ts working-directory: agent-computer env: OPENBOT_COMPUTER_BROWSER: '1' diff --git a/.github/workflows/desktop.yml b/.github/workflows/desktop.yml index 49591470c0..fd4c963fee 100644 --- a/.github/workflows/desktop.yml +++ b/.github/workflows/desktop.yml @@ -9,10 +9,10 @@ name: Desktop # it at runtime with an `expect`, so a bad row panics the window rather than failing a type check. on: pull_request: - paths: ["desktop/**", "shared/harnesses.json", "shared/marks/**", "package.json", ".github/workflows/desktop.yml"] + paths: ["desktop/**", "shared/harnesses.json", "shared/marks/**", "shared/recorder-extension.json", "package.json", ".github/workflows/desktop.yml"] push: branches: [main] - paths: ["desktop/**", "shared/harnesses.json", "shared/marks/**", "package.json", ".github/workflows/desktop.yml"] + paths: ["desktop/**", "shared/harnesses.json", "shared/marks/**", "shared/recorder-extension.json", "package.json", ".github/workflows/desktop.yml"] workflow_call: secrets: OPENBOT_GOOGLE_MODEL_OAUTH_CLIENT_SECRET: @@ -143,6 +143,18 @@ jobs: - name: Rust regression tests run: cargo test --locked --lib --bins working-directory: desktop/src-tauri + # The binary run as the Recorder's browsers run it: the native messaging host Chrome starts, and + # the install that writes its manifests. The host name and extension id come from + # shared/recorder-extension.json, which the Rust reads at compile time. + - name: Recorder native host tests + run: cargo test --locked --test recorder_host + working-directory: desktop/src-tauri + # Only a release build is a GUI-subsystem executable on Windows, and only a release build proves + # the debug-only test seam is absent. + - name: Recorder native host tests (release) + if: matrix.platform.name == 'windows' + run: cargo test --locked --release --test recorder_host + working-directory: desktop/src-tauri # Hosted Windows runners are administrators. Exercise setup again with an actual Users-only # account: DISM's feature reads worked as CI's administrator but refused ordinary app users. - name: Windows standard-user setup regression diff --git a/.gitignore b/.gitignore index 4a7ca18abb..5b6adc7865 100644 --- a/.gitignore +++ b/.gitignore @@ -53,3 +53,6 @@ venv/ .pytest_cache/ .ruff_cache/ .mypy_cache/ + +# The Recorder extension's signing key never enters the repository (Decision 9). +extension/**/*.pem diff --git a/CHANGELOG.md b/CHANGELOG.md index 1b261f7c54..ec8d267841 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,7 +8,7 @@ Newest first. `Unreleased` is what is on `main` and not yet tagged. ## Unreleased -**Before upgrading.** Nine things change for an existing deployment: +**Before upgrading.** Twelve things change for an existing deployment: - A hosted computer's browser is now headed by default. It uses a little more memory than headless did; set `COMPUTER_BROWSER_MODE=headless` (Helm: `computers.browserMode: headless`) to keep the old behaviour. @@ -34,6 +34,20 @@ Newest first. `Unreleased` is what is on `main` and not yet tagged. - Migration `0060_app_access_requests` adds one table, `app_access_requests`, where a person's request for an administrator to switch on a work app, grant it to a Bot, or connect a Shared app's team account is kept, with who answered it and when. It starts empty and nothing is backfilled. +- Migration `0066_recorder_extension` adds six tables for the OpenBot Recorder and each person's own + Learning space (below): `recorder_connect_codes`, `recorder_connections`, + `recorder_trajectory_bindings`, `learning_person_spaces`, `learning_skill_sharing` and + `learning_open_skills`. Nothing existing changes, and the tables stay empty until somebody + connects a browser or changes Settings → Learning. +- The CopilotKit runtime moves to 1.79.1. A built-in Bot whose `openai/...` model is reached + through an `OPENAI_BASE_URL` that is not OpenAI or Azure now calls that endpoint's Chat + Completions API (`/chat/completions`) instead of the Responses API. That is what most + OpenAI-compatible endpoints serve, so Bots behind one of them now work; a gateway that also serves + the Responses API loses Responses-only features, such as reasoning summaries, without an error. + `COPILOTKIT_OPENAI_API=responses` keeps the Responses API (`chat` forces Chat Completions); any + other value is an error when a built-in Bot runs. Deployments on OpenAI or Azure are unchanged. +- Migration `0067_recorder_activity` adds two nullable columns to `recorder_connections`, where each + OpenBot Recorder's latest recording activity is kept. Nothing existing changes. - Migration `0061_suggestion_superseded` widens the check on `proactive_suggestions.status` to allow `superseded` (below). Open suggestions from before the upgrade stay open until that Bot's next research run replaces them. @@ -337,6 +351,215 @@ apps, which nobody could connect there. An app's own page, reached from an older available yet and that an administrator has to finish setting it up, instead of naming the setting. The administrator's page for the app still names `COMPOSIO_API_KEY`. +### A built-in Bot behind an OpenAI proxy answers again + +A built-in Bot behind a gateway or proxy in front of OpenAI (an `OPENAI_BASE_URL` that is not OpenAI or +Azure) answers again on `gpt-5.6-terra` and on every other model that takes tools only through the +Responses API (`gpt-5.6` and later, the `-pro` models, `gpt-5.3-codex`, also under a gateway's `openai/` +name). OpenBot now sets `COPILOTKIT_OPENAI_API=responses` for those at boot, where CopilotKit 1.79.1 had +sent them to Chat Completions and every turn failed with "Function tools with reasoning_effort are not +supported". Every other model still uses Chat Completions there, and a value you set yourself still wins. + +### Teach a Bot by doing the task in your own browser: the OpenBot Recorder + +The OpenBot Recorder is a Chrome and Edge extension. A person names the task, does it in their own +browser, and the Recorder sends what they did, step by step (each click, entry, choice, submit, +Enter and page change, with what an agent needs to find that element again, and the one request a +step sent when it saved something), masked in the browser first, straight to Intelligence as a +trajectory in that person's own Learning space; the page's own traffic is never recorded, except the +agent's AG-UI events (below). Every Bot +they use then gets the Skills learned there, beside the Bot's own. OpenBot connects the browser and +hands out each capture pass; it receives nothing that is recorded. + +- **The agent's side too.** On a page whose agent speaks AG-UI (over SSE or the Intelligence + WebSocket, OpenBot's own chat included), the Recorder also sends the agent's side, in exactly the + shape the CopilotKit SDK emits so Intelligence cannot tell the two apart: `agent.run`, + `agent.message` (the person's own messages too), `tool.call` and `thread.linked`, a click inside a + chat message with its Thread and message, and `agent.event` for what those leave out (a tool + call's arguments and result, state, reasoning, errors), never one row per streamed token. + CopilotKit's suggestion runs are left out, as its own capture leaves them out. +- **Clicks and typing come from CopilotKit main's own capture code**, carried as a Bun patch over + `@copilotkit/learning` 0.0.1 (`patches/`, which the images and OpenBot Desktop copy) until main is + published. +- **Settings → Browsers** connects the Recorder in this browser and lists every browser connected + as you, each with a Disconnect. OpenBot Desktop can connect the Recorder on its own computer too. +- **Only a connected browser gets a capture pass.** The runtime's `trajectory//connect` route + answers only the Recorder's own bearer, and binds the trajectory to that person; no Bot is + picked or named. A signed-in session alone gets no pass, and the Recorder's bearer cannot run a + Bot or read a conversation. +- **One Learning space per person.** Every recording joins the person's own Intelligence Learning + container, `user-` and 32 hex digits derived from their OpenBot user id. OpenBot makes it when + the browser connects (and again on a capture pass if it is still missing) with the same + credential that makes the `openbot` container: a service token on a CopilotKit local evaluation + install, or a CopilotKit CLI login against managed Intelligence. Intelligence's runtime key + cannot make one. With neither, a missing space cannot be made, and the Recorder's popup says + Learning will not see the recording; the recording still goes to Intelligence. +- **Every Bot a person uses draws on their space.** A run gets the Bot's own learned Skills, then + the person's own, then any another person opened (below). On a name clash the Bot's own wins, + then the person's own; two people's opened Skills with one name are both kept, each named + `/`. A person's space that is missing or unreachable never fails a run: it + goes on with the Skills it has. Admin → Automatic Learning off, or a Bot excluded there, still + means no learned Skills at all. +- **Settings → Learning** chooses who else's Bots use a person's Skills: only their own (the + default), every Bot in the deployment, or the Skills they open one by one, and only Skills in + their own space can be opened. Opening Skills to others needs **Admin → Enterprise controls → + Skill sharing**, off by default: while it is off nobody can open their Skills, runs offer nobody + another person's, and each person's choice is kept for when it is turned on. Runs read what is + open from memory, refreshed at once on the replica that saved a change and within 30 seconds on + the others. +- **What the Recorder collects is said before the first Connect** (Settings → Browsers, the + Recorder's setup page, the Browser page and its Record form): your steps, what each step saves + (the request and its answer), the conversation with an AG-UI agent on the page (your own messages + too) and your narration, masked in your browser first. +- **Narration is off until you choose it**, asked before your first recording and remembered, and it + is transcribed on your own computer only (Chrome's on-device speech recognition, with the language + downloaded once on your click); where on-device recognition is not available, narration stays off + and the page says why. +- **Every recorded tab shows that it is being recorded**, from its first step (a page cannot hide the mark: it + comes back at once if removed or hidden), with one click to leave + that site out of recordings; a left-out site records nothing, and the Recorder's options list the + sites left out. A page you have only looked at is held while you stay on its site, and sent when you + act there, move to another tab or stop, so leaving a site out at any point in a visit sends nothing + of it. +- **A Recorder's connection ends** whenever your sign-in ends (signing out, signing out everywhere, an + admin removing you or requiring SSO, your organization ending its session), or after it goes unused for + as long as an OpenBot sign-in lasts; signing out does not finish, and says so, if they cannot be ended; a recording kept for a person who never connects again is deleted unsent + after 7 days. A page on your own computer is trusted to connect without a click only at OpenBot + Desktop's own address, as its native host confirms it, and connecting to another OpenBot or as + another person always asks you first, saying which connection it replaces. +- **Intelligence must allow trajectories:** the `trajectories` licence feature, + `FF_LEARNING_TRAJECTORIES=true`, a realtime gateway that can ask Intelligence whether an + organization may capture, and an `INTELLIGENCE_GATEWAY_WS_URL` that people's browsers can reach. +- **`OPENBOT_RECORDER_EXTRA_EXTENSION_IDS`** lists other extension ids to trust besides the + Recorder's own, for a fork or a build signed with another key. Each must be 32 letters from a to + p, comma separated; **a value that is not stops the server at boot.** +- Connecting, disconnecting and refusals are audited as `recorder.connected`, `recorder.revoked`, + `recorder.connect_refused`, `recorder.bearer_refused` and `recorder.trajectory_refused`; changing + Settings → Learning as `learning.sharing_changed`, `learning.skill_opened` and + `learning.skill_closed`. + +### When OpenBot is open in Chrome or Edge, the Browser button uses that browser + +With the OpenBot Recorder in that browser, one click on Browser opens Record and Stop right in OpenBot, +and the red dot follows the recording at once. If the Recorder is not connected yet, the same click +connects it (OpenBot Desktop and your organization's own OpenBot, when its address is set by policy, +need no Connect click; any other site still asks you to confirm). If it is not installed, the click +opens its store page in a new tab. Only the OpenBot the Recorder is connected to can start, stop or see +its recordings; any other website is told it is not connected. Settings, Browsers shows Connected +straight away when no Connect click is needed. OpenBot Desktop's own window, Safari and Firefox are +unchanged. + +### Your own everyday Chrome or Edge, with the Recorder in it + +On OpenBot Desktop the Browser button now works with your own everyday Chrome or Edge: your profile, +extensions, password manager and logins, not a separate copy. + +- **Once the OpenBot Recorder is published** on the Chrome Web Store and Edge Add-ons, OpenBot Desktop + registers it with your Chrome and Edge, and the browser installs it from the store and asks you once to + turn it on (on a Mac the next time you restart the browser, on Windows while it runs, on Linux in Edge + only). Before it is there, the Browser button opens the Recorder's store page in your Chrome (or Edge) + so you can add it at once. Until the Recorder is published, the Browser button opens the OpenBot browser + as before. Uninstalling Desktop on Windows takes the registration back out. +- **The Recorder connects through OpenBot Desktop as it installs, with nothing to click.** "Connect with + OpenBot Desktop" in the Recorder's popup also connects at once; connecting from a web page still asks + you to confirm. Once it is connected, the Browser button brings that browser to the front. +- **The red dot follows a recording in any of your browsers**, your everyday one included, on Desktop and + hosted: the Recorder tells OpenBot when it starts and stops recording, and every 30 seconds while it + records, and the dot goes out 90 seconds after a browser stops reporting. +- **The Recorder says what it collects** first on its setup page (above its request for your microphone) + and on its Record form, before anything is recorded: every step in every tab you use until you stop, what is on those pages, your clicks and + typing (secrets masked in your browser first), what each step saves and what you say, all going + straight from your browser to your organization's own Intelligence, with a pass from your + organization's OpenBot, which is told that you are recording, the task's name, when it started and + how many steps it holds; when you record in a browser shown inside OpenBot, your narration passes + through it to the Recorder without being stored. +- Migration `0067_recorder_activity` adds two nullable columns to `recorder_connections` (each + Recorder's latest activity). Nothing existing changes. + +### A recording follows every tab you use + +While a recording runs, the OpenBot Recorder now records in every tab you use: any tab you switch to or +open, in any window, in the OpenBot browser and in your everyday Chrome. Before, it recorded only the +tab you started in and tabs opened from it, so steps in a tab you switched to were lost. Tabs you do +not visit during a recording are never recorded, and nothing on the browser's own pages (a new tab, +settings) or the Recorder's own is. A tab that reloaded as you switched to it no longer loses the saves +it sends. + +### Your own browser, opened from OpenBot, with the Recorder already in it + +Wherever OpenBot can run a browser for you, the sidebar has a **Browser** button below Connections. It +opens your own browser with the OpenBot Recorder already in it and connected as you: no install and no +Connect step. On OpenBot Desktop one click opens your browser window, or brings it to the front, and +you stay where you are in OpenBot; record with the Recorder's button in that window's toolbar. On +hosted OpenBot the click takes you to the browser in the page. While a recording runs, the Browser +button shows a red dot; hover it for the recording's name and how long it has run. Name a task and press Record; the page shows the recording's name, how long it has run and +how many actions it holds, and after Stop it says what was saved to your Learning space (for example +"7 actions and 1 narrated phrase"), or that nothing was recorded, and says the same after a reload. Nothing a Bot does is recorded. When +your browser cannot open because of how OpenBot is set up, OpenBot tells you to ask an admin (on +single-user OpenBot, such as OpenBot Desktop, to quit and reopen OpenBot), the message stays until you +close it, and the reason is in the server's and the computer's logs. + +- **On OpenBot Desktop** it is your installed Google Chrome, or Microsoft Edge where Chrome is missing, + in a separate OpenBot profile, as its own window; opening it again brings that window to the front. + OpenBot starts it itself rather than through Playwright, so tabs behind the one in front behave as + background tabs, and OpenBot follows the tab you are on. It is an everyday browser: install your password manager from the Chrome Web Store (a password manager's + link to its desktop app works, such as 1Password's), or sign in to Chrome and sync brings your + extensions, bookmarks and passwords; saved passwords use the system keychain. Once you turn narration + on, the Recorder hears your microphone itself, and the page shows whether it is listening, muted or + cannot hear. + Desktop runs this browser as a fourth process beside the server, app and worker, only where Chrome or + Edge is installed. Installation copies the Recorder, prebuilt, out of the release's computer image into + your OpenBot folder, existing installations download their OpenBot files once more after updating, and Stop + and Quit let the browser close cleanly first. Chrome's extensions page lists the Recorder as an + extension it cannot verify, because it is not installed from the Chrome Web Store. +- **On hosted OpenBot** the browser is the computer's Chromium, shown in the page with tabs, an address + bar, back, forward and reload, and you click and type in it straight away. Only the Recorder is + allowed in it. It runs only on a computer of its own (each person's own, as the supervisor and the + sandbox give one): where Bots share one computer, which can run commands, the Browser button is + hidden and the Browser page says why. While you record with narration on, the page transcribes it + with your own browser's on-device speech recognition (Firefox has none, and the page says so; the + narration switch reads off whenever it cannot hear you), and each phrase is placed before the steps + you took while saying it. It reaches the open web, and OpenBot and the Intelligence gateway on their own + ports, never anything else inside the deployment. +- **Only you** can see or use your browser, admins included. It appears in Settings → Browsers as your + OpenBot browser and can be disconnected there, which signs out the Recorder in it at once (a recording + in progress stops and is kept, and goes to Intelligence when you next record). Each open replaces its + previous connection, and Record after a Disconnect connects it again first. +- A person's hosted browser closes after going idle as a Bot's does, and the computer's browser cap + closes the least recently used first; OpenBot Desktop's own window is never closed for being idle. +- On Linux, OpenBot Desktop also finds Chrome and Edge on the `PATH` (Flatpak builds cannot hold the + Recorder and are not used), and the Browser page says why when there is none. On Windows, Stop and + Quit ask the browser to close before ending it. Uninstalling on Windows, and removing the Linux + package, deletes the OpenBot browser's profile and the Recorder's files. +- `OPENBOT_RECORDER_DIR` on a computer is the built Recorder it puts in a person's browser; the computer + image sets it, and without it the computer still serves Bots and refuses to open a person's browser, + saying why. `OPENBOT_BROWSER_URL` on the server runs people's browsers on a separate computer + (OpenBot Desktop sets it). The local Chrome helper (`scripts/start-local-chrome-computer.ts`) now also + starts when only Edge is installed. +- A computer's network rules can allow a site by name on given ports only (`ports` on a domain rule). +- **Each browser on a shared computer follows its own network policy.** Browsers there now identify + themselves to the computer's network filter, so a Bot allowed to reach a service is no longer refused + because another browser on that computer is not, and no browser borrows another's allowances. +- A hosted person's browser says who it is as a hosted Bot's does (no `HeadlessChrome`), but never + carries the deployment's Web Bot Auth signature: a person browsing is not the deployment's Bot. +- A person's browser always runs the Recorder that came with its OpenBot, even in a profile that ran an + earlier one, and keeps it pinned to the toolbar once. +- A Recorder that is slow to answer is refused instead of taking down the computer and every browser + on it. Typing on a live screen that records a demonstration reaches the page in the order typed. +- Opening the Browser page twice in quick succession (reloading it while your browser is still + opening, for one) opens your browser once, connected; before, both opens could be refused, most + often the first after the Recorder was updated. When a browser cannot be opened, the server's and + the computer's logs say why and at which step. +- **Google loads in Bots' browsers again.** Behind the computer's network filter, Google's pages never + finished loading (Chromium restarts the request for client hints and the restart was lost); the + computer now launches every filtered browser with that restart switched off. +- When a page does not open, OpenBot names the site and says it did not load or was refused, instead of + showing a browser error, the address bar keeps the address that failed, and a slow page says it is + slow rather than that the browser could not be reached. +- A package can no longer give a Bot an id shaped like `person-` and 32 hex digits. +- The Recorder times a narrated phrase from its first word when Chrome reuses a result slot; it was + stamped up to seconds early. + ### The desktop window answers while software is installing Clicking or focusing the desktop window while it downloaded local software showed a spinning cursor diff --git a/Dockerfile b/Dockerfile index 65f52759f0..68092a8913 100644 --- a/Dockerfile +++ b/Dockerfile @@ -52,12 +52,15 @@ WORKDIR /src # Manifests first, so editing a source file does not reinstall the world. COPY package.json bun.lock ./ +# The dependency patches package.json names in patchedDependencies: the install stops without them. +COPY patches patches COPY tsconfig.base.json bunfig.toml ./ COPY app/package.json app/package.json COPY server/package.json server/package.json COPY worker/package.json worker/package.json COPY shared/package.json shared/package.json COPY toolkit/package.json toolkit/package.json +COPY extension/package.json extension/package.json RUN bun install --frozen-lockfile # The lockfile travels with the manifest, because `--frozen-lockfile` with no lockfile in the context @@ -70,13 +73,14 @@ RUN cd agent-computer && bun install --frozen-lockfile # A second tree with the build-time dependencies left out, for the runtime stage to take. Vite, # biome and the test tooling are a gigabyte that nothing in a running container imports. -RUN mkdir -p /prod && cp package.json bun.lock /prod/ \ - && cd /prod && mkdir -p app server worker shared toolkit \ +RUN mkdir -p /prod && cp package.json bun.lock /prod/ && cp -R patches /prod/patches \ + && cd /prod && mkdir -p app server worker shared toolkit extension \ && cp /src/app/package.json app/package.json \ && cp /src/server/package.json server/package.json \ && cp /src/worker/package.json worker/package.json \ && cp /src/shared/package.json shared/package.json \ && cp /src/toolkit/package.json toolkit/package.json \ + && cp /src/extension/package.json extension/package.json \ && bun install --frozen-lockfile --production @@ -128,6 +132,8 @@ WORKDIR /app COPY --from=deps /prod/node_modules node_modules COPY --from=deps /prod/server/node_modules server/node_modules COPY --from=deps /prod/toolkit/node_modules toolkit/node_modules +# shared files import zod at runtime, and the isolated install resolves it from shared/node_modules +COPY --from=deps /prod/shared/node_modules shared/node_modules COPY --from=deps /src/package.json package.json COPY --from=deps /src/bun.lock bun.lock # The browser's tree is a separate install root with its own lockfile rather than a workspace of the diff --git a/README.md b/README.md index 46e48f03f2..7d6304c0a0 100644 --- a/README.md +++ b/README.md @@ -64,7 +64,7 @@ A Bot is any endpoint speaking [AG-UI](https://github.com/ag-ui-protocol/ag-ui), ## Requirements - Docker, for PostgreSQL and the shipped Bots. -- [Bun](https://bun.sh) 1.3+, for the app and API server. +- [Bun](https://bun.sh) 1.4+, for the app, the API server and the tests (some tests fail on 1.3). - A CopilotKit Intelligence project and license. A free plan is available, and Intelligence can be self-hosted. - A model key. The proof-of-concept Bot uses OpenAI; the LangGraph Bot can use OpenAI, Anthropic, or Google. diff --git a/agent-computer/Dockerfile b/agent-computer/Dockerfile index f706ec83cd..e63e08a4c3 100644 --- a/agent-computer/Dockerfile +++ b/agent-computer/Dockerfile @@ -1,10 +1,33 @@ -# The Bot's computer installs only the Playwright browser it launches. +# The Bot's computer installs only the Playwright browser it launches, and carries the built OpenBot +# Recorder for the person's browser it opens (the OpenBot Browser). # # The image tag and Playwright dependency must be pinned to the same exact version. Bump both or # neither. FROM node:24.18.1-bookworm-slim AS node-toolchain FROM oven/bun:1.4.3@sha256:ec06c3b6cea04192ae6770c434f668ca41d343ad19fa6472216c7b48be39c598 AS bun-toolchain +# The Recorder, built here from the repository root as the root Dockerfile builds the app: the +# workspace manifests and lockfile first, so a source edit does not reinstall the world, then the +# sources the build reads. It needs the root install because the extension is a workspace and its +# pages are drawn with the app's own components (`app/src`). Only `extension/dist` leaves this stage. +FROM bun-toolchain AS recorder-build +WORKDIR /src +COPY package.json bun.lock ./ +# The dependency patches package.json names in patchedDependencies: the install stops without them. +COPY patches patches +COPY tsconfig.base.json bunfig.toml ./ +COPY app/package.json app/package.json +COPY server/package.json server/package.json +COPY worker/package.json worker/package.json +COPY shared/package.json shared/package.json +COPY toolkit/package.json toolkit/package.json +COPY extension/package.json extension/package.json +RUN bun install --frozen-lockfile +COPY app/src app/src +COPY shared shared +COPY extension extension +RUN bun run --cwd extension build + FROM ubuntu:24.04 # The Bun image digest pins the amd64/arm64 release bytes, including if its tag changes. @@ -32,9 +55,22 @@ COPY agent-computer/package.json agent-computer/bun.lock ./ RUN bun install --frozen-lockfile COPY agent-computer/src ./src -# index.ts imports ../../shared/file-download at runtime (the other shared imports are type-only and -# erased), so the one file it needs goes where that path resolves from /app/src. +# index.ts imports ../../shared/file-download at runtime (shared/computer-control is type-only and +# erased), demonstration.ts imports ../../shared/demonstration-rules, and the browser routes import +# ../../shared/openbot-browser, so the files they need go where those paths resolve from /app/src. COPY shared/file-download.ts /shared/file-download.ts +COPY shared/demonstration-rules.ts /shared/demonstration-rules.ts +# The OpenBot Browser's contract, which the browser routes check requests against. It imports zod, +# and a module resolves its imports from its own directory upward, which from /shared never reaches +# /app/node_modules: so /shared is given that tree to resolve from, rather than a second install. +COPY shared/openbot-browser.ts /shared/openbot-browser.ts +COPY shared/recorder-extension.ts shared/recorder-extension.json /shared/ +RUN ln -s /app/node_modules /shared/node_modules + +# The built Recorder, put into every person's browser this computer opens. Without it the computer +# still serves Bots and refuses to open a person's browser, saying why. +COPY --from=recorder-build /src/extension/dist /recorder +ENV OPENBOT_RECORDER_DIR=/recorder # Durable working directory. Bot-produced files that must survive replacement live here. RUN mkdir -p /workspace diff --git a/agent-computer/bun.lock b/agent-computer/bun.lock index 49615ea92a..f0ec9ceb2f 100644 --- a/agent-computer/bun.lock +++ b/agent-computer/bun.lock @@ -9,6 +9,7 @@ "spiffe": "^0.5.1", "web-bot-auth": "0.2.0", "yaml": "^2.9.0", + "zod": "4.4.3", }, "devDependencies": { "typescript": "^5.9.3", @@ -150,6 +151,8 @@ "yargs-parser": ["yargs-parser@21.1.1", "", {}, "sha512-tVpsJW7DdjecAiFpbIB1e3qxIQsE6NoPc5/eTdrbbIC4h0LVsWhnoa3g+m2HclBIujHzsxZ4VJVA+GUuc2/LBw=="], + "zod": ["zod@4.4.3", "", {}, "sha512-ytENFjIJFl2UwYglde2jchW2Hwm4GJFLDiSXWdTrJQBIN9Fcyp7n4DhxJEiWNAJMV1/BqWfW/kkg71UDcHJyTQ=="], + "tsyringe/tslib": ["tslib@1.14.1", "", {}, "sha512-Xni35NKzjgMrwevysHTCArtLDpPvye8zV/0E4EyYn43P7/7qvQwPh9BGkHewbMulVntbigmcT7rdX3BNo9wRJg=="], } } diff --git a/agent-computer/package.json b/agent-computer/package.json index 52c561299c..fabbc5ab36 100644 --- a/agent-computer/package.json +++ b/agent-computer/package.json @@ -12,7 +12,8 @@ "playwright": "1.62.1", "spiffe": "^0.5.1", "web-bot-auth": "0.2.0", - "yaml": "^2.9.0" + "yaml": "^2.9.0", + "zod": "4.4.3" }, "devDependencies": { "typescript": "^5.9.3" diff --git a/agent-computer/src/browser-eviction.ts b/agent-computer/src/browser-eviction.ts index 8fa9bcc489..be87a70883 100644 --- a/agent-computer/src/browser-eviction.ts +++ b/agent-computer/src/browser-eviction.ts @@ -17,24 +17,64 @@ * starts where it left off, which is already what `stop` means here. */ +import { + type BrowserWindow, + isPersonBrowserId, +} from "../../shared/openbot-browser"; + /** Enough of a live entry for either decision. Keeps this file independent of what else is on one. */ export type Evictable = { usedAt: number }; +/** + * Which browsers the idle sweep leaves alone and the cap closes last, from where people's browsers show here. + * + * A NATIVE WINDOW, AND ONLY A NATIVE WINDOW. A person browsing in a window on their own desktop (OpenBot Desktop) sends + * this process nothing, so nothing here can tell them from a person who walked away, and their browser always looks + * like the one used longest ago: by use alone it would be closed out from under them. A cloud browser is different. + * The person reaches it only through OpenBot's Browser page, which asks for its state every second and streams its + * screen, and every one of those touches it; once nobody is on that page it is a root, sandbox-less Chromium nobody + * can see. So it closes when idle, by the Bots' own rule and timeout, and is ranked by use alongside theirs rather + * than ahead of them. Closing it loses nothing but the open tabs: its profile is on disk and the next open starts + * from it, as a Bot's does. + */ +export function keptOpen(window: BrowserWindow): (botId: string) => boolean { + return window === "native" ? isPersonBrowserId : () => false; +} + /** * Which to close because there are too many running. * * Least recently used first: the Bot that has been quiet longest. Applied after a launch, so the Bot * that just asked is the most recently used and is therefore never the one closed. + * + * A person's native window (`keptOpen`) counts toward the cap like any other and goes last. It is in + * use whenever the person is in front of it, which this process cannot see: browsing in a native + * window sends it nothing, so its last use would read as long ago and it would be the first closed, + * out from under the person, to make room for a Bot. So every Bot's browser is closed before it, and + * it goes only when nothing else is left to close. A person's cloud browser is ranked by use like a + * Bot's. The browser that just launched stays either way, whoever it belongs to. */ export function chooseEvictions( running: Iterable<[string, Evictable]>, max: number, + goesLast: (botId: string) => boolean = () => false, ): string[] { const entries = [...running]; if (entries.length <= max) return []; + // A cap of zero keeps nothing, the newest included; that is the mistake reading the setting guards. + const newest = + max > 0 + ? entries.reduce((latest, entry) => + entry[1].usedAt >= latest[1].usedAt ? entry : latest, + ) + : undefined; return entries - .sort(([, a], [, b]) => a.usedAt - b.usedAt) + .filter((entry) => entry !== newest) + .sort( + ([aId, a], [bId, b]) => + Number(goesLast(aId)) - Number(goesLast(bId)) || a.usedAt - b.usedAt, + ) .slice(0, entries.length - max) .map(([botId]) => botId); } @@ -45,16 +85,22 @@ export function chooseEvictions( * The other half of the answer: a deployment under the cap still holds a browser for a Bot used once * last Tuesday, and that memory is doing nothing for anybody. A timeout of zero or less switches this * off, so a deployment can keep browsers resident if it would rather. + * + * Never a browser `stays` names, which is a person's native window (`keptOpen`). A person using + * their browser in a native window sends this process nothing, so nothing here can tell that from a + * person who walked away, and closing it would close the window they are typing in. It stays open + * until they close it. */ export function chooseIdle( running: Iterable<[string, Evictable]>, idleTimeoutMs: number, now: number, + stays: (botId: string) => boolean = () => false, ): string[] { if (idleTimeoutMs <= 0) return []; const cutoff = now - idleTimeoutMs; return [...running] - .filter(([, entry]) => entry.usedAt <= cutoff) + .filter(([botId, entry]) => entry.usedAt <= cutoff && !stays(botId)) .map(([botId]) => botId); } diff --git a/agent-computer/src/browser-host.ts b/agent-computer/src/browser-host.ts new file mode 100644 index 0000000000..45679d5bc5 --- /dev/null +++ b/agent-computer/src/browser-host.ts @@ -0,0 +1,742 @@ +/** + * The OpenBot Browser's routes on this computer: open a person's browser with the Recorder in it and + * connected, and act for the person in it (record, narrate, tabs, navigate, close). + * + * Under the computer token like every other route, with the person's browser id in + * `x-openbot-bot-id`; `index.ts` checks the token, this checks the id. The server is the only caller: + * it decides who the signed-in person is, mints their Recorder connection, and forwards here. The + * shapes are the contract in `shared/openbot-browser.ts`, which every slice builds against. + * + * Every answer but narration's is the browser's state, so the caller never has to ask twice to show + * what a change did. + * + * Its own module, taking the browsers it acts on, so a test can serve these exact routes over a + * person's browser launched headless, which the computer's own environment refuses for a native one. + */ +import type { BrowserContext, Page } from "playwright"; +import { + type BrowserOpenRequest, + type BrowserState, + type BrowserTab, + browserComputerPaths, + browserNarrationRequestSchema, + browserNavigateRequestSchema, + browserOpenRequestSchema, + browserRecordingRequestSchema, + browserTabsRequestSchema, + isPersonBrowserId, + type LauncherState, + type Narration, +} from "../../shared/openbot-browser"; +import { navigationFailure } from "./navigation"; +import { BROWSER_NOT_OPEN, BrowserNotOpenError } from "./person-access"; +import { + NoInstalledBrowserError, + PersonOpenError, + type PersonOpenStep, +} from "./person-browser"; +import type { OpenPersonBrowser, Profiles } from "./profiles"; +import { findRecorder } from "./recorder"; +import { RecorderUnavailableError } from "./recorder-launcher"; +import { addressFrom } from "./request-validation"; + +export type BrowserHostOptions = { + profiles: Profiles; + /** `OPENBOT_RECORDER_DIR`, read at every open so a rebuilt Recorder is the one the next browser gets. */ + recorderDir: () => string | undefined; + navigationTimeoutMs: number; +}; + +/** What the person is told when their browser did not open for a reason that is not the computer's setup: the reason + * itself (often a page of Playwright's own text) goes to the computer's log. */ +const NOT_OPENED = "Your browser could not be opened. Try again."; + +/** How long one tab may take to say whether it is showing, before it is left out of the question. */ +const VISIBILITY_MS = 1_000; + +/** + * How long the browser has to say what it is showing before the state route answers that it is not answering. + * + * A browser that answers at all answers this in milliseconds: the end-to-end run measured 2 to 19 ms. One whose main + * thread is stuck never answers DevTools at all, and the route used to wait for it for as long as its caller did: + * OpenBot gave up after 45 s, after its own server had already closed the person's request with an empty reply at 10. + * The Browser page asks every second, and then had no browser to show and no sentence, and the sidebar lost its Browser + * button. Bounded well under OpenBot's own wait for this route (server/src/browser/routes.ts, STATE_TIMEOUT_MS), so the + * person hears it from here, where it is known that the browser is open and not answering. + */ +export const STATE_ANSWER_MS = 5_000; +/** + * How long the state waits for the Recorder's own report. Within {@link STATE_ANSWER_MS}: a Recorder still starting, or + * busy, is left out of the answer rather than making a browser that answers look like one that does not. The page asks + * again a second later, and a Recorder that has started by then is in that answer. + */ +const RECORDER_REPORT_MS = 3_000; + +/** What the state route answers, for OpenBot's log, when the browser did not answer in time. */ +export const NOT_ANSWERING = `The browser did not answer within ${STATE_ANSWER_MS / 1000} s.`; + +/** The work's value if it settles within `ms`, else nothing; it is left running, and a rejection is the caller's. */ +async function within( + work: Promise, + ms: number, +): Promise<{ value: T } | undefined> { + let timer: ReturnType | undefined; + const late = new Promise((resolve) => { + timer = setTimeout(() => resolve(undefined), ms); + }); + try { + return await Promise.race([work.then((value) => ({ value })), late]); + } finally { + clearTimeout(timer); + } +} + +/** + * Where Chromium's error page lives. A tab whose page did not load shows it, and Playwright reports + * the error page's own address, `chrome-error://chromewebdata/`, as the tab's. The tab itself still + * names the address that failed, as a browser's address bar does: DevTools' target for the tab says + * so. Measured on Playwright 1.62.1's Chromium for a name that does not resolve and a port that + * refuses. + */ +const ERROR_PAGE = "chrome-error://"; +/** How long a failed navigation waits for the tab to show its error page. */ +const ERROR_PAGE_MS = 2_000; + +function json(body: unknown, status = 200): Response { + return new Response(JSON.stringify(body), { + status, + headers: { "content-type": "application/json" }, + }); +} + +/** Enough of a contract schema to check a body with, whichever copy of zod built it. */ +type Schema = { + safeParse(input: unknown): + | { success: true; data: T } + | { + success: false; + error: { issues: { path: PropertyKey[]; message: string }[] }; + }; +}; + +/** The first thing wrong with a body, in the words of the field it is about. */ +function parseBody( + schema: Schema, + body: unknown, +): { ok: true; value: T } | { ok: false; error: string; response: Response } { + const parsed = schema.safeParse(body); + if (parsed.success) return { ok: true, value: parsed.data }; + const issue = parsed.error.issues[0]; + const where = issue?.path.length + ? `${issue.path.map(String).join(".")}: ` + : ""; + const error = `${where}${issue?.message ?? "That request is not valid."}`; + return { ok: false, error, response: json({ error }, 400) }; +} + +const messageOf = (error: unknown): string => + error instanceof Error ? error.message : String(error); + +/** A person's browser's opens: how many have arrived and not finished, and the newest connection any brought. */ +type Opening = { + /** Settles when the last open to arrive has finished, so the next waits for it. */ + turn: Promise; + pending: number; + /** None while no open among them brought one: each then leaves the Recorder the connection it holds. */ + newest?: NonNullable; +}; + +export function createBrowserHost(options: BrowserHostOptions) { + const { profiles } = options; + /** Each tab's DevTools target id, which stays the same for the life of the tab. */ + const tabIds = new WeakMap(); + /** How many documents each tab's main frame has committed, counted from when it was first asked. */ + const commits = new WeakMap(); + + const commitsOf = (page: Page): number => { + let counted = commits.get(page); + if (!counted) { + const counter = { count: 0 }; + page.on("framenavigated", (frame) => { + if (frame === page.mainFrame()) counter.count += 1; + }); + commits.set(page, counter); + counted = counter; + } + return counted.count; + }; + + const tabIdOf = async ( + browser: OpenPersonBrowser, + page: Page, + ): Promise => { + const known = tabIds.get(page); + if (known) return known; + const session = await browser.context.newCDPSession(page); + try { + const { targetInfo } = await session.send("Target.getTargetInfo"); + tabIds.set(page, targetInfo.targetId); + return targetInfo.targetId; + } finally { + await session.detach().catch(() => undefined); + } + }; + + const pageFor = async ( + browser: OpenPersonBrowser, + tabId: string, + ): Promise => { + for (const page of browser.pages()) { + if ((await tabIdOf(browser, page).catch(() => undefined)) === tabId) + return page; + } + return undefined; + }; + + /** + * In a native window the person changes tabs in the browser itself, which tells this process + * nothing, so the tab in front is asked of the pages: the one the browser is showing, and of two + * windows showing a tab each, the one with the focus. With no window in focus (the person is in + * OpenBot), the tab already in front stays while it still shows; and a minimised browser, where + * none shows, keeps the one it had. + * + * This reads true only because a native window has no focus emulation (`native-browser.ts`): with + * it, every tab said it was showing and focused, and the person's own switch was never seen. + */ + const followNativeTabs = async (browser: OpenPersonBrowser) => { + if (browser.window !== "native") return; + const pages = browser.pages(); + const hidden = { showing: false, focused: false }; + const shown = await Promise.all( + pages.map((page) => + Promise.race([ + page + .evaluate(() => ({ + showing: document.visibilityState === "visible", + focused: document.hasFocus(), + })) + .catch(() => hidden), + new Promise