diff --git a/crates/code_assistant_core/src/tools/impls/browser.rs b/crates/code_assistant_core/src/tools/impls/browser.rs deleted file mode 100644 index 0140d476..00000000 --- a/crates/code_assistant_core/src/tools/impls/browser.rs +++ /dev/null @@ -1,2165 +0,0 @@ -//! Browser agency tools. -//! -//! These drive a real browser across tool calls so the agent can operate web -//! software: test an app under development, or act on a portal under the user's -//! identity. State lives in the session-scoped `web::BrowserSessionManager` -//! (see [`crate::tools::services`]); one live browser is kept per **profile** -//! name, so the model addresses "the elster browser" by name instead of -//! juggling session ids. -//! -//! - `browser_navigate` — open-or-reuse a profile's browser and go to a URL. -//! - `browser_read` — re-observe the current page without acting. -//! - `browser_act` — click / type / press / wait, a sequence per call. -//! - `browser_close` — close a profile's browser (flushing a persistent one). -//! - `browser_profiles` — list the persistent profiles on disk and how long ago -//! each was last logged in, so a fresh session can reuse an existing login. -//! - `browser_login` — headful human-in-the-loop login handoff: opens a visible -//! window, pauses on the `PermissionMediator` seam for the user to log in, -//! then swaps that window for a headless browser on the same profile — -//! transferring the cookie jar so the login survives — and resumes -//! authenticated. The user can close the login window without killing the -//! session, and the model never sees credentials. -//! -//! Every tool returns a screenshot (the model's eyes, via `render_images`) -//! plus the page's url/title/text. - -use crate::tools::core::{ - ImageData, Render, ResourcesTracker, Tool, ToolContext, ToolResult, ToolSpec, cap_base64_image, - capabilities, -}; -use crate::tools::services::ToolServicesAccess; -use anyhow::Result; -use base64::Engine; -use serde::{Deserialize, Serialize}; -use serde_json::json; -use std::path::{Path, PathBuf}; -use std::sync::Arc; -use std::time::{Duration, SystemTime, UNIX_EPOCH}; -use tools_core::permissions::{ - PermissionDecision, PermissionMediator, PermissionRequest, PermissionRequestReason, -}; -use web::{ - BrowserLaunchConfig, BrowserProfile, BrowserSession, BrowserSessionManager, BrowserTimeout, - PageObservation, -}; - -/// The profile label used when the model does not name one: a single reusable -/// ephemeral (throwaway) browser. -pub(crate) const DEFAULT_PROFILE: &str = "default"; - -/// Resolve a profile name to a launch config. The reserved `"default"` name (or -/// `None`) is an ephemeral throwaway browser; any other name is a persistent -/// profile under `/browser-profiles/`, so a login can be -/// reused across runs. -pub(crate) fn launch_config_for(profile: &str, headful: bool) -> BrowserLaunchConfig { - if profile == DEFAULT_PROFILE { - return BrowserLaunchConfig { - profile: BrowserProfile::Ephemeral, - headful, - }; - } - let dir = profiles_root().join(sanitize_profile(profile)); - BrowserLaunchConfig { - profile: BrowserProfile::Persistent(dir), - headful, - } -} - -/// Get the live browser for `profile`, opening one if none exists yet. -pub(crate) async fn get_or_open( - manager: &BrowserSessionManager, - profile: &str, - headful: bool, -) -> Result> { - if let Some(session) = manager.get_by_label(profile) { - return Ok(session); - } - let session = - Arc::new(BrowserSession::open(launch_config_for(profile, headful), profile).await?); - manager.register(session.clone(), profile); - Ok(session) -} - -/// Shared output of every browser tool: what the page looks like now. -#[derive(Serialize, Deserialize)] -pub struct BrowserOutput { - pub profile: String, - #[serde(skip_serializing_if = "Option::is_none")] - pub observation: Option, - /// Base64 PNG screenshot, surfaced to the model as an image via - /// `render_images`. - #[serde(skip_serializing_if = "Option::is_none")] - pub screenshot_base64: Option, - #[serde(skip_serializing_if = "Option::is_none")] - pub error: Option, - /// Things the model should know that are not failures, e.g. a page that - /// did not finish loading but is shown as far as it got. - #[serde(default, skip_serializing_if = "Vec::is_empty")] - pub notes: Vec, -} - -impl BrowserOutput { - fn failure(profile: &str, error: impl Into) -> Self { - Self { - profile: profile.to_string(), - observation: None, - screenshot_base64: None, - error: Some(error.into()), - notes: Vec::new(), - } - } - - /// The standard output when this context has no browser registry. - fn unavailable(profile: &str) -> Self { - Self::failure( - profile, - "Browser tools are not available in this context (no browser session registry).", - ) - } - - /// Observe + screenshot the session into a success output. Any capture - /// error is folded into `error` rather than failing the whole tool call. - /// `include_text` false suppresses the (often large, step-to-step - /// redundant) page-text dump, keeping the screenshot and element list. - async fn capture( - profile: &str, - session: &BrowserSession, - full_page: bool, - include_text: bool, - ) -> Self { - // Let a navigation triggered by the preceding action settle first, so - // the text (`observe`) and the screenshot show the same page rather - // than racing a mid-transition document. - session.settle().await; - let observation = match session.observe_with(include_text).await { - Ok(observation) => Some(observation), - // A page that cannot be read will not render a screenshot either; - // report it now instead of waiting out a second timeout. - Err(e) if e.downcast_ref::().is_some() => { - return Self::failure( - profile, - format!( - "The page is not responding ({e}). It may be busy or stuck in a \ - script; browser_navigate loads it again." - ), - ); - } - // Other failures (e.g. a document torn down mid-navigation) still - // leave the screenshot worth taking. - Err(_) => None, - }; - let screenshot_base64 = match session.screenshot(full_page).await { - Ok(png) => Some(base64::engine::general_purpose::STANDARD.encode(png)), - Err(_) => None, - }; - Self { - profile: profile.to_string(), - observation, - screenshot_base64, - error: None, - notes: Vec::new(), - } - } -} - -impl Render for BrowserOutput { - fn status(&self) -> String { - if let Some(e) = &self.error { - return format!("Browser error: {e}"); - } - match &self.observation { - Some(obs) => format!("{} — {}", obs.url, obs.title), - None => "Browser action completed".to_string(), - } - } - - fn render(&self, _tracker: &mut ResourcesTracker) -> String { - if let Some(e) = &self.error { - let mut out = format!("Browser error: {e}"); - if let Some(obs) = &self.observation { - push_dialog_notes(&mut out, obs); - } - return out; - } - let Some(obs) = &self.observation else { - return "Browser action completed (no page observed).".to_string(); - }; - // Keep the textual dump bounded; the screenshot carries the visual - // detail. - let mut text = obs.text.trim().to_string(); - const MAX: usize = 4000; - if text.len() > MAX { - text.truncate(MAX); - text.push_str("\n… (truncated; see screenshot for the rest)"); - } - let mut out = format!( - "Profile: {}\nURL: {}\nTitle: {}", - self.profile, obs.url, obs.title - ); - // Disclose the viewport size (CSS px) so the model can address px - // coordinates; it can't read the true size off the resized screenshot. - if obs.viewport_width > 0.0 && obs.viewport_height > 0.0 { - out.push_str(&format!( - "\nViewport: {}×{} (CSS px)", - obs.viewport_width as i64, obs.viewport_height as i64 - )); - } - push_dialog_notes(&mut out, obs); - for note in &self.notes { - out.push_str(&format!("\nNote: {note}")); - } - out.push_str("\n\n"); - out.push_str(&text); - // List the actionable elements with their selectors, so the model can - // target them directly with browser_act instead of guessing. - if !obs.elements.is_empty() { - out.push_str("\n\nInteractive elements:"); - for el in &obs.elements { - if el.label.is_empty() { - out.push_str(&format!("\n [{}] {}", el.role, el.selector)); - } else { - out.push_str(&format!( - "\n [{}] \"{}\" → {}", - el.role, el.label, el.selector - )); - } - } - } - out - } - - fn render_images(&self) -> Vec { - // An error tool result must be text-only (Anthropic rejects images in a - // tool_result with is_error=true). Some error paths still capture a - // screenshot for context (e.g. browser_act showing where a sequence - // stopped); drop it here so the result stays valid — the failing - // step's text still explains what happened. - if self.error.is_some() { - return Vec::new(); - } - self.screenshot_base64 - .iter() - .map(|data| ImageData { - media_type: "image/png".to_string(), - base64_data: data.clone(), - }) - .collect() - } - - fn cap_images(&mut self, max_edge: u32) { - if let Some(data) = self.screenshot_base64.take() { - let capped = cap_base64_image("image/png", &data, max_edge) - .map(|(_media_type, capped)| capped) - .unwrap_or(data); - self.screenshot_base64 = Some(capped); - } - } -} - -impl ToolResult for BrowserOutput { - fn is_success(&self) -> bool { - self.error.is_none() - } -} - -/// Report JavaScript dialogs the session answered on its own, so the model -/// knows a confirm was cancelled (or accepted) on its behalf. -fn push_dialog_notes(out: &mut String, obs: &PageObservation) { - for dialog in &obs.dialogs { - let verdict = if dialog.accepted { - "accepted" - } else { - "dismissed" - }; - out.push_str(&format!( - "\nNote: a {} dialog \"{}\" was {verdict}.", - dialog.kind, dialog.message - )); - } -} - -/// Navigate, treating a load that does not finish in time as a note rather than -/// a failure: the page is usually usable, and a slow iframe or tracker is what -/// holds back its `load` event. Returns that note, if any. -async fn navigate_tolerating_slow_load( - session: &BrowserSession, - url: &str, -) -> Result> { - match session.navigate(url).await { - Ok(()) => Ok(None), - Err(e) if e.downcast_ref::().is_some() => Ok(Some(format!( - "the page did not finish loading ({e}); it is shown as far as it got." - ))), - Err(e) => Err(e), - } -} - -/// Common scope tags for browser tools (agent + default sub-agents). -fn agent_scopes() -> Vec<&'static str> { - vec![ - capabilities::SCOPE_AGENT, - capabilities::SCOPE_AGENT_DIFF, - capabilities::SCOPE_SUBAGENT_DEFAULT, - capabilities::SCOPE_SUBAGENT_DEFAULT_DIFF, - ] -} - -// --------------------------------------------------------------------------- -// browser_navigate -// --------------------------------------------------------------------------- - -#[derive(Deserialize, Serialize)] -pub struct BrowserNavigateInput { - pub url: String, - #[serde(default, skip_serializing_if = "Option::is_none")] - pub profile: Option, -} - -pub struct BrowserNavigateTool; - -#[async_trait::async_trait] -impl Tool for BrowserNavigateTool { - type Input = BrowserNavigateInput; - type Output = BrowserOutput; - - fn spec(&self) -> ToolSpec { - let mut caps = vec![capabilities::READ_ONLY]; - caps.extend(agent_scopes()); - caps.push(capabilities::SCOPE_SUBAGENT_READ_ONLY); - ToolSpec { - name: "browser_navigate".into(), - description: concat!( - "Open (or reuse) a browser and navigate to a URL, then return a screenshot ", - "and the page's text. Use this to try out and inspect web software.\n", - "Pass `profile` to use a persistent, named browser whose login/cookies ", - "survive across runs (e.g. \"elster\"); omit it for a throwaway browser. ", - "To log in to a persistent profile, use `browser_login` first." - ) - .into(), - parameters_schema: json!({ - "type": "object", - "properties": { - "url": {"type": "string", "description": "URL to navigate to"}, - "profile": { - "type": "string", - "description": "Named persistent profile to reuse a login; omit for a throwaway browser" - } - }, - "required": ["url"] - }), - annotations: Some(json!({"readOnlyHint": true, "openWorldHint": true})), - capabilities: ToolSpec::capabilities(&caps), - multiline_params: &[], - hidden: false, - title_template: Some("Navigating to {url}"), - } - } - - async fn execute<'a>( - &self, - context: &mut ToolContext<'a>, - input: &mut Self::Input, - ) -> Result { - let profile = input - .profile - .as_deref() - .unwrap_or(DEFAULT_PROFILE) - .to_string(); - let Some(manager) = context.browser_sessions() else { - return Ok(BrowserOutput::unavailable(&profile)); - }; - let session = match get_or_open(manager, &profile, false).await { - Ok(s) => s, - Err(e) => { - return Ok(BrowserOutput::failure( - &profile, - format!("Failed to open browser: {e}"), - )); - } - }; - let note = match navigate_tolerating_slow_load(&session, &input.url).await { - Ok(note) => note, - Err(e) => { - return Ok(BrowserOutput::failure( - &profile, - format!("Navigation failed: {e}"), - )); - } - }; - let mut out = BrowserOutput::capture(&profile, &session, false, true).await; - out.notes.extend(note); - Ok(out) - } -} - -// --------------------------------------------------------------------------- -// browser_read -// --------------------------------------------------------------------------- - -#[derive(Deserialize, Serialize)] -pub struct BrowserReadInput { - #[serde(default, skip_serializing_if = "Option::is_none")] - pub profile: Option, - /// Capture the entire scrollable page instead of just the viewport. - #[serde(default, skip_serializing_if = "std::ops::Not::not")] - pub full_page: bool, - /// Omit the page's full text dump, returning only the screenshot and the - /// interactive-element list. Cuts token cost on long forms where the text - /// barely changes between steps. - #[serde(default, skip_serializing_if = "std::ops::Not::not")] - pub no_text: bool, -} - -pub struct BrowserReadTool; - -#[async_trait::async_trait] -impl Tool for BrowserReadTool { - type Input = BrowserReadInput; - type Output = BrowserOutput; - - fn spec(&self) -> ToolSpec { - let mut caps = vec![capabilities::READ_ONLY]; - caps.extend(agent_scopes()); - caps.push(capabilities::SCOPE_SUBAGENT_READ_ONLY); - ToolSpec { - name: "browser_read".into(), - description: concat!( - "Re-observe the current page of a browser profile without acting: returns a ", - "fresh screenshot and the page text. Use after something on the page changes." - ) - .into(), - parameters_schema: json!({ - "type": "object", - "properties": { - "profile": {"type": "string", "description": "Profile to read; omit for the throwaway browser"}, - "full_page": {"type": "boolean", "description": "Capture the whole scrollable page instead of just the viewport"}, - "no_text": {"type": "boolean", "description": "Omit the page-text dump; return only the screenshot and the interactive-element list (saves tokens on long forms)"} - } - }), - annotations: Some(json!({"readOnlyHint": true})), - capabilities: ToolSpec::capabilities(&caps), - multiline_params: &[], - hidden: false, - title_template: Some("Reading browser page"), - } - } - - async fn execute<'a>( - &self, - context: &mut ToolContext<'a>, - input: &mut Self::Input, - ) -> Result { - let profile = input - .profile - .as_deref() - .unwrap_or(DEFAULT_PROFILE) - .to_string(); - let Some(manager) = context.browser_sessions() else { - return Ok(BrowserOutput::unavailable(&profile)); - }; - match manager.get_by_label(&profile) { - Some(session) => { - Ok( - BrowserOutput::capture(&profile, &session, input.full_page, !input.no_text) - .await, - ) - } - None => Ok(BrowserOutput::failure( - &profile, - "No open browser for this profile. Use browser_navigate first.", - )), - } - } -} - -// --------------------------------------------------------------------------- -// browser_act -// --------------------------------------------------------------------------- - -/// One interaction step. A `browser_act` call runs a sequence of these. -#[derive(Deserialize, Serialize)] -#[serde(rename_all = "snake_case")] -pub enum BrowserAction { - /// Click the first element matching the CSS selector. - Click { selector: String }, - /// Focus a field and type text into it, appending to any existing content. - /// Not for credentials — use `browser_login` for those. Prefer `fill` to - /// replace a prefilled field. - Type { selector: String, text: String }, - /// Replace a field's content: clear it, then type `text`. The editing verb - /// for prefilled inputs (no manual clearing needed). - Fill { selector: String, text: String }, - /// Empty a field's current content. - Clear { selector: String }, - /// Press a key or chord (e.g. "Enter", "Meta+A", "Control+a", "Shift+Tab") - /// on the element matching the selector. The element is focused first. Omit - /// `selector` to send the key to whatever is currently focused (e.g. arrow - /// keys for a focused game canvas, or a chord for select-all). - Press { - #[serde(default)] - selector: Option, - key: String, - }, - /// Wait until an element appears (or the timeout elapses). - WaitFor { - selector: String, - #[serde(default)] - timeout_ms: Option, - }, - /// Scroll: with `selector` and no delta, scroll that element into view; - /// with `selector` **and** a `(dx, dy)` delta, scroll *inside* that element - /// (for a modal/dialog with its own scroll container); with no selector, - /// scroll the page by `(dx, dy)` pixels (positive `dy` scrolls down). - Scroll { - #[serde(default)] - selector: Option, - #[serde(default)] - dx: Option, - #[serde(default)] - dy: Option, - }, - /// Click at a coordinate. Each value carries a CSS unit: `"40vw"`/`"50%"` - /// (fraction of the viewport axis) or `"640px"` (exact CSS pixels — the - /// viewport size is shown in a read). For canvas/WebGL surfaces and anything - /// without a stable selector (games, maps, drag targets). - ClickAt { x: String, y: String }, - /// Move the mouse to a coordinate (same unit rules as `click_at`) without - /// clicking — drives hover states and canvas pointer-move handlers. - MoveAt { x: String, y: String }, -} - -/// Which viewport axis a coordinate is measured against. -#[derive(Clone, Copy)] -enum Axis { - X, - Y, -} - -/// Resolve a unit-bearing coordinate (`"40vw"`, `"50%"`, `"640px"`) to CSS -/// pixels along `axis`, given the viewport size. `vw`/`vh` are always width/ -/// height; `%` follows the axis; `px` passes through. Typographic units -/// (`rem`/`em`) and bare numbers are rejected so the model states a groundable -/// unit. The result is clamped to the viewport so a click can't land off-page. -fn resolve_coord(value: &str, axis: Axis, vw: f64, vh: f64) -> Result { - let s = value.trim(); - let split = s - .find(|c: char| c.is_alphabetic() || c == '%') - .unwrap_or(s.len()); - let (num_str, unit) = s.split_at(split); - let num: f64 = num_str.trim().parse().map_err(|_| { - anyhow::anyhow!("invalid coordinate '{value}' (expected e.g. \"40vw\", \"50%\", \"640px\")") - })?; - let px = match unit.trim() { - "px" => num, - "vw" => num / 100.0 * vw, - "vh" => num / 100.0 * vh, - "%" => match axis { - Axis::X => num / 100.0 * vw, - Axis::Y => num / 100.0 * vh, - }, - "" => { - return Err(anyhow::anyhow!( - "coordinate '{value}' needs a unit — use vw/vh/%/px" - )); - } - other => { - return Err(anyhow::anyhow!( - "unit '{other}' not supported for coordinates; use vw/vh/%/px" - )); - } - }; - let max = match axis { - Axis::X => vw, - Axis::Y => vh, - }; - Ok(px.clamp(0.0, max)) -} - -#[derive(Deserialize, Serialize)] -pub struct BrowserActInput { - pub actions: Vec, - #[serde(default, skip_serializing_if = "Option::is_none")] - pub profile: Option, - /// Omit the page's full text dump from the result, returning only the - /// screenshot and interactive-element list. Cuts token cost when stepping - /// through a long form whose text barely changes. - #[serde(default, skip_serializing_if = "std::ops::Not::not")] - pub no_text: bool, - /// Accept `confirm`/`prompt` dialogs raised during this call instead of - /// dismissing them (the default). - #[serde(default, skip_serializing_if = "std::ops::Not::not")] - pub accept_dialogs: bool, -} - -pub struct BrowserActTool; - -impl BrowserActTool { - async fn run_action(session: &BrowserSession, action: &BrowserAction) -> Result<()> { - match action { - BrowserAction::Click { selector } => session.click(selector).await, - BrowserAction::Type { selector, text } => session.type_text(selector, text).await, - BrowserAction::Fill { selector, text } => session.fill(selector, text).await, - BrowserAction::Clear { selector } => session.clear(selector).await, - BrowserAction::Press { selector, key } => match selector { - Some(sel) => session.press_key(sel, key).await, - None => session.press_key_global(key).await, - }, - BrowserAction::WaitFor { - selector, - timeout_ms, - } => { - let timeout = Duration::from_millis(timeout_ms.unwrap_or(5000)); - let appeared = session.wait_for(selector, timeout).await?; - if appeared { - Ok(()) - } else { - Err(anyhow::anyhow!("timed out waiting for '{selector}'")) - } - } - BrowserAction::Scroll { selector, dx, dy } => { - session - .scroll(selector.as_deref(), dx.unwrap_or(0.0), dy.unwrap_or(0.0)) - .await - } - BrowserAction::ClickAt { x, y } => { - let (vw, vh) = session.viewport_size().await?; - let px = resolve_coord(x, Axis::X, vw, vh)?; - let py = resolve_coord(y, Axis::Y, vw, vh)?; - session.click_at(px, py).await - } - BrowserAction::MoveAt { x, y } => { - let (vw, vh) = session.viewport_size().await?; - let px = resolve_coord(x, Axis::X, vw, vh)?; - let py = resolve_coord(y, Axis::Y, vw, vh)?; - session.move_mouse(px, py).await - } - } - } -} - -#[async_trait::async_trait] -impl Tool for BrowserActTool { - type Input = BrowserActInput; - type Output = BrowserOutput; - - fn spec(&self) -> ToolSpec { - ToolSpec { - name: "browser_act".into(), - description: concat!( - "Interact with the current page of a browser profile: a sequence of ", - "click / type / fill / clear / press / wait_for / scroll / click_at / move_at ", - "steps, executed in order. Returns a screenshot and text of the resulting page.\n", - "Each action is an object with one key: {\"click\": {\"selector\": \"#go\"}}, ", - "{\"type\": {\"selector\": \"#user\", \"text\": \"hello\"}} (appends), ", - "{\"fill\": {\"selector\": \"#user\", \"text\": \"hello\"}} (REPLACES the field's ", - "content — use this to edit a prefilled field), ", - "{\"clear\": {\"selector\": \"#user\"}} (empty a field), ", - "{\"press\": {\"selector\": \"#user\", \"key\": \"Enter\"}} — `key` may be a chord ", - "like \"Meta+A\", \"Control+a\" or \"Shift+Tab\"; omit selector to send the key to ", - "whatever is focused (e.g. arrow keys for a game canvas), ", - "{\"wait_for\": {\"selector\": \"#result\", \"timeout_ms\": 5000}}, ", - "{\"scroll\": {\"dy\": 600}} (scroll the page down 600px), ", - "{\"scroll\": {\"selector\": \"#footer\"}} (scroll an element into view), ", - "{\"scroll\": {\"selector\": \"#dialog\", \"dy\": 400}} (scroll INSIDE a modal/dialog ", - "with its own scroll container), ", - "{\"click_at\": {\"x\": \"40vw\", \"y\": \"30vh\"}} (click at a coordinate, for ", - "canvas/WebGL surfaces without selectors), or {\"move_at\": {\"x\": \"40vw\", ", - "\"y\": \"30vh\"}} (move the mouse for hover/pointer-move).\n", - "Selectors are CSS by default, but you can also target by visible text/role, ", - "which survives a re-render that changes hashed ids: \"text=Speichern\" (visible ", - "text/label, substring, case-insensitive), \"role=button\" or ", - "\"role=button[name=Save]\", \"aria=Save\" (aria-label). Elements are scrolled into ", - "view automatically before acting.\n", - "Coordinate values carry a CSS unit — think about which unit you mean: \"40vw\"/", - "\"30vh\" or \"50%\" express a fraction of the viewport axis (robust — use these ", - "when eyeballing from the screenshot); \"640px\" is exact CSS pixels (only when you ", - "know the size — the read output shows the Viewport dimensions). rem/em are not ", - "accepted. Prefer selectors when available (see the interactive-element list from a ", - "read); use coordinates only for canvas/game surfaces. ", - "Pass \"no_text\": true to omit the page-text dump from the result (screenshot and ", - "element list only) to save tokens on long forms. ", - "JavaScript dialogs are answered automatically and reported in the result: alerts ", - "are acknowledged, confirm/prompt dialogs are dismissed (Cancel) unless you pass ", - "\"accept_dialogs\": true for that call. ", - "Do not type passwords or 2FA codes here — use browser_login." - ) - .into(), - parameters_schema: json!({ - "type": "object", - "properties": { - "actions": { - "type": "array", - "description": "Ordered interaction steps", - "items": { - "type": "object", - "properties": { - "click": {"type": "object", "properties": {"selector": {"type": "string"}}, "required": ["selector"]}, - "type": {"type": "object", "properties": {"selector": {"type": "string"}, "text": {"type": "string"}}, "required": ["selector", "text"]}, - "fill": {"type": "object", "properties": {"selector": {"type": "string"}, "text": {"type": "string"}}, "required": ["selector", "text"]}, - "clear": {"type": "object", "properties": {"selector": {"type": "string"}}, "required": ["selector"]}, - "press": {"type": "object", "properties": {"selector": {"type": "string"}, "key": {"type": "string", "description": "A key or chord, e.g. \"Enter\", \"Meta+A\", \"Control+a\", \"Shift+Tab\""}}, "required": ["key"]}, - "wait_for": {"type": "object", "properties": {"selector": {"type": "string"}, "timeout_ms": {"type": "integer"}}, "required": ["selector"]}, - "scroll": {"type": "object", "properties": {"selector": {"type": "string"}, "dx": {"type": "number"}, "dy": {"type": "number"}}}, - "click_at": {"type": "object", "properties": {"x": {"type": "string", "description": "x coordinate with CSS unit: vw/% (of width) or px"}, "y": {"type": "string", "description": "y coordinate with CSS unit: vh/% (of height) or px"}}, "required": ["x", "y"]}, - "move_at": {"type": "object", "properties": {"x": {"type": "string"}, "y": {"type": "string"}}, "required": ["x", "y"]} - } - } - }, - "profile": {"type": "string", "description": "Profile to act on; omit for the throwaway browser"}, - "no_text": {"type": "boolean", "description": "Omit the page-text dump from the result (screenshot and element list only)"}, - "accept_dialogs": {"type": "boolean", "description": "Accept (OK) confirm/prompt dialogs raised by this call instead of dismissing them"} - }, - "required": ["actions"] - }), - annotations: Some(json!({"readOnlyHint": false})), - capabilities: ToolSpec::capabilities(&agent_scopes()), - multiline_params: &[], - hidden: false, - title_template: Some("Interacting with the browser"), - } - } - - async fn execute<'a>( - &self, - context: &mut ToolContext<'a>, - input: &mut Self::Input, - ) -> Result { - let profile = input - .profile - .as_deref() - .unwrap_or(DEFAULT_PROFILE) - .to_string(); - let Some(manager) = context.browser_sessions() else { - return Ok(BrowserOutput::unavailable(&profile)); - }; - let Some(session) = manager.get_by_label(&profile) else { - return Ok(BrowserOutput::failure( - &profile, - "No open browser for this profile. Use browser_navigate first.", - )); - }; - - // The opt-in covers this call only, including dialogs raised while - // capturing its result. - session.set_accept_dialogs(input.accept_dialogs); - let mut failure = None; - for (i, action) in input.actions.iter().enumerate() { - if let Err(e) = Self::run_action(&session, action).await { - failure = Some(format!("Action {} failed: {e}", i + 1)); - break; - } - } - // On failure, capture the page as it stands so the model can see where - // the sequence stopped, but report the failing step. - let mut out = BrowserOutput::capture(&profile, &session, false, !input.no_text).await; - session.set_accept_dialogs(false); - out.error = match (failure, out.error.take()) { - (Some(failure), Some(capture)) => Some(format!("{failure}. {capture}")), - (failure, capture) => failure.or(capture), - }; - Ok(out) - } -} - -// --------------------------------------------------------------------------- -// browser_close -// --------------------------------------------------------------------------- - -#[derive(Deserialize, Serialize)] -pub struct BrowserCloseInput { - #[serde(default, skip_serializing_if = "Option::is_none")] - pub profile: Option, -} - -pub struct BrowserCloseTool; - -#[async_trait::async_trait] -impl Tool for BrowserCloseTool { - type Input = BrowserCloseInput; - type Output = BrowserOutput; - - fn spec(&self) -> ToolSpec { - ToolSpec { - name: "browser_close".into(), - description: - "Close a browser profile's window, flushing a persistent profile's session to disk." - .into(), - parameters_schema: json!({ - "type": "object", - "properties": { - "profile": {"type": "string", "description": "Profile to close; omit for the throwaway browser"} - } - }), - annotations: Some(json!({"readOnlyHint": false})), - capabilities: ToolSpec::capabilities(&agent_scopes()), - multiline_params: &[], - hidden: false, - title_template: Some("Closing browser"), - } - } - - async fn execute<'a>( - &self, - context: &mut ToolContext<'a>, - input: &mut Self::Input, - ) -> Result { - let profile = input - .profile - .as_deref() - .unwrap_or(DEFAULT_PROFILE) - .to_string(); - let Some(manager) = context.browser_sessions() else { - return Ok(BrowserOutput::unavailable(&profile)); - }; - match manager.remove_by_label(&profile) { - Some(session) => { - session.close().await; - Ok(BrowserOutput { - profile, - observation: None, - screenshot_base64: None, - error: None, - notes: Vec::new(), - }) - } - None => Ok(BrowserOutput::failure( - &profile, - "No open browser for this profile.", - )), - } - } -} - -// --------------------------------------------------------------------------- -// browser_login — human-in-the-loop login handoff -// --------------------------------------------------------------------------- - -#[derive(Deserialize, Serialize)] -pub struct BrowserLoginInput { - pub url: String, - pub profile: String, -} - -pub struct BrowserLoginTool; - -/// The handoff itself, factored out so tests can drive it headlessly. In -/// production `headful` is always true: a visible window opens, the human logs -/// in, and only their approval lets the agent continue in that same -/// authenticated session. -async fn login_handoff( - manager: &BrowserSessionManager, - handler: &dyn PermissionMediator, - tool_id: Option<&str>, - profile: &str, - url: &str, - headful: bool, -) -> Result { - // A login needs a fresh window: replace any existing (possibly headless) - // session for this profile. - if let Some(existing) = manager.remove_by_label(profile) { - existing.close().await; - } - let session = match BrowserSession::open(launch_config_for(profile, headful), profile).await { - Ok(s) => Arc::new(s), - Err(e) => { - return Ok(BrowserOutput::failure( - profile, - format!("Failed to open browser: {e}"), - )); - } - }; - if let Err(e) = navigate_tolerating_slow_load(&session, url).await { - session.close().await; - return Ok(BrowserOutput::failure( - profile, - format!("Navigation failed: {e}"), - )); - } - - // Pause for the human to log in, then resume on approval. This travels the - // same seam as any other permission prompt (TUI prompt / Telegram keyboard). - let params = json!({ - "action": "browser_login", - "profile": profile, - "url": url, - "instructions": "A browser window has opened. Log in there (password, 2FA, \ - certificate as needed), then approve to let the agent continue.", - }); - let decision = handler - .request_permission(PermissionRequest { - tool_id, - tool_name: "browser_login", - reason: PermissionRequestReason::ToolInvocation { params: ¶ms }, - }) - .await?; - - match decision { - PermissionDecision::Denied => { - session.close().await; - Ok(BrowserOutput::failure( - profile, - "User declined the login handoff.", - )) - } - PermissionDecision::GrantedOnce - | PermissionDecision::GrantedSession - | PermissionDecision::GrantedPersistent => { - // Swap the visible login window for a headless browser on the same - // profile, carrying the login across. The agent then browses in the - // background, and the user can close the login window without killing - // the session (the finding this fixes: a user-closed window left a - // dead session the manager knew nothing about). - match finalize_login_headless(profile, session, url, headful).await { - Ok(headless) => { - manager.register(headless.clone(), profile); - // Note the login so a later session can discover it via - // browser_profiles instead of asking the user to log in again. - record_login_in(&profiles_root(), profile, url); - Ok(BrowserOutput::capture(profile, &headless, false, true).await) - } - Err(e) => Ok(BrowserOutput::failure( - profile, - format!("Login succeeded but switching to a background browser failed: {e}"), - )), - } - } - } -} - -/// Page shown briefly in the login window after a successful handoff, so the -/// user sees it worked and knows the window is safe to close. -const LOGIN_SUCCESS_PAGE: &str = "data:text/html,\ -

✅ Login successful

\ -

You can close this window — the agent now continues in the background.

\ - "; - -/// After a granted login, replace the visible headful browser with a headless -/// one on the same profile, transferring the full cookie jar (including -/// in-memory session cookies a disk flush would drop) so the login survives. -/// Chrome locks the profile dir, so the headful window must fully close before -/// the headless one can start. -async fn finalize_login_headless( - profile: &str, - headful: Arc, - url: &str, - was_headful: bool, -) -> Result> { - // Capture the jar while the authenticated window is still alive. - let cookies = headful.export_cookies().await.unwrap_or_default(); - // Reassure the user in the visible window, give them a moment to read it, - // then close (releasing the profile-dir lock). - let _ = headful.navigate(LOGIN_SUCCESS_PAGE).await; - if was_headful { - tokio::time::sleep(Duration::from_millis(1500)).await; - } - headful.close().await; - - // Relaunch the same profile headless and restore the login. - let headless = - Arc::new(BrowserSession::open(launch_config_for(profile, false), profile).await?); - if url.starts_with("http") { - // Land on an http origin so cookies can be set, inject the jar, then - // reload so the (now present) session cookies take effect. - let _ = headless.navigate(url).await; - let _ = headless.import_cookies(cookies).await; - } - navigate_tolerating_slow_load(&headless, url).await?; - Ok(headless) -} - -#[async_trait::async_trait] -impl Tool for BrowserLoginTool { - type Input = BrowserLoginInput; - type Output = BrowserOutput; - - fn spec(&self) -> ToolSpec { - let mut caps = vec![capabilities::READ_ONLY]; - caps.extend(agent_scopes()); - ToolSpec { - name: "browser_login".into(), - description: concat!( - "Log in to a website AS THE USER without ever seeing their credentials. ", - "Opens a VISIBLE browser window on the named persistent profile, navigates to ", - "the login URL, then pauses and asks the user to complete the login (password, ", - "2FA, certificate — whatever the site needs) in that window and approve. On ", - "approval the agent continues in the same authenticated window, and the session ", - "is saved under the profile for reuse.\n", - "Tell the user what you are doing before calling this. Afterwards use the same ", - "`profile` name with browser_navigate / browser_act." - ) - .into(), - parameters_schema: json!({ - "type": "object", - "properties": { - "url": {"type": "string", "description": "Login page URL"}, - "profile": { - "type": "string", - "description": "Persistent profile name to store the login under (e.g. \"elster\")" - } - }, - "required": ["url", "profile"] - }), - annotations: Some(json!({"readOnlyHint": true, "openWorldHint": true})), - capabilities: ToolSpec::capabilities(&caps), - multiline_params: &[], - hidden: false, - title_template: Some("Logging in at {url}"), - } - } - - async fn execute<'a>( - &self, - context: &mut ToolContext<'a>, - input: &mut Self::Input, - ) -> Result { - let profile = input.profile.clone(); - // The handoff needs a frontend that can prompt the human. - let Some(handler) = context.permission_handler else { - return Ok(BrowserOutput::failure( - &profile, - "Login handoff needs an interactive frontend, which this context does not have.", - )); - }; - let tool_id = context.tool_id.clone(); - let Some(manager) = context.browser_sessions() else { - return Ok(BrowserOutput::unavailable(&profile)); - }; - login_handoff( - manager, - handler, - tool_id.as_deref(), - &profile, - &input.url, - true, - ) - .await - } -} - -// --------------------------------------------------------------------------- -// browser_profiles — discover existing profiles and their last login -// --------------------------------------------------------------------------- - -/// The directory holding all persistent browser profiles and their sidecar -/// metadata files (`/browser-profiles`). -fn profiles_root() -> PathBuf { - crate::config_dir::config_dir().join("browser-profiles") -} - -/// Sanitize a profile name to a filesystem-safe token — the same rule the launch -/// path uses, so a profile's dir and its `.meta.json` sidecar agree. -fn sanitize_profile(profile: &str) -> String { - profile - .chars() - .map(|c| { - if c.is_alphanumeric() || c == '-' || c == '_' { - c - } else { - '_' - } - }) - .collect() -} - -/// Sidecar metadata recorded next to a persistent profile after a login. Kept -/// *beside* the profile dir (not inside it) so Chrome's user-data-dir stays -/// pristine and there is no central index to race on across sessions. -#[derive(Debug, Serialize, Deserialize)] -struct ProfileMeta { - /// The login URL last used for this profile. - url: String, - /// When the last login handoff succeeded (unix seconds). - logged_in_at_unix: i64, -} - -fn meta_path_in(root: &Path, profile: &str) -> PathBuf { - root.join(format!("{}.meta.json", sanitize_profile(profile))) -} - -/// Record a successful login for `profile`. Best-effort — a metadata write must -/// never fail a login — and skipped for the ephemeral default profile, which -/// has no persistent dir. -fn record_login_in(root: &Path, profile: &str, url: &str) { - if profile == DEFAULT_PROFILE { - return; - } - let now = SystemTime::now() - .duration_since(UNIX_EPOCH) - .map(|d| d.as_secs() as i64) - .unwrap_or(0); - let meta = ProfileMeta { - url: url.to_string(), - logged_in_at_unix: now, - }; - let _ = std::fs::create_dir_all(root); - if let Ok(json) = serde_json::to_string_pretty(&meta) { - let _ = std::fs::write(meta_path_in(root, profile), json); - } -} - -fn read_meta_in(root: &Path, profile: &str) -> Option { - let data = std::fs::read_to_string(meta_path_in(root, profile)).ok()?; - serde_json::from_str(&data).ok() -} - -/// List the persistent profiles on disk (subdirectories of the root), each with -/// its login metadata if recorded, sorted by name. -fn list_profiles_in(root: &Path) -> Vec<(String, Option)> { - let Ok(entries) = std::fs::read_dir(root) else { - return Vec::new(); - }; - let mut out: Vec<(String, Option)> = entries - .flatten() - .filter(|e| e.file_type().map(|t| t.is_dir()).unwrap_or(false)) - .filter_map(|e| e.file_name().to_str().map(String::from)) - .map(|name| { - let meta = read_meta_in(root, &name); - (name, meta) - }) - .collect(); - out.sort_by(|a, b| a.0.cmp(&b.0)); - out -} - -/// Render a duration as a coarse, human-friendly "N units ago", so the model can -/// judge at a glance whether a login is likely still fresh. -fn humanize_ago(seconds: i64) -> String { - let s = seconds.max(0); - const MIN: i64 = 60; - const HOUR: i64 = 60 * MIN; - const DAY: i64 = 24 * HOUR; - const WEEK: i64 = 7 * DAY; - const MONTH: i64 = 30 * DAY; - const YEAR: i64 = 365 * DAY; - if s < MIN { - return "just now".to_string(); - } - let (n, unit) = if s < HOUR { - (s / MIN, "minute") - } else if s < DAY { - (s / HOUR, "hour") - } else if s < WEEK { - (s / DAY, "day") - } else if s < MONTH { - (s / WEEK, "week") - } else if s < YEAR { - (s / MONTH, "month") - } else { - (s / YEAR, "year") - }; - format!("{n} {unit}{} ago", if n == 1 { "" } else { "s" }) -} - -/// Best-effort host extraction for display, e.g. `https://x.de/a` → `x.de`. -fn host_of(url: &str) -> String { - let after = url.split("://").nth(1).unwrap_or(url); - let host = after.split('/').next().unwrap_or(after); - host.rsplit('@').next().unwrap_or(host).to_string() -} - -/// Seconds elapsed since a recorded unix timestamp, floored at zero. -fn seconds_since(unix: i64) -> i64 { - let now = SystemTime::now() - .duration_since(UNIX_EPOCH) - .map(|d| d.as_secs() as i64) - .unwrap_or(0); - (now - unix).max(0) -} - -#[derive(Deserialize, Serialize, Default)] -pub struct BrowserProfilesInput {} - -#[derive(Serialize, Deserialize)] -pub struct BrowserProfilesOutput { - text: String, -} - -impl Render for BrowserProfilesOutput { - fn status(&self) -> String { - "Listed browser profiles".to_string() - } - - fn render(&self, _tracker: &mut ResourcesTracker) -> String { - self.text.clone() - } -} - -impl ToolResult for BrowserProfilesOutput { - fn is_success(&self) -> bool { - true - } -} - -pub struct BrowserProfilesTool; - -#[async_trait::async_trait] -impl Tool for BrowserProfilesTool { - type Input = BrowserProfilesInput; - type Output = BrowserProfilesOutput; - - fn spec(&self) -> ToolSpec { - let mut caps = vec![capabilities::READ_ONLY]; - caps.extend(agent_scopes()); - ToolSpec { - name: "browser_profiles".into(), - description: concat!( - "List the browser profiles that already exist on disk, so you can reuse an ", - "existing login instead of asking the user to log in again. Each persistent ", - "profile shows how long ago it was LAST logged in and to which site. That is the ", - "last recorded login, not a guarantee it is still valid — verify by navigating. ", - "\"default\" is the ephemeral throwaway browser (no persisted login). To use a ", - "profile pass its name to browser_navigate / browser_act; to create one use ", - "browser_login." - ) - .into(), - parameters_schema: json!({ "type": "object", "properties": {} }), - annotations: Some(json!({"readOnlyHint": true})), - capabilities: ToolSpec::capabilities(&caps), - multiline_params: &[], - hidden: false, - title_template: Some("Listing browser profiles"), - } - } - - async fn execute<'a>( - &self, - context: &mut ToolContext<'a>, - _input: &mut Self::Input, - ) -> Result { - let profiles = list_profiles_in(&profiles_root()); - let manager = context.browser_sessions(); - - let mut lines = vec![ - "Profiles:".to_string(), - " default — ephemeral (throwaway)".to_string(), - ]; - for (name, meta) in &profiles { - let open = manager - .map(|m| m.get_by_label(name).is_some()) - .unwrap_or(false); - let open_suffix = if open { " · open" } else { "" }; - let detail = match meta { - Some(m) => format!( - "last login: {}, {} (verify by navigating)", - host_of(&m.url), - humanize_ago(seconds_since(m.logged_in_at_unix)), - ), - None => "no login recorded yet".to_string(), - }; - lines.push(format!(" {name} — persistent · {detail}{open_suffix}")); - } - if profiles.is_empty() { - lines.push(String::new()); - lines.push( - "No persistent profiles yet. Use browser_login to create one (it persists the \ - login for reuse)." - .to_string(), - ); - } - Ok(BrowserProfilesOutput { - text: lines.join("\n"), - }) - } -} - -#[cfg(test)] -mod tests { - use super::*; - use crate::mocks::ToolTestFixture; - use tools_core::permissions::PermissionDecision; - - /// A self-contained page (base64 data URL, no server): a field, a button - /// whose JS writes the typed value into a result span, and a title. - fn demo_page_url() -> String { - let html = concat!( - "Login Demo", - "

Welcome

", - "", - "", - "", - "" - ); - let b64 = base64::engine::general_purpose::STANDARD.encode(html); - format!("data:text/html;base64,{b64}") - } - - #[test] - fn humanize_ago_rounds_coarsely() { - assert_eq!(humanize_ago(0), "just now"); - assert_eq!(humanize_ago(59), "just now"); - assert_eq!(humanize_ago(60), "1 minute ago"); - assert_eq!(humanize_ago(3 * 60), "3 minutes ago"); - assert_eq!(humanize_ago(3600), "1 hour ago"); - assert_eq!(humanize_ago(5 * 3600), "5 hours ago"); - assert_eq!(humanize_ago(24 * 3600), "1 day ago"); - assert_eq!(humanize_ago(3 * 24 * 3600), "3 days ago"); - assert_eq!(humanize_ago(10 * 24 * 3600), "1 week ago"); - assert_eq!(humanize_ago(40 * 24 * 3600), "1 month ago"); - assert_eq!(humanize_ago(400 * 24 * 3600), "1 year ago"); - // Never negative, even if a clock skew makes "now" earlier. - assert_eq!(humanize_ago(-500), "just now"); - } - - #[test] - fn host_of_extracts_display_host() { - assert_eq!( - host_of("https://www.elster.de/eportal/start"), - "www.elster.de" - ); - assert_eq!(host_of("http://x.de"), "x.de"); - assert_eq!(host_of("elster.de/path"), "elster.de"); - assert_eq!(host_of("https://user@host.de/x"), "host.de"); - } - - #[test] - fn profiles_are_listed_with_recorded_login() { - let tmp = tempfile::tempdir().unwrap(); - let root = tmp.path(); - // Two profile dirs; only one has a recorded login. - std::fs::create_dir_all(root.join("elster")).unwrap(); - std::fs::create_dir_all(root.join("fresh")).unwrap(); - record_login_in(root, "elster", "https://www.elster.de/eportal"); - - let listed = list_profiles_in(root); - let names: Vec<&str> = listed.iter().map(|(n, _)| n.as_str()).collect(); - assert_eq!(names, vec!["elster", "fresh"], "listed and sorted by name"); - - let meta = listed - .iter() - .find(|(n, _)| n == "elster") - .and_then(|(_, m)| m.as_ref()) - .expect("elster has a recorded login"); - assert_eq!(host_of(&meta.url), "www.elster.de"); - - let fresh = listed.iter().find(|(n, _)| n == "fresh").unwrap(); - assert!(fresh.1.is_none(), "fresh profile has no login recorded"); - } - - #[test] - fn record_login_skips_ephemeral_default() { - let tmp = tempfile::tempdir().unwrap(); - record_login_in(tmp.path(), DEFAULT_PROFILE, "https://x.de"); - assert!( - read_meta_in(tmp.path(), DEFAULT_PROFILE).is_none(), - "the throwaway default profile must not get a sidecar" - ); - } - - #[tokio::test] - async fn browser_profiles_lists_the_default_profile() -> Result<()> { - // Regardless of what's on disk, the output always leads with the header - // and the ever-present ephemeral default. - let mut fixture = ToolTestFixture::new().with_browser_sessions(); - let mut context = fixture.context(); - let mut input = BrowserProfilesInput::default(); - let out = BrowserProfilesTool - .execute(&mut context, &mut input) - .await?; - let text = out.render(&mut ResourcesTracker::default()); - assert!(text.starts_with("Profiles:"), "got: {text}"); - assert!( - text.contains("default — ephemeral (throwaway)"), - "got: {text}" - ); - Ok(()) - } - - #[test] - fn launch_config_maps_default_to_ephemeral_and_names_to_persistent() { - let default = launch_config_for(DEFAULT_PROFILE, false); - assert!(matches!(default.profile, BrowserProfile::Ephemeral)); - - let named = launch_config_for("elster", false); - match named.profile { - BrowserProfile::Persistent(path) => { - assert_eq!(path.file_name().unwrap(), "elster"); - assert!(path.to_string_lossy().contains("browser-profiles")); - } - _ => panic!("named profile should be persistent"), - } - } - - #[test] - fn launch_config_sanitizes_path_traversal_in_profile_names() { - let named = launch_config_for("../evil name", false); - let BrowserProfile::Persistent(path) = named.profile else { - panic!("expected persistent"); - }; - let last = path.file_name().unwrap().to_string_lossy().to_string(); - assert!(!last.contains('/'), "no path separators: {last}"); - assert!(!last.contains(".."), "no traversal: {last}"); - assert_eq!(last, "___evil_name"); - } - - #[tokio::test] - async fn navigate_act_read_close_round_trip() -> Result<()> { - let mut fixture = ToolTestFixture::new().with_browser_sessions(); - - // Navigate opens a browser and returns a screenshot + the page text. - let mut context = fixture.context(); - let mut nav = BrowserNavigateInput { - url: demo_page_url(), - profile: None, - }; - let out = BrowserNavigateTool.execute(&mut context, &mut nav).await?; - assert!(out.error.is_none(), "navigate error: {:?}", out.error); - let obs = out.observation.as_ref().expect("observation"); - assert_eq!(obs.title, "Login Demo"); - assert!(obs.text.contains("Welcome"), "text: {}", obs.text); - // The observation surfaces the actionable elements, and render() lists - // them with their selectors so the model can target them directly. - assert!( - obs.elements.iter().any(|e| e.selector == "#go"), - "should discover the submit button, got: {:?}", - obs.elements - ); - let rendered = out.render(&mut ResourcesTracker::default()); - assert!( - rendered.contains("Interactive elements:") && rendered.contains("#go"), - "render should list interactive elements, got:\n{rendered}" - ); - assert_eq!( - out.render_images().len(), - 1, - "screenshot should be attached" - ); - assert_eq!(out.render_images()[0].media_type, "image/png"); - - // Act: type into the field, click the button (JS writes the result). - let mut act = BrowserActInput { - actions: vec![ - BrowserAction::Type { - selector: "#user".into(), - text: "stephan".into(), - }, - BrowserAction::Click { - selector: "#go".into(), - }, - BrowserAction::WaitFor { - selector: "#who".into(), - timeout_ms: Some(2000), - }, - ], - profile: None, - no_text: false, - accept_dialogs: false, - }; - let out = BrowserActTool.execute(&mut context, &mut act).await?; - assert!(out.error.is_none(), "act error: {:?}", out.error); - - // Read: the typed value round-tripped into the page. - let mut read = BrowserReadInput { - profile: None, - full_page: false, - no_text: false, - }; - let out = BrowserReadTool.execute(&mut context, &mut read).await?; - let obs = out.observation.as_ref().expect("observation"); - assert!( - obs.text.contains("Hello stephan"), - "expected typed value in page, got: {}", - obs.text - ); - - // Close removes the session from the registry. - let mut close = BrowserCloseInput { profile: None }; - let out = BrowserCloseTool.execute(&mut context, &mut close).await?; - assert!(out.error.is_none(), "close error: {:?}", out.error); - assert!( - fixture - .browser_sessions() - .unwrap() - .get_by_label("default") - .is_none(), - "closed session should be gone" - ); - Ok(()) - } - - #[tokio::test] - async fn fill_clear_and_no_text_work_through_the_act_tool() -> Result<()> { - // A page with a prefilled input; fill should replace, clear should empty. - let html = concat!( - "Form", - "

Editing

", - "", - "" - ); - let url = format!( - "data:text/html;base64,{}", - base64::engine::general_purpose::STANDARD.encode(html) - ); - - let mut fixture = ToolTestFixture::new().with_browser_sessions(); - { - let mut context = fixture.context(); - let mut nav = BrowserNavigateInput { url, profile: None }; - BrowserNavigateTool.execute(&mut context, &mut nav).await?; - - // Fill replaces the prefilled value, and no_text suppresses the dump - // while still returning the element list. - let mut act = BrowserActInput { - actions: vec![BrowserAction::Fill { - selector: "#name".into(), - text: "replaced".into(), - }], - profile: None, - no_text: true, - accept_dialogs: false, - }; - let out = BrowserActTool.execute(&mut context, &mut act).await?; - assert!(out.error.is_none(), "fill error: {:?}", out.error); - let obs = out.observation.as_ref().expect("observation"); - assert!(obs.text.is_empty(), "no_text should suppress the text dump"); - assert!( - obs.elements.iter().any(|e| e.selector == "#name"), - "elements should still be present with no_text" - ); - } - - let session = fixture - .browser_sessions() - .unwrap() - .get_by_label("default") - .unwrap(); - let value = session - .eval("document.getElementById('name').value") - .await?; - assert_eq!( - value.as_str().unwrap_or_default(), - "replaced", - "fill should replace the prefilled value" - ); - - // Clear empties the field. - { - let mut context = fixture.context(); - let mut act = BrowserActInput { - actions: vec![BrowserAction::Clear { - selector: "#name".into(), - }], - profile: None, - no_text: false, - accept_dialogs: false, - }; - let out = BrowserActTool.execute(&mut context, &mut act).await?; - assert!(out.error.is_none(), "clear error: {:?}", out.error); - } - let value = session - .eval("document.getElementById('name').value") - .await?; - assert_eq!(value.as_str().unwrap_or_default(), "", "clear should empty"); - - Ok(()) - } - - #[tokio::test] - async fn act_reports_dialogs_and_accepts_them_only_on_request() -> Result<()> { - let html = concat!( - "Dialogs", - "", - "", - "" - ); - let url = format!( - "data:text/html;base64,{}", - base64::engine::general_purpose::STANDARD.encode(html) - ); - let click_confirm = |accept_dialogs| BrowserActInput { - actions: vec![BrowserAction::Click { - selector: "#confirm".into(), - }], - profile: None, - no_text: false, - accept_dialogs, - }; - - let mut fixture = ToolTestFixture::new().with_browser_sessions(); - let mut context = fixture.context(); - let mut nav = BrowserNavigateInput { url, profile: None }; - BrowserNavigateTool.execute(&mut context, &mut nav).await?; - - // Dismissed by default, and the model is told so. - let out = BrowserActTool - .execute(&mut context, &mut click_confirm(false)) - .await?; - assert!(out.error.is_none(), "act error: {:?}", out.error); - let rendered = out.render(&mut ResourcesTracker::default()); - assert!( - rendered.contains("confirm dialog \"Sicher?\" was dismissed"), - "render should report the dialog, got:\n{rendered}" - ); - assert!( - out.observation - .as_ref() - .unwrap() - .text - .trim() - .ends_with("no") - ); - - // Accepted when the call opts in. - let out = BrowserActTool - .execute(&mut context, &mut click_confirm(true)) - .await?; - let rendered = out.render(&mut ResourcesTracker::default()); - assert!( - rendered.contains("confirm dialog \"Sicher?\" was accepted"), - "got:\n{rendered}" - ); - assert!( - out.observation - .as_ref() - .unwrap() - .text - .trim() - .ends_with("yes") - ); - - // The opt-in is per call: the next call dismisses again. - let out = BrowserActTool - .execute(&mut context, &mut click_confirm(false)) - .await?; - let rendered = out.render(&mut ResourcesTracker::default()); - assert!(rendered.contains("was dismissed"), "got:\n{rendered}"); - Ok(()) - } - - /// Serve a page whose iframe never loads (so `load` never fires) and a page - /// whose `#spin` button hangs the renderer in a script loop. - async fn spawn_hanging_site() -> std::net::SocketAddr { - use axum::response::Html; - use axum::{Router, routing::get}; - - let app = Router::new() - .route( - "/", - get(|| async { - Html(concat!( - "Busy", - "" - )) - }), - ) - .route( - "/slow", - get(|| async { - Html(concat!( - "Slow", - "

Main content

" - )) - }), - ) - .route( - "/never", - get(|| async { - tokio::time::sleep(Duration::from_secs(600)).await; - Html("") - }), - ); - let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap(); - let addr = listener.local_addr().unwrap(); - tokio::spawn(async move { axum::serve(listener, app).await.unwrap() }); - addr - } - - /// Register a throwaway browser with short limits as the default profile, - /// so the tools pick it up instead of launching one with the real limits. - async fn register_short_timeout_browser(fixture: &ToolTestFixture) -> Result<()> { - let session = BrowserSession::open(BrowserLaunchConfig::default(), DEFAULT_PROFILE) - .await? - .with_timeouts(web::BrowserTimeouts { - command: Duration::from_secs(1), - navigation: Duration::from_secs(2), - }); - fixture - .browser_sessions() - .unwrap() - .register(Arc::new(session), DEFAULT_PROFILE); - Ok(()) - } - - #[tokio::test] - async fn navigate_shows_a_page_that_never_finishes_loading() -> Result<()> { - let addr = spawn_hanging_site().await; - let mut fixture = ToolTestFixture::new().with_browser_sessions(); - register_short_timeout_browser(&fixture).await?; - - let mut context = fixture.context(); - let mut nav = BrowserNavigateInput { - url: format!("http://{addr}/slow"), - profile: None, - }; - let out = BrowserNavigateTool.execute(&mut context, &mut nav).await?; - assert!( - out.error.is_none(), - "a slow load is no failure: {:?}", - out.error - ); - let rendered = out.render(&mut ResourcesTracker::default()); - assert!(rendered.contains("Main content"), "got:\n{rendered}"); - assert!( - rendered.contains("did not finish loading"), - "the model should learn the page is incomplete, got:\n{rendered}" - ); - assert_eq!(out.render_images().len(), 1, "screenshot attached"); - Ok(()) - } - - #[tokio::test] - async fn act_on_a_hung_page_reports_it_quickly() -> Result<()> { - let addr = spawn_hanging_site().await; - let mut fixture = ToolTestFixture::new().with_browser_sessions(); - register_short_timeout_browser(&fixture).await?; - - let mut context = fixture.context(); - let mut nav = BrowserNavigateInput { - url: format!("http://{addr}/"), - profile: None, - }; - BrowserNavigateTool.execute(&mut context, &mut nav).await?; - - // The click itself succeeds; the page hangs right after. Capturing the - // result must give up on the first unanswered read, not wait out the - // screenshot as well. - let start = std::time::Instant::now(); - let mut act = BrowserActInput { - actions: vec![BrowserAction::Click { - selector: "#spin".into(), - }], - profile: None, - no_text: false, - accept_dialogs: false, - }; - let out = BrowserActTool.execute(&mut context, &mut act).await?; - assert!( - start.elapsed() < Duration::from_millis(4500), - "took {:?}", - start.elapsed() - ); - let error = out.error.as_deref().expect("a hung page is an error"); - assert!(error.contains("not responding"), "got: {error}"); - Ok(()) - } - - #[tokio::test] - async fn act_resolves_a_text_selector_and_a_chord() -> Result<()> { - // A button targeted by visible text, and a chord recorded on keydown. - let html = concat!( - "t", - "", - "", - "", - "" - ); - let url = format!( - "data:text/html;base64,{}", - base64::engine::general_purpose::STANDARD.encode(html) - ); - - let mut fixture = ToolTestFixture::new().with_browser_sessions(); - { - let mut context = fixture.context(); - let mut nav = BrowserNavigateInput { url, profile: None }; - BrowserNavigateTool.execute(&mut context, &mut nav).await?; - - let mut act = BrowserActInput { - actions: vec![ - BrowserAction::Press { - selector: Some("#f".into()), - key: "Control+a".into(), - }, - BrowserAction::Click { - selector: "text=Speichern".into(), - }, - ], - profile: None, - no_text: false, - accept_dialogs: false, - }; - let out = BrowserActTool.execute(&mut context, &mut act).await?; - assert!(out.error.is_none(), "act error: {:?}", out.error); - } - - let session = fixture - .browser_sessions() - .unwrap() - .get_by_label("default") - .unwrap(); - let title = session.eval("document.title").await?; - assert_eq!( - title.as_str().unwrap_or_default(), - "hit", - "text= selector should have clicked the button" - ); - let log = session - .eval("document.getElementById('log').textContent") - .await?; - assert!( - log.as_str().unwrap_or_default().contains("/true"), - "Control+a chord should set ctrlKey, got: {log}" - ); - Ok(()) - } - - #[tokio::test] - async fn scroll_moves_the_viewport_and_full_page_capture_works() -> Result<()> { - // A page much taller than the viewport. - let html = "\ -
TOP
\ -
BOTTOM
\ - "; - let url = format!( - "data:text/html;base64,{}", - base64::engine::general_purpose::STANDARD.encode(html) - ); - - let mut fixture = ToolTestFixture::new().with_browser_sessions(); - { - let mut context = fixture.context(); - let mut nav = BrowserNavigateInput { url, profile: None }; - BrowserNavigateTool.execute(&mut context, &mut nav).await?; - - // Scroll down by pixels. - let mut act = BrowserActInput { - actions: vec![BrowserAction::Scroll { - selector: None, - dx: None, - dy: Some(1200.0), - }], - profile: None, - no_text: false, - accept_dialogs: false, - }; - let out = BrowserActTool.execute(&mut context, &mut act).await?; - assert!(out.error.is_none(), "scroll error: {:?}", out.error); - } - - let session = fixture - .browser_sessions() - .unwrap() - .get_by_label("default") - .unwrap(); - - // The viewport actually moved down. - let y = session.eval("window.scrollY").await?; - assert!( - y.as_f64().unwrap_or(0.0) >= 900.0, - "page should have scrolled down, scrollY={y}" - ); - - // Scrolling an element into view reaches the bottom element. - session.scroll(Some("#bottom"), 0.0, 0.0).await?; - let y2 = session.eval("window.scrollY").await?; - assert!( - y2.as_f64().unwrap_or(0.0) > 3000.0, - "scroll-into-view should reach the bottom element, scrollY={y2}" - ); - - // A full-page screenshot succeeds (captures beyond the viewport). - let png = session.screenshot(true).await?; - assert!(png.starts_with(b"\x89PNG"), "full-page screenshot is a PNG"); - Ok(()) - } - - #[tokio::test] - async fn coordinate_click_and_global_key_reach_the_page() -> Result<()> { - // A full-viewport surface that records the last click's coordinates, plus - // a focused input that records the last key it received. No selectors on - // the click target — coordinates are the only way to hit it. - let html = concat!( - "Canvas", - "", - "
", - "", - "", - "", - "" - ); - let url = format!( - "data:text/html;base64,{}", - base64::engine::general_purpose::STANDARD.encode(html) - ); - - let mut fixture = ToolTestFixture::new().with_browser_sessions(); - { - let mut context = fixture.context(); - let mut nav = BrowserNavigateInput { url, profile: None }; - BrowserNavigateTool.execute(&mut context, &mut nav).await?; - - // Click at exact CSS pixels inside the pad; the handler records them. - let mut act = BrowserActInput { - actions: vec![BrowserAction::ClickAt { - x: "120px".into(), - y: "80px".into(), - }], - profile: None, - no_text: false, - accept_dialogs: false, - }; - let out = BrowserActTool.execute(&mut context, &mut act).await?; - assert!(out.error.is_none(), "click_at error: {:?}", out.error); - } - - let session = fixture - .browser_sessions() - .unwrap() - .get_by_label("default") - .unwrap(); - let hit = session - .eval("document.getElementById('hit').innerText") - .await?; - assert_eq!( - hit.as_str().unwrap_or_default(), - "120,80", - "px click should land at the given CSS-pixel coordinates" - ); - - // Focus the field, then a global key press (no selector) lands on the - // focused element. - session - .eval("document.getElementById('field').focus()") - .await?; - { - let mut context = fixture.context(); - let mut act = BrowserActInput { - actions: vec![BrowserAction::Press { - selector: None, - key: "a".into(), - }], - profile: None, - no_text: false, - accept_dialogs: false, - }; - let out = BrowserActTool.execute(&mut context, &mut act).await?; - assert!(out.error.is_none(), "global press error: {:?}", out.error); - } - let key = session - .eval("document.getElementById('key').innerText") - .await?; - assert_eq!( - key.as_str().unwrap_or_default(), - "a", - "selector-less press should reach the focused element" - ); - Ok(()) - } - - #[test] - fn resolve_coord_maps_units_and_rejects_typographic() { - // Viewport 1000 × 500 CSS px. - let (vw, vh) = (1000.0, 500.0); - // vw/vh are always width/height. - assert_eq!(resolve_coord("40vw", Axis::X, vw, vh).unwrap(), 400.0); - assert_eq!(resolve_coord("30vh", Axis::Y, vw, vh).unwrap(), 150.0); - // % follows the axis. - assert_eq!(resolve_coord("50%", Axis::X, vw, vh).unwrap(), 500.0); - assert_eq!(resolve_coord("50%", Axis::Y, vw, vh).unwrap(), 250.0); - // px passes through unchanged. - assert_eq!(resolve_coord("640px", Axis::X, vw, vh).unwrap(), 640.0); - // Out-of-range clamps to the viewport. - assert_eq!(resolve_coord("150vw", Axis::X, vw, vh).unwrap(), 1000.0); - assert_eq!(resolve_coord("-10px", Axis::Y, vw, vh).unwrap(), 0.0); - // Typographic units and bare numbers are rejected. - assert!(resolve_coord("25rem", Axis::X, vw, vh).is_err()); - assert!(resolve_coord("2em", Axis::Y, vw, vh).is_err()); - assert!(resolve_coord("640", Axis::X, vw, vh).is_err()); - } - - #[tokio::test] - async fn coordinate_units_map_to_the_same_css_pixel() -> Result<()> { - // A full-viewport pad that records the CSS-pixel coords of the last click. - let html = concat!( - "Pad", - "
", - "", - "" - ); - let url = format!( - "data:text/html;base64,{}", - base64::engine::general_purpose::STANDARD.encode(html) - ); - - let mut fixture = ToolTestFixture::new().with_browser_sessions(); - { - let mut context = fixture.context(); - let mut nav = BrowserNavigateInput { url, profile: None }; - BrowserNavigateTool.execute(&mut context, &mut nav).await?; - } - - let session = fixture - .browser_sessions() - .unwrap() - .get_by_label("default") - .unwrap(); - let (vw, vh) = session.viewport_size().await?; - assert!(vw > 0.0 && vh > 0.0, "viewport size should be known"); - - // The centre of the viewport, expressed three ways, must land on the - // same CSS pixel (±1 for rounding). - let (cx, cy) = ((vw / 2.0).round(), (vh / 2.0).round()); - for (xu, yu) in [ - ("50%".to_string(), "50%".to_string()), - ("50vw".to_string(), "50vh".to_string()), - (format!("{cx}px"), format!("{cy}px")), - ] { - session - .eval("document.getElementById('hit').innerText=''") - .await?; - let px = resolve_coord(&xu, Axis::X, vw, vh)?; - let py = resolve_coord(&yu, Axis::Y, vw, vh)?; - session.click_at(px, py).await?; - let hit = session - .eval("document.getElementById('hit').innerText") - .await?; - let got = hit.as_str().unwrap_or_default().to_string(); - let parts: Vec = got.split(',').filter_map(|s| s.parse().ok()).collect(); - assert_eq!(parts.len(), 2, "click {xu},{yu} recorded '{got}'"); - assert!( - (parts[0] - cx).abs() <= 1.0 && (parts[1] - cy).abs() <= 1.0, - "unit {xu}/{yu} should land at centre {cx},{cy}, got {got}" - ); - } - Ok(()) - } - - #[tokio::test] - async fn acting_without_an_open_browser_is_a_clear_error() -> Result<()> { - let mut fixture = ToolTestFixture::new().with_browser_sessions(); - let mut context = fixture.context(); - let mut read = BrowserReadInput { - profile: None, - full_page: false, - no_text: false, - }; - let out = BrowserReadTool.execute(&mut context, &mut read).await?; - assert!(out.error.unwrap().contains("browser_navigate")); - Ok(()) - } - - /// Mediator returning a fixed decision, standing in for the human at the - /// browser. Lets us drive the handoff headlessly (no window popped). - struct ScriptedMediator(PermissionDecision); - - #[async_trait::async_trait] - impl PermissionMediator for ScriptedMediator { - async fn request_permission( - &self, - _request: PermissionRequest<'_>, - ) -> Result { - Ok(self.0) - } - } - - #[tokio::test] - async fn login_handoff_grant_keeps_authenticated_session() -> Result<()> { - let manager = BrowserSessionManager::new(4); - let mediator = ScriptedMediator(PermissionDecision::GrantedOnce); - // Ephemeral profile ("default") so the test touches no config dir. - let out = login_handoff( - &manager, - &mediator, - None, - "default", - &demo_page_url(), - false, - ) - .await?; - assert!(out.error.is_none(), "grant error: {:?}", out.error); - assert!(out.observation.is_some(), "authenticated page observed"); - assert!( - manager.get_by_label("default").is_some(), - "session should be kept after approval" - ); - manager.close_all().await; - Ok(()) - } - - #[tokio::test] - async fn login_handoff_deny_closes_and_reports() -> Result<()> { - let manager = BrowserSessionManager::new(4); - let mediator = ScriptedMediator(PermissionDecision::Denied); - let out = login_handoff( - &manager, - &mediator, - None, - "default", - &demo_page_url(), - false, - ) - .await?; - assert!(out.error.unwrap().contains("declined")); - assert!( - manager.get_by_label("default").is_none(), - "no session should remain after a denied handoff" - ); - Ok(()) - } - - #[tokio::test] - async fn browser_login_without_a_handler_is_a_clear_error() -> Result<()> { - // No permission handler ⇒ no way to ask the human ⇒ graceful error, - // and crucially no browser is launched. - let mut fixture = ToolTestFixture::new().with_browser_sessions(); - let mut context = fixture.context(); - let mut input = BrowserLoginInput { - url: demo_page_url(), - profile: "elster".into(), - }; - let out = BrowserLoginTool.execute(&mut context, &mut input).await?; - assert!(out.error.unwrap().contains("interactive frontend")); - Ok(()) - } - - #[test] - fn error_output_is_text_only_even_when_a_screenshot_was_captured() { - // browser_act's failure path captures a screenshot for context, then - // sets an error. Anthropic rejects images in a tool_result with - // is_error=true, so render_images must be empty on error. - let out = BrowserOutput { - profile: "default".into(), - observation: None, - screenshot_base64: Some("ZmFrZQ==".into()), - error: Some("Action 1 failed: no such element '#missing'".into()), - notes: Vec::new(), - }; - assert!(!out.is_success()); - assert!( - out.render_images().is_empty(), - "an error result must carry no images" - ); - } -} - -#[cfg(test)] -mod registration_check { - use crate::tools::scope::ToolScope; - - #[test] - fn browser_tools_are_exposed_to_the_agent() { - let registry = crate::tools::test_registry(); - let names: Vec = registry - .get_tool_definitions_with_capability(ToolScope::Agent.tag()) - .into_iter() - .map(|d| d.name) - .collect(); - for expected in [ - "browser_navigate", - "browser_read", - "browser_act", - "browser_close", - "browser_login", - "browser_profiles", - ] { - assert!( - names.contains(&expected.to_string()), - "missing {expected}; have: {names:?}" - ); - } - } -} diff --git a/crates/code_assistant_core/src/tools/impls/browser/batch.rs b/crates/code_assistant_core/src/tools/impls/browser/batch.rs new file mode 100644 index 00000000..3e5a1bbc --- /dev/null +++ b/crates/code_assistant_core/src/tools/impls/browser/batch.rs @@ -0,0 +1,219 @@ +//! `browser_batch`: several browser steps in one call. + +use super::{BrowserOutput, DEFAULT_PROFILE, computer, devtools, page, spec, tabs}; +use crate::tools::core::{Tool, ToolContext, ToolSpec}; +use crate::tools::services::ToolServicesAccess; +use anyhow::Result; +use serde::de::DeserializeOwned; +use serde::{Deserialize, Serialize}; +use serde_json::{Value, json}; +use web::BrowserSessionManager; + +const MAX_STEPS: usize = 50; + +#[derive(Deserialize, Serialize, Clone)] +pub struct BatchStep { + pub name: String, + #[serde(default)] + pub input: Value, +} + +#[derive(Deserialize, Serialize)] +pub struct BatchInput { + pub actions: Vec, +} + +pub struct BrowserBatchTool; + +#[async_trait::async_trait] +impl Tool for BrowserBatchTool { + type Input = BatchInput; + type Output = BrowserOutput; + + fn spec(&self) -> ToolSpec { + let mut spec = spec( + "browser_batch", + "Run several browser tool calls in one round trip, in order, stopping at the first \ + error. Each step is {name, input} with the input that tool takes, e.g. \ + [{\"name\": \"browser_computer\", \"input\": {\"action\": \"left_click\", \"ref\": \ + \"ref_4\"}}, {\"name\": \"browser_computer\", \"input\": {\"action\": \"type\", \ + \"text\": \"hi\"}}, {\"name\": \"browser_computer\", \"input\": {\"action\": \ + \"screenshot\"}}]. Screenshots come back in order. Use it whenever you can predict \ + two or more steps; coordinates in a step refer to the screenshot taken before this \ + call. Steps run back to back, so this is also how to time input precisely: the page \ + keeps running between separate calls.", + json!({ + "type": "object", + "properties": { + "actions": { + "type": "array", + "items": { + "type": "object", + "properties": { + "name": {"type": "string", "description": "A browser tool, e.g. browser_computer, browser_navigate, browser_form_input"}, + "input": {"type": "object", "description": "That tool's input"} + }, + "required": ["name", "input"] + } + } + }, + "required": ["actions"] + }), + false, + "Running browser steps", + ); + // Each step names its own profile and tab. + if let Some(props) = spec.parameters_schema["properties"].as_object_mut() { + props.remove("profile"); + props.remove("tab_id"); + } + spec + } + + async fn execute<'a>( + &self, + context: &mut ToolContext<'a>, + input: &mut Self::Input, + ) -> Result { + let Some(manager) = context.browser_sessions() else { + return Ok(BrowserOutput::unavailable(DEFAULT_PROFILE)); + }; + Ok(run_batch(manager, &input.actions).await) + } +} + +pub(crate) async fn run_batch( + manager: &BrowserSessionManager, + steps: &[BatchStep], +) -> BrowserOutput { + if steps.len() > MAX_STEPS { + return BrowserOutput::failure( + DEFAULT_PROFILE, + format!("at most {MAX_STEPS} steps per batch"), + ); + } + let mut combined = BrowserOutput { + profile: DEFAULT_PROFILE.to_string(), + ..Default::default() + }; + let mut sections = Vec::new(); + for (i, step) in steps.iter().enumerate() { + let n = i + 1; + let out = match run_step(manager, step).await { + Ok(out) => out, + Err(e) => BrowserOutput::failure(DEFAULT_PROFILE, e.to_string()), + }; + let label = step_label(step); + if !out.text.is_empty() { + sections.push(format!("[{n}] {label}\n{}", out.text)); + } + combined.profile = out.profile; + if let Some(e) = out.error { + combined.error = Some(format!("Step {n} ({label}) failed: {e}")); + break; + } + combined.images.extend(out.images); + } + combined.text = sections.join("\n\n"); + combined +} + +fn step_label(step: &BatchStep) -> String { + let name = step.name.trim_start_matches("browser_"); + match step.input.get("action").and_then(Value::as_str) { + Some(action) => format!("{name} {action}"), + None => name.to_string(), + } +} + +async fn run_step(manager: &BrowserSessionManager, step: &BatchStep) -> Result { + fn parse(step: &BatchStep) -> Result { + serde_json::from_value(step.input.clone()) + .map_err(|e| anyhow::anyhow!("invalid input for {}: {e}", step.name)) + } + let name = step.name.trim_start_matches("browser_"); + Ok(match name { + "navigate" => page::navigate(manager, &parse(step)?).await, + "read_page" => page::read_page(manager, &parse(step)?).await, + "find" => page::find(manager, &parse(step)?).await, + "get_page_text" => page::get_page_text(manager, &parse(step)?).await, + "form_input" => page::form_input(manager, &parse(step)?).await, + "javascript" => page::javascript(manager, &parse(step)?).await, + "computer" => computer::computer(manager, &parse(step)?).await, + "read_console_messages" => devtools::read_console(manager, &parse(step)?).await, + "read_network_requests" => devtools::read_network(manager, &parse(step)?).await, + "resize_window" => tabs::resize_window(manager, &parse(step)?).await, + "tabs_context" => tabs::tabs_context(manager, &parse(step)?).await, + "tabs_create" => tabs::tabs_create(manager, &parse(step)?).await, + "tabs_select" => tabs::tabs_select(manager, &parse(step)?).await, + "tabs_close" => tabs::tabs_close(manager, &parse(step)?).await, + other => anyhow::bail!("browser_{other} cannot run in a batch"), + }) +} + +#[cfg(test)] +mod tests { + use super::super::test_support::data_url; + use super::*; + use crate::mocks::ToolTestFixture; + use crate::tools::core::{Render, ResourcesTracker}; + + fn step(name: &str, input: Value) -> BatchStep { + BatchStep { + name: name.into(), + input, + } + } + + #[tokio::test] + async fn steps_run_in_order_and_stop_at_the_first_error() -> Result<()> { + let page = data_url( + "", + ); + let mut fixture = ToolTestFixture::new().with_browser_sessions(); + let mut context = fixture.context(); + let mut input = BatchInput { + actions: vec![ + step("browser_navigate", json!({"url": page})), + step("browser_find", json!({"query": "button"})), + step("computer", json!({"action": "screenshot", "scale": 0.25})), + step( + "browser_javascript", + json!({"text": "document.title = 'x'; 1 + 1"}), + ), + step( + "browser_computer", + json!({"action": "left_click", "ref": "ref_999"}), + ), + step("browser_javascript", json!({"text": "'never runs'"})), + ], + }; + let out = BrowserBatchTool.execute(&mut context, &mut input).await?; + let text = out.text.clone(); + assert!(text.contains("[1] navigate\n[t1]"), "{text}"); + assert!(text.contains("[2] find\n- button \"Go\" [ref_"), "{text}"); + assert!( + text.contains("[3] computer screenshot\nScreenshot of t1 — 320×200 px"), + "{text}" + ); + assert!(text.contains("[4] javascript\n2"), "{text}"); + assert!(!text.contains("never runs"), "{text}"); + assert_eq!(out.images.len(), 1); + let error = out.error.as_deref().unwrap(); + assert!( + error.starts_with("Step 5 (computer left_click) failed: unknown ref"), + "{error}" + ); + assert!( + out.render(&mut ResourcesTracker::default()) + .contains("Browser error: Step 5") + ); + + let mut bad = BatchInput { + actions: vec![step("browser_login", json!({}))], + }; + let out = BrowserBatchTool.execute(&mut context, &mut bad).await?; + assert!(out.error.unwrap().contains("cannot run in a batch")); + Ok(()) + } +} diff --git a/crates/code_assistant_core/src/tools/impls/browser/computer.rs b/crates/code_assistant_core/src/tools/impls/browser/computer.rs new file mode 100644 index 00000000..42976758 --- /dev/null +++ b/crates/code_assistant_core/src/tools/impls/browser/computer.rs @@ -0,0 +1,660 @@ +//! `browser_computer`: mouse, keyboard, screenshots — by `ref_N` or by +//! coordinates in the latest screenshot's frame. + +use super::{BrowserOutput, Resolved, Target, browser_tool, spec}; +use crate::tools::core::ToolSpec; +use anyhow::{Result, anyhow}; +use serde::{Deserialize, Serialize}; +use serde_json::json; +use std::time::Duration; +use web::{BrowserSessionManager, Button, Tab}; + +/// Pixels one wheel tick scrolls. +const TICK_PX: f64 = 100.0; + +#[derive(Deserialize, Serialize, Debug, Clone, Copy, PartialEq, Eq)] +#[serde(rename_all = "snake_case")] +pub enum ComputerAction { + LeftClick, + RightClick, + DoubleClick, + TripleClick, + Type, + Key, + Screenshot, + Wait, + Scroll, + ScrollTo, + Hover, + LeftClickDrag, + Zoom, + HoldKey, + KeyDown, + KeyUp, + LeftMouseDown, + LeftMouseUp, +} + +#[derive(Deserialize, Serialize, Debug, Clone, Copy)] +#[serde(rename_all = "snake_case")] +pub enum ScrollDirection { + Up, + Down, + Left, + Right, +} + +#[derive(Deserialize, Serialize)] +pub struct ComputerInput { + pub action: ComputerAction, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub coordinate: Option<[f64; 2]>, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub start_coordinate: Option<[f64; 2]>, + #[serde(default, rename = "ref", skip_serializing_if = "Option::is_none")] + pub r#ref: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub text: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub modifiers: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub scroll_direction: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub scroll_amount: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub duration: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub region: Option<[f64; 4]>, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub repeat: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub scale: Option, + #[serde(default, skip_serializing_if = "std::ops::Not::not")] + pub accept_dialogs: bool, + #[serde(flatten)] + pub target: Target, +} + +pub struct BrowserComputerTool; + +impl BrowserComputerTool { + fn make_spec() -> ToolSpec { + spec( + "browser_computer", + "Mouse, keyboard and screenshots in a browser tab. Target elements by `ref` (from \ + browser_read_page/browser_find) or by `coordinate` in the pixel frame of the most \ + recent screenshot (each screenshot reports its size). Take a screenshot before \ + clicking by coordinate.\n\ + Actions: left_click, right_click, double_click, triple_click (coordinate or ref; \ + `modifiers` like \"ctrl+shift\"), type (`text` into the focused element), key \ + (`text`: space-separated keys or chords like \"Enter\", \"ctrl+a\", \"Backspace\"; \ + `repeat`), hold_key (`text` held down for `duration` seconds), key_down / key_up \ + (`text` stays down across steps until released, e.g. walk while jumping), \ + left_mouse_down / left_mouse_up (at coordinate/ref or where the mouse is), screenshot (`scale` < 1 for a smaller image), wait (`duration` seconds, \ + max 10), scroll (`scroll_direction`, `scroll_amount` ticks, at `coordinate`/`ref` or \ + the center), scroll_to (ref), hover (coordinate or ref), left_click_drag \ + (`start_coordinate` → `coordinate`), zoom (`region` [x0, y0, x1, y1] of the \ + screenshot, enlarged).\n\ + JavaScript dialogs are answered automatically and reported: alerts acknowledged, \ + confirm/prompt dismissed unless `accept_dialogs` is true.\n\ + The page keeps running in real time between calls, also while you think. For \ + timing-sensitive input (games, animations) put the steps in one browser_batch, \ + where they run back to back and `wait`/`hold_key` durations are exact.", + json!({ + "type": "object", + "properties": { + "action": {"type": "string", "enum": [ + "left_click", "right_click", "double_click", "triple_click", "type", "key", + "screenshot", "wait", "scroll", "scroll_to", "hover", "left_click_drag", "zoom", + "hold_key", "key_down", "key_up", "left_mouse_down", "left_mouse_up" + ]}, + "coordinate": {"type": "array", "items": {"type": "number"}, "description": "[x, y] in the latest screenshot's pixels"}, + "start_coordinate": {"type": "array", "items": {"type": "number"}, "description": "[x, y] where left_click_drag starts"}, + "ref": {"type": "string", "description": "ref_N of the element to act on"}, + "text": {"type": "string", "description": "Text to type, or keys to press"}, + "modifiers": {"type": "string", "description": "Modifier keys held during a click, e.g. \"ctrl\", \"cmd+shift\""}, + "scroll_direction": {"type": "string", "enum": ["up", "down", "left", "right"]}, + "scroll_amount": {"type": "integer", "description": "Wheel ticks (default 3)"}, + "duration": {"type": "number", "description": "Seconds to wait or hold a key (max 10)"}, + "region": {"type": "array", "items": {"type": "number"}, "description": "[x0, y0, x1, y1] to zoom into"}, + "repeat": {"type": "integer", "description": "Times to repeat the key sequence"}, + "scale": {"type": "number", "description": "Image scale for screenshot/zoom, e.g. 0.5"}, + "accept_dialogs": {"type": "boolean", "description": "Accept (OK) confirm/prompt dialogs this action raises"} + }, + "required": ["action"] + }), + false, + "Browser: {action}", + ) + } +} + +browser_tool!(BrowserComputerTool, ComputerInput, computer); + +pub(crate) async fn computer( + manager: &BrowserSessionManager, + input: &ComputerInput, +) -> BrowserOutput { + let mut r = match Resolved::new(manager, &input.target, false).await { + Ok(r) => r, + Err(out) => return out, + }; + r.tab.set_accept_dialogs(input.accept_dialogs); + let result = act(&r.tab, input).await; + r.tab.set_accept_dialogs(false); + match result { + Ok(Acted { text, image }) => { + let notes = r.notes().await; + let mut out = r.output(text).with_notes(notes); + if let Some(png) = image { + out = out.with_image(&png); + } + out + } + Err(e) => { + let notes = r.notes().await; + let mut out = r.failure(e); + out = out.with_notes(notes); + out + } + } +} + +struct Acted { + text: String, + image: Option>, +} + +impl Acted { + fn text(text: impl Into) -> Self { + Self { + text: text.into(), + image: None, + } + } +} + +async fn act(tab: &Tab, input: &ComputerInput) -> Result { + use ComputerAction::*; + let action = input.action; + match action { + LeftClick | RightClick | DoubleClick | TripleClick => { + let (button, count) = match action { + RightClick => (Button::Right, 1), + DoubleClick => (Button::Left, 2), + TripleClick => (Button::Left, 3), + _ => (Button::Left, 1), + }; + let modifiers = parse_modifiers(input.modifiers.as_deref())?; + let at = point(tab, input).await?; + tab.click_point(at, button, count, modifiers).await?; + tab.settle().await; + Ok(Acted::text(format!( + "{} {}", + past_tense(action), + target_label(input) + ))) + } + Hover => { + let at = point(tab, input).await?; + tab.hover_point(at).await?; + Ok(Acted::text(format!("Hovered {}", target_label(input)))) + } + Type => { + let text = input + .text + .as_deref() + .ok_or_else(|| anyhow!("type needs `text`"))?; + tab.type_into_focused(text).await?; + Ok(Acted::text(format!( + "Typed {} characters", + text.chars().count() + ))) + } + Key => { + let keys = input + .text + .as_deref() + .ok_or_else(|| anyhow!("key needs `text`"))?; + let repeat = input.repeat.unwrap_or(1).clamp(1, 100); + tab.press_keys(keys, repeat).await?; + tab.settle().await; + let times = if repeat > 1 { + format!(" ×{repeat}") + } else { + String::new() + }; + Ok(Acted::text(format!("Pressed {keys}{times}"))) + } + Screenshot => { + let shot = tab.screenshot_frame(input.scale).await?; + Ok(Acted { + text: format!( + "Screenshot of {} — {}×{} px (coordinates refer to this frame)", + tab.id(), + shot.width, + shot.height + ), + image: Some(shot.png), + }) + } + Zoom => { + let region = input + .region + .ok_or_else(|| anyhow!("zoom needs `region` [x0, y0, x1, y1]"))?; + let shot = tab.zoom(region, input.scale).await?; + Ok(Acted { + text: format!( + "Zoomed into {region:?}: {}×{} px (for inspection; click coordinates still refer to the last screenshot)", + shot.width, shot.height + ), + image: Some(shot.png), + }) + } + Wait => { + let seconds = input.duration.unwrap_or(1.0).clamp(0.0, 10.0); + tokio::time::sleep(Duration::from_secs_f64(seconds)).await; + Ok(Acted::text(format!("Waited {seconds}s"))) + } + HoldKey => { + let keys = input + .text + .as_deref() + .ok_or_else(|| anyhow!("hold_key needs `text`"))?; + let seconds = input.duration.unwrap_or(1.0).clamp(0.0, 10.0); + tab.hold_keys(keys, Duration::from_secs_f64(seconds)) + .await?; + Ok(Acted::text(format!("Held {keys} for {seconds}s"))) + } + KeyDown => { + let keys = input + .text + .as_deref() + .ok_or_else(|| anyhow!("key_down needs `text`"))?; + tab.key_down(keys).await?; + Ok(Acted::text(format!("Pressed {keys} down"))) + } + KeyUp => { + let keys = input + .text + .as_deref() + .ok_or_else(|| anyhow!("key_up needs `text`"))?; + tab.key_up(keys).await?; + Ok(Acted::text(format!("Released {keys}"))) + } + LeftMouseDown | LeftMouseUp => { + let at = if input.coordinate.is_some() || input.r#ref.is_some() { + Some(point(tab, input).await?) + } else { + None + }; + if action == LeftMouseDown { + tab.mouse_down(at, Button::Left).await?; + Ok(Acted::text(format!( + "Pressed the left button {}", + where_label(input) + ))) + } else { + tab.mouse_up(at, Button::Left).await?; + tab.settle().await; + Ok(Acted::text(format!( + "Released the left button {}", + where_label(input) + ))) + } + } + Scroll => { + let direction = input.scroll_direction.unwrap_or(ScrollDirection::Down); + let ticks = input.scroll_amount.unwrap_or(3).clamp(1, 50) as f64; + let (dx, dy) = match direction { + ScrollDirection::Up => (0.0, -ticks * TICK_PX), + ScrollDirection::Down => (0.0, ticks * TICK_PX), + ScrollDirection::Left => (-ticks * TICK_PX, 0.0), + ScrollDirection::Right => (ticks * TICK_PX, 0.0), + }; + let at = if input.coordinate.is_some() || input.r#ref.is_some() { + point(tab, input).await? + } else { + let (w, h) = tab.viewport_size().await?; + web::Point { + x: w / 2.0, + y: h / 2.0, + } + }; + tab.wheel(at, dx, dy).await?; + // Let smooth scrolling land before anyone looks. + tokio::time::sleep(Duration::from_millis(300)).await; + Ok(Acted::text( + format!("Scrolled {direction:?} {ticks} ticks").to_lowercase(), + )) + } + ScrollTo => { + let r = input + .r#ref + .as_deref() + .ok_or_else(|| anyhow!("scroll_to needs `ref`"))?; + tab.scroll_to_ref(r).await?; + Ok(Acted::text(format!("Scrolled {r} into view"))) + } + LeftClickDrag => { + let [sx, sy] = input + .start_coordinate + .ok_or_else(|| anyhow!("left_click_drag needs `start_coordinate`"))?; + let [ex, ey] = input + .coordinate + .ok_or_else(|| anyhow!("left_click_drag needs `coordinate`"))?; + tab.drag(tab.frame_point(sx, sy), tab.frame_point(ex, ey)) + .await?; + tab.settle().await; + Ok(Acted::text(format!( + "Dragged from ({sx}, {sy}) to ({ex}, {ey})" + ))) + } + } +} + +/// Where an action lands: the center of `ref`, or `coordinate` mapped from +/// the screenshot frame to CSS pixels. +async fn point(tab: &Tab, input: &ComputerInput) -> Result { + if let Some(r) = &input.r#ref { + return tab.ref_point(r).await; + } + let [x, y] = input + .coordinate + .ok_or_else(|| anyhow!("{:?} needs `coordinate` or `ref`", input.action))?; + Ok(tab.frame_point(x, y)) +} + +fn target_label(input: &ComputerInput) -> String { + match (&input.r#ref, input.coordinate) { + (Some(r), _) => r.clone(), + (None, Some([x, y])) => format!("at ({x}, {y})"), + _ => String::new(), + } +} + +/// Like [`target_label`], but "where the mouse is" without a target. +fn where_label(input: &ComputerInput) -> String { + match target_label(input) { + label if label.is_empty() => "where the mouse is".to_string(), + label => label, + } +} + +fn past_tense(action: ComputerAction) -> &'static str { + match action { + ComputerAction::RightClick => "Right-clicked", + ComputerAction::DoubleClick => "Double-clicked", + ComputerAction::TripleClick => "Triple-clicked", + _ => "Clicked", + } +} + +/// `"ctrl+shift"` → the CDP modifier bitmask (Alt=1, Ctrl=2, Meta=4, Shift=8). +fn parse_modifiers(spec: Option<&str>) -> Result { + let mut mask = 0; + for part in spec + .unwrap_or("") + .split('+') + .map(str::trim) + .filter(|p| !p.is_empty()) + { + mask |= match part.to_ascii_lowercase().as_str() { + "alt" | "option" | "opt" => 1, + "ctrl" | "control" => 2, + "meta" | "cmd" | "command" | "super" | "win" | "windows" => 4, + "shift" => 8, + other => return Err(anyhow!("unknown modifier '{other}'")), + }; + } + Ok(mask) +} + +#[cfg(test)] +mod tests { + use super::super::page::{ + BrowserNavigateTool, BrowserReadPageTool, NavigateInput, ReadPageInput, + }; + use super::super::test_support::*; + use super::*; + use crate::mocks::ToolTestFixture; + use crate::tools::core::{Render, ResourcesTracker, Tool}; + + fn computer(action: ComputerAction) -> ComputerInput { + ComputerInput { + action, + coordinate: None, + start_coordinate: None, + r#ref: None, + text: None, + modifiers: None, + scroll_direction: None, + scroll_amount: None, + duration: None, + region: None, + repeat: None, + scale: None, + accept_dialogs: false, + target: Target::default(), + } + } + + fn render(out: &BrowserOutput) -> String { + out.render(&mut ResourcesTracker::default()) + } + + #[test] + fn modifiers_parse_to_a_bitmask() { + assert_eq!(parse_modifiers(None).unwrap(), 0); + assert_eq!(parse_modifiers(Some("ctrl+shift")).unwrap(), 10); + assert_eq!(parse_modifiers(Some("Cmd")).unwrap(), 4); + assert!(parse_modifiers(Some("hyper")).is_err()); + } + + #[tokio::test] + async fn click_by_ref_type_and_screenshot_coordinates() -> Result<()> { + let page = data_url( + "\ + \ + \ + \ + ", + ); + let mut fixture = ToolTestFixture::new().with_browser_sessions(); + let mut context = fixture.context(); + BrowserNavigateTool + .execute( + &mut context, + &mut NavigateInput { + url: page, + target: Target::default(), + }, + ) + .await?; + let mut read = ReadPageInput { + filter: super::super::page::ReadFilter::Interactive, + ref_id: None, + depth: None, + max_chars: None, + target: Target::default(), + }; + let tree = render(&BrowserReadPageTool.execute(&mut context, &mut read).await?); + + // Click the field by ref, type, fix it up with keys. + let mut click = computer(ComputerAction::LeftClick); + click.r#ref = Some(ref_on_line(&tree, "textbox \"Field\"")); + let out = BrowserComputerTool + .execute(&mut context, &mut click) + .await?; + assert!(out.error.is_none(), "{:?}", out.error); + let mut typing = computer(ComputerAction::Type); + typing.text = Some("Grüße!".into()); + BrowserComputerTool + .execute(&mut context, &mut typing) + .await?; + let mut key = computer(ComputerAction::Key); + key.text = Some("Backspace".into()); + let out = BrowserComputerTool.execute(&mut context, &mut key).await?; + assert_eq!(render(&out), "Pressed Backspace"); + + // A half-size screenshot: its frame is 640×400, so (650, 420) in the + // page is (325, 210) in the frame. + let mut shot = computer(ComputerAction::Screenshot); + shot.scale = Some(0.5); + let out = BrowserComputerTool.execute(&mut context, &mut shot).await?; + assert!(render(&out).contains("640×400 px"), "{}", render(&out)); + assert_eq!(out.render_images().len(), 1); + let mut click = computer(ComputerAction::LeftClick); + click.coordinate = Some([325.0, 210.0]); + BrowserComputerTool + .execute(&mut context, &mut click) + .await?; + + let session = fixture + .browser_sessions() + .unwrap() + .get_by_label("default") + .unwrap(); + let tab = session.active_tab()?; + assert_eq!( + tab.javascript( + "[document.getElementById('f').value, document.getElementById('b').textContent]" + ) + .await?, + r#"["Grüße","hit"]"# + ); + + // A confirm is dismissed and reported, or accepted on request. + let mut context = fixture.context(); + let mut confirm = computer(ComputerAction::LeftClick); + confirm.r#ref = Some(ref_on_line(&tree, "button \"Confirm\"")); + let out = BrowserComputerTool + .execute(&mut context, &mut confirm) + .await?; + assert!( + render(&out).contains("Note: A confirm dialog \"Sure?\" was dismissed."), + "{}", + render(&out) + ); + confirm.accept_dialogs = true; + let out = BrowserComputerTool + .execute(&mut context, &mut confirm) + .await?; + assert!(render(&out).contains("was accepted"), "{}", render(&out)); + assert_eq!(tab.javascript("document.title").await?, "yes"); + Ok(()) + } + + #[tokio::test] + async fn keys_are_held_for_a_duration_and_across_steps() -> Result<()> { + let page = data_url( + "", + ); + let mut fixture = ToolTestFixture::new().with_browser_sessions(); + let mut context = fixture.context(); + BrowserNavigateTool + .execute( + &mut context, + &mut NavigateInput { + url: page, + target: Target::default(), + }, + ) + .await?; + + let mut hold = computer(ComputerAction::HoldKey); + hold.text = Some("w".into()); + hold.duration = Some(0.3); + let out = BrowserComputerTool.execute(&mut context, &mut hold).await?; + assert_eq!(render(&out), "Held w for 0.3s"); + + // While w is held, every result says so; after key_up it stops. + for (action, keys, held_note) in [ + (ComputerAction::KeyDown, "w", true), + (ComputerAction::Key, "space", true), + (ComputerAction::KeyUp, "w", false), + ] { + let mut step = computer(action); + step.text = Some(keys.into()); + let out = BrowserComputerTool.execute(&mut context, &mut step).await?; + assert!(out.error.is_none(), "{:?}", out.error); + assert_eq!( + render(&out).contains("Note: Still held down: w (release with"), + held_note, + "{}", + render(&out) + ); + } + + let session = fixture + .browser_sessions() + .unwrap() + .get_by_label("default") + .unwrap(); + let log = session + .active_tab()? + .javascript("window.log.join(' ')") + .await?; + let events: Vec<&str> = log.split(' ').collect(); + let codes: Vec<&str> = events + .iter() + .map(|e| e.split('@').next().unwrap()) + .collect(); + assert_eq!( + codes, + [ + "keydown:KeyW", + "keyup:KeyW", + "keydown:KeyW", + "keydown:Space", + "keyup:Space", + "keyup:KeyW" + ], + "{log}" + ); + let at = |i: usize| events[i].split('@').nth(1).unwrap().parse::().unwrap(); + assert!(at(1) - at(0) >= 280.0, "{log}"); + Ok(()) + } + + #[tokio::test] + async fn a_hung_page_fails_fast() -> Result<()> { + let page = data_url( + "", + ); + let mut fixture = ToolTestFixture::new().with_browser_sessions(); + register_short_timeout_browser(&fixture).await?; + let mut context = fixture.context(); + BrowserNavigateTool + .execute( + &mut context, + &mut NavigateInput { + url: page, + target: Target::default(), + }, + ) + .await?; + let mut click = computer(ComputerAction::LeftClick); + click.coordinate = Some([20.0, 15.0]); + BrowserComputerTool + .execute(&mut context, &mut click) + .await?; + tokio::time::sleep(Duration::from_millis(200)).await; + + let start = std::time::Instant::now(); + let out = BrowserComputerTool + .execute(&mut context, &mut computer(ComputerAction::Screenshot)) + .await?; + assert!( + start.elapsed() < Duration::from_secs(4), + "{:?}", + start.elapsed() + ); + assert!(out.error.unwrap().contains("timed out")); + Ok(()) + } +} diff --git a/crates/code_assistant_core/src/tools/impls/browser/describe.rs b/crates/code_assistant_core/src/tools/impls/browser/describe.rs new file mode 100644 index 00000000..56d6c95d --- /dev/null +++ b/crates/code_assistant_core/src/tools/impls/browser/describe.rs @@ -0,0 +1,377 @@ +//! One-line descriptions of browser tool calls, shared by the frontends so a +//! call reads the same everywhere ("Click ref_4", "Find \"Sign in\""). + +use serde_json::Value; + +/// Every browser tool name, for frontends that register renderers by name. +pub const BROWSER_TOOL_NAMES: [&str; 18] = [ + "browser_navigate", + "browser_computer", + "browser_read_page", + "browser_find", + "browser_get_page_text", + "browser_form_input", + "browser_javascript", + "browser_read_console_messages", + "browser_read_network_requests", + "browser_resize_window", + "browser_tabs_context", + "browser_tabs_create", + "browser_tabs_select", + "browser_tabs_close", + "browser_batch", + "browser_close", + "browser_login", + "browser_profiles", +]; + +/// Whether the output of `tool` is structured text (a tree, log lines) that +/// reads best in a monospace font. +pub fn has_structured_output(tool: &str) -> bool { + matches!( + tool, + "browser_read_page" + | "browser_find" + | "browser_read_console_messages" + | "browser_read_network_requests" + | "browser_javascript" + | "browser_tabs_context" + ) +} + +/// Describe a browser tool call from its parameters. `param` returns a +/// parameter's value as shown in the UI (strings unquoted, arrays/objects as +/// JSON), or `None` when it is absent or still streaming. +pub fn describe_call(tool: &str, param: &dyn Fn(&str) -> Option) -> String { + let text = |name: &str, max: usize| param(name).map(|v| truncate(&v, max)); + match tool { + "browser_navigate" => match param("url").as_deref() { + Some("back") => "Go back".to_string(), + Some("forward") => "Go forward".to_string(), + Some(url) => format!("Navigate to {}", truncate(url, 70)), + None => "Navigate".to_string(), + }, + "browser_computer" => describe_computer(param), + "browser_read_page" => { + let mut out = if param("filter").as_deref() == Some("interactive") { + "Read interactive elements".to_string() + } else { + "Read page".to_string() + }; + if let Some(r) = param("ref_id") { + out.push_str(&format!(" under {r}")); + } + out + } + "browser_find" => match text("query", 50) { + Some(q) => format!("Find \"{q}\""), + None => "Find on the page".to_string(), + }, + "browser_get_page_text" => "Read page text".to_string(), + "browser_form_input" => match (param("ref"), text("value", 40)) { + (Some(r), Some(v)) => format!("Set {r} to {v}"), + (Some(r), None) => format!("Set {r}"), + _ => "Set form field".to_string(), + }, + "browser_javascript" => match param("text") { + Some(code) => format!( + "Run {}", + truncate(code.lines().next().unwrap_or_default().trim(), 60) + ), + None => "Run JavaScript".to_string(), + }, + "browser_read_console_messages" => { + if param("only_errors").as_deref() == Some("true") { + "Read console errors".to_string() + } else { + "Read console messages".to_string() + } + } + "browser_read_network_requests" => match param("request_id") { + Some(id) => format!("Read response {id}"), + None => "Read network requests".to_string(), + }, + "browser_resize_window" => { + let size = match (param("preset"), param("width"), param("height")) { + (Some(preset), _, _) => Some(preset), + (None, Some(w), Some(h)) => Some(format!("{w}×{h}")), + _ => None, + }; + match (size, param("color_scheme")) { + (Some(size), Some(scheme)) => format!("Resize to {size}, {scheme} mode"), + (Some(size), None) => format!("Resize to {size}"), + (None, Some(scheme)) => format!("Switch to {scheme} mode"), + (None, None) => "Resize viewport".to_string(), + } + } + "browser_tabs_context" => "List tabs".to_string(), + "browser_tabs_create" => "Open a tab".to_string(), + "browser_tabs_select" => match param("tab_id") { + Some(id) => format!("Switch to tab {id}"), + None => "Switch tab".to_string(), + }, + "browser_tabs_close" => match param("tab_id") { + Some(id) => format!("Close tab {id}"), + None => "Close tab".to_string(), + }, + "browser_batch" => match param("actions") + .and_then(|json| serde_json::from_str::(&json).ok()) + .and_then(|v| v.as_array().map(Vec::len)) + { + Some(n) => format!("Browser steps ({n} step{})", if n == 1 { "" } else { "s" }), + None => "Browser steps".to_string(), + }, + "browser_close" => "Close browser".to_string(), + "browser_login" => match param("url") { + Some(url) => format!("Log in at {}", truncate(&url, 60)), + None => "Log in".to_string(), + }, + "browser_profiles" => "List browser profiles".to_string(), + other => other.to_string(), + } +} + +/// The steps of a `browser_batch` call, each described like a single call. +pub fn describe_batch_steps(actions_json: &str) -> Vec { + let Ok(Value::Array(steps)) = serde_json::from_str::(actions_json) else { + return Vec::new(); + }; + steps + .iter() + .map(|step| { + let name = step.get("name").and_then(Value::as_str).unwrap_or(""); + let tool = if name.starts_with("browser_") { + name.to_string() + } else { + format!("browser_{name}") + }; + let input = step.get("input").cloned().unwrap_or(Value::Null); + describe_call(&tool, &|key| { + input.get(key).map(|v| match v { + Value::String(s) => s.clone(), + other => other.to_string(), + }) + }) + }) + .collect() +} + +/// A short label for where a call runs: a non-default profile and/or a tab. +pub fn target_label(param: &dyn Fn(&str) -> Option) -> Option { + let profile = param("profile").filter(|p| !p.is_empty() && p != "default"); + let tab = param("tab_id").filter(|t| !t.is_empty()); + match (profile, tab) { + (Some(p), Some(t)) => Some(format!("{p} · {t}")), + (Some(p), None) => Some(p), + (None, Some(t)) => Some(t), + (None, None) => None, + } +} + +fn describe_computer(param: &dyn Fn(&str) -> Option) -> String { + let target = param("ref") + .or_else(|| param("coordinate").map(|c| compact_coordinate(&c))) + .map(|t| format!(" {t}")) + .unwrap_or_default(); + let modifiers = param("modifiers") + .map(|m| format!("{m}+")) + .unwrap_or_default(); + match param("action").as_deref() { + Some("left_click") => format!("{modifiers}Click{target}"), + Some("right_click") => format!("Right-click{target}"), + Some("double_click") => format!("Double-click{target}"), + Some("triple_click") => format!("Triple-click{target}"), + Some("hover") => format!("Hover{target}"), + Some("scroll_to") => format!("Scroll to{target}"), + Some("left_click_drag") => "Drag".to_string(), + Some("type") => match param("text") { + Some(text) => format!("Type \"{}\"", truncate(&text, 40)), + None => "Type".to_string(), + }, + Some("key") => match param("text") { + Some(keys) => { + let times = param("repeat") + .filter(|r| r != "1") + .map(|r| format!(" ×{r}")) + .unwrap_or_default(); + format!("Press {}{times}", truncate(&keys, 40)) + } + None => "Press keys".to_string(), + }, + Some("scroll") => match param("scroll_direction") { + Some(direction) => format!("Scroll {direction}"), + None => "Scroll".to_string(), + }, + Some("hold_key") => { + let duration = param("duration") + .map(|d| format!(" for {d}s")) + .unwrap_or_default(); + format!("Hold {}{duration}", param("text").unwrap_or_default()) + } + Some("key_down") => format!("Hold down {}", param("text").unwrap_or_default()), + Some("key_up") => format!("Release {}", param("text").unwrap_or_default()), + Some("left_mouse_down") => format!("Press the mouse{target}"), + Some("left_mouse_up") => format!("Release the mouse{target}"), + Some("screenshot") => "Screenshot".to_string(), + Some("zoom") => "Zoom in".to_string(), + Some("wait") => match param("duration") { + Some(s) => format!("Wait {s}s"), + None => "Wait".to_string(), + }, + _ => "Use the browser".to_string(), + } +} + +/// `[120, 340]` (or `[120.0,340.5]`) → `(120, 340)`. +fn compact_coordinate(json: &str) -> String { + match serde_json::from_str::>(json) { + Ok(v) if v.len() == 2 => format!("({}, {})", v[0].round(), v[1].round()), + _ => json.to_string(), + } +} + +fn truncate(s: &str, max_chars: usize) -> String { + let s = s.replace('\n', " "); + if s.chars().count() > max_chars { + let cut: String = s.chars().take(max_chars.saturating_sub(1)).collect(); + format!("{cut}…") + } else { + s + } +} + +#[cfg(test)] +mod tests { + use super::*; + + fn describe(tool: &str, params: &[(&str, &str)]) -> String { + describe_call(tool, &|key| { + params + .iter() + .find(|(k, _)| *k == key) + .map(|(_, v)| v.to_string()) + }) + } + + #[test] + fn computer_calls_name_the_action_and_target() { + assert_eq!( + describe( + "browser_computer", + &[("action", "left_click"), ("ref", "ref_4")] + ), + "Click ref_4" + ); + assert_eq!( + describe( + "browser_computer", + &[ + ("action", "left_click"), + ("coordinate", "[120.4, 340]"), + ("modifiers", "ctrl") + ] + ), + "ctrl+Click (120, 340)" + ); + assert_eq!( + describe( + "browser_computer", + &[("action", "key"), ("text", "Backspace"), ("repeat", "3")] + ), + "Press Backspace ×3" + ); + assert_eq!( + describe("browser_computer", &[("action", "screenshot")]), + "Screenshot" + ); + assert_eq!( + describe( + "browser_computer", + &[("action", "hold_key"), ("text", "w"), ("duration", "0.5")] + ), + "Hold w for 0.5s" + ); + } + + #[test] + fn page_tools_describe_their_input() { + assert_eq!( + describe("browser_navigate", &[("url", "https://x.test")]), + "Navigate to https://x.test" + ); + assert_eq!(describe("browser_navigate", &[("url", "back")]), "Go back"); + assert_eq!( + describe("browser_read_page", &[("filter", "interactive")]), + "Read interactive elements" + ); + assert_eq!( + describe("browser_find", &[("query", "Sign in")]), + "Find \"Sign in\"" + ); + assert_eq!( + describe( + "browser_form_input", + &[("ref", "ref_7"), ("value", "Green")] + ), + "Set ref_7 to Green" + ); + assert_eq!( + describe("browser_javascript", &[("text", "document.title\n// more")]), + "Run document.title" + ); + assert_eq!( + describe( + "browser_resize_window", + &[("preset", "mobile"), ("color_scheme", "dark")] + ), + "Resize to mobile, dark mode" + ); + } + + #[test] + fn batch_counts_and_describes_its_steps() { + let json = r#"[{"name":"browser_computer","input":{"action":"left_click","ref":"ref_4"}}, + {"name":"find","input":{"query":"Save"}}]"#; + assert_eq!( + describe("browser_batch", &[("actions", json)]), + "Browser steps (2 steps)" + ); + assert_eq!( + describe_batch_steps(json), + vec!["Click ref_4".to_string(), "Find \"Save\"".to_string()] + ); + } + + #[test] + fn target_label_shows_a_named_profile_and_tab() { + let label = |params: &[(&str, &str)]| { + target_label(&|key| { + params + .iter() + .find(|(k, _)| *k == key) + .map(|(_, v)| v.to_string()) + }) + }; + assert_eq!(label(&[("profile", "default")]), None); + assert_eq!( + label(&[("profile", "elster"), ("tab_id", "t2")]).as_deref(), + Some("elster · t2") + ); + assert_eq!(label(&[("tab_id", "t2")]).as_deref(), Some("t2")); + } + + #[test] + fn every_registered_browser_tool_has_a_name_here() { + let mut registry = crate::tools::core::ToolRegistry::new(); + super::super::register(&mut registry); + let mut registered: Vec = registry + .get_tool_definitions_with_capability(crate::tools::scope::ToolScope::Agent.tag()) + .into_iter() + .map(|d| d.name) + .collect(); + registered.sort(); + let mut listed: Vec = BROWSER_TOOL_NAMES.iter().map(|s| s.to_string()).collect(); + listed.sort(); + assert_eq!(registered, listed); + } +} diff --git a/crates/code_assistant_core/src/tools/impls/browser/devtools.rs b/crates/code_assistant_core/src/tools/impls/browser/devtools.rs new file mode 100644 index 00000000..13aab904 --- /dev/null +++ b/crates/code_assistant_core/src/tools/impls/browser/devtools.rs @@ -0,0 +1,211 @@ +//! What a tab logged: console messages and network requests. + +use super::{BrowserOutput, Resolved, Target, browser_tool, spec}; +use crate::tools::core::ToolSpec; +use serde::{Deserialize, Serialize}; +use serde_json::json; +use web::BrowserSessionManager; + +const DEFAULT_LIMIT: usize = 50; +const MAX_LIMIT: usize = 200; +/// How much of a response body to return. +const MAX_BODY_CHARS: usize = 20_000; + +// --------------------------------------------------------------------------- +// browser_read_console_messages +// --------------------------------------------------------------------------- + +#[derive(Deserialize, Serialize)] +pub struct ReadConsoleInput { + #[serde(default, skip_serializing_if = "Option::is_none")] + pub limit: Option, + #[serde(default, skip_serializing_if = "std::ops::Not::not")] + pub only_errors: bool, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub pattern: Option, + #[serde(flatten)] + pub target: Target, +} + +pub struct BrowserReadConsoleTool; + +impl BrowserReadConsoleTool { + fn make_spec() -> ToolSpec { + spec( + "browser_read_console_messages", + "Read the tab's console output (log, info, warning, error, debug), uncaught \ + exceptions and browser messages such as failed resource loads — most recent last.", + json!({ + "type": "object", + "properties": { + "limit": {"type": "integer", "description": "Max entries (default 50, max 200)"}, + "only_errors": {"type": "boolean", "description": "Only errors and exceptions"}, + "pattern": {"type": "string", "description": "Substring filter on the message text"} + } + }), + true, + "Reading console messages", + ) + } +} + +browser_tool!(BrowserReadConsoleTool, ReadConsoleInput, read_console); + +pub(crate) async fn read_console( + manager: &BrowserSessionManager, + input: &ReadConsoleInput, +) -> BrowserOutput { + let r = match Resolved::new(manager, &input.target, false).await { + Ok(r) => r, + Err(out) => return out, + }; + let limit = input.limit.unwrap_or(DEFAULT_LIMIT).clamp(1, MAX_LIMIT); + let lines = r + .tab + .console_messages(input.only_errors, input.pattern.as_deref(), limit); + if lines.is_empty() { + return r.output("No console messages."); + } + r.output(lines.join("\n")) +} + +// --------------------------------------------------------------------------- +// browser_read_network_requests +// --------------------------------------------------------------------------- + +#[derive(Deserialize, Serialize)] +pub struct ReadNetworkInput { + #[serde(default, skip_serializing_if = "Option::is_none")] + pub limit: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub url_pattern: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub request_id: Option, + #[serde(flatten)] + pub target: Target, +} + +pub struct BrowserReadNetworkTool; + +impl BrowserReadNetworkTool { + fn make_spec() -> ToolSpec { + spec( + "browser_read_network_requests", + "List the tab's network requests (`[id] METHOD status type url (mime)`, most recent \ + last), or fetch one response body by `request_id`.", + json!({ + "type": "object", + "properties": { + "limit": {"type": "integer", "description": "Max entries when listing (default 50, max 200)"}, + "url_pattern": {"type": "string", "description": "Substring filter on the URL"}, + "request_id": {"type": "string", "description": "Return this request's response body instead of listing"} + } + }), + true, + "Reading network requests", + ) + } +} + +browser_tool!(BrowserReadNetworkTool, ReadNetworkInput, read_network); + +pub(crate) async fn read_network( + manager: &BrowserSessionManager, + input: &ReadNetworkInput, +) -> BrowserOutput { + let r = match Resolved::new(manager, &input.target, false).await { + Ok(r) => r, + Err(out) => return out, + }; + if let Some(id) = &input.request_id { + return match r.tab.response_body(id, MAX_BODY_CHARS).await { + Ok(body) => r.output(body), + Err(e) => r.failure(e), + }; + } + let limit = input.limit.unwrap_or(DEFAULT_LIMIT).clamp(1, MAX_LIMIT); + let lines = r.tab.network_requests(input.url_pattern.as_deref(), limit); + if lines.is_empty() { + return r.output("No network requests."); + } + r.output(lines.join("\n")) +} + +#[cfg(test)] +mod tests { + use super::super::page::{BrowserNavigateTool, NavigateInput}; + use super::*; + use crate::mocks::ToolTestFixture; + use crate::tools::core::{Render, ResourcesTracker, Tool}; + use anyhow::Result; + + #[tokio::test] + async fn console_errors_and_a_response_body() -> Result<()> { + use axum::response::Html; + use axum::{Router, routing::get}; + let app = Router::new() + .route( + "/", + get(|| async { + Html( + "", + ) + }), + ) + .route("/api", get(|| async { "pong" })); + let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await?; + let addr = listener.local_addr()?; + tokio::spawn(async move { axum::serve(listener, app).await.unwrap() }); + + let mut fixture = ToolTestFixture::new().with_browser_sessions(); + let mut context = fixture.context(); + BrowserNavigateTool + .execute( + &mut context, + &mut NavigateInput { + url: format!("http://{addr}/"), + target: Target::default(), + }, + ) + .await?; + tokio::time::sleep(std::time::Duration::from_millis(300)).await; + let render = |out: BrowserOutput| out.render(&mut ResourcesTracker::default()); + + let mut console = ReadConsoleInput { + limit: None, + only_errors: true, + pattern: None, + target: Target::default(), + }; + let errors = render( + BrowserReadConsoleTool + .execute(&mut context, &mut console) + .await?, + ); + assert!(errors.starts_with("[error] broken"), "{errors}"); + assert!(!errors.contains("ready"), "{errors}"); + + let mut network = ReadNetworkInput { + limit: None, + url_pattern: Some("/api".into()), + request_id: None, + target: Target::default(), + }; + let listed = render( + BrowserReadNetworkTool + .execute(&mut context, &mut network) + .await?, + ); + assert!(listed.contains("GET 200 fetch"), "{listed}"); + let id = listed.trim_start_matches('[').split(']').next().unwrap(); + network.request_id = Some(id.to_string()); + let body = render( + BrowserReadNetworkTool + .execute(&mut context, &mut network) + .await?, + ); + assert_eq!(body, "pong"); + Ok(()) + } +} diff --git a/crates/code_assistant_core/src/tools/impls/browser/mod.rs b/crates/code_assistant_core/src/tools/impls/browser/mod.rs new file mode 100644 index 00000000..33561d11 --- /dev/null +++ b/crates/code_assistant_core/src/tools/impls/browser/mod.rs @@ -0,0 +1,534 @@ +//! Browser tools, shaped after the computer-use style browser tools models +//! are trained on: a page is read as an accessibility tree whose elements +//! carry `ref_N` handles, acted on by ref or by screenshot coordinates, and +//! observed only when the model asks (screenshot, tree, text, console, +//! network) instead of after every step. +//! +//! State lives in the session-scoped `web::BrowserSessionManager` (see +//! [`crate::tools::services`]): one live browser per **profile**, each with +//! tabs. The `default` profile is a throwaway browser; a named profile +//! persists its login (see `browser_login` / `browser_profiles`). +//! +//! - [`page`] — `browser_navigate`, `browser_read_page`, `browser_find`, +//! `browser_get_page_text`, `browser_form_input`, `browser_javascript` +//! - [`computer`] — `browser_computer`: mouse, keyboard, screenshot, zoom +//! - [`devtools`] — `browser_read_console_messages`, +//! `browser_read_network_requests` +//! - [`tabs`] — `browser_tabs_*`, `browser_resize_window` +//! - [`batch`] — `browser_batch`: several steps in one call +//! - [`profiles`] — `browser_close`, `browser_login`, `browser_profiles` +//! - [`describe`] — one-line descriptions of calls, for the frontends + +mod batch; +mod computer; +pub mod describe; +mod devtools; +mod page; +mod profiles; +mod tabs; + +pub use batch::BrowserBatchTool; +pub use computer::BrowserComputerTool; +pub use devtools::{BrowserReadConsoleTool, BrowserReadNetworkTool}; +pub use page::{ + BrowserFindTool, BrowserFormInputTool, BrowserGetPageTextTool, BrowserJavascriptTool, + BrowserNavigateTool, BrowserReadPageTool, +}; +pub use profiles::{BrowserCloseTool, BrowserLoginTool, BrowserProfilesTool}; +pub use tabs::{ + BrowserResizeWindowTool, BrowserTabsCloseTool, BrowserTabsContextTool, BrowserTabsCreateTool, + BrowserTabsSelectTool, +}; + +use crate::tools::core::{ + ImageData, Render, ResourcesTracker, ToolResult, ToolSpec, cap_base64_image, capabilities, +}; +use anyhow::Result; +use base64::Engine; +use serde::{Deserialize, Serialize}; +use serde_json::Value; +use std::sync::Arc; +use web::{BrowserLaunchConfig, BrowserProfile, BrowserSession, BrowserSessionManager, Tab}; + +/// Register every browser tool. +pub fn register(registry: &mut crate::tools::core::ToolRegistry) { + registry.register(Box::new(BrowserNavigateTool)); + registry.register(Box::new(BrowserComputerTool)); + registry.register(Box::new(BrowserReadPageTool)); + registry.register(Box::new(BrowserFindTool)); + registry.register(Box::new(BrowserGetPageTextTool)); + registry.register(Box::new(BrowserFormInputTool)); + registry.register(Box::new(BrowserJavascriptTool)); + registry.register(Box::new(BrowserReadConsoleTool)); + registry.register(Box::new(BrowserReadNetworkTool)); + registry.register(Box::new(BrowserResizeWindowTool)); + registry.register(Box::new(BrowserTabsContextTool)); + registry.register(Box::new(BrowserTabsCreateTool)); + registry.register(Box::new(BrowserTabsSelectTool)); + registry.register(Box::new(BrowserTabsCloseTool)); + registry.register(Box::new(BrowserBatchTool)); + registry.register(Box::new(BrowserCloseTool)); + registry.register(Box::new(BrowserLoginTool)); + registry.register(Box::new(BrowserProfilesTool)); +} + +/// The profile used when the model does not name one: a reusable ephemeral +/// (throwaway) browser. +pub(crate) const DEFAULT_PROFILE: &str = "default"; + +/// Resolve a profile name to a launch config. The reserved `"default"` name is +/// an ephemeral throwaway browser; any other name is a persistent profile under +/// `/browser-profiles/`, so a login can be reused across runs. +pub(crate) fn launch_config_for(profile: &str, headful: bool) -> BrowserLaunchConfig { + if profile == DEFAULT_PROFILE { + return BrowserLaunchConfig { + profile: BrowserProfile::Ephemeral, + headful, + }; + } + let dir = profiles::profiles_root().join(profiles::sanitize_profile(profile)); + BrowserLaunchConfig { + profile: BrowserProfile::Persistent(dir), + headful, + } +} + +/// Get the live browser for `profile`, opening one if none exists yet. +pub(crate) async fn get_or_open( + manager: &BrowserSessionManager, + profile: &str, +) -> Result> { + if let Some(session) = manager.get_by_label(profile) { + return Ok(session); + } + let session = Arc::new(BrowserSession::open(launch_config_for(profile, false), profile).await?); + manager.register(session.clone(), profile); + Ok(session) +} + +/// Which browser and tab a call targets. Flattened into every tool's input. +#[derive(Debug, Default, Clone, Deserialize, Serialize)] +pub struct Target { + #[serde(default, skip_serializing_if = "Option::is_none")] + pub profile: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub tab_id: Option, +} + +impl Target { + pub(crate) fn profile(&self) -> &str { + self.profile.as_deref().unwrap_or(DEFAULT_PROFILE) + } +} + +/// The schema properties every tool shares. +pub(crate) fn target_properties(schema: &mut Value) { + let props = schema["properties"].as_object_mut().expect("object schema"); + props.insert( + "profile".into(), + serde_json::json!({"type": "string", "description": "Named persistent profile (see browser_profiles); omit for the throwaway browser"}), + ); + props.insert( + "tab_id".into(), + serde_json::json!({"type": "string", "description": "Tab to act on; omit for the active tab"}), + ); +} + +/// A resolved browser + tab, plus what changed around the call. +pub(crate) struct Resolved { + pub profile: String, + pub session: Arc, + pub tab: Arc, + /// Tabs the page opened since the last call. + adopted: Vec, + url_before: String, +} + +impl Resolved { + /// Resolve `target`, opening the profile's browser if `open` and none is + /// running. A failure comes back as the output to return. + pub async fn new( + manager: &BrowserSessionManager, + target: &Target, + open: bool, + ) -> std::result::Result { + let profile = target.profile().to_string(); + let session = if open { + get_or_open(manager, &profile).await.map_err(|e| { + BrowserOutput::failure(&profile, format!("Failed to open browser: {e}")) + })? + } else { + manager.get_by_label(&profile).ok_or_else(|| { + BrowserOutput::failure( + &profile, + format!( + "No browser is open for profile '{profile}'. Use browser_navigate first." + ), + ) + })? + }; + let adopted = session.sync_tabs().await.unwrap_or_default(); + let tab = session + .tab(target.tab_id.as_deref()) + .map_err(|e| BrowserOutput::failure(&profile, e.to_string()))?; + let url_before = tab.location().await.0; + Ok(Self { + profile, + session, + tab, + adopted, + url_before, + }) + } + + /// Things the model should know after an action: dialogs answered on its + /// behalf, tabs the page opened, and a navigation the action caused. + pub async fn notes(&mut self) -> Vec { + let mut notes = Vec::new(); + for dialog in self.tab.take_dialogs() { + let verdict = if dialog.accepted { + "accepted" + } else { + "dismissed" + }; + notes.push(format!( + "A {} dialog \"{}\" was {verdict}.", + dialog.kind, dialog.message + )); + } + self.adopted + .extend(self.session.sync_tabs().await.unwrap_or_default()); + for id in self.adopted.drain(..) { + if let Ok(tab) = self.session.tab(Some(&id)) { + let (url, title) = tab.location().await; + notes.push(format!( + "The page opened a new tab {id}: {url} — {title} (switch with browser_tabs_select)." + )); + } + } + let held = self.tab.held_inputs(); + if !held.is_empty() { + notes.push(format!( + "Still held down: {} (release with key_up / left_mouse_up).", + held.join(", ") + )); + } + let (url, title) = self.tab.location().await; + if url != self.url_before { + notes.push(format!("The page navigated to {url} — {title}")); + self.url_before = url; + } + notes + } + + pub fn output(&self, text: impl Into) -> BrowserOutput { + BrowserOutput { + profile: self.profile.clone(), + text: text.into(), + ..Default::default() + } + } + + pub fn failure(&self, error: impl std::fmt::Display) -> BrowserOutput { + BrowserOutput::failure(&self.profile, error.to_string()) + } +} + +/// Text plus optional screenshots, the result of every browser tool. +/// +/// Every field defaults, so outputs stored by older versions (with other +/// fields) still load. +#[derive(Debug, Default, Serialize, Deserialize)] +pub struct BrowserOutput { + #[serde(default)] + pub profile: String, + #[serde(default)] + pub text: String, + /// Base64 PNGs, surfaced to the model via `render_images`. + #[serde(default, skip_serializing_if = "Vec::is_empty")] + pub images: Vec, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub error: Option, +} + +impl BrowserOutput { + pub(crate) fn failure(profile: &str, error: impl Into) -> Self { + Self { + profile: profile.to_string(), + error: Some(error.into()), + ..Default::default() + } + } + + /// The output when this context has no browser registry. + pub(crate) fn unavailable(profile: &str) -> Self { + Self::failure( + profile, + "Browser tools are not available in this context (no browser session registry).", + ) + } + + pub(crate) fn with_image(mut self, png: &[u8]) -> Self { + self.images + .push(base64::engine::general_purpose::STANDARD.encode(png)); + self + } + + /// Append notes (dialogs, new tabs, navigation) after the main text. + pub(crate) fn with_notes(mut self, notes: Vec) -> Self { + for note in notes { + if !self.text.is_empty() { + self.text.push('\n'); + } + self.text.push_str(&format!("Note: {note}")); + } + self + } +} + +impl Render for BrowserOutput { + fn status(&self) -> String { + match &self.error { + Some(e) => format!("Browser error: {e}"), + None => self.text.lines().next().unwrap_or("Done").to_string(), + } + } + + fn render(&self, _tracker: &mut ResourcesTracker) -> String { + match &self.error { + Some(e) if self.text.is_empty() => format!("Browser error: {e}"), + Some(e) => format!("{}\nBrowser error: {e}", self.text), + None if self.text.is_empty() => "Done.".to_string(), + None => self.text.clone(), + } + } + + fn render_images(&self) -> Vec { + // An error tool result must be text-only (Anthropic rejects images in a + // tool_result with is_error=true). + if self.error.is_some() { + return Vec::new(); + } + self.images + .iter() + .map(|data| ImageData { + media_type: "image/png".to_string(), + base64_data: data.clone(), + }) + .collect() + } + + fn cap_images(&mut self, max_edge: u32) { + for data in &mut self.images { + if let Some((_, capped)) = cap_base64_image("image/png", data, max_edge) { + *data = capped; + } + } + } +} + +impl ToolResult for BrowserOutput { + fn is_success(&self) -> bool { + self.error.is_none() + } +} + +/// Build a browser tool's spec. `read_only` tools are also offered to +/// read-only sub-agents. +pub(crate) fn spec( + name: &'static str, + description: &'static str, + mut parameters_schema: Value, + read_only: bool, + title_template: &'static str, +) -> ToolSpec { + target_properties(&mut parameters_schema); + let mut caps = vec![ + capabilities::SCOPE_AGENT, + capabilities::SCOPE_AGENT_DIFF, + capabilities::SCOPE_SUBAGENT_DEFAULT, + capabilities::SCOPE_SUBAGENT_DEFAULT_DIFF, + ]; + if read_only { + caps.push(capabilities::READ_ONLY); + caps.push(capabilities::SCOPE_SUBAGENT_READ_ONLY); + } + ToolSpec { + name: name.into(), + description: description.into(), + parameters_schema, + annotations: Some(serde_json::json!({"readOnlyHint": read_only, "openWorldHint": true})), + capabilities: ToolSpec::capabilities(&caps), + multiline_params: &[], + hidden: false, + title_template: Some(title_template), + } +} + +/// Implement `Tool` for a browser tool whose work is `run(manager, input)`. +macro_rules! browser_tool { + ($tool:ident, $input:ty, $run:path) => { + #[async_trait::async_trait] + impl crate::tools::core::Tool for $tool { + type Input = $input; + type Output = $crate::tools::impls::browser::BrowserOutput; + + fn spec(&self) -> crate::tools::core::ToolSpec { + Self::make_spec() + } + + async fn execute<'a>( + &self, + context: &mut crate::tools::core::ToolContext<'a>, + input: &mut Self::Input, + ) -> anyhow::Result { + use crate::tools::services::ToolServicesAccess; + let Some(manager) = context.browser_sessions() else { + return Ok($crate::tools::impls::browser::BrowserOutput::unavailable( + input.target.profile(), + )); + }; + Ok($run(manager, input).await) + } + } + }; +} +pub(crate) use browser_tool; + +#[cfg(test)] +pub(crate) mod test_support { + use super::*; + use crate::mocks::ToolTestFixture; + use std::time::Duration; + + pub fn data_url(html: &str) -> String { + format!( + "data:text/html;base64,{}", + base64::engine::general_purpose::STANDARD.encode(html) + ) + } + + /// The first `ref_N` in `text` on a line containing `needle`. + pub fn ref_on_line(text: &str, needle: &str) -> String { + let line = text + .lines() + .find(|l| l.contains(needle)) + .unwrap_or_else(|| panic!("no line with {needle:?} in:\n{text}")); + line.split(['[', ']']) + .find(|p| p.starts_with("ref_")) + .unwrap_or_else(|| panic!("no ref on {line:?}")) + .to_string() + } + + /// Register a throwaway browser with short limits as the default profile, + /// so tools pick it up instead of launching one with the real limits. + pub async fn register_short_timeout_browser(fixture: &ToolTestFixture) -> Result<()> { + let session = BrowserSession::open(BrowserLaunchConfig::default(), DEFAULT_PROFILE) + .await? + .with_timeouts(web::BrowserTimeouts { + command: Duration::from_secs(1), + navigation: Duration::from_secs(2), + }); + fixture + .browser_sessions() + .unwrap() + .register(Arc::new(session), DEFAULT_PROFILE); + Ok(()) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn error_output_is_text_only() { + // Anthropic rejects images in a tool_result with is_error=true. + let out = BrowserOutput { + profile: "default".into(), + text: "Screenshot".into(), + images: vec!["ZmFrZQ==".into()], + error: Some("step 2 failed".into()), + }; + assert!(!out.is_success()); + assert!(out.render_images().is_empty()); + assert_eq!( + out.render(&mut ResourcesTracker::default()), + "Screenshot\nBrowser error: step 2 failed" + ); + } + + #[test] + fn outputs_of_older_versions_still_load() { + let old = serde_json::json!({ + "profile": "default", + "observation": {"url": "https://x.test", "title": "X", "text": "hi"}, + "screenshot_base64": "ZmFrZQ==" + }); + let out: BrowserOutput = serde_json::from_value(old).unwrap(); + assert!(out.is_success()); + } + + #[test] + fn launch_config_maps_default_to_ephemeral_and_names_to_persistent() { + let default = launch_config_for(DEFAULT_PROFILE, false); + assert!(matches!(default.profile, BrowserProfile::Ephemeral)); + + let named = launch_config_for("elster", false); + match named.profile { + BrowserProfile::Persistent(path) => { + assert_eq!(path.file_name().unwrap(), "elster"); + assert!(path.to_string_lossy().contains("browser-profiles")); + } + _ => panic!("named profile should be persistent"), + } + } + + #[test] + fn launch_config_sanitizes_path_traversal_in_profile_names() { + let named = launch_config_for("../evil name", false); + let BrowserProfile::Persistent(path) = named.profile else { + panic!("expected persistent"); + }; + let last = path.file_name().unwrap().to_string_lossy().to_string(); + assert!(!last.contains('/'), "no path separators: {last}"); + assert!(!last.contains(".."), "no traversal: {last}"); + assert_eq!(last, "___evil_name"); + } + + #[test] + fn browser_tools_are_exposed_to_the_agent() { + use crate::tools::scope::ToolScope; + let registry = crate::tools::test_registry(); + let names: Vec = registry + .get_tool_definitions_with_capability(ToolScope::Agent.tag()) + .into_iter() + .map(|d| d.name) + .collect(); + for expected in [ + "browser_navigate", + "browser_computer", + "browser_read_page", + "browser_find", + "browser_get_page_text", + "browser_form_input", + "browser_javascript", + "browser_read_console_messages", + "browser_read_network_requests", + "browser_resize_window", + "browser_tabs_context", + "browser_tabs_create", + "browser_tabs_select", + "browser_tabs_close", + "browser_batch", + "browser_close", + "browser_login", + "browser_profiles", + ] { + assert!( + names.contains(&expected.to_string()), + "missing {expected}; have: {names:?}" + ); + } + } +} diff --git a/crates/code_assistant_core/src/tools/impls/browser/page.rs b/crates/code_assistant_core/src/tools/impls/browser/page.rs new file mode 100644 index 00000000..f6fa506d --- /dev/null +++ b/crates/code_assistant_core/src/tools/impls/browser/page.rs @@ -0,0 +1,549 @@ +//! Reading and changing a page: navigate, the accessibility tree, text search, +//! page text, form input and scripts. + +use super::{BrowserOutput, Resolved, Target, browser_tool, spec}; +use crate::tools::core::ToolSpec; +use serde::{Deserialize, Serialize}; +use serde_json::json; +use web::{BrowserSessionManager, BrowserTimeout}; + +/// Default and ceiling for text-producing reads, in characters. +const DEFAULT_MAX_CHARS: usize = 50_000; + +// --------------------------------------------------------------------------- +// browser_navigate +// --------------------------------------------------------------------------- + +#[derive(Deserialize, Serialize)] +pub struct NavigateInput { + pub url: String, + #[serde(flatten)] + pub target: Target, +} + +pub struct BrowserNavigateTool; + +impl BrowserNavigateTool { + fn make_spec() -> ToolSpec { + spec( + "browser_navigate", + "Navigate a browser tab to a URL, or go \"back\"/\"forward\" in its history. Opens the \ + browser if none is running. Returns the resulting URL and title; look at the page \ + with browser_read_page, browser_computer (screenshot) or browser_get_page_text.", + json!({ + "type": "object", + "properties": { + "url": {"type": "string", "description": "URL to load (scheme optional, defaults to https), or \"back\"/\"forward\""} + }, + "required": ["url"] + }), + true, + "Navigating to {url}", + ) + } +} + +browser_tool!(BrowserNavigateTool, NavigateInput, navigate); + +pub(crate) async fn navigate( + manager: &BrowserSessionManager, + input: &NavigateInput, +) -> BrowserOutput { + let mut r = match Resolved::new(manager, &input.target, true).await { + Ok(r) => r, + Err(out) => return out, + }; + let mut notes = Vec::new(); + let result = match input.url.trim() { + "back" => r.tab.history(-1).await, + "forward" => r.tab.history(1).await, + url => { + let url = + if url.contains("://") || url.starts_with("about:") || url.starts_with("data:") { + url.to_string() + } else { + format!("https://{url}") + }; + match r.tab.navigate(&url).await { + // A load event that never comes (a slow iframe or tracker) is + // no failure: the page is usually usable. + Err(e) if e.downcast_ref::().is_some() => { + notes.push(format!( + "The page did not finish loading ({e}); it is shown as far as it got." + )); + Ok(()) + } + other => other, + } + } + }; + if let Err(e) = result { + return r.failure(format!("Navigation failed: {e}")); + } + r.tab.settle().await; + let (url, title) = r.tab.location().await; + // The navigation itself is the headline, not a note. + notes.extend( + r.notes() + .await + .into_iter() + .filter(|n| !n.starts_with("The page navigated")), + ); + r.output(format!("[{}] {url}\nTitle: {title}", r.tab.id())) + .with_notes(notes) +} + +// --------------------------------------------------------------------------- +// browser_read_page +// --------------------------------------------------------------------------- + +#[derive(Deserialize, Serialize, Default, Clone, Copy, PartialEq, Eq)] +#[serde(rename_all = "snake_case")] +pub enum ReadFilter { + #[default] + All, + Interactive, +} + +#[derive(Deserialize, Serialize)] +pub struct ReadPageInput { + #[serde(default)] + pub filter: ReadFilter, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub ref_id: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub depth: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub max_chars: Option, + #[serde(flatten)] + pub target: Target, +} + +pub struct BrowserReadPageTool; + +impl BrowserReadPageTool { + fn make_spec() -> ToolSpec { + spec( + "browser_read_page", + "Read the page as a YAML-style accessibility tree: `- role \"name\" [ref_N] state`. \ + Use the ref_N handles with browser_computer, browser_form_input and \ + browser_find. Prefer this over screenshots for text and structure.", + json!({ + "type": "object", + "properties": { + "filter": {"type": "string", "enum": ["all", "interactive"], "description": "'interactive' lists only clickable/typable elements; 'all' (default) the full tree"}, + "ref_id": {"type": "string", "description": "Only the subtree under this ref_N"}, + "depth": {"type": "integer", "description": "Maximum tree depth (default 15)"}, + "max_chars": {"type": "integer", "description": "Maximum output characters (default 50000)"} + } + }), + true, + "Reading the page", + ) + } +} + +browser_tool!(BrowserReadPageTool, ReadPageInput, read_page); + +pub(crate) async fn read_page( + manager: &BrowserSessionManager, + input: &ReadPageInput, +) -> BrowserOutput { + let r = match Resolved::new(manager, &input.target, false).await { + Ok(r) => r, + Err(out) => return out, + }; + let lines = match r + .tab + .read_page( + input.filter == ReadFilter::Interactive, + input.ref_id.as_deref(), + input.depth.unwrap_or(15), + ) + .await + { + Ok(lines) => lines, + Err(e) => return r.failure(e), + }; + let (url, title) = r.tab.location().await; + let header = format!("[{}] {url} — {title}", r.tab.id()); + let max_chars = input.max_chars.unwrap_or(DEFAULT_MAX_CHARS); + r.output(format!("{header}\n{}", join_bounded(&lines, max_chars))) +} + +/// Join lines up to `max_chars`, cutting at a line boundary with a note. +fn join_bounded(lines: &[String], max_chars: usize) -> String { + let mut out = String::new(); + for line in lines { + if out.len() + line.len() + 1 > max_chars { + out.push_str(&format!( + "… (truncated at {max_chars} characters; narrow it with ref_id, depth or filter)" + )); + return out; + } + out.push_str(line); + out.push('\n'); + } + out.truncate(out.trim_end().len()); + out +} + +// --------------------------------------------------------------------------- +// browser_find +// --------------------------------------------------------------------------- + +#[derive(Deserialize, Serialize)] +pub struct FindInput { + pub query: String, + #[serde(flatten)] + pub target: Target, +} + +pub struct BrowserFindTool; + +impl BrowserFindTool { + fn make_spec() -> ToolSpec { + spec( + "browser_find", + "Search the page's accessibility tree for elements whose role, name or state \ + contains `query` (case-insensitive). Returns up to 20 matching lines with ref_N \ + handles.", + json!({ + "type": "object", + "properties": { + "query": {"type": "string", "description": "Text to look for, e.g. \"Sign in\", \"search\", \"checkbox\""} + }, + "required": ["query"] + }), + true, + "Finding \"{query}\"", + ) + } +} + +browser_tool!(BrowserFindTool, FindInput, find); + +pub(crate) async fn find(manager: &BrowserSessionManager, input: &FindInput) -> BrowserOutput { + let r = match Resolved::new(manager, &input.target, false).await { + Ok(r) => r, + Err(out) => return out, + }; + match r.tab.find(&input.query, 20).await { + Ok(lines) if lines.is_empty() => { + r.output(format!("No elements match \"{}\".", input.query)) + } + Ok(lines) => r.output(lines.join("\n")), + Err(e) => r.failure(e), + } +} + +// --------------------------------------------------------------------------- +// browser_get_page_text +// --------------------------------------------------------------------------- + +#[derive(Deserialize, Serialize)] +pub struct GetPageTextInput { + #[serde(default, skip_serializing_if = "Option::is_none")] + pub max_chars: Option, + #[serde(flatten)] + pub target: Target, +} + +pub struct BrowserGetPageTextTool; + +impl BrowserGetPageTextTool { + fn make_spec() -> ToolSpec { + spec( + "browser_get_page_text", + "Extract the visible text of the page (its main/article content if it has one, \ + else the body).", + json!({ + "type": "object", + "properties": { + "max_chars": {"type": "integer", "description": "Maximum characters (default 50000)"} + } + }), + true, + "Reading the page text", + ) + } +} + +browser_tool!(BrowserGetPageTextTool, GetPageTextInput, get_page_text); + +pub(crate) async fn get_page_text( + manager: &BrowserSessionManager, + input: &GetPageTextInput, +) -> BrowserOutput { + let r = match Resolved::new(manager, &input.target, false).await { + Ok(r) => r, + Err(out) => return out, + }; + match r + .tab + .page_text(input.max_chars.unwrap_or(DEFAULT_MAX_CHARS)) + .await + { + Ok(text) => { + let (url, title) = r.tab.location().await; + r.output(format!("[{}] {url} — {title}\n\n{text}", r.tab.id())) + } + Err(e) => r.failure(e), + } +} + +// --------------------------------------------------------------------------- +// browser_form_input +// --------------------------------------------------------------------------- + +#[derive(Deserialize, Serialize)] +pub struct FormInputInput { + #[serde(rename = "ref")] + pub r#ref: String, + pub value: serde_json::Value, + #[serde(flatten)] + pub target: Target, +} + +pub struct BrowserFormInputTool; + +impl BrowserFormInputTool { + fn make_spec() -> ToolSpec { + spec( + "browser_form_input", + "Set a form control by ref: a select by option value or visible text (an array for \ + multi-select), a checkbox/radio/switch by true/false, a text field or \ + contenteditable by text. Fires the input/change events pages listen for.", + json!({ + "type": "object", + "properties": { + "ref": {"type": "string", "description": "ref_N from browser_read_page or browser_find"}, + "value": {"description": "The value: string, boolean, number, or an array of strings for a multi-select"} + }, + "required": ["ref", "value"] + }), + false, + "Setting {ref}", + ) + } +} + +browser_tool!(BrowserFormInputTool, FormInputInput, form_input); + +pub(crate) async fn form_input( + manager: &BrowserSessionManager, + input: &FormInputInput, +) -> BrowserOutput { + let mut r = match Resolved::new(manager, &input.target, false).await { + Ok(r) => r, + Err(out) => return out, + }; + match r.tab.form_input(&input.r#ref, &input.value).await { + Ok(done) => { + let notes = r.notes().await; + r.output(format!("{}: {done}", input.r#ref)) + .with_notes(notes) + } + Err(e) => r.failure(e), + } +} + +// --------------------------------------------------------------------------- +// browser_javascript +// --------------------------------------------------------------------------- + +#[derive(Deserialize, Serialize)] +pub struct JavascriptInput { + pub text: String, + #[serde(flatten)] + pub target: Target, +} + +pub struct BrowserJavascriptTool; + +impl BrowserJavascriptTool { + fn make_spec() -> ToolSpec { + spec( + "browser_javascript", + "Execute JavaScript in the page for debugging and inspection. REPL semantics: \ + top-level await works and the value of the last expression is returned (as JSON) — \ + write the expression, not `return`.", + json!({ + "type": "object", + "properties": { + "text": {"type": "string", "description": "The code to run"} + }, + "required": ["text"] + }), + false, + "Running JavaScript", + ) + } +} + +browser_tool!(BrowserJavascriptTool, JavascriptInput, javascript); + +pub(crate) async fn javascript( + manager: &BrowserSessionManager, + input: &JavascriptInput, +) -> BrowserOutput { + let mut r = match Resolved::new(manager, &input.target, false).await { + Ok(r) => r, + Err(out) => return out, + }; + match r.tab.javascript(&input.text).await { + Ok(value) => { + let notes = r.notes().await; + r.output(value).with_notes(notes) + } + Err(e) => r.failure(format!("JavaScript error: {e}")), + } +} + +#[cfg(test)] +mod tests { + use super::super::test_support::*; + use super::*; + use crate::mocks::ToolTestFixture; + use crate::tools::core::{Render, ResourcesTracker, Tool}; + use anyhow::Result; + + fn render(out: &BrowserOutput) -> String { + out.render(&mut ResourcesTracker::default()) + } + + #[tokio::test] + async fn navigate_read_find_fill_and_inspect() -> Result<()> { + let page = data_url( + "Form

Welcome

\ + \ + \ +

Main text

", + ); + let mut fixture = ToolTestFixture::new().with_browser_sessions(); + let mut context = fixture.context(); + + let mut nav = NavigateInput { + url: page, + target: Target::default(), + }; + let out = BrowserNavigateTool.execute(&mut context, &mut nav).await?; + assert!(out.error.is_none(), "{:?}", out.error); + assert!(render(&out).contains("Title: Form"), "{}", render(&out)); + assert!(out.images.is_empty(), "navigate returns no screenshot"); + + let mut read = ReadPageInput { + filter: ReadFilter::Interactive, + ref_id: None, + depth: None, + max_chars: None, + target: Target::default(), + }; + let tree = render(&BrowserReadPageTool.execute(&mut context, &mut read).await?); + let name = ref_on_line(&tree, "textbox \"Name\""); + let size = ref_on_line(&tree, "combobox \"Size\""); + + let mut find = FindInput { + query: "welcome".into(), + target: Target::default(), + }; + let found = render(&BrowserFindTool.execute(&mut context, &mut find).await?); + assert!(found.starts_with("- heading \"Welcome\""), "{found}"); + + for (r, value, expect) in [ + (&name, json!("Ada"), "set value"), + (&size, json!("Large"), "selected Large"), + ] { + let mut input = FormInputInput { + r#ref: r.clone(), + value, + target: Target::default(), + }; + let out = BrowserFormInputTool + .execute(&mut context, &mut input) + .await?; + assert!(render(&out).contains(expect), "{}", render(&out)); + } + + let mut js = JavascriptInput { + text: "[document.getElementById('n').value, document.getElementById('s').value]".into(), + target: Target::default(), + }; + let out = BrowserJavascriptTool.execute(&mut context, &mut js).await?; + assert_eq!(render(&out), r#"["Ada","Large"]"#); + + let mut text = GetPageTextInput { + max_chars: None, + target: Target::default(), + }; + let out = BrowserGetPageTextTool + .execute(&mut context, &mut text) + .await?; + assert!(render(&out).ends_with("\n\nMain text"), "{}", render(&out)); + Ok(()) + } + + #[tokio::test] + async fn reading_without_an_open_browser_is_a_clear_error() -> Result<()> { + let mut fixture = ToolTestFixture::new().with_browser_sessions(); + let mut context = fixture.context(); + let mut read = ReadPageInput { + filter: ReadFilter::All, + ref_id: None, + depth: None, + max_chars: None, + target: Target::default(), + }; + let out = BrowserReadPageTool.execute(&mut context, &mut read).await?; + assert!(out.error.unwrap().contains("browser_navigate")); + Ok(()) + } + + #[tokio::test] + async fn a_page_that_never_finishes_loading_is_shown_with_a_note() -> Result<()> { + use axum::response::Html; + use axum::{Router, routing::get}; + let app = Router::new() + .route( + "/slow", + get(|| async { + Html( + "Slow

Main content

\ + ", + ) + }), + ) + .route( + "/never", + get(|| async { + tokio::time::sleep(std::time::Duration::from_secs(600)).await; + Html("") + }), + ); + let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await?; + let addr = listener.local_addr()?; + tokio::spawn(async move { axum::serve(listener, app).await.unwrap() }); + + let mut fixture = ToolTestFixture::new().with_browser_sessions(); + register_short_timeout_browser(&fixture).await?; + let mut context = fixture.context(); + let mut nav = NavigateInput { + url: format!("http://{addr}/slow"), + target: Target::default(), + }; + let out = BrowserNavigateTool.execute(&mut context, &mut nav).await?; + assert!(out.error.is_none(), "{:?}", out.error); + let text = render(&out); + assert!(text.contains("Title: Slow"), "{text}"); + assert!(text.contains("did not finish loading"), "{text}"); + Ok(()) + } + + #[test] + fn join_bounded_cuts_at_a_line() { + let lines = vec!["- a".to_string(), "- bbbb".to_string(), "- c".to_string()]; + assert_eq!(join_bounded(&lines, 100), "- a\n- bbbb\n- c"); + let cut = join_bounded(&lines, 8); + assert!(cut.starts_with("- a\n…"), "{cut}"); + } +} diff --git a/crates/code_assistant_core/src/tools/impls/browser/profiles.rs b/crates/code_assistant_core/src/tools/impls/browser/profiles.rs new file mode 100644 index 00000000..630c4807 --- /dev/null +++ b/crates/code_assistant_core/src/tools/impls/browser/profiles.rs @@ -0,0 +1,698 @@ +//! Profiles: closing a profile's browser, the human-in-the-loop login, and +//! listing the persistent profiles on disk. + +use super::{BrowserOutput, DEFAULT_PROFILE, Target, browser_tool, launch_config_for, spec}; +use crate::tools::core::{ + Render, ResourcesTracker, Tool, ToolContext, ToolResult, ToolSpec, capabilities, +}; +use crate::tools::services::ToolServicesAccess; +use anyhow::Result; +use serde::{Deserialize, Serialize}; +use serde_json::json; +use std::path::{Path, PathBuf}; +use std::sync::Arc; +use std::time::{Duration, SystemTime, UNIX_EPOCH}; +use tools_core::permissions::{ + PermissionDecision, PermissionMediator, PermissionRequest, PermissionRequestReason, +}; +use web::{BrowserSession, BrowserSessionManager, BrowserTimeout, Tab}; + +// --------------------------------------------------------------------------- +// browser_close +// --------------------------------------------------------------------------- + +#[derive(Deserialize, Serialize)] +pub struct CloseInput { + #[serde(flatten)] + pub target: Target, +} + +pub struct BrowserCloseTool; + +impl BrowserCloseTool { + fn make_spec() -> ToolSpec { + let mut spec = spec( + "browser_close", + "Close a profile's browser with all its tabs, flushing a persistent profile's \ + session to disk.", + json!({"type": "object", "properties": {}}), + false, + "Closing browser", + ); + if let Some(props) = spec.parameters_schema["properties"].as_object_mut() { + props.remove("tab_id"); + } + spec + } +} + +browser_tool!(BrowserCloseTool, CloseInput, close); + +pub(crate) async fn close(manager: &BrowserSessionManager, input: &CloseInput) -> BrowserOutput { + let profile = input.target.profile(); + match manager.remove_by_label(profile) { + Some(session) => { + session.close().await; + BrowserOutput { + profile: profile.to_string(), + text: format!("Closed the browser of profile '{profile}'."), + ..Default::default() + } + } + None => BrowserOutput::failure(profile, "No browser is open for this profile."), + } +} + +// --------------------------------------------------------------------------- +// browser_login — human-in-the-loop login handoff +// --------------------------------------------------------------------------- + +#[derive(Deserialize, Serialize)] +pub struct BrowserLoginInput { + pub url: String, + pub profile: String, +} + +pub struct BrowserLoginTool; + +/// Navigate, tolerating a load event that never comes: a slow portal page +/// must not fail a login the user already completed. +async fn navigate_tolerating_slow_load(tab: &Tab, url: &str) -> Result<()> { + match tab.navigate(url).await { + Err(e) if e.downcast_ref::().is_none() => Err(e), + _ => Ok(()), + } +} + +/// The handoff itself, factored out so tests can drive it headlessly. In +/// production `headful` is always true: a visible window opens, the human logs +/// in, and only their approval lets the agent continue in that same +/// authenticated session. +async fn login_handoff( + manager: &BrowserSessionManager, + handler: &dyn PermissionMediator, + tool_id: Option<&str>, + profile: &str, + url: &str, + headful: bool, +) -> Result { + // A login needs a fresh window: replace any existing (possibly headless) + // session for this profile. + if let Some(existing) = manager.remove_by_label(profile) { + existing.close().await; + } + let session = match BrowserSession::open(launch_config_for(profile, headful), profile).await { + Ok(s) => Arc::new(s), + Err(e) => { + return Ok(BrowserOutput::failure( + profile, + format!("Failed to open browser: {e}"), + )); + } + }; + let navigated = match session.active_tab() { + Ok(tab) => navigate_tolerating_slow_load(&tab, url).await, + Err(e) => Err(e), + }; + if let Err(e) = navigated { + session.close().await; + return Ok(BrowserOutput::failure( + profile, + format!("Navigation failed: {e}"), + )); + } + + // Pause for the human to log in, then resume on approval. This travels the + // same seam as any other permission prompt (TUI prompt / Telegram keyboard). + let params = json!({ + "action": "browser_login", + "profile": profile, + "url": url, + "instructions": "A browser window has opened. Log in there (password, 2FA, \ + certificate as needed), then approve to let the agent continue.", + }); + let decision = handler + .request_permission(PermissionRequest { + tool_id, + tool_name: "browser_login", + reason: PermissionRequestReason::ToolInvocation { params: ¶ms }, + }) + .await?; + + match decision { + PermissionDecision::Denied => { + session.close().await; + Ok(BrowserOutput::failure( + profile, + "User declined the login handoff.", + )) + } + PermissionDecision::GrantedOnce + | PermissionDecision::GrantedSession + | PermissionDecision::GrantedPersistent => { + // Swap the visible login window for a headless browser on the same + // profile, carrying the login across. The agent then browses in the + // background, and the user can close the login window without killing + // the session. + match finalize_login_headless(profile, session, url, headful).await { + Ok(headless) => { + manager.register(headless.clone(), profile); + // Note the login so a later session can discover it via + // browser_profiles instead of asking the user to log in again. + record_login_in(&profiles_root(), profile, url); + let (url, title) = match headless.active_tab() { + Ok(tab) => tab.location().await, + Err(_) => Default::default(), + }; + Ok(BrowserOutput { + profile: profile.to_string(), + text: format!( + "Logged in; profile '{profile}' now browses in the background.\n\ + {url}\nTitle: {title}" + ), + ..Default::default() + }) + } + Err(e) => Ok(BrowserOutput::failure( + profile, + format!("Login succeeded but switching to a background browser failed: {e}"), + )), + } + } + } +} + +/// Page shown briefly in the login window after a successful handoff, so the +/// user sees it worked and knows the window is safe to close. +const LOGIN_SUCCESS_PAGE: &str = "data:text/html,\ +

✅ Login successful

\ +

You can close this window — the agent now continues in the background.

\ + "; + +/// After a granted login, replace the visible headful browser with a headless +/// one on the same profile, transferring the full cookie jar (including +/// in-memory session cookies a disk flush would drop) so the login survives. +/// Chrome locks the profile dir, so the headful window must fully close before +/// the headless one can start. +async fn finalize_login_headless( + profile: &str, + headful: Arc, + url: &str, + was_headful: bool, +) -> Result> { + // Capture the jar while the authenticated window is still alive. + let cookies = headful.export_cookies().await.unwrap_or_default(); + // Reassure the user in the visible window, give them a moment to read it, + // then close (releasing the profile-dir lock). + if let Ok(tab) = headful.active_tab() { + let _ = tab.navigate(LOGIN_SUCCESS_PAGE).await; + } + if was_headful { + tokio::time::sleep(Duration::from_millis(1500)).await; + } + headful.close().await; + + // Relaunch the same profile headless and restore the login. + let headless = + Arc::new(BrowserSession::open(launch_config_for(profile, false), profile).await?); + let tab = headless.active_tab()?; + if url.starts_with("http") { + // Land on an http origin so cookies can be set, inject the jar, then + // reload so the (now present) session cookies take effect. + let _ = tab.navigate(url).await; + let _ = tab.import_cookies(cookies).await; + } + navigate_tolerating_slow_load(&tab, url).await?; + Ok(headless) +} + +#[async_trait::async_trait] +impl Tool for BrowserLoginTool { + type Input = BrowserLoginInput; + type Output = BrowserOutput; + + fn spec(&self) -> ToolSpec { + let caps = [ + capabilities::READ_ONLY, + capabilities::SCOPE_AGENT, + capabilities::SCOPE_AGENT_DIFF, + capabilities::SCOPE_SUBAGENT_DEFAULT, + capabilities::SCOPE_SUBAGENT_DEFAULT_DIFF, + ]; + ToolSpec { + name: "browser_login".into(), + description: concat!( + "Log in to a website AS THE USER without ever seeing their credentials. ", + "Opens a VISIBLE browser window on the named persistent profile at the login ", + "URL, then pauses until the user has logged in there (password, 2FA, ", + "certificate) and approves. The login is saved under the profile for reuse.\n", + "Tell the user what you are doing before calling this. Afterwards pass the same ", + "`profile` to the browser tools." + ) + .into(), + parameters_schema: json!({ + "type": "object", + "properties": { + "url": {"type": "string", "description": "Login page URL"}, + "profile": { + "type": "string", + "description": "Persistent profile name to store the login under (e.g. \"elster\")" + } + }, + "required": ["url", "profile"] + }), + annotations: Some(json!({"readOnlyHint": true, "openWorldHint": true})), + capabilities: ToolSpec::capabilities(&caps), + multiline_params: &[], + hidden: false, + title_template: Some("Logging in at {url}"), + } + } + + async fn execute<'a>( + &self, + context: &mut ToolContext<'a>, + input: &mut Self::Input, + ) -> Result { + let profile = input.profile.clone(); + // The handoff needs a frontend that can prompt the human. + let Some(handler) = context.permission_handler else { + return Ok(BrowserOutput::failure( + &profile, + "Login handoff needs an interactive frontend, which this context does not have.", + )); + }; + let tool_id = context.tool_id.clone(); + let Some(manager) = context.browser_sessions() else { + return Ok(BrowserOutput::unavailable(&profile)); + }; + login_handoff( + manager, + handler, + tool_id.as_deref(), + &profile, + &input.url, + true, + ) + .await + } +} + +// --------------------------------------------------------------------------- +// browser_profiles — discover existing profiles and their last login +// --------------------------------------------------------------------------- + +/// The directory holding all persistent browser profiles and their sidecar +/// metadata files (`/browser-profiles`). +pub(crate) fn profiles_root() -> PathBuf { + crate::config_dir::config_dir().join("browser-profiles") +} + +/// Sanitize a profile name to a filesystem-safe token — the same rule the launch +/// path uses, so a profile's dir and its `.meta.json` sidecar agree. +pub(crate) fn sanitize_profile(profile: &str) -> String { + profile + .chars() + .map(|c| { + if c.is_alphanumeric() || c == '-' || c == '_' { + c + } else { + '_' + } + }) + .collect() +} + +/// Sidecar metadata recorded next to a persistent profile after a login. Kept +/// *beside* the profile dir (not inside it) so Chrome's user-data-dir stays +/// pristine and there is no central index to race on across sessions. +#[derive(Debug, Serialize, Deserialize)] +struct ProfileMeta { + /// The login URL last used for this profile. + url: String, + /// When the last login handoff succeeded (unix seconds). + logged_in_at_unix: i64, +} + +fn meta_path_in(root: &Path, profile: &str) -> PathBuf { + root.join(format!("{}.meta.json", sanitize_profile(profile))) +} + +/// Record a successful login for `profile`. Best-effort — a metadata write must +/// never fail a login — and skipped for the ephemeral default profile, which +/// has no persistent dir. +fn record_login_in(root: &Path, profile: &str, url: &str) { + if profile == DEFAULT_PROFILE { + return; + } + let now = SystemTime::now() + .duration_since(UNIX_EPOCH) + .map(|d| d.as_secs() as i64) + .unwrap_or(0); + let meta = ProfileMeta { + url: url.to_string(), + logged_in_at_unix: now, + }; + let _ = std::fs::create_dir_all(root); + if let Ok(json) = serde_json::to_string_pretty(&meta) { + let _ = std::fs::write(meta_path_in(root, profile), json); + } +} + +fn read_meta_in(root: &Path, profile: &str) -> Option { + let data = std::fs::read_to_string(meta_path_in(root, profile)).ok()?; + serde_json::from_str(&data).ok() +} + +/// List the persistent profiles on disk (subdirectories of the root), each with +/// its login metadata if recorded, sorted by name. +fn list_profiles_in(root: &Path) -> Vec<(String, Option)> { + let Ok(entries) = std::fs::read_dir(root) else { + return Vec::new(); + }; + let mut out: Vec<(String, Option)> = entries + .flatten() + .filter(|e| e.file_type().map(|t| t.is_dir()).unwrap_or(false)) + .filter_map(|e| e.file_name().to_str().map(String::from)) + .map(|name| { + let meta = read_meta_in(root, &name); + (name, meta) + }) + .collect(); + out.sort_by(|a, b| a.0.cmp(&b.0)); + out +} + +/// Render a duration as a coarse, human-friendly "N units ago", so the model can +/// judge at a glance whether a login is likely still fresh. +fn humanize_ago(seconds: i64) -> String { + let s = seconds.max(0); + const MIN: i64 = 60; + const HOUR: i64 = 60 * MIN; + const DAY: i64 = 24 * HOUR; + const WEEK: i64 = 7 * DAY; + const MONTH: i64 = 30 * DAY; + const YEAR: i64 = 365 * DAY; + if s < MIN { + return "just now".to_string(); + } + let (n, unit) = if s < HOUR { + (s / MIN, "minute") + } else if s < DAY { + (s / HOUR, "hour") + } else if s < WEEK { + (s / DAY, "day") + } else if s < MONTH { + (s / WEEK, "week") + } else if s < YEAR { + (s / MONTH, "month") + } else { + (s / YEAR, "year") + }; + format!("{n} {unit}{} ago", if n == 1 { "" } else { "s" }) +} + +/// Best-effort host extraction for display, e.g. `https://x.de/a` → `x.de`. +fn host_of(url: &str) -> String { + let after = url.split("://").nth(1).unwrap_or(url); + let host = after.split('/').next().unwrap_or(after); + host.rsplit('@').next().unwrap_or(host).to_string() +} + +/// Seconds elapsed since a recorded unix timestamp, floored at zero. +fn seconds_since(unix: i64) -> i64 { + let now = SystemTime::now() + .duration_since(UNIX_EPOCH) + .map(|d| d.as_secs() as i64) + .unwrap_or(0); + (now - unix).max(0) +} + +#[derive(Deserialize, Serialize, Default)] +pub struct BrowserProfilesInput {} + +#[derive(Serialize, Deserialize)] +pub struct BrowserProfilesOutput { + text: String, +} + +impl Render for BrowserProfilesOutput { + fn status(&self) -> String { + "Listed browser profiles".to_string() + } + + fn render(&self, _tracker: &mut ResourcesTracker) -> String { + self.text.clone() + } +} + +impl ToolResult for BrowserProfilesOutput { + fn is_success(&self) -> bool { + true + } +} + +pub struct BrowserProfilesTool; + +#[async_trait::async_trait] +impl Tool for BrowserProfilesTool { + type Input = BrowserProfilesInput; + type Output = BrowserProfilesOutput; + + fn spec(&self) -> ToolSpec { + let caps = [ + capabilities::READ_ONLY, + capabilities::SCOPE_AGENT, + capabilities::SCOPE_AGENT_DIFF, + capabilities::SCOPE_SUBAGENT_DEFAULT, + capabilities::SCOPE_SUBAGENT_DEFAULT_DIFF, + ]; + ToolSpec { + name: "browser_profiles".into(), + description: concat!( + "List the browser profiles that already exist on disk, so you can reuse an ", + "existing login instead of asking the user to log in again. Each persistent ", + "profile shows how long ago it was LAST logged in and to which site. That is the ", + "last recorded login, not a guarantee it is still valid — verify by navigating. ", + "\"default\" is the ephemeral throwaway browser (no persisted login). To use a ", + "profile pass it as `profile` to the browser tools; to create one use ", + "browser_login." + ) + .into(), + parameters_schema: json!({ "type": "object", "properties": {} }), + annotations: Some(json!({"readOnlyHint": true})), + capabilities: ToolSpec::capabilities(&caps), + multiline_params: &[], + hidden: false, + title_template: Some("Listing browser profiles"), + } + } + + async fn execute<'a>( + &self, + context: &mut ToolContext<'a>, + _input: &mut Self::Input, + ) -> Result { + let profiles = list_profiles_in(&profiles_root()); + let manager = context.browser_sessions(); + + let mut lines = vec![ + "Profiles:".to_string(), + " default — ephemeral (throwaway)".to_string(), + ]; + for (name, meta) in &profiles { + let open = manager + .map(|m| m.get_by_label(name).is_some()) + .unwrap_or(false); + let open_suffix = if open { " · open" } else { "" }; + let detail = match meta { + Some(m) => format!( + "last login: {}, {} (verify by navigating)", + host_of(&m.url), + humanize_ago(seconds_since(m.logged_in_at_unix)), + ), + None => "no login recorded yet".to_string(), + }; + lines.push(format!(" {name} — persistent · {detail}{open_suffix}")); + } + if profiles.is_empty() { + lines.push(String::new()); + lines.push( + "No persistent profiles yet. Use browser_login to create one (it persists the \ + login for reuse)." + .to_string(), + ); + } + Ok(BrowserProfilesOutput { + text: lines.join("\n"), + }) + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::mocks::ToolTestFixture; + + const DEMO_PAGE: &str = + "Login Demo

Welcome

"; + + #[test] + fn humanize_ago_rounds_coarsely() { + assert_eq!(humanize_ago(0), "just now"); + assert_eq!(humanize_ago(59), "just now"); + assert_eq!(humanize_ago(60), "1 minute ago"); + assert_eq!(humanize_ago(3 * 60), "3 minutes ago"); + assert_eq!(humanize_ago(3600), "1 hour ago"); + assert_eq!(humanize_ago(5 * 3600), "5 hours ago"); + assert_eq!(humanize_ago(24 * 3600), "1 day ago"); + assert_eq!(humanize_ago(3 * 24 * 3600), "3 days ago"); + assert_eq!(humanize_ago(10 * 24 * 3600), "1 week ago"); + assert_eq!(humanize_ago(40 * 24 * 3600), "1 month ago"); + assert_eq!(humanize_ago(400 * 24 * 3600), "1 year ago"); + // Never negative, even if a clock skew makes "now" earlier. + assert_eq!(humanize_ago(-500), "just now"); + } + + #[test] + fn host_of_extracts_display_host() { + assert_eq!( + host_of("https://www.elster.de/eportal/start"), + "www.elster.de" + ); + assert_eq!(host_of("http://x.de"), "x.de"); + assert_eq!(host_of("elster.de/path"), "elster.de"); + assert_eq!(host_of("https://user@host.de/x"), "host.de"); + } + + #[test] + fn profiles_are_listed_with_recorded_login() { + let tmp = tempfile::tempdir().unwrap(); + let root = tmp.path(); + // Two profile dirs; only one has a recorded login. + std::fs::create_dir_all(root.join("elster")).unwrap(); + std::fs::create_dir_all(root.join("fresh")).unwrap(); + record_login_in(root, "elster", "https://www.elster.de/eportal"); + + let listed = list_profiles_in(root); + let names: Vec<&str> = listed.iter().map(|(n, _)| n.as_str()).collect(); + assert_eq!(names, vec!["elster", "fresh"], "listed and sorted by name"); + + let meta = listed + .iter() + .find(|(n, _)| n == "elster") + .and_then(|(_, m)| m.as_ref()) + .expect("elster has a recorded login"); + assert_eq!(host_of(&meta.url), "www.elster.de"); + + let fresh = listed.iter().find(|(n, _)| n == "fresh").unwrap(); + assert!(fresh.1.is_none(), "fresh profile has no login recorded"); + } + + #[test] + fn record_login_skips_ephemeral_default() { + let tmp = tempfile::tempdir().unwrap(); + record_login_in(tmp.path(), DEFAULT_PROFILE, "https://x.de"); + assert!( + read_meta_in(tmp.path(), DEFAULT_PROFILE).is_none(), + "the throwaway default profile must not get a sidecar" + ); + } + + #[tokio::test] + async fn browser_profiles_lists_the_default_profile() -> Result<()> { + // Regardless of what's on disk, the output always leads with the header + // and the ever-present ephemeral default. + let mut fixture = ToolTestFixture::new().with_browser_sessions(); + let mut context = fixture.context(); + let mut input = BrowserProfilesInput::default(); + let out = BrowserProfilesTool + .execute(&mut context, &mut input) + .await?; + let text = out.render(&mut ResourcesTracker::default()); + assert!(text.starts_with("Profiles:"), "got: {text}"); + assert!( + text.contains("default — ephemeral (throwaway)"), + "got: {text}" + ); + Ok(()) + } + + /// Mediator returning a fixed decision, standing in for the human at the + /// browser. Lets us drive the handoff headlessly (no window popped). + struct ScriptedMediator(PermissionDecision); + + #[async_trait::async_trait] + impl PermissionMediator for ScriptedMediator { + async fn request_permission( + &self, + _request: PermissionRequest<'_>, + ) -> Result { + Ok(self.0) + } + } + + #[tokio::test] + async fn login_handoff_grant_keeps_authenticated_session() -> Result<()> { + let manager = BrowserSessionManager::new(4); + let mediator = ScriptedMediator(PermissionDecision::GrantedOnce); + // Ephemeral profile ("default") so the test touches no config dir. + let out = login_handoff( + &manager, + &mediator, + None, + "default", + &super::super::test_support::data_url(DEMO_PAGE), + false, + ) + .await?; + assert!(out.error.is_none(), "grant error: {:?}", out.error); + assert!( + out.text.contains("Login Demo"), + "authenticated page reported: {}", + out.text + ); + assert!( + manager.get_by_label("default").is_some(), + "session should be kept after approval" + ); + manager.close_all().await; + Ok(()) + } + + #[tokio::test] + async fn login_handoff_deny_closes_and_reports() -> Result<()> { + let manager = BrowserSessionManager::new(4); + let mediator = ScriptedMediator(PermissionDecision::Denied); + let out = login_handoff( + &manager, + &mediator, + None, + "default", + &super::super::test_support::data_url(DEMO_PAGE), + false, + ) + .await?; + assert!(out.error.unwrap().contains("declined")); + assert!( + manager.get_by_label("default").is_none(), + "no session should remain after a denied handoff" + ); + Ok(()) + } + + #[tokio::test] + async fn browser_login_without_a_handler_is_a_clear_error() -> Result<()> { + // No permission handler ⇒ no way to ask the human ⇒ graceful error, + // and crucially no browser is launched. + let mut fixture = ToolTestFixture::new().with_browser_sessions(); + let mut context = fixture.context(); + let mut input = BrowserLoginInput { + url: super::super::test_support::data_url(DEMO_PAGE), + profile: "elster".into(), + }; + let out = BrowserLoginTool.execute(&mut context, &mut input).await?; + assert!(out.error.unwrap().contains("interactive frontend")); + Ok(()) + } +} diff --git a/crates/code_assistant_core/src/tools/impls/browser/tabs.rs b/crates/code_assistant_core/src/tools/impls/browser/tabs.rs new file mode 100644 index 00000000..22212bfc --- /dev/null +++ b/crates/code_assistant_core/src/tools/impls/browser/tabs.rs @@ -0,0 +1,485 @@ +//! Tabs of a profile's browser, and the viewport a tab emulates. + +use super::{BrowserOutput, Resolved, Target, browser_tool, get_or_open, spec}; +use crate::tools::core::ToolSpec; +use serde::{Deserialize, Serialize}; +use serde_json::json; +use web::BrowserSessionManager; + +/// Drop the `tab_id` property for tools that are not about one tab. +fn without_tab_id(mut spec: ToolSpec) -> ToolSpec { + if let Some(props) = spec.parameters_schema["properties"].as_object_mut() { + props.remove("tab_id"); + } + spec +} + +/// Make `tab_id` required for tools about one specific tab. +fn requiring_tab_id(mut spec: ToolSpec) -> ToolSpec { + spec.parameters_schema["properties"]["tab_id"] = + json!({"type": "string", "description": "The tab, as listed by browser_tabs_context"}); + spec.parameters_schema["required"] = json!(["tab_id"]); + spec +} + +// --------------------------------------------------------------------------- +// browser_tabs_context +// --------------------------------------------------------------------------- + +#[derive(Deserialize, Serialize)] +pub struct TabsContextInput { + #[serde(flatten)] + pub target: Target, +} + +pub struct BrowserTabsContextTool; + +impl BrowserTabsContextTool { + fn make_spec() -> ToolSpec { + without_tab_id(spec( + "browser_tabs_context", + "List the browser's tabs with their id, URL and title; the active tab is the one \ + tools use when no tab_id is given.", + json!({"type": "object", "properties": {}}), + true, + "Listing tabs", + )) + } +} + +browser_tool!(BrowserTabsContextTool, TabsContextInput, tabs_context); + +pub(crate) async fn tabs_context( + manager: &BrowserSessionManager, + input: &TabsContextInput, +) -> BrowserOutput { + let profile = input.target.profile(); + let Some(session) = manager.get_by_label(profile) else { + return BrowserOutput { + profile: profile.to_string(), + text: format!("No browser is open for profile '{profile}'."), + ..Default::default() + }; + }; + let _ = session.sync_tabs().await; + let lines: Vec = session + .tabs() + .await + .into_iter() + .map(|t| { + let active = if t.active { " (active)" } else { "" }; + format!("{}{active}: {} — {}", t.id, t.url, t.title) + }) + .collect(); + BrowserOutput { + profile: profile.to_string(), + text: lines.join("\n"), + ..Default::default() + } +} + +// --------------------------------------------------------------------------- +// browser_tabs_create +// --------------------------------------------------------------------------- + +#[derive(Deserialize, Serialize)] +pub struct TabsCreateInput { + #[serde(default, skip_serializing_if = "std::ops::Not::not")] + pub foreground: bool, + #[serde(flatten)] + pub target: Target, +} + +pub struct BrowserTabsCreateTool; + +impl BrowserTabsCreateTool { + fn make_spec() -> ToolSpec { + without_tab_id(spec( + "browser_tabs_create", + "Open a new blank tab (opening the browser if needed) and return its tab_id. It \ + opens in the background unless `foreground` is true; load a page with \ + browser_navigate and its tab_id.", + json!({ + "type": "object", + "properties": { + "foreground": {"type": "boolean", "description": "Make it the active tab"} + } + }), + false, + "Opening a tab", + )) + } +} + +browser_tool!(BrowserTabsCreateTool, TabsCreateInput, tabs_create); + +pub(crate) async fn tabs_create( + manager: &BrowserSessionManager, + input: &TabsCreateInput, +) -> BrowserOutput { + let profile = input.target.profile(); + let was_open = manager.get_by_label(profile).is_some(); + let session = match get_or_open(manager, profile).await { + Ok(s) => s, + Err(e) => return BrowserOutput::failure(profile, format!("Failed to open browser: {e}")), + }; + // A browser that was just opened already has its fresh blank tab. + let tab = if was_open { + session.create_tab(input.foreground).await + } else { + session.active_tab() + }; + match tab { + Ok(tab) => { + let active = session + .active_tab() + .map(|t| t.id() == tab.id()) + .unwrap_or(false); + BrowserOutput { + profile: profile.to_string(), + text: format!( + "Opened tab {}{}", + tab.id(), + if active { + " (active)" + } else { + " (in the background)" + } + ), + ..Default::default() + } + } + Err(e) => BrowserOutput::failure(profile, e.to_string()), + } +} + +// --------------------------------------------------------------------------- +// browser_tabs_select / browser_tabs_close +// --------------------------------------------------------------------------- + +#[derive(Deserialize, Serialize)] +pub struct TabIdInput { + #[serde(flatten)] + pub target: Target, +} + +pub struct BrowserTabsSelectTool; + +impl BrowserTabsSelectTool { + fn make_spec() -> ToolSpec { + requiring_tab_id(spec( + "browser_tabs_select", + "Make a tab the active one, which tools use when no tab_id is given.", + json!({"type": "object", "properties": {}}), + false, + "Selecting tab {tab_id}", + )) + } +} + +browser_tool!(BrowserTabsSelectTool, TabIdInput, tabs_select); + +pub(crate) async fn tabs_select( + manager: &BrowserSessionManager, + input: &TabIdInput, +) -> BrowserOutput { + let r = match Resolved::new(manager, &input.target, false).await { + Ok(r) => r, + Err(out) => return out, + }; + if let Err(e) = r.session.select_tab(r.tab.id()) { + return r.failure(e); + } + let (url, title) = r.tab.location().await; + r.output(format!("Active tab {}: {url} — {title}", r.tab.id())) +} + +pub struct BrowserTabsCloseTool; + +impl BrowserTabsCloseTool { + fn make_spec() -> ToolSpec { + requiring_tab_id(spec( + "browser_tabs_close", + "Close a tab. Closing the last tab closes the browser.", + json!({"type": "object", "properties": {}}), + false, + "Closing tab {tab_id}", + )) + } +} + +browser_tool!(BrowserTabsCloseTool, TabIdInput, tabs_close); + +pub(crate) async fn tabs_close( + manager: &BrowserSessionManager, + input: &TabIdInput, +) -> BrowserOutput { + let r = match Resolved::new(manager, &input.target, false).await { + Ok(r) => r, + Err(out) => return out, + }; + let id = r.tab.id().to_string(); + if r.session.tabs().await.len() <= 1 { + if let Some(session) = manager.remove_by_label(&r.profile) { + session.close().await; + } + return r.output(format!( + "Closed {id}, the last tab, so the browser was closed." + )); + } + match r.session.close_tab(&id).await { + Ok(()) => { + let active = r + .session + .active_tab() + .map(|t| t.id().to_string()) + .unwrap_or_default(); + r.output(format!("Closed {id}. Active tab: {active}")) + } + Err(e) => r.failure(e), + } +} + +// --------------------------------------------------------------------------- +// browser_resize_window +// --------------------------------------------------------------------------- + +#[derive(Deserialize, Serialize, Clone, Copy, PartialEq, Eq)] +#[serde(rename_all = "snake_case")] +pub enum Preset { + Mobile, + Tablet, + Desktop, +} + +#[derive(Deserialize, Serialize)] +pub struct ResizeInput { + #[serde(default, skip_serializing_if = "Option::is_none")] + pub preset: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub width: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub height: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub color_scheme: Option, + #[serde(flatten)] + pub target: Target, +} + +pub struct BrowserResizeWindowTool; + +impl BrowserResizeWindowTool { + fn make_spec() -> ToolSpec { + spec( + "browser_resize_window", + "Emulate a viewport size for a tab: presets mobile (375×812), tablet (768×1024), \ + desktop (1280×800, the default), or width + height. Below 768 px wide it also \ + emulates a phone (touch, Android user agent) — reload the page afterwards. \ + `color_scheme` emulates prefers-color-scheme.", + json!({ + "type": "object", + "properties": { + "preset": {"type": "string", "enum": ["mobile", "tablet", "desktop"]}, + "width": {"type": "integer"}, + "height": {"type": "integer"}, + "color_scheme": {"type": "string", "enum": ["light", "dark"]} + } + }), + false, + "Resizing the viewport", + ) + } +} + +browser_tool!(BrowserResizeWindowTool, ResizeInput, resize_window); + +pub(crate) async fn resize_window( + manager: &BrowserSessionManager, + input: &ResizeInput, +) -> BrowserOutput { + let r = match Resolved::new(manager, &input.target, false).await { + Ok(r) => r, + Err(out) => return out, + }; + let size = match (input.preset, input.width, input.height) { + (Some(Preset::Mobile), _, _) => Some((375, 812)), + (Some(Preset::Tablet), _, _) => Some((768, 1024)), + (Some(Preset::Desktop), _, _) => Some((web::DEFAULT_VIEWPORT.0, web::DEFAULT_VIEWPORT.1)), + (None, Some(w), Some(h)) => Some((w, h)), + (None, None, None) => None, + _ => return r.failure("give a preset, or both width and height"), + }; + let mut done = Vec::new(); + if let Some((w, h)) = size { + let mobile = w < 768; + if let Err(e) = r.tab.set_viewport(w, h, mobile).await { + return r.failure(e); + } + done.push(if mobile { + format!("Viewport {w}×{h} with phone emulation (reload the page to apply touch and user agent)") + } else { + format!("Viewport {w}×{h}") + }); + } + if let Some(scheme) = &input.color_scheme { + if let Err(e) = r.tab.set_color_scheme(Some(scheme)).await { + return r.failure(e); + } + done.push(format!("prefers-color-scheme: {scheme}")); + } + if done.is_empty() { + return r.failure("nothing to change: give a preset, width + height, or color_scheme"); + } + r.output(done.join("; ")) +} + +#[cfg(test)] +mod tests { + use super::super::page::{BrowserNavigateTool, NavigateInput}; + use super::*; + use crate::mocks::ToolTestFixture; + use crate::tools::core::{Render, ResourcesTracker, Tool}; + use anyhow::Result; + + fn render(out: BrowserOutput) -> String { + out.render(&mut ResourcesTracker::default()) + } + + fn on_tab(tab_id: &str) -> TabIdInput { + TabIdInput { + target: Target { + profile: None, + tab_id: Some(tab_id.into()), + }, + } + } + + #[tokio::test] + async fn tabs_open_list_select_and_close() -> Result<()> { + let mut fixture = ToolTestFixture::new().with_browser_sessions(); + let mut context = fixture.context(); + let mut list = TabsContextInput { + target: Target::default(), + }; + assert!( + render( + BrowserTabsContextTool + .execute(&mut context, &mut list) + .await? + ) + .contains("No browser") + ); + + let mut create = TabsCreateInput { + foreground: false, + target: Target::default(), + }; + assert_eq!( + render( + BrowserTabsCreateTool + .execute(&mut context, &mut create) + .await? + ), + "Opened tab t1 (active)" + ); + assert_eq!( + render( + BrowserTabsCreateTool + .execute(&mut context, &mut create) + .await? + ), + "Opened tab t2 (in the background)" + ); + let mut nav = NavigateInput { + url: "data:text/html,Two".into(), + target: Target { + profile: None, + tab_id: Some("t2".into()), + }, + }; + BrowserNavigateTool.execute(&mut context, &mut nav).await?; + let listed = render( + BrowserTabsContextTool + .execute(&mut context, &mut list) + .await?, + ); + assert!(listed.starts_with("t1 (active): about:blank"), "{listed}"); + assert!( + listed.contains("t2: data:text/html,Two — Two"), + "{listed}" + ); + + let out = render( + BrowserTabsSelectTool + .execute(&mut context, &mut on_tab("t2")) + .await?, + ); + assert!(out.starts_with("Active tab t2"), "{out}"); + assert_eq!( + render( + BrowserTabsCloseTool + .execute(&mut context, &mut on_tab("t2")) + .await? + ), + "Closed t2. Active tab: t1" + ); + let out = render( + BrowserTabsCloseTool + .execute(&mut context, &mut on_tab("t1")) + .await?, + ); + assert!( + out.contains("the last tab, so the browser was closed"), + "{out}" + ); + assert!( + fixture + .browser_sessions() + .unwrap() + .get_by_label("default") + .is_none() + ); + Ok(()) + } + + #[tokio::test] + async fn resize_emulates_a_phone_and_dark_mode() -> Result<()> { + let mut fixture = ToolTestFixture::new().with_browser_sessions(); + let mut context = fixture.context(); + let mut nav = NavigateInput { + url: "data:text/html,x".into(), + target: Target::default(), + }; + BrowserNavigateTool.execute(&mut context, &mut nav).await?; + let mut resize = ResizeInput { + preset: Some(Preset::Mobile), + width: None, + height: None, + color_scheme: Some("dark".into()), + target: Target::default(), + }; + let out = render( + BrowserResizeWindowTool + .execute(&mut context, &mut resize) + .await?, + ); + assert!( + out.starts_with("Viewport 375×812 with phone emulation"), + "{out}" + ); + assert!(out.ends_with("prefers-color-scheme: dark"), "{out}"); + + let tab = fixture + .browser_sessions() + .unwrap() + .get_by_label("default") + .unwrap() + .active_tab()?; + assert_eq!( + tab.javascript("[innerWidth, matchMedia('(prefers-color-scheme: dark)').matches]") + .await?, + "[375,true]" + ); + Ok(()) + } +} diff --git a/crates/code_assistant_core/src/tools/impls/mod.rs b/crates/code_assistant_core/src/tools/impls/mod.rs index dbb5c7a9..814cf6d7 100644 --- a/crates/code_assistant_core/src/tools/impls/mod.rs +++ b/crates/code_assistant_core/src/tools/impls/mod.rs @@ -26,10 +26,6 @@ pub mod write_file; pub mod write_stdin; // Re-export all tools for registration -pub use browser::{ - BrowserActTool, BrowserCloseTool, BrowserLoginTool, BrowserNavigateTool, BrowserProfilesTool, - BrowserReadTool, -}; pub use delete_files::DeleteFilesTool; pub use edit::EditTool; pub use execute_command::ExecuteCommandTool; diff --git a/crates/code_assistant_core/src/tools/mod.rs b/crates/code_assistant_core/src/tools/mod.rs index 7cc34967..c6f3cabb 100644 --- a/crates/code_assistant_core/src/tools/mod.rs +++ b/crates/code_assistant_core/src/tools/mod.rs @@ -89,21 +89,14 @@ pub fn test_registry() -> Arc { /// missing. pub fn register_default_tools(registry: &mut ToolRegistry, config: &ToolsConfig) { use impls::{ - BrowserActTool, BrowserCloseTool, BrowserLoginTool, BrowserNavigateTool, - BrowserProfilesTool, BrowserReadTool, CancelWakeupTool, DeleteFilesTool, EditTool, - ExecuteCommandTool, GetSessionContentTool, GlobFilesTool, ListFilesTool, ListProjectsTool, - ListSkillsTool, NameSessionTool, PerplexityAskTool, ReadFilesTool, ReadSkillTool, - ReplaceInFileTool, ScheduleWakeupTool, SearchFilesTool, SearchSessionsTool, SpawnAgentTool, - UpdatePlanTool, ViewDocumentsTool, ViewImagesTool, WebFetchTool, WebSearchTool, - WriteFileTool, WriteStdinTool, + CancelWakeupTool, DeleteFilesTool, EditTool, ExecuteCommandTool, GetSessionContentTool, + GlobFilesTool, ListFilesTool, ListProjectsTool, ListSkillsTool, NameSessionTool, + PerplexityAskTool, ReadFilesTool, ReadSkillTool, ReplaceInFileTool, ScheduleWakeupTool, + SearchFilesTool, SearchSessionsTool, SpawnAgentTool, UpdatePlanTool, ViewDocumentsTool, + ViewImagesTool, WebFetchTool, WebSearchTool, WriteFileTool, WriteStdinTool, }; - registry.register(Box::new(BrowserNavigateTool)); - registry.register(Box::new(BrowserReadTool)); - registry.register(Box::new(BrowserActTool)); - registry.register(Box::new(BrowserCloseTool)); - registry.register(Box::new(BrowserLoginTool)); - registry.register(Box::new(BrowserProfilesTool)); + impls::browser::register(registry); registry.register(Box::new(DeleteFilesTool)); registry.register(Box::new(EditTool)); registry.register(Box::new(ExecuteCommandTool)); diff --git a/crates/ui_gpui/src/blocks/container.rs b/crates/ui_gpui/src/blocks/container.rs index d7c3e824..67bf60f9 100644 --- a/crates/ui_gpui/src/blocks/container.rs +++ b/crates/ui_gpui/src/blocks/container.rs @@ -336,12 +336,11 @@ impl MessageContainer { { override_state } else { - // No renderer (unknown tool) → collapsed; otherwise ask the - // renderer (cards expanded by default, browser cards collapsed). + // No renderer (unknown tool) → collapsed. let starts_collapsed = crate::tool_cards::ToolBlockRendererRegistry::global() .as_ref() .and_then(|registry| registry.resolve(&name)) - .map(|r| r.starts_collapsed()) + .map(|r| r.style() == crate::tool_cards::ToolBlockStyle::Inline) .unwrap_or(true); if starts_collapsed { ToolBlockState::Collapsed diff --git a/crates/ui_gpui/src/lib.rs b/crates/ui_gpui/src/lib.rs index d94d95bd..bad391cb 100644 --- a/crates/ui_gpui/src/lib.rs +++ b/crates/ui_gpui/src/lib.rs @@ -547,7 +547,7 @@ impl Gpui { tbr_registry.register(Arc::new(tool_cards::sub_agent_card::SubAgentCardRenderer)); tbr_registry.register(Arc::new(tool_cards::code_card::CodeCardRenderer)); tbr_registry.register(Arc::new(tool_cards::session_card::SessionCardRenderer)); - tbr_registry.register(Arc::new(tool_cards::browser_card::BrowserCardRenderer)); + tbr_registry.register(Arc::new(tool_cards::browser_renderer::BrowserToolRenderer)); // MCP tools have dynamic `mcp____` names; one inline // fallback renderer handles all of them. tbr_registry.set_mcp_fallback(Arc::new(McpToolRenderer::new())); diff --git a/crates/ui_gpui/src/shared/file_icons.rs b/crates/ui_gpui/src/shared/file_icons.rs index 26229b80..ae025e2c 100644 --- a/crates/ui_gpui/src/shared/file_icons.rs +++ b/crates/ui_gpui/src/shared/file_icons.rs @@ -244,8 +244,7 @@ impl FileIcons { "glob_files" => TOOL_GLOB_FILES, "web_search" => TOOL_WEB_SEARCH, "web_fetch" => TOOL_WEB_FETCH, - "browser_navigate" | "browser_read" | "browser_act" | "browser_close" - | "browser_login" | "browser_profiles" => TOOL_WEB_FETCH, + name if name.starts_with("browser_") => TOOL_WEB_FETCH, "delete_files" => TOOL_DELETE_FILES, "open_project" => TOOL_OPEN_PROJECT, "user_input" => TOOL_USER_INPUT, diff --git a/crates/ui_gpui/src/tool_cards/browser_card.rs b/crates/ui_gpui/src/tool_cards/browser_card.rs deleted file mode 100644 index c6470ebc..00000000 --- a/crates/ui_gpui/src/tool_cards/browser_card.rs +++ /dev/null @@ -1,389 +0,0 @@ -//! Browser card renderer for the heavier `browser_*` tool blocks -//! (`browser_navigate`, `browser_act`, `browser_login`). -//! -//! Each is its own tool call, so it already gets its own block. This renders -//! that block as a collapsible card: a header with the action and status, and — -//! on expand — the page screenshot captured at that step, plus a short URL/title -//! caption. The lightweight `browser_read` / `browser_close` render inline -//! instead (see [`super::inline_renderer`]). - -use super::{CardRenderContext, ToolBlockRenderer, ToolBlockStyle, animated_card_body}; -use crate::blocks::{BlockView, ToolUseBlock}; -use crate::shared::file_icons; -use code_assistant_core::ui::ToolStatus; -use gpui_kit::prelude::FluentBuilder; -use gpui_kit::{ - Animation, AnimationExt, AnyElement, ClickEvent, Context, ImageSource, InteractiveElement, - IntoElement, ObjectFit, ParentElement, SharedString, StatefulInteractiveElement, Styled, - StyledImage, Transformation, Window, div, img, percentage, px, rems, -}; -use std::time::Duration; - -// Only the tools whose screenshot is worth a card. browser_read and -// browser_close render inline (see InlineToolRenderer). -const BROWSER_TOOLS: [&str; 3] = ["browser_navigate", "browser_act", "browser_login"]; - -/// Maximum height of a screenshot inside a card body. -const SCREENSHOT_MAX_HEIGHT: f32 = 380.0; - -pub struct BrowserCardRenderer; - -impl ToolBlockRenderer for BrowserCardRenderer { - fn supported_tools(&self) -> Vec { - BROWSER_TOOLS.iter().map(|s| s.to_string()).collect() - } - - fn style(&self) -> ToolBlockStyle { - ToolBlockStyle::Card - } - - /// Start collapsed: a browsing run produces many screenshots, so the user - /// expands the step they care about rather than seeing them all at once. - fn starts_collapsed(&self) -> bool { - true - } - - fn describe(&self, tool: &ToolUseBlock) -> String { - describe(tool) - } - - fn render( - &self, - tool: &ToolUseBlock, - _is_generating: bool, - theme: &gpui_kit::component::theme::Theme, - card_ctx: Option<&CardRenderContext>, - _window: &mut Window, - cx: &mut Context, - ) -> Option { - let card_ctx = card_ctx?; - let scale = card_ctx.animation_scale; - let is_collapsed = card_ctx.is_collapsed; - - let is_dark = theme.background.l < 0.5; - let header_bg = if is_dark { - gpui_kit::hsla(0.0, 0.0, 0.15, 1.0) - } else { - gpui_kit::hsla(0.0, 0.0, 0.93, 1.0) - }; - let header_text_color = theme.muted_foreground; - - let mut card = div() - .w_full() - .border_1() - .border_color(theme.border) - .rounded_md() - .overflow_hidden(); - - // ---- Header ---- - let icon = file_icons::get().get_tool_icon(&tool.name); - let header_left = div() - .flex() - .flex_row() - .items_center() - .gap_1p5() - .min_w_0() - .flex_grow(1.0) - .child(file_icons::render_icon_container( - &icon, - 13.0, - header_text_color, - "🌐", - )) - .child( - div() - .text_size(rems(0.75)) - .text_color(header_text_color) - .overflow_hidden() - .child(describe(tool)), - ); - - let mut header_right = div().flex().flex_row().items_center().gap_2(); - match tool.status { - ToolStatus::Running | ToolStatus::Pending => { - header_right = header_right.child( - gpui_kit::svg() - .size(px(12.)) - .path(SharedString::from("icons/arrow_circle.svg")) - .text_color(header_text_color) - .with_animation( - SharedString::from(format!("browser-spin-{}", tool.id)), - Animation::new(Duration::from_secs(2)).repeat(), - |svg, delta| { - svg.with_transformation(Transformation::rotate(percentage(delta))) - }, - ), - ); - } - ToolStatus::Error => { - header_right = header_right.child( - gpui_kit::svg() - .size(px(13.0)) - .path(SharedString::from("icons/close.svg")) - .text_color(theme.danger), - ); - } - ToolStatus::Success => {} - } - - let chevron_icon = if is_collapsed { - file_icons::get().get_type_icon(file_icons::CHEVRON_DOWN) - } else { - file_icons::get().get_type_icon(file_icons::CHEVRON_UP) - }; - header_right = header_right.child( - div() - .flex_none() - .flex() - .items_center() - .justify_center() - .size(px(24.)) - .rounded(px(6.)) - .group_hover("browser-header", |s| s.bg(header_text_color.opacity(0.1))) - .child(file_icons::render_icon( - &chevron_icon, - 14.0, - header_text_color.opacity(0.4), - "▾", - )), - ); - - card = card.child( - div() - .id(SharedString::from(format!("browser-header-{}", tool.id))) - .group("browser-header") - .px_3() - .py_1p5() - .bg(header_bg) - .cursor_pointer() - .flex() - .flex_row() - .justify_between() - .items_center() - .map(|d| { - if scale <= 0.0 { - d.rounded(px(4.)) - } else { - d.rounded_t(px(4.)) - } - }) - .on_click(cx.listener(|view, _event: &ClickEvent, _window, cx| { - view.toggle_tool_collapsed(cx); - })) - .child(header_left) - .child(header_right), - ); - - // ---- Body (animated) ---- - if scale > 0.0 { - let body_inner = self.render_body(tool, theme); - card = card.child(animated_card_body( - body_inner, - scale, - card_ctx.content_height.clone(), - )); - } - - Some(card.into_any_element()) - } -} - -impl BrowserCardRenderer { - fn render_body( - &self, - tool: &ToolUseBlock, - theme: &gpui_kit::component::theme::Theme, - ) -> gpui_kit::Div { - let mut body = div() - .flex() - .flex_col() - .gap_2() - .p_3() - .bg(theme.background) - .rounded_b(px(4.)) - .overflow_hidden(); - - // Screenshot(s) captured at this step. - for (media_type, base64_data) in &tool.images { - if let Some(image) = crate::shared::image::parse_base64_image(media_type, base64_data) { - body = body.child( - div() - .flex_none() - .border_1() - .border_color(theme.border) - .rounded_md() - .overflow_hidden() - .bg(theme.popover) - .shadow_sm() - .child( - img(ImageSource::Image(image)) - .max_h(px(SCREENSHOT_MAX_HEIGHT)) - .max_w_full() - .object_fit(ObjectFit::Contain), - ), - ); - } - } - - // Caption: the URL/title lines for success, or the error text. - if let Some(output) = tool.output.as_deref() { - let is_error = tool.status == ToolStatus::Error; - let caption = caption_from_output(output, is_error); - if !caption.is_empty() { - let color = if is_error { - theme.danger - } else { - theme.muted_foreground - }; - body = body.child(div().text_size(rems(0.75)).text_color(color).child(caption)); - } - } else if matches!(tool.status, ToolStatus::Running | ToolStatus::Pending) { - body = body.child( - div() - .text_size(rems(0.75)) - .text_color(theme.muted_foreground.opacity(0.7)) - .child("Working…"), - ); - } - - body - } -} - -/// A one-line header description per browser tool, from its parameters. -fn describe(tool: &ToolUseBlock) -> String { - let param = |name: &str| { - tool.parameters - .iter() - .find(|p| p.name == name) - .map(|p| p.value.clone()) - }; - let profile_suffix = match param("profile") { - Some(p) if !p.is_empty() && p != "default" => format!(" · {p}"), - _ => String::new(), - }; - - let base = match tool.name.as_str() { - "browser_navigate" => match param("url") { - Some(url) => format!("Navigate to {}", truncate(&url, 70)), - None => "Navigate".to_string(), - }, - "browser_login" => match param("url") { - Some(url) => format!("Log in at {}", truncate(&url, 60)), - None => "Log in".to_string(), - }, - "browser_act" => describe_act(param("actions").as_deref()), - other => other.to_string(), - }; - format!("{base}{profile_suffix}") -} - -/// For `browser_act`, summarize the action count when the JSON parses. -fn describe_act(actions_json: Option<&str>) -> String { - if let Some(json) = actions_json - && let Ok(serde_json::Value::Array(items)) = serde_json::from_str::(json) - { - let n = items.len(); - return format!("Interact ({n} step{})", if n == 1 { "" } else { "s" }); - } - "Interact with page".to_string() -} - -/// Show the first few informative lines of the tool output as a caption. For -/// success that is the `Profile/URL/Title` header the tool emits; for errors it -/// is the error message. -fn caption_from_output(output: &str, is_error: bool) -> String { - if is_error { - return output.trim().lines().take(3).collect::>().join("\n"); - } - output - .lines() - .filter(|l| l.starts_with("URL:") || l.starts_with("Title:")) - .take(2) - .collect::>() - .join(" · ") -} - -fn truncate(s: &str, max_chars: usize) -> String { - let count = s.chars().count(); - if count > max_chars { - let truncated: String = s.chars().take(max_chars.saturating_sub(1)).collect(); - format!("{truncated}…") - } else { - s.to_string() - } -} - -#[cfg(test)] -mod tests { - use super::*; - use crate::tool_cards::tests::make_tool; - use std::sync::Arc; - - #[test] - fn describe_navigate_shows_url() { - let t = make_tool("browser_navigate", &[("url", "https://example.com")]); - assert_eq!(describe(&t), "Navigate to https://example.com"); - } - - #[test] - fn describe_appends_non_default_profile() { - let t = make_tool( - "browser_navigate", - &[("url", "https://elster.de"), ("profile", "elster")], - ); - assert_eq!(describe(&t), "Navigate to https://elster.de · elster"); - } - - #[test] - fn describe_default_profile_has_no_suffix() { - let t = make_tool( - "browser_navigate", - &[("url", "https://x.com"), ("profile", "default")], - ); - assert_eq!(describe(&t), "Navigate to https://x.com"); - } - - #[test] - fn describe_act_counts_steps() { - let two = make_tool( - "browser_act", - &[( - "actions", - r##"[{"click":{"selector":"#a"}},{"type":{"selector":"#b","text":"x"}}]"##, - )], - ); - assert_eq!(describe(&two), "Interact (2 steps)"); - let one = make_tool( - "browser_act", - &[("actions", r##"[{"click":{"selector":"#a"}}]"##)], - ); - assert_eq!(describe(&one), "Interact (1 step)"); - } - - #[test] - fn caption_extracts_url_and_title_on_success() { - let out = "Profile: default\nURL: https://x.com\nTitle: Hi\n\nbody text"; - assert_eq!( - caption_from_output(out, false), - "URL: https://x.com · Title: Hi" - ); - } - - #[test] - fn caption_shows_error_text() { - assert_eq!( - caption_from_output("Browser error: boom", true), - "Browser error: boom" - ); - } - - #[test] - fn registry_registers_all_browser_tools() { - let mut registry = crate::tool_cards::ToolBlockRendererRegistry::default(); - registry.register(Arc::new(BrowserCardRenderer)); - for name in BROWSER_TOOLS { - assert!(registry.get(name).is_some(), "missing renderer for {name}"); - } - } -} diff --git a/crates/ui_gpui/src/tool_cards/browser_renderer.rs b/crates/ui_gpui/src/tool_cards/browser_renderer.rs new file mode 100644 index 00000000..ea281dbe --- /dev/null +++ b/crates/ui_gpui/src/tool_cards/browser_renderer.rs @@ -0,0 +1,186 @@ +//! Inline renderer for the `browser_*` tools. +//! +//! Browser work is many small calls (read, click, type, screenshot), so each +//! renders as one line — "Click ref_4", "Find \"Sign in\"" — like the other +//! inline tools. Expanding it shows the result: trees and logs in a monospace +//! font, cut to a readable length, and screenshots at a size where the page +//! can actually be read. + +use super::{CardRenderContext, ToolBlockRenderer, ToolBlockStyle}; +use crate::blocks::{BlockView, ToolUseBlock}; +use code_assistant_core::tools::impls::browser::describe::{ + BROWSER_TOOL_NAMES, describe_call, has_structured_output, target_label, +}; +use code_assistant_core::ui::ToolStatus; +use gpui_kit::{ + AnyElement, Context, Element, ImageSource, ObjectFit, ParentElement, Styled, StyledImage, + Window, div, img, px, rems, +}; + +/// How many output lines an expanded block shows; the model saw all of them. +const MAX_OUTPUT_LINES: usize = 40; + +/// Maximum height of a screenshot in an expanded block. +const SCREENSHOT_MAX_HEIGHT: f32 = 380.0; + +pub struct BrowserToolRenderer; + +impl ToolBlockRenderer for BrowserToolRenderer { + fn supported_tools(&self) -> Vec { + BROWSER_TOOL_NAMES.iter().map(|s| s.to_string()).collect() + } + + fn style(&self) -> ToolBlockStyle { + ToolBlockStyle::Inline + } + + fn describe(&self, tool: &ToolUseBlock) -> String { + describe_call(&tool.name, &|name| param(tool, name)) + } + + fn header_tag(&self, tool: &ToolUseBlock) -> Option { + target_label(&|name| param(tool, name)) + } + + fn render( + &self, + tool: &ToolUseBlock, + _is_generating: bool, + theme: &gpui_kit::component::theme::Theme, + _card_ctx: Option<&CardRenderContext>, + _window: &mut Window, + _cx: &mut Context, + ) -> Option { + let output = tool.output.as_deref().unwrap_or("").trim(); + if output.is_empty() && tool.images.is_empty() { + return None; + } + + let mut container = div() + .pl(px(8.)) + .ml(px(8.)) + .border_l_2() + .border_color(theme.border) + .py(px(4.)) + .flex() + .flex_col() + .gap_2() + .overflow_hidden(); + + if !output.is_empty() { + let color = if tool.status == ToolStatus::Error { + theme.danger + } else { + theme.muted_foreground + }; + let mut text = div() + .text_size(rems(0.8125)) + .text_color(color) + .overflow_hidden(); + if has_structured_output(&tool.name) { + text = text.font_family("Menlo").line_height(rems(0.8125 * 1.4)); + } + container = container.child(text.child(cap_lines(output, MAX_OUTPUT_LINES))); + } + + for (media_type, base64_data) in &tool.images { + if let Some(image) = crate::shared::image::parse_base64_image(media_type, base64_data) { + container = container.child( + div() + .flex_none() + .border_1() + .border_color(theme.border) + .rounded_md() + .overflow_hidden() + .bg(theme.popover) + .shadow_sm() + .child( + img(ImageSource::Image(image)) + .max_h(px(SCREENSHOT_MAX_HEIGHT)) + .max_w_full() + .object_fit(ObjectFit::Contain), + ), + ); + } + } + + Some(container.into_any()) + } +} + +fn param(tool: &ToolUseBlock, name: &str) -> Option { + tool.parameters + .iter() + .find(|p| p.name == name) + .map(|p| p.value.clone()) + .filter(|v| !v.is_empty()) +} + +/// The first `max` lines of `text`, with a count of the rest. +fn cap_lines(text: &str, max: usize) -> String { + let total = text.lines().count(); + if total <= max { + return text.to_string(); + } + let head: Vec<&str> = text.lines().take(max).collect(); + format!("{}\n… {} more lines", head.join("\n"), total - max) +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::tool_cards::tests::make_tool; + use std::sync::Arc; + + #[test] + fn describes_calls_with_the_shared_wording() { + let click = make_tool( + "browser_computer", + &[("action", "left_click"), ("ref", "ref_4")], + ); + assert_eq!(BrowserToolRenderer.describe(&click), "Click ref_4"); + let nav = make_tool("browser_navigate", &[("url", "https://example.com")]); + assert_eq!( + BrowserToolRenderer.describe(&nav), + "Navigate to https://example.com" + ); + } + + #[test] + fn tags_a_named_profile_and_tab() { + let t = make_tool( + "browser_navigate", + &[("url", "https://elster.de"), ("profile", "elster")], + ); + assert_eq!( + BrowserToolRenderer.header_tag(&t).as_deref(), + Some("elster") + ); + let t = make_tool( + "browser_navigate", + &[("url", "https://x.com"), ("profile", "default")], + ); + assert_eq!(BrowserToolRenderer.header_tag(&t), None); + } + + #[test] + fn long_output_is_capped_with_a_count() { + let text = (1..=45) + .map(|i| format!("line {i}")) + .collect::>() + .join("\n"); + let capped = cap_lines(&text, 40); + assert!(capped.ends_with("line 40\n… 5 more lines"), "{capped}"); + assert_eq!(cap_lines("a\nb", 40), "a\nb"); + } + + #[test] + fn registry_registers_all_browser_tools_inline() { + let mut registry = crate::tool_cards::ToolBlockRendererRegistry::default(); + registry.register(Arc::new(BrowserToolRenderer)); + for name in BROWSER_TOOL_NAMES { + let renderer = registry.get(name).expect(name); + assert_eq!(renderer.style(), ToolBlockStyle::Inline); + } + } +} diff --git a/crates/ui_gpui/src/tool_cards/inline_renderer.rs b/crates/ui_gpui/src/tool_cards/inline_renderer.rs index 417f80d2..3bab447d 100644 --- a/crates/ui_gpui/src/tool_cards/inline_renderer.rs +++ b/crates/ui_gpui/src/tool_cards/inline_renderer.rs @@ -76,24 +76,6 @@ impl InlineToolRenderer { template: "List skills in {project}", fallback: "List skills", }, - // Lightweight browser tools: no card, just a line you can expand - // (browser_read shows its screenshot inline; browser_close is a - // one-liner). The heavier browser_navigate/act/login use cards. - DescribeTemplate { - tool_name: "browser_read", - template: "Read browser page", - fallback: "Read browser page", - }, - DescribeTemplate { - tool_name: "browser_close", - template: "Close browser", - fallback: "Close browser", - }, - DescribeTemplate { - tool_name: "browser_profiles", - template: "List browser profiles", - fallback: "List browser profiles", - }, ]; let tools: Vec = templates.iter().map(|t| t.tool_name.to_string()).collect(); diff --git a/crates/ui_gpui/src/tool_cards/mod.rs b/crates/ui_gpui/src/tool_cards/mod.rs index cd393119..a534435f 100644 --- a/crates/ui_gpui/src/tool_cards/mod.rs +++ b/crates/ui_gpui/src/tool_cards/mod.rs @@ -15,7 +15,7 @@ //! with meaningful visual output (e.g. `execute_command`, `edit`). pub mod animated_card; -pub mod browser_card; +pub mod browser_renderer; pub mod code_card; pub mod diff_card; pub mod diff_prepare; @@ -101,14 +101,6 @@ pub trait ToolBlockRenderer: Send + Sync { /// Whether this tool renders as inline or card. fn style(&self) -> ToolBlockStyle; - /// Whether the block starts collapsed when there is no stored user override. - /// Inline tools start collapsed, cards start expanded; a card renderer can - /// override this to start collapsed (e.g. browser cards, so a run's many - /// screenshots don't flood the transcript until expanded). - fn starts_collapsed(&self) -> bool { - self.style() == ToolBlockStyle::Inline - } - /// Generate a one-line description from parameters (for inline tools). fn describe(&self, tool: &ToolUseBlock) -> String { tool.name.clone() diff --git a/crates/ui_terminal/src/tool_renderers/browser_renderer.rs b/crates/ui_terminal/src/tool_renderers/browser_renderer.rs new file mode 100644 index 00000000..f34be8dd --- /dev/null +++ b/crates/ui_terminal/src/tool_renderers/browser_renderer.rs @@ -0,0 +1,192 @@ +//! Compact renderer for the `browser_*` tools. +//! +//! Browser work is many small calls, and some return a lot of text (a page's +//! accessibility tree, its text, logs). Each call renders as its header, a +//! one-line description ("Click ref_4") and the first line of its result — +//! never the full output, which the model already has. + +use ratatui::prelude::*; +use ratatui::style::{Color, Style}; + +use super::{ + ToolRenderer, push_error_history_line, render_error_line, render_tool_header, tool_header_line, +}; +use crate::message::ToolUseBlock; +use crate::text_util::truncate_to_width; +use code_assistant_core::tools::impls::browser::describe::{ + BROWSER_TOOL_NAMES, describe_batch_steps, describe_call, target_label, +}; +use code_assistant_core::ui::ToolStatus; + +pub struct BrowserToolRenderer; + +/// One line under the header: what the call does, or what it returned. +#[derive(Debug, PartialEq)] +enum BrowserLine { + Action(String), + Result(String), +} + +impl ToolRenderer for BrowserToolRenderer { + fn supported_tools(&self) -> &'static [&'static str] { + &BROWSER_TOOL_NAMES + } + + fn render(&self, tool_block: &ToolUseBlock, area: Rect, buf: &mut Buffer) { + if area.height < 1 { + return; + } + let mut y = render_tool_header(tool_block, area, buf, area.y); + let max_len = area.width.saturating_sub(2) as usize; + for line in browser_lines(tool_block) { + if y >= area.y + area.height { + break; + } + let (text, style) = styled(&line); + buf.set_string(area.x + 2, y, truncate_to_width(&text, max_len), style); + y += 1; + } + render_error_line(tool_block, area, buf, y); + } + + fn calculate_height(&self, tool_block: &ToolUseBlock, _width: u16) -> u16 { + let mut height = 1 + browser_lines(tool_block).len() as u16; + if tool_block.status == ToolStatus::Error && tool_block.status_message.is_some() { + height += 1; + } + height + } + + fn render_history_lines(&self, tool_block: &ToolUseBlock) -> Vec> { + let mut lines = vec![tool_header_line(tool_block)]; + for line in browser_lines(tool_block) { + let (text, style) = styled(&line); + lines.push(Line::from(vec![Span::raw(" "), Span::styled(text, style)])); + } + push_error_history_line(tool_block, &mut lines); + lines + } +} + +fn styled(line: &BrowserLine) -> (String, Style) { + match line { + BrowserLine::Action(text) => (text.clone(), Style::default().fg(Color::Gray)), + BrowserLine::Result(text) => (format!("→ {text}"), Style::default().fg(Color::DarkGray)), + } +} + +fn browser_lines(tool: &ToolUseBlock) -> Vec { + let param = |name: &str| { + tool.parameters + .get(name) + .map(|p| p.value.clone()) + .filter(|v| !v.is_empty()) + }; + let mut lines = Vec::new(); + + if tool.name == "browser_batch" { + let steps = param("actions") + .map(|json| describe_batch_steps(&json)) + .unwrap_or_default(); + for (i, step) in steps.iter().enumerate() { + lines.push(BrowserLine::Action(format!("{}. {step}", i + 1))); + } + return lines; + } + + let mut action = describe_call(&tool.name, ¶m); + if let Some(target) = target_label(¶m) { + action.push_str(&format!(" [{target}]")); + } + lines.push(BrowserLine::Action(action)); + + // The first line of a successful result; errors show via the status line. + if tool.status == ToolStatus::Success + && let Some(output) = tool.output.as_deref() + { + let mut rest = output.trim().lines().filter(|l| !l.trim().is_empty()); + if let Some(first) = rest.next() { + let more = rest.count(); + let suffix = if more > 0 { + format!(" (+{more} lines)") + } else { + String::new() + }; + lines.push(BrowserLine::Result(format!("{}{suffix}", first.trim()))); + } + } + lines +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::message::ParameterValue; + use indexmap::IndexMap; + + fn make_tool(name: &str, params: &[(&str, &str)], output: Option<&str>) -> ToolUseBlock { + let mut parameters = IndexMap::new(); + for (k, v) in params { + parameters.insert(k.to_string(), ParameterValue::new(v.to_string())); + } + ToolUseBlock { + name: name.to_string(), + id: "test-id".to_string(), + parameters, + status: ToolStatus::Success, + status_message: None, + output: output.map(str::to_string), + } + } + + #[test] + fn a_long_result_shows_only_its_first_line() { + let tool = make_tool( + "browser_read_page", + &[("filter", "interactive"), ("profile", "elster")], + Some("[t1] https://x.test — X\n- button \"Go\" [ref_1]\n- link \"Docs\" [ref_2]"), + ); + assert_eq!( + browser_lines(&tool), + vec![ + BrowserLine::Action("Read interactive elements [elster]".into()), + BrowserLine::Result("[t1] https://x.test — X (+2 lines)".into()), + ] + ); + } + + #[test] + fn a_click_shows_its_action_and_result() { + let tool = make_tool( + "browser_computer", + &[("action", "left_click"), ("ref", "ref_4")], + Some("Clicked ref_4"), + ); + assert_eq!( + browser_lines(&tool), + vec![ + BrowserLine::Action("Click ref_4".into()), + BrowserLine::Result("Clicked ref_4".into()), + ] + ); + } + + #[test] + fn a_batch_lists_its_steps() { + let tool = make_tool( + "browser_batch", + &[( + "actions", + r#"[{"name":"browser_computer","input":{"action":"screenshot"}},{"name":"browser_find","input":{"query":"Save"}}]"#, + )], + Some("[1] computer screenshot\n…"), + ); + assert_eq!( + browser_lines(&tool), + vec![ + BrowserLine::Action("1. Screenshot".into()), + BrowserLine::Action("2. Find \"Save\"".into()), + ] + ); + } +} diff --git a/crates/ui_terminal/src/tool_renderers/mod.rs b/crates/ui_terminal/src/tool_renderers/mod.rs index ede2e360..d483b6ad 100644 --- a/crates/ui_terminal/src/tool_renderers/mod.rs +++ b/crates/ui_terminal/src/tool_renderers/mod.rs @@ -3,6 +3,7 @@ //! Each tool (or group of tools) can register a custom renderer that controls //! how the tool block appears in both the live viewport and scrollback history. +pub mod browser_renderer; pub mod command_renderer; pub mod compact_renderer; pub mod diff_renderer; @@ -183,6 +184,7 @@ pub fn push_error_history_line(tool_block: &ToolUseBlock, lines: &mut Vec chromiumoxide::types::MethodId { + "Accessibility.getFullAXTree".into() + } +} + +impl chromiumoxide::Command for GetFullAxTreeRaw { + type Response = RawAxTree; +} + +#[derive(Debug, Deserialize)] +pub(crate) struct RawAxTree { + #[serde(default)] + pub nodes: Vec, +} + +#[derive(Debug, Deserialize)] +#[serde(rename_all = "camelCase")] +pub(crate) struct RawAxNode { + pub node_id: String, + #[serde(default)] + pub ignored: bool, + #[serde(default)] + pub role: Option, + #[serde(default)] + pub name: Option, + #[serde(default)] + pub value: Option, + #[serde(default)] + pub properties: Vec, + #[serde(default)] + pub parent_id: Option, + #[serde(default)] + pub child_ids: Vec, + #[serde(default, rename = "backendDOMNodeId")] + pub backend_dom_node_id: Option, +} + +#[derive(Debug, Deserialize)] +pub(crate) struct RawAxValue { + #[serde(default)] + pub value: Option, +} + +#[derive(Debug, Deserialize)] +pub(crate) struct RawAxProperty { + pub name: String, + pub value: RawAxValue, +} + +impl RawAxValue { + fn text(&self) -> String { + match &self.value { + Some(serde_json::Value::String(s)) => s.clone(), + Some(serde_json::Value::Null) | None => String::new(), + Some(other) => other.to_string(), + } + } +} + +impl RawAxNode { + fn role(&self) -> String { + self.role.as_ref().map(RawAxValue::text).unwrap_or_default() + } + + fn name(&self) -> String { + self.name.as_ref().map(RawAxValue::text).unwrap_or_default() + } + + fn property(&self, name: &str) -> Option<&RawAxValue> { + self.properties + .iter() + .find(|p| p.name == name) + .map(|p| &p.value) + } +} + +/// `ref_N` handles for DOM nodes, stable across reads of the same document. +#[derive(Default)] +pub(crate) struct RefMap { + by_backend: HashMap, + by_ref: HashMap, + next: u32, +} + +impl RefMap { + pub fn ref_for(&mut self, backend_id: i64) -> String { + if let Some(r) = self.by_backend.get(&backend_id) { + return r.clone(); + } + self.next += 1; + let r = format!("ref_{}", self.next); + self.by_backend.insert(backend_id, r.clone()); + self.by_ref.insert(r.clone(), backend_id); + r + } + + pub fn backend_id(&self, r: &str) -> Option { + self.by_ref.get(r).copied() + } +} + +/// Roles a user acts on; `filter: interactive` keeps only these. +const INTERACTIVE_ROLES: &[&str] = &[ + "button", + "checkbox", + "combobox", + "link", + "listbox", + "menuitem", + "menuitemcheckbox", + "menuitemradio", + "option", + "radio", + "scrollbar", + "searchbox", + "slider", + "spinbutton", + "switch", + "tab", + "textbox", + "treeitem", +]; + +/// Roles that only structure the tree; without a name they are not printed +/// and their children move up a level. +const TRANSPARENT_ROLES: &[&str] = &["generic", "none", "presentation", "LineBreak"]; + +/// How to render a snapshot. +pub(crate) struct RenderOptions<'a> { + pub interactive_only: bool, + /// Print only the subtree under this ref. + pub root_ref: Option<&'a str>, + pub max_depth: usize, +} + +/// Render the tree as YAML-style lines, `- role "name" [ref_N] attrs`, +/// indented by depth. With `interactive_only` the result is a flat list of +/// actionable elements. +pub(crate) fn render( + nodes: &[RawAxNode], + refs: &mut RefMap, + opts: &RenderOptions, +) -> Result, String> { + let by_id: HashMap<&str, &RawAxNode> = nodes.iter().map(|n| (n.node_id.as_str(), n)).collect(); + let start: Vec<&RawAxNode> = match opts.root_ref { + Some(r) => { + let backend = refs + .backend_id(r) + .ok_or_else(|| format!("unknown ref '{r}' (read the page again)"))?; + let node = nodes + .iter() + .find(|n| n.backend_dom_node_id == Some(backend)) + .ok_or_else(|| format!("{r} is no longer in the page (read the page again)"))?; + vec![node] + } + None => nodes.iter().filter(|n| n.parent_id.is_none()).collect(), + }; + let mut lines = Vec::new(); + for node in start { + walk(node, &by_id, refs, opts, 0, None, &mut lines); + } + Ok(lines) +} + +fn walk( + node: &RawAxNode, + by_id: &HashMap<&str, &RawAxNode>, + refs: &mut RefMap, + opts: &RenderOptions, + depth: usize, + parent_name: Option<&str>, + lines: &mut Vec, +) { + let role = node.role(); + let name = node.name(); + let skip_self = node.ignored + || role == "InlineTextBox" + || (TRANSPARENT_ROLES.contains(&role.as_str()) && name.is_empty()) + // Text that only repeats its parent's name (a link's or button's + // label) adds nothing. + || (role == "StaticText" && (name.trim().is_empty() || Some(name.as_str()) == parent_name)); + let print = + !skip_self && (!opts.interactive_only || INTERACTIVE_ROLES.contains(&role.as_str())); + + let child_depth = if skip_self || opts.interactive_only { + depth + } else { + depth + 1 + }; + if print && depth < opts.max_depth { + lines.push(line( + node, + &role, + &name, + refs, + if opts.interactive_only { 0 } else { depth }, + )); + } + if child_depth > opts.max_depth { + return; + } + let next_parent_name = if skip_self { + parent_name + } else { + Some(name.as_str()) + }; + for child in node + .child_ids + .iter() + .filter_map(|id| by_id.get(id.as_str())) + { + walk( + child, + by_id, + refs, + opts, + child_depth, + next_parent_name, + lines, + ); + } +} + +fn line(node: &RawAxNode, role: &str, name: &str, refs: &mut RefMap, depth: usize) -> String { + let role = match role { + "RootWebArea" => "document", + "StaticText" => "text", + other => other, + }; + let mut out = format!("{}- {role}", " ".repeat(depth)); + if !name.is_empty() { + out.push_str(&format!(" {}", quote(name, 100))); + } + if let Some(backend) = node.backend_dom_node_id { + out.push_str(&format!(" [{}]", refs.ref_for(backend))); + } + let value = node + .value + .as_ref() + .map(RawAxValue::text) + .unwrap_or_default(); + if !value.is_empty() && value != name { + out.push_str(&format!(" value={}", quote(&value, 100))); + } + for (prop, label) in [ + ("checked", "checked"), + ("pressed", "pressed"), + ("selected", "selected"), + ("expanded", "expanded"), + ("disabled", "disabled"), + ("required", "required"), + ("invalid", "invalid"), + ("focused", "focused"), + ("level", "level"), + ] { + let Some(v) = node.property(prop) else { + continue; + }; + match v.value.as_ref() { + Some(serde_json::Value::Bool(true)) => out.push_str(&format!(" {label}")), + Some(serde_json::Value::Bool(false)) | None => { + // `checked`/`expanded`/`pressed` false is state worth seeing. + if matches!(prop, "checked" | "expanded" | "pressed") { + out.push_str(&format!(" {label}=false")); + } + } + Some(serde_json::Value::String(s)) if s == "false" => { + if matches!(prop, "checked" | "expanded" | "pressed") { + out.push_str(&format!(" {label}=false")); + } + } + Some(serde_json::Value::String(s)) if s == "true" => out.push_str(&format!(" {label}")), + Some(other) => out.push_str(&format!( + " {label}={}", + RawAxValue { + value: Some(other.clone()) + } + .text() + )), + } + } + if role == "link" + && let Some(url) = node.property("url").map(RawAxValue::text) + && !url.is_empty() + { + out.push_str(&format!(" href={}", quote(&url, 120))); + } + out +} + +fn quote(s: &str, max_chars: usize) -> String { + let s = s.replace('\n', " "); + let s = if s.chars().count() > max_chars { + let cut: String = s.chars().take(max_chars).collect(); + format!("{cut}…") + } else { + s + }; + format!("{s:?}") +} + +#[cfg(test)] +mod tests { + use super::*; + + /// A document with a heading, a link whose text node repeats its name, a + /// nameless wrapper div, a filled textbox, and an unchecked checkbox. + fn sample() -> Vec { + let json = serde_json::json!({"nodes": [ + {"nodeId": "1", "role": {"value": "RootWebArea"}, "name": {"value": "Demo"}, + "childIds": ["2", "3", "5"], "backendDOMNodeId": 10}, + {"nodeId": "2", "parentId": "1", "role": {"value": "heading"}, "name": {"value": "Welcome"}, + "properties": [{"name": "level", "value": {"value": 1}}], "backendDOMNodeId": 11}, + {"nodeId": "3", "parentId": "1", "role": {"value": "link"}, "name": {"value": "Docs"}, + "properties": [{"name": "url", "value": {"value": "https://x.test/docs"}}], + "childIds": ["4"], "backendDOMNodeId": 12}, + {"nodeId": "4", "parentId": "3", "role": {"value": "StaticText"}, "name": {"value": "Docs"}, + "backendDOMNodeId": 13}, + {"nodeId": "5", "parentId": "1", "role": {"value": "generic"}, "name": {"value": ""}, + "childIds": ["6", "7", "8"], "backendDOMNodeId": 14}, + {"nodeId": "6", "parentId": "5", "role": {"value": "textbox"}, "name": {"value": "User"}, + "value": {"value": "stephan"}, "properties": [{"name": "focused", "value": {"value": true}}], + "backendDOMNodeId": 15}, + {"nodeId": "7", "parentId": "5", "role": {"value": "checkbox"}, "name": {"value": "Remember"}, + "properties": [{"name": "checked", "value": {"value": "false"}}], "backendDOMNodeId": 16}, + {"nodeId": "8", "parentId": "5", "ignored": true, "role": {"value": "none"}, "backendDOMNodeId": 17} + ]}); + serde_json::from_value::(json).unwrap().nodes + } + + fn opts() -> RenderOptions<'static> { + RenderOptions { + interactive_only: false, + root_ref: None, + max_depth: 15, + } + } + + #[test] + fn renders_an_indented_tree_with_refs_and_states() { + let mut refs = RefMap::default(); + let lines = render(&sample(), &mut refs, &opts()).unwrap(); + assert_eq!( + lines, + vec![ + r#"- document "Demo" [ref_1]"#, + r#" - heading "Welcome" [ref_2] level=1"#, + r#" - link "Docs" [ref_3] href="https://x.test/docs""#, + r#" - textbox "User" [ref_4] value="stephan" focused"#, + r#" - checkbox "Remember" [ref_5] checked=false"#, + ] + ); + } + + #[test] + fn interactive_filter_is_a_flat_list_of_actionable_elements() { + let mut refs = RefMap::default(); + let lines = render( + &sample(), + &mut refs, + &RenderOptions { + interactive_only: true, + ..opts() + }, + ) + .unwrap(); + assert_eq!( + lines, + vec![ + r#"- link "Docs" [ref_1] href="https://x.test/docs""#, + r#"- textbox "User" [ref_2] value="stephan" focused"#, + r#"- checkbox "Remember" [ref_3] checked=false"#, + ] + ); + } + + #[test] + fn refs_are_stable_across_reads_and_scope_a_subtree() { + let mut refs = RefMap::default(); + render(&sample(), &mut refs, &opts()).unwrap(); + // The link keeps its ref on a second read and roots a subtree. + let lines = render( + &sample(), + &mut refs, + &RenderOptions { + root_ref: Some("ref_3"), + ..opts() + }, + ) + .unwrap(); + assert_eq!( + lines, + vec![r#"- link "Docs" [ref_3] href="https://x.test/docs""#] + ); + assert!( + render( + &sample(), + &mut refs, + &RenderOptions { + root_ref: Some("ref_99"), + ..opts() + } + ) + .is_err() + ); + } + + #[test] + fn depth_limits_the_tree() { + let mut refs = RefMap::default(); + let lines = render( + &sample(), + &mut refs, + &RenderOptions { + max_depth: 1, + ..opts() + }, + ) + .unwrap(); + assert_eq!(lines, vec![r#"- document "Demo" [ref_1]"#]); + } +} diff --git a/crates/web/src/browser.rs b/crates/web/src/browser.rs index 33190519..3c003e35 100644 --- a/crates/web/src/browser.rs +++ b/crates/web/src/browser.rs @@ -13,6 +13,7 @@ //! default preserves the old behavior. use anyhow::Result; +use chromiumoxide::handler::viewport::Viewport; use chromiumoxide::{Browser, BrowserConfig}; use futures::StreamExt; use std::path::PathBuf; @@ -80,6 +81,11 @@ pub(crate) fn resolve_user_data_dir( } } +/// Viewport of a headless browser in CSS pixels: a common laptop size, so +/// pages lay out as for a desktop user (chromiumoxide defaults to 800×600). +/// The device scale factor stays 1, so screenshot pixels are CSS pixels. +pub const DEFAULT_VIEWPORT: (u32, u32) = (1280, 800); + /// How long a graceful [`LaunchedBrowser::close`] may take before the process /// is killed. const CLOSE_TIMEOUT: Duration = Duration::from_secs(10); @@ -100,7 +106,19 @@ impl LaunchedBrowser { let mut builder = BrowserConfig::builder().user_data_dir(&data_dir); if config.headful { - builder = builder.with_head(); + // A window a human uses: let the page fill it instead of emulating + // a fixed viewport inside it. + builder = builder + .with_head() + .viewport(None) + .window_size(DEFAULT_VIEWPORT.0, DEFAULT_VIEWPORT.1 + 100); + } else { + builder = builder.viewport(Viewport { + width: DEFAULT_VIEWPORT.0, + height: DEFAULT_VIEWPORT.1, + device_scale_factor: Some(1.0), + ..Viewport::default() + }); } let browser_config = builder.build().map_err(|e| anyhow::anyhow!("{e}"))?; diff --git a/crates/web/src/browser_session.rs b/crates/web/src/browser_session.rs index 3f79813c..9f22818a 100644 --- a/crates/web/src/browser_session.rs +++ b/crates/web/src/browser_session.rs @@ -1,245 +1,60 @@ //! Interactive browser sessions for agent tools. //! //! [`crate::WebClient`] covers the one-shot case: fetch a page, extract it, -//! discard it. Browser *agency* needs the opposite — a page the agent drives +//! discard it. Browser *agency* needs the opposite — pages the agent drives //! over many tool calls: navigate, look (screenshot / read), click, type, wait. //! -//! This mirrors the `pty_session` crate one-to-one: -//! - [`BrowserSession`] — one live page on a launched browser, with the -//! interaction verbs, kept across tool calls. -//! - [`BrowserSessionManager`] — an id-keyed registry with an LRU cap, one per -//! agent session, so browser sessions survive across tool calls but die with -//! their agent session. +//! This mirrors the `pty_session` crate: +//! - [`BrowserSession`] — one launched browser with its tabs ([`Tab`]), kept +//! across tool calls. +//! - [`BrowserSessionManager`] — a registry with an LRU cap, one per agent +//! session, so browser sessions survive across tool calls but die with their +//! agent session. use crate::browser::LaunchedBrowser; +use crate::tab::{BrowserTimeouts, Tab}; use anyhow::Result; -use chromiumoxide::cdp::browser_protocol::input::{ - DispatchKeyEventParams, DispatchKeyEventType, InsertTextParams, -}; -use chromiumoxide::cdp::browser_protocol::network::{CookieParam, CookieSameSite, TimeSinceEpoch}; -use chromiumoxide::cdp::browser_protocol::page::{ - CaptureScreenshotFormat, DialogType, EventJavascriptDialogOpening, HandleJavaScriptDialogParams, -}; -use chromiumoxide::element::Element; -use chromiumoxide::keys::{KeyDefinition, get_key_definition}; -use chromiumoxide::layout::Point; -use chromiumoxide::page::{Page, ScreenshotParams}; -use futures::StreamExt; +use chromiumoxide::cdp::browser_protocol::network::CookieParam; +use chromiumoxide::cdp::browser_protocol::target::GetTargetsParams; use std::collections::HashMap; -use std::sync::atomic::{AtomicBool, Ordering}; use std::sync::{Arc, Mutex}; -use std::time::{Duration, Instant}; +use std::time::Instant; use tokio::sync::Mutex as AsyncMutex; -use tokio::task::JoinHandle; -/// JS that discovers the actionable elements on the page and returns them as an -/// array of `{selector, role, label}`. Best-effort: it prefers `#id` selectors, -/// falls back to an `:nth-of-type` path, skips hidden/disabled elements, and is -/// bounded so a huge page can't blow up the observation. -/// -/// It also descends into open shadow roots and puts elements that live inside a -/// modal/dialog (or a fixed high-z-index overlay) first, so a dialog's buttons -/// are never dropped by the cap when the page behind it is long. -const DISCOVER_ELEMENTS_JS: &str = r#" -(() => { - const MAX = 200; - const SEL = 'a,button,input,textarea,select,summary,[role=button],[role=link],[role=checkbox],[role=tab],[role=menuitem],[role=menuitemcheckbox],[role=menuitemradio],[role=switch],[role=option],[contenteditable=true],[onclick],[tabindex]'; - const seen = new Set(); - const out = []; - - const visible = (el) => { - if (el.disabled) return false; - const rects = el.getClientRects(); - if (!rects.length) return false; - const r = rects[0]; - if (r.width < 1 || r.height < 1) return false; - const style = getComputedStyle(el); - if (style.visibility === 'hidden' || style.display === 'none') return false; - return true; - }; - - const cssPath = (el) => { - if (el.id) return '#' + CSS.escape(el.id); - const parts = []; - let node = el; - while (node && node.nodeType === 1 && node.tagName !== 'HTML') { - let sel = node.tagName.toLowerCase(); - if (node.id) { parts.unshift('#' + CSS.escape(node.id)); break; } - const parent = node.parentNode; - if (parent && parent.children) { - const sameTag = Array.from(parent.children).filter(c => c.tagName === node.tagName); - if (sameTag.length > 1) { - sel += ':nth-of-type(' + (sameTag.indexOf(node) + 1) + ')'; - } - } - parts.unshift(sel); - node = node.parentNode && node.parentNode.host ? node.parentNode.host : node.parentNode; - } - return parts.join(' > '); - }; - - const roleOf = (el) => { - const r = el.getAttribute('role'); - if (r) return r; - const tag = el.tagName.toLowerCase(); - if (tag === 'input') return (el.getAttribute('type') || 'text'); - return tag; - }; - - const labelOf = (el) => { - const pick = (s) => (s || '').replace(/\s+/g, ' ').trim(); - let l = pick(el.getAttribute('aria-label')); - if (!l) l = pick(el.textContent); - if (!l) l = pick(el.value); - if (!l) l = pick(el.getAttribute('placeholder')); - if (!l) l = pick(el.getAttribute('name')); - if (!l) l = pick(el.getAttribute('alt')); - if (!l) l = pick(el.getAttribute('title')); - return l.slice(0, 80); - }; - - // Rank: elements inside a dialog / high-z fixed overlay first, so the - // topmost interactive surface is never truncated away. - const inDialog = (el) => { - let node = el; - while (node && node.nodeType === 1) { - const role = node.getAttribute && node.getAttribute('role'); - if (node.tagName === 'DIALOG' || role === 'dialog' || role === 'alertdialog' || node.getAttribute && node.getAttribute('aria-modal') === 'true') { - return true; - } - if (node.parentNode && node.parentNode.host) { node = node.parentNode.host; continue; } - node = node.parentNode; - } - return false; - }; - - // Gather across the main document and any open shadow roots. - const collect = (root, acc) => { - let nodes = []; - try { nodes = Array.from(root.querySelectorAll(SEL)); } catch (e) {} - for (const el of nodes) acc.push(el); - let all = []; - try { all = Array.from(root.querySelectorAll('*')); } catch (e) {} - for (const el of all) { - if (el.shadowRoot) collect(el.shadowRoot, acc); - } - }; - - const candidates = []; - collect(document, candidates); - - // Stable sort: dialog elements first, keeping document order otherwise. - const ranked = candidates - .map((el, i) => ({ el, i, dlg: inDialog(el) ? 0 : 1 })) - .sort((a, b) => (a.dlg - b.dlg) || (a.i - b.i)); - - for (const { el } of ranked) { - if (out.length >= MAX) break; - if (!visible(el)) continue; - const selector = cssPath(el); - if (!selector || seen.has(selector)) continue; - seen.add(selector); - out.push({ selector, role: roleOf(el), label: labelOf(el) }); - } - return out; -})() -"#; - -/// One actionable element discovered on the page, so the model can target it by -/// selector instead of guessing. -#[derive(Debug, Clone, serde::Serialize, serde::Deserialize)] -pub struct InteractiveElement { - /// A CSS selector that resolves to this element (`#id` when available, else - /// an `:nth-of-type` path). - pub selector: String, - /// The element's ARIA role or tag name (button, a, input, checkbox, …). - pub role: String, - /// A short human label: visible text, aria-label, placeholder, name, … - pub label: String, -} - -/// A JavaScript dialog (`alert` / `confirm` / `prompt` / `beforeunload`) the -/// session answered on its own, reported so the model knows it happened. +/// A tab as listed for the model. #[derive(Debug, Clone, serde::Serialize, serde::Deserialize)] -pub struct HandledDialog { - pub kind: String, - pub message: String, - /// Whether the dialog was accepted (OK) or dismissed (Cancel). - pub accepted: bool, -} - -/// What the model sees after acting: where it is and what's on the page. -#[derive(Debug, Clone, Default, serde::Serialize, serde::Deserialize)] -pub struct PageObservation { +pub struct TabInfo { + pub id: String, pub url: String, pub title: String, - /// Visible text (`document.body.innerText`), the cheap textual companion to - /// a screenshot. - pub text: String, - /// Actionable elements (bounded), so the model targets real selectors - /// instead of guessing from the screenshot. - #[serde(default)] - pub elements: Vec, - /// Viewport size in CSS pixels (`window.innerWidth`/`innerHeight`). Disclosed - /// so the model can express coordinate clicks in `px` — it cannot read the - /// true size off a screenshot the API has already resized. - #[serde(default)] - pub viewport_width: f64, - #[serde(default)] - pub viewport_height: f64, - /// Dialogs answered since the previous observation. - #[serde(default)] - pub dialogs: Vec, + pub active: bool, } -/// How long a [`BrowserSession`] waits for the page before giving up. -#[derive(Debug, Clone, Copy)] -pub struct BrowserTimeouts { - /// One interaction: a click, a read, a screenshot, a script. - pub command: Duration, - /// Loading a page until its `load` event. - pub navigation: Duration, +#[derive(Default)] +struct Tabs { + list: Vec>, + /// The tab a call without an explicit tab id targets. + active: Option, + next_id: u32, } -impl Default for BrowserTimeouts { - fn default() -> Self { - Self { - command: Duration::from_secs(15), - navigation: Duration::from_secs(30), - } +impl Tabs { + fn mint_id(&mut self) -> String { + self.next_id += 1; + format!("t{}", self.next_id) } -} - -/// The page did not answer within its limit: it is busy, hung, or (for a -/// navigation) still loading something. Callers can downcast an -/// `anyhow::Error` to this to tell a stuck page from a failed action. -#[derive(Debug)] -pub struct BrowserTimeout { - pub what: &'static str, - pub after: Duration, -} -impl std::fmt::Display for BrowserTimeout { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - write!( - f, - "{} timed out after {}s", - self.what, - self.after.as_secs_f64() - ) + fn get(&self, id: &str) -> Option> { + self.list.iter().find(|t| t.id() == id).cloned() } } -impl std::error::Error for BrowserTimeout {} - -/// One live page on a launched browser, driven across many tool calls. +/// One launched browser and its tabs, driven across many tool calls. pub struct BrowserSession { /// Kept alive so the browser process outlives individual tool calls; behind /// an async mutex only because a graceful [`close`](Self::close) needs `&mut`. launched: AsyncMutex, - /// The page every interaction targets. `Page` is internally reference - /// counted and its methods take `&self`, so all verbs below are `&self`. - page: Page, + tabs: Mutex, label: String, /// Whether this is an ephemeral throwaway browser (no persistent profile). /// Ephemeral sessions are dropped at the end of an agent turn (see @@ -247,77 +62,34 @@ pub struct BrowserSession { /// `browser_navigate` on the default profile can't leak a Chrome process; /// persistent named profiles survive across turns on purpose. ephemeral: bool, - /// Dialogs answered since the last observation (drained by `observe`). - dialogs: Arc>>, - /// Whether `confirm`/`prompt` dialogs are accepted rather than dismissed. - accept_dialogs: Arc, - /// Answers dialogs as they open (aborted on drop). - dialog_task: JoinHandle<()>, - timeouts: BrowserTimeouts, + timeouts: Arc>, } impl BrowserSession { - /// Launch a browser for `config` and open a blank page to drive. + /// Launch a browser for `config` with one blank tab. pub async fn open( config: crate::browser::BrowserLaunchConfig, label: impl Into, ) -> Result { let ephemeral = matches!(config.profile, crate::browser::BrowserProfile::Ephemeral); let launched = LaunchedBrowser::launch(config).await?; - let page = launched.browser.new_page("about:blank").await?; - let dialogs = Arc::new(Mutex::new(Vec::new())); - let accept_dialogs = Arc::new(AtomicBool::new(false)); - let dialog_task = - spawn_dialog_handler(&page, dialogs.clone(), accept_dialogs.clone()).await?; - Ok(Self { + let session = Self { launched: AsyncMutex::new(launched), - page, + tabs: Mutex::new(Tabs::default()), label: label.into(), ephemeral, - dialogs, - accept_dialogs, - dialog_task, - timeouts: BrowserTimeouts::default(), - }) + timeouts: Arc::new(Mutex::new(BrowserTimeouts::default())), + }; + session.create_tab(true).await?; + Ok(session) } - /// Use other limits than [`BrowserTimeouts::default`]. - pub fn with_timeouts(mut self, timeouts: BrowserTimeouts) -> Self { - self.timeouts = timeouts; + /// Use other limits than [`BrowserTimeouts::default`], for every tab. + pub fn with_timeouts(self, timeouts: BrowserTimeouts) -> Self { + *self.timeouts.lock().unwrap() = timeouts; self } - /// Run one interaction with the page, giving up after `limit`. - /// - /// chromiumoxide's own per-command timeout does not hold when the - /// renderer is stuck: a click on a hung page was measured to block for - /// minutes. Bounding each verb here keeps a busy or broken page from - /// stalling the agent. - async fn bounded( - &self, - what: &'static str, - limit: Duration, - interaction: impl std::future::Future>, - ) -> Result { - match tokio::time::timeout(limit, interaction).await { - Ok(result) => result, - Err(_) => Err(BrowserTimeout { what, after: limit }.into()), - } - } - - /// Typing presses a key per character, so long text gets more time. - fn typing_limit(&self, text: &str) -> Duration { - self.timeouts.command + Duration::from_millis(20) * text.chars().count() as u32 - } - - /// Accept (`true`) or dismiss (`false`, the default) `confirm` and `prompt` - /// dialogs from now on. `alert` and `beforeunload` are always accepted: an - /// alert has nothing to decide, and a `beforeunload` prompt only appears - /// when leaving the page was already requested. - pub fn set_accept_dialogs(&self, accept: bool) { - self.accept_dialogs.store(accept, Ordering::Relaxed); - } - pub fn label(&self) -> &str { &self.label } @@ -327,555 +99,140 @@ impl BrowserSession { self.ephemeral } - /// Navigate to a URL and wait for its `load` event. (`goto` already waits; - /// a further `wait_for_navigation` has no timeout and could hang on a page - /// that starts a script redirect right after loading.) - pub async fn navigate(&self, url: &str) -> Result<()> { - self.bounded("navigation", self.timeouts.navigation, async { - self.page.goto(url).await?; - Ok(()) - }) - .await - } - - /// Capture a PNG screenshot — the model's eyes. `full_page` captures the - /// entire scrollable page instead of just the current viewport. - pub async fn screenshot(&self, full_page: bool) -> Result> { - self.bounded("screenshot", self.timeouts.command, async { - let params = ScreenshotParams::builder() - .format(CaptureScreenshotFormat::Png) - .full_page(full_page) - .build(); - Ok(self.page.screenshot(params).await?) - }) - .await - } - - /// Scroll the page or an element. With a `selector` and no delta, scroll - /// that element into view. With a `selector` **and** a non-zero `(dx, dy)`, - /// scroll *inside* that element — the fix for modal/dialog content that - /// lives in its own scroll container (a plain `window.scrollBy` moves the - /// page behind it, not the dialog). With no selector, scroll the page by - /// `(dx, dy)` (positive `dy` scrolls down). The selector is JSON-encoded - /// into the script, so it cannot break out of the string. - pub async fn scroll(&self, selector: Option<&str>, dx: f64, dy: f64) -> Result<()> { - self.bounded("scroll", self.timeouts.command, async { - match selector { - Some(sel) => { - let sel_json = serde_json::to_string(sel)?; - let js = if dx == 0.0 && dy == 0.0 { - format!( - "(() => {{ const e = document.querySelector({sel_json}); \ - if (!e) return false; \ - e.scrollIntoView({{block: 'center', inline: 'center'}}); \ - return true; }})()" - ) - } else { - // Scroll within the element's own scroll container (or the - // nearest scrollable ancestor if the element itself does not - // scroll), so a dialog's inner content moves. - format!( - "(() => {{ let e = document.querySelector({sel_json}); \ - if (!e) return false; \ - const scrollable = (n) => {{ \ - while (n && n !== document.body) {{ \ - const s = getComputedStyle(n); \ - if (/(auto|scroll)/.test(s.overflowY + s.overflow) && n.scrollHeight > n.clientHeight) return n; \ - n = n.parentElement; \ - }} \ - return e; \ - }}; \ - const target = (e.scrollHeight > e.clientHeight) ? e : scrollable(e); \ - target.scrollBy({dx}, {dy}); \ - return true; }})()" - ) - }; - let found = self - .page - .evaluate(js) - .await? - .into_value::() - .unwrap_or(false); - if !found { - anyhow::bail!("no element matches selector '{sel}'"); - } - } - None => { - self.page - .evaluate(format!("window.scrollBy({dx}, {dy})")) - .await?; - } + /// Open a new blank tab. `foreground` makes it the tab that calls without + /// a tab id target. + pub async fn create_tab(&self, foreground: bool) -> Result> { + let page = self + .launched + .lock() + .await + .browser + .new_page("about:blank") + .await?; + let id = self.tabs.lock().unwrap().mint_id(); + let tab = Arc::new(Tab::new(id.clone(), page, self.timeouts.clone()).await?); + let mut tabs = self.tabs.lock().unwrap(); + tabs.list.push(tab.clone()); + if foreground || tabs.active.is_none() { + tabs.active = Some(id); } - Ok(()) - }) - .await - } - - /// Read the current location, title, visible text, and the actionable - /// elements on the page. - pub async fn observe(&self) -> Result { - self.observe_with(true).await - } - - /// Like [`observe`](Self::observe), but `include_text` can suppress the - /// (often large and redundant) `innerText` dump — the model keeps the - /// screenshot plus the interactive-element list, and avoids re-reading a - /// long form's text on every step. - pub async fn observe_with(&self, include_text: bool) -> Result { - self.bounded("reading the page", self.timeouts.command, async { - let url = self.page.url().await?.unwrap_or_default(); - let title = self.page.get_title().await?.unwrap_or_default(); - let text = if include_text { - self.page - .evaluate("document.body ? document.body.innerText : ''") - .await? - .into_value::() - .unwrap_or_default() - } else { - String::new() - }; - // Element discovery is best-effort: a failure (e.g. mid-navigation) - // just yields an empty list rather than failing the observation. - let elements = match self.page.evaluate(DISCOVER_ELEMENTS_JS).await { - Ok(v) => v - .into_value::>() - .unwrap_or_default(), - Err(_) => Vec::new(), - }; - let (viewport_width, viewport_height) = - self.viewport_size().await.unwrap_or((0.0, 0.0)); - let dialogs = std::mem::take(&mut *self.dialogs.lock().unwrap()); - Ok(PageObservation { - url, - title, - text, - elements, - viewport_width, - viewport_height, - dialogs, - }) - }) - .await - } - - /// The viewport size in CSS pixels — the reference frame for coordinate - /// clicks (CDP mouse events use CSS pixels, so a coordinate resolved against - /// this lands where intended regardless of screenshot scaling or DPR). - pub async fn viewport_size(&self) -> Result<(f64, f64)> { - self.bounded("reading the viewport size", self.timeouts.command, async { - let dims = self - .page - .evaluate("[window.innerWidth, window.innerHeight]") - .await? - .into_value::<(f64, f64)>() - .unwrap_or((0.0, 0.0)); - Ok(dims) - }) - .await + Ok(tab) } - /// Find an element by a selector, turning chromiumoxide's opaque CDP miss - /// ("Could not find node with given id") into a message that names the - /// selector. - /// - /// Besides plain CSS, this understands robust prefixes that survive a site - /// re-rendering with fresh hashed ids (a real problem on portals like - /// ELSTER): - /// - `text=Foo` — the first visible element whose trimmed text / - /// aria-label / value contains `Foo` (case-insensitive). - /// - `role=button` — the first element with that ARIA role (or, for a bare - /// tag, that tag). `role=button[name=Save]` also matches on text. - /// - `aria=Save` — the first element whose aria-label matches. + /// Adopt tabs the page opened itself (`target=_blank`, `window.open`) and + /// forget tabs that were closed. Returns the ids of newly adopted tabs. /// - /// These are resolved to a concrete node in the page, so a fragile hashed - /// `#id` is never needed. - async fn find(&self, selector: &str) -> Result { - if let Some(css) = self.resolve_semantic_selector(selector).await? { - return self - .page - .find_element(&css) - .await - .map_err(|_| anyhow::anyhow!("no element matches selector '{selector}'")); - } - self.page - .find_element(selector) - .await - .map_err(|_| anyhow::anyhow!("no element matches selector '{selector}'")) - } - - /// If `selector` uses a semantic prefix (`text=`, `role=`, `aria=`), locate - /// the matching element in the page and stamp it with a unique data - /// attribute, returning a concrete CSS selector for it. Returns `Ok(None)` - /// for a plain CSS selector (handled directly by the caller). - async fn resolve_semantic_selector(&self, selector: &str) -> Result> { - let sel = selector.trim(); - let (kind, query) = if let Some(q) = sel.strip_prefix("text=") { - ("text", q) - } else if let Some(q) = sel.strip_prefix("role=") { - ("role", q) - } else if let Some(q) = sel.strip_prefix("aria=") { - ("aria", q) - } else { - return Ok(None); + /// Only pages with an opener are adopted: Chrome's own initial blank tab + /// has none and stays out of the list. + pub async fn sync_tabs(&self) -> Result> { + let (pages, opened) = { + let launched = self.launched.lock().await; + let pages = launched.browser.pages().await?; + let targets = launched + .browser + .execute(GetTargetsParams::default()) + .await? + .result + .target_infos; + let opened: Vec<_> = targets + .into_iter() + .filter(|t| t.opener_id.is_some()) + .map(|t| t.target_id) + .collect(); + (pages, opened) }; - let kind_json = serde_json::to_string(kind)?; - let query_json = serde_json::to_string(query.trim())?; - // Tag the match with a unique attribute so we can hand back a stable CSS - // selector even on a page that mints fresh ids every render. - let js = format!( - r#"(() => {{ - const kind = {kind_json}; - const q = {query_json}; - const SEL = 'a,button,input,textarea,select,summary,[role],[onclick],[tabindex],label'; - const norm = (s) => (s || '').replace(/\s+/g, ' ').trim().toLowerCase(); - const visible = (el) => {{ - const r = el.getClientRects(); - if (!r.length) return false; - const st = getComputedStyle(el); - return st.visibility !== 'hidden' && st.display !== 'none'; - }}; - const labelText = (el) => norm(el.getAttribute('aria-label')) || norm(el.textContent) || norm(el.value) || norm(el.getAttribute('placeholder')) || norm(el.title); - const roleOf = (el) => (el.getAttribute('role') || el.tagName.toLowerCase()); - // role=button[name=Save] → role + optional name filter. - let wantRole = q, wantName = null; - const m = q.match(/^([^\[]+)\[name=(.+)\]$/); - if (kind === 'role' && m) {{ wantRole = m[1].trim(); wantName = norm(m[2]); }} - const needle = norm(q); - const collect = (root, acc) => {{ - let nodes = []; - try {{ nodes = Array.from(root.querySelectorAll(SEL)); }} catch (e) {{}} - for (const el of nodes) acc.push(el); - let all = []; - try {{ all = Array.from(root.querySelectorAll('*')); }} catch (e) {{}} - for (const el of all) if (el.shadowRoot) collect(el.shadowRoot, acc); - }}; - const cands = []; - collect(document, cands); - const match = cands.find((el) => {{ - if (!visible(el)) return false; - if (kind === 'text') return labelText(el).includes(needle); - if (kind === 'aria') return norm(el.getAttribute('aria-label')).includes(needle); - if (kind === 'role') {{ - if (norm(roleOf(el)) !== norm(wantRole)) return false; - return wantName ? labelText(el).includes(wantName) : true; - }} - return false; - }}); - if (!match) return null; - const token = 'ca-sel-' + Math.random().toString(36).slice(2); - match.setAttribute('data-ca-sel', token); - return '[data-ca-sel="' + token + '"]'; -}})()"# - ); - let resolved = self - .page - .evaluate(js) - .await? - .into_value::>() - .unwrap_or(None); - match resolved { - Some(css) => Ok(Some(css)), - None => Err(anyhow::anyhow!("no element matches selector '{selector}'")), + let known: Vec<_> = { + let tabs = self.tabs.lock().unwrap(); + tabs.list + .iter() + .map(|t| t.page().target_id().clone()) + .collect() + }; + let mut adopted = Vec::new(); + for page in pages + .iter() + .filter(|p| !known.contains(p.target_id()) && opened.contains(p.target_id())) + { + let id = self.tabs.lock().unwrap().mint_id(); + let tab = Arc::new(Tab::new(id.clone(), page.clone(), self.timeouts.clone()).await?); + self.tabs.lock().unwrap().list.push(tab); + adopted.push(id); } - } - - /// Click the first element matching a selector. The element is scrolled - /// into view first (via `scrollIntoView`, which handles nested scroll - /// containers), so an off-screen or collapsed target no longer fails with - /// "Node is either not visible or not an HTMLElement". - pub async fn click(&self, selector: &str) -> Result<()> { - self.bounded("click", self.timeouts.command, async { - let element = self.find(selector).await?; - let _ = element.scroll_into_view().await; - element.click().await?; - Ok(()) - }) - .await - } - - /// Click at viewport coordinates `(x, y)`. For canvas/WebGL surfaces and - /// anything without a stable selector (games, maps, drag targets). - pub async fn click_at(&self, x: f64, y: f64) -> Result<()> { - self.bounded("click", self.timeouts.command, async { - self.page.click(Point { x, y }).await?; - Ok(()) - }) - .await - } - - /// Move the mouse to viewport coordinates `(x, y)` without clicking — drives - /// hover states and canvas pointer-move handlers. - pub async fn move_mouse(&self, x: f64, y: f64) -> Result<()> { - self.bounded("mouse move", self.timeouts.command, async { - self.page.move_mouse(Point { x, y }).await?; - Ok(()) - }) - .await - } - - /// Focus a field and type text into it, appending to any existing value. - /// The element is scrolled into view first. Never used for credentials — - /// those go through the human-in-the-loop login handoff. Prefer - /// [`fill`](Self::fill) to replace a prefilled field. - pub async fn type_text(&self, selector: &str, text: &str) -> Result<()> { - self.bounded("typing", self.typing_limit(text), async { - let element = self.find(selector).await?; - let _ = element.scroll_into_view().await; - element.focus().await?; - self.type_chars(text).await - }) - .await - } - - /// Clear a field, then type `text` — the replace semantics editing a - /// prefilled input needs (the old workflow required End + repeated - /// Backspace). Works for ``/`