From 7f8e8e2be32d4d5a3f6c739578a2807abba632da Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Stephan=20A=C3=9Fmus?= Date: Fri, 2 Oct 2026 11:52:20 +0200 Subject: [PATCH 01/13] refactor(web): give a browser session tabs MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The page verbs move from BrowserSession to a new Tab, which also owns its answered dialogs. A BrowserSession now holds the launched browser and its tabs: it creates, selects and closes them, and sync_tabs adopts tabs the page opened itself (target=_blank, window.open) without making them active. The session's timeouts are shared with every tab. Session-level verbs that act on the active tab stay for now, for the tools that predate tabs. Headless browsers get a 1280×800 viewport at device scale 1 instead of chromiumoxide's 800×600, so pages lay out for a desktop and screenshot pixels are CSS pixels. A headful login window fills its real window. --- crates/web/src/browser.rs | 20 +- crates/web/src/browser_session.rs | 1157 +++++------------------------ crates/web/src/lib.rs | 7 +- crates/web/src/tab.rs | 1074 ++++++++++++++++++++++++++ crates/web/src/tests.rs | 59 ++ 5 files changed, 1337 insertions(+), 980 deletions(-) create mode 100644 crates/web/src/tab.rs diff --git a/crates/web/src/browser.rs b/crates/web/src/browser.rs index 33190519..3c003e35 100644 --- a/crates/web/src/browser.rs +++ b/crates/web/src/browser.rs @@ -13,6 +13,7 @@ //! default preserves the old behavior. use anyhow::Result; +use chromiumoxide::handler::viewport::Viewport; use chromiumoxide::{Browser, BrowserConfig}; use futures::StreamExt; use std::path::PathBuf; @@ -80,6 +81,11 @@ pub(crate) fn resolve_user_data_dir( } } +/// Viewport of a headless browser in CSS pixels: a common laptop size, so +/// pages lay out as for a desktop user (chromiumoxide defaults to 800×600). +/// The device scale factor stays 1, so screenshot pixels are CSS pixels. +pub const DEFAULT_VIEWPORT: (u32, u32) = (1280, 800); + /// How long a graceful [`LaunchedBrowser::close`] may take before the process /// is killed. const CLOSE_TIMEOUT: Duration = Duration::from_secs(10); @@ -100,7 +106,19 @@ impl LaunchedBrowser { let mut builder = BrowserConfig::builder().user_data_dir(&data_dir); if config.headful { - builder = builder.with_head(); + // A window a human uses: let the page fill it instead of emulating + // a fixed viewport inside it. + builder = builder + .with_head() + .viewport(None) + .window_size(DEFAULT_VIEWPORT.0, DEFAULT_VIEWPORT.1 + 100); + } else { + builder = builder.viewport(Viewport { + width: DEFAULT_VIEWPORT.0, + height: DEFAULT_VIEWPORT.1, + device_scale_factor: Some(1.0), + ..Viewport::default() + }); } let browser_config = builder.build().map_err(|e| anyhow::anyhow!("{e}"))?; diff --git a/crates/web/src/browser_session.rs b/crates/web/src/browser_session.rs index 3f79813c..aada02d0 100644 --- a/crates/web/src/browser_session.rs +++ b/crates/web/src/browser_session.rs @@ -1,245 +1,59 @@ //! Interactive browser sessions for agent tools. //! //! [`crate::WebClient`] covers the one-shot case: fetch a page, extract it, -//! discard it. Browser *agency* needs the opposite — a page the agent drives +//! discard it. Browser *agency* needs the opposite — pages the agent drives //! over many tool calls: navigate, look (screenshot / read), click, type, wait. //! -//! This mirrors the `pty_session` crate one-to-one: -//! - [`BrowserSession`] — one live page on a launched browser, with the -//! interaction verbs, kept across tool calls. -//! - [`BrowserSessionManager`] — an id-keyed registry with an LRU cap, one per -//! agent session, so browser sessions survive across tool calls but die with -//! their agent session. +//! This mirrors the `pty_session` crate: +//! - [`BrowserSession`] — one launched browser with its tabs ([`Tab`]), kept +//! across tool calls. +//! - [`BrowserSessionManager`] — a registry with an LRU cap, one per agent +//! session, so browser sessions survive across tool calls but die with their +//! agent session. use crate::browser::LaunchedBrowser; +use crate::tab::{BrowserTimeouts, Tab}; use anyhow::Result; -use chromiumoxide::cdp::browser_protocol::input::{ - DispatchKeyEventParams, DispatchKeyEventType, InsertTextParams, -}; -use chromiumoxide::cdp::browser_protocol::network::{CookieParam, CookieSameSite, TimeSinceEpoch}; -use chromiumoxide::cdp::browser_protocol::page::{ - CaptureScreenshotFormat, DialogType, EventJavascriptDialogOpening, HandleJavaScriptDialogParams, -}; -use chromiumoxide::element::Element; -use chromiumoxide::keys::{KeyDefinition, get_key_definition}; -use chromiumoxide::layout::Point; -use chromiumoxide::page::{Page, ScreenshotParams}; -use futures::StreamExt; +use chromiumoxide::cdp::browser_protocol::network::CookieParam; use std::collections::HashMap; -use std::sync::atomic::{AtomicBool, Ordering}; use std::sync::{Arc, Mutex}; use std::time::{Duration, Instant}; use tokio::sync::Mutex as AsyncMutex; -use tokio::task::JoinHandle; -/// JS that discovers the actionable elements on the page and returns them as an -/// array of `{selector, role, label}`. Best-effort: it prefers `#id` selectors, -/// falls back to an `:nth-of-type` path, skips hidden/disabled elements, and is -/// bounded so a huge page can't blow up the observation. -/// -/// It also descends into open shadow roots and puts elements that live inside a -/// modal/dialog (or a fixed high-z-index overlay) first, so a dialog's buttons -/// are never dropped by the cap when the page behind it is long. -const DISCOVER_ELEMENTS_JS: &str = r#" -(() => { - const MAX = 200; - const SEL = 'a,button,input,textarea,select,summary,[role=button],[role=link],[role=checkbox],[role=tab],[role=menuitem],[role=menuitemcheckbox],[role=menuitemradio],[role=switch],[role=option],[contenteditable=true],[onclick],[tabindex]'; - const seen = new Set(); - const out = []; - - const visible = (el) => { - if (el.disabled) return false; - const rects = el.getClientRects(); - if (!rects.length) return false; - const r = rects[0]; - if (r.width < 1 || r.height < 1) return false; - const style = getComputedStyle(el); - if (style.visibility === 'hidden' || style.display === 'none') return false; - return true; - }; - - const cssPath = (el) => { - if (el.id) return '#' + CSS.escape(el.id); - const parts = []; - let node = el; - while (node && node.nodeType === 1 && node.tagName !== 'HTML') { - let sel = node.tagName.toLowerCase(); - if (node.id) { parts.unshift('#' + CSS.escape(node.id)); break; } - const parent = node.parentNode; - if (parent && parent.children) { - const sameTag = Array.from(parent.children).filter(c => c.tagName === node.tagName); - if (sameTag.length > 1) { - sel += ':nth-of-type(' + (sameTag.indexOf(node) + 1) + ')'; - } - } - parts.unshift(sel); - node = node.parentNode && node.parentNode.host ? node.parentNode.host : node.parentNode; - } - return parts.join(' > '); - }; - - const roleOf = (el) => { - const r = el.getAttribute('role'); - if (r) return r; - const tag = el.tagName.toLowerCase(); - if (tag === 'input') return (el.getAttribute('type') || 'text'); - return tag; - }; - - const labelOf = (el) => { - const pick = (s) => (s || '').replace(/\s+/g, ' ').trim(); - let l = pick(el.getAttribute('aria-label')); - if (!l) l = pick(el.textContent); - if (!l) l = pick(el.value); - if (!l) l = pick(el.getAttribute('placeholder')); - if (!l) l = pick(el.getAttribute('name')); - if (!l) l = pick(el.getAttribute('alt')); - if (!l) l = pick(el.getAttribute('title')); - return l.slice(0, 80); - }; - - // Rank: elements inside a dialog / high-z fixed overlay first, so the - // topmost interactive surface is never truncated away. - const inDialog = (el) => { - let node = el; - while (node && node.nodeType === 1) { - const role = node.getAttribute && node.getAttribute('role'); - if (node.tagName === 'DIALOG' || role === 'dialog' || role === 'alertdialog' || node.getAttribute && node.getAttribute('aria-modal') === 'true') { - return true; - } - if (node.parentNode && node.parentNode.host) { node = node.parentNode.host; continue; } - node = node.parentNode; - } - return false; - }; - - // Gather across the main document and any open shadow roots. - const collect = (root, acc) => { - let nodes = []; - try { nodes = Array.from(root.querySelectorAll(SEL)); } catch (e) {} - for (const el of nodes) acc.push(el); - let all = []; - try { all = Array.from(root.querySelectorAll('*')); } catch (e) {} - for (const el of all) { - if (el.shadowRoot) collect(el.shadowRoot, acc); - } - }; - - const candidates = []; - collect(document, candidates); - - // Stable sort: dialog elements first, keeping document order otherwise. - const ranked = candidates - .map((el, i) => ({ el, i, dlg: inDialog(el) ? 0 : 1 })) - .sort((a, b) => (a.dlg - b.dlg) || (a.i - b.i)); - - for (const { el } of ranked) { - if (out.length >= MAX) break; - if (!visible(el)) continue; - const selector = cssPath(el); - if (!selector || seen.has(selector)) continue; - seen.add(selector); - out.push({ selector, role: roleOf(el), label: labelOf(el) }); - } - return out; -})() -"#; - -/// One actionable element discovered on the page, so the model can target it by -/// selector instead of guessing. -#[derive(Debug, Clone, serde::Serialize, serde::Deserialize)] -pub struct InteractiveElement { - /// A CSS selector that resolves to this element (`#id` when available, else - /// an `:nth-of-type` path). - pub selector: String, - /// The element's ARIA role or tag name (button, a, input, checkbox, …). - pub role: String, - /// A short human label: visible text, aria-label, placeholder, name, … - pub label: String, -} - -/// A JavaScript dialog (`alert` / `confirm` / `prompt` / `beforeunload`) the -/// session answered on its own, reported so the model knows it happened. +/// A tab as listed for the model. #[derive(Debug, Clone, serde::Serialize, serde::Deserialize)] -pub struct HandledDialog { - pub kind: String, - pub message: String, - /// Whether the dialog was accepted (OK) or dismissed (Cancel). - pub accepted: bool, -} - -/// What the model sees after acting: where it is and what's on the page. -#[derive(Debug, Clone, Default, serde::Serialize, serde::Deserialize)] -pub struct PageObservation { +pub struct TabInfo { + pub id: String, pub url: String, pub title: String, - /// Visible text (`document.body.innerText`), the cheap textual companion to - /// a screenshot. - pub text: String, - /// Actionable elements (bounded), so the model targets real selectors - /// instead of guessing from the screenshot. - #[serde(default)] - pub elements: Vec, - /// Viewport size in CSS pixels (`window.innerWidth`/`innerHeight`). Disclosed - /// so the model can express coordinate clicks in `px` — it cannot read the - /// true size off a screenshot the API has already resized. - #[serde(default)] - pub viewport_width: f64, - #[serde(default)] - pub viewport_height: f64, - /// Dialogs answered since the previous observation. - #[serde(default)] - pub dialogs: Vec, + pub active: bool, } -/// How long a [`BrowserSession`] waits for the page before giving up. -#[derive(Debug, Clone, Copy)] -pub struct BrowserTimeouts { - /// One interaction: a click, a read, a screenshot, a script. - pub command: Duration, - /// Loading a page until its `load` event. - pub navigation: Duration, +#[derive(Default)] +struct Tabs { + list: Vec>, + /// The tab a call without an explicit tab id targets. + active: Option, + next_id: u32, } -impl Default for BrowserTimeouts { - fn default() -> Self { - Self { - command: Duration::from_secs(15), - navigation: Duration::from_secs(30), - } +impl Tabs { + fn mint_id(&mut self) -> String { + self.next_id += 1; + format!("t{}", self.next_id) } -} - -/// The page did not answer within its limit: it is busy, hung, or (for a -/// navigation) still loading something. Callers can downcast an -/// `anyhow::Error` to this to tell a stuck page from a failed action. -#[derive(Debug)] -pub struct BrowserTimeout { - pub what: &'static str, - pub after: Duration, -} -impl std::fmt::Display for BrowserTimeout { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - write!( - f, - "{} timed out after {}s", - self.what, - self.after.as_secs_f64() - ) + fn get(&self, id: &str) -> Option> { + self.list.iter().find(|t| t.id() == id).cloned() } } -impl std::error::Error for BrowserTimeout {} - -/// One live page on a launched browser, driven across many tool calls. +/// One launched browser and its tabs, driven across many tool calls. pub struct BrowserSession { /// Kept alive so the browser process outlives individual tool calls; behind /// an async mutex only because a graceful [`close`](Self::close) needs `&mut`. launched: AsyncMutex, - /// The page every interaction targets. `Page` is internally reference - /// counted and its methods take `&self`, so all verbs below are `&self`. - page: Page, + tabs: Mutex, label: String, /// Whether this is an ephemeral throwaway browser (no persistent profile). /// Ephemeral sessions are dropped at the end of an agent turn (see @@ -247,77 +61,34 @@ pub struct BrowserSession { /// `browser_navigate` on the default profile can't leak a Chrome process; /// persistent named profiles survive across turns on purpose. ephemeral: bool, - /// Dialogs answered since the last observation (drained by `observe`). - dialogs: Arc>>, - /// Whether `confirm`/`prompt` dialogs are accepted rather than dismissed. - accept_dialogs: Arc, - /// Answers dialogs as they open (aborted on drop). - dialog_task: JoinHandle<()>, - timeouts: BrowserTimeouts, + timeouts: Arc>, } impl BrowserSession { - /// Launch a browser for `config` and open a blank page to drive. + /// Launch a browser for `config` with one blank tab. pub async fn open( config: crate::browser::BrowserLaunchConfig, label: impl Into, ) -> Result { let ephemeral = matches!(config.profile, crate::browser::BrowserProfile::Ephemeral); let launched = LaunchedBrowser::launch(config).await?; - let page = launched.browser.new_page("about:blank").await?; - let dialogs = Arc::new(Mutex::new(Vec::new())); - let accept_dialogs = Arc::new(AtomicBool::new(false)); - let dialog_task = - spawn_dialog_handler(&page, dialogs.clone(), accept_dialogs.clone()).await?; - Ok(Self { + let session = Self { launched: AsyncMutex::new(launched), - page, + tabs: Mutex::new(Tabs::default()), label: label.into(), ephemeral, - dialogs, - accept_dialogs, - dialog_task, - timeouts: BrowserTimeouts::default(), - }) + timeouts: Arc::new(Mutex::new(BrowserTimeouts::default())), + }; + session.create_tab(true).await?; + Ok(session) } - /// Use other limits than [`BrowserTimeouts::default`]. - pub fn with_timeouts(mut self, timeouts: BrowserTimeouts) -> Self { - self.timeouts = timeouts; + /// Use other limits than [`BrowserTimeouts::default`], for every tab. + pub fn with_timeouts(self, timeouts: BrowserTimeouts) -> Self { + *self.timeouts.lock().unwrap() = timeouts; self } - /// Run one interaction with the page, giving up after `limit`. - /// - /// chromiumoxide's own per-command timeout does not hold when the - /// renderer is stuck: a click on a hung page was measured to block for - /// minutes. Bounding each verb here keeps a busy or broken page from - /// stalling the agent. - async fn bounded( - &self, - what: &'static str, - limit: Duration, - interaction: impl std::future::Future>, - ) -> Result { - match tokio::time::timeout(limit, interaction).await { - Ok(result) => result, - Err(_) => Err(BrowserTimeout { what, after: limit }.into()), - } - } - - /// Typing presses a key per character, so long text gets more time. - fn typing_limit(&self, text: &str) -> Duration { - self.timeouts.command + Duration::from_millis(20) * text.chars().count() as u32 - } - - /// Accept (`true`) or dismiss (`false`, the default) `confirm` and `prompt` - /// dialogs from now on. `alert` and `beforeunload` are always accepted: an - /// alert has nothing to decide, and a `beforeunload` prompt only appears - /// when leaving the page was already requested. - pub fn set_accept_dialogs(&self, accept: bool) { - self.accept_dialogs.store(accept, Ordering::Relaxed); - } - pub fn label(&self) -> &str { &self.label } @@ -327,728 +98,192 @@ impl BrowserSession { self.ephemeral } - /// Navigate to a URL and wait for its `load` event. (`goto` already waits; - /// a further `wait_for_navigation` has no timeout and could hang on a page - /// that starts a script redirect right after loading.) - pub async fn navigate(&self, url: &str) -> Result<()> { - self.bounded("navigation", self.timeouts.navigation, async { - self.page.goto(url).await?; - Ok(()) - }) - .await + /// Open a new blank tab. `foreground` makes it the tab that calls without + /// a tab id target. + pub async fn create_tab(&self, foreground: bool) -> Result> { + let page = self + .launched + .lock() + .await + .browser + .new_page("about:blank") + .await?; + let id = self.tabs.lock().unwrap().mint_id(); + let tab = Arc::new(Tab::new(id.clone(), page, self.timeouts.clone()).await?); + let mut tabs = self.tabs.lock().unwrap(); + tabs.list.push(tab.clone()); + if foreground || tabs.active.is_none() { + tabs.active = Some(id); + } + Ok(tab) } - /// Capture a PNG screenshot — the model's eyes. `full_page` captures the - /// entire scrollable page instead of just the current viewport. - pub async fn screenshot(&self, full_page: bool) -> Result> { - self.bounded("screenshot", self.timeouts.command, async { - let params = ScreenshotParams::builder() - .format(CaptureScreenshotFormat::Png) - .full_page(full_page) - .build(); - Ok(self.page.screenshot(params).await?) - }) - .await + /// Adopt tabs the page opened itself (`target=_blank`, `window.open`) and + /// forget tabs that were closed. Returns the ids of newly adopted tabs. + pub async fn sync_tabs(&self) -> Result> { + let pages = self.launched.lock().await.browser.pages().await?; + let known: Vec<_> = { + let tabs = self.tabs.lock().unwrap(); + tabs.list + .iter() + .map(|t| t.page().target_id().clone()) + .collect() + }; + let mut adopted = Vec::new(); + for page in pages.iter().filter(|p| !known.contains(p.target_id())) { + let id = self.tabs.lock().unwrap().mint_id(); + let tab = Arc::new(Tab::new(id.clone(), page.clone(), self.timeouts.clone()).await?); + self.tabs.lock().unwrap().list.push(tab); + adopted.push(id); + } + let open: Vec<_> = pages.iter().map(|p| p.target_id().clone()).collect(); + let mut tabs = self.tabs.lock().unwrap(); + tabs.list.retain(|t| open.contains(t.page().target_id())); + let active_gone = tabs + .active + .as_deref() + .is_some_and(|id| tabs.get(id).is_none()); + if active_gone { + tabs.active = tabs.list.last().map(|t| t.id().to_string()); + } + Ok(adopted) + } + + /// The tab `id`, or the active tab when `id` is `None`. + pub fn tab(&self, id: Option<&str>) -> Result> { + let tabs = self.tabs.lock().unwrap(); + let id = match id { + Some(id) => id, + None => tabs + .active + .as_deref() + .ok_or_else(|| anyhow::anyhow!("this browser has no open tab"))?, + }; + tabs.get(id) + .ok_or_else(|| anyhow::anyhow!("no tab '{id}' (list them with browser_tabs_context)")) } - /// Scroll the page or an element. With a `selector` and no delta, scroll - /// that element into view. With a `selector` **and** a non-zero `(dx, dy)`, - /// scroll *inside* that element — the fix for modal/dialog content that - /// lives in its own scroll container (a plain `window.scrollBy` moves the - /// page behind it, not the dialog). With no selector, scroll the page by - /// `(dx, dy)` (positive `dy` scrolls down). The selector is JSON-encoded - /// into the script, so it cannot break out of the string. - pub async fn scroll(&self, selector: Option<&str>, dx: f64, dy: f64) -> Result<()> { - self.bounded("scroll", self.timeouts.command, async { - match selector { - Some(sel) => { - let sel_json = serde_json::to_string(sel)?; - let js = if dx == 0.0 && dy == 0.0 { - format!( - "(() => {{ const e = document.querySelector({sel_json}); \ - if (!e) return false; \ - e.scrollIntoView({{block: 'center', inline: 'center'}}); \ - return true; }})()" - ) - } else { - // Scroll within the element's own scroll container (or the - // nearest scrollable ancestor if the element itself does not - // scroll), so a dialog's inner content moves. - format!( - "(() => {{ let e = document.querySelector({sel_json}); \ - if (!e) return false; \ - const scrollable = (n) => {{ \ - while (n && n !== document.body) {{ \ - const s = getComputedStyle(n); \ - if (/(auto|scroll)/.test(s.overflowY + s.overflow) && n.scrollHeight > n.clientHeight) return n; \ - n = n.parentElement; \ - }} \ - return e; \ - }}; \ - const target = (e.scrollHeight > e.clientHeight) ? e : scrollable(e); \ - target.scrollBy({dx}, {dy}); \ - return true; }})()" - ) - }; - let found = self - .page - .evaluate(js) - .await? - .into_value::() - .unwrap_or(false); - if !found { - anyhow::bail!("no element matches selector '{sel}'"); - } - } - None => { - self.page - .evaluate(format!("window.scrollBy({dx}, {dy})")) - .await?; - } - } + /// The active tab. + pub fn active_tab(&self) -> Result> { + self.tab(None) + } + + /// Make tab `id` the one calls without a tab id target. + pub fn select_tab(&self, id: &str) -> Result<()> { + let tab = self.tab(Some(id))?; + self.tabs.lock().unwrap().active = Some(tab.id().to_string()); Ok(()) - }) - .await } - /// Read the current location, title, visible text, and the actionable - /// elements on the page. - pub async fn observe(&self) -> Result { - self.observe_with(true).await + /// Close tab `id`. Closing the active tab activates the most recent other. + pub async fn close_tab(&self, id: &str) -> Result<()> { + let tab = self.tab(Some(id))?; + { + let mut tabs = self.tabs.lock().unwrap(); + tabs.list.retain(|t| t.id() != id); + if tabs.active.as_deref() == Some(id) { + tabs.active = tabs.list.last().map(|t| t.id().to_string()); + } + } + tab.page().clone().close().await?; + Ok(()) } - /// Like [`observe`](Self::observe), but `include_text` can suppress the - /// (often large and redundant) `innerText` dump — the model keeps the - /// screenshot plus the interactive-element list, and avoids re-reading a - /// long form's text on every step. - pub async fn observe_with(&self, include_text: bool) -> Result { - self.bounded("reading the page", self.timeouts.command, async { - let url = self.page.url().await?.unwrap_or_default(); - let title = self.page.get_title().await?.unwrap_or_default(); - let text = if include_text { - self.page - .evaluate("document.body ? document.body.innerText : ''") - .await? - .into_value::() - .unwrap_or_default() - } else { - String::new() - }; - // Element discovery is best-effort: a failure (e.g. mid-navigation) - // just yields an empty list rather than failing the observation. - let elements = match self.page.evaluate(DISCOVER_ELEMENTS_JS).await { - Ok(v) => v - .into_value::>() - .unwrap_or_default(), - Err(_) => Vec::new(), - }; - let (viewport_width, viewport_height) = - self.viewport_size().await.unwrap_or((0.0, 0.0)); - let dialogs = std::mem::take(&mut *self.dialogs.lock().unwrap()); - Ok(PageObservation { + /// The open tabs with their location. + pub async fn tabs(&self) -> Vec { + let (list, active) = { + let tabs = self.tabs.lock().unwrap(); + (tabs.list.clone(), tabs.active.clone()) + }; + let mut out = Vec::new(); + for tab in list { + let (url, title) = tab.location().await; + out.push(TabInfo { + active: active.as_deref() == Some(tab.id()), + id: tab.id().to_string(), url, title, - text, - elements, - viewport_width, - viewport_height, - dialogs, - }) - }) - .await + }); + } + out } - /// The viewport size in CSS pixels — the reference frame for coordinate - /// clicks (CDP mouse events use CSS pixels, so a coordinate resolved against - /// this lands where intended regardless of screenshot scaling or DPR). - pub async fn viewport_size(&self) -> Result<(f64, f64)> { - self.bounded("reading the viewport size", self.timeouts.command, async { - let dims = self - .page - .evaluate("[window.innerWidth, window.innerHeight]") - .await? - .into_value::<(f64, f64)>() - .unwrap_or((0.0, 0.0)); - Ok(dims) - }) - .await + /// Export the whole cookie jar (shared by all tabs). + pub async fn export_cookies(&self) -> Result> { + self.active_tab()?.export_cookies().await } - /// Find an element by a selector, turning chromiumoxide's opaque CDP miss - /// ("Could not find node with given id") into a message that names the - /// selector. - /// - /// Besides plain CSS, this understands robust prefixes that survive a site - /// re-rendering with fresh hashed ids (a real problem on portals like - /// ELSTER): - /// - `text=Foo` — the first visible element whose trimmed text / - /// aria-label / value contains `Foo` (case-insensitive). - /// - `role=button` — the first element with that ARIA role (or, for a bare - /// tag, that tag). `role=button[name=Save]` also matches on text. - /// - `aria=Save` — the first element whose aria-label matches. - /// - /// These are resolved to a concrete node in the page, so a fragile hashed - /// `#id` is never needed. - async fn find(&self, selector: &str) -> Result { - if let Some(css) = self.resolve_semantic_selector(selector).await? { - return self - .page - .find_element(&css) - .await - .map_err(|_| anyhow::anyhow!("no element matches selector '{selector}'")); - } - self.page - .find_element(selector) - .await - .map_err(|_| anyhow::anyhow!("no element matches selector '{selector}'")) + /// Close the browser gracefully so a persistent profile flushes its cookies + /// to disk. After this the session is dead. Dropping without calling this + /// still kills the process (via `kill_on_drop`) but skips the flush. + pub async fn close(&self) { + self.launched.lock().await.close().await; } - /// If `selector` uses a semantic prefix (`text=`, `role=`, `aria=`), locate - /// the matching element in the page and stamp it with a unique data - /// attribute, returning a concrete CSS selector for it. Returns `Ok(None)` - /// for a plain CSS selector (handled directly by the caller). - async fn resolve_semantic_selector(&self, selector: &str) -> Result> { - let sel = selector.trim(); - let (kind, query) = if let Some(q) = sel.strip_prefix("text=") { - ("text", q) - } else if let Some(q) = sel.strip_prefix("role=") { - ("role", q) - } else if let Some(q) = sel.strip_prefix("aria=") { - ("aria", q) - } else { - return Ok(None); - }; - let kind_json = serde_json::to_string(kind)?; - let query_json = serde_json::to_string(query.trim())?; - // Tag the match with a unique attribute so we can hand back a stable CSS - // selector even on a page that mints fresh ids every render. - let js = format!( - r#"(() => {{ - const kind = {kind_json}; - const q = {query_json}; - const SEL = 'a,button,input,textarea,select,summary,[role],[onclick],[tabindex],label'; - const norm = (s) => (s || '').replace(/\s+/g, ' ').trim().toLowerCase(); - const visible = (el) => {{ - const r = el.getClientRects(); - if (!r.length) return false; - const st = getComputedStyle(el); - return st.visibility !== 'hidden' && st.display !== 'none'; - }}; - const labelText = (el) => norm(el.getAttribute('aria-label')) || norm(el.textContent) || norm(el.value) || norm(el.getAttribute('placeholder')) || norm(el.title); - const roleOf = (el) => (el.getAttribute('role') || el.tagName.toLowerCase()); - // role=button[name=Save] → role + optional name filter. - let wantRole = q, wantName = null; - const m = q.match(/^([^\[]+)\[name=(.+)\]$/); - if (kind === 'role' && m) {{ wantRole = m[1].trim(); wantName = norm(m[2]); }} - const needle = norm(q); - const collect = (root, acc) => {{ - let nodes = []; - try {{ nodes = Array.from(root.querySelectorAll(SEL)); }} catch (e) {{}} - for (const el of nodes) acc.push(el); - let all = []; - try {{ all = Array.from(root.querySelectorAll('*')); }} catch (e) {{}} - for (const el of all) if (el.shadowRoot) collect(el.shadowRoot, acc); - }}; - const cands = []; - collect(document, cands); - const match = cands.find((el) => {{ - if (!visible(el)) return false; - if (kind === 'text') return labelText(el).includes(needle); - if (kind === 'aria') return norm(el.getAttribute('aria-label')).includes(needle); - if (kind === 'role') {{ - if (norm(roleOf(el)) !== norm(wantRole)) return false; - return wantName ? labelText(el).includes(wantName) : true; - }} - return false; - }}); - if (!match) return null; - const token = 'ca-sel-' + Math.random().toString(36).slice(2); - match.setAttribute('data-ca-sel', token); - return '[data-ca-sel="' + token + '"]'; -}})()"# - ); - let resolved = self - .page - .evaluate(js) - .await? - .into_value::>() - .unwrap_or(None); - match resolved { - Some(css) => Ok(Some(css)), - None => Err(anyhow::anyhow!("no element matches selector '{selector}'")), - } - } + // Transitional: the verbs below act on the active tab, for callers that + // predate tabs. They go away with the selector-based tools. - /// Click the first element matching a selector. The element is scrolled - /// into view first (via `scrollIntoView`, which handles nested scroll - /// containers), so an off-screen or collapsed target no longer fails with - /// "Node is either not visible or not an HTMLElement". + pub async fn navigate(&self, url: &str) -> Result<()> { + self.active_tab()?.navigate(url).await + } + pub async fn screenshot(&self, full_page: bool) -> Result> { + self.active_tab()?.screenshot(full_page).await + } + pub async fn scroll(&self, selector: Option<&str>, dx: f64, dy: f64) -> Result<()> { + self.active_tab()?.scroll(selector, dx, dy).await + } + pub async fn observe(&self) -> Result { + self.active_tab()?.observe().await + } + pub async fn observe_with(&self, include_text: bool) -> Result { + self.active_tab()?.observe_with(include_text).await + } + pub async fn viewport_size(&self) -> Result<(f64, f64)> { + self.active_tab()?.viewport_size().await + } pub async fn click(&self, selector: &str) -> Result<()> { - self.bounded("click", self.timeouts.command, async { - let element = self.find(selector).await?; - let _ = element.scroll_into_view().await; - element.click().await?; - Ok(()) - }) - .await + self.active_tab()?.click(selector).await } - - /// Click at viewport coordinates `(x, y)`. For canvas/WebGL surfaces and - /// anything without a stable selector (games, maps, drag targets). pub async fn click_at(&self, x: f64, y: f64) -> Result<()> { - self.bounded("click", self.timeouts.command, async { - self.page.click(Point { x, y }).await?; - Ok(()) - }) - .await + self.active_tab()?.click_at(x, y).await } - - /// Move the mouse to viewport coordinates `(x, y)` without clicking — drives - /// hover states and canvas pointer-move handlers. pub async fn move_mouse(&self, x: f64, y: f64) -> Result<()> { - self.bounded("mouse move", self.timeouts.command, async { - self.page.move_mouse(Point { x, y }).await?; - Ok(()) - }) - .await + self.active_tab()?.move_mouse(x, y).await } - - /// Focus a field and type text into it, appending to any existing value. - /// The element is scrolled into view first. Never used for credentials — - /// those go through the human-in-the-loop login handoff. Prefer - /// [`fill`](Self::fill) to replace a prefilled field. pub async fn type_text(&self, selector: &str, text: &str) -> Result<()> { - self.bounded("typing", self.typing_limit(text), async { - let element = self.find(selector).await?; - let _ = element.scroll_into_view().await; - element.focus().await?; - self.type_chars(text).await - }) - .await + self.active_tab()?.type_text(selector, text).await } - - /// Clear a field, then type `text` — the replace semantics editing a - /// prefilled input needs (the old workflow required End + repeated - /// Backspace). Works for ``/`