From ef328aa7ca54154af96b173d980961062e626020 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Stephan=20A=C3=9Fmus?= Date: Sat, 3 Oct 2026 11:17:33 +0200 Subject: [PATCH 1/4] feat(web): record a tab as a contact sheet of screencast frames Tab::record collects Page.screencastFrame frames for a stretch of real time and lays evenly spread moments out as one PNG contact sheet, each cell labelled with its offset (built-in 5x7 bitmap font). Cells with no repaint since the one before repeat it and are marked. Headless Chrome only reports repaints inside its window to a screencast, and that window was the default 800x600 while the viewport is emulated at 1280x800: launch with a matching window, and grow it when a larger viewport is emulated. --- Cargo.lock | 1 + crates/web/Cargo.toml | 1 + crates/web/src/browser.rs | 16 ++- crates/web/src/lib.rs | 2 + crates/web/src/recording.rs | 257 ++++++++++++++++++++++++++++++++++++ crates/web/src/tab.rs | 115 +++++++++++++++- crates/web/src/tests.rs | 112 ++++++++++++++++ 7 files changed, 497 insertions(+), 7 deletions(-) create mode 100644 crates/web/src/recording.rs diff --git a/Cargo.lock b/Cargo.lock index 1dabb2e5..d54b3952 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -12680,6 +12680,7 @@ dependencies = [ "chromiumoxide", "futures", "htmd", + "image", "percent-encoding", "rand 0.9.4", "regex", diff --git a/crates/web/Cargo.toml b/crates/web/Cargo.toml index 4fa452ff..e0684902 100644 --- a/crates/web/Cargo.toml +++ b/crates/web/Cargo.toml @@ -10,6 +10,7 @@ base64 = "0.22" chromiumoxide = "0.9" futures = "0.3" htmd = "0.1.6" +image = "0.25" percent-encoding = "2.3" rand = "0.9" regex = "1.12" diff --git a/crates/web/src/browser.rs b/crates/web/src/browser.rs index 3c003e35..547a3ffa 100644 --- a/crates/web/src/browser.rs +++ b/crates/web/src/browser.rs @@ -113,12 +113,16 @@ impl LaunchedBrowser { .viewport(None) .window_size(DEFAULT_VIEWPORT.0, DEFAULT_VIEWPORT.1 + 100); } else { - builder = builder.viewport(Viewport { - width: DEFAULT_VIEWPORT.0, - height: DEFAULT_VIEWPORT.1, - device_scale_factor: Some(1.0), - ..Viewport::default() - }); + // The window as large as the viewport: headless Chrome's default + // 800×600 window would cut off what a screencast sees. + builder = builder + .viewport(Viewport { + width: DEFAULT_VIEWPORT.0, + height: DEFAULT_VIEWPORT.1, + device_scale_factor: Some(1.0), + ..Viewport::default() + }) + .window_size(DEFAULT_VIEWPORT.0, DEFAULT_VIEWPORT.1); } let browser_config = builder.build().map_err(|e| anyhow::anyhow!("{e}"))?; diff --git a/crates/web/src/lib.rs b/crates/web/src/lib.rs index a47addd0..6fa51e5d 100644 --- a/crates/web/src/lib.rs +++ b/crates/web/src/lib.rs @@ -4,6 +4,7 @@ mod browser_session; mod client; mod page_log; mod perplexity; +mod recording; mod tab; #[cfg(test)] mod tests; @@ -14,6 +15,7 @@ pub use browser_session::{ pub use chromiumoxide::layout::Point; pub use client::{PageMetadata, WebClient, WebPage, WebSearchResult}; pub use perplexity::{PerplexityCitation, PerplexityClient, PerplexityMessage, PerplexityResponse}; +pub use recording::{Recording, SheetFrame}; pub use tab::{ BrowserTimeout, BrowserTimeouts, Button, HandledDialog, MAX_SCREENSHOT_EDGE, Screenshot, Tab, }; diff --git a/crates/web/src/recording.rs b/crates/web/src/recording.rs new file mode 100644 index 00000000..b06b3967 --- /dev/null +++ b/crates/web/src/recording.rs @@ -0,0 +1,257 @@ +//! Recordings: frames a page painted over a stretch of real time, laid out as +//! one contact sheet so a model can judge motion from a single image. +//! +//! [`crate::Tab::record`] collects the frames (CDP screencast); this module +//! picks which frame each cell shows and draws the sheet. + +use crate::tab::MAX_SCREENSHOT_EDGE; +use anyhow::Result; +use image::{Rgba, RgbaImage, imageops}; +use std::io::Cursor; + +/// Pixels between cells and around the sheet. +const GAP: u32 = 4; +/// Height of the label strip above each cell. +const BAND: u32 = 18; +/// Glyphs are drawn at this many pixels per font pixel. +const GLYPH_SCALE: u32 = 2; +const BACKGROUND: Rgba = Rgba([40, 40, 40, 255]); +const LABEL: Rgba = Rgba([255, 255, 255, 255]); +/// Labels of cells that repeat the previous one are dimmed. +const LABEL_REPEATED: Rgba = Rgba([140, 140, 140, 255]); + +/// A recording laid out as a contact sheet, cells left to right, top to +/// bottom. +pub struct Recording { + pub png: Vec, + pub width: u32, + pub height: u32, + pub columns: u32, + pub rows: u32, + /// Size of one cell (one frame) in the sheet. + pub cell_width: u32, + pub cell_height: u32, + /// One per cell. + pub frames: Vec, + /// Frames the browser sent: it only sends one when the page repaints. + pub received: usize, +} + +/// What one cell of the sheet shows. +#[derive(Debug, Clone, Copy, PartialEq)] +pub struct SheetFrame { + /// Seconds after the start of the recording that the cell stands for. + pub at: f64, + /// The cell shows the same frame as the one before: nothing was painted + /// in between. + pub repeated: bool, +} + +impl Recording { + /// `(x, y, width, height)` of cell `i`'s frame in the sheet (below its + /// label). + pub fn cell_rect(&self, i: usize) -> (u32, u32, u32, u32) { + let (col, row) = (i as u32 % self.columns, i as u32 / self.columns); + ( + GAP + col * (self.cell_width + GAP), + GAP + row * (BAND + self.cell_height + GAP) + BAND, + self.cell_width, + self.cell_height, + ) + } +} + +/// Columns and rows for `n` cells: as square as possible, wider than tall. +pub(crate) fn grid(n: u32) -> (u32, u32) { + let columns = (n as f64).sqrt().ceil() as u32; + (columns, n.div_ceil(columns)) +} + +/// The size of one cell for a `vw`×`vh` (CSS px) viewport: `scale` if given, +/// reduced so the whole sheet stays within [`MAX_SCREENSHOT_EDGE`]. +pub(crate) fn cell_size( + vw: f64, + vh: f64, + columns: u32, + rows: u32, + scale: Option, +) -> (u32, u32) { + let edge = MAX_SCREENSHOT_EDGE as f64; + let fit_w = (edge - ((columns + 1) * GAP) as f64) / columns as f64 / vw; + let fit_h = (edge - ((rows + 1) * GAP + rows * BAND) as f64) / rows as f64 / vh; + let fit = fit_w.min(fit_h); + let scale = scale.filter(|s| *s > 0.0).map_or(fit, |s| s.min(fit)); + ( + ((vw * scale).floor() as u32).max(1), + ((vh * scale).floor() as u32).max(1), + ) +} + +/// The moments the cells stand for: `n` evenly spread from the start to the +/// end of `duration` seconds. +pub(crate) fn cell_times(n: usize, duration: f64) -> Vec { + (0..n) + .map(|i| duration * i as f64 / (n - 1).max(1) as f64) + .collect() +} + +/// For each moment in `times`, the frame on screen then: the last one painted +/// at or before it, else (before the first paint) the first. `frame_times` +/// are ascending. +pub(crate) fn pick_frames(frame_times: &[f64], times: &[f64]) -> Vec { + times + .iter() + .map(|t| { + frame_times + .iter() + .rposition(|f| *f <= *t + 1e-6) + .unwrap_or(0) + }) + .collect() +} + +/// Lay out `frames` (encoded images, decoded here) for the cells at `times` +/// as a contact sheet of `columns`×`rows` cells of `cell` size. +pub(crate) fn contact_sheet( + frames: &[(f64, Vec)], + times: &[f64], + columns: u32, + rows: u32, + cell: (u32, u32), +) -> Result { + anyhow::ensure!(!frames.is_empty(), "the page sent no frames"); + let (cw, ch) = cell; + let width = GAP + columns * (cw + GAP); + let height = GAP + rows * (BAND + ch + GAP); + let mut sheet = RgbaImage::from_pixel(width, height, BACKGROUND); + + let frame_times: Vec = frames.iter().map(|(t, _)| *t).collect(); + let picks = pick_frames(&frame_times, times); + let mut recording = Recording { + png: Vec::new(), + width, + height, + columns, + rows, + cell_width: cw, + cell_height: ch, + frames: Vec::with_capacity(times.len()), + received: frames.len(), + }; + let mut decoded: Option<(usize, RgbaImage)> = None; + for (i, (&at, &pick)) in times.iter().zip(&picks).enumerate() { + let repeated = i > 0 && picks[i - 1] == pick; + if decoded.as_ref().is_none_or(|(n, _)| *n != pick) { + let mut img = image::load_from_memory(&frames[pick].1)?.to_rgba8(); + if img.dimensions() != (cw, ch) { + img = imageops::resize(&img, cw, ch, imageops::FilterType::Triangle); + } + decoded = Some((pick, img)); + } + let (x, y, _, _) = recording.cell_rect(i); + let img = &decoded.as_ref().expect("decoded above").1; + imageops::replace(&mut sheet, img, x as i64, y as i64); + let color = if repeated { LABEL_REPEATED } else { LABEL }; + draw_text( + &mut sheet, + x + 2, + y - BAND + 2, + &format!("#{} +{at:.2}s", i + 1), + color, + ); + recording.frames.push(SheetFrame { at, repeated }); + } + + let mut png = Vec::new(); + sheet.write_to(&mut Cursor::new(&mut png), image::ImageFormat::Png)?; + recording.png = png; + Ok(recording) +} + +/// Draw `text` with its top-left at `(x, y)`; characters without a glyph +/// are skipped (left blank). +fn draw_text(img: &mut RgbaImage, x: u32, y: u32, text: &str, color: Rgba) { + let advance = (GLYPH_WIDTH + 1) * GLYPH_SCALE; + for (n, c) in text.chars().enumerate() { + let Some(rows) = glyph(c) else { continue }; + let gx = x + n as u32 * advance; + for (row, bits) in rows.iter().enumerate() { + for col in 0..GLYPH_WIDTH { + if bits & (1 << (GLYPH_WIDTH - 1 - col)) == 0 { + continue; + } + for dy in 0..GLYPH_SCALE { + for dx in 0..GLYPH_SCALE { + let (px, py) = ( + gx + col * GLYPH_SCALE + dx, + y + row as u32 * GLYPH_SCALE + dy, + ); + if px < img.width() && py < img.height() { + img.put_pixel(px, py, color); + } + } + } + } + } + } +} + +const GLYPH_WIDTH: u32 = 5; + +/// A 5×7 bitmap glyph, one byte per row, high bit left — just the +/// characters labels use. +fn glyph(c: char) -> Option<[u8; 7]> { + Some(match c { + '0' => [0x0E, 0x11, 0x13, 0x15, 0x19, 0x11, 0x0E], + '1' => [0x04, 0x0C, 0x04, 0x04, 0x04, 0x04, 0x0E], + '2' => [0x0E, 0x11, 0x01, 0x02, 0x04, 0x08, 0x1F], + '3' => [0x1F, 0x02, 0x04, 0x02, 0x01, 0x11, 0x0E], + '4' => [0x02, 0x06, 0x0A, 0x12, 0x1F, 0x02, 0x02], + '5' => [0x1F, 0x10, 0x1E, 0x01, 0x01, 0x11, 0x0E], + '6' => [0x06, 0x08, 0x10, 0x1E, 0x11, 0x11, 0x0E], + '7' => [0x1F, 0x01, 0x02, 0x04, 0x08, 0x08, 0x08], + '8' => [0x0E, 0x11, 0x11, 0x0E, 0x11, 0x11, 0x0E], + '9' => [0x0E, 0x11, 0x11, 0x0F, 0x01, 0x02, 0x0C], + '+' => [0x00, 0x04, 0x04, 0x1F, 0x04, 0x04, 0x00], + '.' => [0x00, 0x00, 0x00, 0x00, 0x00, 0x0C, 0x0C], + 's' => [0x00, 0x00, 0x0E, 0x10, 0x0E, 0x01, 0x1E], + '#' => [0x0A, 0x0A, 0x1F, 0x0A, 0x1F, 0x0A, 0x0A], + _ => return None, + }) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn grids_are_square_or_wider() { + assert_eq!(grid(9), (3, 3)); + assert_eq!(grid(4), (2, 2)); + assert_eq!(grid(6), (3, 2)); + assert_eq!(grid(2), (2, 1)); + assert_eq!(grid(16), (4, 4)); + } + + #[test] + fn cells_fit_the_sheet_into_the_edge_limit() { + let (cw, ch) = cell_size(1280.0, 800.0, 3, 3, None); + assert!(GAP + 3 * (cw + GAP) <= MAX_SCREENSHOT_EDGE); + assert!(GAP + 3 * (BAND + ch + GAP) <= MAX_SCREENSHOT_EDGE); + assert!(cw > 500, "{cw}"); + // A smaller scale is taken as asked, a larger one is capped. + assert_eq!(cell_size(1280.0, 800.0, 3, 3, Some(0.25)), (320, 200)); + assert_eq!(cell_size(1280.0, 800.0, 3, 3, Some(2.0)), (cw, ch)); + } + + #[test] + fn each_moment_shows_the_last_frame_painted_by_then() { + assert_eq!(cell_times(5, 1.0), [0.0, 0.25, 0.5, 0.75, 1.0]); + let frames = [0.01, 0.2, 0.3, 0.9]; + assert_eq!( + pick_frames(&frames, &cell_times(5, 1.0)), + [0, 1, 2, 2, 3], + "before the first paint the first frame stands in" + ); + } +} diff --git a/crates/web/src/tab.rs b/crates/web/src/tab.rs index 86980214..64005287 100644 --- a/crates/web/src/tab.rs +++ b/crates/web/src/tab.rs @@ -6,7 +6,11 @@ use crate::ax_tree::{GetFullAxTreeRaw, RefMap, RenderOptions}; use crate::page_log::PageLog; +use crate::recording::{self, Recording}; use anyhow::Result; +use chromiumoxide::cdp::browser_protocol::browser::{ + Bounds, GetWindowForTargetParams, SetWindowBoundsParams, +}; use chromiumoxide::cdp::browser_protocol::dom::ResolveNodeParams; use chromiumoxide::cdp::browser_protocol::dom::{ BackendNodeId, GetContentQuadsParams, ScrollIntoViewIfNeededParams, @@ -23,7 +27,8 @@ use chromiumoxide::cdp::browser_protocol::network::GetResponseBodyParams; use chromiumoxide::cdp::browser_protocol::network::{CookieParam, CookieSameSite, TimeSinceEpoch}; use chromiumoxide::cdp::browser_protocol::page::{ CaptureScreenshotFormat, CaptureScreenshotParams, DialogType, EventJavascriptDialogOpening, - HandleJavaScriptDialogParams, Viewport, + EventScreencastFrame, HandleJavaScriptDialogParams, ScreencastFrameAckParams, + StartScreencastFormat, StartScreencastParams, StopScreencastParams, Viewport, }; use chromiumoxide::cdp::js_protocol::runtime::{ CallArgument, CallFunctionOnParams, EvaluateParams, RemoteObject, @@ -352,6 +357,90 @@ impl Tab { .await } + /// Record the page for `duration` of real time and lay `frames` moments, + /// evenly spread from start to end, out as one contact sheet. Each cell + /// is a frame the page painted, its size `scale` × CSS pixels (default: + /// as large as fits [`MAX_SCREENSHOT_EDGE`]). The browser only sends a + /// frame when the page repaints; a cell with nothing new since the one + /// before repeats it and is marked so. Does not change the coordinate + /// frame. + pub async fn record( + &self, + duration: Duration, + frames: usize, + scale: Option, + ) -> Result { + anyhow::ensure!(frames > 0, "a recording needs at least one frame"); + let limit = self.timeouts().command * 2 + duration; + self.bounded("recording", limit, async { + let (_, _, vw, vh) = self.scroll_and_viewport().await?; + let (columns, rows) = recording::grid(frames as u32); + let (cw, ch) = recording::cell_size(vw, vh, columns, rows, scale); + let mut events = self.page.event_listener::().await?; + let start = std::time::SystemTime::now(); + let started = Instant::now(); + self.page + .execute( + StartScreencastParams::builder() + .format(StartScreencastFormat::Jpeg) + .quality(90) + .max_width(cw as i64) + .max_height(ch as i64) + .build(), + ) + .await?; + let collected = self + .collect_frames(&mut events, start, started, duration) + .await; + // Stop even when collecting failed, so frames do not keep coming. + let stopped = self.page.execute(StopScreencastParams::default()).await; + let captured = collected?; + stopped?; + let times = recording::cell_times(frames, duration.as_secs_f64()); + recording::contact_sheet(&captured, ×, columns, rows, (cw, ch)) + }) + .await + } + + /// Screencast frames until `duration` after `started`, acknowledged as + /// they come (the browser sends no more until it is), each with its time + /// in seconds since the start. + async fn collect_frames( + &self, + events: &mut (impl futures::Stream> + Unpin), + start: std::time::SystemTime, + started: Instant, + duration: Duration, + ) -> Result)>> { + use base64::Engine; + let start_epoch = start + .duration_since(std::time::UNIX_EPOCH) + .unwrap_or_default() + .as_secs_f64(); + let deadline = tokio::time::Instant::from_std(started + duration); + let mut frames = Vec::new(); + loop { + let event = match tokio::time::timeout_at(deadline, events.next()).await { + Ok(Some(event)) => event, + Ok(None) | Err(_) => break, + }; + self.page + .execute(ScreencastFrameAckParams::new(event.session_id)) + .await?; + // When the frame was painted; its arrival if the browser omits that. + let at = event + .metadata + .timestamp + .as_ref() + .map(|t| *t.inner() - start_epoch) + .unwrap_or_else(|| started.elapsed().as_secs_f64()); + let jpeg = base64::engine::general_purpose::STANDARD + .decode(AsRef::::as_ref(&event.data))?; + frames.push((at, jpeg)); + } + Ok(frames) + } + /// Scroll offset and viewport size, in CSS pixels. async fn scroll_and_viewport(&self) -> Result<(f64, f64, f64, f64)> { Ok(self @@ -689,6 +778,7 @@ impl Tab { .build() .map_err(anyhow::Error::msg)?; self.page.execute(touch).await?; + self.grow_window(width, height).await; let original = { let known = self.original_user_agent.lock().unwrap().clone(); @@ -718,6 +808,29 @@ impl Tab { .await } + /// Make the browser window at least `width`×`height`. Headless Chrome + /// only reports repaints inside its window to a screencast, so an emulated + /// viewport larger than the window would record as frozen where it + /// sticks out. Best effort: a window that cannot be resized stays as is. + async fn grow_window(&self, width: u32, height: u32) { + let Ok(window) = self.page.execute(GetWindowForTargetParams::default()).await else { + return; + }; + let bounds = &window.result.bounds; + let (w, h) = (bounds.width.unwrap_or(0), bounds.height.unwrap_or(0)); + if w >= width as i64 && h >= height as i64 { + return; + } + let grown = Bounds::builder() + .width(w.max(width as i64)) + .height(h.max(height as i64)) + .build(); + let _ = self + .page + .execute(SetWindowBoundsParams::new(window.result.window_id, grown)) + .await; + } + /// Emulate `prefers-color-scheme` (`"light"` / `"dark"`); `None` follows /// the browser again. pub async fn set_color_scheme(&self, scheme: Option<&str>) -> Result<()> { diff --git a/crates/web/src/tests.rs b/crates/web/src/tests.rs index 762ff7a3..c576cfb2 100644 --- a/crates/web/src/tests.rs +++ b/crates/web/src/tests.rs @@ -1104,3 +1104,115 @@ async fn keys_and_buttons_can_be_held() { session.close().await; } + +/// Where the red box is in cell `i` of a contact sheet: the mean x of its +/// red pixels, in cell pixels. +#[cfg(test)] +fn red_x_in_cell(sheet: &image::RgbaImage, rec: &super::Recording, i: usize) -> Option { + let (cx, cy, cw, ch) = rec.cell_rect(i); + let (mut sum, mut n) = (0u64, 0u64); + for y in cy..cy + ch { + for x in cx..cx + cw { + let p = sheet.get_pixel(x, y); + if p[0] > 200 && p[1] < 60 && p[2] < 60 { + sum += (x - cx) as u64; + n += 1; + } + } + } + (n > 0).then(|| sum as f64 / n as f64) +} + +/// A page with a red box sliding across at `top`, wrapping around. +#[cfg(test)] +fn sliding_box_url(top: u32) -> String { + data_url(&format!( + "\ +
\ + " + )) +} + +/// A recording turns the next second of the page into one contact sheet: +/// frames spread evenly over the time, each labelled with its offset, the +/// moving box in a different place in each — also where the viewport extends +/// past headless Chrome's default 800×600 window. A page that does not +/// repaint sends few frames; the sheet repeats the last one and says so. +#[tokio::test] +async fn recording_makes_a_contact_sheet_of_the_motion() { + use std::time::Duration; + + let session = BrowserSession::open(BrowserLaunchConfig::default(), "test") + .await + .unwrap(); + let tab = session.active_tab().unwrap(); + tab.navigate(&sliding_box_url(300)).await.unwrap(); + tokio::time::sleep(Duration::from_millis(300)).await; + + let rec = tab.record(Duration::from_secs(1), 9, None).await.unwrap(); + let rate = rec.received; + assert_eq!((rec.columns, rec.rows), (3, 3)); + assert_eq!(png_size(&rec.png), (rec.width, rec.height)); + assert!(rec.width <= super::MAX_SCREENSHOT_EDGE && rec.height <= super::MAX_SCREENSHOT_EDGE); + let at: Vec = rec.frames.iter().map(|f| f.at).collect(); + assert_eq!(at, [0.0, 0.125, 0.25, 0.375, 0.5, 0.625, 0.75, 0.875, 1.0]); + // About 60 frames a second. A loaded machine paints fewer and may skip + // a cell, which then repeats the one before; every fresh cell shows the + // box somewhere new. + assert!( + rec.received >= 9, + "only {} frames for a moving page", + rec.received + ); + let fresh = rec.frames[1..].iter().filter(|f| !f.repeated).count(); + assert!(fresh >= 6, "{:?}", rec.frames); + let sheet = image::load_from_memory(&rec.png).unwrap().to_rgba8(); + let xs: Vec = (0..9) + .map(|i| red_x_in_cell(&sheet, &rec, i).unwrap_or_else(|| panic!("no box in cell {i}"))) + .collect(); + for i in 1..9 { + let moved = (xs[i] - xs[i - 1]).abs() > 2.0; + assert_eq!( + moved, !rec.frames[i].repeated, + "cell {i}: {xs:?} {:?}", + rec.frames + ); + } + + // A viewport taller than the window: the box below 800 px still records + // at the full rate (outside the window, only a fraction of the frames + // came). + tab.set_viewport(1000, 1100, false).await.unwrap(); + tab.navigate(&sliding_box_url(950)).await.unwrap(); + tokio::time::sleep(Duration::from_millis(300)).await; + let rec = tab.record(Duration::from_secs(1), 4, None).await.unwrap(); + assert!( + rec.received * 3 >= rate * 2, + "{} frames, {rate} in the default viewport", + rec.received + ); + + // A still page: the sheet repeats what little the browser sent. + tab.javascript("document.getElementById('box').remove(); 0") + .await + .unwrap(); + tokio::time::sleep(Duration::from_millis(200)).await; + let rec = tab + .record(Duration::from_millis(500), 4, Some(0.25)) + .await + .unwrap(); + assert_eq!((rec.columns, rec.rows), (2, 2)); + assert_eq!(rec.cell_rect(0).2, 250, "a quarter of 1000 px"); + assert!(rec.received >= 1); + assert!( + rec.frames[1..].iter().all(|f| f.repeated), + "{:?}", + rec.frames + ); + + session.close().await; +} From 58ebe705d82dd6c6d762cead1d6d23e670fcb56f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Stephan=20A=C3=9Fmus?= Date: Sat, 3 Oct 2026 11:37:27 +0200 Subject: [PATCH 2/4] fix(web): cut recorded frames to the viewport and grow a short window A screencast frame shows the browser window, not the emulated viewport, and headless Chrome reserves 143 px of the window height for browser UI: a 1280x800 window gave 1280x657 frames, which were stretched to the cell. Frames are now cut to the viewport; the window gets 200 px on top of the viewport, and a frame showing less than the viewport grows the window by what is missing. The sheet starts from a screenshot of the page as the recording begins (a still page paints nothing), and a cell counts as unchanged when it looks like the one before rather than when no frame arrived: a repaint can leave the picture as it was. Frames are requested at about cell size, which keeps decoding cheap. --- crates/web/src/browser.rs | 10 +- crates/web/src/recording.rs | 79 +++++++++++--- crates/web/src/tab.rs | 200 ++++++++++++++++++++++++++---------- crates/web/src/tests.rs | 57 ++++++++-- 4 files changed, 268 insertions(+), 78 deletions(-) diff --git a/crates/web/src/browser.rs b/crates/web/src/browser.rs index 547a3ffa..e43818c1 100644 --- a/crates/web/src/browser.rs +++ b/crates/web/src/browser.rs @@ -113,8 +113,9 @@ impl LaunchedBrowser { .viewport(None) .window_size(DEFAULT_VIEWPORT.0, DEFAULT_VIEWPORT.1 + 100); } else { - // The window as large as the viewport: headless Chrome's default - // 800×600 window would cut off what a screencast sees. + // A window with room for the viewport: a screencast sees only + // what is inside the window, and headless Chrome's default is + // 800×600, part of it reserved for browser UI. builder = builder .viewport(Viewport { width: DEFAULT_VIEWPORT.0, @@ -122,7 +123,10 @@ impl LaunchedBrowser { device_scale_factor: Some(1.0), ..Viewport::default() }) - .window_size(DEFAULT_VIEWPORT.0, DEFAULT_VIEWPORT.1); + .window_size( + DEFAULT_VIEWPORT.0, + DEFAULT_VIEWPORT.1 + crate::tab::WINDOW_UI_ALLOWANCE, + ); } let browser_config = builder.build().map_err(|e| anyhow::anyhow!("{e}"))?; diff --git a/crates/web/src/recording.rs b/crates/web/src/recording.rs index b06b3967..3b858f1b 100644 --- a/crates/web/src/recording.rs +++ b/crates/web/src/recording.rs @@ -17,7 +17,7 @@ const BAND: u32 = 18; const GLYPH_SCALE: u32 = 2; const BACKGROUND: Rgba = Rgba([40, 40, 40, 255]); const LABEL: Rgba = Rgba([255, 255, 255, 255]); -/// Labels of cells that repeat the previous one are dimmed. +/// Labels of cells that look like the one before are dimmed. const LABEL_REPEATED: Rgba = Rgba([140, 140, 140, 255]); /// A recording laid out as a contact sheet, cells left to right, top to @@ -33,7 +33,8 @@ pub struct Recording { pub cell_height: u32, /// One per cell. pub frames: Vec, - /// Frames the browser sent: it only sends one when the page repaints. + /// Frames the page painted during the recording (the browser only sends + /// one when something changed), not counting the one it started with. pub received: usize, } @@ -42,8 +43,8 @@ pub struct Recording { pub struct SheetFrame { /// Seconds after the start of the recording that the cell stands for. pub at: f64, - /// The cell shows the same frame as the one before: nothing was painted - /// in between. + /// The cell looks the same as the one before: nothing visible changed in + /// between. pub repeated: bool, } @@ -95,6 +96,18 @@ pub(crate) fn cell_times(n: usize, duration: f64) -> Vec { .collect() } +/// One captured frame. +pub(crate) struct Frame { + /// Seconds after the start of the recording that it was painted. + pub at: f64, + /// The image, as JPEG or PNG. + pub encoded: Vec, + /// The part of the image that is the viewport, from its top left, as a + /// fraction of its width and height: a screencast frame shows the whole + /// window, which can be larger. + pub viewport: (f64, f64), +} + /// For each moment in `times`, the frame on screen then: the last one painted /// at or before it, else (before the first paint) the first. `frame_times` /// are ascending. @@ -110,22 +123,23 @@ pub(crate) fn pick_frames(frame_times: &[f64], times: &[f64]) -> Vec { .collect() } -/// Lay out `frames` (encoded images, decoded here) for the cells at `times` -/// as a contact sheet of `columns`×`rows` cells of `cell` size. +/// Lay out `frames` (encoded images, decoded here; the first is the page as +/// the recording started) for the cells at `times` as a contact sheet of +/// `columns`×`rows` cells of `cell` size. pub(crate) fn contact_sheet( - frames: &[(f64, Vec)], + frames: &[Frame], times: &[f64], columns: u32, rows: u32, cell: (u32, u32), ) -> Result { - anyhow::ensure!(!frames.is_empty(), "the page sent no frames"); + anyhow::ensure!(!frames.is_empty(), "no frames to lay out"); let (cw, ch) = cell; let width = GAP + columns * (cw + GAP); let height = GAP + rows * (BAND + ch + GAP); let mut sheet = RgbaImage::from_pixel(width, height, BACKGROUND); - let frame_times: Vec = frames.iter().map(|(t, _)| *t).collect(); + let frame_times: Vec = frames.iter().map(|f| f.at).collect(); let picks = pick_frames(&frame_times, times); let mut recording = Recording { png: Vec::new(), @@ -136,16 +150,29 @@ pub(crate) fn contact_sheet( cell_width: cw, cell_height: ch, frames: Vec::with_capacity(times.len()), - received: frames.len(), + received: frames.len() - 1, }; let mut decoded: Option<(usize, RgbaImage)> = None; for (i, (&at, &pick)) in times.iter().zip(&picks).enumerate() { - let repeated = i > 0 && picks[i - 1] == pick; + let mut repeated = i > 0; if decoded.as_ref().is_none_or(|(n, _)| *n != pick) { - let mut img = image::load_from_memory(&frames[pick].1)?.to_rgba8(); + let frame = &frames[pick]; + let mut img = image::load_from_memory(&frame.encoded)?.to_rgba8(); + let (fw, fh) = img.dimensions(); + let (rx, ry) = frame.viewport; + let (vw, vh) = ( + ((fw as f64 * rx).round() as u32).clamp(1, fw), + ((fh as f64 * ry).round() as u32).clamp(1, fh), + ); + if (vw, vh) != (fw, fh) { + img = imageops::crop_imm(&img, 0, 0, vw, vh).to_image(); + } if img.dimensions() != (cw, ch) { img = imageops::resize(&img, cw, ch, imageops::FilterType::Triangle); } + repeated &= decoded + .as_ref() + .is_some_and(|(_, before)| looks_same(before, &img)); decoded = Some((pick, img)); } let (x, y, _, _) = recording.cell_rect(i); @@ -168,6 +195,21 @@ pub(crate) fn contact_sheet( Ok(recording) } +/// Whether two frames show the same picture. A repaint can leave the page as +/// it was, and frames are JPEGs, so this tolerates compression noise: only a +/// clearly different pixel (or a few) counts as a change. +fn looks_same(a: &RgbaImage, b: &RgbaImage) -> bool { + const NOISE: u8 = 40; + const PIXELS: usize = 3; + a.dimensions() == b.dimensions() + && a.pixels() + .zip(b.pixels()) + .filter(|(p, q)| p.0.iter().zip(q.0).any(|(x, y)| x.abs_diff(y) > NOISE)) + .take(PIXELS + 1) + .count() + <= PIXELS +} + /// Draw `text` with its top-left at `(x, y)`; characters without a glyph /// are skipped (left blank). fn draw_text(img: &mut RgbaImage, x: u32, y: u32, text: &str, color: Rgba) { @@ -224,6 +266,19 @@ fn glyph(c: char) -> Option<[u8; 7]> { mod tests { use super::*; + #[test] + fn a_few_noisy_pixels_are_no_change() { + let white = RgbaImage::from_pixel(50, 40, Rgba([255, 255, 255, 255])); + let mut noisy = white.clone(); + noisy.put_pixel(3, 3, Rgba([230, 240, 255, 255])); + assert!(looks_same(&white, &noisy)); + let mut moved = white.clone(); + for y in 10..20 { + moved.put_pixel(25, y, Rgba([255, 0, 0, 255])); + } + assert!(!looks_same(&white, &moved)); + } + #[test] fn grids_are_square_or_wider() { assert_eq!(grid(9), (3, 3)); diff --git a/crates/web/src/tab.rs b/crates/web/src/tab.rs index 64005287..521f18c4 100644 --- a/crates/web/src/tab.rs +++ b/crates/web/src/tab.rs @@ -9,7 +9,7 @@ use crate::page_log::PageLog; use crate::recording::{self, Recording}; use anyhow::Result; use chromiumoxide::cdp::browser_protocol::browser::{ - Bounds, GetWindowForTargetParams, SetWindowBoundsParams, + Bounds, GetWindowForTargetParams, SetWindowBoundsParams, WindowId, }; use chromiumoxide::cdp::browser_protocol::dom::ResolveNodeParams; use chromiumoxide::cdp::browser_protocol::dom::{ @@ -323,7 +323,9 @@ impl Tab { self.bounded("screenshot", self.timeouts().command, async { let (sx, sy, vw, vh) = self.scroll_and_viewport().await?; let scale = fit_scale(scale.unwrap_or(1.0), vw, vh); - let png = self.capture(sx, sy, vw, vh, scale).await?; + let png = self + .capture(sx, sy, vw, vh, scale, CaptureScreenshotFormat::Png) + .await?; *self.frame_scale.lock().unwrap() = scale; Ok(Screenshot { png, @@ -347,7 +349,16 @@ impl Tab { let (sx, sy, _, _) = self.scroll_and_viewport().await?; let (w, h) = (x1 - x0, y1 - y0); let enlarge = (MAX_SCREENSHOT_EDGE as f64 / w.max(h)).min(4.0) * scale.unwrap_or(1.0); - let png = self.capture(sx + x0, sy + y0, w, h, enlarge).await?; + let png = self + .capture( + sx + x0, + sy + y0, + w, + h, + enlarge, + CaptureScreenshotFormat::Png, + ) + .await?; Ok(Screenshot { png, width: (w * enlarge).round() as u32, @@ -361,9 +372,8 @@ impl Tab { /// evenly spread from start to end, out as one contact sheet. Each cell /// is a frame the page painted, its size `scale` × CSS pixels (default: /// as large as fits [`MAX_SCREENSHOT_EDGE`]). The browser only sends a - /// frame when the page repaints; a cell with nothing new since the one - /// before repeats it and is marked so. Does not change the coordinate - /// frame. + /// frame when the page repaints; a cell that looks the same as the one + /// before is marked so. Does not change the coordinate frame. pub async fn record( &self, duration: Duration, @@ -373,28 +383,43 @@ impl Tab { anyhow::ensure!(frames > 0, "a recording needs at least one frame"); let limit = self.timeouts().command * 2 + duration; self.bounded("recording", limit, async { - let (_, _, vw, vh) = self.scroll_and_viewport().await?; + let (sx, sy, vw, vh) = self.scroll_and_viewport().await?; let (columns, rows) = recording::grid(frames as u32); let (cw, ch) = recording::cell_size(vw, vh, columns, rows, scale); let mut events = self.page.event_listener::().await?; - let start = std::time::SystemTime::now(); - let started = Instant::now(); - self.page - .execute( - StartScreencastParams::builder() - .format(StartScreencastFormat::Jpeg) - .quality(90) - .max_width(cw as i64) - .max_height(ch as i64) - .build(), - ) - .await?; - let collected = self - .collect_frames(&mut events, start, started, duration) - .await; - // Stop even when collecting failed, so frames do not keep coming. + // A frame shows the whole window, which can be larger than the + // viewport and is cut to it afterwards: frames sized for the + // window at the cell scale leave the viewport at least cell size. + let scale = cw as f64 / vw; + let mut screencast = StartScreencastParams::builder() + .format(StartScreencastFormat::Jpeg) + .quality(JPEG_QUALITY); + if let Some((_, w, h)) = self.window_bounds().await { + screencast = screencast + .max_width((w as f64 * scale).ceil() as i64) + .max_height((h as f64 * scale).ceil() as i64); + } + self.page.execute(screencast.build()).await?; + let captured = async { + // The page as the recording starts: the screencast only sends + // what is painted from now on, and a still page paints nothing. + // A JPEG like the screencast's, so the same picture compares + // as the same. + let first = self + .capture(sx, sy, vw, vh, scale, CaptureScreenshotFormat::Jpeg) + .await?; + let first = recording::Frame { + at: 0.0, + encoded: first, + viewport: (1.0, 1.0), + }; + let painted = self.collect_frames(&mut events, duration, (vw, vh)).await?; + Ok::<_, anyhow::Error>(std::iter::once(first).chain(painted).collect()) + } + .await; + // Stop even when capturing failed, so frames do not keep coming. let stopped = self.page.execute(StopScreencastParams::default()).await; - let captured = collected?; + let captured: Vec = captured?; stopped?; let times = recording::cell_times(frames, duration.as_secs_f64()); recording::contact_sheet(&captured, ×, columns, rows, (cw, ch)) @@ -402,23 +427,31 @@ impl Tab { .await } - /// Screencast frames until `duration` after `started`, acknowledged as - /// they come (the browser sends no more until it is), each with its time - /// in seconds since the start. + /// Screencast frames painted from now until `duration` from now, + /// acknowledged as they come (the browser sends no more until it is), + /// each with its time in seconds from now. Frames painted earlier are + /// dropped. + /// + /// A frame shows the browser window, which can be smaller than the + /// `viewport` (CSS px) when it is emulated. Then the window grows by what + /// is missing, and frames that do not show the whole viewport are + /// dropped. async fn collect_frames( &self, events: &mut (impl futures::Stream> + Unpin), - start: std::time::SystemTime, - started: Instant, duration: Duration, - ) -> Result)>> { + viewport: (f64, f64), + ) -> Result> { use base64::Engine; - let start_epoch = start + let (vw, vh) = viewport; + let started = Instant::now(); + let start_epoch = std::time::SystemTime::now() .duration_since(std::time::UNIX_EPOCH) .unwrap_or_default() .as_secs_f64(); let deadline = tokio::time::Instant::from_std(started + duration); let mut frames = Vec::new(); + let mut grown_for = None; loop { let event = match tokio::time::timeout_at(deadline, events.next()).await { Ok(Some(event)) => event, @@ -427,16 +460,32 @@ impl Tab { self.page .execute(ScreencastFrameAckParams::new(event.session_id)) .await?; + let meta = &event.metadata; // When the frame was painted; its arrival if the browser omits that. - let at = event - .metadata + let at = meta .timestamp .as_ref() .map(|t| *t.inner() - start_epoch) .unwrap_or_else(|| started.elapsed().as_secs_f64()); - let jpeg = base64::engine::general_purpose::STANDARD - .decode(AsRef::::as_ref(&event.data))?; - frames.push((at, jpeg)); + if at < 0.0 { + continue; + } + let (dw, dh) = (meta.device_width, meta.device_height); + if dw + 0.5 < vw || dh + 0.5 < vh { + // Once per size: growing takes a moment to show in frames. + if grown_for != Some((dw, dh)) { + grown_for = Some((dw, dh)); + self.grow_window_by((vw - dw).max(0.0), (vh - dh).max(0.0)) + .await; + } + continue; + } + frames.push(recording::Frame { + at, + encoded: base64::engine::general_purpose::STANDARD + .decode(AsRef::::as_ref(&event.data))?, + viewport: (vw / dw, vh / dh), + }); } Ok(frames) } @@ -450,11 +499,19 @@ impl Tab { .into_value::<(f64, f64, f64, f64)>()?) } - /// Capture the document rectangle `(x, y, w, h)` (CSS px) as PNG at `scale`. - async fn capture(&self, x: f64, y: f64, w: f64, h: f64, scale: f64) -> Result> { + /// Capture the document rectangle `(x, y, w, h)` (CSS px) at `scale`, a + /// JPEG at the quality recordings use or a PNG. + async fn capture( + &self, + x: f64, + y: f64, + w: f64, + h: f64, + scale: f64, + format: CaptureScreenshotFormat, + ) -> Result> { use base64::Engine; - let params = CaptureScreenshotParams::builder() - .format(CaptureScreenshotFormat::Png) + let mut params = CaptureScreenshotParams::builder() .clip(Viewport { x, y, @@ -463,6 +520,10 @@ impl Tab { scale, }) .build(); + if format == CaptureScreenshotFormat::Jpeg { + params.quality = Some(JPEG_QUALITY); + } + params.format = Some(format); let data = self.page.execute(params).await?.result.data; Ok(base64::engine::general_purpose::STANDARD.decode(AsRef::::as_ref(&data))?) } @@ -808,26 +869,50 @@ impl Tab { .await } - /// Make the browser window at least `width`×`height`. Headless Chrome - /// only reports repaints inside its window to a screencast, so an emulated - /// viewport larger than the window would record as frozen where it - /// sticks out. Best effort: a window that cannot be resized stays as is. + /// Make the browser window large enough for a `width`×`height` viewport. + /// Headless Chrome only shows a screencast what is inside its window, and + /// its window has room for browser UI (143 px of its height on macOS), so + /// the window gets [`WINDOW_UI_ALLOWANCE`] on top; a recording grows it + /// further if that is not enough. Best effort: a window that cannot be + /// resized stays as is. async fn grow_window(&self, width: u32, height: u32) { - let Ok(window) = self.page.execute(GetWindowForTargetParams::default()).await else { + let Some((id, w, h)) = self.window_bounds().await else { return; }; - let bounds = &window.result.bounds; - let (w, h) = (bounds.width.unwrap_or(0), bounds.height.unwrap_or(0)); - if w >= width as i64 && h >= height as i64 { - return; + let (width, height) = (width as i64, (height + WINDOW_UI_ALLOWANCE) as i64); + if w < width || h < height { + self.set_window_size(id, w.max(width), h.max(height)).await; } - let grown = Bounds::builder() - .width(w.max(width as i64)) - .height(h.max(height as i64)) - .build(); + } + + /// Make the browser window `dw` × `dh` pixels larger. + async fn grow_window_by(&self, dw: f64, dh: f64) { + if let Some((id, w, h)) = self.window_bounds().await { + let (w, h) = (w + dw.ceil() as i64, h + dh.ceil() as i64); + self.set_window_size(id, w, h).await; + } + } + + async fn window_bounds(&self) -> Option<(WindowId, i64, i64)> { + let window = self + .page + .execute(GetWindowForTargetParams::default()) + .await + .ok()? + .result; + let bounds = &window.bounds; + Some(( + window.window_id, + bounds.width.unwrap_or(0), + bounds.height.unwrap_or(0), + )) + } + + async fn set_window_size(&self, id: WindowId, width: i64, height: i64) { + let bounds = Bounds::builder().width(width).height(height).build(); let _ = self .page - .execute(SetWindowBoundsParams::new(window.result.window_id, grown)) + .execute(SetWindowBoundsParams::new(id, bounds)) .await; } @@ -1259,6 +1344,13 @@ fn truncate_chars(text: &str, max_chars: usize) -> String { /// API, past which images are downscaled again (and coordinates would drift). pub const MAX_SCREENSHOT_EDGE: u32 = 1568; +/// Room a headless browser window gets beyond its viewport for the browser +/// UI it reserves (143 px of its height on macOS). +pub(crate) const WINDOW_UI_ALLOWANCE: u32 = 200; + +/// JPEG quality of recorded frames. +const JPEG_QUALITY: i64 = 90; + /// `requested`, reduced as needed so a `vw`×`vh` capture fits the edge limit. fn fit_scale(requested: f64, vw: f64, vh: f64) -> f64 { let requested = if requested > 0.0 { requested } else { 1.0 }; diff --git a/crates/web/src/tests.rs b/crates/web/src/tests.rs index c576cfb2..1f88588d 100644 --- a/crates/web/src/tests.rs +++ b/crates/web/src/tests.rs @@ -1105,22 +1105,28 @@ async fn keys_and_buttons_can_be_held() { session.close().await; } -/// Where the red box is in cell `i` of a contact sheet: the mean x of its -/// red pixels, in cell pixels. +/// Where the red box is in cell `i` of a contact sheet and its size: the +/// mean x of its red pixels and their bounding box, in cell pixels. #[cfg(test)] -fn red_x_in_cell(sheet: &image::RgbaImage, rec: &super::Recording, i: usize) -> Option { +fn red_box_in_cell( + sheet: &image::RgbaImage, + rec: &super::Recording, + i: usize, +) -> Option<(f64, u32, u32)> { let (cx, cy, cw, ch) = rec.cell_rect(i); let (mut sum, mut n) = (0u64, 0u64); + let (mut x0, mut y0, mut x1, mut y1) = (u32::MAX, u32::MAX, 0, 0); for y in cy..cy + ch { for x in cx..cx + cw { let p = sheet.get_pixel(x, y); if p[0] > 200 && p[1] < 60 && p[2] < 60 { sum += (x - cx) as u64; n += 1; + (x0, y0, x1, y1) = (x0.min(x), y0.min(y), x1.max(x), y1.max(y)); } } } - (n > 0).then(|| sum as f64 / n as f64) + (n > 0).then(|| (sum as f64 / n as f64, x1 - x0 + 1, y1 - y0 + 1)) } /// A page with a red box sliding across at `top`, wrapping around. @@ -1171,9 +1177,14 @@ async fn recording_makes_a_contact_sheet_of_the_motion() { let fresh = rec.frames[1..].iter().filter(|f| !f.repeated).count(); assert!(fresh >= 6, "{:?}", rec.frames); let sheet = image::load_from_memory(&rec.png).unwrap().to_rgba8(); - let xs: Vec = (0..9) - .map(|i| red_x_in_cell(&sheet, &rec, i).unwrap_or_else(|| panic!("no box in cell {i}"))) + let boxes: Vec<(f64, u32, u32)> = (0..9) + .map(|i| red_box_in_cell(&sheet, &rec, i).unwrap_or_else(|| panic!("no box in cell {i}"))) .collect(); + // Square in every cell: frames are cut to the viewport, not stretched. + for (_, w, h) in &boxes { + assert!(w.abs_diff(*h) <= 2, "{boxes:?}"); + } + let xs: Vec = boxes.iter().map(|b| b.0).collect(); for i in 1..9 { let moved = (xs[i] - xs[i - 1]).abs() > 2.0; assert_eq!( @@ -1196,7 +1207,36 @@ async fn recording_makes_a_contact_sheet_of_the_motion() { rec.received ); - // A still page: the sheet repeats what little the browser sent. + // A window too small for the viewport grows as soon as a frame shows it; + // from then on the frames cover the viewport. + tab.set_viewport(1280, 800, false).await.unwrap(); + tab.navigate(&sliding_box_url(600)).await.unwrap(); + let window = tab + .page() + .execute(chromiumoxide::cdp::browser_protocol::browser::GetWindowForTargetParams::default()) + .await + .unwrap() + .result; + tab.page() + .execute( + chromiumoxide::cdp::browser_protocol::browser::SetWindowBoundsParams::new( + window.window_id, + chromiumoxide::cdp::browser_protocol::browser::Bounds::builder() + .width(1280) + .height(600) + .build(), + ), + ) + .await + .unwrap(); + tokio::time::sleep(Duration::from_millis(300)).await; + let rec = tab.record(Duration::from_secs(1), 4, None).await.unwrap(); + assert!(!rec.frames[3].repeated, "{:?}", rec.frames); + let sheet = image::load_from_memory(&rec.png).unwrap().to_rgba8(); + let (_, w, h) = red_box_in_cell(&sheet, &rec, 3).expect("the box low in the viewport"); + assert!(w.abs_diff(h) <= 2, "{w}×{h}"); + + // A still page: every cell looks like the one before. tab.javascript("document.getElementById('box').remove(); 0") .await .unwrap(); @@ -1206,8 +1246,7 @@ async fn recording_makes_a_contact_sheet_of_the_motion() { .await .unwrap(); assert_eq!((rec.columns, rec.rows), (2, 2)); - assert_eq!(rec.cell_rect(0).2, 250, "a quarter of 1000 px"); - assert!(rec.received >= 1); + assert_eq!(rec.cell_rect(0).2, 320, "a quarter of 1280 px"); assert!( rec.frames[1..].iter().all(|f| f.repeated), "{:?}", From 7ca0d9b6bd45618e5b1478b90f617de0f446743c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Stephan=20A=C3=9Fmus?= Date: Sat, 3 Oct 2026 11:37:27 +0200 Subject: [PATCH 3/4] feat(core): browser_computer record action record films `duration` seconds (max 5) and returns one contact sheet of `frames` evenly spaced frames (default 9), with the layout, timing and unchanged cells in the text. It runs in browser_batch between key_down/key_up like any other step and gets the same notes. describe.rs names it "Record 1s"; docs updated. --- .../src/tools/impls/browser/batch.rs | 33 ++++ .../src/tools/impls/browser/computer.rs | 153 +++++++++++++++++- .../src/tools/impls/browser/describe.rs | 15 ++ docs/browser-tools.md | 24 ++- 4 files changed, 216 insertions(+), 9 deletions(-) diff --git a/crates/code_assistant_core/src/tools/impls/browser/batch.rs b/crates/code_assistant_core/src/tools/impls/browser/batch.rs index 3e5a1bbc..e5fd8fff 100644 --- a/crates/code_assistant_core/src/tools/impls/browser/batch.rs +++ b/crates/code_assistant_core/src/tools/impls/browser/batch.rs @@ -209,6 +209,39 @@ mod tests { .contains("Browser error: Step 5") ); + // Input held across a recording: the page moves while it is filmed. + let mut walk = BatchInput { + actions: vec![ + step( + "browser_javascript", + json!({"text": "window.x = 0; let held = false, last = 0;\ + document.addEventListener('keydown', () => held = true);\ + document.addEventListener('keyup', () => held = false);\ + const step = (t) => { if (held) window.x += t - last; last = t; \ + requestAnimationFrame(step); };\ + requestAnimationFrame(step); 0"}), + ), + step("computer", json!({"action": "key_down", "text": "w"})), + step( + "computer", + json!({"action": "record", "duration": 0.4, "frames": 4, "scale": 0.2}), + ), + step("computer", json!({"action": "key_up", "text": "w"})), + step("javascript", json!({"text": "window.x"})), + ], + }; + let out = BrowserBatchTool.execute(&mut context, &mut walk).await?; + assert!(out.error.is_none(), "{:?}", out.error); + let text = out.text.clone(); + assert!( + text.contains("[3] computer record\n4 frames over 0.4s"), + "{text}" + ); + assert!(text.contains("Note: Still held down: w"), "{text}"); + assert_eq!(out.images.len(), 1); + let walked: f64 = text.rsplit('\n').next().unwrap().parse()?; + assert!(walked >= 350.0, "held for {walked} ms\n{text}"); + let mut bad = BatchInput { actions: vec![step("browser_login", json!({}))], }; diff --git a/crates/code_assistant_core/src/tools/impls/browser/computer.rs b/crates/code_assistant_core/src/tools/impls/browser/computer.rs index 42976758..4d5b4831 100644 --- a/crates/code_assistant_core/src/tools/impls/browser/computer.rs +++ b/crates/code_assistant_core/src/tools/impls/browser/computer.rs @@ -12,6 +12,9 @@ use web::{BrowserSessionManager, Button, Tab}; /// Pixels one wheel tick scrolls. const TICK_PX: f64 = 100.0; +/// Longest recording, in seconds. +const MAX_RECORD_SECS: f64 = 5.0; + #[derive(Deserialize, Serialize, Debug, Clone, Copy, PartialEq, Eq)] #[serde(rename_all = "snake_case")] pub enum ComputerAction { @@ -33,6 +36,7 @@ pub enum ComputerAction { KeyUp, LeftMouseDown, LeftMouseUp, + Record, } #[derive(Deserialize, Serialize, Debug, Clone, Copy)] @@ -68,6 +72,8 @@ pub struct ComputerInput { #[serde(default, skip_serializing_if = "Option::is_none")] pub repeat: Option, #[serde(default, skip_serializing_if = "Option::is_none")] + pub frames: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] pub scale: Option, #[serde(default, skip_serializing_if = "std::ops::Not::not")] pub accept_dialogs: bool, @@ -90,11 +96,13 @@ impl BrowserComputerTool { (`text`: space-separated keys or chords like \"Enter\", \"ctrl+a\", \"Backspace\"; \ `repeat`), hold_key (`text` held down for `duration` seconds), key_down / key_up \ (`text` stays down across steps until released, e.g. walk while jumping), \ - left_mouse_down / left_mouse_up (at coordinate/ref or where the mouse is), screenshot (`scale` < 1 for a smaller image), wait (`duration` seconds, \ - max 10), scroll (`scroll_direction`, `scroll_amount` ticks, at `coordinate`/`ref` or \ - the center), scroll_to (ref), hover (coordinate or ref), left_click_drag \ - (`start_coordinate` → `coordinate`), zoom (`region` [x0, y0, x1, y1] of the \ - screenshot, enlarged).\n\ + left_mouse_down / left_mouse_up (at coordinate/ref or where the mouse is), screenshot \ + (`scale` < 1 for a smaller image), wait (`duration` seconds, max 10), scroll \ + (`scroll_direction`, `scroll_amount` ticks, at `coordinate`/`ref` or the center), \ + scroll_to (ref), hover (coordinate or ref), left_click_drag (`start_coordinate` → \ + `coordinate`), zoom (`region` [x0, y0, x1, y1] of the screenshot, enlarged), record \ + (`duration` seconds, max 5, as one contact sheet of `frames` evenly spaced frames, \ + to judge motion and animations).\n\ JavaScript dialogs are answered automatically and reported: alerts acknowledged, \ confirm/prompt dismissed unless `accept_dialogs` is true.\n\ The page keeps running in real time between calls, also while you think. For \ @@ -106,7 +114,7 @@ impl BrowserComputerTool { "action": {"type": "string", "enum": [ "left_click", "right_click", "double_click", "triple_click", "type", "key", "screenshot", "wait", "scroll", "scroll_to", "hover", "left_click_drag", "zoom", - "hold_key", "key_down", "key_up", "left_mouse_down", "left_mouse_up" + "hold_key", "key_down", "key_up", "left_mouse_down", "left_mouse_up", "record" ]}, "coordinate": {"type": "array", "items": {"type": "number"}, "description": "[x, y] in the latest screenshot's pixels"}, "start_coordinate": {"type": "array", "items": {"type": "number"}, "description": "[x, y] where left_click_drag starts"}, @@ -115,10 +123,11 @@ impl BrowserComputerTool { "modifiers": {"type": "string", "description": "Modifier keys held during a click, e.g. \"ctrl\", \"cmd+shift\""}, "scroll_direction": {"type": "string", "enum": ["up", "down", "left", "right"]}, "scroll_amount": {"type": "integer", "description": "Wheel ticks (default 3)"}, - "duration": {"type": "number", "description": "Seconds to wait or hold a key (max 10)"}, + "duration": {"type": "number", "description": "Seconds to wait or hold a key (max 10), or to record (max 5)"}, "region": {"type": "array", "items": {"type": "number"}, "description": "[x0, y0, x1, y1] to zoom into"}, "repeat": {"type": "integer", "description": "Times to repeat the key sequence"}, - "scale": {"type": "number", "description": "Image scale for screenshot/zoom, e.g. 0.5"}, + "frames": {"type": "integer", "description": "Frames in a recording (2-16, default 9)"}, + "scale": {"type": "number", "description": "Image scale for screenshot/zoom/recorded frames, e.g. 0.5"}, "accept_dialogs": {"type": "boolean", "description": "Accept (OK) confirm/prompt dialogs this action raises"} }, "required": ["action"] @@ -251,6 +260,22 @@ async fn act(tab: &Tab, input: &ComputerInput) -> Result { image: Some(shot.png), }) } + Record => { + let seconds = input.duration.unwrap_or(1.0); + if !(seconds > 0.0 && seconds <= MAX_RECORD_SECS) { + return Err(anyhow!( + "record `duration` must be more than 0 and at most {MAX_RECORD_SECS} seconds" + )); + } + let frames = input.frames.unwrap_or(9).clamp(2, 16) as usize; + let rec = tab + .record(Duration::from_secs_f64(seconds), frames, input.scale) + .await?; + Ok(Acted { + text: describe_recording(&rec, seconds), + image: Some(rec.png), + }) + } Wait => { let seconds = input.duration.unwrap_or(1.0).clamp(0.0, 10.0); tokio::time::sleep(Duration::from_secs_f64(seconds)).await; @@ -353,6 +378,43 @@ async fn act(tab: &Tab, input: &ComputerInput) -> Result { } } +/// What a contact sheet shows: its layout and timing, and which cells look +/// the same as the one before. +fn describe_recording(rec: &web::Recording, seconds: f64) -> String { + let n = rec.frames.len(); + let mut text = format!( + "{n} frames over {seconds}s ({}×{}, left to right, top to bottom), one every {}s", + rec.columns, + rec.rows, + round_ms(seconds / (n - 1) as f64) + ); + let repeated: Vec = rec + .frames + .iter() + .enumerate() + .filter(|(_, f)| f.repeated) + .map(|(i, _)| format!("#{}", i + 1)) + .collect(); + if !repeated.is_empty() { + text.push_str(&format!( + "\nUnchanged from the frame before: {} (the page painted {} frame{} in {seconds}s)", + repeated.join(", "), + rec.received, + if rec.received == 1 { "" } else { "s" } + )); + } + text.push_str(&format!( + "\nContact sheet {}×{} px (click coordinates still refer to the last screenshot)", + rec.width, rec.height + )); + text +} + +/// Seconds to the millisecond, without trailing zeros. +fn round_ms(seconds: f64) -> f64 { + (seconds * 1000.0).round() / 1000.0 +} + /// Where an action lands: the center of `ref`, or `coordinate` mapped from /// the screenshot frame to CSS pixels. async fn point(tab: &Tab, input: &ComputerInput) -> Result { @@ -433,6 +495,7 @@ mod tests { duration: None, region: None, repeat: None, + frames: None, scale: None, accept_dialogs: false, target: Target::default(), @@ -443,6 +506,33 @@ mod tests { out.render(&mut ResourcesTracker::default()) } + #[test] + fn a_recording_names_its_layout_and_unchanged_frames() { + let frame = |at, repeated| web::SheetFrame { at, repeated }; + let rec = web::Recording { + png: Vec::new(), + width: 652, + height: 452, + columns: 2, + rows: 2, + cell_width: 320, + cell_height: 200, + frames: vec![ + frame(0.0, false), + frame(1.0 / 3.0, false), + frame(2.0 / 3.0, true), + frame(1.0, true), + ], + received: 1, + }; + assert_eq!( + describe_recording(&rec, 1.0), + "4 frames over 1s (2×2, left to right, top to bottom), one every 0.333s\n\ + Unchanged from the frame before: #3, #4 (the page painted 1 frame in 1s)\n\ + Contact sheet 652×452 px (click coordinates still refer to the last screenshot)" + ); + } + #[test] fn modifiers_parse_to_a_bitmask() { assert_eq!(parse_modifiers(None).unwrap(), 0); @@ -621,6 +711,53 @@ mod tests { Ok(()) } + #[tokio::test] + async fn a_recording_comes_back_as_one_contact_sheet() -> Result<()> { + let page = data_url( + "
", + ); + let mut fixture = ToolTestFixture::new().with_browser_sessions(); + let mut context = fixture.context(); + BrowserNavigateTool + .execute( + &mut context, + &mut NavigateInput { + url: page, + target: Target::default(), + }, + ) + .await?; + + let mut record = computer(ComputerAction::Record); + record.duration = Some(0.6); + record.frames = Some(4); + record.scale = Some(0.25); + let out = BrowserComputerTool + .execute(&mut context, &mut record) + .await?; + assert!(out.error.is_none(), "{:?}", out.error); + let text = render(&out); + assert!( + text.starts_with( + "4 frames over 0.6s (2×2, left to right, top to bottom), one every 0.2s" + ), + "{text}" + ); + assert!(text.contains("click coordinates still refer to"), "{text}"); + assert_eq!(out.render_images().len(), 1); + + // Too long a recording is refused rather than cut short. + let mut long = computer(ComputerAction::Record); + long.duration = Some(30.0); + let out = BrowserComputerTool.execute(&mut context, &mut long).await?; + assert!(out.error.unwrap().contains("at most 5 seconds")); + Ok(()) + } + #[tokio::test] async fn a_hung_page_fails_fast() -> Result<()> { let page = data_url( diff --git a/crates/code_assistant_core/src/tools/impls/browser/describe.rs b/crates/code_assistant_core/src/tools/impls/browser/describe.rs index 56d6c95d..e6816ca3 100644 --- a/crates/code_assistant_core/src/tools/impls/browser/describe.rs +++ b/crates/code_assistant_core/src/tools/impls/browser/describe.rs @@ -214,6 +214,10 @@ fn describe_computer(param: &dyn Fn(&str) -> Option) -> String { Some("left_mouse_up") => format!("Release the mouse{target}"), Some("screenshot") => "Screenshot".to_string(), Some("zoom") => "Zoom in".to_string(), + Some("record") => match param("duration") { + Some(s) => format!("Record {s}s"), + None => "Record 1s".to_string(), + }, Some("wait") => match param("duration") { Some(s) => format!("Wait {s}s"), None => "Wait".to_string(), @@ -291,6 +295,17 @@ mod tests { ), "Hold w for 0.5s" ); + assert_eq!( + describe( + "browser_computer", + &[("action", "record"), ("duration", "2")] + ), + "Record 2s" + ); + assert_eq!( + describe("browser_computer", &[("action", "record")]), + "Record 1s" + ); } #[test] diff --git a/docs/browser-tools.md b/docs/browser-tools.md index b12ff842..dd87b2e5 100644 --- a/docs/browser-tools.md +++ b/docs/browser-tools.md @@ -17,7 +17,7 @@ The history and the login design are in `docs/browser-agency-plan.md`. | `browser_read_page` | accessibility tree, `- role "name" [ref_N] state` | yes | | `browser_find` | tree lines matching a query (max 20) | yes | | `browser_get_page_text` | main/article text, else the body | yes | -| `browser_computer` | click/type/key/scroll/hover/drag/wait, screenshot, zoom | no | +| `browser_computer` | click/type/key/scroll/hover/drag/wait, screenshot, zoom, record | no | | `browser_form_input` | set a select, checkbox or field by ref | no | | `browser_javascript` | run JS with REPL semantics | no | | `browser_read_console_messages` | console, exceptions, browser log | yes | @@ -58,11 +58,33 @@ Together the definitions are about 12.8k characters (~3.3k tokens). Closing the last tab closes the browser. - **Console and network** are collected per tab in the background (last 500 each), so they can be read after the fact. +- **Real time.** The page keeps running between calls, also while the model + thinks. Timing-sensitive input goes in one `browser_batch`, whose steps run + back to back (`key_down w`, `record`, `key_up w`). +- **Recording.** `browser_computer` `record` films `duration` seconds (max 5) + and returns one contact sheet: `frames` cells (default 9, 3×3) evenly spread + from start to end, left to right, top to bottom, each labelled `#n +0.25s`. + The sheet starts from a screenshot of the page as the recording begins; + after that, frames come from a CDP screencast (`Page.startScreencast`), + which only sends a frame when the page repaints. A cell shows the last + frame painted by its moment. Cells that look like the one before are listed + as unchanged in the text and dimmed in the sheet. The sheet stays within the + 1568 px edge limit; `scale` shrinks its cells. It does not change the + coordinate frame. +- **Headless window.** A screencast frame shows the browser window, not the + emulated viewport, and headless Chrome reserves part of its window for + browser UI (143 px of its height on macOS; its default window is 800×600). + So a headless browser gets a window 200 px taller than its viewport, + emulating a larger viewport grows it, frames are cut to the viewport, and a + frame that still shows less than the viewport grows the window by what is + missing (that frame is dropped). ## Engine (`web` crate) - `BrowserSession` — one launched browser and its tabs. - `Tab` (`tab.rs`) — one page and its state: refs, screenshot frame, dialogs, console/network log (`page_log.rs`), and every page verb. +- `recording.rs` — contact sheets: which frame each cell shows, layout, and + a built-in 5×7 bitmap font for the labels (pure, unit-tested). - `ax_tree.rs` — the accessibility tree, read through a raw CDP command with lenient structs, and its rendering (pure, unit-tested). From 53839f76b42a50acbcffc6b3622efde67a9412c4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Stephan=20A=C3=9Fmus?= Date: Sat, 3 Oct 2026 22:53:16 +0200 Subject: [PATCH 4/4] refactor(web): fit the window with Browser.setContentsSize before recording Instead of a guessed 200 px allowance for headless Chrome's reserved browser UI and growing the window by the shortfall frames report, record sets the window contents to the viewport with Browser.setContentsSize (Chrome 140+, a clear error otherwise). The launch config and set_viewport are back as on main; frames are again requested at cell size. --- crates/web/src/browser.rs | 20 +++----- crates/web/src/tab.rs | 105 ++++++++++++-------------------------- crates/web/src/tests.rs | 5 +- docs/browser-tools.md | 13 +++-- 4 files changed, 48 insertions(+), 95 deletions(-) diff --git a/crates/web/src/browser.rs b/crates/web/src/browser.rs index e43818c1..3c003e35 100644 --- a/crates/web/src/browser.rs +++ b/crates/web/src/browser.rs @@ -113,20 +113,12 @@ impl LaunchedBrowser { .viewport(None) .window_size(DEFAULT_VIEWPORT.0, DEFAULT_VIEWPORT.1 + 100); } else { - // A window with room for the viewport: a screencast sees only - // what is inside the window, and headless Chrome's default is - // 800×600, part of it reserved for browser UI. - builder = builder - .viewport(Viewport { - width: DEFAULT_VIEWPORT.0, - height: DEFAULT_VIEWPORT.1, - device_scale_factor: Some(1.0), - ..Viewport::default() - }) - .window_size( - DEFAULT_VIEWPORT.0, - DEFAULT_VIEWPORT.1 + crate::tab::WINDOW_UI_ALLOWANCE, - ); + builder = builder.viewport(Viewport { + width: DEFAULT_VIEWPORT.0, + height: DEFAULT_VIEWPORT.1, + device_scale_factor: Some(1.0), + ..Viewport::default() + }); } let browser_config = builder.build().map_err(|e| anyhow::anyhow!("{e}"))?; diff --git a/crates/web/src/tab.rs b/crates/web/src/tab.rs index 521f18c4..a984d98e 100644 --- a/crates/web/src/tab.rs +++ b/crates/web/src/tab.rs @@ -9,7 +9,7 @@ use crate::page_log::PageLog; use crate::recording::{self, Recording}; use anyhow::Result; use chromiumoxide::cdp::browser_protocol::browser::{ - Bounds, GetWindowForTargetParams, SetWindowBoundsParams, WindowId, + GetWindowForTargetParams, SetContentsSizeParams, }; use chromiumoxide::cdp::browser_protocol::dom::ResolveNodeParams; use chromiumoxide::cdp::browser_protocol::dom::{ @@ -387,24 +387,24 @@ impl Tab { let (columns, rows) = recording::grid(frames as u32); let (cw, ch) = recording::cell_size(vw, vh, columns, rows, scale); let mut events = self.page.event_listener::().await?; - // A frame shows the whole window, which can be larger than the - // viewport and is cut to it afterwards: frames sized for the - // window at the cell scale leave the viewport at least cell size. - let scale = cw as f64 / vw; - let mut screencast = StartScreencastParams::builder() - .format(StartScreencastFormat::Jpeg) - .quality(JPEG_QUALITY); - if let Some((_, w, h)) = self.window_bounds().await { - screencast = screencast - .max_width((w as f64 * scale).ceil() as i64) - .max_height((h as f64 * scale).ceil() as i64); - } - self.page.execute(screencast.build()).await?; + // A screencast shows the window, not the emulated viewport. + self.fit_window_to(vw, vh).await?; + self.page + .execute( + StartScreencastParams::builder() + .format(StartScreencastFormat::Jpeg) + .quality(JPEG_QUALITY) + .max_width(cw as i64) + .max_height(ch as i64) + .build(), + ) + .await?; let captured = async { // The page as the recording starts: the screencast only sends // what is painted from now on, and a still page paints nothing. // A JPEG like the screencast's, so the same picture compares // as the same. + let scale = cw as f64 / vw; let first = self .capture(sx, sy, vw, vh, scale, CaptureScreenshotFormat::Jpeg) .await?; @@ -432,10 +432,9 @@ impl Tab { /// each with its time in seconds from now. Frames painted earlier are /// dropped. /// - /// A frame shows the browser window, which can be smaller than the - /// `viewport` (CSS px) when it is emulated. Then the window grows by what - /// is missing, and frames that do not show the whole viewport are - /// dropped. + /// A frame shows the browser window, fitted to the `viewport` (CSS px) + /// beforehand; frames from before it fitted, which do not show the whole + /// viewport, are dropped, and larger ones are cut to it. async fn collect_frames( &self, events: &mut (impl futures::Stream> + Unpin), @@ -451,7 +450,6 @@ impl Tab { .as_secs_f64(); let deadline = tokio::time::Instant::from_std(started + duration); let mut frames = Vec::new(); - let mut grown_for = None; loop { let event = match tokio::time::timeout_at(deadline, events.next()).await { Ok(Some(event)) => event, @@ -472,12 +470,6 @@ impl Tab { } let (dw, dh) = (meta.device_width, meta.device_height); if dw + 0.5 < vw || dh + 0.5 < vh { - // Once per size: growing takes a moment to show in frames. - if grown_for != Some((dw, dh)) { - grown_for = Some((dw, dh)); - self.grow_window_by((vw - dw).max(0.0), (vh - dh).max(0.0)) - .await; - } continue; } frames.push(recording::Frame { @@ -839,7 +831,6 @@ impl Tab { .build() .map_err(anyhow::Error::msg)?; self.page.execute(touch).await?; - self.grow_window(width, height).await; let original = { let known = self.original_user_agent.lock().unwrap().clone(); @@ -869,51 +860,27 @@ impl Tab { .await } - /// Make the browser window large enough for a `width`×`height` viewport. - /// Headless Chrome only shows a screencast what is inside its window, and - /// its window has room for browser UI (143 px of its height on macOS), so - /// the window gets [`WINDOW_UI_ALLOWANCE`] on top; a recording grows it - /// further if that is not enough. Best effort: a window that cannot be - /// resized stays as is. - async fn grow_window(&self, width: u32, height: u32) { - let Some((id, w, h)) = self.window_bounds().await else { - return; - }; - let (width, height) = (width as i64, (height + WINDOW_UI_ALLOWANCE) as i64); - if w < width || h < height { - self.set_window_size(id, w.max(width), h.max(height)).await; - } - } - - /// Make the browser window `dw` × `dh` pixels larger. - async fn grow_window_by(&self, dw: f64, dh: f64) { - if let Some((id, w, h)) = self.window_bounds().await { - let (w, h) = (w + dw.ceil() as i64, h + dh.ceil() as i64); - self.set_window_size(id, w, h).await; - } - } - - async fn window_bounds(&self) -> Option<(WindowId, i64, i64)> { + /// Size the browser window so its contents are `width`×`height` CSS px: + /// a screencast shows the window, which neither follows an emulated + /// viewport nor is the size `--window-size` asks for (headless Chrome + /// reserves part of it for browser UI). Needs `Browser.setContentsSize`, + /// Chrome 140 or newer. + async fn fit_window_to(&self, width: f64, height: f64) -> Result<()> { let window = self .page .execute(GetWindowForTargetParams::default()) - .await - .ok()? + .await? .result; - let bounds = &window.bounds; - Some(( - window.window_id, - bounds.width.unwrap_or(0), - bounds.height.unwrap_or(0), - )) - } - - async fn set_window_size(&self, id: WindowId, width: i64, height: i64) { - let bounds = Bounds::builder().width(width).height(height).build(); - let _ = self - .page - .execute(SetWindowBoundsParams::new(id, bounds)) - .await; + let size = SetContentsSizeParams::builder() + .window_id(window.window_id) + .width(width.round() as i64) + .height(height.round() as i64) + .build() + .map_err(anyhow::Error::msg)?; + self.page.execute(size).await.map_err(|e| { + anyhow::anyhow!("recording needs Chrome 140 or newer (Browser.setContentsSize): {e}") + })?; + Ok(()) } /// Emulate `prefers-color-scheme` (`"light"` / `"dark"`); `None` follows @@ -1344,10 +1311,6 @@ fn truncate_chars(text: &str, max_chars: usize) -> String { /// API, past which images are downscaled again (and coordinates would drift). pub const MAX_SCREENSHOT_EDGE: u32 = 1568; -/// Room a headless browser window gets beyond its viewport for the browser -/// UI it reserves (143 px of its height on macOS). -pub(crate) const WINDOW_UI_ALLOWANCE: u32 = 200; - /// JPEG quality of recorded frames. const JPEG_QUALITY: i64 = 90; diff --git a/crates/web/src/tests.rs b/crates/web/src/tests.rs index 1f88588d..53ec9883 100644 --- a/crates/web/src/tests.rs +++ b/crates/web/src/tests.rs @@ -1196,7 +1196,7 @@ async fn recording_makes_a_contact_sheet_of_the_motion() { // A viewport taller than the window: the box below 800 px still records // at the full rate (outside the window, only a fraction of the frames - // came). + // would come). tab.set_viewport(1000, 1100, false).await.unwrap(); tab.navigate(&sliding_box_url(950)).await.unwrap(); tokio::time::sleep(Duration::from_millis(300)).await; @@ -1207,8 +1207,7 @@ async fn recording_makes_a_contact_sheet_of_the_motion() { rec.received ); - // A window too small for the viewport grows as soon as a frame shows it; - // from then on the frames cover the viewport. + // A window too small for the viewport is fitted to it before recording. tab.set_viewport(1280, 800, false).await.unwrap(); tab.navigate(&sliding_box_url(600)).await.unwrap(); let window = tab diff --git a/docs/browser-tools.md b/docs/browser-tools.md index dd87b2e5..8c80c34b 100644 --- a/docs/browser-tools.md +++ b/docs/browser-tools.md @@ -71,13 +71,12 @@ Together the definitions are about 12.8k characters (~3.3k tokens). as unchanged in the text and dimmed in the sheet. The sheet stays within the 1568 px edge limit; `scale` shrinks its cells. It does not change the coordinate frame. -- **Headless window.** A screencast frame shows the browser window, not the - emulated viewport, and headless Chrome reserves part of its window for - browser UI (143 px of its height on macOS; its default window is 800×600). - So a headless browser gets a window 200 px taller than its viewport, - emulating a larger viewport grows it, frames are cut to the viewport, and a - frame that still shows less than the viewport grows the window by what is - missing (that frame is dropped). +- **Window size for recordings.** A screencast frame shows the browser + window, not the emulated viewport, and `--window-size` sets the outer size + (headless Chrome reserves part of it for browser UI; its default window is + 800×600). So `record` first fits the window contents to the viewport with + `Browser.setContentsSize`, which needs **Chrome 140 or newer**; frames are + still cut to the viewport, and any from before the fit are dropped. ## Engine (`web` crate)