diff --git a/Cargo.lock b/Cargo.lock index 1dabb2e5..d54b3952 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -12680,6 +12680,7 @@ dependencies = [ "chromiumoxide", "futures", "htmd", + "image", "percent-encoding", "rand 0.9.4", "regex", diff --git a/crates/code_assistant_core/src/tools/impls/browser/batch.rs b/crates/code_assistant_core/src/tools/impls/browser/batch.rs index 3e5a1bbc..e5fd8fff 100644 --- a/crates/code_assistant_core/src/tools/impls/browser/batch.rs +++ b/crates/code_assistant_core/src/tools/impls/browser/batch.rs @@ -209,6 +209,39 @@ mod tests { .contains("Browser error: Step 5") ); + // Input held across a recording: the page moves while it is filmed. + let mut walk = BatchInput { + actions: vec![ + step( + "browser_javascript", + json!({"text": "window.x = 0; let held = false, last = 0;\ + document.addEventListener('keydown', () => held = true);\ + document.addEventListener('keyup', () => held = false);\ + const step = (t) => { if (held) window.x += t - last; last = t; \ + requestAnimationFrame(step); };\ + requestAnimationFrame(step); 0"}), + ), + step("computer", json!({"action": "key_down", "text": "w"})), + step( + "computer", + json!({"action": "record", "duration": 0.4, "frames": 4, "scale": 0.2}), + ), + step("computer", json!({"action": "key_up", "text": "w"})), + step("javascript", json!({"text": "window.x"})), + ], + }; + let out = BrowserBatchTool.execute(&mut context, &mut walk).await?; + assert!(out.error.is_none(), "{:?}", out.error); + let text = out.text.clone(); + assert!( + text.contains("[3] computer record\n4 frames over 0.4s"), + "{text}" + ); + assert!(text.contains("Note: Still held down: w"), "{text}"); + assert_eq!(out.images.len(), 1); + let walked: f64 = text.rsplit('\n').next().unwrap().parse()?; + assert!(walked >= 350.0, "held for {walked} ms\n{text}"); + let mut bad = BatchInput { actions: vec![step("browser_login", json!({}))], }; diff --git a/crates/code_assistant_core/src/tools/impls/browser/computer.rs b/crates/code_assistant_core/src/tools/impls/browser/computer.rs index 42976758..4d5b4831 100644 --- a/crates/code_assistant_core/src/tools/impls/browser/computer.rs +++ b/crates/code_assistant_core/src/tools/impls/browser/computer.rs @@ -12,6 +12,9 @@ use web::{BrowserSessionManager, Button, Tab}; /// Pixels one wheel tick scrolls. const TICK_PX: f64 = 100.0; +/// Longest recording, in seconds. +const MAX_RECORD_SECS: f64 = 5.0; + #[derive(Deserialize, Serialize, Debug, Clone, Copy, PartialEq, Eq)] #[serde(rename_all = "snake_case")] pub enum ComputerAction { @@ -33,6 +36,7 @@ pub enum ComputerAction { KeyUp, LeftMouseDown, LeftMouseUp, + Record, } #[derive(Deserialize, Serialize, Debug, Clone, Copy)] @@ -68,6 +72,8 @@ pub struct ComputerInput { #[serde(default, skip_serializing_if = "Option::is_none")] pub repeat: Option, #[serde(default, skip_serializing_if = "Option::is_none")] + pub frames: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] pub scale: Option, #[serde(default, skip_serializing_if = "std::ops::Not::not")] pub accept_dialogs: bool, @@ -90,11 +96,13 @@ impl BrowserComputerTool { (`text`: space-separated keys or chords like \"Enter\", \"ctrl+a\", \"Backspace\"; \ `repeat`), hold_key (`text` held down for `duration` seconds), key_down / key_up \ (`text` stays down across steps until released, e.g. walk while jumping), \ - left_mouse_down / left_mouse_up (at coordinate/ref or where the mouse is), screenshot (`scale` < 1 for a smaller image), wait (`duration` seconds, \ - max 10), scroll (`scroll_direction`, `scroll_amount` ticks, at `coordinate`/`ref` or \ - the center), scroll_to (ref), hover (coordinate or ref), left_click_drag \ - (`start_coordinate` → `coordinate`), zoom (`region` [x0, y0, x1, y1] of the \ - screenshot, enlarged).\n\ + left_mouse_down / left_mouse_up (at coordinate/ref or where the mouse is), screenshot \ + (`scale` < 1 for a smaller image), wait (`duration` seconds, max 10), scroll \ + (`scroll_direction`, `scroll_amount` ticks, at `coordinate`/`ref` or the center), \ + scroll_to (ref), hover (coordinate or ref), left_click_drag (`start_coordinate` → \ + `coordinate`), zoom (`region` [x0, y0, x1, y1] of the screenshot, enlarged), record \ + (`duration` seconds, max 5, as one contact sheet of `frames` evenly spaced frames, \ + to judge motion and animations).\n\ JavaScript dialogs are answered automatically and reported: alerts acknowledged, \ confirm/prompt dismissed unless `accept_dialogs` is true.\n\ The page keeps running in real time between calls, also while you think. For \ @@ -106,7 +114,7 @@ impl BrowserComputerTool { "action": {"type": "string", "enum": [ "left_click", "right_click", "double_click", "triple_click", "type", "key", "screenshot", "wait", "scroll", "scroll_to", "hover", "left_click_drag", "zoom", - "hold_key", "key_down", "key_up", "left_mouse_down", "left_mouse_up" + "hold_key", "key_down", "key_up", "left_mouse_down", "left_mouse_up", "record" ]}, "coordinate": {"type": "array", "items": {"type": "number"}, "description": "[x, y] in the latest screenshot's pixels"}, "start_coordinate": {"type": "array", "items": {"type": "number"}, "description": "[x, y] where left_click_drag starts"}, @@ -115,10 +123,11 @@ impl BrowserComputerTool { "modifiers": {"type": "string", "description": "Modifier keys held during a click, e.g. \"ctrl\", \"cmd+shift\""}, "scroll_direction": {"type": "string", "enum": ["up", "down", "left", "right"]}, "scroll_amount": {"type": "integer", "description": "Wheel ticks (default 3)"}, - "duration": {"type": "number", "description": "Seconds to wait or hold a key (max 10)"}, + "duration": {"type": "number", "description": "Seconds to wait or hold a key (max 10), or to record (max 5)"}, "region": {"type": "array", "items": {"type": "number"}, "description": "[x0, y0, x1, y1] to zoom into"}, "repeat": {"type": "integer", "description": "Times to repeat the key sequence"}, - "scale": {"type": "number", "description": "Image scale for screenshot/zoom, e.g. 0.5"}, + "frames": {"type": "integer", "description": "Frames in a recording (2-16, default 9)"}, + "scale": {"type": "number", "description": "Image scale for screenshot/zoom/recorded frames, e.g. 0.5"}, "accept_dialogs": {"type": "boolean", "description": "Accept (OK) confirm/prompt dialogs this action raises"} }, "required": ["action"] @@ -251,6 +260,22 @@ async fn act(tab: &Tab, input: &ComputerInput) -> Result { image: Some(shot.png), }) } + Record => { + let seconds = input.duration.unwrap_or(1.0); + if !(seconds > 0.0 && seconds <= MAX_RECORD_SECS) { + return Err(anyhow!( + "record `duration` must be more than 0 and at most {MAX_RECORD_SECS} seconds" + )); + } + let frames = input.frames.unwrap_or(9).clamp(2, 16) as usize; + let rec = tab + .record(Duration::from_secs_f64(seconds), frames, input.scale) + .await?; + Ok(Acted { + text: describe_recording(&rec, seconds), + image: Some(rec.png), + }) + } Wait => { let seconds = input.duration.unwrap_or(1.0).clamp(0.0, 10.0); tokio::time::sleep(Duration::from_secs_f64(seconds)).await; @@ -353,6 +378,43 @@ async fn act(tab: &Tab, input: &ComputerInput) -> Result { } } +/// What a contact sheet shows: its layout and timing, and which cells look +/// the same as the one before. +fn describe_recording(rec: &web::Recording, seconds: f64) -> String { + let n = rec.frames.len(); + let mut text = format!( + "{n} frames over {seconds}s ({}×{}, left to right, top to bottom), one every {}s", + rec.columns, + rec.rows, + round_ms(seconds / (n - 1) as f64) + ); + let repeated: Vec = rec + .frames + .iter() + .enumerate() + .filter(|(_, f)| f.repeated) + .map(|(i, _)| format!("#{}", i + 1)) + .collect(); + if !repeated.is_empty() { + text.push_str(&format!( + "\nUnchanged from the frame before: {} (the page painted {} frame{} in {seconds}s)", + repeated.join(", "), + rec.received, + if rec.received == 1 { "" } else { "s" } + )); + } + text.push_str(&format!( + "\nContact sheet {}×{} px (click coordinates still refer to the last screenshot)", + rec.width, rec.height + )); + text +} + +/// Seconds to the millisecond, without trailing zeros. +fn round_ms(seconds: f64) -> f64 { + (seconds * 1000.0).round() / 1000.0 +} + /// Where an action lands: the center of `ref`, or `coordinate` mapped from /// the screenshot frame to CSS pixels. async fn point(tab: &Tab, input: &ComputerInput) -> Result { @@ -433,6 +495,7 @@ mod tests { duration: None, region: None, repeat: None, + frames: None, scale: None, accept_dialogs: false, target: Target::default(), @@ -443,6 +506,33 @@ mod tests { out.render(&mut ResourcesTracker::default()) } + #[test] + fn a_recording_names_its_layout_and_unchanged_frames() { + let frame = |at, repeated| web::SheetFrame { at, repeated }; + let rec = web::Recording { + png: Vec::new(), + width: 652, + height: 452, + columns: 2, + rows: 2, + cell_width: 320, + cell_height: 200, + frames: vec![ + frame(0.0, false), + frame(1.0 / 3.0, false), + frame(2.0 / 3.0, true), + frame(1.0, true), + ], + received: 1, + }; + assert_eq!( + describe_recording(&rec, 1.0), + "4 frames over 1s (2×2, left to right, top to bottom), one every 0.333s\n\ + Unchanged from the frame before: #3, #4 (the page painted 1 frame in 1s)\n\ + Contact sheet 652×452 px (click coordinates still refer to the last screenshot)" + ); + } + #[test] fn modifiers_parse_to_a_bitmask() { assert_eq!(parse_modifiers(None).unwrap(), 0); @@ -621,6 +711,53 @@ mod tests { Ok(()) } + #[tokio::test] + async fn a_recording_comes_back_as_one_contact_sheet() -> Result<()> { + let page = data_url( + "
", + ); + let mut fixture = ToolTestFixture::new().with_browser_sessions(); + let mut context = fixture.context(); + BrowserNavigateTool + .execute( + &mut context, + &mut NavigateInput { + url: page, + target: Target::default(), + }, + ) + .await?; + + let mut record = computer(ComputerAction::Record); + record.duration = Some(0.6); + record.frames = Some(4); + record.scale = Some(0.25); + let out = BrowserComputerTool + .execute(&mut context, &mut record) + .await?; + assert!(out.error.is_none(), "{:?}", out.error); + let text = render(&out); + assert!( + text.starts_with( + "4 frames over 0.6s (2×2, left to right, top to bottom), one every 0.2s" + ), + "{text}" + ); + assert!(text.contains("click coordinates still refer to"), "{text}"); + assert_eq!(out.render_images().len(), 1); + + // Too long a recording is refused rather than cut short. + let mut long = computer(ComputerAction::Record); + long.duration = Some(30.0); + let out = BrowserComputerTool.execute(&mut context, &mut long).await?; + assert!(out.error.unwrap().contains("at most 5 seconds")); + Ok(()) + } + #[tokio::test] async fn a_hung_page_fails_fast() -> Result<()> { let page = data_url( diff --git a/crates/code_assistant_core/src/tools/impls/browser/describe.rs b/crates/code_assistant_core/src/tools/impls/browser/describe.rs index 56d6c95d..e6816ca3 100644 --- a/crates/code_assistant_core/src/tools/impls/browser/describe.rs +++ b/crates/code_assistant_core/src/tools/impls/browser/describe.rs @@ -214,6 +214,10 @@ fn describe_computer(param: &dyn Fn(&str) -> Option) -> String { Some("left_mouse_up") => format!("Release the mouse{target}"), Some("screenshot") => "Screenshot".to_string(), Some("zoom") => "Zoom in".to_string(), + Some("record") => match param("duration") { + Some(s) => format!("Record {s}s"), + None => "Record 1s".to_string(), + }, Some("wait") => match param("duration") { Some(s) => format!("Wait {s}s"), None => "Wait".to_string(), @@ -291,6 +295,17 @@ mod tests { ), "Hold w for 0.5s" ); + assert_eq!( + describe( + "browser_computer", + &[("action", "record"), ("duration", "2")] + ), + "Record 2s" + ); + assert_eq!( + describe("browser_computer", &[("action", "record")]), + "Record 1s" + ); } #[test] diff --git a/crates/web/Cargo.toml b/crates/web/Cargo.toml index 4fa452ff..e0684902 100644 --- a/crates/web/Cargo.toml +++ b/crates/web/Cargo.toml @@ -10,6 +10,7 @@ base64 = "0.22" chromiumoxide = "0.9" futures = "0.3" htmd = "0.1.6" +image = "0.25" percent-encoding = "2.3" rand = "0.9" regex = "1.12" diff --git a/crates/web/src/lib.rs b/crates/web/src/lib.rs index a47addd0..6fa51e5d 100644 --- a/crates/web/src/lib.rs +++ b/crates/web/src/lib.rs @@ -4,6 +4,7 @@ mod browser_session; mod client; mod page_log; mod perplexity; +mod recording; mod tab; #[cfg(test)] mod tests; @@ -14,6 +15,7 @@ pub use browser_session::{ pub use chromiumoxide::layout::Point; pub use client::{PageMetadata, WebClient, WebPage, WebSearchResult}; pub use perplexity::{PerplexityCitation, PerplexityClient, PerplexityMessage, PerplexityResponse}; +pub use recording::{Recording, SheetFrame}; pub use tab::{ BrowserTimeout, BrowserTimeouts, Button, HandledDialog, MAX_SCREENSHOT_EDGE, Screenshot, Tab, }; diff --git a/crates/web/src/recording.rs b/crates/web/src/recording.rs new file mode 100644 index 00000000..3b858f1b --- /dev/null +++ b/crates/web/src/recording.rs @@ -0,0 +1,312 @@ +//! Recordings: frames a page painted over a stretch of real time, laid out as +//! one contact sheet so a model can judge motion from a single image. +//! +//! [`crate::Tab::record`] collects the frames (CDP screencast); this module +//! picks which frame each cell shows and draws the sheet. + +use crate::tab::MAX_SCREENSHOT_EDGE; +use anyhow::Result; +use image::{Rgba, RgbaImage, imageops}; +use std::io::Cursor; + +/// Pixels between cells and around the sheet. +const GAP: u32 = 4; +/// Height of the label strip above each cell. +const BAND: u32 = 18; +/// Glyphs are drawn at this many pixels per font pixel. +const GLYPH_SCALE: u32 = 2; +const BACKGROUND: Rgba = Rgba([40, 40, 40, 255]); +const LABEL: Rgba = Rgba([255, 255, 255, 255]); +/// Labels of cells that look like the one before are dimmed. +const LABEL_REPEATED: Rgba = Rgba([140, 140, 140, 255]); + +/// A recording laid out as a contact sheet, cells left to right, top to +/// bottom. +pub struct Recording { + pub png: Vec, + pub width: u32, + pub height: u32, + pub columns: u32, + pub rows: u32, + /// Size of one cell (one frame) in the sheet. + pub cell_width: u32, + pub cell_height: u32, + /// One per cell. + pub frames: Vec, + /// Frames the page painted during the recording (the browser only sends + /// one when something changed), not counting the one it started with. + pub received: usize, +} + +/// What one cell of the sheet shows. +#[derive(Debug, Clone, Copy, PartialEq)] +pub struct SheetFrame { + /// Seconds after the start of the recording that the cell stands for. + pub at: f64, + /// The cell looks the same as the one before: nothing visible changed in + /// between. + pub repeated: bool, +} + +impl Recording { + /// `(x, y, width, height)` of cell `i`'s frame in the sheet (below its + /// label). + pub fn cell_rect(&self, i: usize) -> (u32, u32, u32, u32) { + let (col, row) = (i as u32 % self.columns, i as u32 / self.columns); + ( + GAP + col * (self.cell_width + GAP), + GAP + row * (BAND + self.cell_height + GAP) + BAND, + self.cell_width, + self.cell_height, + ) + } +} + +/// Columns and rows for `n` cells: as square as possible, wider than tall. +pub(crate) fn grid(n: u32) -> (u32, u32) { + let columns = (n as f64).sqrt().ceil() as u32; + (columns, n.div_ceil(columns)) +} + +/// The size of one cell for a `vw`×`vh` (CSS px) viewport: `scale` if given, +/// reduced so the whole sheet stays within [`MAX_SCREENSHOT_EDGE`]. +pub(crate) fn cell_size( + vw: f64, + vh: f64, + columns: u32, + rows: u32, + scale: Option, +) -> (u32, u32) { + let edge = MAX_SCREENSHOT_EDGE as f64; + let fit_w = (edge - ((columns + 1) * GAP) as f64) / columns as f64 / vw; + let fit_h = (edge - ((rows + 1) * GAP + rows * BAND) as f64) / rows as f64 / vh; + let fit = fit_w.min(fit_h); + let scale = scale.filter(|s| *s > 0.0).map_or(fit, |s| s.min(fit)); + ( + ((vw * scale).floor() as u32).max(1), + ((vh * scale).floor() as u32).max(1), + ) +} + +/// The moments the cells stand for: `n` evenly spread from the start to the +/// end of `duration` seconds. +pub(crate) fn cell_times(n: usize, duration: f64) -> Vec { + (0..n) + .map(|i| duration * i as f64 / (n - 1).max(1) as f64) + .collect() +} + +/// One captured frame. +pub(crate) struct Frame { + /// Seconds after the start of the recording that it was painted. + pub at: f64, + /// The image, as JPEG or PNG. + pub encoded: Vec, + /// The part of the image that is the viewport, from its top left, as a + /// fraction of its width and height: a screencast frame shows the whole + /// window, which can be larger. + pub viewport: (f64, f64), +} + +/// For each moment in `times`, the frame on screen then: the last one painted +/// at or before it, else (before the first paint) the first. `frame_times` +/// are ascending. +pub(crate) fn pick_frames(frame_times: &[f64], times: &[f64]) -> Vec { + times + .iter() + .map(|t| { + frame_times + .iter() + .rposition(|f| *f <= *t + 1e-6) + .unwrap_or(0) + }) + .collect() +} + +/// Lay out `frames` (encoded images, decoded here; the first is the page as +/// the recording started) for the cells at `times` as a contact sheet of +/// `columns`×`rows` cells of `cell` size. +pub(crate) fn contact_sheet( + frames: &[Frame], + times: &[f64], + columns: u32, + rows: u32, + cell: (u32, u32), +) -> Result { + anyhow::ensure!(!frames.is_empty(), "no frames to lay out"); + let (cw, ch) = cell; + let width = GAP + columns * (cw + GAP); + let height = GAP + rows * (BAND + ch + GAP); + let mut sheet = RgbaImage::from_pixel(width, height, BACKGROUND); + + let frame_times: Vec = frames.iter().map(|f| f.at).collect(); + let picks = pick_frames(&frame_times, times); + let mut recording = Recording { + png: Vec::new(), + width, + height, + columns, + rows, + cell_width: cw, + cell_height: ch, + frames: Vec::with_capacity(times.len()), + received: frames.len() - 1, + }; + let mut decoded: Option<(usize, RgbaImage)> = None; + for (i, (&at, &pick)) in times.iter().zip(&picks).enumerate() { + let mut repeated = i > 0; + if decoded.as_ref().is_none_or(|(n, _)| *n != pick) { + let frame = &frames[pick]; + let mut img = image::load_from_memory(&frame.encoded)?.to_rgba8(); + let (fw, fh) = img.dimensions(); + let (rx, ry) = frame.viewport; + let (vw, vh) = ( + ((fw as f64 * rx).round() as u32).clamp(1, fw), + ((fh as f64 * ry).round() as u32).clamp(1, fh), + ); + if (vw, vh) != (fw, fh) { + img = imageops::crop_imm(&img, 0, 0, vw, vh).to_image(); + } + if img.dimensions() != (cw, ch) { + img = imageops::resize(&img, cw, ch, imageops::FilterType::Triangle); + } + repeated &= decoded + .as_ref() + .is_some_and(|(_, before)| looks_same(before, &img)); + decoded = Some((pick, img)); + } + let (x, y, _, _) = recording.cell_rect(i); + let img = &decoded.as_ref().expect("decoded above").1; + imageops::replace(&mut sheet, img, x as i64, y as i64); + let color = if repeated { LABEL_REPEATED } else { LABEL }; + draw_text( + &mut sheet, + x + 2, + y - BAND + 2, + &format!("#{} +{at:.2}s", i + 1), + color, + ); + recording.frames.push(SheetFrame { at, repeated }); + } + + let mut png = Vec::new(); + sheet.write_to(&mut Cursor::new(&mut png), image::ImageFormat::Png)?; + recording.png = png; + Ok(recording) +} + +/// Whether two frames show the same picture. A repaint can leave the page as +/// it was, and frames are JPEGs, so this tolerates compression noise: only a +/// clearly different pixel (or a few) counts as a change. +fn looks_same(a: &RgbaImage, b: &RgbaImage) -> bool { + const NOISE: u8 = 40; + const PIXELS: usize = 3; + a.dimensions() == b.dimensions() + && a.pixels() + .zip(b.pixels()) + .filter(|(p, q)| p.0.iter().zip(q.0).any(|(x, y)| x.abs_diff(y) > NOISE)) + .take(PIXELS + 1) + .count() + <= PIXELS +} + +/// Draw `text` with its top-left at `(x, y)`; characters without a glyph +/// are skipped (left blank). +fn draw_text(img: &mut RgbaImage, x: u32, y: u32, text: &str, color: Rgba) { + let advance = (GLYPH_WIDTH + 1) * GLYPH_SCALE; + for (n, c) in text.chars().enumerate() { + let Some(rows) = glyph(c) else { continue }; + let gx = x + n as u32 * advance; + for (row, bits) in rows.iter().enumerate() { + for col in 0..GLYPH_WIDTH { + if bits & (1 << (GLYPH_WIDTH - 1 - col)) == 0 { + continue; + } + for dy in 0..GLYPH_SCALE { + for dx in 0..GLYPH_SCALE { + let (px, py) = ( + gx + col * GLYPH_SCALE + dx, + y + row as u32 * GLYPH_SCALE + dy, + ); + if px < img.width() && py < img.height() { + img.put_pixel(px, py, color); + } + } + } + } + } + } +} + +const GLYPH_WIDTH: u32 = 5; + +/// A 5×7 bitmap glyph, one byte per row, high bit left — just the +/// characters labels use. +fn glyph(c: char) -> Option<[u8; 7]> { + Some(match c { + '0' => [0x0E, 0x11, 0x13, 0x15, 0x19, 0x11, 0x0E], + '1' => [0x04, 0x0C, 0x04, 0x04, 0x04, 0x04, 0x0E], + '2' => [0x0E, 0x11, 0x01, 0x02, 0x04, 0x08, 0x1F], + '3' => [0x1F, 0x02, 0x04, 0x02, 0x01, 0x11, 0x0E], + '4' => [0x02, 0x06, 0x0A, 0x12, 0x1F, 0x02, 0x02], + '5' => [0x1F, 0x10, 0x1E, 0x01, 0x01, 0x11, 0x0E], + '6' => [0x06, 0x08, 0x10, 0x1E, 0x11, 0x11, 0x0E], + '7' => [0x1F, 0x01, 0x02, 0x04, 0x08, 0x08, 0x08], + '8' => [0x0E, 0x11, 0x11, 0x0E, 0x11, 0x11, 0x0E], + '9' => [0x0E, 0x11, 0x11, 0x0F, 0x01, 0x02, 0x0C], + '+' => [0x00, 0x04, 0x04, 0x1F, 0x04, 0x04, 0x00], + '.' => [0x00, 0x00, 0x00, 0x00, 0x00, 0x0C, 0x0C], + 's' => [0x00, 0x00, 0x0E, 0x10, 0x0E, 0x01, 0x1E], + '#' => [0x0A, 0x0A, 0x1F, 0x0A, 0x1F, 0x0A, 0x0A], + _ => return None, + }) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn a_few_noisy_pixels_are_no_change() { + let white = RgbaImage::from_pixel(50, 40, Rgba([255, 255, 255, 255])); + let mut noisy = white.clone(); + noisy.put_pixel(3, 3, Rgba([230, 240, 255, 255])); + assert!(looks_same(&white, &noisy)); + let mut moved = white.clone(); + for y in 10..20 { + moved.put_pixel(25, y, Rgba([255, 0, 0, 255])); + } + assert!(!looks_same(&white, &moved)); + } + + #[test] + fn grids_are_square_or_wider() { + assert_eq!(grid(9), (3, 3)); + assert_eq!(grid(4), (2, 2)); + assert_eq!(grid(6), (3, 2)); + assert_eq!(grid(2), (2, 1)); + assert_eq!(grid(16), (4, 4)); + } + + #[test] + fn cells_fit_the_sheet_into_the_edge_limit() { + let (cw, ch) = cell_size(1280.0, 800.0, 3, 3, None); + assert!(GAP + 3 * (cw + GAP) <= MAX_SCREENSHOT_EDGE); + assert!(GAP + 3 * (BAND + ch + GAP) <= MAX_SCREENSHOT_EDGE); + assert!(cw > 500, "{cw}"); + // A smaller scale is taken as asked, a larger one is capped. + assert_eq!(cell_size(1280.0, 800.0, 3, 3, Some(0.25)), (320, 200)); + assert_eq!(cell_size(1280.0, 800.0, 3, 3, Some(2.0)), (cw, ch)); + } + + #[test] + fn each_moment_shows_the_last_frame_painted_by_then() { + assert_eq!(cell_times(5, 1.0), [0.0, 0.25, 0.5, 0.75, 1.0]); + let frames = [0.01, 0.2, 0.3, 0.9]; + assert_eq!( + pick_frames(&frames, &cell_times(5, 1.0)), + [0, 1, 2, 2, 3], + "before the first paint the first frame stands in" + ); + } +} diff --git a/crates/web/src/tab.rs b/crates/web/src/tab.rs index 86980214..a984d98e 100644 --- a/crates/web/src/tab.rs +++ b/crates/web/src/tab.rs @@ -6,7 +6,11 @@ use crate::ax_tree::{GetFullAxTreeRaw, RefMap, RenderOptions}; use crate::page_log::PageLog; +use crate::recording::{self, Recording}; use anyhow::Result; +use chromiumoxide::cdp::browser_protocol::browser::{ + GetWindowForTargetParams, SetContentsSizeParams, +}; use chromiumoxide::cdp::browser_protocol::dom::ResolveNodeParams; use chromiumoxide::cdp::browser_protocol::dom::{ BackendNodeId, GetContentQuadsParams, ScrollIntoViewIfNeededParams, @@ -23,7 +27,8 @@ use chromiumoxide::cdp::browser_protocol::network::GetResponseBodyParams; use chromiumoxide::cdp::browser_protocol::network::{CookieParam, CookieSameSite, TimeSinceEpoch}; use chromiumoxide::cdp::browser_protocol::page::{ CaptureScreenshotFormat, CaptureScreenshotParams, DialogType, EventJavascriptDialogOpening, - HandleJavaScriptDialogParams, Viewport, + EventScreencastFrame, HandleJavaScriptDialogParams, ScreencastFrameAckParams, + StartScreencastFormat, StartScreencastParams, StopScreencastParams, Viewport, }; use chromiumoxide::cdp::js_protocol::runtime::{ CallArgument, CallFunctionOnParams, EvaluateParams, RemoteObject, @@ -318,7 +323,9 @@ impl Tab { self.bounded("screenshot", self.timeouts().command, async { let (sx, sy, vw, vh) = self.scroll_and_viewport().await?; let scale = fit_scale(scale.unwrap_or(1.0), vw, vh); - let png = self.capture(sx, sy, vw, vh, scale).await?; + let png = self + .capture(sx, sy, vw, vh, scale, CaptureScreenshotFormat::Png) + .await?; *self.frame_scale.lock().unwrap() = scale; Ok(Screenshot { png, @@ -342,7 +349,16 @@ impl Tab { let (sx, sy, _, _) = self.scroll_and_viewport().await?; let (w, h) = (x1 - x0, y1 - y0); let enlarge = (MAX_SCREENSHOT_EDGE as f64 / w.max(h)).min(4.0) * scale.unwrap_or(1.0); - let png = self.capture(sx + x0, sy + y0, w, h, enlarge).await?; + let png = self + .capture( + sx + x0, + sy + y0, + w, + h, + enlarge, + CaptureScreenshotFormat::Png, + ) + .await?; Ok(Screenshot { png, width: (w * enlarge).round() as u32, @@ -352,6 +368,120 @@ impl Tab { .await } + /// Record the page for `duration` of real time and lay `frames` moments, + /// evenly spread from start to end, out as one contact sheet. Each cell + /// is a frame the page painted, its size `scale` × CSS pixels (default: + /// as large as fits [`MAX_SCREENSHOT_EDGE`]). The browser only sends a + /// frame when the page repaints; a cell that looks the same as the one + /// before is marked so. Does not change the coordinate frame. + pub async fn record( + &self, + duration: Duration, + frames: usize, + scale: Option, + ) -> Result { + anyhow::ensure!(frames > 0, "a recording needs at least one frame"); + let limit = self.timeouts().command * 2 + duration; + self.bounded("recording", limit, async { + let (sx, sy, vw, vh) = self.scroll_and_viewport().await?; + let (columns, rows) = recording::grid(frames as u32); + let (cw, ch) = recording::cell_size(vw, vh, columns, rows, scale); + let mut events = self.page.event_listener::().await?; + // A screencast shows the window, not the emulated viewport. + self.fit_window_to(vw, vh).await?; + self.page + .execute( + StartScreencastParams::builder() + .format(StartScreencastFormat::Jpeg) + .quality(JPEG_QUALITY) + .max_width(cw as i64) + .max_height(ch as i64) + .build(), + ) + .await?; + let captured = async { + // The page as the recording starts: the screencast only sends + // what is painted from now on, and a still page paints nothing. + // A JPEG like the screencast's, so the same picture compares + // as the same. + let scale = cw as f64 / vw; + let first = self + .capture(sx, sy, vw, vh, scale, CaptureScreenshotFormat::Jpeg) + .await?; + let first = recording::Frame { + at: 0.0, + encoded: first, + viewport: (1.0, 1.0), + }; + let painted = self.collect_frames(&mut events, duration, (vw, vh)).await?; + Ok::<_, anyhow::Error>(std::iter::once(first).chain(painted).collect()) + } + .await; + // Stop even when capturing failed, so frames do not keep coming. + let stopped = self.page.execute(StopScreencastParams::default()).await; + let captured: Vec = captured?; + stopped?; + let times = recording::cell_times(frames, duration.as_secs_f64()); + recording::contact_sheet(&captured, ×, columns, rows, (cw, ch)) + }) + .await + } + + /// Screencast frames painted from now until `duration` from now, + /// acknowledged as they come (the browser sends no more until it is), + /// each with its time in seconds from now. Frames painted earlier are + /// dropped. + /// + /// A frame shows the browser window, fitted to the `viewport` (CSS px) + /// beforehand; frames from before it fitted, which do not show the whole + /// viewport, are dropped, and larger ones are cut to it. + async fn collect_frames( + &self, + events: &mut (impl futures::Stream> + Unpin), + duration: Duration, + viewport: (f64, f64), + ) -> Result> { + use base64::Engine; + let (vw, vh) = viewport; + let started = Instant::now(); + let start_epoch = std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .unwrap_or_default() + .as_secs_f64(); + let deadline = tokio::time::Instant::from_std(started + duration); + let mut frames = Vec::new(); + loop { + let event = match tokio::time::timeout_at(deadline, events.next()).await { + Ok(Some(event)) => event, + Ok(None) | Err(_) => break, + }; + self.page + .execute(ScreencastFrameAckParams::new(event.session_id)) + .await?; + let meta = &event.metadata; + // When the frame was painted; its arrival if the browser omits that. + let at = meta + .timestamp + .as_ref() + .map(|t| *t.inner() - start_epoch) + .unwrap_or_else(|| started.elapsed().as_secs_f64()); + if at < 0.0 { + continue; + } + let (dw, dh) = (meta.device_width, meta.device_height); + if dw + 0.5 < vw || dh + 0.5 < vh { + continue; + } + frames.push(recording::Frame { + at, + encoded: base64::engine::general_purpose::STANDARD + .decode(AsRef::::as_ref(&event.data))?, + viewport: (vw / dw, vh / dh), + }); + } + Ok(frames) + } + /// Scroll offset and viewport size, in CSS pixels. async fn scroll_and_viewport(&self) -> Result<(f64, f64, f64, f64)> { Ok(self @@ -361,11 +491,19 @@ impl Tab { .into_value::<(f64, f64, f64, f64)>()?) } - /// Capture the document rectangle `(x, y, w, h)` (CSS px) as PNG at `scale`. - async fn capture(&self, x: f64, y: f64, w: f64, h: f64, scale: f64) -> Result> { + /// Capture the document rectangle `(x, y, w, h)` (CSS px) at `scale`, a + /// JPEG at the quality recordings use or a PNG. + async fn capture( + &self, + x: f64, + y: f64, + w: f64, + h: f64, + scale: f64, + format: CaptureScreenshotFormat, + ) -> Result> { use base64::Engine; - let params = CaptureScreenshotParams::builder() - .format(CaptureScreenshotFormat::Png) + let mut params = CaptureScreenshotParams::builder() .clip(Viewport { x, y, @@ -374,6 +512,10 @@ impl Tab { scale, }) .build(); + if format == CaptureScreenshotFormat::Jpeg { + params.quality = Some(JPEG_QUALITY); + } + params.format = Some(format); let data = self.page.execute(params).await?.result.data; Ok(base64::engine::general_purpose::STANDARD.decode(AsRef::::as_ref(&data))?) } @@ -718,6 +860,29 @@ impl Tab { .await } + /// Size the browser window so its contents are `width`×`height` CSS px: + /// a screencast shows the window, which neither follows an emulated + /// viewport nor is the size `--window-size` asks for (headless Chrome + /// reserves part of it for browser UI). Needs `Browser.setContentsSize`, + /// Chrome 140 or newer. + async fn fit_window_to(&self, width: f64, height: f64) -> Result<()> { + let window = self + .page + .execute(GetWindowForTargetParams::default()) + .await? + .result; + let size = SetContentsSizeParams::builder() + .window_id(window.window_id) + .width(width.round() as i64) + .height(height.round() as i64) + .build() + .map_err(anyhow::Error::msg)?; + self.page.execute(size).await.map_err(|e| { + anyhow::anyhow!("recording needs Chrome 140 or newer (Browser.setContentsSize): {e}") + })?; + Ok(()) + } + /// Emulate `prefers-color-scheme` (`"light"` / `"dark"`); `None` follows /// the browser again. pub async fn set_color_scheme(&self, scheme: Option<&str>) -> Result<()> { @@ -1146,6 +1311,9 @@ fn truncate_chars(text: &str, max_chars: usize) -> String { /// API, past which images are downscaled again (and coordinates would drift). pub const MAX_SCREENSHOT_EDGE: u32 = 1568; +/// JPEG quality of recorded frames. +const JPEG_QUALITY: i64 = 90; + /// `requested`, reduced as needed so a `vw`×`vh` capture fits the edge limit. fn fit_scale(requested: f64, vw: f64, vh: f64) -> f64 { let requested = if requested > 0.0 { requested } else { 1.0 }; diff --git a/crates/web/src/tests.rs b/crates/web/src/tests.rs index 762ff7a3..53ec9883 100644 --- a/crates/web/src/tests.rs +++ b/crates/web/src/tests.rs @@ -1104,3 +1104,153 @@ async fn keys_and_buttons_can_be_held() { session.close().await; } + +/// Where the red box is in cell `i` of a contact sheet and its size: the +/// mean x of its red pixels and their bounding box, in cell pixels. +#[cfg(test)] +fn red_box_in_cell( + sheet: &image::RgbaImage, + rec: &super::Recording, + i: usize, +) -> Option<(f64, u32, u32)> { + let (cx, cy, cw, ch) = rec.cell_rect(i); + let (mut sum, mut n) = (0u64, 0u64); + let (mut x0, mut y0, mut x1, mut y1) = (u32::MAX, u32::MAX, 0, 0); + for y in cy..cy + ch { + for x in cx..cx + cw { + let p = sheet.get_pixel(x, y); + if p[0] > 200 && p[1] < 60 && p[2] < 60 { + sum += (x - cx) as u64; + n += 1; + (x0, y0, x1, y1) = (x0.min(x), y0.min(y), x1.max(x), y1.max(y)); + } + } + } + (n > 0).then(|| (sum as f64 / n as f64, x1 - x0 + 1, y1 - y0 + 1)) +} + +/// A page with a red box sliding across at `top`, wrapping around. +#[cfg(test)] +fn sliding_box_url(top: u32) -> String { + data_url(&format!( + "\ +
\ + " + )) +} + +/// A recording turns the next second of the page into one contact sheet: +/// frames spread evenly over the time, each labelled with its offset, the +/// moving box in a different place in each — also where the viewport extends +/// past headless Chrome's default 800×600 window. A page that does not +/// repaint sends few frames; the sheet repeats the last one and says so. +#[tokio::test] +async fn recording_makes_a_contact_sheet_of_the_motion() { + use std::time::Duration; + + let session = BrowserSession::open(BrowserLaunchConfig::default(), "test") + .await + .unwrap(); + let tab = session.active_tab().unwrap(); + tab.navigate(&sliding_box_url(300)).await.unwrap(); + tokio::time::sleep(Duration::from_millis(300)).await; + + let rec = tab.record(Duration::from_secs(1), 9, None).await.unwrap(); + let rate = rec.received; + assert_eq!((rec.columns, rec.rows), (3, 3)); + assert_eq!(png_size(&rec.png), (rec.width, rec.height)); + assert!(rec.width <= super::MAX_SCREENSHOT_EDGE && rec.height <= super::MAX_SCREENSHOT_EDGE); + let at: Vec = rec.frames.iter().map(|f| f.at).collect(); + assert_eq!(at, [0.0, 0.125, 0.25, 0.375, 0.5, 0.625, 0.75, 0.875, 1.0]); + // About 60 frames a second. A loaded machine paints fewer and may skip + // a cell, which then repeats the one before; every fresh cell shows the + // box somewhere new. + assert!( + rec.received >= 9, + "only {} frames for a moving page", + rec.received + ); + let fresh = rec.frames[1..].iter().filter(|f| !f.repeated).count(); + assert!(fresh >= 6, "{:?}", rec.frames); + let sheet = image::load_from_memory(&rec.png).unwrap().to_rgba8(); + let boxes: Vec<(f64, u32, u32)> = (0..9) + .map(|i| red_box_in_cell(&sheet, &rec, i).unwrap_or_else(|| panic!("no box in cell {i}"))) + .collect(); + // Square in every cell: frames are cut to the viewport, not stretched. + for (_, w, h) in &boxes { + assert!(w.abs_diff(*h) <= 2, "{boxes:?}"); + } + let xs: Vec = boxes.iter().map(|b| b.0).collect(); + for i in 1..9 { + let moved = (xs[i] - xs[i - 1]).abs() > 2.0; + assert_eq!( + moved, !rec.frames[i].repeated, + "cell {i}: {xs:?} {:?}", + rec.frames + ); + } + + // A viewport taller than the window: the box below 800 px still records + // at the full rate (outside the window, only a fraction of the frames + // would come). + tab.set_viewport(1000, 1100, false).await.unwrap(); + tab.navigate(&sliding_box_url(950)).await.unwrap(); + tokio::time::sleep(Duration::from_millis(300)).await; + let rec = tab.record(Duration::from_secs(1), 4, None).await.unwrap(); + assert!( + rec.received * 3 >= rate * 2, + "{} frames, {rate} in the default viewport", + rec.received + ); + + // A window too small for the viewport is fitted to it before recording. + tab.set_viewport(1280, 800, false).await.unwrap(); + tab.navigate(&sliding_box_url(600)).await.unwrap(); + let window = tab + .page() + .execute(chromiumoxide::cdp::browser_protocol::browser::GetWindowForTargetParams::default()) + .await + .unwrap() + .result; + tab.page() + .execute( + chromiumoxide::cdp::browser_protocol::browser::SetWindowBoundsParams::new( + window.window_id, + chromiumoxide::cdp::browser_protocol::browser::Bounds::builder() + .width(1280) + .height(600) + .build(), + ), + ) + .await + .unwrap(); + tokio::time::sleep(Duration::from_millis(300)).await; + let rec = tab.record(Duration::from_secs(1), 4, None).await.unwrap(); + assert!(!rec.frames[3].repeated, "{:?}", rec.frames); + let sheet = image::load_from_memory(&rec.png).unwrap().to_rgba8(); + let (_, w, h) = red_box_in_cell(&sheet, &rec, 3).expect("the box low in the viewport"); + assert!(w.abs_diff(h) <= 2, "{w}×{h}"); + + // A still page: every cell looks like the one before. + tab.javascript("document.getElementById('box').remove(); 0") + .await + .unwrap(); + tokio::time::sleep(Duration::from_millis(200)).await; + let rec = tab + .record(Duration::from_millis(500), 4, Some(0.25)) + .await + .unwrap(); + assert_eq!((rec.columns, rec.rows), (2, 2)); + assert_eq!(rec.cell_rect(0).2, 320, "a quarter of 1280 px"); + assert!( + rec.frames[1..].iter().all(|f| f.repeated), + "{:?}", + rec.frames + ); + + session.close().await; +} diff --git a/docs/browser-tools.md b/docs/browser-tools.md index b12ff842..8c80c34b 100644 --- a/docs/browser-tools.md +++ b/docs/browser-tools.md @@ -17,7 +17,7 @@ The history and the login design are in `docs/browser-agency-plan.md`. | `browser_read_page` | accessibility tree, `- role "name" [ref_N] state` | yes | | `browser_find` | tree lines matching a query (max 20) | yes | | `browser_get_page_text` | main/article text, else the body | yes | -| `browser_computer` | click/type/key/scroll/hover/drag/wait, screenshot, zoom | no | +| `browser_computer` | click/type/key/scroll/hover/drag/wait, screenshot, zoom, record | no | | `browser_form_input` | set a select, checkbox or field by ref | no | | `browser_javascript` | run JS with REPL semantics | no | | `browser_read_console_messages` | console, exceptions, browser log | yes | @@ -58,11 +58,32 @@ Together the definitions are about 12.8k characters (~3.3k tokens). Closing the last tab closes the browser. - **Console and network** are collected per tab in the background (last 500 each), so they can be read after the fact. +- **Real time.** The page keeps running between calls, also while the model + thinks. Timing-sensitive input goes in one `browser_batch`, whose steps run + back to back (`key_down w`, `record`, `key_up w`). +- **Recording.** `browser_computer` `record` films `duration` seconds (max 5) + and returns one contact sheet: `frames` cells (default 9, 3×3) evenly spread + from start to end, left to right, top to bottom, each labelled `#n +0.25s`. + The sheet starts from a screenshot of the page as the recording begins; + after that, frames come from a CDP screencast (`Page.startScreencast`), + which only sends a frame when the page repaints. A cell shows the last + frame painted by its moment. Cells that look like the one before are listed + as unchanged in the text and dimmed in the sheet. The sheet stays within the + 1568 px edge limit; `scale` shrinks its cells. It does not change the + coordinate frame. +- **Window size for recordings.** A screencast frame shows the browser + window, not the emulated viewport, and `--window-size` sets the outer size + (headless Chrome reserves part of it for browser UI; its default window is + 800×600). So `record` first fits the window contents to the viewport with + `Browser.setContentsSize`, which needs **Chrome 140 or newer**; frames are + still cut to the viewport, and any from before the fit are dropped. ## Engine (`web` crate) - `BrowserSession` — one launched browser and its tabs. - `Tab` (`tab.rs`) — one page and its state: refs, screenshot frame, dialogs, console/network log (`page_log.rs`), and every page verb. +- `recording.rs` — contact sheets: which frame each cell shows, layout, and + a built-in 5×7 bitmap font for the labels (pure, unit-tested). - `ax_tree.rs` — the accessibility tree, read through a raw CDP command with lenient structs, and its rendering (pure, unit-tested).