diff --git a/.claude/skills/knot-api/SKILL.md b/.claude/skills/knot-api/SKILL.md index 37c5d064..fa9ebbf9 100644 --- a/.claude/skills/knot-api/SKILL.md +++ b/.claude/skills/knot-api/SKILL.md @@ -64,6 +64,8 @@ Keep the password in an env var, never in argv/history. Failed logins are thrott | Workspace export | `GET /api/workspace/export` → zip (markdown + manifest) | | Workspace import | `POST /api/workspace/import` (multipart zip upload) | | Single-doc export | `GET /api/docs/{id}/export` | +| **Export doc as Word / OpenDocument** | `POST /api/docs/{id}/export/docx` or `…/export/odt` → the file; body empty, or multipart with one PNG per diagram key (≤ 10 MiB total, ≤ 2 MiB and ≤ 100 parts). Header `X-Knot-Export-Fallbacks: diagrams=…, images=…, unknown=…` counts what was left out. Viewer+ | +| Diagrams to render for that export | `GET /api/docs/{id}/export/diagrams` → `{"limits":{…}, "diagrams":[{"key","kind":"mermaid","source"} \| {"key","kind":"board","board_id","label"}]}` | | Upload attachment | `POST /api/docs/{id}/blobs` · fetch: `GET /api/blobs/{id}` | | Workspace info | `GET /api/workspace` · templates: `GET /api/workspace/templates` | @@ -90,6 +92,11 @@ kmut -X PATCH "$KNOT/api/docs/$DOC_ID/tasks/2" \ # Full workspace backup kget /api/workspace/export > knot-backup.zip + +# Export a doc as Word (or …/export/odt). An empty body means no diagram +# pictures: Mermaid blocks and boards become "(not included)" placeholders. +kmut -X POST "$KNOT/api/docs/$DOC_ID/export/docx" -D headers.txt -o page.docx +grep -i '^x-knot-export-fallbacks' headers.txt ``` ## Gotchas @@ -99,5 +106,6 @@ kget /api/workspace/export > knot-backup.zip - `403 {"code":"auth.csrf","message":"missing or mismatched CSRF token"}` on POST/PUT/PATCH/DELETE → `X-CSRF-Token` header missing or ≠ `csrf` cookie value. - `403 {"code":"acl.editor_required"}` → the user is a workspace Viewer; writes need Editor+. - Search returns `{"results":[]}` silently for queries < 2 chars; `limit` is clamped to 20 server-side. +- Word/OpenDocument export: `503 export.busy` (with `Retry-After: 5`) when two exports already run on that pod — wait and retry; `413 export.too_large` when the multipart body passes 10 MiB — resend with smaller or no pictures. A picture part over 2 MiB, past the 100th part, or named after no diagram on the page is ignored, not an error. - Real-time editing runs over a WebSocket CRDT protocol (yrs) — not curl territory; use the markdown endpoints instead. - First-run bootstrap / break-glass admin (no HTTP): `knot-server admin create --email … --display-name …` (password via stdin) directly against the DB (`KNOT_DATABASE_URL`). diff --git a/ARCHITECTURE.md b/ARCHITECTURE.md index c797de79..30052b99 100644 --- a/ARCHITECTURE.md +++ b/ARCHITECTURE.md @@ -38,6 +38,7 @@ There is **no Redis**. Cross-pod CRDT update fan-out and ACL revocation both rid | `knot-crdt` | yrs `Engine`, `Rooms` registry, `Room` actor, `PgBus` over LISTEN/NOTIFY | | `knot-docs` | ACL evaluation engine + ACL invalidation listener | | `knot-markdown` | Markdown ↔ ProseMirror round-trip (canonical schema) | +| `knot-export` | Page → DOCX/ODT (pure; walks the Y.Doc) | | `knot-obs` | structured logging, OTLP traces, Prometheus exporter, well-known metric describes | | `knot-test-support` | `fresh_db()` against the dev-compose Postgres (NEVER testcontainers) | | `tools/schemagen` | generates `schema.rs` (server) + `schema.ts` (client) from `tools/schema.json` | diff --git a/CHANGELOG.md b/CHANGELOG.md index 56f68d91..f5835649 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -23,6 +23,9 @@ so this log can be regenerated from history (e.g. with `git-cliff`). the card hides it until the next upgrade, on every device. New users start caught up, and development builds never show it. `GET /api/whats-new`, `POST /api/whats-new/seen`. (#45) +- Word (DOCX) and OpenDocument (ODT) export from the page header's download + menu, with diagrams as pictures. `GET /api/docs/{id}/export/diagrams`, + `POST /api/docs/{id}/export/{docx|odt}`. (#46) ### Security - `ctutils` 0.4.5 fixes RUSTSEC-2026-0385: its constant-time helpers could diff --git a/Cargo.lock b/Cargo.lock index 1819356f..417ab1b7 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -446,6 +446,12 @@ version = "1.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1fd0f2584146f6f2ef48085050886acf353beff7305ebd1ae69500e27c67f64b" +[[package]] +name = "byteorder-lite" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8f1fe948ff07f4bd06c30984e69f5b4899c516a3ef74f34df92a2df2ab535495" + [[package]] name = "bytes" version = "1.12.1" @@ -1139,6 +1145,15 @@ dependencies = [ "getrandom 0.4.3", ] +[[package]] +name = "fdeflate" +version = "0.3.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1e6853b52649d4ac5c0bd02320cddc5ba956bdb407c4b75a2c6b75bf51500f8c" +dependencies = [ + "simd-adler32", +] + [[package]] name = "ff" version = "0.13.1" @@ -1190,6 +1205,7 @@ version = "1.1.9" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "843fba2746e448b37e26a819579957415c8cef339bf08564fe8b7ddbd959573c" dependencies = [ + "crc32fast", "miniz_oxide", "zlib-rs", ] @@ -1342,7 +1358,7 @@ dependencies = [ "libc", "log", "rustversion", - "windows-link 0.1.3", + "windows-link 0.2.1", "windows-result 0.4.1", ] @@ -1806,6 +1822,22 @@ dependencies = [ "icu_properties", ] +[[package]] +name = "image-webp" +version = "0.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "525e9ff3e1a4be2fbea1fdf0e98686a6d98b4d8f937e1bf7402245af1909e8c3" +dependencies = [ + "byteorder-lite", + "quick-error", +] + +[[package]] +name = "imagesize" +version = "0.15.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "65b27460c2c92b037f3f94c538ed9a3342f3fdf923606781629ccb35f82d042a" + [[package]] name = "indexmap" version = "1.9.3" @@ -2022,6 +2054,22 @@ dependencies = [ "uuid", ] +[[package]] +name = "knot-export" +version = "0.0.0" +dependencies = [ + "image-webp", + "imagesize", + "png", + "roxmltree", + "sha2 0.10.9", + "thiserror 2.0.20", + "url", + "uuid", + "yrs", + "zip", +] + [[package]] name = "knot-markdown" version = "0.0.0" @@ -2073,6 +2121,7 @@ dependencies = [ "knot-config", "knot-crdt", "knot-docs", + "knot-export", "knot-markdown", "knot-obs", "knot-storage", @@ -2901,6 +2950,19 @@ version = "0.3.34" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f6b464fbc74e149a392436b17d523f769e057cb6877f6a5c4618bc6f11800548" +[[package]] +name = "png" +version = "0.18.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "60769b8b31b2a9f263dae2776c37b1b28ae246943cf719eb6946a1db05128a61" +dependencies = [ + "bitflags 2.13.1", + "crc32fast", + "fdeflate", + "flate2", + "miniz_oxide", +] + [[package]] name = "portable-atomic" version = "1.15.0" @@ -3071,6 +3133,12 @@ dependencies = [ "winapi", ] +[[package]] +name = "quick-error" +version = "2.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a993555f31e5a609f617c12db6250dedcac1b0a85076912c436e6fc9b2c8e6a3" + [[package]] name = "quick-xml" version = "0.38.4" @@ -3424,6 +3492,12 @@ dependencies = [ "windows-sys 0.52.0", ] +[[package]] +name = "roxmltree" +version = "0.20.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6c20b6793b5c2fa6553b250154b78d6d0db37e72700ae35fad9387a46f487c97" + [[package]] name = "rsa" version = "0.9.10" @@ -4275,7 +4349,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "32497e9a4c7b38532efcdebeef879707aa9f794296a4f0244f6f69e9bc8574bd" dependencies = [ "fastrand", - "getrandom 0.3.4", + "getrandom 0.4.3", "once_cell", "rustix", "windows-sys 0.52.0", diff --git a/Cargo.toml b/Cargo.toml index 228fe313..ce4d817e 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -6,6 +6,7 @@ members = [ "crates/knot-config", "crates/knot-crdt", "crates/knot-docs", + "crates/knot-export", "crates/knot-markdown", "crates/knot-obs", "crates/knot-storage", diff --git a/crates/knot-export/Cargo.toml b/crates/knot-export/Cargo.toml new file mode 100644 index 00000000..85975435 --- /dev/null +++ b/crates/knot-export/Cargo.toml @@ -0,0 +1,20 @@ +[package] +name = "knot-export" +version = "0.0.0" +edition.workspace = true +license.workspace = true +publish = false + +[dependencies] +yrs.workspace = true +sha2.workspace = true +url.workspace = true +thiserror.workspace = true +uuid.workspace = true +zip = { version = "8", default-features = false, features = ["deflate"] } +image-webp = "0.2" +png = "0.18" +imagesize = { version = "0.15", default-features = false, features = ["png", "jpeg", "gif", "webp"] } + +[dev-dependencies] +roxmltree = "0.20" diff --git a/crates/knot-export/examples/kitchen_sink.rs b/crates/knot-export/examples/kitchen_sink.rs new file mode 100644 index 00000000..8be3108a --- /dev/null +++ b/crates/knot-export/examples/kitchen_sink.rs @@ -0,0 +1,391 @@ +//! Writes `kitchen-sink.docx` and `kitchen-sink.odt` into a directory: one +//! page with every block and inline kind and every fallback, for opening in +//! Word, LibreOffice and friends by hand. +//! +//! `cargo run -p knot-export --example kitchen_sink -- ` + +use std::path::PathBuf; + +use knot_export::assets::{Assets, ImageAsset, Meta, PicFormat, Picture}; +use knot_export::{ + Align, Block, Cell, ExportDoc, ImageSource, Inline, LinkTarget, List, ListItem, Marks, Run, + Table, write_docx, write_odt, +}; + +const PHOTO: &str = "11111111-1111-4111-8111-111111111111"; +const WIDE: &str = "22222222-2222-4222-8222-222222222222"; +const GONE: &str = "33333333-3333-4333-8333-333333333333"; +const HUGE: &str = "44444444-4444-4444-8444-444444444444"; +const DOC: &str = "55555555-5555-4555-8555-555555555555"; +const USER: &str = "66666666-6666-4666-8666-666666666666"; +const GHOST: &str = "77777777-7777-4777-8777-777777777777"; +const BOARD: &str = "88888888-8888-4888-8888-888888888888"; + +fn main() -> Result<(), Box> { + let dir: PathBuf = std::env::args_os() + .nth(1) + .ok_or("usage: kitchen_sink ")? + .into(); + std::fs::create_dir_all(&dir)?; + + let (doc, assets) = sample(); + let meta = Meta { + title: "Kitchen sink".into(), + exported_on: "23 Oct 2026".into(), + base_url: "https://knot.example".into(), + }; + for (name, out) in [ + ("kitchen-sink.docx", write_docx(&doc, &meta, &assets)?), + ("kitchen-sink.odt", write_odt(&doc, &meta, &assets)?), + ] { + let path = dir.join(name); + std::fs::write(&path, &out.bytes)?; + println!( + "{} ({} bytes, {:?})", + path.display(), + out.bytes.len(), + out.fallbacks + ); + } + Ok(()) +} + +fn run(text: &str, marks: Marks) -> Run { + Run { + text: text.into(), + marks, + } +} + +fn plain(text: &str) -> Inline { + Inline::Run(run(text, Marks::default())) +} + +fn marked(text: &str, set: impl FnOnce(&mut Marks)) -> Inline { + let mut marks = Marks::default(); + set(&mut marks); + Inline::Run(run(text, marks)) +} + +fn para(content: Vec) -> Block { + Block::Paragraph(content) +} + +fn text_para(text: &str) -> Block { + para(vec![plain(text)]) +} + +fn item(checked: Option, blocks: Vec) -> ListItem { + ListItem { checked, blocks } +} + +fn cell(col: usize, header: bool, align: Align, blocks: Vec) -> Cell { + Cell { + col, + colspan: 1, + rowspan: 1, + header, + align, + blocks, + } +} + +/// A solid-colour PNG with a lighter diagonal, so scaling shows. +fn png(width: u32, height: u32, rgb: [u8; 3]) -> Vec { + let mut data = Vec::with_capacity((width * height * 3) as usize); + for y in 0..height { + for x in 0..width { + let on_diagonal = x * height / width.max(1) == y; + let px = if on_diagonal { [255, 255, 255] } else { rgb }; + data.extend_from_slice(&px); + } + } + let mut out = Vec::new(); + let mut enc = png::Encoder::new(&mut out, width, height); + enc.set_color(png::ColorType::Rgb); + enc.set_depth(png::BitDepth::Eight); + let mut writer = enc.write_header().expect("png header"); + writer.write_image_data(&data).expect("png data"); + writer.finish().expect("png end"); + out +} + +fn picture(width_px: u32, height_px: u32, scale: u32, rgb: [u8; 3]) -> Picture { + Picture { + format: PicFormat::Png, + bytes: png(width_px, height_px, rgb), + width_px, + height_px, + scale, + } +} + +fn sample() -> (ExportDoc, Assets) { + let mut assets = Assets::default(); + assets.images.insert( + PHOTO.into(), + ImageAsset::Embedded(picture(320, 180, 1, [40, 110, 180])), + ); + // Wider than the page, so it is shrunk to fit. + assets.images.insert( + WIDE.into(), + ImageAsset::Embedded(picture(1600, 400, 1, [180, 90, 40])), + ); + assets.images.insert(GONE.into(), ImageAsset::Unavailable); + assets.images.insert(HUGE.into(), ImageAsset::OverBudget); + // Diagrams are rendered at twice their display size. + assets + .diagrams + .insert("mermaid-ok".into(), picture(640, 320, 2, [60, 140, 90])); + assets.diagrams.insert( + format!("board-{BOARD}"), + picture(800, 400, 2, [120, 70, 150]), + ); + assets.user_names.insert(USER.into(), "Ada Lovelace".into()); + + let blocks = vec![ + Block::Heading { + level: 1, + content: vec![plain("Headings and text")], + }, + Block::Heading { + level: 2, + content: vec![plain("Level two")], + }, + Block::Heading { + level: 3, + content: vec![plain("Level three")], + }, + Block::Heading { + level: 6, + content: vec![plain("Level six")], + }, + para(vec![ + plain("Plain, "), + marked("bold", |m| m.bold = true), + plain(", "), + marked("italic", |m| m.italic = true), + plain(", "), + marked("underline", |m| m.underline = true), + plain(", "), + marked("strike", |m| m.strike = true), + plain(", "), + marked("code", |m| m.code = true), + plain(" and "), + marked("all of them", |m| { + m.bold = true; + m.italic = true; + m.underline = true; + m.strike = true; + }), + plain("."), + Inline::Break, + plain( + "After a line break:\ttab, and two spaces. <&> \"quotes\" \u{1F600} \u{05E9}\u{05DC}\u{05D5}\u{05DD}", + ), + ]), + para(vec![ + plain("Links: "), + Inline::Link { + target: LinkTarget::Web("https://example.com/?a=1&b=2".into()), + runs: vec![run("a web page", Marks::default())], + }, + plain(", "), + Inline::Link { + target: LinkTarget::Doc(DOC.into()), + runs: vec![run("another page", Marks::default())], + }, + plain(", a mention of "), + Inline::Link { + target: LinkTarget::User(USER.into()), + runs: vec![run("@ada", Marks::default())], + }, + plain(", a mention of a deleted user "), + Inline::Link { + target: LinkTarget::User(GHOST.into()), + runs: vec![run("@ghost", Marks::default())], + }, + plain(" and "), + Inline::Link { + target: LinkTarget::Plain, + runs: vec![run("a link that stays text", Marks::default())], + }, + plain("."), + ]), + Block::Rule, + Block::Quote(vec![ + text_para("A quotation."), + Block::Quote(vec![text_para("A quotation inside it.")]), + Block::List(List { + ordered: false, + start: 1, + items: vec![item(None, vec![text_para("A list in a quote")])], + }), + ]), + Block::Code { + language: "rust".into(), + text: "fn main() {\n println!(\"hi\");\n\t// tab\n}".into(), + }, + Block::Heading { + level: 1, + content: vec![plain("Lists")], + }, + Block::List(List { + ordered: false, + start: 1, + items: vec![ + item( + None, + vec![ + text_para("Bullet one"), + Block::List(List { + ordered: true, + start: 3, + items: vec![ + item(None, vec![text_para("Nested, starts at 3")]), + item(None, vec![text_para("Nested, four")]), + ], + }), + ], + ), + item( + None, + vec![text_para("Bullet two"), text_para("Second paragraph")], + ), + ], + }), + Block::List(List { + ordered: true, + start: 1, + items: vec![ + item(None, vec![text_para("First")]), + item(None, vec![text_para("Second")]), + ], + }), + Block::List(List { + ordered: false, + start: 1, + items: vec![ + item(Some(true), vec![text_para("Done")]), + item(Some(false), vec![text_para("Still to do")]), + ], + }), + Block::Heading { + level: 1, + content: vec![plain("Tables")], + }, + Block::Table(Table { + shares: vec![0.3, 0.4, 0.3], + rows: vec![ + vec![ + cell(0, true, Align::Left, vec![text_para("Left")]), + cell(1, true, Align::Center, vec![text_para("Centre")]), + cell(2, true, Align::Right, vec![text_para("Right")]), + ], + vec![ + Cell { + colspan: 2, + ..cell(0, false, Align::Left, vec![text_para("Spans two columns")]) + }, + Cell { + rowspan: 2, + ..cell(2, false, Align::Right, vec![text_para("Spans two rows")]) + }, + ], + vec![ + cell(0, false, Align::Left, vec![text_para("a")]), + cell( + 1, + false, + Align::Center, + vec![ + text_para("A nested table:"), + Block::Table(Table { + shares: vec![0.5, 0.5], + rows: vec![vec![ + cell(0, false, Align::Left, vec![text_para("x")]), + cell(1, false, Align::Left, vec![text_para("y")]), + ]], + }), + ], + ), + ], + vec![cell( + 0, + false, + Align::Left, + vec![Block::Image { + source: ImageSource::Blob(PHOTO.into()), + alt: "A picture inside a cell".into(), + }], + )], + ], + }), + Block::Heading { + level: 1, + content: vec![plain("Pictures")], + }, + Block::Image { + source: ImageSource::Blob(PHOTO.into()), + alt: "A blue picture".into(), + }, + Block::Image { + source: ImageSource::Blob(WIDE.into()), + alt: "A wide orange picture, shrunk to the page".into(), + }, + // The same blob again shares one stored picture. + Block::Image { + source: ImageSource::Blob(PHOTO.into()), + alt: "The blue picture again".into(), + }, + Block::Image { + source: ImageSource::External("https://img.example/remote.png".into()), + alt: "An external image".into(), + }, + Block::Mermaid { + key: "mermaid-ok".into(), + source: "graph TD\n A --> B".into(), + }, + Block::Board { + key: format!("board-{BOARD}"), + board_id: BOARD.into(), + label: "Roadmap".into(), + }, + Block::Attachment { + url: "/api/blobs/99999999-9999-4999-8999-999999999999".into(), + name: "budget.xlsx".into(), + size: Some(49152), + }, + Block::Attachment { + url: "https://elsewhere.example/file.pdf".into(), + name: "elsewhere.pdf".into(), + size: None, + }, + Block::Heading { + level: 1, + content: vec![plain("Fallbacks")], + }, + Block::Mermaid { + key: "mermaid-missing".into(), + source: "graph LR\n X --> Y".into(), + }, + Block::Board { + key: "board-missing".into(), + board_id: "00000000-0000-4000-8000-000000000000".into(), + label: "Sprint board".into(), + }, + Block::Image { + source: ImageSource::Blob(GONE.into()), + alt: "Unavailable image".into(), + }, + Block::Image { + source: ImageSource::Blob(HUGE.into()), + alt: "Image over the size budget".into(), + }, + Block::Image { + source: ImageSource::Unsupported, + alt: "Local file".into(), + }, + ]; + (ExportDoc { blocks }, assets) +} diff --git a/crates/knot-export/src/assets.rs b/crates/knot-export/src/assets.rs new file mode 100644 index 00000000..85efc435 --- /dev/null +++ b/crates/knot-export/src/assets.rs @@ -0,0 +1,70 @@ +//! Inputs prepared by the caller (images, diagrams, user names) and the output +//! of an export. + +use std::collections::HashMap; + +/// Document metadata shown in the title block. +pub struct Meta { + pub title: String, + /// Already formatted, e.g. "23 Oct 2026". + pub exported_on: String, + pub base_url: String, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum PicFormat { + Png, + Jpeg, + Gif, +} + +#[derive(Debug, Clone)] +pub struct Picture { + pub format: PicFormat, + pub bytes: Vec, + pub width_px: u32, + pub height_px: u32, + /// 1 for images, 2 for diagrams (rendered at twice the display size). + pub scale: u32, +} + +#[derive(Debug, Clone)] +pub enum ImageAsset { + Embedded(Picture), + Unavailable, + OverBudget, +} + +#[derive(Default)] +pub struct Assets { + /// Key: blob id. Absent means unavailable. + pub images: HashMap, + /// Key: diagram key. Absent means fallback. + pub diagrams: HashMap, + /// User id to display name. + pub user_names: HashMap, +} + +#[derive(Default, Debug, Clone, Copy, PartialEq)] +pub struct Fallbacks { + pub diagram: u32, + pub image_unavailable: u32, + pub image_over_budget: u32, + pub unknown_element: u32, +} + +pub struct Output { + pub bytes: Vec, + pub fallbacks: Fallbacks, +} + +#[derive(Debug, thiserror::Error)] +pub enum ExportError { + #[error("zip: {0}")] + Zip(String), + #[error("io: {0}")] + Io(String), + /// The document's XML passed [`crate::xml::MAX_XML_BYTES`]. + #[error("document too large: over {} MiB of XML", crate::xml::MAX_XML_BYTES >> 20)] + TooLarge, +} diff --git a/crates/knot-export/src/docx/body.rs b/crates/knot-export/src/docx/body.rs new file mode 100644 index 00000000..5966c714 --- /dev/null +++ b/crates/knot-export/src/docx/body.rs @@ -0,0 +1,309 @@ +//! `word/document.xml`: the title block, the blocks and the section +//! properties. +//! +//! Block emitters take `avail_twips`, the width their content may use. It +//! starts at the text width and shrinks inside quotes, list items and table +//! cells, so tables and pictures can size themselves to it. + +use crate::model::{Block, ExportDoc, Inline, LinkTarget, List, ListItem, Marks, Run}; +use crate::xml::esc_text; + +use super::numbering::{INDENT_TWIPS, LEVELS}; +use super::{ + Ctx, FOOTER_RID, MARGIN_TWIPS, PAGE_HEIGHT_TWIPS, PAGE_WIDTH_TWIPS, TEXT_WIDTH_TWIPS, picture, + table, +}; + +/// Left indent of the Quote style, in twips. +pub(super) const QUOTE_INDENT_TWIPS: u32 = 567; + +pub(super) fn document(cx: &mut Ctx, doc: &ExportDoc) { + cx.xml.push_str(concat!( + r#""#, + r#""#, + "", + )); + let meta = cx.meta; + text_paragraph(cx, "Title", &meta.title); + let subtitle = format!("Exported from knot · {}", meta.exported_on); + text_paragraph(cx, "Subtitle", &subtitle); + + blocks(cx, &doc.blocks, TEXT_WIDTH_TWIPS); + + cx.xml.push_str(&format!( + concat!( + r#""#, + r#""#, + r#""#, + "", + ), + footer = FOOTER_RID, + width = PAGE_WIDTH_TWIPS, + height = PAGE_HEIGHT_TWIPS, + m = MARGIN_TWIPS, + )); +} + +/// Write blocks whose content may use `avail_twips` of width. +pub(super) fn blocks(cx: &mut Ctx, items: &[Block], avail_twips: u32) { + for b in items { + block(cx, b, avail_twips); + } +} + +fn block(cx: &mut Ctx, b: &Block, avail_twips: u32) { + if cx.full() { + return; + } + match b { + Block::Paragraph(content) => { + let style = if cx.quote_depth > 0 { + Some("Quote") + } else { + cx.header_cell.then_some("TableHeader") + }; + open_paragraph(cx, style); + inlines(cx, content); + cx.xml.push_str(""); + } + Block::Heading { level, content } => { + let style = format!("Heading{}", (*level).clamp(1, 6)); + open_paragraph(cx, Some(&style)); + inlines(cx, content); + cx.xml.push_str(""); + } + Block::Quote(inner) => { + cx.quote_depth += 1; + blocks(cx, inner, avail_twips.saturating_sub(QUOTE_INDENT_TWIPS)); + cx.quote_depth -= 1; + } + Block::Code { text, .. } => text_paragraph(cx, "Code", text), + Block::Rule => { + open_paragraph(cx, Some("HorizontalRule")); + cx.xml.push_str(""); + } + Block::List(l) => list(cx, l, avail_twips), + Block::Table(t) => table::table(cx, t, avail_twips), + Block::Mermaid { key, source } => picture::mermaid(cx, key, source, avail_twips), + Block::Image { source, alt } => picture::image(cx, source, alt, avail_twips), + Block::Board { key, label, .. } => picture::board(cx, key, label, avail_twips), + Block::Attachment { url, name, size } => picture::attachment(cx, url, name, *size), + } +} + +/// A paragraph in `style` holding `text` as one unmarked run. +fn text_paragraph(cx: &mut Ctx, style: &str, text: &str) { + open_paragraph(cx, Some(style)); + run(cx, text, &Marks::default(), None); + cx.xml.push_str(""); +} + +/// Open a `w:p`, with its paragraph style unless it is Normal, and the +/// indent of the list item it continues. +pub(super) fn open_paragraph(cx: &mut Ctx, style: Option<&str>) { + let indent = cx.block_indent; + open_paragraph_with(cx, style, None, (indent > 0).then_some(indent)); +} + +/// Open a `w:p` with optional style, list membership (`numId`, `ilvl`) and +/// left indent, aligned like the table cell it is in. `w:pPr` children are +/// written in schema order. +fn open_paragraph_with( + cx: &mut Ctx, + style: Option<&str>, + num: Option<(u32, u32)>, + indent: Option, +) { + let jc = cx.cell_jc; + cx.xml.push_str(""); + if style.is_none() && num.is_none() && indent.is_none() && jc.is_none() { + return; + } + cx.xml.push_str(""); + if let Some(style) = style { + cx.xml.push_str(r#""#); + } + if let Some((num_id, ilvl)) = num { + cx.xml.push_str(&format!( + r#""# + )); + } + if let Some(left) = indent { + cx.xml.push_str(&format!(r#""#)); + } + if let Some(jc) = jc { + cx.xml.push_str(&format!(r#""#)); + } + cx.xml.push_str(""); +} + +/// A list. Every list gets its own `w:num`, so a numbered list restarts at +/// its `start`. Items are written flat, one paragraph per block; the nesting +/// depth is the `ilvl`, clamped to the nine levels Word has. +fn list(cx: &mut Ctx, l: &List, avail_twips: u32) { + let ilvl = cx.list_depth.min(LEVELS - 1); + let num_id = cx.add_num(l.ordered, ilvl, l.start); + let outer_indent = std::mem::take(&mut cx.block_indent); + cx.list_depth += 1; + let inner = avail_twips.saturating_sub(INDENT_TWIPS); + for item in &l.items { + if cx.full() { + break; + } + list_item(cx, item, (num_id, ilvl), inner); + } + cx.list_depth -= 1; + cx.block_indent = outer_indent; +} + +/// One item. Its first paragraph carries the bullet or number. A checklist +/// item has no `numPr`: its first run starts with a box character and the +/// paragraph is indented like a list item instead. Later blocks are +/// indented to line up with the text and carry no marker. +fn list_item(cx: &mut Ctx, item: &ListItem, (num_id, ilvl): (u32, u32), avail_twips: u32) { + let indent = INDENT_TWIPS * (ilvl + 1); + let (lead, tail) = match item.blocks.split_first() { + Some((Block::Paragraph(content), tail)) => (content.as_slice(), tail), + // No paragraph to hold the marker: write an empty one. + _ => (&[][..], &item.blocks[..]), + }; + match item.checked { + Some(checked) => { + open_paragraph_with(cx, Some("ListParagraph"), None, Some(indent)); + let marker = if checked { "\u{2611} " } else { "\u{2610} " }; + run(cx, marker, &Marks::default(), None); + } + None => open_paragraph_with(cx, Some("ListParagraph"), Some((num_id, ilvl)), None), + } + inlines(cx, lead); + cx.xml.push_str(""); + + for b in tail { + match b { + Block::Paragraph(content) => { + open_paragraph_with(cx, Some("ListParagraph"), None, Some(indent)); + inlines(cx, content); + cx.xml.push_str(""); + } + // A nested list brings its own indent. + Block::List(_) => block(cx, b, avail_twips), + _ => { + cx.block_indent = indent; + block(cx, b, avail_twips); + cx.block_indent = 0; + } + } + } +} + +fn inlines(cx: &mut Ctx, content: &[Inline]) { + for inline in content { + match inline { + Inline::Run(r) => run(cx, &r.text, &r.marks, None), + Inline::Break => cx.xml.push_str(""), + Inline::Link { target, runs } => link(cx, target, runs), + } + } +} + +/// Web and page links become hyperlinks. A mention becomes the plain text +/// `@`, or keeps its stored text when the user is gone. Anything +/// else stays plain text, so no unsafe target reaches the file. +fn link(cx: &mut Ctx, target: &LinkTarget, runs: &[Run]) { + let (meta, assets) = (cx.meta, cx.assets); + let url = match target { + LinkTarget::Web(url) => url.clone(), + LinkTarget::Doc(id) => format!("{}/doc/{id}", meta.base_url.trim_end_matches('/')), + LinkTarget::User(id) if assets.user_names.contains_key(id) => { + let marks = runs + .first() + .map_or_else(Marks::default, |r| r.marks.clone()); + run(cx, &format!("@{}", assets.user_names[id]), &marks, None); + return; + } + LinkTarget::User(_) | LinkTarget::Plain => { + for r in runs { + run(cx, &r.text, &r.marks, None); + } + return; + } + }; + let id = cx.hyperlink(&url); + cx.xml + .push_str(&format!(r#""#)); + for r in runs { + run(cx, &r.text, &r.marks, Some("Hyperlink")); + } + cx.xml.push_str(""); +} + +/// One `w:r`. Tabs become `w:tab`, line breaks `w:br`, and the text between +/// them a space-preserving `w:t`. A run takes one character style, so a +/// link's style wins over inline code. +pub(super) fn run(cx: &mut Ctx, text: &str, marks: &Marks, char_style: Option<&str>) { + if text.is_empty() { + return; + } + let out = &mut cx.xml; + out.push_str(""); + run_props( + out, + marks, + char_style.or(marks.code.then_some("InlineCode")), + ); + let mut rest = text; + while let Some(i) = rest.find(['\t', '\n', '\r']) { + push_text(out, &rest[..i]); + let tail = &rest[i..]; + out.push_str(if tail.starts_with('\t') { + "" + } else { + "" + }); + rest = &tail[if tail.starts_with("\r\n") { 2 } else { 1 }..]; + } + push_text(out, rest); + out.push_str(""); +} + +/// `w:rPr` in schema order: `rStyle, b, bCs, i, iCs, strike, u`. +fn run_props(out: &mut String, marks: &Marks, style: Option<&str>) { + if style.is_none() && !(marks.bold || marks.italic || marks.strike || marks.underline) { + return; + } + out.push_str(""); + if let Some(style) = style { + out.push_str(r#""#); + } + if marks.bold { + out.push_str(""); + } + if marks.italic { + out.push_str(""); + } + if marks.strike { + out.push_str(""); + } + if marks.underline { + out.push_str(r#""#); + } + out.push_str(""); +} + +fn push_text(out: &mut String, text: &str) { + let text = esc_text(text); + if !text.is_empty() { + out.push_str(r#""#); + out.push_str(&text); + out.push_str(""); + } +} diff --git a/crates/knot-export/src/docx/mod.rs b/crates/knot-export/src/docx/mod.rs new file mode 100644 index 00000000..2e850e29 --- /dev/null +++ b/crates/knot-export/src/docx/mod.rs @@ -0,0 +1,358 @@ +//! DOCX (Office Open XML) writer: hand-written XML in an in-memory zip. +//! +//! All formatting goes through named styles. The parts a company template +//! would replace (styles, numbering, footer) come in as [`DocxDefaults`]. + +mod body; +mod numbering; +mod picture; +mod styles; +mod table; + +use std::borrow::Cow; +use std::collections::HashMap; + +use crate::assets::{Assets, ExportError, Fallbacks, Meta, Output, PicFormat, Picture}; +use crate::model::ExportDoc; +use crate::xml::{MAX_XML_BYTES, ZipOut, esc_attr, esc_text}; + +/// A4 portrait with 2.5 cm margins, in twips. +const PAGE_WIDTH_TWIPS: u32 = 11906; +const PAGE_HEIGHT_TWIPS: u32 = 16838; +const MARGIN_TWIPS: u32 = 1417; +/// Width available to top-level blocks (16 cm). +const TEXT_WIDTH_TWIPS: u32 = PAGE_WIDTH_TWIPS - 2 * MARGIN_TWIPS; +/// Height available to the body (24.7 cm); the footer sits in the bottom +/// margin. +const TEXT_HEIGHT_TWIPS: u32 = PAGE_HEIGHT_TWIPS - 2 * MARGIN_TWIPS; + +const REL_TYPE: &str = "http://schemas.openxmlformats.org/officeDocument/2006/relationships/"; + +/// Relationships every document has, as (type, target). They take ids +/// `rId1`..; generated relationships follow. +const FIXED_RELS: [(&str, &str); 4] = [ + ("styles", "styles.xml"), + ("numbering", "numbering.xml"), + ("settings", "settings.xml"), + ("footer", "footer1.xml"), +]; +/// Id of the `footer1.xml` relationship (the fourth fixed one). +const FOOTER_RID: &str = "rId4"; + +/// The template-ready package defaults (spec §10): the contents of +/// `word/styles.xml`, `word/numbering.xml` and `word/footer1.xml`. +pub struct DocxDefaults { + pub styles_xml: Cow<'static, str>, + pub numbering_xml: Cow<'static, str>, + pub footer_xml: Cow<'static, str>, +} + +impl DocxDefaults { + /// knot's neutral look (spec §7). + pub fn builtin() -> Self { + Self { + styles_xml: Cow::Borrowed(styles::STYLES_XML), + numbering_xml: Cow::Owned(numbering::builtin_xml()), + footer_xml: Cow::Borrowed(styles::FOOTER_XML), + } + } +} + +/// Write `doc` as a DOCX package with the built-in defaults. +pub fn write_docx(doc: &ExportDoc, meta: &Meta, assets: &Assets) -> Result { + write_docx_with(doc, meta, assets, &DocxDefaults::builtin()) +} + +/// Write `doc` as a DOCX package on top of the given package defaults. +pub fn write_docx_with( + doc: &ExportDoc, + meta: &Meta, + assets: &Assets, + defaults: &DocxDefaults, +) -> Result { + let mut cx = Ctx::new(meta, assets); + body::document(&mut cx, doc); + if cx.full() { + return Err(ExportError::TooLarge); + } + + let numbering_xml = numbering::with_nums(&defaults.numbering_xml, &cx.nums) + .ok_or_else(|| ExportError::Io("numbering part has no ".into()))?; + let rels_xml = cx.rels_xml(); + + // Room for the media, stored as they are, and the XML before + // compression, so the buffer never grows by copying itself. + let media_bytes: usize = cx.media.iter().map(|(_, bytes)| bytes.len()).sum(); + let xml_bytes = cx.xml.len() + + rels_xml.len() + + numbering_xml.len() + + defaults.styles_xml.len() + + defaults.footer_xml.len(); + let mut zip = ZipOut::with_capacity(media_bytes + xml_bytes); + zip.deflated("[Content_Types].xml", CONTENT_TYPES.as_bytes())?; + zip.deflated("_rels/.rels", ROOT_RELS.as_bytes())?; + zip.deflated("docProps/core.xml", core_xml(meta).as_bytes())?; + zip.deflated("docProps/app.xml", APP_XML.as_bytes())?; + zip.deflated("word/document.xml", cx.xml.as_bytes())?; + zip.deflated("word/_rels/document.xml.rels", rels_xml.as_bytes())?; + zip.deflated("word/styles.xml", defaults.styles_xml.as_bytes())?; + zip.deflated("word/numbering.xml", numbering_xml.as_bytes())?; + zip.deflated("word/settings.xml", SETTINGS_XML.as_bytes())?; + zip.deflated("word/footer1.xml", defaults.footer_xml.as_bytes())?; + for (path, bytes) in &cx.media { + // Images are compressed already. + zip.stored(path, bytes)?; + } + Ok(Output { + bytes: zip.finish()?, + fallbacks: cx.fallbacks, + }) +} + +struct Relationship { + kind: &'static str, + target: String, + external: bool, +} + +/// Writer state for one document. +struct Ctx<'a> { + meta: &'a Meta, + assets: &'a Assets, + /// `word/document.xml` as written so far. + xml: String, + fallbacks: Fallbacks, + /// How many quotes enclose the block being written. + quote_depth: u32, + /// How many lists enclose the block being written. + list_depth: u32, + /// Left indent, in twips, for the paragraph a list item's non-paragraph + /// block opens; 0 outside list items. + block_indent: u32, + /// Whether the block being written is in a header cell, whose + /// paragraphs take the TableHeader style. + header_cell: bool, + /// `w:jc` of the table cell the block being written is in; `None` + /// outside cells and in left-aligned ones. + cell_jc: Option<&'static str>, + /// The `w:num` elements of every list so far, for `numbering.xml`. + nums: String, + /// Last `w:numId` handed out. + last_num_id: u32, + /// `rels[i]` has id `rId{i + 1}`. + rels: Vec, + /// Hyperlink target to its relationship id. + hyperlinks: HashMap, + /// Last drawing id handed out; ids are document-wide. + last_doc_pr: u32, + /// Media parts as (package path, bytes); `media[i]` is `image{i + 1}`. + /// The bytes are borrowed from `assets`, never copied. + media: Vec<(String, &'a [u8])>, + /// Picture key to its media number, relationship id and extension, so a + /// picture used twice is stored once. + media_index: HashMap, +} + +impl<'a> Ctx<'a> { + fn new(meta: &'a Meta, assets: &'a Assets) -> Self { + let rels = FIXED_RELS + .iter() + .map(|(kind, target)| Relationship { + kind, + target: (*target).to_owned(), + external: false, + }) + .collect(); + Self { + meta, + assets, + xml: String::new(), + fallbacks: Fallbacks::default(), + quote_depth: 0, + list_depth: 0, + block_indent: 0, + header_cell: false, + cell_jc: None, + nums: String::new(), + last_num_id: 0, + rels, + hyperlinks: HashMap::new(), + last_doc_pr: 0, + media: Vec::new(), + media_index: HashMap::new(), + } + } + + /// Whether the XML written so far has passed [`MAX_XML_BYTES`]. Once it + /// has, the emitters stop writing and the export fails. + fn full(&self) -> bool { + self.xml.len() + self.nums.len() > MAX_XML_BYTES + } + + fn add_rel(&mut self, kind: &'static str, target: String, external: bool) -> String { + self.rels.push(Relationship { + kind, + target, + external, + }); + format!("rId{}", self.rels.len()) + } + + /// Relationship id of an external hyperlink; one per unique target. + fn hyperlink(&mut self, target: &str) -> String { + if let Some(id) = self.hyperlinks.get(target) { + return id.clone(); + } + let id = self.add_rel("hyperlink", target.to_owned(), true); + self.hyperlinks.insert(target.to_owned(), id.clone()); + id + } + + /// Add a `w:num` for a new list and return its id. + fn add_num(&mut self, ordered: bool, ilvl: u32, start: u32) -> u32 { + self.last_num_id += 1; + self.nums + .push_str(&numbering::num(self.last_num_id, ordered, ilvl, start)); + self.last_num_id + } + + /// The next drawing id (`wp:docPr id`), unique and positive. + fn doc_pr_id(&mut self) -> u32 { + self.last_doc_pr += 1; + self.last_doc_pr + } + + /// The media part of the picture `key`, stored on first use, as + /// (number, relationship id, extension). + fn media_part(&mut self, key: &str, pic: &'a Picture) -> (usize, String, &'static str) { + if let Some((n, rid, ext)) = self.media_index.get(key) { + return (*n, rid.clone(), ext); + } + let n = self.media.len() + 1; + let ext = extension(pic.format); + self.media + .push((format!("word/media/image{n}.{ext}"), &pic.bytes)); + let rid = self.add_rel("image", format!("media/image{n}.{ext}"), false); + self.media_index + .insert(key.to_owned(), (n, rid.clone(), ext)); + (n, rid, ext) + } + + fn rels_xml(&self) -> String { + let mut s = String::from(concat!( + r#""#, + r#""#, + )); + for (i, rel) in self.rels.iter().enumerate() { + let mode = if rel.external { + r#" TargetMode="External""# + } else { + "" + }; + s.push_str(&format!( + r#""#, + i + 1, + rel.kind, + esc_attr(&rel.target), + )); + } + s.push_str(""); + s + } +} + +/// File extension of a picture's media part. JPEG is `jpeg`, the one +/// extension `[Content_Types].xml` declares for it. +fn extension(format: PicFormat) -> &'static str { + match format { + PicFormat::Png => "png", + PicFormat::Jpeg => "jpeg", + PicFormat::Gif => "gif", + } +} + +const CONTENT_TYPES: &str = concat!( + r#""#, + r#""#, + r#""#, + r#""#, + r#""#, + r#""#, + r#""#, + r#""#, + r#""#, + r#""#, + r#""#, + r#""#, + r#""#, + r#""#, + r#""#, +); + +const ROOT_RELS: &str = concat!( + r#""#, + r#""#, + r#""#, + r#""#, + r#""#, + r#""#, +); + +const APP_XML: &str = concat!( + r#""#, + r#""#, + r#"knot"#, +); + +/// Word 2013+ compatibility mode, so Word doesn't open the file in +/// Compatibility Mode. No `updateFields`: it would prompt on open. +const SETTINGS_XML: &str = concat!( + r#""#, + r#""#, + r#""#, + r#""#, + r#""#, + r#""#, +); + +/// Core properties: the title and the export date. No creator, so the +/// exporter's name stays out of forwarded files. +fn core_xml(meta: &Meta) -> String { + let dates = w3cdtf(&meta.exported_on) + .map(|d| { + format!( + concat!( + r#"{d}"#, + r#"{d}"#, + ), + d = d + ) + }) + .unwrap_or_default(); + format!( + concat!( + r#""#, + r#""#, + r#"{title}{dates}"#, + ), + title = esc_text(&meta.title), + dates = dates, + ) +} + +/// `Meta::exported_on` ("23 Oct 2026", the server's UTC date) as a W3CDTF +/// timestamp at midnight UTC; `None` when it isn't in that form. +fn w3cdtf(date: &str) -> Option { + const MONTHS: [&str; 12] = [ + "Jan", "Feb", "Mar", "Apr", "May", "Jun", "Jul", "Aug", "Sep", "Oct", "Nov", "Dec", + ]; + let mut parts = date.split(' '); + let day: u8 = parts.next()?.parse().ok()?; + let month = parts.next()?; + let month = MONTHS.iter().position(|m| *m == month)? + 1; + let year: u16 = parts.next()?.parse().ok()?; + (parts.next().is_none() && (1..=31).contains(&day) && year <= 9999) + .then(|| format!("{year:04}-{month:02}-{day:02}T00:00:00Z")) +} diff --git a/crates/knot-export/src/docx/numbering.rs b/crates/knot-export/src/docx/numbering.rs new file mode 100644 index 00000000..1e4578b5 --- /dev/null +++ b/crates/knot-export/src/docx/numbering.rs @@ -0,0 +1,92 @@ +//! `word/numbering.xml`: the list definitions and the `w:num` instances. +//! +//! Two abstract definitions, both nine levels deep: 0 for bullets and 1 for +//! decimals. Every list in the document gets its own `w:num` pointing at one +//! of them, so numbered lists restart where they should. + +/// `w:abstractNumId` of the bullet definition. +pub(super) const BULLET_ABSTRACT: u32 = 0; +/// `w:abstractNumId` of the decimal definition. +pub(super) const DECIMAL_ABSTRACT: u32 = 1; +/// Levels per definition; the deepest `w:ilvl` is one less. +pub(super) const LEVELS: u32 = 9; +/// Indent step per level, in twips. +pub(super) const INDENT_TWIPS: u32 = 720; +/// How far the marker hangs left of the text, in twips. +const HANGING_TWIPS: u32 = 360; + +const BULLETS: [&str; 3] = ["\u{2022}", "\u{25E6}", "\u{25AA}"]; + +const HEAD: &str = concat!( + r#""#, + r#""#, +); + +/// The built-in numbering part: both `w:abstractNum`s and no `w:num`. +pub(super) fn builtin_xml() -> String { + let mut s = String::from(HEAD); + abstract_num(&mut s, BULLET_ABSTRACT, |level| { + ("bullet", BULLETS[level as usize % BULLETS.len()].to_owned()) + }); + abstract_num(&mut s, DECIMAL_ABSTRACT, |level| { + ("decimal", format!("%{}.", level + 1)) + }); + s.push_str(""); + s +} + +fn abstract_num(out: &mut String, id: u32, level: impl Fn(u32) -> (&'static str, String)) { + out.push_str(&format!( + r#""# + )); + for ilvl in 0..LEVELS { + let (format, text) = level(ilvl); + out.push_str(&format!( + concat!( + r#""#, + r#""#, + r#""#, + ), + ilvl = ilvl, + format = format, + text = text, + left = INDENT_TWIPS * (ilvl + 1), + hanging = HANGING_TWIPS, + )); + } + out.push_str(""); +} + +/// One `w:num` for a list instance. An ordered list restarts at `start` on +/// its own level, even when that is 1, so it never continues an earlier list. +pub(super) fn num(num_id: u32, ordered: bool, ilvl: u32, start: u32) -> String { + if ordered { + format!( + concat!( + r#""#, + r#""#, + r#""#, + ), + num_id = num_id, + DECIMAL_ABSTRACT = DECIMAL_ABSTRACT, + ilvl = ilvl, + start = start, + ) + } else { + format!(r#""#) + } +} + +/// `numbering` with `nums` inserted before its closing tag, after every +/// `w:abstractNum`. `None` when the part has no closing tag to insert at. +pub(super) fn with_nums(numbering: &str, nums: &str) -> Option { + if nums.is_empty() { + return Some(numbering.to_owned()); + } + let at = numbering.rfind("")?; + let mut s = String::with_capacity(numbering.len() + nums.len()); + s.push_str(&numbering[..at]); + s.push_str(nums); + s.push_str(&numbering[at..]); + Some(s) +} diff --git a/crates/knot-export/src/docx/picture.rs b/crates/knot-export/src/docx/picture.rs new file mode 100644 index 00000000..34427509 --- /dev/null +++ b/crates/knot-export/src/docx/picture.rs @@ -0,0 +1,161 @@ +//! Pictures, and what stands in for them: inline drawings for embedded +//! images and rendered diagrams, captions for the ones that are missing, and +//! links for external images and attachments. + +use crate::assets::{ImageAsset, Picture}; +use crate::media::{base_url, fit_emu, human_size}; +use crate::model::{ImageSource, Marks}; +use crate::xml::esc_attr; + +use super::body::{open_paragraph, run}; +use super::{Ctx, TEXT_HEIGHT_TWIPS}; + +/// EMU per twip (914400 EMU per inch, 1440 twips per inch). +const EMU_PER_TWIP: u64 = 635; + +/// Tallest picture, in EMU: one that still fits on a page. Word lays an +/// inline picture out in a Normal paragraph, whose 1.15 line spacing +/// (`w:line="276"`) it applies to the picture's line too, followed by +/// 160 twips of spacing. About 21.2 cm. +const MAX_HEIGHT_EMU: u64 = (TEXT_HEIGHT_TWIPS as u64 - 160) * 240 / 276 * EMU_PER_TWIP; + +/// A body image. Embedded ones become drawings; the rest become a caption +/// or a link, and the fallback counts say which. +pub(super) fn image(cx: &mut Ctx, source: &ImageSource, alt: &str, avail_twips: u32) { + match source { + ImageSource::Blob(id) => { + let assets = cx.assets; + match assets.images.get(id) { + Some(ImageAsset::Embedded(pic)) => { + drawing(cx, &format!("image:{id}"), pic, alt, avail_twips); + } + Some(ImageAsset::OverBudget) => { + cx.fallbacks.image_over_budget += 1; + let url = format!("{}/api/blobs/{id}", base_url(&cx.meta.base_url)); + caption_link(cx, "[Image too large to include]", &url); + } + Some(ImageAsset::Unavailable) | None => unavailable(cx, alt), + } + } + ImageSource::Unsupported => unavailable(cx, alt), + ImageSource::External(url) => { + open_paragraph(cx, None); + link_run(cx, if alt.is_empty() { url } else { alt }, url); + cx.xml.push_str(""); + } + } +} + +/// A Mermaid block: the rendered diagram, or its source and a note. +pub(super) fn mermaid(cx: &mut Ctx, key: &str, source: &str, avail_twips: u32) { + let assets = cx.assets; + if let Some(pic) = assets.diagrams.get(key) { + drawing( + cx, + &format!("diagram:{key}"), + pic, + "Mermaid diagram", + avail_twips, + ); + return; + } + cx.fallbacks.diagram += 1; + open_paragraph(cx, Some("Code")); + run(cx, source, &Marks::default(), None); + cx.xml.push_str(""); + caption(cx, "Mermaid diagram (not included)"); +} + +/// A board block: the rendered diagram, or a note with the board's label. +pub(super) fn board(cx: &mut Ctx, key: &str, label: &str, avail_twips: u32) { + let descr = format!("Diagram: {label}"); + let assets = cx.assets; + if let Some(pic) = assets.diagrams.get(key) { + drawing(cx, &format!("diagram:{key}"), pic, &descr, avail_twips); + return; + } + cx.fallbacks.diagram += 1; + caption(cx, &format!("{descr} (not included)")); +} + +/// A line naming the file and its size, linked back to knot when the +/// attachment lives on this instance. +pub(super) fn attachment(cx: &mut Ctx, url: &str, name: &str, size: Option) { + let mut line = format!("\u{1F4CE} {name}"); + if let Some(size) = size { + line.push_str(&format!(" ({})", human_size(size))); + } + open_paragraph(cx, None); + if url.starts_with('/') { + let target = format!("{}{url}", base_url(&cx.meta.base_url)); + link_run(cx, &line, &target); + } else { + run(cx, &line, &Marks::default(), None); + } + cx.xml.push_str(""); +} + +fn unavailable(cx: &mut Ctx, alt: &str) { + cx.fallbacks.image_unavailable += 1; + let text = if alt.is_empty() { + "[Image not available]".to_owned() + } else { + format!("[Image not available] \u{2014} {alt}") + }; + caption(cx, &text); +} + +fn caption(cx: &mut Ctx, text: &str) { + open_paragraph(cx, Some("Caption")); + run(cx, text, &Marks::default(), None); + cx.xml.push_str(""); +} + +fn caption_link(cx: &mut Ctx, text: &str, url: &str) { + open_paragraph(cx, Some("Caption")); + link_run(cx, text, url); + cx.xml.push_str(""); +} + +/// `text` as a hyperlink to `url`. +fn link_run(cx: &mut Ctx, text: &str, url: &str) { + let id = cx.hyperlink(url); + cx.xml + .push_str(&format!(r#""#)); + run(cx, text, &Marks::default(), Some("Hyperlink")); + cx.xml.push_str(""); +} + +/// An inline drawing of `pic` in its own paragraph, at most `avail_twips` +/// wide and a page high. `key` identifies the picture, so one used twice is +/// stored once. +fn drawing<'a>(cx: &mut Ctx<'a>, key: &str, pic: &'a Picture, descr: &str, avail_twips: u32) { + let (n, rid, ext) = cx.media_part(key, pic); + let (cx_emu, cy_emu) = fit_emu(pic, u64::from(avail_twips) * EMU_PER_TWIP, MAX_HEIGHT_EMU); + // A zero extent is invalid; a collapsed width or height is still drawn. + let (w, h) = (cx_emu.max(1), cy_emu.max(1)); + let id = cx.doc_pr_id(); + let descr = esc_attr(descr); + open_paragraph(cx, None); + cx.xml.push_str(&format!( + concat!( + r#""#, + r#""#, + r#""#, + r#""#, + r#""#, + r#""#, + r#""#, + r#""#, + r#""#, + r#""#, + ), + w = w, + h = h, + id = id, + descr = descr, + n = n, + ext = ext, + rid = rid, + )); +} diff --git a/crates/knot-export/src/docx/styles.rs b/crates/knot-export/src/docx/styles.rs new file mode 100644 index 00000000..0a39f0d9 --- /dev/null +++ b/crates/knot-export/src/docx/styles.rs @@ -0,0 +1,129 @@ +//! The built-in package defaults: style definitions and the footer (the +//! numbering definitions are in [`super::numbering`]). A company template would replace these parts (spec §10), so +//! they live here in one place and reach the writer only through +//! [`super::DocxDefaults`]. +//! +//! Built-in styles carry Word's built-in `w:name` ("heading 1", "Title", ...) +//! so Word, and a later template mapping, treats them as its own. Children of +//! `w:style`, `w:pPr` and `w:rPr` follow the schema order; Word rejects files +//! where they don't. + +pub(super) const STYLES_XML: &str = concat!( + r#""#, + r#""#, + r#""#, + r#""#, + r#""#, + r#""#, + r#""#, + // Paragraph styles. + r#""#, + r#""#, + r#""#, + r#""#, + r#""#, + r#""#, + r#""#, + r#""#, + r#""#, + r#""#, + r#""#, + r#""#, + r#""#, + r#""#, + r#""#, + r#""#, + r#""#, + r#""#, + r#""#, + r#""#, + r#""#, + r#""#, + r#""#, + r#""#, + r#""#, + r#""#, + r#""#, + r#""#, + r#""#, + r#""#, + r#""#, + r#""#, + r#""#, + // The left indent is QUOTE_INDENT_TWIPS in body.rs. + r#""#, + r#""#, + r#""#, + r#""#, + r#""#, + r#""#, + r#""#, + r#""#, + r#""#, + r#""#, + r#""#, + r#""#, + r#""#, + r#""#, + r#""#, + r#""#, + r#""#, + r#""#, + r#""#, + r#""#, + r#""#, + r#""#, + r#""#, + r#""#, + r#""#, + r#""#, + r#""#, + r#""#, + // Character styles. + r#""#, + r#""#, + r#""#, + r#""#, + r#""#, + r#""#, + r#""#, + // Table styles. TableNormal's 108-twip side margins are why cell content + // gets the cell width minus 216. + r#""#, + r#""#, + r#""#, + r#""#, + r#""#, + r#""#, + r#""#, + r#""#, + r#""#, + r#""#, + r#""#, + r#""#, + r#""#, + r#""#, + r#""#, + r#""#, + r#""#, +); + +/// "Page X of Y", centred, with complex `PAGE` / `NUMPAGES` fields. +pub(super) const FOOTER_XML: &str = concat!( + r#""#, + r#""#, + r#""#, + r#"Page "#, + r#""#, + r#" PAGE "#, + r#""#, + r#"1"#, + r#""#, + r#" of "#, + r#""#, + r#" NUMPAGES "#, + r#""#, + r#"1"#, + r#""#, + r#""#, +); diff --git a/crates/knot-export/src/docx/table.rs b/crates/knot-export/src/docx/table.rs new file mode 100644 index 00000000..34cd1754 --- /dev/null +++ b/crates/knot-export/src/docx/table.rs @@ -0,0 +1,154 @@ +//! Tables: a fixed grid of `w:gridCol`s from the column shares, every row +//! filled out to that grid, and spans as `w:gridSpan` and `w:vMerge`. + +use crate::model::{Align, Cell, Table}; +use crate::tables::{Slot, grid_edges, header_rows, layout}; + +use super::body::QUOTE_INDENT_TWIPS; +use super::{Ctx, body}; + +/// Left plus right cell margin of the TableNormal style, in twips: what +/// the content of a cell loses of the cell's width. +const CELL_MARGINS_TWIPS: u32 = 216; + +const V_MERGE_RESTART: &str = r#""#; +const V_MERGE_CONTINUE: &str = ""; + +/// Write `t`, `avail_twips` wide. A table without columns or rows writes +/// nothing. Word joins adjacent tables into one, so a table right after +/// another gets an empty paragraph in between. +pub(super) fn table(cx: &mut Ctx, t: &Table, avail_twips: u32) { + if t.shares.is_empty() || t.rows.is_empty() { + return; + } + if cx.xml.ends_with("") { + cx.xml.push_str(""); + } + let edges = grid_edges(&t.shares, avail_twips); + // A table inside a list item or a quote lines up with the text around + // it, which `avail_twips` has already lost that indent for. + let indent = cx.block_indent + cx.quote_depth * QUOTE_INDENT_TWIPS; + let tbl_ind = if indent > 0 { + format!(r#""#) + } else { + String::new() + }; + cx.xml.push_str(&format!( + concat!( + r#""#, + r#"{tbl_ind}"#, + r#""#, + ), + avail = avail_twips, + tbl_ind = tbl_ind, + )); + for col in edges.windows(2) { + cx.xml + .push_str(&format!(r#""#, col[1] - col[0])); + } + cx.xml.push_str(""); + + let heads = header_rows(t); + // The whole table is one row group: a row span is clamped to the end of + // the table, also when it starts in a header row. + let rows = t.rows.len(); + for (r, slots) in layout(t, move |_| rows).enumerate() { + if cx.full() { + break; + } + cx.xml.push_str(""); + if r < heads { + cx.xml.push_str(""); + } + for slot in slots { + row_slot(cx, slot, &edges); + } + cx.xml.push_str(""); + } + cx.xml.push_str(""); +} + +/// One slot of a row: a cell, a continuation of a merge from a row above +/// (shaded like the cell it continues), or an empty cell filling a gap. +fn row_slot(cx: &mut Ctx, slot: Slot, edges: &[u32]) { + match slot { + Slot::Cell { + cell: c, + pos, + span, + rowspan, + } => cell(cx, c, edges[pos + span] - edges[pos], span, rowspan > 1), + Slot::Covered { pos, span, header } => { + let twips = edges[pos + span] - edges[pos]; + empty_cell(cx, twips, span, Some(V_MERGE_CONTINUE), header); + } + Slot::Gap { pos, end } => { + empty_cell(cx, edges[end] - edges[pos], end - pos, None, false); + } + } +} + +/// A cell with `c`'s blocks, `twips` wide over `span` grid columns. +fn cell(cx: &mut Ctx, c: &Cell, twips: u32, span: usize, restart: bool) { + cx.xml.push_str(""); + tc_pr( + &mut cx.xml, + twips, + span, + restart.then_some(V_MERGE_RESTART), + c.header, + ); + // The content starts at the cell's edge: no quote, list or list-item + // indent from around the table carries into it. + let outer = ( + cx.quote_depth, + cx.list_depth, + cx.block_indent, + cx.header_cell, + cx.cell_jc, + ); + (cx.quote_depth, cx.list_depth, cx.block_indent) = (0, 0, 0); + cx.header_cell = c.header; + cx.cell_jc = match c.align { + Align::Left => None, + Align::Center => Some("center"), + Align::Right => Some("right"), + }; + body::blocks(cx, &c.blocks, twips.saturating_sub(CELL_MARGINS_TWIPS)); + ( + cx.quote_depth, + cx.list_depth, + cx.block_indent, + cx.header_cell, + cx.cell_jc, + ) = outer; + // A cell must end in a paragraph, also when it is empty or ends in a + // table. + if !(cx.xml.ends_with("") || cx.xml.ends_with("")) { + cx.xml.push_str(""); + } + cx.xml.push_str(""); +} + +/// A cell holding only an empty paragraph: a merge continuation or padding. +fn empty_cell(cx: &mut Ctx, twips: u32, span: usize, v_merge: Option<&str>, header: bool) { + cx.xml.push_str(""); + tc_pr(&mut cx.xml, twips, span, v_merge, header); + cx.xml.push_str(""); +} + +/// `w:tcPr` in schema order: `tcW, gridSpan, vMerge, shd`. Header cells are +/// shaded. +fn tc_pr(out: &mut String, twips: u32, span: usize, v_merge: Option<&str>, header: bool) { + out.push_str(&format!(r#""#)); + if span > 1 { + out.push_str(&format!(r#""#)); + } + if let Some(v_merge) = v_merge { + out.push_str(v_merge); + } + if header { + out.push_str(r#""#); + } + out.push_str(""); +} diff --git a/crates/knot-export/src/lib.rs b/crates/knot-export/src/lib.rs new file mode 100644 index 00000000..c41f7a5a --- /dev/null +++ b/crates/knot-export/src/lib.rs @@ -0,0 +1,21 @@ +//! Pure export of a knot page (a yrs document) to an in-memory model that the +//! DOCX and ODT writers consume. + +pub mod assets; +pub mod docx; +pub mod media; +pub mod model; +pub mod odt; +pub mod tables; +pub mod walk; +pub mod xml; + +pub use assets::{Assets, ExportError, Fallbacks, ImageAsset, Meta, Output, PicFormat, Picture}; +pub use docx::write_docx; +pub use media::{picture_from_png, prepare_image}; +pub use model::*; +pub use odt::write_odt; +pub use walk::{ + DiagramRef, MAX_DEPTH, MAX_PAGE_GRID_CELLS, MAX_SPAN, MAX_TABLE_COLUMNS, MAX_TABLE_GRID_CELLS, + TABLE_COLUMN_ROWS, Walked, blob_ids, diagrams, user_ids, walk, +}; diff --git a/crates/knot-export/src/media.rs b/crates/knot-export/src/media.rs new file mode 100644 index 00000000..4b3f1232 --- /dev/null +++ b/crates/knot-export/src/media.rs @@ -0,0 +1,145 @@ +//! Image sniffing, WebP conversion and size fitting. + +use crate::assets::{PicFormat, Picture}; + +/// WebP pixel cap (4096 x 4096), checked before decoding. +const MAX_WEBP_PIXELS: u64 = 16_777_216; +const EMU_PER_PX: u64 = 9525; +const CM_PER_PX: f64 = 2.54 / 96.0; +const PNG_MAGIC: &[u8] = b"\x89PNG\r\n\x1a\n"; + +fn dims(bytes: &[u8]) -> Option<(u32, u32)> { + let s = imagesize::blob_size(bytes).ok()?; + let (w, h) = (u32::try_from(s.width).ok()?, u32::try_from(s.height).ok()?); + (w > 0 && h > 0).then_some((w, h)) +} + +/// Accept PNG, JPEG, GIF and WebP; WebP is converted to PNG. Anything else, or +/// any failure, yields `None`. +pub fn prepare_image(bytes: &[u8]) -> Option { + let format = if bytes.starts_with(PNG_MAGIC) { + PicFormat::Png + } else if bytes.starts_with(&[0xFF, 0xD8, 0xFF]) { + PicFormat::Jpeg + } else if bytes.starts_with(b"GIF87a") || bytes.starts_with(b"GIF89a") { + PicFormat::Gif + } else if bytes.len() >= 12 && &bytes[..4] == b"RIFF" && &bytes[8..12] == b"WEBP" { + return prepare_webp(bytes); + } else { + return None; + }; + let (width_px, height_px) = dims(bytes)?; + Some(Picture { + format, + bytes: bytes.to_vec(), + width_px, + height_px, + scale: 1, + }) +} + +fn prepare_webp(bytes: &[u8]) -> Option { + let (w, h) = dims(bytes)?; + if u64::from(w) * u64::from(h) > MAX_WEBP_PIXELS { + return None; + } + let mut dec = image_webp::WebPDecoder::new(std::io::Cursor::new(bytes)).ok()?; + if dec.dimensions() != (w, h) { + return None; + } + let has_alpha = dec.has_alpha(); + let mut buf = vec![0u8; dec.output_buffer_size()?]; + dec.read_image(&mut buf).ok()?; + let mut out = Vec::new(); + { + let mut enc = png::Encoder::new(&mut out, w, h); + enc.set_color(if has_alpha { + png::ColorType::Rgba + } else { + png::ColorType::Rgb + }); + enc.set_depth(png::BitDepth::Eight); + enc.set_compression(png::Compression::Fast); + let mut writer = enc.write_header().ok()?; + writer.write_image_data(&buf).ok()?; + writer.finish().ok()?; + } + Some(Picture { + format: PicFormat::Png, + bytes: out, + width_px: w, + height_px: h, + scale: 1, + }) +} + +/// A diagram part: PNG only, with its dimensions checked. +pub fn picture_from_png(bytes: Vec, scale: u32) -> Option { + if scale == 0 || !bytes.starts_with(PNG_MAGIC) { + return None; + } + let (width_px, height_px) = dims(&bytes)?; + Some(Picture { + format: PicFormat::Png, + bytes, + width_px, + height_px, + scale, + }) +} + +/// Size in EMU: natural `px / scale` at 96 dpi, shrunk proportionally to +/// fit within `max_width_emu` by `max_height_emu`, never enlarged. +pub fn fit_emu(p: &Picture, max_width_emu: u64, max_height_emu: u64) -> (u64, u64) { + let scale = u64::from(p.scale.max(1)); + let nat_w = u64::from(p.width_px) * EMU_PER_PX / scale; + let nat_h = u64::from(p.height_px) * EMU_PER_PX / scale; + if nat_w <= max_width_emu && nat_h <= max_height_emu { + return (nat_w, nat_h); + } + let (w, h, max_w, max_h) = ( + u128::from(nat_w), + u128::from(nat_h), + u128::from(max_width_emu), + u128::from(max_height_emu), + ); + // The tighter bound wins: width when max_w / w <= max_h / h. + if max_w * h <= max_h * w { + (max_width_emu, (h * max_w / w) as u64) + } else { + ((w * max_h / h) as u64, max_height_emu) + } +} + +/// Size in centimetres at 96 dpi, same rule as [`fit_emu`]. +pub fn fit_cm(p: &Picture, max_width_cm: f64, max_height_cm: f64) -> (f64, f64) { + let scale = f64::from(p.scale.max(1)); + let nat_w = f64::from(p.width_px) / scale * CM_PER_PX; + let nat_h = f64::from(p.height_px) / scale * CM_PER_PX; + if nat_w <= max_width_cm && nat_h <= max_height_cm { + return (nat_w, nat_h); + } + if max_width_cm * nat_h <= max_height_cm * nat_w { + (max_width_cm, nat_h * max_width_cm / nat_w) + } else { + (nat_w * max_height_cm / nat_h, max_height_cm) + } +} + +/// A base URL without trailing slashes, ready to prefix a path. +pub(crate) fn base_url(base: &str) -> &str { + base.trim_end_matches('/') +} + +/// 48 KB, 1.5 MB: binary units, whole kilobytes. +pub(crate) fn human_size(bytes: u64) -> String { + const KB: u64 = 1024; + const MB: u64 = KB * 1024; + const GB: u64 = MB * 1024; + match bytes { + 0..KB => format!("{bytes} B"), + KB..MB => format!("{} KB", (bytes + KB / 2) / KB), + MB..GB => format!("{:.1} MB", bytes as f64 / MB as f64), + _ => format!("{:.1} GB", bytes as f64 / GB as f64), + } +} diff --git a/crates/knot-export/src/model.rs b/crates/knot-export/src/model.rs new file mode 100644 index 00000000..2c2257cc --- /dev/null +++ b/crates/knot-export/src/model.rs @@ -0,0 +1,116 @@ +//! Format-neutral export model. + +#[derive(Debug, Clone, PartialEq)] +pub struct ExportDoc { + pub blocks: Vec, +} + +#[derive(Debug, Clone, PartialEq)] +pub enum Block { + Paragraph(Vec), + /// `level` is clamped to 1..=6. + Heading { + level: u8, + content: Vec, + }, + Quote(Vec), + Code { + language: String, + text: String, + }, + Mermaid { + key: String, + source: String, + }, + Rule, + List(List), + Table(Table), + Image { + source: ImageSource, + alt: String, + }, + Board { + key: String, + board_id: String, + label: String, + }, + Attachment { + url: String, + name: String, + size: Option, + }, +} + +#[derive(Debug, Clone, PartialEq)] +pub enum Inline { + Run(Run), + Link { target: LinkTarget, runs: Vec }, + Break, +} + +#[derive(Debug, Clone, PartialEq)] +pub struct Run { + pub text: String, + pub marks: Marks, +} + +#[derive(Debug, Clone, PartialEq, Default)] +pub struct Marks { + pub bold: bool, + pub italic: bool, + pub underline: bool, + pub strike: bool, + pub code: bool, +} + +#[derive(Debug, Clone, PartialEq)] +pub enum LinkTarget { + Web(String), + Doc(String), + User(String), + Plain, +} + +#[derive(Debug, Clone, PartialEq)] +pub enum ImageSource { + Blob(String), + External(String), + Unsupported, +} + +#[derive(Debug, Clone, PartialEq)] +pub struct List { + pub ordered: bool, + pub start: u32, + pub items: Vec, +} + +#[derive(Debug, Clone, PartialEq)] +pub struct ListItem { + pub checked: Option, + pub blocks: Vec, +} + +#[derive(Debug, Clone, PartialEq)] +pub struct Table { + pub shares: Vec, + pub rows: Vec>, +} + +#[derive(Debug, Clone, PartialEq)] +pub struct Cell { + pub col: usize, + pub colspan: usize, + pub rowspan: usize, + pub header: bool, + pub align: Align, + pub blocks: Vec, +} + +#[derive(Debug, Clone, PartialEq, Default)] +pub enum Align { + #[default] + Left, + Center, + Right, +} diff --git a/crates/knot-export/src/odt/body.rs b/crates/knot-export/src/odt/body.rs new file mode 100644 index 00000000..322ad598 --- /dev/null +++ b/crates/knot-export/src/odt/body.rs @@ -0,0 +1,403 @@ +//! `office:text`: the title block, the blocks and the lists, and the text +//! encoder every piece of text goes through. +//! +//! Block emitters take `avail_cm`, the width their content may use. It +//! starts at the text width and shrinks inside quotes, list items and table +//! cells, so tables and pictures can size themselves to it. + +use crate::model::{Block, ExportDoc, Inline, LinkTarget, List, ListItem, Marks, Run}; +use crate::xml::{esc_attr, esc_text}; + +use super::{Ctx, LIST_INDENT_CM, LIST_LEVELS, QUOTE_INDENT_CM, TEXT_WIDTH_CM, picture, table}; + +const HEADINGS: [&str; 6] = [ + "Heading_20_1", + "Heading_20_2", + "Heading_20_3", + "Heading_20_4", + "Heading_20_5", + "Heading_20_6", +]; + +/// A `text:list` or list item that is open: its element and where it is in +/// `Ctx::body`. +struct OpenElement { + tag: &'static str, + /// Where its start tag begins. + start: usize, + /// Where its content begins, right after the start tag. + content: usize, +} + +impl OpenElement { + /// Write the start tag `<{tag}{attrs}>`. + fn write(cx: &mut Ctx, tag: &'static str, attrs: &str) -> Self { + let start = cx.body.len(); + cx.body.push_str(&format!("<{tag}{attrs}>")); + Self { + tag, + start, + content: cx.body.len(), + } + } + + /// Write the end tag. An element still without content (a list without + /// items, or one reopened after a table with nothing left to hold) goes + /// instead. + fn close(&self, cx: &mut Ctx) { + if cx.body.len() == self.content { + cx.body.truncate(self.start); + } else { + cx.body.push_str(&format!("", self.tag)); + } + } +} + +/// A `text:list` that is open, and the item open in it. +pub(super) struct OpenList { + list: OpenElement, + /// Its list style. + style: &'static str, + /// The `xml:id` of an outermost list, which the list reopened after a + /// table continues. + id: Option, + /// The open item: a `text:list-item`, or a `text:list-header` for an + /// item continued after a table. + item: Option, +} + +pub(super) fn document(cx: &mut Ctx, doc: &ExportDoc) { + let meta = cx.meta; + text_paragraph(cx, "Title", &meta.title); + let subtitle = format!("Exported from knot · {}", meta.exported_on); + text_paragraph(cx, "Subtitle", &subtitle); + blocks(cx, &doc.blocks, TEXT_WIDTH_CM); +} + +/// Write blocks whose content may use `avail_cm` of width. +pub(super) fn blocks(cx: &mut Ctx, items: &[Block], avail_cm: f64) { + for b in items { + block(cx, b, avail_cm); + } +} + +fn block(cx: &mut Ctx, b: &Block, avail_cm: f64) { + if cx.full() { + return; + } + match b { + Block::Paragraph(content) => { + let style = if cx.quote_depth > 0 { + "Quotations" + } else { + match cx.cell { + Some(cell) if cell.header => "Table_20_Heading", + Some(_) => "Table_20_Contents", + None => "Standard", + } + }; + open_paragraph(cx, style); + inlines(cx, content); + cx.body.push_str(""); + } + Block::Heading { level, content } => { + let level = (*level).clamp(1, 6); + let attrs = format!(r#" text:outline-level="{level}""#); + open(cx, "text:h", HEADINGS[usize::from(level - 1)], &attrs); + inlines(cx, content); + cx.body.push_str(""); + } + Block::Quote(inner) => { + cx.quote_depth += 1; + blocks(cx, inner, (avail_cm - QUOTE_INDENT_CM).max(0.0)); + cx.quote_depth -= 1; + } + Block::Code { text, .. } => text_paragraph(cx, "Preformatted_20_Text", text), + Block::Rule => { + open_paragraph(cx, "Horizontal_20_Line"); + cx.body.push_str(""); + } + Block::List(l) => list(cx, l, avail_cm), + Block::Table(t) => table::table(cx, t, avail_cm), + Block::Image { source, alt } => picture::image(cx, source, alt, avail_cm), + Block::Mermaid { key, source } => picture::mermaid(cx, key, source, avail_cm), + Block::Board { key, label, .. } => picture::board(cx, key, label, avail_cm), + Block::Attachment { url, name, size } => picture::attachment(cx, url, name, *size), + } +} + +/// A paragraph in `style` holding `text` as one unmarked run. +pub(super) fn text_paragraph(cx: &mut Ctx, style: &'static str, text: &str) { + open_paragraph(cx, style); + run(cx, text, &Marks::default()); + cx.body.push_str(""); +} + +/// Open a `text:p` in `style`, aligned like the table cell it is in. +pub(super) fn open_paragraph(cx: &mut Ctx, style: &'static str) { + open(cx, "text:p", style, ""); +} + +/// Open a paragraph element `tag` in `style` (see [`Ctx::para_style`]) with +/// extra `attrs`. Leading spaces of its text become `text:s`. +fn open(cx: &mut Ctx, tag: &str, style: &'static str, attrs: &str) { + let style = cx.para_style(style); + cx.body + .push_str(&format!(r#"<{tag} text:style-name="{style}"{attrs}>"#)); + cx.after_char = false; +} + +/// A list. Each one is its own `text:list` that continues no other, so a +/// numbered list starts at its `start`; nested lists nest, and their depth +/// picks the list style's level. ODF list styles have ten levels: a list +/// nested deeper continues the innermost list instead, its items becoming +/// items there, and what follows it in the enclosing item stays in the last +/// of them. +fn list(cx: &mut Ctx, l: &List, avail_cm: f64) { + if cx.lists.len() >= LIST_LEVELS as usize { + for item in &l.items { + if cx.full() { + break; + } + close_item(cx); + list_item(cx, item, None, avail_cm); + } + return; + } + let style = if l.ordered { + "Numbering_20_123" + } else { + "List_20_Bullet" + }; + let id = cx.lists.is_empty().then(|| { + cx.last_list_id += 1; + format!("list{}", cx.last_list_id) + }); + let id_attr = id + .as_ref() + .map(|id| format!(r#" xml:id="{id}""#)) + .unwrap_or_default(); + let attrs = format!(r#" text:style-name="{style}"{id_attr}"#); + let list = OpenElement::write(cx, "text:list", &attrs); + cx.lists.push(OpenList { + list, + style, + id, + item: None, + }); + let inner = (avail_cm - LIST_INDENT_CM).max(0.0); + for (i, item) in l.items.iter().enumerate() { + if cx.full() { + break; + } + let start = (l.ordered && i == 0).then_some(l.start); + list_item(cx, item, start, inner); + close_item(cx); + } + if let Some(open) = cx.lists.pop() { + open.list.close(cx); + } +} + +/// Open an item in the innermost list and write its blocks, leaving it open +/// for the caller to close. Its first paragraph carries the bullet or +/// number, so an item that doesn't start with a paragraph gets an empty one. +/// A checklist item stays in its list, at its level, but takes the +/// label-less `List_20_Check` style, and its first paragraph starts with a +/// box character instead. An ordered list's first item carries its start. +fn list_item(cx: &mut Ctx, item: &ListItem, start: Option, avail_cm: f64) { + let mut attrs = String::new(); + if let Some(start) = start { + attrs.push_str(&format!(r#" text:start-value="{start}""#)); + } + if item.checked.is_some() { + attrs.push_str(r#" text:style-override="List_20_Check""#); + } + let open = OpenElement::write(cx, "text:list-item", &attrs); + if let Some(list) = cx.lists.last_mut() { + list.item = Some(open); + } + + let (lead, tail) = match item.blocks.split_first() { + Some((Block::Paragraph(content), tail)) => (content.as_slice(), tail), + _ => (&[][..], &item.blocks[..]), + }; + open_paragraph(cx, "List_20_Paragraph"); + if let Some(checked) = item.checked { + let marker = if checked { "\u{2611} " } else { "\u{2610} " }; + run(cx, marker, &Marks::default()); + } + inlines(cx, lead); + cx.body.push_str(""); + + for b in tail { + match b { + Block::Paragraph(content) => { + open_paragraph(cx, "List_20_Paragraph"); + inlines(cx, content); + cx.body.push_str(""); + } + _ => block(cx, b, avail_cm), + } + } +} + +/// Close the open item of the innermost list, if any. +fn close_item(cx: &mut Ctx) { + if let Some(item) = cx.lists.last_mut().and_then(|l| l.item.take()) { + item.close(cx); + } +} + +/// Close every open list, innermost first, before a block no list item may +/// hold (a table). Returns them for [`reopen_lists`]. +pub(super) fn close_lists(cx: &mut Ctx) -> Vec { + let lists = std::mem::take(&mut cx.lists); + for l in lists.iter().rev() { + if let Some(item) = &l.item { + item.close(cx); + } + l.list.close(cx); + } + lists +} + +/// Reopen the lists [`close_lists`] closed. The outermost continues the +/// original through `text:continue-list`, and each open item continues as a +/// list header, which has no bullet or number of its own. Whatever ends up +/// with nothing in it goes again when it closes. +pub(super) fn reopen_lists(cx: &mut Ctx, mut lists: Vec) { + for l in &mut lists { + let continues = + l.id.as_ref() + .map(|id| format!(r#" text:continue-list="{id}""#)) + .unwrap_or_default(); + let attrs = format!(r#" text:style-name="{}"{continues}"#, l.style); + l.list = OpenElement::write(cx, "text:list", &attrs); + if l.item.is_some() { + l.item = Some(OpenElement::write(cx, "text:list-header", "")); + } + } + cx.lists = lists; +} + +/// How far, in cm, the open lists and quotes indent the text: where a table +/// written among them starts. +pub(super) fn indent_cm(cx: &Ctx, lists: &[OpenList]) -> f64 { + LIST_INDENT_CM * lists.len() as f64 + QUOTE_INDENT_CM * f64::from(cx.quote_depth) +} + +fn inlines(cx: &mut Ctx, content: &[Inline]) { + for inline in content { + match inline { + Inline::Run(r) => run(cx, &r.text, &r.marks), + Inline::Break => { + cx.body.push_str(""); + cx.after_char = false; + } + Inline::Link { target, runs } => link(cx, target, runs), + } + } +} + +/// Web and page links become `text:a`. A mention becomes the plain text +/// `@`, or keeps its stored text when the user is gone. +/// Anything else stays plain text, so no unsafe target reaches the file. +fn link(cx: &mut Ctx, target: &LinkTarget, runs: &[Run]) { + let (meta, assets) = (cx.meta, cx.assets); + let url = match target { + LinkTarget::Web(url) => url.clone(), + LinkTarget::Doc(id) => format!("{}/doc/{id}", meta.base_url.trim_end_matches('/')), + LinkTarget::User(id) if assets.user_names.contains_key(id) => { + let marks = runs + .first() + .map_or_else(Marks::default, |r| r.marks.clone()); + run(cx, &format!("@{}", assets.user_names[id]), &marks); + return; + } + LinkTarget::User(_) | LinkTarget::Plain => { + for r in runs { + run(cx, &r.text, &r.marks); + } + return; + } + }; + cx.body.push_str(&format!( + r#""#, + esc_attr(&url) + )); + for r in runs { + run(cx, &r.text, &r.marks); + } + cx.body.push_str(""); +} + +/// One run. Inline code takes the `Source_20_Text` character style; the +/// other marks an automatic style in a span inside it. +fn run(cx: &mut Ctx, text: &str, marks: &Marks) { + if text.is_empty() { + return; + } + let direct = cx.text_style(marks); + if marks.code { + cx.body + .push_str(r#""#); + } + if let Some(style) = &direct { + cx.body + .push_str(&format!(r#""#)); + } + push_text(cx, text); + if direct.is_some() { + cx.body.push_str(""); + } + if marks.code { + cx.body.push_str(""); + } +} + +/// Write `text` so a reader sees exactly it. ODF collapses white space in +/// all text, so a tab becomes `text:tab`, a line break (`\n`, `\r\n` or +/// `\r`) `text:line-break`, and a run of spaces one literal space plus +/// `text:s` for the rest. A space that doesn't follow a character written +/// as character data (at the start of the paragraph, or after an element) +/// would be dropped, so such a run is all `text:s`. Characters XML forbids +/// are dropped. +fn push_text(cx: &mut Ctx, text: &str) { + let out = &mut cx.body; + let mut rest = text; + loop { + let end = rest.find([' ', '\t', '\n', '\r']).unwrap_or(rest.len()); + let chars = esc_text(&rest[..end]); + if !chars.is_empty() { + out.push_str(&chars); + cx.after_char = true; + } + rest = &rest[end..]; + let len = match rest.as_bytes().first() { + None => return, + Some(b' ') => { + let n = rest.len() - rest.trim_start_matches(' ').len(); + if cx.after_char { + out.push(' '); + if n > 1 { + out.push_str(&format!(r#""#, n - 1)); + } + } else { + out.push_str(&format!(r#""#)); + } + n + } + Some(b'\t') => { + out.push_str(""); + 1 + } + Some(_) => { + out.push_str(""); + if rest.starts_with("\r\n") { 2 } else { 1 } + } + }; + cx.after_char = false; + rest = &rest[len..]; + } +} diff --git a/crates/knot-export/src/odt/mod.rs b/crates/knot-export/src/odt/mod.rs new file mode 100644 index 00000000..c1e2c79a --- /dev/null +++ b/crates/knot-export/src/odt/mod.rs @@ -0,0 +1,342 @@ +//! ODT (OpenDocument text) writer: hand-written XML in an in-memory zip. +//! +//! All formatting goes through named styles, except the marks of a run and a +//! table's widths and borders, which are automatic styles in `content.xml`. +//! The part a company template would replace (`styles.xml`) comes in as +//! [`OdtDefaults`]. + +mod body; +mod picture; +mod styles; +mod table; + +use std::borrow::Cow; +use std::collections::HashMap; + +use crate::assets::{Assets, ExportError, Fallbacks, Meta, Output, Picture}; +use crate::model::{ExportDoc, Marks}; +use crate::xml::{MAX_XML_BYTES, ZipOut, esc_attr, esc_text}; + +const MIMETYPE: &str = "application/vnd.oasis.opendocument.text"; + +/// Width available to top-level blocks: A4 less two 2.5 cm margins. +const TEXT_WIDTH_CM: f64 = 16.0; +/// Height available to the body: A4 less the 2.5 cm top margin and the +/// 2.5 cm the footer takes with the bottom margin (see `styles::PAGE`). +const TEXT_HEIGHT_CM: f64 = 24.7; +/// Left margin of the Quotations style. +const QUOTE_INDENT_CM: f64 = 1.0; +/// Indent step per list level. +const LIST_INDENT_CM: f64 = 1.27; +/// Levels of a list style; deeper lists continue the deepest level. +const LIST_LEVELS: u32 = 10; + +/// The template-ready package defaults (spec §10): the contents of +/// `styles.xml`, with the named styles, the list styles, the page layout and +/// the footer. +pub struct OdtDefaults { + pub styles_xml: Cow<'static, str>, +} + +impl OdtDefaults { + /// knot's neutral look (spec §7). + pub fn builtin() -> Self { + Self { + styles_xml: Cow::Owned(styles::builtin_xml()), + } + } +} + +/// Write `doc` as an ODT package with the built-in defaults. +pub fn write_odt(doc: &ExportDoc, meta: &Meta, assets: &Assets) -> Result { + write_odt_with(doc, meta, assets, &OdtDefaults::builtin()) +} + +/// Write `doc` as an ODT package on top of the given package defaults. +pub fn write_odt_with( + doc: &ExportDoc, + meta: &Meta, + assets: &Assets, + defaults: &OdtDefaults, +) -> Result { + let mut cx = Ctx::new(meta, assets); + body::document(&mut cx, doc); + if cx.full() { + return Err(ExportError::TooLarge); + } + + // `content.xml` goes into the zip in pieces, never copied into one + // buffer. + let content = cx.content_xml(); + let meta = meta_xml(meta); + // Every other part but the mimetype and the manifest, as (path, media + // type, bytes); the manifest lists them after `content.xml`. + let mut parts: Vec<(&str, &str, &[u8])> = vec![ + ("styles.xml", "text/xml", defaults.styles_xml.as_bytes()), + ("meta.xml", "text/xml", meta.as_bytes()), + ]; + for (path, kind, bytes) in &cx.media { + parts.push((path, kind, bytes)); + } + let manifest = manifest_xml( + std::iter::once(("content.xml", "text/xml")) + .chain(parts.iter().map(|(path, kind, _)| (*path, *kind))), + ); + // Room for every part before compression, so the buffer never grows by + // copying itself. + let data_bytes = content.iter().map(|piece| piece.len()).sum::() + + parts.iter().map(|(_, _, bytes)| bytes.len()).sum::() + + manifest.len(); + let mut zip = ZipOut::with_capacity(data_bytes); + // First and uncompressed, so the file type can be read at a fixed offset. + zip.stored("mimetype", MIMETYPE.as_bytes())?; + zip.deflated_pieces("content.xml", &content)?; + for (path, _, bytes) in &parts { + zip.deflated(path, bytes)?; + } + zip.deflated("META-INF/manifest.xml", manifest.as_bytes())?; + Ok(Output { + bytes: zip.finish()?, + fallbacks: cx.fallbacks, + }) +} + +/// The table cell the block being written is in. +#[derive(Clone, Copy)] +struct CellCx { + header: bool, + /// `fo:text-align` of its paragraphs. + align: &'static str, +} + +/// Writer state for one document. +struct Ctx<'a> { + meta: &'a Meta, + assets: &'a Assets, + /// The `office:text` content written so far. + body: String, + /// The automatic styles `body` uses, for `content.xml`. + auto_styles: String, + fallbacks: Fallbacks, + /// Whether the last thing written in the open paragraph is a character + /// other than white space, so a space may follow as character data. + after_char: bool, + /// How many quotes enclose the block being written, in its table cell. + quote_depth: u32, + /// The lists open around the block being written, outermost first, in + /// its table cell. + lists: Vec, + /// The table cell the block being written is in. + cell: Option, + /// Last `xml:id` number given to an outermost list. + last_list_id: u32, + /// Last table number; it names the table and its styles. + last_table: u32, + /// Automatic text style per set of run marks. + text_styles: HashMap, + /// Automatic paragraph style per (parent style, alignment), for cells. + para_styles: HashMap<(&'static str, &'static str), String>, + /// Whether the table cell styles are in `auto_styles` yet. + cell_styles: bool, + /// Whether the picture frame style is in `auto_styles` yet. + frame_style: bool, + /// Last picture frame number; it names the frame. + last_frame: u32, + /// The picture parts as (path, media type, bytes), in order of first use. + /// The bytes are borrowed from `assets`, never copied. + media: Vec<(String, &'static str, &'a [u8])>, + /// Part number and extension per picture key, so a picture used twice is + /// stored once. + media_index: HashMap, +} + +impl<'a> Ctx<'a> { + fn new(meta: &'a Meta, assets: &'a Assets) -> Self { + Self { + meta, + assets, + body: String::new(), + auto_styles: String::new(), + fallbacks: Fallbacks::default(), + after_char: false, + quote_depth: 0, + lists: Vec::new(), + cell: None, + last_list_id: 0, + last_table: 0, + text_styles: HashMap::new(), + para_styles: HashMap::new(), + cell_styles: false, + frame_style: false, + last_frame: 0, + media: Vec::new(), + media_index: HashMap::new(), + } + } + + /// The picture part of `key`, stored on first use, as (number, + /// extension). + fn media_part(&mut self, key: &str, pic: &'a Picture) -> (usize, &'static str) { + if let Some(&part) = self.media_index.get(key) { + return part; + } + let n = self.media.len() + 1; + let ext = picture::extension(pic.format); + self.media.push(( + format!("Pictures/img{n}.{ext}"), + picture::media_type(ext), + &pic.bytes, + )); + self.media_index.insert(key.to_owned(), (n, ext)); + (n, ext) + } + + /// Whether the XML written so far has passed [`MAX_XML_BYTES`]. Once it + /// has, the emitters stop writing and the export fails. + fn full(&self) -> bool { + self.body.len() + self.auto_styles.len() > MAX_XML_BYTES + } + + fn next_frame_name(&mut self) -> String { + self.last_frame += 1; + format!("Picture {}", self.last_frame) + } + + /// The automatic graphic style of picture frames, written on first use. + fn frame_style(&mut self) -> &'static str { + if !self.frame_style { + self.frame_style = true; + self.auto_styles.push_str(concat!( + r#""#, + r#""#, + )); + } + "fr1" + } + + /// The automatic text style carrying a run's bold, italic, underline and + /// strike marks; `None` when it has none. One style per combination. + fn text_style(&mut self, marks: &Marks) -> Option { + let key = u8::from(marks.bold) + | u8::from(marks.italic) << 1 + | u8::from(marks.underline) << 2 + | u8::from(marks.strike) << 3; + if key == 0 { + return None; + } + if let Some(name) = self.text_styles.get(&key) { + return Some(name.clone()); + } + let name = format!("T{}", self.text_styles.len() + 1); + self.auto_styles.push_str(&format!( + r#""); + self.text_styles.insert(key, name.clone()); + Some(name) + } + + /// The paragraph style to write for `base`: `base` itself, or inside a + /// table cell an automatic style based on it with the cell's alignment. + fn para_style(&mut self, base: &'static str) -> String { + let Some(cell) = self.cell else { + return base.to_owned(); + }; + let key = (base, cell.align); + if let Some(name) = self.para_styles.get(&key) { + return name.clone(); + } + let name = format!("P{}", self.para_styles.len() + 1); + self.auto_styles.push_str(&format!( + concat!( + r#""#, + r#""#, + ), + name = name, + base = base, + align = cell.align, + )); + self.para_styles.insert(key, name.clone()); + name + } + + /// `content.xml`, as the pieces it is made of in order. + fn content_xml(&self) -> [&[u8]; 5] { + [ + CONTENT_HEAD.as_bytes(), + self.auto_styles.as_bytes(), + b"", + self.body.as_bytes(), + b"", + ] + } +} + +const BOLD: &str = + r#" fo:font-weight="bold" style:font-weight-asian="bold" style:font-weight-complex="bold""#; +const ITALIC: &str = + r#" fo:font-style="italic" style:font-style-asian="italic" style:font-style-complex="italic""#; +const UNDERLINE: &str = r#" style:text-underline-style="solid" style:text-underline-width="auto" style:text-underline-color="font-color""#; +const STRIKE: &str = + r#" style:text-line-through-style="solid" style:text-line-through-type="single""#; + +/// `content.xml` up to its automatic styles. +const CONTENT_HEAD: &str = concat!( + r#""#, + r#""#, +); + +/// The title and the generator. No creator, so the exporter's name stays +/// out of forwarded files. +fn meta_xml(meta: &Meta) -> String { + format!( + concat!( + r#""#, + r#""#, + r#"knot{title}"#, + r#""#, + ), + title = esc_text(&meta.title), + ) +} + +/// The manifest: the package itself, then `parts` as (path, media type). +fn manifest_xml<'p>(parts: impl IntoIterator) -> String { + let mut s = String::from(concat!( + r#""#, + r#""#, + )); + s.push_str(&format!( + r#""# + )); + for (path, media_type) in parts { + s.push_str(&format!( + r#""#, + esc_attr(path), + esc_attr(media_type), + )); + } + s.push_str(""); + s +} diff --git a/crates/knot-export/src/odt/picture.rs b/crates/knot-export/src/odt/picture.rs new file mode 100644 index 00000000..318b8afa --- /dev/null +++ b/crates/knot-export/src/odt/picture.rs @@ -0,0 +1,163 @@ +//! Pictures, and what stands in for them: frames for embedded images and +//! rendered diagrams, captions for the ones that are missing, and links for +//! external images and attachments. Placeholders, wording and fallback +//! counting match the DOCX writer. + +use crate::assets::{ImageAsset, PicFormat, Picture}; +use crate::media::{base_url, fit_cm, human_size}; +use crate::model::ImageSource; +use crate::xml::{esc_attr, esc_text}; + +use super::body::{open_paragraph, text_paragraph}; +use super::{Ctx, TEXT_HEIGHT_CM}; + +/// Smallest frame side written, in cm: a zero size is invalid. +const MIN_CM: f64 = 0.001; + +/// Tallest frame, in cm: one that still fits on a page. The frame sits in +/// a Standard paragraph, whose 115% line height applies to its line too, +/// followed by a 0.282 cm bottom margin. About 21.2 cm. +const MAX_HEIGHT_CM: f64 = (TEXT_HEIGHT_CM - 0.282) / 1.15; + +/// A body image. Embedded ones become frames; the rest become a caption or +/// a link, and the fallback counts say which. +pub(super) fn image(cx: &mut Ctx, source: &ImageSource, alt: &str, avail_cm: f64) { + match source { + ImageSource::Blob(id) => { + let assets = cx.assets; + match assets.images.get(id) { + Some(ImageAsset::Embedded(pic)) => { + frame(cx, &format!("image:{id}"), pic, alt, avail_cm); + } + Some(ImageAsset::OverBudget) => { + cx.fallbacks.image_over_budget += 1; + let url = format!("{}/api/blobs/{id}", base_url(&cx.meta.base_url)); + open_paragraph(cx, "Caption"); + link(cx, "[Image too large to include]", &url); + cx.body.push_str(""); + } + Some(ImageAsset::Unavailable) | None => unavailable(cx, alt), + } + } + ImageSource::Unsupported => unavailable(cx, alt), + ImageSource::External(url) => { + open_paragraph(cx, "Standard"); + link(cx, if alt.is_empty() { url } else { alt }, url); + cx.body.push_str(""); + } + } +} + +/// A Mermaid block: the rendered diagram, or its source and a note. +pub(super) fn mermaid(cx: &mut Ctx, key: &str, source: &str, avail_cm: f64) { + let assets = cx.assets; + if let Some(pic) = assets.diagrams.get(key) { + frame( + cx, + &format!("diagram:{key}"), + pic, + "Mermaid diagram", + avail_cm, + ); + return; + } + cx.fallbacks.diagram += 1; + text_paragraph(cx, "Preformatted_20_Text", source); + text_paragraph(cx, "Caption", "Mermaid diagram (not included)"); +} + +/// A board block: the rendered diagram, or a note with the board's label. +pub(super) fn board(cx: &mut Ctx, key: &str, label: &str, avail_cm: f64) { + let descr = format!("Diagram: {label}"); + let assets = cx.assets; + if let Some(pic) = assets.diagrams.get(key) { + frame(cx, &format!("diagram:{key}"), pic, &descr, avail_cm); + return; + } + cx.fallbacks.diagram += 1; + text_paragraph(cx, "Caption", &format!("{descr} (not included)")); +} + +/// A line naming the file and its size, linked back to knot when the +/// attachment lives on this instance. +pub(super) fn attachment(cx: &mut Ctx, url: &str, name: &str, size: Option) { + let mut line = format!("\u{1F4CE} {name}"); + if let Some(size) = size { + line.push_str(&format!(" ({})", human_size(size))); + } + if url.starts_with('/') { + let target = format!("{}{url}", base_url(&cx.meta.base_url)); + open_paragraph(cx, "Standard"); + link(cx, &line, &target); + cx.body.push_str(""); + } else { + text_paragraph(cx, "Standard", &line); + } +} + +fn unavailable(cx: &mut Ctx, alt: &str) { + cx.fallbacks.image_unavailable += 1; + let text = if alt.is_empty() { + "[Image not available]".to_owned() + } else { + format!("[Image not available] \u{2014} {alt}") + }; + text_paragraph(cx, "Caption", &text); +} + +/// `text` as a hyperlink to `url`, in the open paragraph. +fn link(cx: &mut Ctx, text: &str, url: &str) { + cx.body.push_str(&format!( + r#"{}"#, + esc_attr(url), + esc_text(text), + )); + cx.after_char = true; +} + +/// An as-character frame holding `pic` in its own paragraph, at most +/// `avail_cm` wide and a page high. `key` identifies the picture, so one +/// used twice is stored once. +fn frame<'a>(cx: &mut Ctx<'a>, key: &str, pic: &'a Picture, descr: &str, avail_cm: f64) { + let (n, ext) = cx.media_part(key, pic); + let (w, h) = fit_cm(pic, avail_cm, MAX_HEIGHT_CM); + let (w, h) = (w.max(MIN_CM), h.max(MIN_CM)); + let name = cx.next_frame_name(); + let style = cx.frame_style(); + open_paragraph(cx, "Standard"); + cx.body.push_str(&format!( + concat!( + r#""#, + r#""#, + r#"{descr}"#, + ), + style = style, + name = esc_attr(&name), + w = w, + h = h, + n = n, + ext = ext, + descr = esc_text(descr), + )); + cx.after_char = false; +} + +/// File extension of a picture's part. JPEG is `jpeg`, as in the DOCX writer. +pub(super) fn extension(format: PicFormat) -> &'static str { + match format { + PicFormat::Png => "png", + PicFormat::Jpeg => "jpeg", + PicFormat::Gif => "gif", + } +} + +/// Media type of a picture's part, for the manifest. +pub(super) fn media_type(ext: &str) -> &'static str { + match ext { + "png" => "image/png", + "jpeg" => "image/jpeg", + _ => "image/gif", + } +} diff --git a/crates/knot-export/src/odt/styles.rs b/crates/knot-export/src/odt/styles.rs new file mode 100644 index 00000000..6634d534 --- /dev/null +++ b/crates/knot-export/src/odt/styles.rs @@ -0,0 +1,251 @@ +//! The built-in `styles.xml`: fonts, the named styles, the outline and list +//! styles, the page layout and the master page with its footer. A company +//! template would replace this part (spec §10), so it reaches the writer only +//! through [`super::OdtDefaults`]. +//! +//! Style names are LibreOffice's (`Heading_20_1` is "Heading 1"), so +//! LibreOffice, and a later template mapping, treats them as its own. + +use super::{LIST_INDENT_CM, LIST_LEVELS}; + +const HEAD: &str = concat!( + r#""#, + r#""#, + r#""#, + r#""#, + r#""#, + r#""#, +); + +/// The defaults and the title styles. Sizes and spacing follow the DOCX +/// styles (spec §7). +const TITLE_STYLES: &str = concat!( + r#""#, + r#""#, + r#""#, + r#""#, + r#""#, + r#""#, + r#""#, + r#""#, + r#""#, + r#""#, + r#""#, + r##""##, + r#""#, +); + +/// The other paragraph styles, then the character styles. +const BODY_STYLES: &str = concat!( + // The left margin is QUOTE_INDENT_CM in mod.rs. + r#""#, + r#""#, + r##""##, + r#""#, + r#""#, + r##""##, + r#""#, + r#""#, + r#""#, + r#""#, + r#""#, + r#""#, + r#""#, + r#""#, + r#""#, + r#""#, + r#""#, + r#""#, + r#""#, + r##""#, + r#""#, + r#""#, + r#""#, + r##""##, + r#""#, + r#""#, + r#""#, + r#""#, + r#""#, + // Character styles. + r#""#, + r#""##, + r#""#, + r#""#, + r##""#, + r#""#, +); + +/// Heading 1-6 as (font size, space before, space after). +const HEADING_LOOK: [(&str, &str, &str); 6] = [ + ("16pt", "0.635cm", "0.212cm"), + ("14pt", "0.494cm", "0.176cm"), + ("13pt", "0.423cm", "0.141cm"), + ("12pt", "0.353cm", "0.141cm"), + ("11pt", "0.353cm", "0.141cm"), + ("11pt", "0.353cm", "0.141cm"), +]; + +/// A4 portrait with 2.5 cm margins. The footer sits in the bottom margin: +/// 1.25 cm from the page edge, 0.6 cm high, 0.65 cm below the text, so the +/// text ends 2.5 cm from the edge, as in the DOCX. +const PAGE: &str = concat!( + r#""#, + r#""#, + r#""#, + r#""#, + r#""#, + r#""#, + r#"Page "#, + r#"1 of "#, + r#"1"#, + r#""#, + r#""#, +); + +/// The built-in `styles.xml`. +pub(super) fn builtin_xml() -> String { + const BULLETS: [&str; 3] = ["\u{2022}", "\u{25E6}", "\u{25AA}"]; + const NUMBER_END: &str = ""; + let mut s = String::from(HEAD); + s.push_str(""); + s.push_str(TITLE_STYLES); + headings(&mut s); + s.push_str(BODY_STYLES); + outline_style(&mut s); + list_style(&mut s, ("List_20_Bullet", "List Bullet"), true, |level| { + let bullet = BULLETS[(level - 1) as usize % BULLETS.len()]; + ( + format!( + r#""# + ), + "", + ) + }); + list_style( + &mut s, + ("Numbering_20_123", "Numbering 123"), + true, + |level| { + let open = format!( + r#""# + ); + (open, NUMBER_END) + }, + ); + // No label: a checklist item's box is part of its text. + list_style(&mut s, ("List_20_Check", "List Check"), false, |level| { + let open = + format!(r#""#); + (open, NUMBER_END) + }); + s.push_str(""); + s.push_str(PAGE); + s +} + +fn headings(out: &mut String) { + for (i, (size, before, after)) in HEADING_LOOK.iter().enumerate() { + let level = i + 1; + let italic = if level == 6 { + r#" fo:font-style="italic" style:font-style-asian="italic" style:font-style-complex="italic""# + } else { + "" + }; + out.push_str(&format!( + concat!( + r#""#, + r#""#, + r#""#, + r#""#, + ), + level = level, + before = before, + after = after, + size = size, + italic = italic, + )); + } +} + +/// Headings carry outline levels but no numbers. +fn outline_style(out: &mut String) { + out.push_str(r#""#); + for level in 1..=LIST_LEVELS { + out.push_str(&format!( + concat!( + r#""#, + r#""#, + r#""#, + r#""#, + ), + level = level, + )); + } + out.push_str(""); +} + +/// A list style `(name, display name)` with [`LIST_LEVELS`] levels, each +/// indented one [`LIST_INDENT_CM`] step further. `level` gives a level's +/// opening and closing tag. A label hangs half a step left of the text and a +/// tab follows it; without one (checklists) the text starts at the indent. +fn list_style( + out: &mut String, + (name, display): (&str, &str), + labelled: bool, + level: impl Fn(u32) -> (String, &'static str), +) { + out.push_str(&format!( + r#""# + )); + let (followed_by, text_indent) = if labelled { + ("listtab", format!("-{:.3}cm", LIST_INDENT_CM / 2.0)) + } else { + ("nothing", "0cm".to_owned()) + }; + for n in 1..=LIST_LEVELS { + let (open, close) = level(n); + out.push_str(&open); + out.push_str(&format!( + concat!( + r#""#, + r#""#, + r#""#, + ), + followed_by = followed_by, + indent = LIST_INDENT_CM * f64::from(n), + text_indent = text_indent, + )); + out.push_str(close); + } + out.push_str(""); +} diff --git a/crates/knot-export/src/odt/table.rs b/crates/knot-export/src/odt/table.rs new file mode 100644 index 00000000..40f279e7 --- /dev/null +++ b/crates/knot-export/src/odt/table.rs @@ -0,0 +1,189 @@ +//! Tables: a `table:table-column` per grid column, sized from the column +//! shares, and every row filled out to the grid. A span is +//! `number-columns-spanned` / `number-rows-spanned` on its cell plus a +//! `table:covered-table-cell` for each other grid position it covers. + +use crate::model::{Align, Cell, Table}; +use crate::tables::{Slot, grid_edges, header_rows, layout}; + +use super::{CellCx, Ctx, body}; + +/// Left and right padding of a table cell, in cm: what the content of a cell +/// loses of the cell's width on each side. The cell styles below use it. +const CELL_PADDING_CM: f64 = 0.19; +/// What the `style:rel-column-width`s of a table add up to. +const REL_WIDTH_TOTAL: f64 = 65535.0; + +const CELL: &str = "Cell"; +const HEADER_CELL: &str = "HeaderCell"; + +/// Thin grey borders on every cell, and shading on header cells. +const CELL_STYLES: &str = concat!( + r#""#, + r#""#, + r#""#, + r#""#, + r##""#, + r#""#, +); + +/// Write `t`, `avail_cm` wide. A table without columns or rows writes +/// nothing. +pub(super) fn table(cx: &mut Ctx, t: &Table, avail_cm: f64) { + let width = t.shares.len(); + if width == 0 || t.rows.is_empty() { + return; + } + // A list item can't hold a table: close the lists around it, and + // continue them after it. The table lines up with the text it + // interrupts. + let lists = body::close_lists(cx); + let indent_cm = body::indent_cm(cx, &lists); + // Widths in thousandths of a cm; every column gets at least one. + let total = ((avail_cm * 1000.0).round() as u32).max(width as u32); + let edges = grid_edges(&t.shares, total); + cx.last_table += 1; + let name = format!("Table{}", cx.last_table); + styles(cx, &name, &t.shares, &edges, indent_cm); + + cx.body.push_str(&format!( + r#""# + )); + for i in 1..=width { + cx.body.push_str(&format!( + r#""# + )); + } + // The header rows form a group of their own, which no row span leaves. + let heads = header_rows(t); + let rows = t.rows.len(); + let group_end = move |r| if r < heads { heads } else { rows }; + for (r, slots) in layout(t, group_end).enumerate() { + if cx.full() { + break; + } + if r == 0 && heads > 0 { + cx.body.push_str(""); + } + cx.body.push_str(""); + for slot in slots { + row_slot(cx, slot, &edges); + } + cx.body.push_str(""); + if r + 1 == heads { + cx.body.push_str(""); + } + } + cx.body.push_str(""); + body::reopen_lists(cx, lists); +} + +/// A length in thousandths of a cm, never zero (ODF lengths of tables and +/// columns must be positive). +fn cm(thousandths: u32) -> String { + let n = thousandths.max(1); + format!("{}.{:03}cm", n / 1000, n % 1000) +} + +/// The automatic styles of table `name`: its width and indent, each +/// column's absolute and relative width, and, once per document, the cell +/// styles. +fn styles(cx: &mut Ctx, name: &str, shares: &[f64], edges: &[u32], indent_cm: f64) { + let margin = if indent_cm > 0.0 { + format!(r#" fo:margin-left="{indent_cm:.3}cm""#) + } else { + String::new() + }; + cx.auto_styles.push_str(&format!( + concat!( + r#""#, + r#""#, + ), + name = name, + width = cm(edges[edges.len() - 1]), + margin = margin, + )); + for (i, (col, share)) in edges.windows(2).zip(shares).enumerate() { + let rel = ((share * REL_WIDTH_TOTAL).round() as u32).max(1); + cx.auto_styles.push_str(&format!( + concat!( + r#""#, + r#""#, + r#""#, + ), + name = name, + n = i + 1, + width = cm(col[1] - col[0]), + rel = rel, + )); + } + if !cx.cell_styles { + cx.auto_styles.push_str(CELL_STYLES); + cx.cell_styles = true; + } +} + +/// One slot of a row: a cell, covered cells where a row span from above +/// reaches into the row, or one empty cell spanning a gap. +fn row_slot(cx: &mut Ctx, slot: Slot, edges: &[u32]) { + match slot { + Slot::Cell { + cell: c, + pos, + span, + rowspan, + } => cell(cx, c, edges[pos + span] - edges[pos], span, rowspan), + Slot::Covered { span, .. } => covered(cx, span), + Slot::Gap { pos, end } => { + open_cell(cx, CELL, end - pos, 1); + cx.body.push_str(""); + covered(cx, end - pos - 1); + } + } +} + +/// The opening tag of a cell spanning `span` columns and `rows` rows. +fn open_cell(cx: &mut Ctx, style: &str, span: usize, rows: usize) { + cx.body + .push_str(&format!(r#" 1 { + cx.body + .push_str(&format!(r#" table:number-columns-spanned="{span}""#)); + } + if rows > 1 { + cx.body + .push_str(&format!(r#" table:number-rows-spanned="{rows}""#)); + } + cx.body.push('>'); +} + +/// A cell with `c`'s blocks, `thousandths` of a cm wide, then the covered +/// cells of the rest of its column span. +fn cell(cx: &mut Ctx, c: &Cell, thousandths: u32, span: usize, rows: usize) { + open_cell(cx, if c.header { HEADER_CELL } else { CELL }, span, rows); + // The content starts at the cell's edge: no quote around the table + // carries into it (the lists around it are closed already). + let outer = (cx.quote_depth, cx.cell); + cx.quote_depth = 0; + cx.cell = Some(CellCx { + header: c.header, + align: match c.align { + Align::Left => "start", + Align::Center => "center", + Align::Right => "end", + }, + }); + let avail_cm = (f64::from(thousandths) / 1000.0 - 2.0 * CELL_PADDING_CM).max(0.0); + body::blocks(cx, &c.blocks, avail_cm); + (cx.quote_depth, cx.cell) = outer; + cx.body.push_str(""); + covered(cx, span - 1); +} + +fn covered(cx: &mut Ctx, n: usize) { + for _ in 0..n { + cx.body.push_str(""); + } +} diff --git a/crates/knot-export/src/tables.rs b/crates/knot-export/src/tables.rs new file mode 100644 index 00000000..92691aba --- /dev/null +++ b/crates/knot-export/src/tables.rs @@ -0,0 +1,186 @@ +//! Table helpers shared by the writers: column shares, header detection, and +//! the layout of a table's cells on its grid. + +use std::collections::BTreeMap; + +use crate::model::{Cell, Table}; + +/// Committed column weights add up to this (mirrors the editor). +pub const WEIGHT_TOTAL: f64 = 1000.0; + +fn usable(w: Option) -> Option { + w.filter(|v| v.is_finite() && *v > 0.0) +} + +/// Fractions of the table width (summing to 1) for stored column weights. +/// Port of `resolveShares` in `web/src/features/editor/columnWidths.ts`. +pub fn resolve_shares(weights: &[Option]) -> Vec { + let known: Vec = weights.iter().filter_map(|w| usable(*w)).collect(); + let known_total: f64 = known.iter().sum(); + let unset = weights.len() - known.len(); + let leftover = WEIGHT_TOTAL - known_total; + let fallback = if known.is_empty() { + 1.0 + } else if leftover > 0.0 && unset > 0 { + leftover / unset as f64 + } else { + known_total / known.len() as f64 + }; + let filled: Vec = weights + .iter() + .map(|w| usable(*w).unwrap_or(fallback)) + .collect(); + let total: f64 = filled.iter().sum(); + filled.into_iter().map(|w| w / total).collect() +} + +/// Number of leading rows in which every cell is a header cell. +pub fn header_rows(table: &Table) -> usize { + table + .rows + .iter() + .take_while(|r| !r.is_empty() && r.iter().all(|c| c.header)) + .count() +} + +/// Column boundaries from the table's left edge, `shares.len() + 1` of them +/// from 0 to `total` (only `[0]` without columns), in whatever unit `total` +/// is in. Each column gets its share of `total`, rounded, and the last one +/// what is left, so they add up exactly. +pub fn grid_edges(shares: &[f64], total: u32) -> Vec { + let mut edges = Vec::with_capacity(shares.len() + 1); + edges.push(0); + let Some((_, init)) = shares.split_last() else { + return edges; + }; + let mut at = 0; + for share in init { + let width = (share * f64::from(total)).round() as u32; + at += width.min(total - at); + edges.push(at); + } + edges.push(total); + edges +} + +/// What fills a stretch of grid columns in one row of a laid-out table. +#[derive(Debug, Clone, Copy, PartialEq)] +pub enum Slot<'a> { + /// `cell` at grid column `pos`, spanning `span` columns and `rowspan` + /// rows, both clamped by [`layout`] and at least 1. + Cell { + cell: &'a Cell, + pos: usize, + span: usize, + rowspan: usize, + }, + /// Columns `pos..pos + span`, covered by the row span of a cell in a row + /// above. `header` is whether that cell is a header cell. + Covered { + pos: usize, + span: usize, + header: bool, + }, + /// Columns `pos..end`, which no cell fills. + Gap { pos: usize, end: usize }, +} + +/// A row span that still covers rows below the current one. +#[derive(Clone, Copy)] +struct RowSpan { + /// Grid columns it spans. + span: usize, + /// Rows it still covers, counting from the next row laid out. + rows_left: usize, + /// Whether it started in a header cell. + header: bool, +} + +/// Lay `t` out on its grid of `t.shares.len()` columns: for each row, the +/// slots that fill it left to right, so that every row has exactly the grid +/// width. +/// +/// - Each cell goes at its column, or at the next free one when a cell or +/// span before it already reaches past that column. A cell that starts +/// past the grid is dropped. +/// - A column span is clamped to the grid and stops short of a row span +/// from above. +/// - A row span is clamped to the rows from its row up to `group_end(row)`, +/// the end (exclusive row index) of the row group it starts in: the end +/// of the table, or of the header rows when they form a group of their +/// own. No row span is less than 1, whatever `group_end` returns. +/// - Each gap is one slot, however wide, so a crafted wide grid can't +/// multiply the output. +pub fn layout<'a>( + t: &'a Table, + group_end: impl Fn(usize) -> usize + 'a, +) -> impl Iterator>> + 'a { + let width = t.shares.len(); + // Open row spans by their first grid column. They never overlap. + let mut spans = BTreeMap::new(); + t.rows.iter().enumerate().map(move |(r, cells)| { + let rows_to_end = group_end(r).saturating_sub(r).max(1); + layout_row(cells, width, rows_to_end, &mut spans) + }) +} + +/// One row of [`layout`], `rows_to_end` (at least 1) rows from the end of +/// its group, given the row spans reaching into it from above. +fn layout_row<'a>( + cells: &'a [Cell], + width: usize, + rows_to_end: usize, + spans: &mut BTreeMap, +) -> Vec> { + let mut slots = Vec::new(); + let mut opened = Vec::new(); + let mut cells = cells.iter().peekable(); + let mut pos = 0; + while pos < width { + let next_span = spans.range(pos..).next().map(|(col, s)| (*col, *s)); + if let Some((_, s)) = next_span.filter(|(col, _)| *col == pos) { + slots.push(Slot::Covered { + pos, + span: s.span, + header: s.header, + }); + pos += s.span; + continue; + } + // The free columns run from `pos` up to the next row span, which + // starts after `pos`. + let limit = next_span.map_or(width, |(col, _)| col); + if let Some(cell) = cells.next_if(|c| c.col <= pos) { + let span = cell.colspan.clamp(1, limit - pos); + let rowspan = cell.rowspan.clamp(1, rows_to_end); + if rowspan > 1 { + opened.push(( + pos, + RowSpan { + span, + rows_left: rowspan - 1, + header: cell.header, + }, + )); + } + slots.push(Slot::Cell { + cell, + pos, + span, + rowspan, + }); + pos += span; + } else { + // The next cell, if there is one, starts after `pos`. + let end = cells.peek().map_or(limit, |c| c.col.min(limit)); + slots.push(Slot::Gap { pos, end }); + pos = end; + } + } + spans.retain(|_, s| { + s.rows_left -= 1; + s.rows_left > 0 + }); + spans.extend(opened); + slots +} diff --git a/crates/knot-export/src/walk.rs b/crates/knot-export/src/walk.rs new file mode 100644 index 00000000..42bd3ebf --- /dev/null +++ b/crates/knot-export/src/walk.rs @@ -0,0 +1,690 @@ +//! Walks the canonical "default" XmlFragment of a Y.Doc into an [`ExportDoc`]. +//! +//! The walk never panics and never fails: anything it does not understand +//! becomes a plain paragraph and is reported in [`Walked::unknown_elements`]. + +use sha2::{Digest, Sha256}; +use url::Url; +use yrs::types::text::YChange; +use yrs::{Any, Out, ReadTxn, Text, Transact, Xml, XmlFragment, XmlOut, XmlTextRef}; + +use crate::model::{ + Align, Block, Cell, ExportDoc, ImageSource, Inline, LinkTarget, List, ListItem, Marks, Run, + Table, +}; +use crate::tables::resolve_shares; + +/// Maximum nesting of block containers (lists, list items, quotes, tables, +/// cells, unknown elements). Deeper subtrees are flattened to paragraphs so a +/// crafted page cannot overflow the stack. +pub const MAX_DEPTH: usize = 64; + +/// Widest table (in grid columns) that stays a table. +pub const MAX_TABLE_COLUMNS: usize = 64; + +/// Largest table (grid width times row count) that stays a table. Writers +/// emit output per grid position, so this bounds their output. +pub const MAX_TABLE_GRID_CELLS: usize = 50_000; + +/// Grid positions all the tables of a page may have together. A table past +/// it degrades like one over [`MAX_TABLE_GRID_CELLS`]. Writers emit output +/// per grid position (a covered or merge-continuation cell even in an empty +/// row) and per column, so each table counts as +/// `width * (rows + TABLE_COLUMN_ROWS)`. That keeps what a page of many +/// tables can make a writer emit to about 16 MB of XML. +pub const MAX_PAGE_GRID_CELLS: usize = 200_000; + +/// Rows a table's columns count for against [`MAX_PAGE_GRID_CELLS`], on top +/// of its own: an ODT column's style and element weigh about as much as +/// eight covered cells. +pub const TABLE_COLUMN_ROWS: usize = 8; + +/// Largest `colspan` or `rowspan`; bigger values are clamped. +pub const MAX_SPAN: usize = 64; + +/// Result of walking a document. +#[derive(Debug, Clone, PartialEq)] +pub struct Walked { + pub doc: ExportDoc, + /// Tags of block elements the walker did not recognise, in document order. + pub unknown_elements: Vec, +} + +/// What a walk keeps track of across the whole page. +#[derive(Default)] +struct Tally { + /// See [`Walked::unknown_elements`]. + unknown: Vec, + /// What the tables kept so far count for against + /// [`MAX_PAGE_GRID_CELLS`]. + grid_cells: usize, +} + +pub fn walk(doc: &yrs::Doc) -> Walked { + let txn = doc.transact(); + let mut tally = Tally::default(); + let blocks = match txn.get_xml_fragment("default") { + Some(frag) => walk_blocks(&frag, &txn, &mut tally, 0), + None => Vec::new(), + }; + Walked { + doc: ExportDoc { blocks }, + unknown_elements: tally.unknown, + } +} + +/// Read an attribute as a string. Handles the editor's numbers and the +/// import's strings alike. +fn attr(el: &X, txn: &T, name: &str) -> Option { + el.get_attribute(txn, name).map(|v| v.to_string(txn)) +} + +/// Walk the children of a block container. Raw text or `hard_break` children +/// (the import's soft-break shape) are repaired into paragraphs; consecutive +/// ones share a paragraph. +fn walk_blocks( + parent: &F, + txn: &T, + tally: &mut Tally, + depth: usize, +) -> Vec { + if depth > MAX_DEPTH { + tally.unknown.push("depth-limit".to_owned()); + return capped_paragraphs(parent, txn); + } + let mut out = Vec::new(); + let mut stray: Vec = Vec::new(); + let mut last_href: Option = None; + for i in 0..parent.len(txn) { + match parent.get(txn, i) { + Some(XmlOut::Text(t)) => push_text(&mut stray, &mut last_href, &t, txn), + Some(XmlOut::Element(el)) if el.tag().as_ref() == "hard_break" => { + last_href = None; + stray.push(Inline::Break); + } + Some(XmlOut::Element(el)) => { + flush_stray(&mut stray, &mut out); + last_href = None; + out.extend(walk_element(&el, txn, tally, depth)); + } + Some(XmlOut::Fragment(f)) => { + flush_stray(&mut stray, &mut out); + last_href = None; + out.extend(walk_blocks(&f, txn, tally, depth + 1)); + } + None => {} + } + } + flush_stray(&mut stray, &mut out); + out +} + +fn flush_stray(stray: &mut Vec, out: &mut Vec) { + if !stray.is_empty() { + out.push(Block::Paragraph(std::mem::take(stray))); + } +} + +fn walk_list( + el: &yrs::XmlElementRef, + ordered: bool, + txn: &T, + tally: &mut Tally, + depth: usize, +) -> Block { + let start = if ordered { + attr(el, txn, "start") + .and_then(|s| s.trim().parse::().ok()) + .filter(|n| n.is_finite()) + .map(|n| n.round().clamp(1.0, f64::from(u32::MAX)) as u32) + .unwrap_or(1) + } else { + 1 + }; + let mut items = Vec::new(); + for i in 0..el.len(txn) { + match el.get(txn, i) { + Some(XmlOut::Element(item)) if item.tag().as_ref() == "list_item" => { + items.push(ListItem { + checked: attr(&item, txn, "checked").map(|v| v == "true"), + blocks: walk_blocks(&item, txn, tally, depth + 2), + }); + } + Some(XmlOut::Element(other)) => { + // Not a list_item: keep it as an item of its own. + items.push(ListItem { + checked: None, + blocks: walk_element(&other, txn, tally, depth + 1), + }); + } + _ => {} + } + } + Block::List(List { + ordered, + start, + items, + }) +} + +/// A span attribute (`colspan`/`rowspan`): editor number or import string. +/// Anything missing, non-numeric or below 1 counts as 1. +fn span(el: &yrs::XmlElementRef, txn: &T, name: &str) -> usize { + attr(el, txn, name) + .and_then(|s| s.trim().parse::().ok()) + .filter(|n| n.is_finite()) + .map(|n| n.round().clamp(1.0, MAX_SPAN as f64) as usize) + .unwrap_or(1) +} + +/// The `colwidth` attribute: an array of numbers and nulls. +fn colwidth(el: &yrs::XmlElementRef, txn: &T) -> Vec> { + match el.get_attribute(txn, "colwidth") { + Some(Out::Any(Any::Array(items))) => items + .iter() + .map(|a| match a { + Any::Number(n) => Some(*n), + _ => None, + }) + .collect(), + _ => Vec::new(), + } +} + +fn walk_table( + el: &yrs::XmlElementRef, + txn: &T, + tally: &mut Tally, + depth: usize, +) -> Vec { + // Rows still occupied below the current one, per grid column. + let mut carry: Vec = Vec::new(); + let mut width = 0; + let mut rows: Vec> = Vec::new(); + let mut weights_by_col: Vec> = Vec::new(); + for i in 0..el.len(txn) { + let Some(XmlOut::Element(row)) = el.get(txn, i) else { + continue; + }; + let blocked: Vec = carry.iter().map(|&n| n > 0).collect(); + for n in carry.iter_mut() { + *n = n.saturating_sub(1); + } + let first_row = rows.is_empty(); + let mut col = 0; + let mut cells = Vec::new(); + for j in 0..row.len(txn) { + let Some(XmlOut::Element(c)) = row.get(txn, j) else { + continue; + }; + while blocked.get(col).copied().unwrap_or(false) { + col += 1; + } + let colspan = span(&c, txn, "colspan"); + let rowspan = span(&c, txn, "rowspan"); + if carry.len() < col + colspan { + carry.resize(col + colspan, 0); + } + for n in &mut carry[col..col + colspan] { + *n = (*n).max(rowspan - 1); + } + if first_row { + let cw = colwidth(&c, txn); + weights_by_col.resize(col + colspan, None); + for k in 0..colspan { + weights_by_col[col + k] = cw.get(k).copied().flatten(); + } + } + let align = match attr(&c, txn, "align").as_deref() { + Some("center") => Align::Center, + Some("right") => Align::Right, + _ => Align::Left, + }; + cells.push(Cell { + col, + colspan, + rowspan, + header: c.tag().as_ref() == "table_header", + align, + blocks: walk_blocks(&c, txn, tally, depth + 2), + }); + col += colspan; + } + width = width.max(col).max(carry.len()); + rows.push(cells); + } + let grid_cells = width.saturating_mul(rows.len()); + let page_cells = tally + .grid_cells + .saturating_add(width.saturating_mul(rows.len().saturating_add(TABLE_COLUMN_ROWS))); + if width > MAX_TABLE_COLUMNS + || grid_cells > MAX_TABLE_GRID_CELLS + || page_cells > MAX_PAGE_GRID_CELLS + { + // Too big to keep as a table: keep the text, one paragraph per row. + tally.unknown.push("table-limit".to_owned()); + let paragraphs = rows + .iter() + .map(|cells| { + let texts: Vec = cells.iter().map(|c| plain_text(&c.blocks)).collect(); + Block::Paragraph(vec![Inline::Run(Run { + text: texts.join(" | "), + marks: Marks::default(), + })]) + }) + .collect(); + return paragraphs; + } + tally.grid_cells = page_cells; + weights_by_col.resize(width, None); + vec![Block::Table(Table { + shares: resolve_shares(&weights_by_col), + rows, + })] +} + +/// The text of `blocks`, one space between blocks and breaks. +fn plain_text(blocks: &[Block]) -> String { + fn runs(out: &mut Vec, content: &[Inline]) { + let mut s = String::new(); + for inline in content { + match inline { + Inline::Run(r) => s.push_str(&r.text), + Inline::Link { runs, .. } => runs.iter().for_each(|r| s.push_str(&r.text)), + Inline::Break => s.push(' '), + } + } + out.push(s); + } + let mut parts = Vec::new(); + visit_blocks(blocks, &mut |b| match b { + Block::Paragraph(content) | Block::Heading { content, .. } => runs(&mut parts, content), + Block::Code { text, .. } => parts.push(text.clone()), + Block::Mermaid { source, .. } => parts.push(source.clone()), + _ => {} + }); + parts.retain(|p| !p.is_empty()); + parts.join(" ") +} + +/// One element as blocks: a table over the size limits becomes several +/// paragraphs, everything else exactly one block. +fn walk_element( + el: &yrs::XmlElementRef, + txn: &T, + tally: &mut Tally, + depth: usize, +) -> Vec { + if depth <= MAX_DEPTH && el.tag().as_ref() == "table" { + return walk_table(el, txn, tally, depth + 1); + } + vec![walk_single(el, txn, tally, depth)] +} + +fn walk_single( + el: &yrs::XmlElementRef, + txn: &T, + tally: &mut Tally, + depth: usize, +) -> Block { + if depth > MAX_DEPTH { + // Reached through list/element recursion that never passes through + // walk_blocks or inlines; cap here too. + tally.unknown.push("depth-limit".to_owned()); + return Block::Paragraph(vec![Inline::Run(Run { + text: flatten_text(el, txn), + marks: Marks::default(), + })]); + } + let tag = el.tag().to_string(); + match tag.as_str() { + "paragraph" => Block::Paragraph(inlines(el, txn, depth)), + "heading" => { + let level = attr(el, txn, "level") + .and_then(|s| s.trim().parse::().ok()) + .filter(|n| n.is_finite()) + .map(|n| n.round().clamp(1.0, 6.0) as u8) + .unwrap_or(1); + Block::Heading { + level, + content: inlines(el, txn, depth), + } + } + "blockquote" => Block::Quote(walk_blocks(el, txn, tally, depth + 1)), + "code_block" => { + let language = attr(el, txn, "language").unwrap_or_default(); + let text = flatten_text(el, txn); + if language.trim().eq_ignore_ascii_case("mermaid") { + let key = format!("mermaid-{:x}", Sha256::digest(text.as_bytes())); + Block::Mermaid { key, source: text } + } else { + Block::Code { language, text } + } + } + "horizontal_rule" => Block::Rule, + "bullet_list" => walk_list(el, false, txn, tally, depth + 1), + "ordered_list" => walk_list(el, true, txn, tally, depth + 1), + "image" => { + let src = attr(el, txn, "src").unwrap_or_default(); + Block::Image { + source: classify_image(&src), + alt: attr(el, txn, "alt").unwrap_or_default(), + } + } + "excalidraw_board" => { + let board_id = attr(el, txn, "board_id").unwrap_or_default(); + let label = attr(el, txn, "label") + .filter(|l| !l.trim().is_empty()) + .unwrap_or_else(|| "Diagram".to_owned()); + Block::Board { + key: format!("board-{board_id}"), + board_id, + label, + } + } + "attachment" => Block::Attachment { + url: attr(el, txn, "url").unwrap_or_default(), + name: attr(el, txn, "name").unwrap_or_default(), + size: attr(el, txn, "size") + .and_then(|s| s.trim().parse::().ok()) + .filter(|n| n.is_finite() && *n > 0.0) + .map(|n| n.round() as u64), + }, + _ => { + tally.unknown.push(tag); + Block::Paragraph(vec![Inline::Run(Run { + text: flatten_text(el, txn), + marks: Marks::default(), + })]) + } + } +} + +/// Inline content of a textblock. Unknown inline elements keep their text. +fn inlines(parent: &F, txn: &T, depth: usize) -> Vec { + if depth > MAX_DEPTH { + return vec![Inline::Run(Run { + text: flatten_text(parent, txn), + marks: Marks::default(), + })]; + } + let mut out = Vec::new(); + let mut last_href: Option = None; + for i in 0..parent.len(txn) { + match parent.get(txn, i) { + Some(XmlOut::Text(t)) => push_text(&mut out, &mut last_href, &t, txn), + Some(XmlOut::Element(el)) => { + last_href = None; + if el.tag().as_ref() == "hard_break" { + out.push(Inline::Break); + } else { + let text = flatten_text(&el, txn); + if !text.is_empty() { + out.push(Inline::Run(Run { + text, + marks: Marks::default(), + })); + } + } + } + Some(XmlOut::Fragment(f)) => { + last_href = None; + out.extend(inlines(&f, txn, depth + 1)); + } + None => {} + } + } + out +} + +fn push_text( + out: &mut Vec, + last_href: &mut Option, + t: &XmlTextRef, + txn: &T, +) { + for d in t.diff(txn, YChange::identity) { + let text = match d.insert { + Out::Any(Any::String(s)) => s.to_string(), + _ => continue, + }; + let attrs = d.attributes.as_deref(); + let has = |k: &str| attrs.is_some_and(|m| m.contains_key(k)); + let run = Run { + text, + marks: Marks { + bold: has("bold"), + italic: has("italic"), + underline: has("underline"), + strike: has("strike"), + code: has("code"), + }, + }; + let href = attrs.and_then(|m| match m.get("link") { + Some(Any::Map(link)) => match link.get("href") { + Some(Any::String(s)) => Some(s.to_string()), + _ => None, + }, + _ => None, + }); + match href { + None => { + *last_href = None; + out.push(Inline::Run(run)); + } + Some(h) => { + if last_href.as_deref() == Some(h.as_str()) + && let Some(Inline::Link { runs, .. }) = out.last_mut() + { + runs.push(run); + } else { + out.push(Inline::Link { + target: classify(&h), + runs: vec![run], + }); + *last_href = Some(h); + } + } + } + } +} + +/// A UUID in any accepted spelling (hyphenated, simple, braced, urn) as +/// lower-case hyphenated. +fn canonical_id(s: &str) -> Option { + uuid::Uuid::parse_str(s) + .ok() + .map(|u| u.hyphenated().to_string()) +} + +fn classify_image(src: &str) -> ImageSource { + if let Some(id) = src.strip_prefix("/api/blobs/") + && let Some(id) = canonical_id(id) + { + return ImageSource::Blob(id); + } + match Url::parse(src) { + Ok(u) if matches!(u.scheme(), "http" | "https") => ImageSource::External(src.to_owned()), + _ => ImageSource::Unsupported, + } +} + +fn classify(href: &str) -> LinkTarget { + if let Some(rest) = href.strip_prefix("knot://doc/") { + return canonical_id(rest).map_or(LinkTarget::Plain, LinkTarget::Doc); + } + if let Some(rest) = href.strip_prefix("knot://user/") { + return canonical_id(rest).map_or(LinkTarget::Plain, LinkTarget::User); + } + match Url::parse(href) { + Ok(u) if matches!(u.scheme(), "http" | "https" | "mailto") => { + LinkTarget::Web(u.as_str().to_owned()) + } + _ => LinkTarget::Plain, + } +} + +/// Text of one `XmlText` node, mark-free. +fn text_of(t: &XmlTextRef, txn: &T) -> String { + let mut s = String::new(); + for d in t.diff(txn, YChange::identity) { + if let Out::Any(Any::String(x)) = d.insert { + s.push_str(&x); + } + } + s +} + +/// Text nodes below `node` in document order. Iterative (explicit stack), so +/// it is safe at any depth. +fn text_nodes(node: &F, txn: &T) -> Vec { + let mut out = Vec::new(); + let mut stack: Vec = (0..node.len(txn)) + .rev() + .filter_map(|i| node.get(txn, i)) + .collect(); + while let Some(n) = stack.pop() { + match n { + XmlOut::Text(t) => out.push(text_of(&t, txn)), + XmlOut::Element(e) => { + stack.extend((0..e.len(txn)).rev().filter_map(|i| e.get(txn, i))); + } + XmlOut::Fragment(f) => { + stack.extend((0..f.len(txn)).rev().filter_map(|i| f.get(txn, i))); + } + } + } + out +} + +/// All plain text below `node`, concatenated (mark-free). +fn flatten_text(node: &F, txn: &T) -> String { + text_nodes(node, txn).concat() +} + +/// A too-deep subtree as one paragraph per non-empty text node. +fn capped_paragraphs(node: &F, txn: &T) -> Vec { + text_nodes(node, txn) + .into_iter() + .filter(|t| !t.is_empty()) + .map(|text| { + Block::Paragraph(vec![Inline::Run(Run { + text, + marks: Marks::default(), + })]) + }) + .collect() +} + +/// A diagram the server has to render before writing the document. +#[derive(Debug, Clone, PartialEq)] +pub enum DiagramRef { + Mermaid { + key: String, + source: String, + }, + Board { + key: String, + board_id: String, + label: String, + }, +} + +/// Calls `f` for every block, depth first in document order. +fn visit_blocks<'a>(blocks: &'a [Block], f: &mut impl FnMut(&'a Block)) { + for b in blocks { + f(b); + match b { + Block::Quote(inner) => visit_blocks(inner, f), + Block::List(list) => { + for item in &list.items { + visit_blocks(&item.blocks, f); + } + } + Block::Table(table) => { + for cell in table.rows.iter().flatten() { + visit_blocks(&cell.blocks, f); + } + } + _ => {} + } + } +} + +fn push_unique(out: &mut Vec, id: &str) { + if !out.iter().any(|x| x == id) { + out.push(id.to_owned()); + } +} + +/// Mermaid blocks and boards in document order, once per key. +pub fn diagrams(doc: &ExportDoc) -> Vec { + let mut out: Vec = Vec::new(); + visit_blocks(&doc.blocks, &mut |b| { + let (r, key) = match b { + Block::Mermaid { key, source } => ( + DiagramRef::Mermaid { + key: key.clone(), + source: source.clone(), + }, + key, + ), + Block::Board { + key, + board_id, + label, + } => ( + DiagramRef::Board { + key: key.clone(), + board_id: board_id.clone(), + label: label.clone(), + }, + key, + ), + _ => return, + }; + let seen = out.iter().any(|d| match d { + DiagramRef::Mermaid { key: k, .. } | DiagramRef::Board { key: k, .. } => k == key, + }); + if !seen { + out.push(r); + } + }); + out +} + +/// Blob ids of all images stored in knot, in document order, once each. +pub fn blob_ids(doc: &ExportDoc) -> Vec { + let mut out = Vec::new(); + visit_blocks(&doc.blocks, &mut |b| { + if let Block::Image { + source: ImageSource::Blob(id), + .. + } = b + { + push_unique(&mut out, id); + } + }); + out +} + +/// Ids of all mentioned users, in document order, once each. +pub fn user_ids(doc: &ExportDoc) -> Vec { + let mut out = Vec::new(); + visit_blocks(&doc.blocks, &mut |b| { + let inlines = match b { + Block::Paragraph(i) | Block::Heading { content: i, .. } => i, + _ => return, + }; + for i in inlines { + if let Inline::Link { + target: LinkTarget::User(id), + .. + } = i + { + push_unique(&mut out, id); + } + } + }); + out +} diff --git a/crates/knot-export/src/xml.rs b/crates/knot-export/src/xml.rs new file mode 100644 index 00000000..1bf2123a --- /dev/null +++ b/crates/knot-export/src/xml.rs @@ -0,0 +1,124 @@ +//! XML escaping and the in-memory zip writer shared by the DOCX and ODT writers. + +use std::io::{Cursor, Write}; + +use zip::CompressionMethod; +use zip::ZipWriter; +use zip::write::SimpleFileOptions; + +use crate::assets::ExportError; + +/// Most XML one document may write (64 MiB). The walker's limits keep real +/// pages far below it; a writer that passes it stops and fails with +/// [`ExportError::TooLarge`] rather than run the server out of memory. +pub const MAX_XML_BYTES: usize = 64 * 1024 * 1024; + +/// Room in the zip buffer beyond its entries' data: headers, the central +/// directory, and deflate's overhead on data that doesn't compress. +const ZIP_SLACK_BYTES: usize = 64 * 1024; + +fn invalid_xml_char(c: char) -> bool { + matches!(c, + '\u{0}'..='\u{8}' | '\u{B}' | '\u{C}' | '\u{E}'..='\u{1F}' + | '\u{7F}'..='\u{9F}' | '\u{FFFE}' | '\u{FFFF}') +} + +fn esc(s: &str, attr: bool) -> String { + let mut out = String::with_capacity(s.len()); + for c in s.chars() { + if invalid_xml_char(c) { + continue; + } + match c { + '&' => out.push_str("&"), + '<' => out.push_str("<"), + '>' => out.push_str(">"), + '"' if attr => out.push_str("""), + '\'' if attr => out.push_str("'"), + // Attribute-value normalisation would turn these into spaces. + '\t' if attr => out.push_str(" "), + '\n' if attr => out.push_str(" "), + '\r' if attr => out.push_str(" "), + _ => out.push(c), + } + } + out +} + +/// Escape character data, dropping characters XML 1.0 forbids. +pub fn esc_text(s: &str) -> String { + esc(s, false) +} + +/// Escape an attribute value, dropping characters XML 1.0 forbids. Tabs and +/// line breaks become character references so they survive parsing. +pub fn esc_attr(s: &str) -> String { + esc(s, true) +} + +/// A zip archive built in memory. Entries are written in call order. +pub struct ZipOut { + zip: ZipWriter>>, +} + +impl Default for ZipOut { + fn default() -> Self { + Self::new() + } +} + +impl ZipOut { + pub fn new() -> Self { + Self::with_capacity(0) + } + + /// A zip whose entries hold about `data_bytes` in all. The buffer is + /// sized for them up front, so a large package is never copied while + /// it grows. + pub fn with_capacity(data_bytes: usize) -> Self { + let buf = Vec::with_capacity(data_bytes.saturating_add(ZIP_SLACK_BYTES)); + Self { + zip: ZipWriter::new(Cursor::new(buf)), + } + } + + fn add( + &mut self, + name: &str, + pieces: &[&[u8]], + method: CompressionMethod, + ) -> Result<(), ExportError> { + let opts = SimpleFileOptions::default().compression_method(method); + self.zip + .start_file(name, opts) + .map_err(|e| ExportError::Zip(e.to_string()))?; + for piece in pieces { + self.zip + .write_all(piece) + .map_err(|e| ExportError::Io(e.to_string()))?; + } + Ok(()) + } + + /// Store without compression (the ODT mimetype entry, and media files). + pub fn stored(&mut self, name: &str, data: &[u8]) -> Result<(), ExportError> { + self.add(name, &[data], CompressionMethod::Stored) + } + + pub fn deflated(&mut self, name: &str, data: &[u8]) -> Result<(), ExportError> { + self.add(name, &[data], CompressionMethod::Deflated) + } + + /// One deflated entry holding `pieces` one after another, so a part + /// built in pieces is never copied into one buffer first. + pub fn deflated_pieces(&mut self, name: &str, pieces: &[&[u8]]) -> Result<(), ExportError> { + self.add(name, pieces, CompressionMethod::Deflated) + } + + pub fn finish(self) -> Result, ExportError> { + self.zip + .finish() + .map(Cursor::into_inner) + .map_err(|e| ExportError::Zip(e.to_string())) + } +} diff --git a/crates/knot-export/tests/common/docx.rs b/crates/knot-export/tests/common/docx.rs new file mode 100644 index 00000000..2dc2b993 --- /dev/null +++ b/crates/knot-export/tests/common/docx.rs @@ -0,0 +1,363 @@ +//! DOCX-specific roxmltree helpers shared by the DOCX test files. + +use std::collections::HashMap; + +use roxmltree::{Document, Node}; + +use super::xml; + +pub const W: &str = "http://schemas.openxmlformats.org/wordprocessingml/2006/main"; +pub const PKG_REL: &str = "http://schemas.openxmlformats.org/package/2006/relationships"; + +/// The parts of an exported package. +pub struct Docx(pub HashMap>); + +impl Docx { + pub fn part(&self, name: &str) -> &str { + let bytes = self + .0 + .get(name) + .unwrap_or_else(|| panic!("missing part {name}")); + std::str::from_utf8(bytes).expect("UTF-8 part") + } +} + +pub struct Rel { + pub kind: String, + pub target: String, + pub external: bool, +} + +/// `word/_rels/document.xml.rels` by relationship id. +pub fn rels(docx: &Docx) -> HashMap { + let d = xml(docx.part("word/_rels/document.xml.rels")); + d.descendants() + .filter(|n| n.has_tag_name((PKG_REL, "Relationship"))) + .map(|n| { + let rel = Rel { + kind: n.attribute("Type").unwrap().to_owned(), + target: n.attribute("Target").unwrap().to_owned(), + external: n.attribute("TargetMode") == Some("External"), + }; + (n.attribute("Id").unwrap().to_owned(), rel) + }) + .collect() +} + +pub fn elements<'a, 'i>(n: Node<'a, 'i>) -> Vec> { + n.children().filter(Node::is_element).collect() +} + +/// First `w:` child element with this local name. +pub fn child<'a, 'i>(n: Node<'a, 'i>, name: &str) -> Option> { + n.children().find(|c| c.has_tag_name((W, name))) +} + +pub fn wattr<'a>(n: Node<'a, '_>, name: &str) -> Option<&'a str> { + n.attribute((W, name)) +} + +pub fn val<'a>(n: Node<'a, '_>) -> Option<&'a str> { + wattr(n, "val") +} + +pub fn p_style<'a>(p: Node<'a, '_>) -> Option<&'a str> { + child(p, "pPr") + .and_then(|ppr| child(ppr, "pStyle")) + .and_then(val) +} + +pub fn r_style<'a>(r: Node<'a, '_>) -> Option<&'a str> { + child(r, "rPr") + .and_then(|rpr| child(rpr, "rStyle")) + .and_then(val) +} + +/// Concatenated `w:t` text below a node. +pub fn text(n: Node) -> String { + n.descendants() + .filter(|d| d.has_tag_name((W, "t"))) + .filter_map(|t| t.text()) + .collect() +} + +pub fn body<'a, 'i>(d: &'a Document<'i>) -> Node<'a, 'i> { + d.descendants() + .find(|n| n.has_tag_name((W, "body"))) + .expect("w:body") +} + +/// Body children after the title and subtitle, without the closing `sectPr`. +pub fn content<'a, 'i>(d: &'a Document<'i>) -> Vec> { + let kids = elements(body(d)); + assert_eq!(kids.last().unwrap().tag_name().name(), "sectPr"); + kids[2..kids.len() - 1].to_vec() +} + +/// Child elements named `w:`. +pub fn children<'a, 'i>(n: Node<'a, 'i>, name: &str) -> Vec> { + n.children().filter(|c| c.has_tag_name((W, name))).collect() +} + +/// Every `w:tbl` in a part, outer tables before the tables they hold. +pub fn tables<'a, 'i>(d: &'a Document<'i>) -> Vec> { + d.descendants() + .filter(|n| n.has_tag_name((W, "tbl"))) + .collect() +} + +/// The `w:tblW` of a table, in twips; asserts it is given in `dxa`. +pub fn tbl_w(tbl: Node) -> u32 { + let w = child(tbl, "tblPr") + .and_then(|p| child(p, "tblW")) + .expect("tblW"); + assert_eq!(wattr(w, "type"), Some("dxa")); + wattr(w, "w").unwrap().parse().unwrap() +} + +/// The `w:gridCol` widths of a table, in twips. +pub fn grid_cols(tbl: Node) -> Vec { + children(child(tbl, "tblGrid").expect("tblGrid"), "gridCol") + .into_iter() + .map(|c| wattr(c, "w").unwrap().parse().unwrap()) + .collect() +} + +/// A `w:tcPr` child of a table cell. +pub fn tc_prop<'a, 'i>(tc: Node<'a, 'i>, name: &str) -> Option> { + child(tc, "tcPr").and_then(|p| child(p, name)) +} + +/// The `w:tcW` of a table cell, in twips; asserts it is given in `dxa`. +pub fn tc_w(tc: Node) -> u32 { + let w = tc_prop(tc, "tcW").expect("tcW"); + assert_eq!(wattr(w, "type"), Some("dxa")); + wattr(w, "w").unwrap().parse().unwrap() +} + +/// Grid columns a table cell spans (`w:gridSpan`, 1 without one). +pub fn grid_span(tc: Node) -> usize { + tc_prop(tc, "gridSpan") + .and_then(val) + .map_or(1, |v| v.parse().unwrap()) +} + +/// A table cell's `w:vMerge`: "restart", "continue" (also for one without +/// `w:val`), or `None` when the cell isn't vertically merged. +pub fn v_merge<'a>(tc: Node<'a, '_>) -> Option<&'a str> { + tc_prop(tc, "vMerge").map(|m| val(m).unwrap_or("continue")) +} + +pub fn style<'a, 'i>(styles: &'a Document<'i>, id: &str) -> Node<'a, 'i> { + styles + .descendants() + .find(|n| n.has_tag_name((W, "style")) && wattr(*n, "styleId") == Some(id)) + .unwrap_or_else(|| panic!("no style {id}")) +} + +/// Value of `w:style/w:name`. +pub fn style_name<'a>(s: Node<'a, '_>) -> &'a str { + child(s, "name").and_then(val).expect("style name") +} + +/// ECMA-376 (wml.xsd) order of the children Word checks strictly. +pub const SCHEMA_ORDER: &[(&str, &[&str])] = &[ + ( + "pPr", + &[ + "pStyle", + "keepNext", + "keepLines", + "pageBreakBefore", + "framePr", + "widowControl", + "numPr", + "suppressLineNumbers", + "pBdr", + "shd", + "tabs", + "suppressAutoHyphens", + "kinsoku", + "wordWrap", + "overflowPunct", + "topLinePunct", + "autoSpaceDE", + "autoSpaceDN", + "bidi", + "adjustRightInd", + "snapToGrid", + "spacing", + "ind", + "contextualSpacing", + "mirrorIndents", + "suppressOverlap", + "jc", + "textDirection", + "textAlignment", + "textboxTightWrap", + "outlineLvl", + "divId", + "cnfStyle", + "rPr", + "sectPr", + "pPrChange", + ], + ), + ( + "rPr", + &[ + "rStyle", + "rFonts", + "b", + "bCs", + "i", + "iCs", + "caps", + "smallCaps", + "strike", + "dstrike", + "outline", + "shadow", + "emboss", + "imprint", + "noProof", + "snapToGrid", + "vanish", + "webHidden", + "color", + "spacing", + "w", + "kern", + "position", + "sz", + "szCs", + "highlight", + "u", + "effect", + "bdr", + "shd", + "fitText", + "vertAlign", + "rtl", + "cs", + "em", + "lang", + "eastAsianLayout", + "specVanish", + "oMath", + ], + ), + ( + "style", + &[ + "name", + "aliases", + "basedOn", + "next", + "link", + "autoRedefine", + "hidden", + "uiPriority", + "semiHidden", + "unhideWhenUsed", + "qFormat", + "locked", + "personal", + "personalCompose", + "personalReply", + "rsid", + "pPr", + "rPr", + "tblPr", + "trPr", + "tcPr", + "tblStylePr", + ], + ), + ( + "tblPr", + &[ + "tblStyle", + "tblpPr", + "tblOverlap", + "bidiVisual", + "tblStyleRowBandSize", + "tblStyleColBandSize", + "tblW", + "jc", + "tblCellSpacing", + "tblInd", + "tblBorders", + "shd", + "tblLayout", + "tblCellMar", + "tblLook", + "tblCaption", + "tblDescription", + ], + ), + ( + "tcPr", + &[ + "cnfStyle", + "tcW", + "gridSpan", + "hMerge", + "vMerge", + "tcBorders", + "shd", + "noWrap", + "tcMar", + "textDirection", + "tcFitText", + "vAlign", + "hideMark", + "headers", + "cellIns", + "cellDel", + "cellMerge", + "tcPrChange", + ], + ), + ( + "tblBorders", + &[ + "top", "left", "start", "bottom", "right", "end", "insideH", "insideV", + ], + ), + ( + "tblCellMar", + &["top", "left", "start", "bottom", "right", "end"], + ), + ( + "pBdr", + &["top", "left", "bottom", "right", "between", "bar"], + ), + ( + "sectPr", + &[ + "headerReference", + "footerReference", + "footnotePr", + "endnotePr", + "type", + "pgSz", + "pgMar", + "paperSrc", + "pgBorders", + "lnNumType", + "pgNumType", + "cols", + "formProt", + "vAlign", + "noEndnote", + "titlePg", + "textDirection", + "bidi", + "rtlGutter", + "docGrid", + "printerSettings", + "sectPrChange", + ], + ), +]; diff --git a/crates/knot-export/tests/common/mod.rs b/crates/knot-export/tests/common/mod.rs new file mode 100644 index 00000000..4e19bf50 --- /dev/null +++ b/crates/knot-export/tests/common/mod.rs @@ -0,0 +1,219 @@ +#![allow(dead_code)] +//! Fixture tree builder and package readers shared by every knot-export test. + +pub mod docx; +pub mod odt; + +use std::collections::HashMap; +use std::io::{Cursor, Read}; +use std::sync::Arc; + +use knot_export::assets::{Assets, ImageAsset}; +use knot_export::{Block, ExportDoc, ImageSource}; +use yrs::types::Attrs; +use yrs::{ + Any, Doc, Text, Transact, TransactionMut, Xml, XmlElementPrelim, XmlFragment, XmlTextPrelim, +}; + +pub enum N { + El(&'static str, Vec<(&'static str, Any)>, Vec), + Text(Vec<(String, Vec<(&'static str, Any)>)>), +} + +pub fn build(blocks: Vec) -> Doc { + let doc = Doc::new(); + let frag = doc.get_or_insert_xml_fragment("default"); + let mut txn = doc.transact_mut(); + for b in blocks { + add(&frag, &mut txn, b); + } + drop(txn); + doc +} + +fn add(parent: &F, txn: &mut TransactionMut, n: N) { + match n { + N::El(tag, attrs, children) => { + let el = parent.push_back(txn, XmlElementPrelim::empty(tag)); + for (k, v) in attrs { + el.insert_attribute(txn, k, v); + } + for c in children { + add(&el, txn, c); + } + } + N::Text(chunks) => { + let t = parent.push_back(txn, XmlTextPrelim::new("")); + for (s, marks) in chunks { + let attrs: Attrs = marks + .into_iter() + .map(|(k, v)| (Arc::from(k), v)) + .collect::>(); + let pos = t.len(txn); + t.insert_with_attributes(txn, pos, &s, attrs); + } + } + } +} + +pub fn el(tag: &'static str, attrs: Vec<(&'static str, Any)>, children: Vec) -> N { + N::El(tag, attrs, children) +} + +pub fn t(s: &str) -> N { + N::Text(vec![(s.to_owned(), vec![])]) +} + +pub fn p(s: &str) -> N { + el("paragraph", vec![], vec![t(s)]) +} + +/// Heading with the level as an editor number (`f64`). +pub fn h(level: u8, s: &str) -> N { + el( + "heading", + vec![("level", Any::Number(f64::from(level)))], + vec![t(s)], + ) +} + +/// Text carrying the given marks, each as an empty map (editor shape). +pub fn marked(s: &str, marks: &[&'static str]) -> N { + N::Text(vec![( + s.to_owned(), + marks + .iter() + .map(|m| (*m, Any::Map(Arc::new(HashMap::new())))) + .collect(), + )]) +} + +pub fn linked(s: &str, href: &str) -> N { + let mut m = HashMap::new(); + m.insert("href".to_owned(), Any::String(Arc::from(href))); + N::Text(vec![(s.to_owned(), vec![("link", Any::Map(Arc::new(m)))])]) +} + +/// A table inside every per-table limit whose grid, not its content, sets +/// the output size: a first row of one cell spanning all `width` columns, +/// then `rows - 1` empty rows. ODT writes a cell and `width - 1` covered +/// cells for each of them. +pub fn wide_empty_table(width: usize, rows: usize) -> N { + let mut out = vec![el( + "table_row", + vec![], + vec![el( + "table_cell", + vec![("colspan", Any::Number(width as f64))], + vec![p("x")], + )], + )]; + out.extend((1..rows).map(|_| el("table_row", vec![], vec![]))); + el("table", vec![], out) +} + +/// A table inside every per-table limit made of row spans: every `span`th +/// row holds `width` cells spanning `span` rows, the rows between are +/// empty. DOCX writes a `vMerge` continuation cell per column for each of +/// them. +pub fn row_spanning_table(width: usize, span: usize, rows: usize) -> N { + let out = (0..rows) + .map(|r| { + let cells = if r % span == 0 { + (0..width) + .map(|_| { + el( + "table_cell", + vec![("rowspan", Any::Number(span as f64))], + vec![p("y")], + ) + }) + .collect() + } else { + vec![] + }; + el("table_row", vec![], cells) + }) + .collect(); + el("table", vec![], out) +} + +/// `len` bytes that don't compress, like a real picture's. +pub fn noise(len: usize) -> Vec { + let mut x: u32 = 0x9E37_79B9; + (0..len) + .map(|_| { + x ^= x << 13; + x ^= x >> 17; + x ^= x << 5; + x as u8 + }) + .collect() +} + +/// Every entry of a zip archive, keyed by name. Entries are read with +/// `ZipArchive::by_name` only. +pub fn unzip(bytes: &[u8]) -> HashMap> { + let mut zip = zip::ZipArchive::new(Cursor::new(bytes)).expect("a zip archive"); + let names: Vec = zip.file_names().map(str::to_owned).collect(); + names + .into_iter() + .map(|name| { + let mut data = Vec::new(); + zip.by_name(&name) + .expect("listed entry") + .read_to_end(&mut data) + .expect("readable entry"); + (name, data) + }) + .collect() +} + +/// Parse one package part; panics when it is not well-formed. +pub fn xml(s: &str) -> roxmltree::Document<'_> { + roxmltree::Document::parse(s).unwrap_or_else(|e| panic!("malformed XML: {e}")) +} + +pub const UNAVAILABLE_BLOB: &str = "5f1e2d3c-4b5a-4968-8776-655443322110"; +pub const OVER_BUDGET_BLOB: &str = "6a7b8c9d-0e1f-4a2b-9c3d-4e5f60718293"; +pub const FIXTURE_BOARD: &str = "1d2c3b4a-5968-4776-8584-93a2b1c0d9e8"; + +/// A page in which every diagram and image falls back: a Mermaid block and a +/// board without pictures, plus images that are unavailable, over the image +/// budget, and of an unsupported source. +pub fn fallback_fixture() -> (ExportDoc, Assets) { + let doc = ExportDoc { + blocks: vec![ + Block::Mermaid { + key: "mermaid-3f6c1e0b9a4d7f2e8c5b1a0d9e7f6c4b3a2d1e0f9c8b7a6d5e4f3c2b1a0d9e8f" + .into(), + source: "graph TD\n A --> B".into(), + }, + Block::Board { + key: format!("board-{FIXTURE_BOARD}"), + board_id: FIXTURE_BOARD.into(), + label: "Roadmap".into(), + }, + Block::Image { + source: ImageSource::Blob(UNAVAILABLE_BLOB.into()), + alt: "Org chart".into(), + }, + Block::Image { + source: ImageSource::Blob(OVER_BUDGET_BLOB.into()), + alt: "Team photo".into(), + }, + Block::Image { + source: ImageSource::Unsupported, + alt: "Local file".into(), + }, + ], + }; + let mut assets = Assets::default(); + assets + .images + .insert(UNAVAILABLE_BLOB.into(), ImageAsset::Unavailable); + assets + .images + .insert(OVER_BUDGET_BLOB.into(), ImageAsset::OverBudget); + (doc, assets) +} diff --git a/crates/knot-export/tests/common/odt.rs b/crates/knot-export/tests/common/odt.rs new file mode 100644 index 00000000..a106220a --- /dev/null +++ b/crates/knot-export/tests/common/odt.rs @@ -0,0 +1,119 @@ +//! ODT-specific roxmltree helpers shared by the ODT test files. + +use std::collections::HashMap; + +use roxmltree::{Document, Node}; + +pub const OFFICE: &str = "urn:oasis:names:tc:opendocument:xmlns:office:1.0"; +pub const STYLE: &str = "urn:oasis:names:tc:opendocument:xmlns:style:1.0"; +pub const TEXT: &str = "urn:oasis:names:tc:opendocument:xmlns:text:1.0"; +pub const TABLE: &str = "urn:oasis:names:tc:opendocument:xmlns:table:1.0"; +pub const FO: &str = "urn:oasis:names:tc:opendocument:xmlns:xsl-fo-compatible:1.0"; +pub const XLINK: &str = "http://www.w3.org/1999/xlink"; +pub const META: &str = "urn:oasis:names:tc:opendocument:xmlns:meta:1.0"; +pub const DC: &str = "http://purl.org/dc/elements/1.1/"; +pub const MANIFEST: &str = "urn:oasis:names:tc:opendocument:xmlns:manifest:1.0"; +pub const XML_NS: &str = "http://www.w3.org/XML/1998/namespace"; + +/// The parts of an exported package. +pub struct Odt(pub HashMap>); + +impl Odt { + pub fn part(&self, name: &str) -> &str { + let bytes = self + .0 + .get(name) + .unwrap_or_else(|| panic!("missing part {name}")); + std::str::from_utf8(bytes).expect("UTF-8 part") + } +} + +pub fn elements<'a, 'i>(n: Node<'a, 'i>) -> Vec> { + n.children().filter(Node::is_element).collect() +} + +/// First child element `ns:name`. +pub fn child<'a, 'i>(n: Node<'a, 'i>, ns: &str, name: &str) -> Option> { + n.children().find(|c| c.has_tag_name((ns, name))) +} + +/// Child elements `ns:name`. +pub fn children<'a, 'i>(n: Node<'a, 'i>, ns: &str, name: &str) -> Vec> { + n.children() + .filter(|c| c.has_tag_name((ns, name))) + .collect() +} + +/// Every element `ns:name` in a part, in document order. +pub fn all<'a, 'i>(d: &'a Document<'i>, ns: &str, name: &str) -> Vec> { + d.descendants() + .filter(|n| n.has_tag_name((ns, name))) + .collect() +} + +/// The text a reader sees below a node: character data, with `text:s` as +/// its spaces, `text:tab` as a tab and `text:line-break` as a line feed. +pub fn text(n: Node) -> String { + let mut s = String::new(); + for d in n.descendants() { + if d.is_text() { + s.push_str(d.text().unwrap_or("")); + } else if d.has_tag_name((TEXT, "s")) { + let c: usize = d.attribute((TEXT, "c")).map_or(1, |c| c.parse().unwrap()); + s.extend(std::iter::repeat_n(' ', c)); + } else if d.has_tag_name((TEXT, "tab")) { + s.push('\t'); + } else if d.has_tag_name((TEXT, "line-break")) { + s.push('\n'); + } + } + s +} + +/// `office:text` of `content.xml`. +pub fn office_text<'a, 'i>(d: &'a Document<'i>) -> Node<'a, 'i> { + d.descendants() + .find(|n| n.has_tag_name((OFFICE, "text"))) + .expect("office:text") +} + +/// `office:text` children after the title and subtitle. +pub fn content<'a, 'i>(d: &'a Document<'i>) -> Vec> { + elements(office_text(d))[2..].to_vec() +} + +/// `text:style-name` of a paragraph, heading, span, link or list. +pub fn style_name<'a>(n: Node<'a, '_>) -> Option<&'a str> { + n.attribute((TEXT, "style-name")) +} + +/// The style (`style:style`, a list style, a page layout, ...) named `name` +/// in a part. +pub fn style<'a, 'i>(d: &'a Document<'i>, name: &str) -> Node<'a, 'i> { + d.descendants() + .find(|n| n.attribute((STYLE, "name")) == Some(name)) + .unwrap_or_else(|| panic!("no style {name}")) +} + +/// Attribute `ns:name` on one of a style's `*-properties` children. +pub fn prop<'a>(s: Node<'a, '_>, ns: &str, name: &str) -> Option<&'a str> { + s.children() + .filter(|c| c.is_element() && c.tag_name().name().ends_with("-properties")) + .find_map(|c| c.attribute((ns, name))) +} + +/// A length such as "16.000cm" as centimetres. +pub fn cm(len: &str) -> f64 { + len.strip_suffix("cm") + .unwrap_or_else(|| panic!("{len} is not in cm")) + .parse() + .unwrap() +} + +/// How many `text:list` elements enclose a node (not counting the node). +pub fn list_depth(n: Node) -> usize { + n.ancestors() + .skip(1) + .filter(|a| a.has_tag_name((TEXT, "list"))) + .count() +} diff --git a/crates/knot-export/tests/docx.rs b/crates/knot-export/tests/docx.rs new file mode 100644 index 00000000..5beba333 --- /dev/null +++ b/crates/knot-export/tests/docx.rs @@ -0,0 +1,1938 @@ +mod common; + +use std::borrow::Cow; +use std::collections::HashSet; +use std::time::{Duration, Instant}; + +use common::docx::{ + Docx, PKG_REL, SCHEMA_ORDER, W, body, child, children, content, elements, grid_cols, grid_span, + p_style, r_style, rels, style, style_name, tables, tbl_w, tc_prop, tc_w, text, v_merge, val, + wattr, +}; +use common::{unzip, xml}; +use knot_export::assets::{Assets, Fallbacks, ImageAsset, Meta, PicFormat, Picture}; +use knot_export::docx::{DocxDefaults, write_docx_with}; +use knot_export::{ + Align, Block, Cell, ExportDoc, ImageSource, Inline, LinkTarget, List, ListItem, Marks, Run, + Table, write_docx, +}; +use roxmltree::Node; + +const R: &str = "http://schemas.openxmlformats.org/officeDocument/2006/relationships"; +const CT: &str = "http://schemas.openxmlformats.org/package/2006/content-types"; +const DC: &str = "http://purl.org/dc/elements/1.1/"; +const DCTERMS: &str = "http://purl.org/dc/terms/"; +const XML_NS: &str = "http://www.w3.org/XML/1998/namespace"; +const REL_TYPE: &str = "http://schemas.openxmlformats.org/officeDocument/2006/relationships/"; + +const DOC_ID: &str = "0b1c2d3e-4f50-4162-8374-8596a7b8c9d0"; +const USER_ID: &str = "7a6b5c4d-3e2f-4101-9283-746556473829"; +const HOSTILE: &str = "Q4 & \"final\" \u{1}😀 שלום"; + +fn meta() -> Meta { + Meta { + title: "Q4 Plan".into(), + exported_on: "23 Oct 2026".into(), + base_url: "https://knot.example".into(), + } +} + +fn export_with(blocks: Vec, assets: &Assets) -> Docx { + let out = write_docx(&ExportDoc { blocks }, &meta(), assets).expect("export"); + Docx(unzip(&out.bytes)) +} + +fn export(blocks: Vec) -> Docx { + export_with(blocks, &Assets::default()) +} + +fn run(text: &str) -> Run { + Run { + text: text.into(), + marks: Marks::default(), + } +} + +fn para(text: &str) -> Block { + Block::Paragraph(vec![Inline::Run(run(text))]) +} + +fn link(target: LinkTarget, text: &str) -> Inline { + Inline::Link { + target, + runs: vec![run(text)], + } +} + +/// One of every block and inline this task writes. +fn kitchen_sink() -> Vec { + vec![ + Block::Heading { + level: 2, + content: vec![Inline::Run(run("Heading"))], + }, + Block::Paragraph(vec![ + Inline::Run(Run { + text: "all marks".into(), + marks: Marks { + bold: true, + italic: true, + underline: true, + strike: true, + code: false, + }, + }), + Inline::Run(Run { + text: "code".into(), + marks: Marks { + code: true, + ..Marks::default() + }, + }), + Inline::Break, + link(LinkTarget::Web("https://a.example/".into()), "web"), + link(LinkTarget::Doc(DOC_ID.into()), "page"), + ]), + Block::Quote(vec![para("quoted")]), + Block::Code { + language: "rust".into(), + text: "fn main() {\n\tprintln!();\n}".into(), + }, + Block::Rule, + ] +} + +#[test] +fn package_parts_and_relationships_resolve() { + let docx = export(kitchen_sink()); + let names: HashSet<&str> = docx.0.keys().map(String::as_str).collect(); + + let root = xml(docx.part("_rels/.rels")); + for r in root + .descendants() + .filter(|n| n.has_tag_name((PKG_REL, "Relationship"))) + { + let target = r.attribute("Target").unwrap(); + assert!(names.contains(target), "root target {target} missing"); + } + + let rels = rels(&docx); + let doc = xml(docx.part("word/document.xml")); + let ids: Vec<&str> = doc + .descendants() + .flat_map(|n| n.attributes()) + .filter(|a| a.namespace() == Some(R)) + .map(|a| a.value()) + .collect(); + assert!(ids.len() >= 3, "footer and two hyperlinks: {ids:?}"); + for id in &ids { + assert!(rels.contains_key(*id), "{id} not in document.xml.rels"); + } + for (id, rel) in &rels { + if !rel.external { + let part = format!("word/{}", rel.target); + assert!(names.contains(part.as_str()), "{id} → {part} missing"); + } + } + + let ct = xml(docx.part("[Content_Types].xml")); + let overrides: HashSet<&str> = ct + .descendants() + .filter(|n| n.has_tag_name((CT, "Override"))) + .map(|n| n.attribute("PartName").unwrap().trim_start_matches('/')) + .collect(); + let defaults: HashSet<&str> = ct + .descendants() + .filter(|n| n.has_tag_name((CT, "Default"))) + .map(|n| n.attribute("Extension").unwrap()) + .collect(); + for part in &overrides { + assert!(names.contains(part), "override {part} missing"); + } + for ext in ["png", "jpeg", "gif"] { + assert!(defaults.contains(ext), "no default for {ext}"); + } + for name in names.iter().filter(|n| **n != "[Content_Types].xml") { + let ext = name.rsplit('.').next().unwrap(); + assert!( + overrides.contains(name) || defaults.contains(ext), + "{name} has no content type" + ); + } + + let core = xml(docx.part("docProps/core.xml")); + let find = |ns: &str, name: &str| { + core.descendants() + .find(|n| n.has_tag_name((ns, name))) + .and_then(|n| n.text()) + .map(str::to_owned) + }; + assert_eq!(find(DC, "title").as_deref(), Some("Q4 Plan")); + assert_eq!( + find(DCTERMS, "created").as_deref(), + Some("2026-10-23T00:00:00Z") + ); + assert_eq!( + find(DCTERMS, "modified").as_deref(), + Some("2026-10-23T00:00:00Z") + ); + assert!(!core.descendants().any(|n| n.has_tag_name((DC, "creator")))); + + let app = xml(docx.part("docProps/app.xml")); + let application = app + .descendants() + .find(|n| n.tag_name().name() == "Application") + .and_then(|n| n.text()); + assert_eq!(application, Some("knot")); +} + +#[test] +fn empty_page_has_title_subtitle_and_sectpr() { + let docx = export(vec![]); + let d = xml(docx.part("word/document.xml")); + let kids = elements(body(&d)); + let tags: Vec<&str> = kids.iter().map(|n| n.tag_name().name()).collect(); + assert_eq!(tags, ["p", "p", "sectPr"]); + + assert_eq!(p_style(kids[0]), Some("Title")); + assert_eq!(text(kids[0]), "Q4 Plan"); + assert_eq!(p_style(kids[1]), Some("Subtitle")); + assert_eq!(text(kids[1]), "Exported from knot · 23 Oct 2026"); + + let sect = kids[2]; + let size = child(sect, "pgSz").expect("pgSz"); + assert_eq!( + (wattr(size, "w"), wattr(size, "h")), + (Some("11906"), Some("16838")) + ); + let margins = child(sect, "pgMar").expect("pgMar"); + for side in ["top", "right", "bottom", "left"] { + assert_eq!(wattr(margins, side), Some("1417"), "margin {side}"); + } + let footer = child(sect, "footerReference").expect("footerReference"); + assert_eq!(wattr(footer, "type"), Some("default")); + let rels = rels(&docx); + let rel = &rels[footer.attribute((R, "id")).unwrap()]; + assert_eq!(rel.kind, format!("{REL_TYPE}footer")); + assert_eq!(rel.target, "footer1.xml"); +} + +#[test] +fn headings_use_heading_styles_with_outline_levels() { + let docx = export( + (1..=6) + .map(|level| Block::Heading { + level, + content: vec![Inline::Run(run("h"))], + }) + .collect(), + ); + let d = xml(docx.part("word/document.xml")); + let used: Vec<&str> = content(&d).into_iter().filter_map(p_style).collect(); + assert_eq!( + used, + [ + "Heading1", "Heading2", "Heading3", "Heading4", "Heading5", "Heading6" + ] + ); + + let styles = xml(docx.part("word/styles.xml")); + for level in 1..=6u8 { + let s = style(&styles, &format!("Heading{level}")); + assert_eq!(style_name(s), format!("heading {level}")); + let outline = child(s, "pPr") + .and_then(|ppr| child(ppr, "outlineLvl")) + .and_then(val); + assert_eq!(outline, Some((level - 1).to_string().as_str())); + } +} + +#[test] +fn style_ids_carry_builtin_names() { + let docx = export(vec![]); + let styles = xml(docx.part("word/styles.xml")); + for (id, kind, name) in [ + ("Normal", "paragraph", Some("Normal")), + ("Title", "paragraph", Some("Title")), + ("Subtitle", "paragraph", Some("Subtitle")), + ("Quote", "paragraph", Some("Quote")), + ("Code", "paragraph", None), + ("InlineCode", "character", None), + ("ListParagraph", "paragraph", Some("List Paragraph")), + ("TableGrid", "table", Some("Table Grid")), + ("TableHeader", "paragraph", None), + ("Hyperlink", "character", Some("Hyperlink")), + ("Caption", "paragraph", Some("caption")), + ("Footer", "paragraph", Some("footer")), + ("HorizontalRule", "paragraph", None), + ("TableNormal", "table", Some("Normal Table")), + ] { + let s = style(&styles, id); + assert_eq!(wattr(s, "type"), Some(kind), "type of {id}"); + if let Some(name) = name { + assert_eq!(style_name(s), name, "name of {id}"); + } + } + + let default_table = style(&styles, "TableNormal"); + assert_eq!(wattr(default_table, "default"), Some("1")); + let caption = child(style(&styles, "Caption"), "rPr").expect("Caption rPr"); + assert!(child(caption, "i").is_some(), "Caption is italic"); + let footer = child(style(&styles, "Footer"), "pPr").expect("Footer pPr"); + assert_eq!(child(footer, "jc").and_then(val), Some("center")); + let rule = child(style(&styles, "HorizontalRule"), "pPr").expect("rule pPr"); + assert!( + child(rule, "pBdr") + .and_then(|b| child(b, "bottom")) + .is_some() + ); +} + +#[test] +fn every_referenced_style_exists() { + let mut blocks = kitchen_sink(); + blocks.push(sample_table()); + let docx = export(blocks); + let styles = xml(docx.part("word/styles.xml")); + let ids: HashSet<&str> = styles + .descendants() + .filter(|n| n.has_tag_name((W, "style"))) + .filter_map(|n| wattr(n, "styleId")) + .collect(); + let mut seen = HashSet::new(); + for part in ["word/document.xml", "word/footer1.xml"] { + let d = xml(docx.part(part)); + for n in d.descendants().filter(|n| { + ["pStyle", "rStyle", "tblStyle"] + .iter() + .any(|t| n.has_tag_name((W, *t))) + }) { + let id = val(n).expect("style reference"); + assert!(ids.contains(id), "{part} references missing style {id}"); + seen.insert(id.to_owned()); + } + } + for id in [ + "Title", + "Subtitle", + "Heading2", + "InlineCode", + "Hyperlink", + "Quote", + "Code", + "HorizontalRule", + "Footer", + "TableGrid", + "TableHeader", + ] { + assert!(seen.contains(id), "kitchen sink should use {id}"); + } +} + +#[test] +fn properties_follow_schema_order() { + let mut blocks = kitchen_sink(); + blocks.push(list( + true, + 1, + vec![item(vec![para("a"), code("b")]), check_item(true, "c")], + )); + blocks.push(sample_table()); + let docx = export(blocks); + for part in [ + "word/document.xml", + "word/numbering.xml", + "word/styles.xml", + "word/footer1.xml", + "word/settings.xml", + ] { + let d = xml(docx.part(part)); + for n in d + .descendants() + .filter(|n| n.tag_name().namespace() == Some(W)) + { + let name = n.tag_name().name(); + let kids = elements(n); + let props = [ + ("p", "pPr"), + ("r", "rPr"), + ("tbl", "tblPr"), + ("tr", "trPr"), + ("tc", "tcPr"), + ] + .iter() + .find(|(el, _)| *el == name); + if let Some((_, props)) = props { + for (i, k) in kids.iter().enumerate() { + assert!( + k.tag_name().name() != *props || i == 0, + "{part}: {props} is not the first child of {name}" + ); + } + } + let Some((_, order)) = SCHEMA_ORDER.iter().find(|(p, _)| *p == name) else { + continue; + }; + let mut last = None; + for k in kids { + let kname = k.tag_name().name(); + let pos = order + .iter() + .position(|o| *o == kname) + .unwrap_or_else(|| panic!("{part}: unexpected {kname} in {name}")); + assert!( + last.is_none_or(|l| l < pos), + "{part}: {kname} out of schema order in {name}" + ); + last = Some(pos); + } + } + } +} + +#[test] +fn runs_marks_and_breaks() { + let docx = export(vec![Block::Paragraph(vec![ + Inline::Run(Run { + text: "loud".into(), + marks: Marks { + bold: true, + italic: true, + underline: true, + strike: true, + code: false, + }, + }), + Inline::Run(Run { + text: "x = 1".into(), + marks: Marks { + code: true, + ..Marks::default() + }, + }), + Inline::Break, + Inline::Run(run(" a\tb ")), + ])]); + let d = xml(docx.part("word/document.xml")); + let p = content(&d)[0]; + let runs: Vec = p.children().filter(|n| n.has_tag_name((W, "r"))).collect(); + assert_eq!(runs.len(), 4); + + let rpr = child(runs[0], "rPr").expect("marks"); + for mark in ["b", "i", "strike", "u"] { + assert!(child(rpr, mark).is_some(), "missing {mark}"); + } + assert_eq!(child(rpr, "u").and_then(val), Some("single")); + assert_eq!(text(runs[0]), "loud"); + + assert_eq!(r_style(runs[1]), Some("InlineCode")); + assert!(child(runs[2], "br").is_some()); + + let pieces: Vec<(&str, &str)> = elements(runs[3]) + .into_iter() + .map(|n| (n.tag_name().name(), n.text().unwrap_or(""))) + .collect(); + assert_eq!(pieces, [("t", " a"), ("tab", ""), ("t", "b ")]); + + for t in d.descendants().filter(|n| n.has_tag_name((W, "t"))) { + assert_eq!(t.attribute((XML_NS, "space")), Some("preserve")); + } +} + +#[test] +fn links_become_hyperlinks_with_external_targets() { + let mut assets = Assets::default(); + assets.user_names.insert(USER_ID.into(), "Ada".into()); + let web = "https://a.example/x?y=1&z"; + let docx = export_with( + vec![Block::Paragraph(vec![ + link(LinkTarget::Web(web.into()), "web"), + Inline::Run(run(" ")), + link(LinkTarget::Doc("d1".into()), "page"), + link(LinkTarget::Web(web.into()), "again"), + link(LinkTarget::User(USER_ID.into()), "@Old Name"), + link(LinkTarget::User("gone".into()), "@Gone"), + ])], + &assets, + ); + let rels = rels(&docx); + let d = xml(docx.part("word/document.xml")); + let links: Vec = d + .descendants() + .filter(|n| n.has_tag_name((W, "hyperlink"))) + .collect(); + let ids: Vec<&str> = links + .iter() + .map(|l| l.attribute((R, "id")).expect("r:id")) + .collect(); + let targets: Vec<&str> = ids.iter().map(|id| rels[*id].target.as_str()).collect(); + assert_eq!(targets, [web, "https://knot.example/doc/d1", web]); + assert_eq!(ids[0], ids[2], "one relationship per target"); + assert_ne!(ids[0], ids[1]); + for id in &ids { + assert_eq!(rels[*id].kind, format!("{REL_TYPE}hyperlink")); + assert!(rels[*id].external); + } + assert!( + docx.part("word/_rels/document.xml.rels") + .contains("x?y=1&z") + ); + for r in links + .iter() + .flat_map(|l| l.children().filter(|n| n.has_tag_name((W, "r")))) + { + assert_eq!(r_style(r), Some("Hyperlink")); + } + + let p = content(&d)[0]; + assert_eq!(text(p), "web pageagain@Ada@Gone"); + for t in p.descendants().filter(|n| n.text() == Some("@Ada")) { + assert!( + !t.ancestors().any(|a| a.has_tag_name((W, "hyperlink"))), + "mentions are plain text" + ); + } +} + +#[test] +fn unsafe_links_have_no_relationship() { + let docx = export(vec![Block::Paragraph(vec![link( + LinkTarget::Plain, + "knot://time/2026-10-23", + )])]); + let d = xml(docx.part("word/document.xml")); + assert!(!d.descendants().any(|n| n.has_tag_name((W, "hyperlink")))); + let p = content(&d)[0]; + let r = child(p, "r").expect("plain run"); + assert_eq!(text(r), "knot://time/2026-10-23"); + assert_eq!(r_style(r), None); + + let mut kinds: Vec = rels(&docx).into_values().map(|r| r.kind).collect(); + kinds.sort(); + let expected: Vec = ["footer", "numbering", "settings", "styles"] + .iter() + .map(|k| format!("{REL_TYPE}{k}")) + .collect(); + assert_eq!(kinds, expected); +} + +#[test] +fn code_block_is_one_shaded_paragraph() { + let docx = export(vec![Block::Code { + language: "text".into(), + text: "a\n b".into(), + }]); + let d = xml(docx.part("word/document.xml")); + let blocks = content(&d); + assert_eq!(blocks.len(), 1); + assert_eq!(p_style(blocks[0]), Some("Code")); + let pieces: Vec<(&str, &str)> = blocks[0] + .descendants() + .filter(|n| ["t", "br", "tab"].iter().any(|t| n.has_tag_name((W, *t)))) + .map(|n| (n.tag_name().name(), n.text().unwrap_or(""))) + .collect(); + assert_eq!(pieces, [("t", "a"), ("br", ""), ("t", " b")]); + + let styles = xml(docx.part("word/styles.xml")); + let code = style(&styles, "Code"); + let shd = child(code, "pPr") + .and_then(|p| child(p, "shd")) + .expect("shaded"); + assert!(wattr(shd, "fill").is_some_and(|f| f != "auto")); + let font = child(code, "rPr") + .and_then(|r| child(r, "rFonts")) + .and_then(|f| wattr(f, "ascii")); + assert_eq!(font, Some("Consolas")); +} + +#[test] +fn quote_and_rule_styles() { + let docx = export(vec![Block::Quote(vec![para("quoted")]), Block::Rule]); + let d = xml(docx.part("word/document.xml")); + let blocks = content(&d); + assert_eq!(blocks.len(), 2); + assert_eq!(p_style(blocks[0]), Some("Quote")); + assert_eq!(text(blocks[0]), "quoted"); + assert_eq!(p_style(blocks[1]), Some("HorizontalRule")); + assert!(child(blocks[1], "r").is_none()); + + let styles = xml(docx.part("word/styles.xml")); + let indent = child(style(&styles, "Quote"), "pPr") + .and_then(|p| child(p, "ind")) + .and_then(|i| wattr(i, "left")); + assert_eq!(indent, Some("567")); +} + +#[test] +fn settings_declare_compat_15_and_footer_fields() { + let docx = export(vec![]); + let settings = xml(docx.part("word/settings.xml")); + let compat = settings + .descendants() + .find(|n| { + n.has_tag_name((W, "compatSetting")) && wattr(*n, "name") == Some("compatibilityMode") + }) + .expect("compatibilityMode"); + assert_eq!(val(compat), Some("15")); + assert_eq!( + wattr(compat, "uri"), + Some("http://schemas.microsoft.com/office/word") + ); + assert!( + !settings + .descendants() + .any(|n| n.has_tag_name((W, "updateFields"))) + ); + + let footer = xml(docx.part("word/footer1.xml")); + let instr: Vec<&str> = footer + .descendants() + .filter(|n| n.has_tag_name((W, "instrText"))) + .filter_map(|n| n.text()) + .collect(); + assert_eq!(instr, [" PAGE ", " NUMPAGES "]); + let begins = footer + .descendants() + .filter(|n| n.has_tag_name((W, "fldChar")) && wattr(*n, "fldCharType") == Some("begin")) + .count(); + assert_eq!(begins, 2); + let p = footer + .descendants() + .find(|n| n.has_tag_name((W, "p"))) + .expect("footer paragraph"); + assert_eq!(p_style(p), Some("Footer")); + assert_eq!(text(p), "Page 1 of 1"); +} + +#[test] +fn text_with_xml_specials_and_controls_is_well_formed() { + let mut assets = Assets::default(); + assets.user_names.insert(USER_ID.into(), HOSTILE.into()); + let doc = ExportDoc { + blocks: vec![ + para(HOSTILE), + Block::Heading { + level: 1, + content: vec![Inline::Run(run(HOSTILE))], + }, + Block::Code { + language: HOSTILE.into(), + text: HOSTILE.into(), + }, + Block::Paragraph(vec![ + link( + LinkTarget::Web("https://a.example/?q=&b=\"c\"".into()), + HOSTILE, + ), + link(LinkTarget::User(USER_ID.into()), "@x"), + ]), + ], + }; + let meta = Meta { + title: HOSTILE.into(), + ..meta() + }; + let out = write_docx(&doc, &meta, &assets).expect("export"); + let docx = Docx(unzip(&out.bytes)); + for name in docx.0.keys() { + if name.ends_with(".xml") || name.ends_with(".rels") { + xml(docx.part(name)); + } + } + + let clean = HOSTILE.replace('\u{1}', ""); + let core = xml(docx.part("docProps/core.xml")); + let title = core + .descendants() + .find(|n| n.has_tag_name((DC, "title"))) + .and_then(|n| n.text()); + assert_eq!(title, Some(clean.as_str())); + let d = xml(docx.part("word/document.xml")); + let kids = elements(body(&d)); + assert_eq!(text(kids[0]), clean); + let texts: Vec = content(&d).into_iter().map(text).collect(); + assert_eq!( + texts, + [ + clean.clone(), + clean.clone(), + clean.clone(), + format!("{clean}@{clean}") + ] + ); + let rels = rels(&docx); + assert!( + rels.values() + .any(|r| r.target == "https://a.example/?q=&b=\"c\"") + ); +} + +#[test] +fn package_defaults_are_written_as_given() { + let builtin = DocxDefaults::builtin(); + let defaults = DocxDefaults { + styles_xml: Cow::Owned(builtin.styles_xml.replace("Calibri", "Arial")), + numbering_xml: builtin.numbering_xml.clone(), + footer_xml: Cow::Owned(builtin.footer_xml.replace("Page ", "Seite ")), + }; + let out = write_docx_with( + &ExportDoc { + blocks: vec![para("x")], + }, + &meta(), + &Assets::default(), + &defaults, + ) + .expect("export"); + let docx = Docx(unzip(&out.bytes)); + assert_eq!(docx.part("word/styles.xml"), defaults.styles_xml); + assert_eq!(docx.part("word/footer1.xml"), defaults.footer_xml); + assert_eq!(docx.part("word/numbering.xml"), defaults.numbering_xml); + assert!(docx.part("word/styles.xml").contains("Arial")); +} + +fn list(ordered: bool, start: u32, items: Vec) -> Block { + Block::List(List { + ordered, + start, + items, + }) +} + +fn item(blocks: Vec) -> ListItem { + ListItem { + checked: None, + blocks, + } +} + +fn check_item(checked: bool, text: &str) -> ListItem { + ListItem { + checked: Some(checked), + blocks: vec![para(text)], + } +} + +fn code(text: &str) -> Block { + Block::Code { + language: String::new(), + text: text.into(), + } +} + +/// `(numId, ilvl)` of a paragraph's `w:numPr`. +fn num_pr(p: Node) -> Option<(String, u32)> { + let num_pr = child(child(p, "pPr")?, "numPr")?; + let ilvl = child(num_pr, "ilvl").and_then(val)?.parse().unwrap(); + let id = child(num_pr, "numId").and_then(val)?.to_owned(); + Some((id, ilvl)) +} + +/// The `w:ind w:left` of a paragraph. +fn ind_left(p: Node) -> Option { + let ind = child(child(p, "pPr")?, "ind")?; + wattr(ind, "left").map(|v| v.parse().unwrap()) +} + +/// A `w:num` of numbering.xml. +struct Num { + id: String, + abstract_id: String, + /// `(ilvl, startOverride)`. + over: Option<(u32, u32)>, +} + +fn nums(docx: &Docx) -> Vec { + let d = xml(docx.part("word/numbering.xml")); + d.descendants() + .filter(|n| n.has_tag_name((W, "num"))) + .map(|n| Num { + id: wattr(n, "numId").unwrap().to_owned(), + abstract_id: child(n, "abstractNumId").and_then(val).unwrap().to_owned(), + over: child(n, "lvlOverride").map(|o| { + let start = child(o, "startOverride").and_then(val).unwrap(); + ( + wattr(o, "ilvl").unwrap().parse().unwrap(), + start.parse().unwrap(), + ) + }), + }) + .collect() +} + +#[test] +fn each_ordered_list_restarts_with_its_start() { + let docx = export(vec![ + list(true, 1, vec![item(vec![para("a")])]), + para("between"), + list(true, 5, vec![item(vec![para("b")])]), + ]); + let d = xml(docx.part("word/document.xml")); + let ps = content(&d); + let (first, _) = num_pr(ps[0]).expect("first list item numbered"); + let (second, _) = num_pr(ps[2]).expect("second list item numbered"); + assert_ne!(first, second); + let nums = nums(&docx); + let find = |id: &str| nums.iter().find(|n| n.id == id).unwrap(); + assert_eq!(find(&first).abstract_id, "1"); + assert_eq!( + find(&first).over, + Some((0, 1)), + "start 1 is still overridden" + ); + assert_eq!(find(&second).over, Some((0, 5))); +} + +#[test] +fn nested_mixed_lists_use_depth_as_ilvl() { + let docx = export(vec![list( + false, + 1, + vec![item(vec![ + para("a"), + list( + true, + 3, + vec![item(vec![ + para("b"), + list(false, 1, vec![item(vec![para("c")])]), + ])], + ), + ])], + )]); + let d = xml(docx.part("word/document.xml")); + let ps = content(&d); + assert_eq!(ps.len(), 3); + let nums = nums(&docx); + let abstract_of = |p: Node| { + let (id, ilvl) = num_pr(p).expect("numbered"); + let n = nums.iter().find(|n| n.id == id).unwrap(); + (n.abstract_id.clone(), ilvl, n.over) + }; + assert_eq!(abstract_of(ps[0]), ("0".into(), 0, None)); + assert_eq!(abstract_of(ps[1]), ("1".into(), 1, Some((1, 3)))); + assert_eq!(abstract_of(ps[2]), ("0".into(), 2, None)); + for p in &ps { + assert_eq!(p_style(*p), Some("ListParagraph")); + } +} + +#[test] +fn checklist_items_use_box_characters() { + let docx = export(vec![list( + false, + 1, + vec![check_item(true, "done"), check_item(false, "todo")], + )]); + let d = xml(docx.part("word/document.xml")); + let ps = content(&d); + assert_eq!(ps.len(), 2); + assert_eq!(text(ps[0]), "\u{2611} done"); + assert_eq!(text(ps[1]), "\u{2610} todo"); + for p in &ps { + assert_eq!(num_pr(*p), None); + assert_eq!(p_style(*p), Some("ListParagraph")); + assert_eq!(ind_left(*p), Some(720)); + } + let first_t = ps[0] + .descendants() + .find(|n| n.has_tag_name((W, "t"))) + .unwrap(); + assert!(first_t.text().unwrap().starts_with("\u{2611} ")); +} + +#[test] +fn nested_checklist_indents_by_depth() { + let docx = export(vec![list( + false, + 1, + vec![item(vec![ + para("plain"), + list(false, 1, vec![check_item(true, "inner")]), + ])], + )]); + let d = xml(docx.part("word/document.xml")); + let ps = content(&d); + assert_eq!(ind_left(ps[1]), Some(1440)); + assert_eq!(num_pr(ps[1]), None); +} + +#[test] +fn deep_list_clamps_to_level_9() { + let mut inner = vec![item(vec![para("leaf")])]; + for _ in 0..11 { + inner = vec![item(vec![para("x"), list(false, 1, inner)])]; + } + let docx = export(vec![list(false, 1, inner)]); + let d = xml(docx.part("word/document.xml")); + let levels: Vec = content(&d) + .into_iter() + .map(|p| num_pr(p).expect("numbered").1) + .collect(); + assert_eq!(levels.len(), 12); + assert_eq!(levels.iter().max(), Some(&8)); + assert_eq!(levels[..9], [0, 1, 2, 3, 4, 5, 6, 7, 8]); + assert!(levels[9..].iter().all(|l| *l == 8)); +} + +#[test] +fn list_item_with_several_blocks() { + let docx = export(vec![list( + false, + 1, + vec![item(vec![para("lead"), code("let x = 1;"), para("tail")])], + )]); + let d = xml(docx.part("word/document.xml")); + let ps = content(&d); + assert_eq!(ps.len(), 3); + assert!(num_pr(ps[0]).is_some()); + assert_eq!(p_style(ps[1]), Some("Code")); + assert_eq!(num_pr(ps[1]), None); + assert_eq!(ind_left(ps[1]), Some(720)); + assert_eq!(p_style(ps[2]), Some("ListParagraph")); + assert_eq!(num_pr(ps[2]), None); + assert_eq!(ind_left(ps[2]), Some(720)); +} + +#[test] +fn item_without_leading_paragraph_still_has_a_marker() { + let docx = export(vec![list(true, 1, vec![item(vec![code("x")])])]); + let d = xml(docx.part("word/document.xml")); + let ps = content(&d); + assert_eq!(ps.len(), 2); + assert!(num_pr(ps[0]).is_some()); + assert_eq!(text(ps[0]), ""); + assert_eq!(p_style(ps[1]), Some("Code")); +} + +#[test] +fn every_num_id_exists_in_numbering() { + let docx = export(vec![ + list( + false, + 1, + vec![item(vec![ + para("a"), + list(true, 2, vec![item(vec![para("b")])]), + ])], + ), + list(true, 1, vec![item(vec![para("c")])]), + ]); + let d = xml(docx.part("word/document.xml")); + let used: HashSet = d + .descendants() + .filter(|n| n.has_tag_name((W, "numId"))) + .filter_map(|n| val(n).map(str::to_owned)) + .collect(); + assert_eq!(used.len(), 3); + let defined: Vec = nums(&docx).into_iter().map(|n| n.id).collect(); + let unique: HashSet<&String> = defined.iter().collect(); + assert_eq!(unique.len(), defined.len(), "numIds are unique"); + for id in &used { + assert!(defined.contains(id), "numId {id} has no w:num"); + } +} + +#[test] +fn numbering_defines_both_abstract_nums_before_any_num() { + let docx = export(vec![list(true, 1, vec![item(vec![para("a")])])]); + let d = xml(docx.part("word/numbering.xml")); + let kids = elements(d.root_element()); + let tags: Vec<&str> = kids.iter().map(|n| n.tag_name().name()).collect(); + assert_eq!(tags, ["abstractNum", "abstractNum", "num"]); + let lvls_of = |abs: Node<'_, '_>| -> Vec { + elements(abs) + .into_iter() + .filter(|n| n.has_tag_name((W, "lvl"))) + .map(|l| child(l, "lvlText").and_then(val).unwrap().to_owned()) + .collect() + }; + let bullets = lvls_of(kids[0]); + assert_eq!(bullets.len(), 9); + assert_eq!( + bullets[..4], + ["\u{2022}", "\u{25E6}", "\u{25AA}", "\u{2022}"] + ); + let decimals = lvls_of(kids[1]); + assert_eq!(decimals.len(), 9); + assert_eq!(decimals[0], "%1."); + assert_eq!(decimals[8], "%9."); + let lvl2 = elements(kids[1]) + .into_iter() + .filter(|n| n.has_tag_name((W, "lvl"))) + .nth(2) + .unwrap(); + let ind = child(child(lvl2, "pPr").unwrap(), "ind").unwrap(); + assert_eq!(wattr(ind, "left"), Some("2160")); + assert_eq!(wattr(ind, "hanging"), Some("360")); +} + +fn cell(col: usize, blocks: Vec) -> Cell { + Cell { + col, + colspan: 1, + rowspan: 1, + header: false, + align: Align::Left, + blocks, + } +} + +fn td(col: usize, text: &str) -> Cell { + cell(col, vec![para(text)]) +} + +fn th(col: usize, text: &str) -> Cell { + Cell { + header: true, + ..td(col, text) + } +} + +fn table(shares: Vec, rows: Vec>) -> Block { + Block::Table(Table { shares, rows }) +} + +/// `n` equal column shares. +fn even(n: usize) -> Vec { + vec![1.0 / n as f64; n] +} + +/// A table with a header row, both kinds of span and a centred cell. +fn sample_table() -> Block { + table( + even(3), + vec![ + vec![th(0, "h1"), th(1, "h2"), th(2, "h3")], + vec![ + Cell { + colspan: 2, + rowspan: 2, + align: Align::Center, + ..td(0, "merged") + }, + td(2, "x"), + ], + vec![td(2, "y")], + ], + ) +} + +fn rows<'a, 'i>(tbl: Node<'a, 'i>) -> Vec> { + children(tbl, "tr") +} + +fn cells<'a, 'i>(tr: Node<'a, 'i>) -> Vec> { + children(tr, "tc") +} + +/// Each row's cells as "text, span N, restart|continue", leaving out what a +/// cell doesn't have. +fn shape(tbl: Node) -> Vec> { + rows(tbl) + .into_iter() + .map(|tr| { + cells(tr) + .into_iter() + .map(|tc| { + let mut parts = vec![text(tc)]; + if grid_span(tc) > 1 { + parts.push(format!("span {}", grid_span(tc))); + } + parts.extend(v_merge(tc).map(str::to_owned)); + parts.retain(|p| !p.is_empty()); + parts.join(", ") + }) + .collect() + }) + .collect() +} + +/// The `w:jc` of a paragraph. +fn jc<'a>(p: Node<'a, '_>) -> Option<&'a str> { + child(child(p, "pPr")?, "jc").and_then(val) +} + +#[test] +fn grid_columns_follow_shares_and_sum_to_text_width() { + let docx = export(vec![ + table(vec![0.3, 0.7], vec![vec![td(0, "a"), td(1, "b")]]), + table(even(11), vec![vec![td(0, "c")]]), + ]); + let d = xml(docx.part("word/document.xml")); + let tbls = tables(&d); + assert_eq!(tbls.len(), 2); + + let pr = child(tbls[0], "tblPr").expect("tblPr"); + assert_eq!(child(pr, "tblStyle").and_then(val), Some("TableGrid")); + assert_eq!(tbl_w(tbls[0]), 9072); + let layout = child(pr, "tblLayout").and_then(|l| wattr(l, "type")); + assert_eq!(layout, Some("fixed")); + assert_eq!(grid_cols(tbls[0]), [2722, 6350]); + let widths: Vec = cells(rows(tbls[0])[0]).into_iter().map(tc_w).collect(); + assert_eq!(widths, [2722, 6350]); + + let cols = grid_cols(tbls[1]); + assert_eq!(cols[..10], [825; 10]); + assert_eq!( + cols[10], 822, + "the rounding remainder goes on the last column" + ); + assert_eq!(cols.iter().sum::(), 9072); +} + +#[test] +fn spans_are_encoded() { + let docx = export(vec![table( + even(3), + vec![ + vec![ + Cell { + colspan: 2, + rowspan: 2, + ..td(0, "A") + }, + td(2, "B"), + ], + vec![Cell { + rowspan: 2, + ..td(2, "C") + }], + vec![Cell { + colspan: 2, + ..td(0, "D") + }], + ], + )]); + let d = xml(docx.part("word/document.xml")); + let tbl = tables(&d)[0]; + assert_eq!( + shape(tbl), + vec![ + vec!["A, span 2, restart", "B"], + vec!["span 2, continue", "C, restart"], + vec!["D, span 2", "continue"], + ] + ); + + let continuations: Vec = d + .descendants() + .filter(|n| n.has_tag_name((W, "tc")) && v_merge(*n) == Some("continue")) + .collect(); + assert_eq!(continuations.len(), 2); + for tc in continuations { + assert_eq!( + val(tc_prop(tc, "vMerge").unwrap()), + None, + "a bare " + ); + let kids = elements(tc); + let tags: Vec<&str> = kids.iter().map(|k| k.tag_name().name()).collect(); + assert_eq!(tags, ["tcPr", "p"]); + assert!(elements(kids[1]).is_empty(), "an empty "); + } + assert_eq!(tc_w(cells(rows(tbl)[0])[0]), 6048); + assert_eq!(tc_w(cells(rows(tbl)[1])[0]), 6048, "as wide as its restart"); +} + +#[test] +fn spans_are_clamped_to_the_table() { + let docx = export(vec![ + table( + even(2), + vec![ + vec![ + Cell { + rowspan: 5, + ..td(0, "A") + }, + Cell { + colspan: 4, + ..td(1, "B") + }, + ], + vec![td(1, "C")], + ], + ), + // A span that runs into a merged cell stops short of it. + table( + even(3), + vec![ + vec![ + td(0, "a"), + Cell { + rowspan: 2, + ..td(1, "b") + }, + td(2, "c"), + ], + vec![Cell { + colspan: 3, + ..td(0, "wide") + }], + ], + ), + ]); + let d = xml(docx.part("word/document.xml")); + let tbls = tables(&d); + assert_eq!( + shape(tbls[0]), + vec![vec!["A, restart", "B"], vec!["continue", "C"]] + ); + assert_eq!(tc_w(cells(rows(tbls[0])[0])[1]), 4536); + assert_eq!( + shape(tbls[1]), + vec![vec!["a", "b, restart", "c"], vec!["wide", "continue", ""]] + ); +} + +#[test] +fn header_rows_repeat_and_are_shaded() { + let docx = export(vec![table( + even(2), + vec![ + vec![ + Cell { + rowspan: 2, + ..th(0, "H1") + }, + th(1, "H2"), + ], + vec![td(1, "body")], + vec![th(0, "side"), td(1, "x")], + ], + )]); + let d = xml(docx.part("word/document.xml")); + let trs = rows(tables(&d)[0]); + let repeats: Vec = trs + .iter() + .map(|tr| { + child(*tr, "trPr") + .and_then(|p| child(p, "tblHeader")) + .is_some() + }) + .collect(); + assert_eq!(repeats, [true, false, false]); + + let tcs: Vec = trs.iter().flat_map(|tr| cells(*tr)).collect(); + let expected = [ + ("H1", true, Some("TableHeader")), + ("H2", true, Some("TableHeader")), + ("", true, None), + ("body", false, None), + ("side", true, Some("TableHeader")), + ("x", false, None), + ]; + assert_eq!(tcs.len(), expected.len()); + for (tc, (label, shaded, style)) in tcs.iter().zip(expected) { + assert_eq!(text(*tc), label); + let shd = tc_prop(*tc, "shd"); + assert_eq!(shd.is_some(), shaded, "shading of {label:?}"); + if let Some(shd) = shd { + assert_eq!(wattr(shd, "val"), Some("clear")); + assert_eq!(wattr(shd, "fill"), Some("F2F2F2")); + } + let p_styles: Vec> = children(*tc, "p").into_iter().map(p_style).collect(); + assert_eq!(p_styles, [style], "paragraph style in {label:?}"); + } +} + +#[test] +fn rows_are_padded_and_cells_end_with_paragraph() { + let nested = table(even(2), vec![vec![td(0, "in"), td(1, "ner")]]); + let docx = export(vec![table( + even(3), + vec![ + vec![td(0, "a"), td(1, "b"), td(2, "c")], + vec![td(0, "short")], + vec![td(0, "x"), td(2, "z")], + vec![cell(0, vec![nested]), cell(1, vec![])], + ], + )]); + let d = xml(docx.part("word/document.xml")); + let tbl = tables(&d)[0]; + assert_eq!( + shape(tbl), + vec![ + vec!["a", "b", "c"], + vec!["short", "span 2"], + vec!["x", "", "z"], + vec!["inner", "", ""], + ] + ); + for tc in d.descendants().filter(|n| n.has_tag_name((W, "tc"))) { + let last = elements(tc).pop().unwrap(); + assert!( + last.has_tag_name((W, "p")), + "{} ends with a paragraph", + text(tc) + ); + } + let holder = cells(rows(tbl)[3])[0]; + let tags: Vec<&str> = elements(holder) + .iter() + .map(|n| n.tag_name().name()) + .collect(); + assert_eq!(tags, ["tcPr", "tbl", "p"]); +} + +#[test] +fn adjacent_tables_are_separated() { + let t = || table(even(1), vec![vec![td(0, "t")]]); + let docx = export(vec![ + t(), + t(), + table(vec![], vec![]), + t(), + para("after"), + t(), + table(even(1), vec![vec![cell(0, vec![t(), t()])]]), + ]); + let d = xml(docx.part("word/document.xml")); + let blocks = content(&d); + let tags: Vec<&str> = blocks.iter().map(|n| n.tag_name().name()).collect(); + assert_eq!( + tags, + ["tbl", "p", "tbl", "p", "tbl", "p", "tbl", "p", "tbl"] + ); + for sep in [blocks[1], blocks[3], blocks[7]] { + assert!(elements(sep).is_empty(), "an empty separator paragraph"); + } + assert_eq!(text(blocks[5]), "after"); + + let holder = cells(rows(blocks[8])[0])[0]; + let tags: Vec<&str> = elements(holder) + .iter() + .map(|n| n.tag_name().name()) + .collect(); + assert_eq!(tags, ["tcPr", "tbl", "p", "tbl", "p"]); +} + +#[test] +fn nested_table_fits_its_cell() { + let inner = || table(vec![0.3, 0.7], vec![vec![td(0, "i"), td(1, "j")]]); + let docx = export(vec![table( + vec![0.5, 0.5], + vec![ + vec![cell(0, vec![inner()]), td(1, "b")], + vec![Cell { + colspan: 2, + ..cell(0, vec![inner()]) + }], + ], + )]); + let d = xml(docx.part("word/document.xml")); + let tbls = tables(&d); + assert_eq!(tbls.len(), 3); + assert_eq!(tbl_w(tbls[0]), 9072); + assert_eq!(grid_cols(tbls[0]), [4536, 4536]); + for (tbl, width) in [(tbls[1], 4536 - 216), (tbls[2], 9072 - 216)] { + assert_eq!(tbl_w(tbl), width); + assert_eq!(grid_cols(tbl).iter().sum::(), width); + } + assert_eq!(grid_cols(tbls[1]), [1296, 3024]); +} + +#[test] +fn cell_align_sets_paragraph_jc() { + let docx = export(vec![table( + even(3), + vec![vec![ + Cell { + align: Align::Center, + ..cell( + 0, + vec![para("c"), list(false, 1, vec![item(vec![para("li")])])], + ) + }, + Cell { + align: Align::Right, + ..td(1, "r") + }, + td(2, "l"), + ]], + )]); + let d = xml(docx.part("word/document.xml")); + let aligns: Vec>> = cells(rows(tables(&d)[0])[0]) + .into_iter() + .map(|tc| children(tc, "p").into_iter().map(jc).collect()) + .collect(); + assert_eq!( + aligns, + vec![ + vec![Some("center"), Some("center")], + vec![Some("right")], + vec![None] + ] + ); +} + +#[test] +fn cell_content_starts_at_the_cell_edge() { + let inner = || { + table( + even(1), + vec![vec![cell( + 0, + vec![para("cell"), list(false, 1, vec![item(vec![para("li")])])], + )]], + ) + }; + let docx = export(vec![ + Block::Quote(vec![inner()]), + list(false, 1, vec![item(vec![para("lead"), inner()])]), + ]); + let d = xml(docx.part("word/document.xml")); + let tbls = tables(&d); + assert_eq!(tbl_w(tbls[0]), 9072 - 567); + assert_eq!(tbl_w(tbls[1]), 9072 - 720); + for tbl in tbls { + let ps = children(cells(rows(tbl)[0])[0], "p"); + assert_eq!(text(ps[0]), "cell"); + assert_eq!(p_style(ps[0]), None, "no Quote style inside a cell"); + assert_eq!(ind_left(ps[0]), None, "no list indent inside a cell"); + assert_eq!(text(ps[1]), "li"); + assert_eq!(num_pr(ps[1]).map(|(_, ilvl)| ilvl), Some(0)); + } +} + +#[test] +fn empty_tables_write_nothing_and_parse() { + let docx = export(vec![ + table(vec![], vec![]), + table(vec![], vec![vec![]]), + table(vec![1.0], vec![]), + para("after"), + ]); + for name in docx.0.keys() { + if name.ends_with(".xml") || name.ends_with(".rels") { + xml(docx.part(name)); + } + } + let d = xml(docx.part("word/document.xml")); + assert!(tables(&d).is_empty()); + let blocks = content(&d); + assert_eq!(blocks.len(), 1); + assert_eq!(text(blocks[0]), "after"); +} + +#[test] +fn large_page_exports_valid_docx_quickly() { + let mut blocks: Vec = (0..2000) + .map(|i| para(&format!("Paragraph {i} of a long page, with some text."))) + .collect(); + let grid = (0..50) + .map(|r| (0..20).map(|c| td(c, &format!("r{r}c{c}"))).collect()) + .collect(); + blocks.push(table(even(20), grid)); + + let started = Instant::now(); + let out = write_docx(&ExportDoc { blocks }, &meta(), &Assets::default()).expect("export"); + let elapsed = started.elapsed(); + assert!(elapsed < Duration::from_secs(5), "export took {elapsed:?}"); + + let docx = Docx(unzip(&out.bytes)); + for name in docx.0.keys() { + if name.ends_with(".xml") || name.ends_with(".rels") { + xml(docx.part(name)); + } + } + let d = xml(docx.part("word/document.xml")); + assert_eq!(content(&d).len(), 2001); + let trs = rows(tables(&d)[0]); + assert_eq!(trs.len(), 50); + assert!(trs.iter().all(|tr| cells(*tr).len() == 20)); +} + +const WP: &str = "http://schemas.openxmlformats.org/drawingml/2006/wordprocessingDrawing"; +const A: &str = "http://schemas.openxmlformats.org/drawingml/2006/main"; +const PIC: &str = "http://schemas.openxmlformats.org/drawingml/2006/picture"; + +const BLOB_A: &str = "aaaaaaaa-1111-4222-8333-444444444444"; +const BLOB_B: &str = "bbbbbbbb-1111-4222-8333-444444444444"; + +fn picture(format: PicFormat, width_px: u32, height_px: u32, scale: u32) -> Picture { + Picture { + format, + bytes: vec![1, 2, 3, width_px as u8], + width_px, + height_px, + scale, + } +} + +fn image_block(blob: &str, alt: &str) -> Block { + Block::Image { + source: ImageSource::Blob(blob.into()), + alt: alt.into(), + } +} + +/// Every `wp:extent` in the body as (cx, cy). +fn extents(d: &roxmltree::Document) -> Vec<(u64, u64)> { + d.descendants() + .filter(|n| n.has_tag_name((WP, "extent"))) + .map(|n| { + ( + n.attribute("cx").unwrap().parse().unwrap(), + n.attribute("cy").unwrap().parse().unwrap(), + ) + }) + .collect() +} + +#[test] +fn embedded_image_has_media_part_and_unique_docpr() { + let mut assets = Assets::default(); + assets.images.insert( + BLOB_A.into(), + ImageAsset::Embedded(picture(PicFormat::Png, 100, 50, 1)), + ); + assets.images.insert( + BLOB_B.into(), + ImageAsset::Embedded(picture(PicFormat::Jpeg, 80, 40, 1)), + ); + let docx = export_with( + vec![image_block(BLOB_A, "First "), image_block(BLOB_B, "")], + &assets, + ); + assert_eq!(docx.0["word/media/image1.png"], [1, 2, 3, 100]); + assert_eq!(docx.0["word/media/image2.jpeg"], [1, 2, 3, 80]); + + let ct = xml(docx.part("[Content_Types].xml")); + for (ext, mime) in [ + ("png", "image/png"), + ("jpeg", "image/jpeg"), + ("gif", "image/gif"), + ] { + assert!(ct.descendants().any(|n| n.has_tag_name((CT, "Default")) + && n.attribute("Extension") == Some(ext) + && n.attribute("ContentType") == Some(mime))); + } + + let d = xml(docx.part("word/document.xml")); + let root = d.root_element(); + for (prefix, ns) in [("wp", WP), ("a", A), ("pic", PIC)] { + assert_eq!(root.lookup_namespace_uri(Some(prefix)), Some(ns)); + } + let doc_prs: Vec<_> = d + .descendants() + .filter(|n| n.has_tag_name((WP, "docPr"))) + .collect(); + assert_eq!(doc_prs.len(), 2); + let ids: HashSet = doc_prs + .iter() + .map(|n| n.attribute("id").unwrap().parse().unwrap()) + .collect(); + assert_eq!(ids.len(), 2); + assert!(!ids.contains(&0)); + assert_eq!(doc_prs[0].attribute("descr"), Some("First ")); + + let rels = rels(&docx); + let blips: Vec<_> = d + .descendants() + .filter(|n| n.has_tag_name((A, "blip"))) + .map(|n| &rels[n.attribute((R, "embed")).unwrap()]) + .collect(); + assert_eq!(blips.len(), 2); + assert_eq!(blips[0].target, "media/image1.png"); + assert_eq!(blips[1].target, "media/image2.jpeg"); + assert!( + blips + .iter() + .all(|r| r.kind == format!("{REL_TYPE}image") && !r.external) + ); + // Word reports corruption without pic:spPr. + assert_eq!( + d.descendants() + .filter(|n| n.has_tag_name((PIC, "spPr"))) + .count(), + 2 + ); + assert_eq!(extents(&d), [(952_500, 476_250), (762_000, 381_000)]); +} + +#[test] +fn diagram_picture_is_half_size_and_capped() { + let mut assets = Assets::default(); + assets + .diagrams + .insert("m1".into(), picture(PicFormat::Png, 400, 200, 2)); + assets + .diagrams + .insert("m2".into(), picture(PicFormat::Png, 4000, 1000, 2)); + let docx = export_with( + vec![ + Block::Mermaid { + key: "m1".into(), + source: "graph TD".into(), + }, + Block::Mermaid { + key: "m2".into(), + source: "graph LR".into(), + }, + ], + &assets, + ); + let d = xml(docx.part("word/document.xml")); + assert_eq!(extents(&d), [(1_905_000, 952_500), (5_760_720, 1_440_180)]); + let descr: Vec<_> = d + .descendants() + .filter(|n| n.has_tag_name((WP, "docPr"))) + .map(|n| n.attribute("descr").unwrap()) + .collect(); + assert_eq!(descr, ["Mermaid diagram", "Mermaid diagram"]); +} + +#[test] +fn board_picture_is_described_by_its_label() { + let mut assets = Assets::default(); + assets + .diagrams + .insert("b1".into(), picture(PicFormat::Png, 200, 100, 2)); + let docx = export_with( + vec![Block::Board { + key: "b1".into(), + board_id: "x".into(), + label: "Roadmap".into(), + }], + &assets, + ); + let d = xml(docx.part("word/document.xml")); + let pr = d + .descendants() + .find(|n| n.has_tag_name((WP, "docPr"))) + .unwrap(); + assert_eq!(pr.attribute("descr"), Some("Diagram: Roadmap")); +} + +#[test] +fn image_in_table_cell_fits_the_cell() { + let mut assets = Assets::default(); + assets.images.insert( + BLOB_A.into(), + ImageAsset::Embedded(picture(PicFormat::Png, 3000, 1000, 1)), + ); + let cell = Cell { + col: 0, + colspan: 1, + rowspan: 1, + header: false, + align: Align::Left, + blocks: vec![image_block(BLOB_A, "wide")], + }; + let docx = export_with(vec![table(vec![0.5, 0.5], vec![vec![cell]])], &assets); + let d = xml(docx.part("word/document.xml")); + let max = (4536 - 216) * 635u64; + let e = extents(&d); + assert_eq!(e.len(), 1); + assert_eq!(e[0].0, max); + assert_eq!(e[0].1, max / 3); +} + +#[test] +fn tall_image_fits_the_page_height() { + // A phone screenshot: at the 16 cm text width it would be 34.6 cm + // tall, past the page. It shrinks to 21.2 cm, which with the Normal + // paragraph's 1.15 line spacing and spacing after still fits the + // 24.7 cm of body. + let mut assets = Assets::default(); + assets.images.insert( + BLOB_A.into(), + ImageAsset::Embedded(picture(PicFormat::Png, 1170, 2532, 1)), + ); + let docx = export_with(vec![image_block(BLOB_A, "screenshot")], &assets); + let d = xml(docx.part("word/document.xml")); + assert_eq!(extents(&d), [(3_532_240, 7_644_130)]); +} + +#[test] +fn same_blob_twice_shares_one_media_part() { + let mut assets = Assets::default(); + assets.images.insert( + BLOB_A.into(), + ImageAsset::Embedded(picture(PicFormat::Gif, 10, 10, 1)), + ); + let docx = export_with( + vec![image_block(BLOB_A, "a"), image_block(BLOB_A, "b")], + &assets, + ); + let media: Vec<_> = docx + .0 + .keys() + .filter(|k| k.starts_with("word/media/")) + .collect(); + assert_eq!(media, ["word/media/image1.gif"]); + let d = xml(docx.part("word/document.xml")); + let ids: HashSet<_> = d + .descendants() + .filter(|n| n.has_tag_name((WP, "docPr"))) + .map(|n| n.attribute("id").unwrap()) + .collect(); + assert_eq!(ids.len(), 2); +} + +#[test] +fn fallbacks_are_rendered_and_counted() { + let (doc, assets) = common::fallback_fixture(); + let out = write_docx(&doc, &meta(), &assets).expect("export"); + assert_eq!( + out.fallbacks, + Fallbacks { + diagram: 2, + image_unavailable: 2, + image_over_budget: 1, + unknown_element: 0, + } + ); + let docx = Docx(unzip(&out.bytes)); + let d = xml(docx.part("word/document.xml")); + let paras: Vec<(Option<&str>, String)> = content(&d) + .into_iter() + .map(|p| (p_style(p), text(p))) + .collect(); + assert_eq!( + paras, + [ + // The line break is a w:br, which text() skips. + (Some("Code"), "graph TD A --> B".to_owned()), + (Some("Caption"), "Mermaid diagram (not included)".to_owned()), + ( + Some("Caption"), + "Diagram: Roadmap (not included)".to_owned() + ), + ( + Some("Caption"), + "[Image not available] \u{2014} Org chart".to_owned() + ), + (Some("Caption"), "[Image too large to include]".to_owned()), + ( + Some("Caption"), + "[Image not available] \u{2014} Local file".to_owned() + ), + ] + ); + // The over-budget placeholder links to the blob. + let rels = rels(&docx); + let link = d + .descendants() + .find(|n| n.has_tag_name((W, "hyperlink"))) + .expect("hyperlink"); + let rel = &rels[link.attribute((R, "id")).unwrap()]; + assert_eq!( + rel.target, + format!( + "https://knot.example/api/blobs/{}", + common::OVER_BUDGET_BLOB + ) + ); + assert!(rel.external); + assert_eq!(text(link), "[Image too large to include]"); +} + +#[test] +fn absent_and_unlabelled_images_fall_back() { + let out = write_docx( + &ExportDoc { + blocks: vec![ + image_block(BLOB_A, ""), + Block::Image { + source: ImageSource::Unsupported, + alt: String::new(), + }, + ], + }, + &meta(), + &Assets::default(), + ) + .expect("export"); + assert_eq!(out.fallbacks.image_unavailable, 2); + let docx = Docx(unzip(&out.bytes)); + let d = xml(docx.part("word/document.xml")); + let texts: Vec<_> = content(&d).into_iter().map(text).collect(); + assert_eq!(texts, ["[Image not available]", "[Image not available]"]); +} + +#[test] +fn external_image_is_a_link() { + let docx = export(vec![ + Block::Image { + source: ImageSource::External("https://img.example/a.png?x=1&y=2".into()), + alt: "Remote".into(), + }, + Block::Image { + source: ImageSource::External("https://img.example/b.png".into()), + alt: String::new(), + }, + ]); + let d = xml(docx.part("word/document.xml")); + let rels = rels(&docx); + let got: Vec<_> = d + .descendants() + .filter(|n| n.has_tag_name((W, "hyperlink"))) + .map(|l| { + ( + text(l), + rels[l.attribute((R, "id")).unwrap()].target.clone(), + ) + }) + .collect(); + assert_eq!( + got, + [ + ( + "Remote".to_owned(), + "https://img.example/a.png?x=1&y=2".to_owned() + ), + ( + "https://img.example/b.png".to_owned(), + "https://img.example/b.png".to_owned() + ), + ] + ); +} + +#[test] +fn attachment_line_links_back_to_knot() { + let url = "/api/blobs/cccccccc-1111-4222-8333-444444444444"; + let docx = export(vec![ + Block::Attachment { + url: url.into(), + name: "budget.xlsx".into(), + size: Some(49152), + }, + Block::Attachment { + url: "javascript:alert(1)".into(), + name: "odd".into(), + size: None, + }, + ]); + let d = xml(docx.part("word/document.xml")); + let ps = content(&d); + assert_eq!(text(ps[0]), "\u{1F4CE} budget.xlsx (48 KB)"); + assert_eq!(p_style(ps[0]), None); + let link = child(ps[0], "hyperlink").expect("hyperlink"); + let rels = rels(&docx); + assert_eq!( + rels[link.attribute((R, "id")).unwrap()].target, + format!("https://knot.example{url}") + ); + // No size, and no link for a target that isn't a knot path. + assert_eq!(text(ps[1]), "\u{1F4CE} odd"); + assert!(child(ps[1], "hyperlink").is_none()); +} + +#[test] +fn table_in_list_item_is_indented() { + let docx = export(vec![list( + false, + 1, + vec![item(vec![ + para("lead"), + table(even(2), vec![vec![td(0, "a"), td(1, "b")]]), + ])], + )]); + let d = xml(docx.part("word/document.xml")); + let tbl = tables(&d)[0]; + let pr = child(tbl, "tblPr").expect("tblPr"); + let names: Vec<&str> = elements(pr).iter().map(|n| n.tag_name().name()).collect(); + assert_eq!(names, ["tblStyle", "tblW", "tblInd", "tblLayout"]); + let ind = child(pr, "tblInd").unwrap(); + assert_eq!(wattr(ind, "w"), Some("720")); + assert_eq!(wattr(ind, "type"), Some("dxa")); + assert_eq!(tbl_w(tbl), 9072 - 720); +} + +#[test] +fn table_in_quote_is_indented_and_top_level_is_not() { + let docx = export(vec![ + Block::Quote(vec![table(even(1), vec![vec![td(0, "a")]])]), + para("between"), + table(even(1), vec![vec![td(0, "b")]]), + ]); + let d = xml(docx.part("word/document.xml")); + let tbls = tables(&d); + let ind = child(child(tbls[0], "tblPr").unwrap(), "tblInd").unwrap(); + assert_eq!(wattr(ind, "w"), Some("567")); + assert!(child(child(tbls[1], "tblPr").unwrap(), "tblInd").is_none()); +} + +/// Most `word/document.xml` a crafted table may make per grid position the +/// walker counts: a merge continuation is an empty `w:tc` of about 80 bytes. +const BYTES_PER_GRID_CELL: usize = 100; + +#[test] +fn amplification_is_bounded() { + use common::{build, row_spanning_table}; + use knot_export::{MAX_PAGE_GRID_CELLS, MAX_TABLE_GRID_CELLS, walk}; + // A table inside every limit: 64 columns of cells spanning 64 rows. + // Between them the page holds empty rows; the file, a `vMerge` + // continuation cell per column. + let rows = MAX_TABLE_GRID_CELLS / 64; + let walked = walk(&build(vec![row_spanning_table(64, 64, rows)])); + assert!( + walked.unknown_elements.is_empty(), + "{:?}", + walked.unknown_elements + ); + assert!(matches!(walked.doc.blocks.as_slice(), [Block::Table(_)])); + let docx = export(walked.doc.blocks); + let body = docx.part("word/document.xml"); + let restarts = rows.div_ceil(64); + assert_eq!(body.matches("").count(), 64 * (rows - restarts)); + assert!( + body.len() < 64 * rows * BYTES_PER_GRID_CELL, + "{}", + body.len() + ); + + // A page of them: the page budget keeps three, and degrades the rest. + let walked = walk(&build( + (0..6).map(|_| row_spanning_table(64, 64, rows)).collect(), + )); + assert_eq!(walked.unknown_elements, ["table-limit"; 3]); + let docx = export(walked.doc.blocks); + let len = docx.part("word/document.xml").len(); + assert!(len < MAX_PAGE_GRID_CELLS * BYTES_PER_GRID_CELL, "{len}"); +} + +#[test] +fn xml_past_the_cap_fails_the_export() { + use knot_export::ExportError; + use knot_export::xml::MAX_XML_BYTES; + // Only a page far past the walker's limits gets here: the writer stops + // and fails instead of building the rest. + let text = "x".repeat(1 << 20); + let blocks = vec![para(&text); MAX_XML_BYTES / text.len() + 1]; + let result = write_docx(&ExportDoc { blocks }, &meta(), &Assets::default()); + assert!( + matches!(result, Err(ExportError::TooLarge)), + "{:?}", + result.err() + ); +} + +#[test] +fn output_buffer_is_sized_up_front() { + // A megabyte of picture: grown by doubling, the buffer would end up + // about twice that. + let mut assets = Assets::default(); + let pic = Picture { + format: PicFormat::Png, + bytes: common::noise(1 << 20), + width_px: 10, + height_px: 10, + scale: 1, + }; + assets + .images + .insert(BLOB_A.into(), ImageAsset::Embedded(pic)); + let blocks = vec![Block::Image { + source: ImageSource::Blob(BLOB_A.into()), + alt: String::new(), + }]; + let out = write_docx(&ExportDoc { blocks }, &meta(), &assets).expect("export"); + assert!(out.bytes.len() > 1 << 20); + let spare = out.bytes.capacity() - out.bytes.len(); + assert!(spare < 256 * 1024, "{spare} bytes spare"); +} diff --git a/crates/knot-export/tests/fixtures/2x2.webp b/crates/knot-export/tests/fixtures/2x2.webp new file mode 100644 index 00000000..694fcbee Binary files /dev/null and b/crates/knot-export/tests/fixtures/2x2.webp differ diff --git a/crates/knot-export/tests/media.rs b/crates/knot-export/tests/media.rs new file mode 100644 index 00000000..8e54405f --- /dev/null +++ b/crates/knot-export/tests/media.rs @@ -0,0 +1,178 @@ +use knot_export::assets::{PicFormat, Picture}; +use knot_export::media::{fit_cm, fit_emu, picture_from_png, prepare_image}; +use knot_export::xml::{ZipOut, esc_attr, esc_text}; + +fn png_bytes(w: u32, h: u32) -> Vec { + let mut out = Vec::new(); + let mut enc = png::Encoder::new(&mut out, w, h); + enc.set_color(png::ColorType::Rgb); + enc.set_depth(png::BitDepth::Eight); + let mut writer = enc.write_header().unwrap(); + writer + .write_image_data(&vec![200u8; (w * h * 3) as usize]) + .unwrap(); + writer.finish().unwrap(); + out +} + +fn pic(w: u32, h: u32, scale: u32) -> Picture { + Picture { + format: PicFormat::Png, + bytes: Vec::new(), + width_px: w, + height_px: h, + scale, + } +} + +#[test] +fn png_jpeg_gif_dimensions_are_read() { + let p = prepare_image(&png_bytes(3, 2)).unwrap(); + assert_eq!( + (p.format, p.width_px, p.height_px, p.scale), + (PicFormat::Png, 3, 2, 1) + ); + + // SOI, SOF0 (height 2, width 3, 3 components), EOI. + let jpeg = [ + 0xFF, 0xD8, 0xFF, 0xC0, 0x00, 0x11, 0x08, 0x00, 0x02, 0x00, 0x03, 0x03, 0x01, 0x22, 0x00, + 0x02, 0x11, 0x01, 0x03, 0x11, 0x01, 0xFF, 0xD9, + ]; + let p = prepare_image(&jpeg).unwrap(); + assert_eq!( + (p.format, p.width_px, p.height_px, p.scale), + (PicFormat::Jpeg, 3, 2, 1) + ); + + // GIF89a header, logical screen 3x2. + let gif = [ + b'G', b'I', b'F', b'8', b'9', b'a', 3, 0, 2, 0, 0, 0, 0, 0x3B, + ]; + let p = prepare_image(&gif).unwrap(); + assert_eq!( + (p.format, p.width_px, p.height_px, p.scale), + (PicFormat::Gif, 3, 2, 1) + ); +} + +#[test] +fn webp_is_converted_to_png() { + let webp = include_bytes!("fixtures/2x2.webp"); + let p = prepare_image(webp).unwrap(); + assert_eq!(p.format, PicFormat::Png); + assert_eq!((p.width_px, p.height_px, p.scale), (2, 2, 1)); + assert!(p.bytes.starts_with(b"\x89PNG")); + // The converted bytes decode as a 2x2 PNG. + let dec = png::Decoder::new(std::io::Cursor::new(&p.bytes)); + let reader = dec.read_info().unwrap(); + assert_eq!((reader.info().width, reader.info().height), (2, 2)); +} + +#[test] +fn webp_over_pixel_cap_is_refused() { + // RIFF/WEBP/VP8L header declaring 5000x5000, with no pixel data at all: + // refusal must come from the header, before any decoding. + let bits: u32 = 4999 | (4999 << 14); + let mut chunk = vec![0x2F]; + chunk.extend_from_slice(&bits.to_le_bytes()); + let mut webp = b"RIFF".to_vec(); + webp.extend_from_slice(&(4 + 8 + chunk.len() as u32).to_le_bytes()); + webp.extend_from_slice(b"WEBPVP8L"); + webp.extend_from_slice(&(chunk.len() as u32).to_le_bytes()); + webp.extend_from_slice(&chunk); + assert!(prepare_image(&webp).is_none()); +} + +#[test] +fn unknown_bytes_are_refused() { + assert!(prepare_image(b"").is_none()); + assert!(picture_from_png(b"".to_vec(), 2).is_none()); +} + +#[test] +fn picture_from_png_keeps_scale() { + let p = picture_from_png(png_bytes(4, 2), 2).unwrap(); + assert_eq!((p.width_px, p.height_px, p.scale), (4, 2, 2)); +} + +/// 24 cm in EMU: taller than any picture in `fit_scales_down_never_up`. +const TALL_EMU: u64 = 8_640_000; + +#[test] +fn fit_scales_down_never_up() { + assert_eq!( + fit_emu(&pic(800, 400, 1), 5_760_720, TALL_EMU), + (5_760_720, 2_880_360) + ); + assert_eq!( + fit_emu(&pic(100, 50, 1), 5_760_720, TALL_EMU), + (952_500, 476_250) + ); + assert_eq!( + fit_emu(&pic(400, 200, 2), 5_760_720, TALL_EMU), + (1_905_000, 952_500) + ); + + let (w, h) = fit_cm(&pic(1600, 800, 1), 16.0, 24.0); + assert!((w - 16.0).abs() < 1e-9 && (h - 8.0).abs() < 1e-9); + let (w, h) = fit_cm(&pic(96, 48, 1), 16.0, 24.0); + assert!((w - 2.54).abs() < 1e-9 && (h - 1.27).abs() < 1e-9); + let (w, _) = fit_cm(&pic(192, 96, 2), 16.0, 24.0); + assert!((w - 2.54).abs() < 1e-9); +} + +#[test] +fn fit_keeps_a_tall_picture_on_the_page() { + // A phone screenshot: at the text width it would be 34.6 cm tall. + let shot = pic(1170, 2532, 1); + let (w, h) = fit_emu(&shot, 5_760_720, TALL_EMU); + assert_eq!(h, TALL_EMU); + assert_eq!(w, 1170 * TALL_EMU / 2532); + let (w, h) = fit_cm(&shot, 16.0, 24.0); + assert!((h - 24.0).abs() < 1e-9, "{h}"); + assert!((w - 24.0 * 1170.0 / 2532.0).abs() < 1e-9, "{w}"); + + // Tall but narrow: only the height bound applies, and it never enlarges. + let (w, h) = fit_emu(&pic(100, 4000, 1), 5_760_720, TALL_EMU); + assert_eq!((w, h), (100 * TALL_EMU / 4000, TALL_EMU)); + assert_eq!( + fit_emu(&pic(100, 900, 1), 5_760_720, TALL_EMU), + (952_500, 8_572_500) + ); +} + +#[test] +fn escaping_drops_invalid_chars() { + assert_eq!(esc_text("a\u{1}&\u{B}"), "a<b>&"); + assert_eq!(esc_text("\"'\u{85}\u{FFFE}ok"), "\"'ok"); + assert_eq!(esc_attr("a\"b'c<&>\u{0}"), "a"b'c<&>"); +} + +#[test] +fn attribute_escaping_keeps_tabs_and_line_breaks() { + let s = "line 1\nline 2\r\n\tend"; + assert_eq!(esc_attr(s), "line 1 line 2 end"); + // A parser normalises literal whitespace in attributes to spaces; the + // character references survive. + let doc = format!("", esc_attr(s)); + let parsed = roxmltree::Document::parse(&doc).unwrap(); + assert_eq!(parsed.root_element().attribute("v"), Some(s)); + assert_eq!(esc_text("a\tb\nc"), "a\tb\nc"); +} + +#[test] +fn zip_first_entry_can_be_stored() { + let mut z = ZipOut::new(); + z.stored("mimetype", b"application/x").unwrap(); + z.deflated("a.xml", b"") + .unwrap(); + let bytes = z.finish().unwrap(); + let mut ar = zip::ZipArchive::new(std::io::Cursor::new(bytes)).unwrap(); + let first = ar.by_index(0).unwrap(); + assert_eq!(first.name(), "mimetype"); + assert_eq!(first.compression(), zip::CompressionMethod::Stored); + drop(first); + let second = ar.by_index(1).unwrap(); + assert_eq!(second.name(), "a.xml"); + assert_eq!(second.compression(), zip::CompressionMethod::Deflated); +} diff --git a/crates/knot-export/tests/odt.rs b/crates/knot-export/tests/odt.rs new file mode 100644 index 00000000..1866a277 --- /dev/null +++ b/crates/knot-export/tests/odt.rs @@ -0,0 +1,1839 @@ +mod common; + +use std::borrow::Cow; +use std::collections::{HashMap, HashSet}; +use std::io::Cursor; + +use common::odt::{ + DC, FO, MANIFEST, META, OFFICE, Odt, STYLE, TABLE, TEXT, XLINK, XML_NS, all, child, children, + cm, content, elements, list_depth, office_text, prop, style, style_name, text, +}; +use common::{unzip, xml}; +use knot_export::assets::{Assets, Fallbacks, ImageAsset, Meta, PicFormat, Picture}; +use knot_export::odt::{OdtDefaults, write_odt_with}; +use knot_export::{ + Align, Block, Cell, ExportDoc, ImageSource, Inline, LinkTarget, List, ListItem, Marks, Run, + Table, write_odt, +}; +use roxmltree::{Document, Node}; + +const MIMETYPE: &str = "application/vnd.oasis.opendocument.text"; +const DOC_ID: &str = "0b1c2d3e-4f50-4162-8374-8596a7b8c9d0"; +const USER_ID: &str = "7a6b5c4d-3e2f-4101-9283-746556473829"; +const HOSTILE: &str = "Q4 & \"final\" \u{1}😀 שלום"; + +fn meta() -> Meta { + Meta { + title: "Q4 Plan".into(), + exported_on: "23 Oct 2026".into(), + base_url: "https://knot.example".into(), + } +} + +fn export_with(blocks: Vec, assets: &Assets) -> Odt { + let out = write_odt(&ExportDoc { blocks }, &meta(), assets).expect("export"); + Odt(unzip(&out.bytes)) +} + +fn export(blocks: Vec) -> Odt { + export_with(blocks, &Assets::default()) +} + +/// Parse every XML part; panics on the first malformed one. +fn parse_all(odt: &Odt) { + for name in odt.0.keys() { + if name.ends_with(".xml") { + xml(odt.part(name)); + } + } +} + +fn run(text: &str) -> Run { + Run { + text: text.into(), + marks: Marks::default(), + } +} + +fn para(text: &str) -> Block { + Block::Paragraph(vec![Inline::Run(run(text))]) +} + +fn link(target: LinkTarget, text: &str) -> Inline { + Inline::Link { + target, + runs: vec![run(text)], + } +} + +fn code(text: &str) -> Block { + Block::Code { + language: String::new(), + text: text.into(), + } +} + +/// One of every text block and inline. +fn kitchen_sink() -> Vec { + vec![ + Block::Heading { + level: 2, + content: vec![Inline::Run(run("Heading"))], + }, + Block::Paragraph(vec![ + Inline::Run(Run { + text: "all marks".into(), + marks: Marks { + bold: true, + italic: true, + underline: true, + strike: true, + code: false, + }, + }), + Inline::Run(Run { + text: "code".into(), + marks: Marks { + code: true, + ..Marks::default() + }, + }), + Inline::Break, + link(LinkTarget::Web("https://a.example/".into()), "web"), + link(LinkTarget::Doc(DOC_ID.into()), "page"), + ]), + Block::Quote(vec![para("quoted")]), + code("fn main() {\n\tprintln!();\n}"), + Block::Rule, + ] +} + +fn list(ordered: bool, start: u32, items: Vec) -> Block { + Block::List(List { + ordered, + start, + items, + }) +} + +fn item(blocks: Vec) -> ListItem { + ListItem { + checked: None, + blocks, + } +} + +fn check_item(checked: bool, blocks: Vec) -> ListItem { + ListItem { + checked: Some(checked), + blocks, + } +} + +fn cell(col: usize, blocks: Vec) -> Cell { + Cell { + col, + colspan: 1, + rowspan: 1, + header: false, + align: Align::Left, + blocks, + } +} + +fn td(col: usize, text: &str) -> Cell { + cell(col, vec![para(text)]) +} + +fn th(col: usize, text: &str) -> Cell { + Cell { + header: true, + ..td(col, text) + } +} + +fn table(shares: Vec, rows: Vec>) -> Block { + Block::Table(Table { shares, rows }) +} + +/// `n` equal column shares. +fn even(n: usize) -> Vec { + vec![1.0 / n as f64; n] +} + +fn assert_close(got: f64, want: f64, what: &str) { + assert!((got - want).abs() < 1e-9, "{what}: {got} != {want}"); +} + +/// The children of a paragraph as "text", "s ", "tab" or "line-break". +fn pieces(p: Node) -> Vec { + p.children() + .map(|n| { + if n.is_text() { + n.text().unwrap().to_owned() + } else if n.has_tag_name((TEXT, "s")) { + format!("s {}", n.attribute((TEXT, "c")).unwrap_or("-")) + } else { + n.tag_name().name().to_owned() + } + }) + .collect() +} + +/// The style a node names with `ns:style-name`, from `content.xml`'s +/// automatic styles. +fn auto_style<'a, 'i>(d: &'a Document<'i>, n: Node, ns: &str) -> Node<'a, 'i> { + style(d, n.attribute((ns, "style-name")).expect("style-name")) +} + +// Package. + +#[test] +fn mimetype_is_first_and_stored_and_manifest_is_complete() { + let out = write_odt( + &ExportDoc { + blocks: kitchen_sink(), + }, + &meta(), + &Assets::default(), + ) + .expect("export"); + let bytes = &out.bytes; + + // The first local header: stored, no extra field, then the name and the + // bytes of the media type. + assert_eq!(&bytes[..4], b"PK\x03\x04"); + assert_eq!(u16::from_le_bytes([bytes[8], bytes[9]]), 0, "stored"); + let name_len = usize::from(u16::from_le_bytes([bytes[26], bytes[27]])); + assert_eq!( + u16::from_le_bytes([bytes[28], bytes[29]]), + 0, + "no extra field" + ); + assert_eq!(&bytes[30..30 + name_len], b"mimetype"); + let data = 30 + name_len; + assert_eq!(&bytes[data..data + MIMETYPE.len()], MIMETYPE.as_bytes()); + + let mut zip = zip::ZipArchive::new(Cursor::new(bytes)).expect("zip"); + let first = zip.by_index(0).expect("first entry"); + assert_eq!(first.name(), "mimetype"); + assert_eq!(first.compression(), zip::CompressionMethod::Stored); + drop(first); + + let odt = Odt(unzip(bytes)); + assert_eq!(odt.part("mimetype"), MIMETYPE); + parse_all(&odt); + + let manifest = xml(odt.part("META-INF/manifest.xml")); + assert_eq!( + manifest.root_element().attribute((MANIFEST, "version")), + Some("1.3") + ); + let entries: HashMap<&str, &str> = all(&manifest, MANIFEST, "file-entry") + .into_iter() + .map(|e| { + ( + e.attribute((MANIFEST, "full-path")).unwrap(), + e.attribute((MANIFEST, "media-type")).unwrap(), + ) + }) + .collect(); + assert_eq!(entries.get("/"), Some(&MIMETYPE)); + for name in odt.0.keys() { + if name != "mimetype" && !name.starts_with("META-INF/") { + assert!( + entries.contains_key(name.as_str()), + "{name} not in manifest" + ); + } + } + for path in entries.keys().filter(|p| **p != "/") { + assert!(odt.0.contains_key(*path), "manifest lists missing {path}"); + } + for part in ["content.xml", "styles.xml", "meta.xml"] { + assert_eq!(entries.get(part), Some(&"text/xml"), "{part}"); + let d = xml(odt.part(part)); + assert_eq!( + d.root_element().attribute((OFFICE, "version")), + Some("1.3"), + "{part}" + ); + } + + let meta_xml = xml(odt.part("meta.xml")); + let find = |ns: &str, name: &str| { + meta_xml + .descendants() + .find(|n| n.has_tag_name((ns, name))) + .and_then(|n| n.text()) + }; + assert_eq!(find(META, "generator"), Some("knot")); + assert_eq!(find(DC, "title"), Some("Q4 Plan")); + for (ns, name) in [(DC, "creator"), (META, "initial-creator")] { + assert!(find(ns, name).is_none(), "no {name}"); + } +} + +#[test] +fn empty_page_has_title_subtitle_and_footer_fields() { + let odt = export(vec![]); + let d = xml(odt.part("content.xml")); + let kids = elements(office_text(&d)); + assert_eq!(kids.len(), 2); + let expected = [ + ("Title", "Q4 Plan"), + ("Subtitle", "Exported from knot · 23 Oct 2026"), + ]; + for (p, (style, copy)) in kids.iter().zip(expected) { + assert!(p.has_tag_name((TEXT, "p"))); + assert_eq!(style_name(*p), Some(style)); + assert_eq!(text(*p), copy); + } + + let styles = xml(odt.part("styles.xml")); + let master = all(&styles, STYLE, "master-page") + .into_iter() + .find(|m| m.attribute((STYLE, "name")) == Some("Standard")) + .expect("master page Standard"); + let layout = all(&styles, STYLE, "page-layout") + .into_iter() + .find(|l| l.attribute((STYLE, "name")) == master.attribute((STYLE, "page-layout-name"))) + .expect("page layout"); + let page = child(layout, STYLE, "page-layout-properties").expect("page properties"); + assert_eq!(page.attribute((FO, "page-width")), Some("21cm")); + assert_eq!(page.attribute((FO, "page-height")), Some("29.7cm")); + assert_eq!( + page.attribute((STYLE, "print-orientation")), + Some("portrait") + ); + for side in ["top", "left", "right"] { + let margin = page.attribute((FO, format!("margin-{side}").as_str())); + assert_close(cm(margin.unwrap()), 2.5, side); + } + let footer_props = child(layout, STYLE, "footer-style") + .and_then(|f| child(f, STYLE, "header-footer-properties")) + .expect("footer properties"); + let bottom = cm(page.attribute((FO, "margin-bottom")).unwrap()) + + cm(footer_props.attribute((FO, "min-height")).unwrap()) + + cm(footer_props.attribute((FO, "margin-top")).unwrap()); + assert_close(bottom, 2.5, "bottom margin, footer height and spacing"); + + let footer = child(master, STYLE, "footer").expect("footer"); + let p = child(footer, TEXT, "p").expect("footer paragraph"); + assert_eq!(style_name(p), Some("Footer")); + assert!(child(p, TEXT, "page-number").is_some()); + assert!(child(p, TEXT, "page-count").is_some()); + assert_eq!(text(p), "Page 1 of 1"); + let footer_style = style(&styles, "Footer"); + assert_eq!(prop(footer_style, FO, "text-align"), Some("center")); +} + +#[test] +fn package_defaults_are_written_as_given_odt() { + let builtin = OdtDefaults::builtin(); + let defaults = OdtDefaults { + styles_xml: Cow::Owned(builtin.styles_xml.replace("Calibri", "Arial")), + }; + let out = write_odt_with( + &ExportDoc { + blocks: vec![para("x")], + }, + &meta(), + &Assets::default(), + &defaults, + ) + .expect("export"); + let odt = Odt(unzip(&out.bytes)); + assert_eq!(odt.part("styles.xml"), defaults.styles_xml); + assert!(odt.part("styles.xml").contains("Arial")); +} + +// Styles. + +#[test] +fn styles_cover_the_vocabulary() { + let odt = export(vec![]); + let styles = xml(odt.part("styles.xml")); + for (name, family, display) in [ + ("Standard", "paragraph", None), + ("Title", "paragraph", Some("Title")), + ("Subtitle", "paragraph", Some("Subtitle")), + ("Quotations", "paragraph", Some("Quotations")), + ( + "Preformatted_20_Text", + "paragraph", + Some("Preformatted Text"), + ), + ("Source_20_Text", "text", Some("Source Text")), + ("List_20_Paragraph", "paragraph", Some("List Paragraph")), + ("Table_20_Contents", "paragraph", Some("Table Contents")), + ("Table_20_Heading", "paragraph", Some("Table Heading")), + ("Internet_20_link", "text", Some("Internet link")), + ("Caption", "paragraph", Some("Caption")), + ("Footer", "paragraph", Some("Footer")), + ("Horizontal_20_Line", "paragraph", Some("Horizontal Line")), + ] { + let s = style(&styles, name); + assert!(s.has_tag_name((STYLE, "style")), "{name} is a style:style"); + assert_eq!(s.attribute((STYLE, "family")), Some(family), "{name}"); + if let Some(display) = display { + assert_eq!(s.attribute((STYLE, "display-name")), Some(display)); + } + } + + let pre = style(&styles, "Preformatted_20_Text"); + assert_eq!(prop(pre, STYLE, "font-name"), Some("Consolas")); + assert_eq!(prop(pre, FO, "background-color"), Some("#f5f5f5")); + let source = style(&styles, "Source_20_Text"); + assert_eq!(prop(source, STYLE, "font-name"), Some("Consolas")); + assert_eq!( + prop(style(&styles, "Caption"), FO, "font-style"), + Some("italic") + ); + let fonts: Vec<&str> = all(&styles, STYLE, "font-face") + .into_iter() + .filter_map(|f| f.attribute((STYLE, "name"))) + .collect(); + assert_eq!(fonts, ["Calibri", "Consolas"]); + let default = all(&styles, STYLE, "default-style") + .into_iter() + .find(|s| s.attribute((STYLE, "family")) == Some("paragraph")) + .expect("default paragraph style"); + assert_eq!(prop(default, STYLE, "font-name"), Some("Calibri")); +} + +#[test] +fn headings_have_outline_levels_and_styles() { + let heading = |level| Block::Heading { + level, + content: vec![Inline::Run(run("h"))], + }; + let odt = export((0..=7).map(heading).collect()); + let d = xml(odt.part("content.xml")); + let got: Vec<(&str, String, String)> = content(&d) + .into_iter() + .map(|h| { + ( + h.tag_name().name(), + h.attribute((TEXT, "outline-level")).unwrap().to_owned(), + style_name(h).unwrap().to_owned(), + ) + }) + .collect(); + let expected: Vec<(&str, String, String)> = [1, 1, 2, 3, 4, 5, 6, 6] + .into_iter() + .map(|l| ("h", l.to_string(), format!("Heading_20_{l}"))) + .collect(); + assert_eq!(got, expected, "levels 0 and 7 are clamped"); + + let styles = xml(odt.part("styles.xml")); + for level in 1..=6 { + let s = style(&styles, &format!("Heading_20_{level}")); + assert_eq!( + s.attribute((STYLE, "display-name")), + Some(format!("Heading {level}").as_str()) + ); + assert_eq!( + s.attribute((STYLE, "default-outline-level")), + Some(level.to_string().as_str()) + ); + assert_eq!(prop(s, FO, "font-weight"), Some("bold")); + } + let outline = all(&styles, TEXT, "outline-style"); + assert_eq!(outline.len(), 1); + let levels = children(outline[0], TEXT, "outline-level-style"); + assert_eq!(levels.len(), 10); + for l in levels { + assert_eq!(l.attribute((STYLE, "num-format")), Some(""), "unnumbered"); + } +} + +#[test] +fn list_styles_have_ten_levels_and_the_same_indents() { + let odt = export(vec![]); + let styles = xml(odt.part("styles.xml")); + let mut indents = Vec::new(); + for (name, level_tag) in [ + ("List_20_Bullet", "list-level-style-bullet"), + ("Numbering_20_123", "list-level-style-number"), + ("List_20_Check", "list-level-style-number"), + ] { + let s = style(&styles, name); + assert!(s.has_tag_name((TEXT, "list-style")), "{name}"); + let levels = elements(s); + assert_eq!(levels.len(), 10, "{name}"); + let mut margins = Vec::new(); + for (i, l) in levels.into_iter().enumerate() { + assert!(l.has_tag_name((TEXT, level_tag)), "{name} level {i}"); + assert_eq!( + l.attribute((TEXT, "level")), + Some((i + 1).to_string().as_str()) + ); + let align = l + .descendants() + .find(|n| n.has_tag_name((STYLE, "list-level-label-alignment"))) + .expect("label alignment"); + let margin = cm(align.attribute((FO, "margin-left")).unwrap()); + assert_close(margin, 1.27 * (i + 1) as f64, name); + margins.push(margin); + if name == "List_20_Check" { + assert_eq!(l.attribute((STYLE, "num-format")), Some(""), "no label"); + } + } + indents.push(margins); + } + assert_eq!(indents[0], indents[1]); + assert_eq!(indents[0], indents[2]); + let numbers = elements(style(&styles, "Numbering_20_123")); + assert_eq!(numbers[0].attribute((STYLE, "num-format")), Some("1")); +} + +#[test] +fn every_referenced_style_exists_odt() { + let mut blocks = kitchen_sink(); + blocks.push(list( + false, + 1, + vec![ + item(vec![para("plain")]), + check_item(true, vec![para("done")]), + ], + )); + blocks.push(list( + true, + 1, + vec![item(vec![ + para("lead"), + table( + vec![0.5, 0.5], + vec![ + vec![th(0, "h"), th(1, "h")], + vec![ + Cell { + align: Align::Center, + ..td(0, "c") + }, + cell(1, vec![code("x")]), + ], + ], + ), + ])], + )); + let odt = export(blocks); + let styles = xml(odt.part("styles.xml")); + let content_xml = xml(odt.part("content.xml")); + + // Declared styles by (kind, name). A list style's kind is "list". + let mut declared = HashSet::new(); + for d in [&styles, &content_xml] { + for n in d.descendants().filter(Node::is_element) { + let Some(name) = n.attribute((STYLE, "name")) else { + continue; + }; + let kind = match n.tag_name().name() { + "style" => n.attribute((STYLE, "family")).unwrap(), + "list-style" => "list", + other => other, + }; + declared.insert((kind.to_owned(), name.to_owned())); + } + } + + let mut seen = HashSet::new(); + for d in [&styles, &content_xml] { + for n in d.descendants().filter(Node::is_element) { + let family = n.attribute((STYLE, "family")); + let refs: Vec<(&str, Option<&str>)> = + match (n.tag_name().namespace(), n.tag_name().name()) { + (Some(TEXT), "p" | "h") => vec![("paragraph", style_name(n))], + (Some(TEXT), "span" | "a") => vec![("text", style_name(n))], + (Some(TEXT), "list") => vec![("list", style_name(n))], + (Some(TEXT), "list-item") => { + vec![("list", n.attribute((TEXT, "style-override")))] + } + (Some(TABLE), "table" | "table-column" | "table-cell") => { + let kind = match n.tag_name().name() { + "table" => "table", + "table-column" => "table-column", + _ => "table-cell", + }; + vec![(kind, n.attribute((TABLE, "style-name")))] + } + (Some(STYLE), "style") => vec![ + (family.unwrap(), n.attribute((STYLE, "parent-style-name"))), + (family.unwrap(), n.attribute((STYLE, "next-style-name"))), + ], + (Some(STYLE), "master-page") => { + vec![("page-layout", n.attribute((STYLE, "page-layout-name")))] + } + _ => vec![], + }; + for (kind, name) in refs { + let Some(name) = name else { continue }; + assert!( + declared.contains(&(kind.to_owned(), name.to_owned())), + "{} references missing {kind} style {name}", + n.tag_name().name() + ); + seen.insert(name.to_owned()); + } + } + } + for name in [ + "Title", + "Subtitle", + "Heading_20_2", + "Source_20_Text", + "Internet_20_link", + "Quotations", + "Preformatted_20_Text", + "Horizontal_20_Line", + "Footer", + "List_20_Bullet", + "Numbering_20_123", + "List_20_Check", + "List_20_Paragraph", + "Table_20_Contents", + "Table_20_Heading", + ] { + assert!(seen.contains(name), "the sample should use {name}"); + } +} + +// Text. + +#[test] +fn marks_links_and_mentions() { + let mut assets = Assets::default(); + assets.user_names.insert(USER_ID.into(), "Ada".into()); + let bold = Marks { + bold: true, + ..Marks::default() + }; + let web = "https://a.example/x?y=1&z"; + let odt = export_with( + vec![Block::Paragraph(vec![ + Inline::Run(Run { + text: "loud".into(), + marks: Marks { + bold: true, + italic: true, + underline: true, + strike: true, + code: false, + }, + }), + Inline::Run(Run { + text: "b1".into(), + marks: bold.clone(), + }), + Inline::Run(Run { + text: "b2".into(), + marks: bold, + }), + Inline::Run(Run { + text: "x = 1".into(), + marks: Marks { + code: true, + ..Marks::default() + }, + }), + link(LinkTarget::Web(web.into()), "web"), + link(LinkTarget::Doc("d1".into()), "page"), + link(LinkTarget::User(USER_ID.into()), "@Old Name"), + link(LinkTarget::User("gone".into()), "@Gone"), + link(LinkTarget::Plain, "knot://time/2026-10-23"), + ])], + &assets, + ); + let d = xml(odt.part("content.xml")); + let p = content(&d)[0]; + assert_eq!(style_name(p), Some("Standard")); + assert_eq!( + text(p), + "loudb1b2x = 1webpage@Ada@Goneknot://time/2026-10-23" + ); + + let spans = children(p, TEXT, "span"); + assert_eq!(spans.len(), 4); + let loud = auto_style(&d, spans[0], TEXT); + assert_eq!(loud.attribute((STYLE, "family")), Some("text")); + for (ns, name, value) in [ + (FO, "font-weight", "bold"), + (FO, "font-style", "italic"), + (STYLE, "text-underline-style", "solid"), + (STYLE, "text-line-through-style", "solid"), + ] { + assert_eq!(prop(loud, ns, name), Some(value), "{name}"); + } + assert_eq!( + style_name(spans[1]), + style_name(spans[2]), + "one automatic style per mark set" + ); + assert_ne!(style_name(spans[0]), style_name(spans[1])); + assert_eq!(style_name(spans[3]), Some("Source_20_Text")); + assert_eq!(text(spans[3]), "x = 1"); + + let links = children(p, TEXT, "a"); + let got: Vec<(Option<&str>, String)> = links + .iter() + .map(|a| (a.attribute((XLINK, "href")), text(*a))) + .collect(); + assert_eq!( + got, + [ + (Some(web), "web".to_owned()), + (Some("https://knot.example/doc/d1"), "page".to_owned()), + ] + ); + for a in &links { + assert_eq!(a.attribute((XLINK, "type")), Some("simple")); + assert_eq!(style_name(*a), Some("Internet_20_link")); + } + assert!(odt.part("content.xml").contains("x?y=1&z")); +} + +#[test] +fn code_preserves_spaces_and_tabs() { + let odt = export(vec![code(" a\tb c\r\n d\re")]); + let d = xml(odt.part("content.xml")); + let blocks = content(&d); + assert_eq!(blocks.len(), 1, "one paragraph"); + let p = blocks[0]; + assert_eq!(style_name(p), Some("Preformatted_20_Text")); + assert_eq!( + pieces(p), + [ + "s 2", + "a", + "tab", + "b ", + "s 2", + "c", + "line-break", + "s 1", + "d", + "line-break", + "e" + ] + ); + assert_eq!(text(p), " a\tb c\n d\ne"); +} + +#[test] +fn normal_text_preserves_tabs_and_double_spaces() { + let bold = Marks { + bold: true, + ..Marks::default() + }; + let meta = Meta { + title: " Indented title".into(), + ..meta() + }; + let blocks = vec![ + para("a\tb c"), + // A space after another run's trailing space would collapse. + Block::Paragraph(vec![ + Inline::Run(run("x ")), + Inline::Run(Run { + text: " y".into(), + marks: bold, + }), + Inline::Break, + Inline::Run(run(" z ")), + ]), + ]; + let out = write_odt(&ExportDoc { blocks }, &meta, &Assets::default()).expect("export"); + let odt = Odt(unzip(&out.bytes)); + let d = xml(odt.part("content.xml")); + let title = elements(office_text(&d))[0]; + assert_eq!(pieces(title), ["s 2", "Indented ", "s 1", "title"]); + + let blocks = content(&d); + assert_eq!(pieces(blocks[0]), ["a", "tab", "b ", "s 1", "c"]); + assert_eq!(pieces(blocks[1]), ["x ", "span", "line-break", "s 1", "z "]); + let span = child(blocks[1], TEXT, "span").unwrap(); + assert_eq!(pieces(span), ["s 1", "y"]); + assert_eq!(text(blocks[1]), "x y\n z "); +} + +#[test] +fn quote_and_rule_styles_odt() { + let odt = export(vec![Block::Quote(vec![para("quoted")]), Block::Rule]); + let d = xml(odt.part("content.xml")); + let blocks = content(&d); + assert_eq!(blocks.len(), 2); + assert_eq!(style_name(blocks[0]), Some("Quotations")); + assert_eq!(text(blocks[0]), "quoted"); + assert_eq!(style_name(blocks[1]), Some("Horizontal_20_Line")); + assert!(blocks[1].children().next().is_none(), "an empty paragraph"); + + let styles = xml(odt.part("styles.xml")); + let quote = style(&styles, "Quotations"); + assert_close( + cm(prop(quote, FO, "margin-left").unwrap()), + 1.0, + "quote indent", + ); + assert!(prop(style(&styles, "Horizontal_20_Line"), FO, "border-bottom").is_some()); +} + +#[test] +fn text_with_xml_specials_and_controls_is_well_formed_odt() { + let mut assets = Assets::default(); + assets.user_names.insert(USER_ID.into(), HOSTILE.into()); + let doc = ExportDoc { + blocks: vec![ + para(HOSTILE), + Block::Heading { + level: 1, + content: vec![Inline::Run(run(HOSTILE))], + }, + Block::Code { + language: HOSTILE.into(), + text: HOSTILE.into(), + }, + Block::Paragraph(vec![ + link( + LinkTarget::Web("https://a.example/?q=&b=\"c\"".into()), + HOSTILE, + ), + link(LinkTarget::User(USER_ID.into()), "@x"), + ]), + ], + }; + let meta = Meta { + title: HOSTILE.into(), + ..meta() + }; + let out = write_odt(&doc, &meta, &assets).expect("export"); + let odt = Odt(unzip(&out.bytes)); + parse_all(&odt); + + let clean = HOSTILE.replace('\u{1}', ""); + let meta_xml = xml(odt.part("meta.xml")); + let title = meta_xml + .descendants() + .find(|n| n.has_tag_name((DC, "title"))) + .and_then(|n| n.text()); + assert_eq!(title, Some(clean.as_str())); + let d = xml(odt.part("content.xml")); + assert_eq!(text(elements(office_text(&d))[0]), clean); + let texts: Vec = content(&d).into_iter().map(text).collect(); + assert_eq!( + texts, + [ + clean.clone(), + clean.clone(), + clean.clone(), + format!("{clean}@{clean}") + ] + ); + let a = all(&d, TEXT, "a")[0]; + assert_eq!( + a.attribute((XLINK, "href")), + Some("https://a.example/?q=&b=\"c\"") + ); +} + +// Lists. + +#[test] +fn lists_restart_and_checklists_use_boxes() { + let odt = export(vec![ + list( + true, + 5, + vec![item(vec![para("five")]), item(vec![para("six")])], + ), + para("between"), + list(true, 1, vec![item(vec![para("one")])]), + list( + false, + 1, + vec![ + check_item(true, vec![para("done")]), + check_item(false, vec![para("todo")]), + ], + ), + ]); + let d = xml(odt.part("content.xml")); + let blocks = content(&d); + let tags: Vec<&str> = blocks.iter().map(|n| n.tag_name().name()).collect(); + assert_eq!(tags, ["list", "p", "list", "list"]); + let lists = [blocks[0], blocks[2], blocks[3]]; + let styles: Vec> = lists.iter().map(|l| style_name(*l)).collect(); + assert_eq!( + styles, + [ + Some("Numbering_20_123"), + Some("Numbering_20_123"), + Some("List_20_Bullet") + ] + ); + let ids: HashSet<&str> = lists + .iter() + .map(|l| l.attribute((XML_NS, "id")).expect("xml:id")) + .collect(); + assert_eq!(ids.len(), 3, "unique ids"); + for l in lists { + assert_eq!(l.attribute((TEXT, "continue-list")), None); + assert_eq!(l.attribute((TEXT, "continue-numbering")), None); + } + + let starts = |l: Node| -> Vec> { + children(l, TEXT, "list-item") + .into_iter() + .map(|i| i.attribute((TEXT, "start-value")).map(str::to_owned)) + .collect() + }; + assert_eq!(starts(blocks[0]), [Some("5".to_owned()), None]); + assert_eq!(starts(blocks[2]), [Some("1".to_owned())], "restarts at 1"); + assert_eq!(starts(blocks[3]), [None, None]); + + let checks = children(blocks[3], TEXT, "list-item"); + let got: Vec<(Option<&str>, String)> = checks + .iter() + .map(|i| { + ( + i.attribute((TEXT, "style-override")), + text(child(*i, TEXT, "p").unwrap()), + ) + }) + .collect(); + assert_eq!( + got, + [ + (Some("List_20_Check"), "\u{2611} done".to_owned()), + (Some("List_20_Check"), "\u{2610} todo".to_owned()), + ] + ); + for p in all(&d, TEXT, "p") + .into_iter() + .filter(|p| list_depth(*p) > 0) + { + assert_eq!(style_name(p), Some("List_20_Paragraph")); + } +} + +#[test] +fn mixed_nested_checklist_keeps_levels() { + let odt = export(vec![list( + false, + 1, + vec![ + item(vec![para("plain")]), + check_item( + true, + vec![ + para("parent"), + list(false, 1, vec![check_item(true, vec![para("child")])]), + ], + ), + ], + )]); + let d = xml(odt.part("content.xml")); + let got: Vec<(String, usize, Option<&str>)> = all(&d, TEXT, "list-item") + .into_iter() + .map(|i| { + ( + text(child(i, TEXT, "p").unwrap()), + list_depth(i), + i.attribute((TEXT, "style-override")), + ) + }) + .collect(); + assert_eq!( + got, + [ + ("plain".to_owned(), 1, None), + ("\u{2611} parent".to_owned(), 1, Some("List_20_Check")), + ("\u{2611} child".to_owned(), 2, Some("List_20_Check")), + ] + ); + let nested = all(&d, TEXT, "list")[1]; + assert_eq!(style_name(nested), Some("List_20_Bullet")); + assert_eq!( + nested.attribute((XML_NS, "id")), + None, + "only the outer list has an id" + ); +} + +#[test] +fn item_without_leading_paragraph_still_has_a_marker_paragraph() { + let odt = export(vec![list(true, 1, vec![item(vec![code("x")])])]); + let d = xml(odt.part("content.xml")); + let it = all(&d, TEXT, "list-item")[0]; + let ps = children(it, TEXT, "p"); + assert_eq!(ps.len(), 2); + assert_eq!(style_name(ps[0]), Some("List_20_Paragraph")); + assert_eq!(text(ps[0]), ""); + assert_eq!(style_name(ps[1]), Some("Preformatted_20_Text")); +} + +#[test] +fn deep_list_clamps_to_ten_levels() { + let mut inner = vec![item(vec![para("leaf")])]; + for i in 0..11 { + inner = vec![item(vec![ + para(&format!("x{i}")), + list(false, 1, inner), + para(&format!("after{i}")), + ])]; + } + let odt = export(vec![list(false, 1, inner)]); + parse_all(&odt); + let d = xml(odt.part("content.xml")); + let lists = all(&d, TEXT, "list"); + assert_eq!(lists.len(), 10); + assert_eq!(lists.iter().map(|l| list_depth(*l)).max(), Some(9)); + let ps = all(&d, TEXT, "p"); + let texts: Vec = ps.iter().skip(2).map(|p| text(*p)).collect(); + let mut expected: Vec = (0..11).rev().map(|i| format!("x{i}")).collect(); + expected.push("leaf".into()); + expected.extend((0..11).map(|i| format!("after{i}"))); + assert_eq!(texts, expected, "every paragraph, in order"); + // The two levels past the tenth continue the tenth list as items. + let deepest = lists[9]; + let items = children(deepest, TEXT, "list-item"); + let firsts: Vec = items + .iter() + .map(|i| text(child(*i, TEXT, "p").unwrap())) + .collect(); + assert_eq!(firsts, ["x1", "x0", "leaf"]); +} + +// Tables. + +/// The rows of a table, header rows first. +fn rows<'a, 'i>(tbl: Node<'a, 'i>) -> Vec> { + elements(tbl) + .into_iter() + .flat_map(|n| { + if n.has_tag_name((TABLE, "table-header-rows")) { + children(n, TABLE, "table-row") + } else if n.has_tag_name((TABLE, "table-row")) { + vec![n] + } else { + vec![] + } + }) + .collect() +} + +/// Each row's cells as "text, span N, rows N", or "covered". +fn shape(tbl: Node) -> Vec> { + rows(tbl) + .into_iter() + .map(|tr| { + elements(tr) + .into_iter() + .map(|c| { + if c.has_tag_name((TABLE, "covered-table-cell")) { + return "covered".to_owned(); + } + assert!(c.has_tag_name((TABLE, "table-cell"))); + let mut parts = vec![text(c)]; + if let Some(n) = c.attribute((TABLE, "number-columns-spanned")) { + parts.push(format!("span {n}")); + } + if let Some(n) = c.attribute((TABLE, "number-rows-spanned")) { + parts.push(format!("rows {n}")); + } + parts.retain(|p| !p.is_empty()); + parts.join(", ") + }) + .collect() + }) + .collect() +} + +/// Widths of a table's columns in cm, from their automatic styles. +fn column_widths(d: &Document, tbl: Node) -> Vec { + children(tbl, TABLE, "table-column") + .into_iter() + .map(|c| cm(prop(auto_style(d, c, TABLE), STYLE, "column-width").unwrap())) + .collect() +} + +#[test] +fn tables_have_columns_spans_and_covered_cells() { + let odt = export(vec![table( + vec![0.3, 0.7], + vec![ + vec![th(0, "h1"), th(1, "h2")], + vec![Cell { + colspan: 2, + align: Align::Center, + ..td(0, "wide") + }], + vec![ + td(0, "a"), + Cell { + align: Align::Right, + ..td(1, "b") + }, + ], + vec![ + Cell { + rowspan: 2, + ..td(0, "tall") + }, + td(1, "c"), + ], + vec![td(1, "d")], + ], + )]); + parse_all(&odt); + let d = xml(odt.part("content.xml")); + let tbl = all(&d, TABLE, "table")[0]; + let tstyle = auto_style(&d, tbl, TABLE); + assert_eq!(tstyle.attribute((STYLE, "family")), Some("table")); + assert_close( + cm(prop(tstyle, STYLE, "width").unwrap()), + 16.0, + "table width", + ); + assert_eq!( + prop(tstyle, FO, "margin-left"), + None, + "no indent at the top" + ); + + let widths = column_widths(&d, tbl); + assert_eq!(widths.len(), 2); + assert_close(widths[0], 4.8, "first column"); + assert_close(widths[1], 11.2, "second column"); + let rel: Vec = children(tbl, TABLE, "table-column") + .into_iter() + .map(|c| { + let s = auto_style(&d, c, TABLE); + assert_eq!(s.attribute((STYLE, "family")), Some("table-column")); + let w = prop(s, STYLE, "rel-column-width").unwrap(); + w.strip_suffix('*').expect("N*").parse().unwrap() + }) + .collect(); + assert_eq!( + rel, + [ + (0.3f64 * 65535.0).round() as u32, + (0.7f64 * 65535.0).round() as u32 + ] + ); + + let heads = children(tbl, TABLE, "table-header-rows"); + assert_eq!(heads.len(), 1); + assert_eq!(children(heads[0], TABLE, "table-row").len(), 1); + assert_eq!(children(tbl, TABLE, "table-row").len(), 4); + assert_eq!( + shape(tbl), + vec![ + vec!["h1", "h2"], + vec!["wide, span 2", "covered"], + vec!["a", "b"], + vec!["tall, rows 2", "c"], + vec!["covered", "d"], + ] + ); + + for c in all(&d, TABLE, "table-cell") { + let s = auto_style(&d, c, TABLE); + assert_eq!(s.attribute((STYLE, "family")), Some("table-cell")); + assert_eq!(prop(s, FO, "border"), Some("0.5pt solid #bfbfbf")); + let header = c + .ancestors() + .any(|a| a.has_tag_name((TABLE, "table-header-rows"))); + let background = prop(s, FO, "background-color"); + assert_eq!(background, header.then_some("#f2f2f2"), "{}", text(c)); + } + + let p_of = |label: &str| { + let p = all(&d, TEXT, "p") + .into_iter() + .find(|p| text(*p) == label) + .unwrap(); + let s = auto_style(&d, p, TEXT); + assert_eq!(s.attribute((STYLE, "family")), Some("paragraph")); + ( + s.attribute((STYLE, "parent-style-name")), + prop(s, FO, "text-align"), + ) + }; + assert_eq!(p_of("h1"), (Some("Table_20_Heading"), Some("start"))); + assert_eq!(p_of("wide"), (Some("Table_20_Contents"), Some("center"))); + assert_eq!(p_of("a"), (Some("Table_20_Contents"), Some("start"))); + assert_eq!(p_of("b"), (Some("Table_20_Contents"), Some("end"))); +} + +#[test] +fn spans_are_clamped_to_the_table_and_the_header_group() { + let odt = export(vec![ + table( + even(2), + vec![ + vec![ + Cell { + rowspan: 5, + ..td(0, "A") + }, + Cell { + colspan: 4, + ..td(1, "B") + }, + ], + vec![td(1, "C")], + ], + ), + // A header cell's row span stops at the last header row. + table( + even(2), + vec![ + vec![ + Cell { + rowspan: 3, + ..th(0, "H") + }, + th(1, "H2"), + ], + vec![td(1, "x")], + vec![td(0, "y"), td(1, "z")], + ], + ), + // A column span that runs into a merged cell stops short of it. + table( + even(3), + vec![ + vec![ + td(0, "a"), + Cell { + rowspan: 2, + ..td(1, "b") + }, + td(2, "c"), + ], + vec![Cell { + colspan: 3, + ..td(0, "wide") + }], + ], + ), + ]); + parse_all(&odt); + let d = xml(odt.part("content.xml")); + let tbls = all(&d, TABLE, "table"); + assert_eq!( + shape(tbls[0]), + vec![vec!["A, rows 2", "B"], vec!["covered", "C"]] + ); + assert_eq!( + shape(tbls[1]), + vec![vec!["H", "H2"], vec!["", "x"], vec!["y", "z"]] + ); + assert_eq!(children(tbls[1], TABLE, "table-header-rows").len(), 1); + assert_eq!( + shape(tbls[2]), + vec![vec!["a", "b, rows 2", "c"], vec!["wide", "covered", ""]] + ); +} + +#[test] +fn rows_are_padded_to_the_grid() { + let odt = export(vec![table( + even(3), + vec![ + vec![td(0, "a"), td(1, "b"), td(2, "c")], + vec![td(0, "short")], + vec![td(0, "x"), td(2, "z")], + vec![cell(0, vec![]), td(1, "m")], + ], + )]); + let d = xml(odt.part("content.xml")); + let tbl = all(&d, TABLE, "table")[0]; + assert_eq!( + shape(tbl), + vec![ + vec!["a", "b", "c"], + vec!["short", "span 2", "covered"], + vec!["x", "", "z"], + vec!["", "m", ""], + ] + ); + for tr in rows(tbl) { + assert_eq!(elements(tr).len(), 3, "every row spans the grid"); + } +} + +#[test] +fn empty_tables_write_nothing_and_parse_odt() { + let odt = export(vec![ + table(vec![], vec![]), + table(vec![], vec![vec![]]), + table(vec![1.0], vec![]), + para("after"), + list( + false, + 1, + vec![item(vec![para("lead"), table(vec![], vec![vec![]])])], + ), + ]); + parse_all(&odt); + let d = xml(odt.part("content.xml")); + assert!(all(&d, TABLE, "table").is_empty()); + let blocks = content(&d); + let tags: Vec<&str> = blocks.iter().map(|n| n.tag_name().name()).collect(); + assert_eq!(tags, ["p", "list"], "the list isn't split"); + assert_eq!(text(blocks[0]), "after"); +} + +#[test] +fn table_inside_list_item_is_valid_odf() { + let odt = export(vec![list( + true, + 3, + vec![ + item(vec![ + para("lead"), + table(even(2), vec![vec![td(0, "a"), td(1, "b")]]), + para("tail"), + ]), + item(vec![para("next")]), + ], + )]); + parse_all(&odt); + let d = xml(odt.part("content.xml")); + for tbl in all(&d, TABLE, "table") { + assert!( + !tbl.ancestors().any(|a| a.has_tag_name((TEXT, "list"))), + "no table inside a list" + ); + } + let blocks = content(&d); + let tags: Vec<&str> = blocks.iter().map(|n| n.tag_name().name()).collect(); + assert_eq!(tags, ["list", "table", "list"]); + let id = blocks[0].attribute((XML_NS, "id")).expect("xml:id"); + assert_eq!(blocks[2].attribute((TEXT, "continue-list")), Some(id)); + assert_eq!(style_name(blocks[2]), Some("Numbering_20_123")); + assert_eq!(blocks[2].attribute((XML_NS, "id")), None); + + // The rest of the item has no number of its own; the next item does. + let kids = elements(blocks[2]); + let got: Vec<(&str, String, Option<&str>)> = kids + .iter() + .map(|k| { + ( + k.tag_name().name(), + text(*k), + k.attribute((TEXT, "start-value")), + ) + }) + .collect(); + assert_eq!( + got, + [ + ("list-header", "tail".to_owned(), None), + ("list-item", "next".to_owned(), None), + ] + ); + + // It lines up with the item's text. + let tstyle = auto_style(&d, blocks[1], TABLE); + assert_close(cm(prop(tstyle, FO, "margin-left").unwrap()), 1.27, "indent"); + assert_close( + cm(prop(tstyle, STYLE, "width").unwrap()), + 16.0 - 1.27, + "width", + ); +} + +#[test] +fn a_table_ending_an_item_leaves_no_empty_list() { + let t = || table(even(1), vec![vec![td(0, "t")]]); + let odt = export(vec![ + list(false, 1, vec![item(vec![para("a"), t(), t()])]), + list( + false, + 1, + vec![item(vec![ + para("b"), + list(false, 1, vec![item(vec![para("c"), t()])]), + ])], + ), + list(false, 1, vec![]), + ]); + parse_all(&odt); + let d = xml(odt.part("content.xml")); + let tags: Vec<&str> = content(&d).iter().map(|n| n.tag_name().name()).collect(); + assert_eq!(tags, ["list", "table", "table", "list", "table"]); + for n in all(&d, TEXT, "list") + .into_iter() + .chain(all(&d, TEXT, "list-header")) + { + assert!(!elements(n).is_empty(), "an empty {}", n.tag_name().name()); + } +} + +#[test] +fn table_in_nested_item_reopens_every_level() { + let odt = export(vec![list( + false, + 1, + vec![ + item(vec![ + para("outer"), + list( + true, + 2, + vec![ + item(vec![ + para("inner"), + table(even(1), vec![vec![td(0, "t")]]), + para("inner tail"), + ]), + item(vec![para("inner next")]), + ], + ), + para("outer tail"), + ]), + item(vec![para("outer next")]), + ], + )]); + parse_all(&odt); + let d = xml(odt.part("content.xml")); + let blocks = content(&d); + let tags: Vec<&str> = blocks.iter().map(|n| n.tag_name().name()).collect(); + assert_eq!(tags, ["list", "table", "list"]); + let id = blocks[0].attribute((XML_NS, "id")).unwrap(); + let reopened = blocks[2]; + assert_eq!(reopened.attribute((TEXT, "continue-list")), Some(id)); + + let outer = elements(reopened); + let outer_tags: Vec<&str> = outer.iter().map(|n| n.tag_name().name()).collect(); + assert_eq!(outer_tags, ["list-header", "list-item"]); + let header = elements(outer[0]); + let inner_tags: Vec<&str> = header.iter().map(|n| n.tag_name().name()).collect(); + assert_eq!(inner_tags, ["list", "p"]); + assert_eq!(style_name(header[0]), Some("Numbering_20_123")); + assert_eq!(text(header[1]), "outer tail"); + let inner: Vec<(&str, String)> = elements(header[0]) + .into_iter() + .map(|n| (n.tag_name().name(), text(n))) + .collect(); + assert_eq!( + inner, + [ + ("list-header", "inner tail".to_owned()), + ("list-item", "inner next".to_owned()), + ] + ); + assert_eq!(text(outer[1]), "outer next"); + + let tstyle = auto_style(&d, blocks[1], TABLE); + assert_close(cm(prop(tstyle, FO, "margin-left").unwrap()), 2.54, "indent"); + assert_close( + cm(prop(tstyle, STYLE, "width").unwrap()), + 16.0 - 2.54, + "width", + ); +} + +#[test] +fn table_in_quote_is_indented_and_cell_content_starts_at_the_cell_edge() { + let inner = || { + table( + vec![0.25, 0.75], + vec![vec![ + cell( + 0, + vec![para("cell"), list(false, 1, vec![item(vec![para("li")])])], + ), + cell(1, vec![table(even(2), vec![vec![td(0, "n"), td(1, "m")]])]), + ]], + ) + }; + let odt = export(vec![ + Block::Quote(vec![inner()]), + para("between"), + table(even(1), vec![vec![td(0, "top")]]), + ]); + parse_all(&odt); + let d = xml(odt.part("content.xml")); + let tbls = all(&d, TABLE, "table"); + assert_eq!(tbls.len(), 3); + let quoted = auto_style(&d, tbls[0], TABLE); + assert_close(cm(prop(quoted, FO, "margin-left").unwrap()), 1.0, "indent"); + assert_close(cm(prop(quoted, STYLE, "width").unwrap()), 15.0, "width"); + assert_eq!(column_widths(&d, tbls[0]), [3.75, 11.25]); + + // The nested table fills its cell, less the cell's padding, unindented. + let nested = auto_style(&d, tbls[1], TABLE); + assert_eq!(prop(nested, FO, "margin-left"), None); + assert_close( + cm(prop(nested, STYLE, "width").unwrap()), + 11.25 - 0.38, + "nested", + ); + + let first = elements(rows(tbls[0])[0])[0]; + let p = child(first, TEXT, "p").unwrap(); + let s = auto_style(&d, p, TEXT); + assert_eq!( + s.attribute((STYLE, "parent-style-name")), + Some("Table_20_Contents"), + "no Quotations inside a cell" + ); + let li = all(&d, TEXT, "list-item")[0]; + let s = auto_style(&d, child(li, TEXT, "p").unwrap(), TEXT); + assert_eq!( + s.attribute((STYLE, "parent-style-name")), + Some("List_20_Paragraph") + ); + + let top = auto_style(&d, tbls[2], TABLE); + assert_eq!(prop(top, FO, "margin-left"), None); +} + +/// Most `content.xml` a crafted table may make per grid position the walker +/// counts: a covered position is a 27-byte `table:covered-table-cell`. +const BYTES_PER_GRID_CELL: usize = 40; + +#[test] +fn amplification_is_bounded_odt() { + use common::{build, wide_empty_table}; + use knot_export::{MAX_PAGE_GRID_CELLS, MAX_TABLE_GRID_CELLS, walk}; + // A table inside every limit: one cell spanning 64 columns, then rows + // that are empty in the page and a cell plus 63 covered cells each in + // the file. + let rows = MAX_TABLE_GRID_CELLS / 64; + let walked = walk(&build(vec![wide_empty_table(64, rows)])); + assert!( + walked.unknown_elements.is_empty(), + "{:?}", + walked.unknown_elements + ); + assert!(matches!(walked.doc.blocks.as_slice(), [Block::Table(_)])); + let odt = export(walked.doc.blocks); + let content = odt.part("content.xml"); + assert_eq!( + content.matches("").count(), + 63 * rows + ); + assert!( + content.len() < 64 * rows * BYTES_PER_GRID_CELL, + "{}", + content.len() + ); + + // Pages of them, and of one-row tables, whose column styles weigh the + // most: the page budget degrades the tables past it, and bounds both. + for tables in [ + (0..6) + .map(|_| wide_empty_table(64, rows)) + .collect::>(), + (0..400).map(|_| wide_empty_table(64, 1)).collect(), + ] { + let walked = walk(&build(tables)); + assert!(!walked.unknown_elements.is_empty()); + assert!(walked.unknown_elements.iter().all(|u| u == "table-limit")); + let odt = export(walked.doc.blocks); + let len = odt.part("content.xml").len(); + assert!(len < MAX_PAGE_GRID_CELLS * BYTES_PER_GRID_CELL, "{len}"); + } +} + +#[test] +fn xml_past_the_cap_fails_the_export_odt() { + use knot_export::ExportError; + use knot_export::xml::MAX_XML_BYTES; + // Only a page far past the walker's limits gets here: the writer stops + // and fails instead of building the rest. + let text = "x".repeat(1 << 20); + let blocks = vec![para(&text); MAX_XML_BYTES / text.len() + 1]; + let result = write_odt(&ExportDoc { blocks }, &meta(), &Assets::default()); + assert!( + matches!(result, Err(ExportError::TooLarge)), + "{:?}", + result.err() + ); +} + +#[test] +fn output_buffer_is_sized_up_front_odt() { + // Three megabytes of pictures: grown as it fills, the buffer would end + // up with about another megabyte spare. + let mut assets = Assets::default(); + for blob in [BLOB_A, BLOB_B, BLOB_C] { + let pic = Picture { + format: PicFormat::Png, + bytes: common::noise(1 << 20), + width_px: 10, + height_px: 10, + scale: 1, + }; + assets.images.insert(blob.into(), ImageAsset::Embedded(pic)); + } + let out = write_odt( + &ExportDoc { + blocks: [BLOB_A, BLOB_B, BLOB_C] + .iter() + .map(|blob| image_block(blob, "")) + .collect(), + }, + &meta(), + &assets, + ) + .expect("export"); + assert!(out.bytes.len() > 3 << 20); + let spare = out.bytes.capacity() - out.bytes.len(); + assert!(spare < 256 * 1024, "{spare} bytes spare"); + // The content went into the zip in pieces and reads back whole. + let odt = Odt(unzip(&out.bytes)); + xml(odt.part("content.xml")); +} + +#[test] +fn tall_image_fits_the_page_height_odt() { + // A phone screenshot: at the 16 cm text width it would be 34.6 cm + // tall, past the page. It shrinks to 21.2 cm, which with the Standard + // paragraph's 115% line height and bottom margin still fits the + // 24.7 cm of body. + let mut assets = Assets::default(); + assets.images.insert( + BLOB_A.into(), + ImageAsset::Embedded(picture(PicFormat::Png, 1170, 2532, 1)), + ); + let odt = export_with(vec![image_block(BLOB_A, "screenshot")], &assets); + let d = xml(odt.part("content.xml")); + let frames = all(&d, DRAW, "frame"); + assert_eq!(frames.len(), 1); + assert_eq!(frames[0].attribute((SVG, "width")), Some("9.811cm")); + assert_eq!(frames[0].attribute((SVG, "height")), Some("21.233cm")); +} + +const DRAW: &str = "urn:oasis:names:tc:opendocument:xmlns:drawing:1.0"; +const SVG: &str = "urn:oasis:names:tc:opendocument:xmlns:svg-compatible:1.0"; + +const BLOB_A: &str = "aaaaaaaa-1111-4222-8333-444444444444"; +const BLOB_B: &str = "bbbbbbbb-1111-4222-8333-444444444444"; +const BLOB_C: &str = "cccccccc-1111-4222-8333-444444444444"; + +fn picture(format: PicFormat, width_px: u32, height_px: u32, scale: u32) -> Picture { + Picture { + format, + bytes: vec![1, 2, 3, width_px as u8], + width_px, + height_px, + scale, + } +} + +fn image_block(blob: &str, alt: &str) -> Block { + Block::Image { + source: ImageSource::Blob(blob.into()), + alt: alt.into(), + } +} + +#[test] +fn odt_pictures_have_frames_and_manifest_entries() { + let mut assets = Assets::default(); + assets.images.insert( + BLOB_A.into(), + ImageAsset::Embedded(picture(PicFormat::Png, 160, 80, 1)), + ); + assets.images.insert( + BLOB_B.into(), + ImageAsset::Embedded(picture(PicFormat::Jpeg, 80, 40, 1)), + ); + assets + .diagrams + .insert("m1".into(), picture(PicFormat::Png, 4000, 1000, 2)); + let odt = export_with( + vec![ + image_block(BLOB_A, "First "), + image_block(BLOB_B, ""), + image_block(BLOB_A, "again"), + Block::Mermaid { + key: "m1".into(), + source: "graph TD".into(), + }, + ], + &assets, + ); + parse_all(&odt); + assert_eq!(odt.0["Pictures/img1.png"], [1, 2, 3, 160]); + assert_eq!(odt.0["Pictures/img2.jpeg"], [1, 2, 3, 80]); + assert_eq!(odt.0["Pictures/img3.png"], [1, 2, 3, 160]); + assert_eq!( + odt.0.keys().filter(|k| k.starts_with("Pictures/")).count(), + 3 + ); + + let manifest = xml(odt.part("META-INF/manifest.xml")); + let entries: HashMap<&str, &str> = all(&manifest, MANIFEST, "file-entry") + .into_iter() + .map(|e| { + ( + e.attribute((MANIFEST, "full-path")).unwrap(), + e.attribute((MANIFEST, "media-type")).unwrap(), + ) + }) + .collect(); + assert_eq!(entries["Pictures/img1.png"], "image/png"); + assert_eq!(entries["Pictures/img2.jpeg"], "image/jpeg"); + assert_eq!(entries["Pictures/img3.png"], "image/png"); + + let d = xml(odt.part("content.xml")); + let frames = all(&d, DRAW, "frame"); + assert_eq!(frames.len(), 4); + let names: HashSet<_> = frames + .iter() + .map(|f| f.attribute((DRAW, "name")).unwrap()) + .collect(); + assert_eq!(names.len(), 4); + for f in &frames { + assert_eq!(f.attribute((TEXT, "anchor-type")), Some("as-char")); + } + // 160 px at scale 1 is 4.233 cm. + assert_eq!(frames[0].attribute((SVG, "width")), Some("4.233cm")); + assert_eq!(frames[0].attribute((SVG, "height")), Some("2.117cm")); + assert_eq!(frames[1].attribute((SVG, "width")), Some("2.117cm")); + // The third frame shares the first part; the diagram is the third part. + let hrefs: Vec<_> = frames + .iter() + .map(|f| { + child(*f, DRAW, "image") + .unwrap() + .attribute((XLINK, "href")) + .unwrap() + }) + .collect(); + assert_eq!( + hrefs, + [ + "Pictures/img1.png", + "Pictures/img2.jpeg", + "Pictures/img1.png", + "Pictures/img3.png" + ] + ); + // Capped at the 16 cm text width. + assert_eq!(frames[3].attribute((SVG, "width")), Some("16.000cm")); + assert_eq!(frames[3].attribute((SVG, "height")), Some("4.000cm")); + let descr: Vec<_> = frames + .iter() + .map(|f| text(child(*f, SVG, "desc").unwrap())) + .collect(); + assert_eq!(descr, ["First ", "", "again", "Mermaid diagram"]); + let style_ref = frames[0].attribute((DRAW, "style-name")).unwrap(); + style(&d, style_ref); +} + +#[test] +fn odt_fallbacks_match_docx() { + let (doc, assets) = common::fallback_fixture(); + let out = write_odt(&doc, &meta(), &assets).expect("export"); + assert_eq!( + out.fallbacks, + Fallbacks { + diagram: 2, + image_unavailable: 2, + image_over_budget: 1, + unknown_element: 0, + } + ); + let odt = Odt(unzip(&out.bytes)); + parse_all(&odt); + let d = xml(odt.part("content.xml")); + let paras: Vec<(Option<&str>, String)> = content(&d) + .into_iter() + .map(|p| (style_name(p), text(p))) + .collect(); + assert_eq!( + paras, + [ + ( + Some("Preformatted_20_Text"), + "graph TD\n A --> B".to_owned() + ), + (Some("Caption"), "Mermaid diagram (not included)".to_owned()), + ( + Some("Caption"), + "Diagram: Roadmap (not included)".to_owned() + ), + ( + Some("Caption"), + "[Image not available] \u{2014} Org chart".to_owned() + ), + (Some("Caption"), "[Image too large to include]".to_owned()), + ( + Some("Caption"), + "[Image not available] \u{2014} Local file".to_owned() + ), + ] + ); + let link = all(&d, TEXT, "a").into_iter().next().expect("link"); + assert_eq!( + link.attribute((XLINK, "href")), + Some( + format!( + "https://knot.example/api/blobs/{}", + common::OVER_BUDGET_BLOB + ) + .as_str() + ) + ); + assert_eq!(text(link), "[Image too large to include]"); + assert!(all(&d, DRAW, "frame").is_empty()); +} + +#[test] +fn odt_external_images_and_attachments_are_links() { + let url = "/api/blobs/cccccccc-1111-4222-8333-444444444444"; + let odt = export(vec![ + Block::Image { + source: ImageSource::External("https://img.example/a.png?x=1&y=2".into()), + alt: "Remote".into(), + }, + Block::Image { + source: ImageSource::External("https://img.example/b.png".into()), + alt: String::new(), + }, + Block::Attachment { + url: url.into(), + name: "budget.xlsx".into(), + size: Some(49152), + }, + Block::Attachment { + url: "javascript:alert(1)".into(), + name: "odd".into(), + size: None, + }, + ]); + parse_all(&odt); + let d = xml(odt.part("content.xml")); + let got: Vec<_> = all(&d, TEXT, "a") + .into_iter() + .map(|l| (text(l), l.attribute((XLINK, "href")).unwrap().to_owned())) + .collect(); + assert_eq!( + got, + [ + ( + "Remote".to_owned(), + "https://img.example/a.png?x=1&y=2".to_owned() + ), + ( + "https://img.example/b.png".to_owned(), + "https://img.example/b.png".to_owned() + ), + ( + "\u{1F4CE} budget.xlsx (48 KB)".to_owned(), + format!("https://knot.example{url}") + ), + ] + ); + let texts: Vec<_> = content(&d).into_iter().map(text).collect(); + assert_eq!(texts[3], "\u{1F4CE} odd"); +} diff --git a/crates/knot-export/tests/tables.rs b/crates/knot-export/tests/tables.rs new file mode 100644 index 00000000..b70ce0c2 --- /dev/null +++ b/crates/knot-export/tests/tables.rs @@ -0,0 +1,498 @@ +mod common; + +use common::*; +use knot_export::tables::{Slot, grid_edges, header_rows, layout, resolve_shares}; +use knot_export::{Align, Block, Cell, Table, walk}; +use yrs::Any; + +fn approx(a: &[f64], b: &[f64]) { + assert_eq!(a.len(), b.len(), "{a:?} vs {b:?}"); + for (x, y) in a.iter().zip(b) { + assert!((x - y).abs() < 1e-9, "{a:?} vs {b:?}"); + } +} + +fn widths(ws: &[Option]) -> Any { + Any::Array( + ws.iter() + .map(|w| w.map_or(Any::Null, Any::Number)) + .collect::>() + .into(), + ) +} + +fn cell(tag: &'static str, attrs: Vec<(&'static str, Any)>, text: &str) -> N { + el(tag, attrs, vec![p(text)]) +} + +fn td(text: &str) -> N { + cell("table_cell", vec![], text) +} + +fn th(text: &str) -> N { + cell("table_header", vec![], text) +} + +fn row(cells: Vec) -> N { + el("table_row", vec![], cells) +} + +fn table_of(rows: Vec) -> Table { + let w = walk(&build(vec![el("table", vec![], rows)])); + assert!(w.unknown_elements.is_empty(), "{:?}", w.unknown_elements); + match w.doc.blocks.into_iter().next() { + Some(Block::Table(t)) => t, + other => panic!("not a table: {other:?}"), + } +} + +#[test] +fn shares_all_unset_are_equal() { + assert_eq!(resolve_shares(&[None; 4]), vec![0.25; 4]); +} + +#[test] +fn shares_normalise_known() { + assert_eq!(resolve_shares(&[Some(200.), Some(600.)]), vec![0.25, 0.75]); +} + +#[test] +fn shares_fill_unset_from_leftover() { + approx( + &resolve_shares(&[Some(400.), None, Some(600.)]), + &[400. / 1500., 500. / 1500., 600. / 1500.], + ); + approx( + &resolve_shares(&[Some(300.), None, None, None]), + &[0.3, 0.7 / 3., 0.7 / 3., 0.7 / 3.], + ); + approx( + &resolve_shares(&[Some(500.), Some(250.), None]), + &[0.5, 0.25, 0.25], + ); +} + +#[test] +fn shares_ignore_non_positive() { + assert_eq!( + resolve_shares(&[Some(0.), Some(300.), Some(-5.), Some(f64::NAN)]), + resolve_shares(&[None, Some(300.), None, None]) + ); +} + +#[test] +fn shares_empty() { + assert!(resolve_shares(&[]).is_empty()); +} + +#[test] +fn grid_places_cells_after_rowspan() { + let t = table_of(vec![ + row(vec![ + cell("table_cell", vec![("rowspan", Any::Number(2.))], "A"), + td("B"), + ]), + row(vec![td("C")]), + ]); + assert_eq!(t.shares.len(), 2); + assert_eq!(t.rows[0][0].col, 0); + assert_eq!(t.rows[0][0].rowspan, 2); + assert_eq!(t.rows[0][1].col, 1); + assert_eq!(t.rows[1][0].col, 1); + assert_eq!(t.rows[1][0].colspan, 1); +} + +#[test] +fn colspan_weights_come_from_first_row() { + let t = table_of(vec![ + row(vec![cell( + "table_cell", + vec![ + ("colspan", Any::Number(2.)), + ("colwidth", widths(&[Some(300.), Some(700.)])), + ], + "A", + )]), + row(vec![td("B"), td("C")]), + ]); + approx(&t.shares, &[0.3, 0.7]); +} + +#[test] +fn string_spans_and_null_widths_are_read() { + let t = table_of(vec![row(vec![ + cell( + "table_cell", + vec![ + ("colspan", "2".into()), + ("colwidth", widths(&[None, Some(500.)])), + ], + "A", + ), + td("B"), + ])]); + assert_eq!(t.rows[0][0].colspan, 2); + assert_eq!(t.rows[0][1].col, 2); + // Column 0 and 2 are unset and share the 500 left of 1000. + approx(&t.shares, &[0.25, 0.5, 0.25]); +} + +#[test] +fn header_rows_counts_leading_all_header_rows() { + let t = table_of(vec![ + row(vec![th("a"), th("b")]), + row(vec![th("c"), th("d")]), + row(vec![td("e"), td("f")]), + ]); + assert_eq!(header_rows(&t), 2); + let t = table_of(vec![ + row(vec![th("a"), td("b")]), + row(vec![th("c"), td("d")]), + ]); + assert_eq!(header_rows(&t), 0); + assert!(t.rows[0][0].header && !t.rows[0][1].header); +} + +#[test] +fn cell_blocks_are_kept() { + let t = table_of(vec![row(vec![el( + "table_cell", + vec![], + vec![ + el( + "bullet_list", + vec![], + vec![el("list_item", vec![], vec![p("x")])], + ), + el( + "code_block", + vec![("language", "rs".into())], + vec![t("fn main() {}")], + ), + p("one"), + p("two"), + ], + )])]); + let blocks = &t.rows[0][0].blocks; + assert_eq!(blocks.len(), 4); + assert!(matches!(blocks[0], Block::List(_))); + assert!(matches!(blocks[1], Block::Code { .. })); + assert!(matches!(blocks[2], Block::Paragraph(_))); + assert!(matches!(blocks[3], Block::Paragraph(_))); +} + +#[test] +fn empty_table_walks_without_panic() { + let t = table_of(vec![]); + assert!(t.shares.is_empty() && t.rows.is_empty()); + let t = table_of(vec![row(vec![])]); + assert!(t.shares.is_empty()); + assert_eq!(t.rows.len(), 1); + assert_eq!(header_rows(&t), 0); +} + +#[test] +fn align_attribute_is_read() { + let t = table_of(vec![row(vec![ + cell("table_cell", vec![("align", "center".into())], "a"), + cell("table_cell", vec![("align", "right".into())], "b"), + td("c"), + ])]); + assert_eq!(t.rows[0][0].align, Align::Center); + assert_eq!(t.rows[0][1].align, Align::Right); + assert_eq!(t.rows[0][2].align, Align::Left); +} + +fn texts(w: &knot_export::Walked) -> Vec { + w.doc + .blocks + .iter() + .map(|b| match b { + Block::Paragraph(inl) => match inl.as_slice() { + [knot_export::Inline::Run(r)] => r.text.clone(), + other => panic!("{other:?}"), + }, + other => panic!("not a paragraph: {other:?}"), + }) + .collect() +} + +#[test] +fn spans_are_clamped_to_max_span() { + let attrs = vec![ + ("colspan", Any::Number(1000.)), + ("rowspan", Any::Number(1000.)), + ]; + let t = table_of(vec![row(vec![cell("table_cell", attrs, "A")])]); + assert_eq!(t.rows[0][0].colspan, knot_export::MAX_SPAN); + assert_eq!(t.rows[0][0].rowspan, knot_export::MAX_SPAN); + assert_eq!(t.shares.len(), 64); +} + +#[test] +fn too_wide_table_degrades_to_paragraphs() { + let names: Vec = (0..=knot_export::MAX_TABLE_COLUMNS) + .map(|i| format!("c{i}")) + .collect(); + let w = walk(&build(vec![ + el( + "table", + vec![], + vec![ + row(names.iter().map(|n| td(n)).collect()), + row(vec![td("second")]), + ], + ), + p("after"), + ])); + assert_eq!(w.unknown_elements, ["table-limit"]); + assert_eq!( + texts(&w), + [names.join(" | "), "second".into(), "after".into()] + ); +} + +#[test] +fn too_many_grid_cells_degrade() { + let table = |n: usize| { + let rows = (0..n) + .map(|r| row((0..10).map(|c| td(&format!("{r}.{c}"))).collect())) + .collect(); + walk(&build(vec![el("table", vec![], rows)])) + }; + let ok = table(5_000); + assert!(ok.unknown_elements.is_empty()); + assert!(matches!(ok.doc.blocks.as_slice(), [Block::Table(_)])); + + let over = table(5_001); + assert_eq!(over.unknown_elements, ["table-limit"]); + assert_eq!(over.doc.blocks.len(), 5_001); + assert_eq!( + texts(&over)[0], + "0.0 | 0.1 | 0.2 | 0.3 | 0.4 | 0.5 | 0.6 | 0.7 | 0.8 | 0.9" + ); +} + +#[test] +fn tables_past_the_page_budget_degrade() { + use knot_export::{MAX_PAGE_GRID_CELLS, MAX_TABLE_GRID_CELLS, TABLE_COLUMN_ROWS}; + // 64 x 773 grid positions each, inside the per-table limits; with + // their columns each counts 64 x 781 = 49,984, and four of them fill + // the page budget. + let rows = MAX_TABLE_GRID_CELLS / 64 - TABLE_COLUMN_ROWS; + let per_table = 64 * (rows + TABLE_COLUMN_ROWS); + assert!(4 * per_table <= MAX_PAGE_GRID_CELLS && 5 * per_table > MAX_PAGE_GRID_CELLS); + let mut blocks: Vec = (0..5).map(|_| wide_empty_table(64, rows)).collect(); + // The budget counts kept tables only: a small table after the + // degraded one still fits. + blocks.push(el("table", vec![], vec![row(vec![td("a"), td("b")])])); + let w = walk(&build(blocks)); + + assert_eq!(w.unknown_elements, ["table-limit"]); + let kept: Vec = w + .doc + .blocks + .iter() + .enumerate() + .filter_map(|(i, b)| matches!(b, Block::Table(_)).then_some(i)) + .collect(); + // Four tables, the fifth as one paragraph per row, then the small one. + assert_eq!(kept, [0, 1, 2, 3, 4 + rows]); + assert_eq!(w.doc.blocks.len(), 5 + rows); + assert!(matches!(&w.doc.blocks[4], Block::Paragraph(_))); +} + +fn grid_cell(col: usize, colspan: usize, rowspan: usize) -> Cell { + Cell { + col, + colspan, + rowspan, + header: false, + align: Align::Left, + blocks: vec![], + } +} + +fn grid(width: usize, rows: Vec>) -> Table { + Table { + shares: vec![1.0 / width as f64; width], + rows, + } +} + +/// `t` laid out with `group_end`, each slot as a short string. Also checks +/// that every row's slots tile the grid exactly, left to right. +fn shape(t: &Table, group_end: impl Fn(usize) -> usize) -> Vec> { + layout(t, group_end) + .map(|slots| { + let mut at = 0; + let row = slots + .iter() + .map(|s| { + let (pos, end, text) = match *s { + Slot::Cell { + pos, span, rowspan, .. + } => (pos, pos + span, format!("cell {pos}+{span} r{rowspan}")), + Slot::Covered { pos, span, header } => { + let h = if header { " h" } else { "" }; + (pos, pos + span, format!("covered {pos}+{span}{h}")) + } + Slot::Gap { pos, end } => (pos, end, format!("gap {pos}..{end}")), + }; + assert!(pos == at && end > pos, "{text} doesn't continue at {at}"); + at = end; + text + }) + .collect(); + assert_eq!(at, t.shares.len(), "the row fills the grid"); + row + }) + .collect() +} + +#[test] +fn layout_covers_row_spans_and_places_cells_after_them() { + let t = table_of(vec![ + row(vec![ + cell("table_cell", vec![("rowspan", Any::Number(2.))], "A"), + td("B"), + ]), + row(vec![td("C")]), + row(vec![td("D"), td("E")]), + ]); + assert_eq!( + shape(&t, |_| t.rows.len()), + [ + vec!["cell 0+1 r2", "cell 1+1 r1"], + vec!["covered 0+1", "cell 1+1 r1"], + vec!["cell 0+1 r1", "cell 1+1 r1"], + ] + ); + // Each cell slot carries the cell it places. + let rows: Vec<_> = layout(&t, |_| t.rows.len()).collect(); + match rows[1][1] { + Slot::Cell { cell, .. } => assert!(std::ptr::eq(cell, &t.rows[1][0])), + other => panic!("not a cell: {other:?}"), + } +} + +#[test] +fn layout_clamps_column_spans_to_the_grid_and_short_of_row_spans() { + // A column span past the grid ends at its edge. + let t = grid(2, vec![vec![grid_cell(0, 1, 1), grid_cell(1, 4, 1)]]); + assert_eq!(shape(&t, |_| 1), [vec!["cell 0+1 r1", "cell 1+1 r1"]]); + + // A column span that runs into a row span from above stops short of it. + let t = grid( + 3, + vec![ + vec![grid_cell(0, 1, 1), grid_cell(1, 1, 2), grid_cell(2, 1, 1)], + vec![grid_cell(0, 3, 1)], + ], + ); + assert_eq!( + shape(&t, |_| 2), + [ + vec!["cell 0+1 r1", "cell 1+1 r2", "cell 2+1 r1"], + vec!["cell 0+1 r1", "covered 1+1", "gap 2..3"], + ] + ); +} + +#[test] +fn layout_clamps_row_spans_to_the_group_end() { + let rows = || vec![vec![grid_cell(0, 1, 9)], vec![], vec![]]; + // Clamped to the end of the table. + let t = grid(1, rows()); + assert_eq!( + shape(&t, |_| 3), + [ + vec!["cell 0+1 r3"], + vec!["covered 0+1"], + vec!["covered 0+1"] + ] + ); + // Clamped to a group of its first row only: nothing is covered below. + assert_eq!( + shape(&t, |r| if r < 1 { 1 } else { 3 }), + [vec!["cell 0+1 r1"], vec!["gap 0..1"], vec!["gap 0..1"]] + ); + // A group end at or before the row (a caller bug) still lays out, with + // every row span at 1, instead of panicking. + assert_eq!( + shape(&t, |_| 0), + [vec!["cell 0+1 r1"], vec!["gap 0..1"], vec!["gap 0..1"]] + ); +} + +#[test] +fn layout_ends_a_row_span_after_its_rows() { + let t = grid(2, vec![vec![grid_cell(0, 2, 2)], vec![], vec![]]); + assert_eq!( + shape(&t, |_| 3), + [vec!["cell 0+2 r2"], vec!["covered 0+2"], vec!["gap 0..2"]] + ); +} + +#[test] +fn layout_fills_each_gap_with_one_slot() { + let t = grid( + 4, + vec![ + vec![grid_cell(2, 1, 1)], + vec![], + // A cell past the grid has nowhere to go. + vec![grid_cell(9, 1, 1)], + vec![grid_cell(0, 1, 1), grid_cell(2, 1, 2)], + vec![], + ], + ); + assert_eq!( + shape(&t, |_| 5), + [ + vec!["gap 0..2", "cell 2+1 r1", "gap 3..4"], + vec!["gap 0..4"], + vec!["gap 0..4"], + vec!["cell 0+1 r1", "gap 1..2", "cell 2+1 r2", "gap 3..4"], + vec!["gap 0..2", "covered 2+1", "gap 3..4"], + ] + ); +} + +#[test] +fn layout_moves_an_overlapped_cell_to_the_next_free_column() { + let t = grid(3, vec![vec![grid_cell(0, 2, 1), grid_cell(1, 1, 1)]]); + assert_eq!(shape(&t, |_| 1), [vec!["cell 0+2 r1", "cell 2+1 r1"]]); +} + +#[test] +fn layout_marks_rows_covered_by_a_header_cell() { + let mut head = grid_cell(0, 1, 2); + head.header = true; + let t = grid(2, vec![vec![head, grid_cell(1, 1, 1)], vec![]]); + assert_eq!( + shape(&t, |_| 2), + [ + vec!["cell 0+1 r2", "cell 1+1 r1"], + vec!["covered 0+1 h", "gap 1..2"], + ] + ); +} + +#[test] +fn layout_of_an_empty_grid_has_empty_rows() { + let t = grid(0, vec![vec![grid_cell(0, 1, 1)], vec![]]); + assert_eq!(shape(&t, |_| 2), [Vec::::new(), vec![]]); + assert_eq!(layout(&grid(3, vec![]), |_| 0).count(), 0); +} + +#[test] +fn grid_edges_add_up_to_the_total() { + assert_eq!(grid_edges(&[0.3, 0.7], 9072), [0, 2722, 9072]); + // Each column's width is rounded; the last one takes what is left. + assert_eq!(grid_edges(&[1.0 / 3.0; 3], 1000), [0, 333, 666, 1000]); + assert_eq!(grid_edges(&[1.0], 500), [0, 500]); + assert_eq!(grid_edges(&[], 500), [0]); + // Shares that round up past the total never push an edge beyond it. + assert_eq!(grid_edges(&[0.6, 0.6, 0.6], 10), [0, 6, 10, 10]); +} diff --git a/crates/knot-export/tests/walk.rs b/crates/knot-export/tests/walk.rs new file mode 100644 index 00000000..c6debb49 --- /dev/null +++ b/crates/knot-export/tests/walk.rs @@ -0,0 +1,733 @@ +mod common; + +use common::*; +use knot_export::{ + Block, DiagramRef, ImageSource, Inline, LinkTarget, List, ListItem, Marks, Run, blob_ids, + diagrams, user_ids, walk, +}; +use sha2::{Digest, Sha256}; +use yrs::Any; + +const DOC_ID: &str = "0b1c2d3e-4f50-4162-8374-8596a7b8c9d0"; +const USER_ID: &str = "7a6b5c4d-3e2f-4101-9283-746556473829"; + +fn run(text: &str) -> Run { + Run { + text: text.into(), + marks: Marks::default(), + } +} + +fn single_link(href: &str) -> Inline { + let w = walk(&build(vec![el( + "paragraph", + vec![], + vec![linked("x", href)], + )])); + match &w.doc.blocks[..] { + [Block::Paragraph(i)] if i.len() == 1 => i[0].clone(), + other => panic!("unexpected {other:?}"), + } +} + +fn target(href: &str) -> LinkTarget { + match single_link(href) { + Inline::Link { target, .. } => target, + other => panic!("not a link: {other:?}"), + } +} + +#[test] +fn empty_doc_walks_to_no_blocks() { + assert_eq!(walk(&build(vec![])).doc.blocks, vec![]); +} + +#[test] +fn headings_accept_number_and_string_levels() { + let w = walk(&build(vec![ + h(2, "a"), + el("heading", vec![("level", "3".into())], vec![t("b")]), + el("heading", vec![("level", "9".into())], vec![t("c")]), + ])); + let levels: Vec = w + .doc + .blocks + .iter() + .map(|b| match b { + Block::Heading { level, .. } => *level, + other => panic!("{other:?}"), + }) + .collect(); + assert_eq!(levels, vec![2, 3, 6]); + assert!(w.unknown_elements.is_empty()); +} + +#[test] +fn marks_are_read_by_key_presence() { + let all = Marks { + bold: true, + italic: true, + underline: true, + strike: true, + code: true, + }; + let names = ["bold", "italic", "underline", "strike", "code"]; + let bools = N::Text(vec![( + "y".into(), + names.iter().map(|n| (*n, Any::Bool(true))).collect(), + )]); + let w = walk(&build(vec![el( + "paragraph", + vec![], + vec![marked("x", &names), bools], + )])); + assert_eq!( + w.doc.blocks, + vec![Block::Paragraph(vec![ + Inline::Run(Run { + text: "x".into(), + marks: all.clone() + }), + Inline::Run(Run { + text: "y".into(), + marks: all + }), + ])] + ); +} + +#[test] +fn link_targets_are_classified() { + assert_eq!( + target("https://a.example"), + LinkTarget::Web("https://a.example/".into()) + ); + assert_eq!(target("mailto:x@y"), LinkTarget::Web("mailto:x@y".into())); + assert_eq!( + target(&format!("knot://doc/{DOC_ID}")), + LinkTarget::Doc(DOC_ID.into()) + ); + assert_eq!( + target(&format!("knot://user/{USER_ID}")), + LinkTarget::User(USER_ID.into()) + ); + assert_eq!( + target("knot://time/2026-10-23T10:00:00Z"), + LinkTarget::Plain + ); + assert_eq!(target("javascript:alert(1)"), LinkTarget::Plain); + assert_eq!(target("/relative"), LinkTarget::Plain); + assert_eq!(target("https://bad host/with space"), LinkTarget::Plain); + assert_eq!( + target("HTTPS://A.example"), + LinkTarget::Web("https://a.example/".into()) + ); + assert_eq!(target("knot://doc/not-a-uuid"), LinkTarget::Plain); + assert_eq!(target("knot://user/a b"), LinkTarget::Plain); +} + +#[test] +fn adjacent_runs_with_same_href_merge() { + let w = walk(&build(vec![el( + "paragraph", + vec![], + vec![ + linked("a", "https://a.example"), + N::Text(vec![( + "b".into(), + vec![ + ("bold", Any::Bool(true)), + ("link", { + let mut m = std::collections::HashMap::new(); + m.insert("href".to_owned(), Any::String("https://a.example".into())); + Any::Map(std::sync::Arc::new(m)) + }), + ], + )]), + linked("c", "https://other.example"), + ], + )])); + let Block::Paragraph(inl) = &w.doc.blocks[0] else { + panic!() + }; + assert_eq!(inl.len(), 2); + match &inl[0] { + Inline::Link { runs, .. } => { + assert_eq!(runs.len(), 2); + assert_eq!(runs[0].text, "a"); + assert!(runs[1].marks.bold); + } + other => panic!("{other:?}"), + } +} + +#[test] +fn code_quote_rule_and_hard_break() { + let w = walk(&build(vec![ + el( + "code_block", + vec![("language", "rust".into())], + vec![t("a\n\tb")], + ), + el("blockquote", vec![], vec![p("one"), p("two")]), + el("horizontal_rule", vec![], vec![]), + el( + "paragraph", + vec![], + vec![t("a"), el("hard_break", vec![], vec![]), t("b")], + ), + ])); + assert_eq!( + w.doc.blocks, + vec![ + Block::Code { + language: "rust".into(), + text: "a\n\tb".into() + }, + Block::Quote(vec![ + Block::Paragraph(vec![Inline::Run(run("one"))]), + Block::Paragraph(vec![Inline::Run(run("two"))]), + ]), + Block::Rule, + Block::Paragraph(vec![ + Inline::Run(run("a")), + Inline::Break, + Inline::Run(run("b")), + ]), + ] + ); + assert!(w.unknown_elements.is_empty()); +} + +#[test] +fn mermaid_key_hashes_plain_text_content() { + let source = "graph TD; A-->B"; + let w = walk(&build(vec![el( + "code_block", + vec![("language", "mermaid".into())], + vec![t("graph TD; "), marked("A-->B", &["bold"])], + )])); + assert_eq!( + w.doc.blocks, + vec![Block::Mermaid { + key: format!("mermaid-{:x}", Sha256::digest(source.as_bytes())), + source: source.into(), + }] + ); +} + +#[test] +fn unknown_block_becomes_paragraph_and_is_reported() { + let w = walk(&build(vec![el( + "mystery", + vec![], + vec![el("paragraph", vec![], vec![t("kept")])], + )])); + assert_eq!( + w.doc.blocks, + vec![Block::Paragraph(vec![Inline::Run(run("kept"))])] + ); + assert_eq!(w.unknown_elements, vec!["mystery".to_string()]); +} + +fn para(text: &str) -> Block { + Block::Paragraph(vec![Inline::Run(run(text))]) +} + +fn item(text: &str) -> N { + el("list_item", vec![], vec![p(text)]) +} + +fn only_list(w: &knot_export::Walked) -> &List { + match &w.doc.blocks[..] { + [Block::List(l)] => l, + other => panic!("unexpected {other:?}"), + } +} + +#[test] +fn ordered_list_start_from_number_or_string() { + let w = walk(&build(vec![ + el( + "ordered_list", + vec![("start", Any::Number(5.0))], + vec![item("a")], + ), + el("ordered_list", vec![("start", "3".into())], vec![item("b")]), + el("ordered_list", vec![], vec![item("c")]), + ])); + let lists: Vec<(bool, u32)> = w + .doc + .blocks + .iter() + .map(|b| match b { + Block::List(l) => (l.ordered, l.start), + other => panic!("{other:?}"), + }) + .collect(); + assert_eq!(lists, vec![(true, 5), (true, 3), (true, 1)]); +} + +#[test] +fn checked_accepts_bool_and_string() { + let w = walk(&build(vec![el( + "bullet_list", + vec![], + vec![ + el( + "list_item", + vec![("checked", Any::Bool(true))], + vec![p("a")], + ), + el("list_item", vec![("checked", "false".into())], vec![p("b")]), + item("c"), + ], + )])); + let l = only_list(&w); + assert!(!l.ordered); + let checked: Vec> = l.items.iter().map(|i| i.checked).collect(); + assert_eq!(checked, vec![Some(true), Some(false), None]); +} + +#[test] +fn nested_mixed_lists_keep_structure() { + let w = walk(&build(vec![el( + "bullet_list", + vec![], + vec![el( + "list_item", + vec![], + vec![ + p("top"), + el( + "ordered_list", + vec![("start", Any::Number(2.0))], + vec![el( + "list_item", + vec![], + vec![ + p("mid"), + el( + "bullet_list", + vec![], + vec![ + el("list_item", vec![("checked", "true".into())], vec![p("x")]), + el("list_item", vec![("checked", "no".into())], vec![p("y")]), + ], + ), + ], + )], + ), + ], + )], + )])); + assert_eq!( + w.doc.blocks, + vec![Block::List(List { + ordered: false, + start: 1, + items: vec![ListItem { + checked: None, + blocks: vec![ + para("top"), + Block::List(List { + ordered: true, + start: 2, + items: vec![ListItem { + checked: None, + blocks: vec![ + para("mid"), + Block::List(List { + ordered: false, + start: 1, + items: vec![ + ListItem { + checked: Some(true), + blocks: vec![para("x")] + }, + ListItem { + checked: Some(false), + blocks: vec![para("y")] + }, + ], + }), + ], + }], + }), + ], + }], + })] + ); + assert!(w.unknown_elements.is_empty()); +} + +#[test] +fn text_directly_under_list_item_is_wrapped() { + let w = walk(&build(vec![el( + "bullet_list", + vec![], + vec![el( + "list_item", + vec![], + vec![t("a"), el("hard_break", vec![], vec![]), t("b")], + )], + )])); + assert_eq!( + only_list(&w).items, + vec![ListItem { + checked: None, + blocks: vec![Block::Paragraph(vec![ + Inline::Run(run("a")), + Inline::Break, + Inline::Run(run("b")), + ])], + }] + ); + assert!(w.unknown_elements.is_empty()); +} + +#[test] +fn text_under_blockquote_is_wrapped() { + let w = walk(&build(vec![el( + "blockquote", + vec![], + vec![t("loose"), p("real")], + )])); + assert_eq!( + w.doc.blocks, + vec![Block::Quote(vec![para("loose"), para("real")])] + ); + assert!(w.unknown_elements.is_empty()); +} + +const BLOB_A: &str = "11111111-2222-4333-8444-555555555555"; +const BLOB_B: &str = "66666666-7777-4888-8999-aaaaaaaaaaaa"; + +fn image(src: &str) -> N { + el( + "image", + vec![("src", src.into()), ("alt", "pic".into())], + vec![], + ) +} + +fn mermaid(source: &str) -> N { + el( + "code_block", + vec![("language", "mermaid".into())], + vec![t(source)], + ) +} + +fn board(id: &str) -> N { + el("excalidraw_board", vec![("board_id", id.into())], vec![]) +} + +#[test] +fn image_sources_are_classified() { + let blob = format!("/api/blobs/{BLOB_A}"); + let w = walk(&build(vec![ + image(&blob), + image("https://x/y.png"), + image("http://x/y.png"), + image("data:image/png;base64,AA"), + image("img/a.png"), + ])); + let sources: Vec = w + .doc + .blocks + .iter() + .map(|b| match b { + Block::Image { source, alt } => { + assert_eq!(alt, "pic"); + source.clone() + } + other => panic!("{other:?}"), + }) + .collect(); + assert_eq!( + sources, + vec![ + ImageSource::Blob(BLOB_A.into()), + ImageSource::External("https://x/y.png".into()), + ImageSource::External("http://x/y.png".into()), + ImageSource::Unsupported, + ImageSource::Unsupported, + ] + ); + assert!(w.unknown_elements.is_empty()); +} + +#[test] +fn board_and_attachment_attributes() { + let w = walk(&build(vec![ + board("b1"), + el( + "excalidraw_board", + vec![("board_id", "b2".into()), ("label", "Flow".into())], + vec![], + ), + el( + "attachment", + vec![ + ("url", "/api/blobs/x".into()), + ("name", "a.pdf".into()), + ("size", Any::Number(2048.0)), + ], + vec![], + ), + el( + "attachment", + vec![ + ("url", "/api/blobs/y".into()), + ("name", "b.pdf".into()), + ("size", Any::Number(0.0)), + ], + vec![], + ), + ])); + assert_eq!( + w.doc.blocks, + vec![ + Block::Board { + key: "board-b1".into(), + board_id: "b1".into(), + label: "Diagram".into() + }, + Block::Board { + key: "board-b2".into(), + board_id: "b2".into(), + label: "Flow".into() + }, + Block::Attachment { + url: "/api/blobs/x".into(), + name: "a.pdf".into(), + size: Some(2048) + }, + Block::Attachment { + url: "/api/blobs/y".into(), + name: "b.pdf".into(), + size: None + }, + ] + ); +} + +fn cell_of(children: Vec) -> N { + el( + "table", + vec![], + vec![el( + "table_row", + vec![], + vec![el("table_cell", vec![], children)], + )], + ) +} + +#[test] +fn diagrams_are_listed_once_in_order() { + let w = walk(&build(vec![ + mermaid("graph TD; A-->B"), + mermaid("graph TD; A-->B"), + cell_of(vec![board("b1"), board("b1")]), + ])); + let key = format!("mermaid-{:x}", Sha256::digest(b"graph TD; A-->B")); + assert_eq!( + diagrams(&w.doc), + vec![ + DiagramRef::Mermaid { + key, + source: "graph TD; A-->B".into() + }, + DiagramRef::Board { + key: "board-b1".into(), + board_id: "b1".into(), + label: "Diagram".into() + }, + ] + ); +} + +#[test] +fn blob_and_user_ids_are_collected_from_nested_blocks() { + let w = walk(&build(vec![ + el( + "bullet_list", + vec![], + vec![el( + "list_item", + vec![], + vec![image(&format!("/api/blobs/{BLOB_B}"))], + )], + ), + image(&format!("/api/blobs/{BLOB_A}")), + image(&format!("/api/blobs/{BLOB_B}")), + image("https://x/y.png"), + cell_of(vec![el( + "paragraph", + vec![], + vec![linked("@u", &format!("knot://user/{USER_ID}"))], + )]), + el( + "paragraph", + vec![], + vec![linked("@u", &format!("knot://user/{USER_ID}"))], + ), + ])); + assert_eq!( + blob_ids(&w.doc), + vec![BLOB_B.to_string(), BLOB_A.to_string()] + ); + assert_eq!(user_ids(&w.doc), vec![USER_ID.to_string()]); +} + +#[test] +fn unknown_block_element_degrades_and_is_reported() { + let w = walk(&build(vec![el("callout", vec![], vec![p("hi")])])); + assert_eq!( + w.doc.blocks, + vec![Block::Paragraph(vec![Inline::Run(run("hi"))])] + ); + assert_eq!(w.unknown_elements, vec!["callout".to_string()]); +} + +fn quote_depth(blocks: &[Block]) -> usize { + // Iterative on purpose: the model itself may be deep. + let mut depth = 0; + let mut cur = blocks; + while let Some(Block::Quote(inner)) = cur.first() { + depth += 1; + cur = inner; + } + depth +} + +fn all_text(blocks: &[Block]) -> String { + let mut s = String::new(); + let mut stack: Vec<&Block> = blocks.iter().rev().collect(); + while let Some(b) = stack.pop() { + match b { + Block::Paragraph(inl) => { + for i in inl { + if let Inline::Run(r) = i { + s.push_str(&r.text); + } + } + } + Block::Quote(inner) => stack.extend(inner.iter().rev()), + _ => {} + } + } + s +} + +#[test] +fn deep_nesting_is_capped_without_stack_overflow() { + use yrs::{Transact, XmlElementPrelim, XmlFragment, XmlTextPrelim}; + let doc = yrs::Doc::new(); + { + let frag = doc.get_or_insert_xml_fragment("default"); + let mut txn = doc.transact_mut(); + let mut cur = frag.push_back(&mut txn, XmlElementPrelim::empty("blockquote")); + for _ in 1..5_000 { + cur = cur.push_back(&mut txn, XmlElementPrelim::empty("blockquote")); + } + let para = cur.push_back(&mut txn, XmlElementPrelim::empty("paragraph")); + para.push_back(&mut txn, XmlTextPrelim::new("deep")); + } + let (text, depth, unknown) = std::thread::Builder::new() + .stack_size(256 * 1024) + .spawn(move || { + let w = walk(&doc); + let summary = ( + all_text(&w.doc.blocks), + quote_depth(&w.doc.blocks), + w.unknown_elements.clone(), + ); + // The capped model is shallow, so dropping it is safe. + drop(w); + summary + }) + .unwrap() + .join() + .expect("walk must not overflow the stack"); + assert!(text.contains("deep"), "{text:?}"); + assert!(unknown.iter().any(|u| u == "depth-limit"), "{unknown:?}"); + assert!(depth <= knot_export::MAX_DEPTH + 1, "{depth}"); +} + +#[test] +fn deeply_nested_lists_are_capped_without_stack_overflow() { + use yrs::{Transact, XmlElementPrelim, XmlFragment, XmlTextPrelim}; + let doc = yrs::Doc::new(); + { + let frag = doc.get_or_insert_xml_fragment("default"); + let mut txn = doc.transact_mut(); + let mut cur = frag.push_back(&mut txn, XmlElementPrelim::empty("bullet_list")); + for i in 1..5_000 { + let tag = if i % 2 == 0 { + "bullet_list" + } else { + "ordered_list" + }; + cur = cur.push_back(&mut txn, XmlElementPrelim::empty(tag)); + } + let para = cur.push_back(&mut txn, XmlElementPrelim::empty("paragraph")); + para.push_back(&mut txn, XmlTextPrelim::new("deep")); + } + let unknown = std::thread::Builder::new() + .stack_size(256 * 1024) + .spawn(move || { + let w = walk(&doc); + let u = w.unknown_elements.clone(); + // Leak the shallow capped model; the test only needs the walk. + std::mem::forget(w); + u + }) + .unwrap() + .join() + .expect("walk must not overflow the stack on nested lists"); + assert!(unknown.iter().any(|u| u == "depth-limit"), "{unknown:?}"); +} + +#[test] +fn heading_level_non_finite_defaults_to_1() { + let w = walk(&build(vec![ + el("heading", vec![("level", "NaN".into())], vec![t("a")]), + el("heading", vec![("level", "inf".into())], vec![t("b")]), + el("heading", vec![("level", "-inf".into())], vec![t("c")]), + ])); + for b in &w.doc.blocks { + assert!(matches!(b, Block::Heading { level: 1, .. }), "{b:?}"); + } +} + +#[test] +fn ids_are_canonicalised() { + let upper = DOC_ID.to_uppercase(); + assert_eq!( + target(&format!("knot://doc/{upper}")), + LinkTarget::Doc(DOC_ID.into()) + ); + assert_eq!( + target(&format!("knot://doc/{}", DOC_ID.replace('-', ""))), + LinkTarget::Doc(DOC_ID.into()) + ); + assert_eq!( + target(&format!("knot://user/{}", USER_ID.to_uppercase())), + LinkTarget::User(USER_ID.into()) + ); + let img = walk(&build(vec![el( + "image", + vec![( + "src", + format!("/api/blobs/{{{}}}", DOC_ID.to_uppercase()).into(), + )], + vec![], + )])); + assert!(matches!( + &img.doc.blocks[..], + [Block::Image { source: ImageSource::Blob(id), .. }] if id == DOC_ID + )); +} diff --git a/crates/knot-obs/src/metrics.rs b/crates/knot-obs/src/metrics.rs index a4f9b539..2d554e20 100644 --- a/crates/knot-obs/src/metrics.rs +++ b/crates/knot-obs/src/metrics.rs @@ -70,6 +70,21 @@ pub fn init(addr: &str) -> Result<(), MetricsError> { "Collab socket upgraded -> initial document state queued to the client" ); + // Export + describe_counter!( + "knot_export_total", + "Page exports to DOCX/ODT, by format (docx|odt|unknown) and outcome (ok|client_error|busy|server_error); a request timed out or abandoned before its response counts as client_error (408)" + ); + describe_histogram!( + "knot_export_duration_seconds", + Unit::Seconds, + "Page export request duration, from the request to the written file, or to the request timeout or the client going away" + ); + describe_counter!( + "knot_export_fallback_total", + "Content an export could not include as-is, by kind (diagram|image_unavailable|image_over_budget|unknown_element)" + ); + // Storage / pool describe_gauge!("knot_db_pool_size", "Total connections in the pool"); describe_gauge!("knot_db_pool_idle", "Idle connections in the pool"); diff --git a/crates/knot-server/Cargo.toml b/crates/knot-server/Cargo.toml index f487f576..ae813345 100644 --- a/crates/knot-server/Cargo.toml +++ b/crates/knot-server/Cargo.toml @@ -19,6 +19,7 @@ knot-auth = { path = "../knot-auth" } knot-config = { path = "../knot-config" } knot-crdt = { path = "../knot-crdt" } knot-docs = { path = "../knot-docs" } +knot-export = { path = "../knot-export" } knot-markdown = { path = "../knot-markdown" } knot-obs = { path = "../knot-obs" } knot-storage = { path = "../knot-storage" } diff --git a/crates/knot-server/src/export.rs b/crates/knot-server/src/export.rs new file mode 100644 index 00000000..ad24f6bd --- /dev/null +++ b/crates/knot-server/src/export.rs @@ -0,0 +1,395 @@ +//! Page export to DOCX and ODT: snapshot the live page, resolve what needs IO +//! (images, diagram pictures, mention names), and write the file in memory. +//! The pure work lives in `knot-export`; this module feeds it. + +use std::collections::HashMap; +use std::fmt::Write as _; +use std::time::Duration; + +use chrono::{DateTime, Utc}; +use knot_crdt::DocHandle; +use knot_export::{ + Assets, DiagramRef, ExportDoc, Fallbacks, ImageAsset, Meta, picture_from_png, prepare_image, +}; +use knot_storage::BlobStoreError; +use uuid::Uuid; + +use crate::AppState; +use crate::routes::api::markdown::live_snapshot; + +/// Image bytes one export may embed (40 MiB), counted after preparation. +pub const IMAGE_BUDGET_BYTES: i64 = 40 * 1024 * 1024; +/// Diagram pictures are rendered at twice their display size. +pub const DIAGRAM_SCALE: u32 = 2; +/// Exports one server process runs at a time. +pub const MAX_CONCURRENT_EXPORTS: usize = 2; +/// How long an export waits for a free slot before it is refused as busy. +pub const BUSY_WAIT: Duration = Duration::from_secs(5); +/// Longest file name before the extension, in code points. +const MAX_FILE_STEM_CHARS: usize = 120; + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum Format { + Docx, + Odt, +} + +impl Format { + /// `docx` or `odt`, exactly as they appear in the route. + pub fn parse(s: &str) -> Option { + match s { + "docx" => Some(Self::Docx), + "odt" => Some(Self::Odt), + _ => None, + } + } + + pub fn content_type(self) -> &'static str { + match self { + Self::Docx => "application/vnd.openxmlformats-officedocument.wordprocessingml.document", + Self::Odt => "application/vnd.oasis.opendocument.text", + } + } + + pub fn extension(self) -> &'static str { + match self { + Self::Docx => "docx", + Self::Odt => "odt", + } + } +} + +/// A written export. +pub struct Built { + pub bytes: Vec, + pub fallbacks: Fallbacks, + pub title: String, +} + +#[derive(Debug, thiserror::Error)] +pub enum ExportFailure { + #[error("room: {0}")] + Room(String), + #[error("store: {0}")] + Store(String), + #[error("write: {0}")] + Write(String), +} + +fn store_err(e: impl std::fmt::Display) -> ExportFailure { + ExportFailure::Store(e.to_string()) +} + +fn not_configured(what: &str) -> ExportFailure { + ExportFailure::Store(format!("{what} not configured")) +} + +/// The page's live state, copied into a transient doc by the same +/// [`live_snapshot`] the markdown export uses. The room actor only encodes +/// its state, so it keeps serving editors while the export walks the copy. +pub async fn snapshot(state: &AppState, doc_id: Uuid) -> Result { + let (transient, _seq) = live_snapshot(state, doc_id) + .await + .map_err(|e| ExportFailure::Room(e.to_string()))?; + Ok(transient) +} + +/// [`resolve_assets_with_budget`] with the standard image budget. +pub async fn resolve_assets( + state: &AppState, + user_id: Uuid, + doc: &ExportDoc, + diagrams: HashMap>, +) -> Result { + resolve_assets_with_budget(state, user_id, doc, diagrams, IMAGE_BUDGET_BYTES).await +} + +/// Everything the writers need from outside the page: +/// - each knot image, if `user_id` may read the page it belongs to and it +/// fits in `budget`; +/// - a picture for each diagram on the page, from `diagrams` (keys that +/// aren't on the page are ignored); +/// - the current display name of each mentioned user. +pub async fn resolve_assets_with_budget( + state: &AppState, + user_id: Uuid, + doc: &ExportDoc, + mut diagrams: HashMap>, + budget: i64, +) -> Result { + let mut assets = Assets::default(); + + let mut spent = 0; + for id in knot_export::blob_ids(doc) { + let image = resolve_image(state, user_id, &id, budget, &mut spent).await?; + assets.images.insert(id, image); + } + + for diagram in knot_export::diagrams(doc) { + let (DiagramRef::Mermaid { key, .. } | DiagramRef::Board { key, .. }) = diagram; + let picture = diagrams + .remove(&key) + .and_then(|png| picture_from_png(png, DIAGRAM_SCALE)); + if let Some(picture) = picture { + assets.diagrams.insert(key, picture); + } + } + + for id in knot_export::user_ids(doc) { + let users = state + .users + .as_ref() + .ok_or_else(|| not_configured("users"))?; + let Ok(mentioned) = Uuid::parse_str(&id) else { + continue; + }; + if let Some(user) = users.find_by_id(mentioned).await.map_err(store_err)? { + assets.user_names.insert(id, user.display_name); + } + } + Ok(assets) +} + +/// One knot image. Access is checked on the blob's own page, like +/// `GET /api/blobs/{id}`, so an image copied from a page the reader can't +/// open stays out. `spent` is the running total of embedded bytes. +async fn resolve_image( + state: &AppState, + user_id: Uuid, + id: &str, + budget: i64, + spent: &mut i64, +) -> Result { + let (Some(blob_meta), Some(acl), Some(blobs)) = ( + state.blob_meta.as_ref(), + state.acl.as_ref(), + state.blob_store.as_ref(), + ) else { + return Err(not_configured("blob storage")); + }; + let Ok(blob_id) = Uuid::parse_str(id) else { + return Ok(ImageAsset::Unavailable); + }; + let Some(meta) = blob_meta.find(blob_id).await.map_err(store_err)? else { + return Ok(ImageAsset::Unavailable); + }; + let role = acl + .effective_role(meta.workspace_id, meta.doc_id, user_id) + .await + .map_err(store_err)?; + if role.is_none() { + return Ok(ImageAsset::Unavailable); + } + // Decide on the stored size first, so an image past the budget is never + // fetched. + if spent.saturating_add(meta.byte_size) > budget { + return Ok(ImageAsset::OverBudget); + } + let bytes = match blobs.get(blob_id).await { + Ok(bytes) => bytes, + Err(BlobStoreError::NotFound) => return Ok(ImageAsset::Unavailable), + Err(e) => return Err(store_err(e)), + }; + let picture = match tokio::task::spawn_blocking(move || prepare_image(&bytes)).await { + Ok(Some(picture)) => picture, + Ok(None) => return Ok(ImageAsset::Unavailable), + Err(e) => { + tracing::warn!(%blob_id, error = %e, "export image preparation failed"); + return Ok(ImageAsset::Unavailable); + } + }; + // Charge what gets embedded: a WebP converted to PNG can outgrow the + // stored file. + let total = spent.saturating_add(i64::try_from(picture.bytes.len()).unwrap_or(i64::MAX)); + if total > budget { + return Ok(ImageAsset::OverBudget); + } + *spent = total; + Ok(ImageAsset::Embedded(picture)) +} + +/// Export the page's live state as `format`. `diagram_pngs` holds the +/// browser's diagram pictures by key. The caller holds an export permit. +pub async fn build( + state: &AppState, + user_id: Uuid, + doc_id: Uuid, + format: Format, + diagram_pngs: HashMap>, +) -> Result { + let handle = snapshot(state, doc_id).await?; + let walked = tokio::task::spawn_blocking(move || knot_export::walk(handle.inner())) + .await + .map_err(|e| ExportFailure::Write(format!("walk: {e}")))?; + let assets = resolve_assets(state, user_id, &walked.doc, diagram_pngs).await?; + let docs = state.docs.as_ref().ok_or_else(|| not_configured("docs"))?; + let title = docs + .get(doc_id) + .await + .map_err(store_err)? + .ok_or_else(|| ExportFailure::Store(format!("doc {doc_id} not found")))? + .title; + let meta = Meta { + title: title.clone(), + exported_on: exported_on(Utc::now()), + base_url: state.base_url.clone(), + }; + let doc = walked.doc; + let output = tokio::task::spawn_blocking(move || match format { + Format::Docx => knot_export::write_docx(&doc, &meta, &assets), + Format::Odt => knot_export::write_odt(&doc, &meta, &assets), + }) + .await + .map_err(|e| ExportFailure::Write(e.to_string()))? + .map_err(|e| ExportFailure::Write(e.to_string()))?; + let mut fallbacks = output.fallbacks; + fallbacks.unknown_element = walked.unknown_elements.len() as u32; + record_fallbacks(doc_id, &fallbacks); + Ok(Built { + bytes: output.bytes, + fallbacks, + title, + }) +} + +/// Count and log each kind of content the export could not include as-is. +fn record_fallbacks(doc_id: Uuid, fallbacks: &Fallbacks) { + for (kind, count) in [ + ("diagram", fallbacks.diagram), + ("image_unavailable", fallbacks.image_unavailable), + ("image_over_budget", fallbacks.image_over_budget), + ("unknown_element", fallbacks.unknown_element), + ] { + if count > 0 { + metrics::counter!("knot_export_fallback_total", "kind" => kind) + .increment(u64::from(count)); + tracing::warn!(%doc_id, kind, count, "export fallback"); + } + } +} + +/// The download name for a page titled `title`. The browser applies the +/// same rule (`exportFileName`), so both must change together. +pub fn file_name(title: &str, format: Format) -> String { + fn trim(s: &str) -> &str { + s.trim_matches([' ', '.']) + } + let replaced: String = title + .chars() + .map(|c| match c { + '/' | '\\' | ':' | '*' | '?' | '"' | '<' | '>' | '|' => '-', + c if c.is_control() => '-', + c => c, + }) + .collect(); + let capped: String = trim(&replaced).chars().take(MAX_FILE_STEM_CHARS).collect(); + let stem = match trim(&capped) { + "" => "Untitled", + stem => stem, + }; + format!("{stem}.{}", format.extension()) +} + +/// An RFC 6266 attachment header: a quoted ASCII fallback (`_` for anything +/// else) for old clients, and the exact name as RFC 8187 `filename*`. +pub fn content_disposition(file_name: &str) -> String { + let fallback: String = file_name + .chars() + .map(|c| match c { + '"' | '\\' => '_', + ' '..='~' => c, + _ => '_', + }) + .collect(); + let mut encoded = String::new(); + for b in file_name.bytes() { + if b.is_ascii_alphanumeric() || b"!#$&+-.^_`|~".contains(&b) { + encoded.push(char::from(b)); + } else { + let _ = write!(encoded, "%{b:02X}"); + } + } + format!("attachment; filename=\"{fallback}\"; filename*=UTF-8''{encoded}") +} + +/// The export date shown under the title, e.g. "23 Oct 2026". The DOCX +/// writer derives the document's dates from this exact form. +pub fn exported_on(now: DateTime) -> String { + now.format("%-d %b %Y").to_string() +} + +#[cfg(test)] +mod tests { + use super::*; + use chrono::TimeZone; + + #[test] + fn file_name_replaces_reserved_characters() { + assert_eq!( + file_name("Q4 / Plan: ?", Format::Docx), + "Q4 - Plan- -draft--.docx" + ); + } + + #[test] + fn file_name_falls_back_to_untitled() { + assert_eq!(file_name(" .. ", Format::Odt), "Untitled.odt"); + } + + #[test] + fn file_name_caps_code_points_and_replaces_controls() { + let long = format!("{}😀😀", "a".repeat(119)); + assert_eq!( + file_name(&long, Format::Docx), + format!("{}😀.docx", "a".repeat(119)) + ); + assert_eq!(file_name("a\u{1}b\u{9f}c", Format::Odt), "a-b-c.odt"); + assert_eq!(file_name(" Notes. ", Format::Odt), "Notes.odt"); + } + + #[test] + fn content_disposition_percent_encodes_utf8() { + let header = content_disposition("Über.docx"); + assert!(header.starts_with("attachment; "), "{header}"); + assert!(header.contains(r#"filename="_ber.docx""#), "{header}"); + assert!( + header.contains("filename*=UTF-8''%C3%9Cber.docx"), + "{header}" + ); + } + + #[test] + fn content_disposition_keeps_the_ascii_fallback_quotable() { + let header = content_disposition("a\"b\\c d.odt"); + assert!(header.contains(r#"filename="a_b_c d.odt""#), "{header}"); + assert!( + header.contains("filename*=UTF-8''a%22b%5Cc%20d.odt"), + "{header}" + ); + } + + #[test] + fn exported_on_is_day_month_year() { + let now = chrono::Utc + .with_ymd_and_hms(2026, 10, 23, 23, 59, 0) + .unwrap(); + assert_eq!(exported_on(now), "23 Oct 2026"); + } + + #[test] + fn format_parses_its_own_extension_only() { + for format in [Format::Docx, Format::Odt] { + assert_eq!(Format::parse(format.extension()), Some(format)); + } + assert_eq!(Format::parse("pdf"), None); + assert_eq!(Format::parse("DOCX"), None); + assert_eq!( + Format::Docx.content_type(), + "application/vnd.openxmlformats-officedocument.wordprocessingml.document" + ); + assert_eq!( + Format::Odt.content_type(), + "application/vnd.oasis.opendocument.text" + ); + } +} diff --git a/crates/knot-server/src/export_check.rs b/crates/knot-server/src/export_check.rs new file mode 100644 index 00000000..1e993e87 --- /dev/null +++ b/crates/knot-server/src/export_check.rs @@ -0,0 +1,138 @@ +//! `knot-server export-check`: run every page through both export writers +//! offline and report what fails or degrades. Meant for a staging copy of the +//! database. It never migrates and never writes. + +use std::panic::{AssertUnwindSafe, catch_unwind}; + +use anyhow::Context as _; +use knot_crdt::{DocHandle, Engine, YrsEngine}; +use knot_export::{Assets, Fallbacks, Meta}; +use knot_storage::{ + DocStore, PgDocStore, PgSnapshotStore, PgUpdatesStore, PgWorkspaceStore, SnapshotStore, + UpdatesStore, WorkspaceStore, +}; +use sqlx::PgPool; +use uuid::Uuid; + +#[derive(Debug)] +pub struct CheckReport { + pub pages: usize, + pub failed: Vec<(Uuid, String)>, + pub fallbacks: Fallbacks, +} + +/// The page as the room would load it: latest snapshot, then the updates +/// after it. Errors are returned, not swallowed. +pub async fn hydrate(pool: &PgPool, doc_id: Uuid) -> anyhow::Result { + let engine = YrsEngine; + let doc = engine.new_doc(); + let mut applied = 0; + if let Some(snap) = PgSnapshotStore::new(pool.clone()).latest(doc_id).await? { + engine.apply_update(&doc, &snap.state_bytes)?; + applied = snap.snapshot_seq; + } + for update in PgUpdatesStore::new(pool.clone()) + .since(doc_id, applied) + .await? + { + engine.apply_update(&doc, &update.update_bytes)?; + } + Ok(doc) +} + +fn add(total: &mut Fallbacks, f: &Fallbacks) { + total.diagram += f.diagram; + total.image_unavailable += f.image_unavailable; + total.image_over_budget += f.image_over_budget; + total.unknown_element += f.unknown_element; +} + +/// Walk and write one hydrated page in both formats. Returns the fallbacks +/// summed over the two writers. +fn check_page(handle: &DocHandle, base_url: &str) -> anyhow::Result { + let walked = knot_export::walk(handle.inner()); + let meta = Meta { + title: "export-check".to_owned(), + exported_on: String::new(), + base_url: base_url.to_owned(), + }; + // Offline: no image or diagram is resolved, so they fall back. + let assets = Assets::default(); + let unknown = u32::try_from(walked.unknown_elements.len()).unwrap_or(u32::MAX); + let mut total = Fallbacks::default(); + for mut output in [ + knot_export::write_docx(&walked.doc, &meta, &assets)?, + knot_export::write_odt(&walked.doc, &meta, &assets)?, + ] { + output.fallbacks.unknown_element += unknown; + add(&mut total, &output.fallbacks); + } + Ok(total) +} + +fn panic_message(payload: &(dyn std::any::Any + Send)) -> String { + payload + .downcast_ref::<&str>() + .map(|s| (*s).to_owned()) + .or_else(|| payload.downcast_ref::().cloned()) + .unwrap_or_else(|| "panic".to_owned()) +} + +pub async fn check_all(pool: &PgPool, base_url: &str) -> anyhow::Result { + let mut report = CheckReport { + pages: 0, + failed: Vec::new(), + fallbacks: Fallbacks::default(), + }; + let Some(ws) = PgWorkspaceStore::new(pool.clone()).get_singleton().await? else { + return Ok(report); + }; + let docs = PgDocStore::new(pool.clone()); + let mut pages = docs.list_alive(ws.id).await?; + pages.extend(docs.list_templates(ws.id).await?); + + for page in pages { + report.pages += 1; + let outcome = match hydrate(pool, page.id).await { + Ok(handle) => catch_unwind(AssertUnwindSafe(|| check_page(&handle, base_url))) + .unwrap_or_else(|p| Err(anyhow::anyhow!("panic: {}", panic_message(&*p)))), + Err(e) => Err(e.context("hydrate")), + }; + match outcome { + Ok(f) => add(&mut report.fallbacks, &f), + Err(e) => report.failed.push((page.id, format!("{e:#}"))), + } + } + Ok(report) +} + +/// Run the check against `cfg.database_url`; fails when any page fails. +pub async fn run(cfg: knot_config::Config) -> anyhow::Result<()> { + if cfg.database_url.is_empty() { + anyhow::bail!("KNOT_DATABASE_URL must be set"); + } + // Not knot_storage::connect: that runs migrations. + let pool = sqlx::postgres::PgPoolOptions::new() + .max_connections(2) + .connect(&cfg.database_url) + .await + .context("database connect")?; + let report = check_all(&pool, &cfg.base_url).await?; + println!( + "checked {} pages, {} failed", + report.pages, + report.failed.len() + ); + let f = &report.fallbacks; + println!("diagram: {}", f.diagram); + println!("image_unavailable: {}", f.image_unavailable); + println!("image_over_budget: {}", f.image_over_budget); + println!("unknown_element: {}", f.unknown_element); + for (id, error) in &report.failed { + println!("failed {id}: {error}"); + } + if !report.failed.is_empty() { + anyhow::bail!("{} pages failed", report.failed.len()); + } + Ok(()) +} diff --git a/crates/knot-server/src/lib.rs b/crates/knot-server/src/lib.rs index e064622b..e000a1c3 100644 --- a/crates/knot-server/src/lib.rs +++ b/crates/knot-server/src/lib.rs @@ -28,6 +28,8 @@ pub mod archive; pub mod auth; pub mod board_room_shim; pub mod comments_listener; +pub mod export; +pub mod export_check; pub mod http_error; pub mod metrics; pub mod protocol; @@ -79,6 +81,10 @@ pub struct AppState { /// 1001 Close and the process can drain within the grace period instead /// of being SIGKILLed mid-rollout. pub shutdown: tokio_util::sync::CancellationToken, + /// Page exports (DOCX/ODT) running at once in this process. + pub export_permits: Arc, + /// How long an export waits for a permit before it is refused as busy. + pub export_busy_wait: std::time::Duration, } impl AppState { @@ -114,6 +120,8 @@ impl AppState { config: Arc::new(Config::default()), build_version: env!("KNOT_BUILD_VERSION"), shutdown: tokio_util::sync::CancellationToken::new(), + export_permits: Arc::new(tokio::sync::Semaphore::new(export::MAX_CONCURRENT_EXPORTS)), + export_busy_wait: export::BUSY_WAIT, } } @@ -179,6 +187,8 @@ impl AppState { config: Arc::new(Config::default()), build_version: env!("KNOT_BUILD_VERSION"), shutdown: tokio_util::sync::CancellationToken::new(), + export_permits: Arc::new(tokio::sync::Semaphore::new(export::MAX_CONCURRENT_EXPORTS)), + export_busy_wait: export::BUSY_WAIT, } } diff --git a/crates/knot-server/src/main.rs b/crates/knot-server/src/main.rs index 91958207..78075957 100644 --- a/crates/knot-server/src/main.rs +++ b/crates/knot-server/src/main.rs @@ -29,6 +29,9 @@ enum Cmd { Admin(admin::AdminArgs), /// Run pending Postgres migrations and exit. Migrate, + /// Export every page offline to DOCX and ODT and report failures and + /// fallbacks. Does not migrate; meant for a staging copy. + ExportCheck, } #[tokio::main] @@ -55,6 +58,12 @@ async fn main() { process::exit(1); } }, + Cmd::ExportCheck => { + if let Err(e) = knot_server::export_check::run(cfg).await { + eprintln!("export-check: {e}"); + process::exit(1); + } + } Cmd::Migrate => { if cfg.database_url.is_empty() { eprintln!("migrate: KNOT_DATABASE_URL must be set"); diff --git a/crates/knot-server/src/routes/api/docs.rs b/crates/knot-server/src/routes/api/docs.rs index ec193eb4..9fa8abad 100644 --- a/crates/knot-server/src/routes/api/docs.rs +++ b/crates/knot-server/src/routes/api/docs.rs @@ -87,6 +87,7 @@ pub fn router(state: AppState) -> Router { post(crate::routes::api::history::restore), ) .merge(crate::routes::api::comments::routes()) + .merge(crate::routes::api::export::routes()) .route( "/api/docs/{id}/contributors", get(crate::routes::api::contributors::byline), diff --git a/crates/knot-server/src/routes/api/export.rs b/crates/knot-server/src/routes/api/export.rs new file mode 100644 index 00000000..ba89f87b --- /dev/null +++ b/crates/knot-server/src/routes/api/export.rs @@ -0,0 +1,480 @@ +//! Page export to Word and OpenDocument: +//! - GET /api/docs/{id}/export/diagrams the page's diagrams, for the +//! browser to render, and the upload limits +//! - POST /api/docs/{id}/export/{docx|odt} multipart, one PNG per diagram +//! key (an empty body means none) → the file +//! +//! Both sit behind `require_doc_role_mw`, so viewers and up may export, and +//! archived pages are refused. The export itself lives in [`crate::export`]. + +use std::collections::HashMap; +use std::time::Instant; + +use axum::{ + Json, Router, + extract::{Path, Request, State}, + http::{HeaderName, HeaderValue, StatusCode, header}, + response::{IntoResponse, Response}, + routing::{get, post}, +}; +use knot_export::{DiagramRef, Fallbacks}; +use multer::{Constraints, Multipart, SizeLimit}; +use serde::Serialize; +use tracing::Instrument; +use uuid::Uuid; + +use crate::AppState; +use crate::auth::{AuthContext, DocArchived, require_viewer}; +use crate::export::{self, DIAGRAM_SCALE, Format}; +use crate::http_error::{doc_archived, json_err}; + +/// Largest request body: 10 MiB, as for blob uploads, so a proxy that lets +/// image uploads through lets exports through too. +pub const MAX_BODY_BYTES: u64 = 10 * 1024 * 1024; +/// Largest diagram picture. A bigger part is dropped and falls back. +pub const MAX_PART_BYTES: usize = 2 * 1024 * 1024; +/// Parts read from one request. Later parts are dropped and fall back. +pub const MAX_PARTS: usize = 100; +/// `Retry-After` seconds on `503 export.busy`. +const RETRY_AFTER_SECS: HeaderValue = HeaderValue::from_static("5"); +const FALLBACKS_HEADER: HeaderName = HeaderName::from_static("x-knot-export-fallbacks"); + +pub fn routes() -> Router { + Router::new() + .route("/api/docs/{id}/export/diagrams", get(diagrams)) + .route("/api/docs/{id}/export/{format}", post(export_file)) +} + +#[derive(Serialize)] +struct DiagramsResponse { + limits: Limits, + diagrams: Vec, +} + +/// What the browser has to fit its pictures into. +#[derive(Serialize)] +struct Limits { + max_body_bytes: u64, + max_part_bytes: usize, + max_parts: usize, + scale: u32, +} + +#[derive(Serialize)] +#[serde(tag = "kind", rename_all = "lowercase")] +enum Diagram { + Mermaid { + key: String, + source: String, + }, + Board { + key: String, + board_id: String, + label: String, + }, +} + +impl From for Diagram { + fn from(diagram: DiagramRef) -> Self { + match diagram { + DiagramRef::Mermaid { key, source } => Self::Mermaid { key, source }, + DiagramRef::Board { + key, + board_id, + label, + } => Self::Board { + key, + board_id, + label, + }, + } + } +} + +/// The diagrams on the page's live state, under the keys the writers look +/// pictures up by. +async fn diagrams( + State(state): State, + Path(doc_id): Path, + req: Request, +) -> Response { + if let Some(refusal) = require_viewer(&req) { + return refusal; + } + // The role layer lets GETs through on archived pages; export refuses + // them on both routes. + if req.extensions().get::().is_some() { + return doc_archived(); + } + let handle = match export::snapshot(&state, doc_id).await { + Ok(handle) => handle, + Err(e) => { + tracing::error!(error = %e, %doc_id, "export diagrams: snapshot"); + return internal(); + } + }; + let listed = tokio::task::spawn_blocking(move || { + knot_export::diagrams(&knot_export::walk(handle.inner()).doc) + }) + .await; + let diagrams = match listed { + Ok(diagrams) => diagrams, + Err(e) => { + tracing::error!(error = %e, %doc_id, "export diagrams: walk"); + return internal(); + } + }; + Json(DiagramsResponse { + limits: Limits { + max_body_bytes: MAX_BODY_BYTES, + max_part_bytes: MAX_PART_BYTES, + max_parts: MAX_PARTS, + scale: DIAGRAM_SCALE, + }, + diagrams: diagrams.into_iter().map(Diagram::from).collect(), + }) + .into_response() +} + +/// Export the page as `format`, counting the outcome and timing it. +#[tracing::instrument(skip_all, name = "docs.export", fields(%doc_id))] +async fn export_file( + State(state): State, + Path((doc_id, format)): Path<(Uuid, String)>, + req: Request, +) -> Response { + let format = Format::parse(&format); + let mut recorded = Recorded::start(format); + let response = export_response(&state, doc_id, format, req).await; + recorded.status = Some(response.status()); + response +} + +/// Counts an export request in `knot_export_total` and times it in +/// `knot_export_duration_seconds` when dropped, so a request that never +/// gets its response counts too: the request timeout drops the handler +/// (the client sees 408), as does the client going away. Both count as +/// that 408, and are logged. +struct Recorded { + started: Instant, + format: &'static str, + /// The response's status, once the handler has one. + status: Option, +} + +impl Recorded { + fn start(format: Option) -> Self { + Self { + started: Instant::now(), + format: format.map_or("unknown", Format::extension), + status: None, + } + } +} + +impl Drop for Recorded { + fn drop(&mut self) { + let status = self.status.unwrap_or_else(|| { + tracing::warn!( + format = self.format, + "export ended without a response: timed out or the client went away" + ); + StatusCode::REQUEST_TIMEOUT + }); + let labels = [("format", self.format), ("outcome", outcome(status))]; + metrics::counter!("knot_export_total", &labels).increment(1); + metrics::histogram!("knot_export_duration_seconds", &labels) + .record(self.started.elapsed().as_secs_f64()); + } +} + +async fn export_response( + state: &AppState, + doc_id: Uuid, + format: Option, + req: Request, +) -> Response { + if let Some(refusal) = require_viewer(&req) { + return refusal; + } + let Some(user_id) = req.extensions().get::().map(|ctx| ctx.user_id) else { + return json_err(StatusCode::UNAUTHORIZED, "auth.session_required", ""); + }; + let Some(format) = format else { + return json_err( + StatusCode::BAD_REQUEST, + "export.bad_format", + "format must be docx or odt", + ); + }; + // Taken before the upload is read, so only the exports holding a permit + // keep pictures, images and output in memory. + let permit = tokio::time::timeout( + state.export_busy_wait, + state.export_permits.clone().acquire_owned(), + ) + .await; + let permit = match permit { + Ok(Ok(permit)) => permit, + Ok(Err(e)) => { + tracing::error!(error = %e, "export permits closed"); + return internal(); + } + Err(_) => return busy(), + }; + let Ok(pictures) = read_pictures(req).await else { + return json_err( + StatusCode::PAYLOAD_TOO_LARGE, + "export.too_large", + "10 MiB cap", + ); + }; + // The build runs on its own task, which owns the permit until the build + // ends. Dropping this handler (the request timeout, or the client going + // away) then can't hand the permit back while the build's blocking walk + // and write still hold the images, pictures and output in memory. + let build = tokio::spawn( + { + let state = state.clone(); + async move { + let built = export::build(&state, user_id, doc_id, format, pictures).await; + drop(permit); + built + } + } + .in_current_span(), + ); + let built = match build.await { + Ok(Ok(built)) => built, + Ok(Err(e)) => { + tracing::error!(error = %e, "export"); + return internal(); + } + Err(e) => { + tracing::error!(error = %e, "export task"); + return internal(); + } + }; + let file_name = export::file_name(&built.title, format); + ( + [ + (header::CONTENT_TYPE, format.content_type().to_owned()), + ( + header::CONTENT_DISPOSITION, + export::content_disposition(&file_name), + ), + (FALLBACKS_HEADER, fallbacks_header(&built.fallbacks)), + ], + built.bytes, + ) + .into_response() +} + +/// The request body was over [`MAX_BODY_BYTES`]. +struct TooLarge; + +/// The diagram pictures in the request, by part name. An empty body, one +/// that isn't multipart, or one that turns out malformed gives what was read +/// so far: a missing picture falls back, it never fails the export. Parts +/// over [`MAX_PART_BYTES`] or past [`MAX_PARTS`] are read and dropped, since +/// multer can't continue after its own per-field limit. +async fn read_pictures(req: Request) -> Result>, TooLarge> { + let mut pictures = HashMap::new(); + let Some(boundary) = req + .headers() + .get(header::CONTENT_TYPE) + .and_then(|v| v.to_str().ok()) + .and_then(|ct| multer::parse_boundary(ct).ok()) + else { + return Ok(pictures); + }; + let constraints = Constraints::new().size_limit(SizeLimit::new().whole_stream(MAX_BODY_BYTES)); + let mut multipart = + Multipart::with_constraints(req.into_body().into_data_stream(), boundary, constraints); + let mut parts = 0; + loop { + let mut field = match multipart.next_field().await { + Ok(Some(field)) => field, + Ok(None) => return Ok(pictures), + Err(e) => return stop_reading(e, pictures), + }; + parts += 1; + let name = field.name().map(str::to_owned); + let mut kept = (parts <= MAX_PARTS && name.is_some()).then(Vec::new); + loop { + match field.chunk().await { + Ok(Some(chunk)) => { + if kept + .as_ref() + .is_some_and(|bytes| bytes.len() + chunk.len() > MAX_PART_BYTES) + { + kept = None; + } + if let Some(bytes) = kept.as_mut() { + bytes.extend_from_slice(&chunk); + } + } + Ok(None) => break, + Err(e) => return stop_reading(e, pictures), + } + } + if let (Some(name), Some(bytes)) = (name, kept) { + pictures.insert(name, bytes); + } + } +} + +fn stop_reading( + e: multer::Error, + pictures: HashMap>, +) -> Result>, TooLarge> { + if matches!(e, multer::Error::StreamSizeExceeded { .. }) { + return Err(TooLarge); + } + tracing::debug!(error = %e, "export pictures: unreadable multipart body"); + Ok(pictures) +} + +/// `X-Knot-Export-Fallbacks`. The client shows `diagrams` and `images`. +fn fallbacks_header(fallbacks: &Fallbacks) -> String { + format!( + "diagrams={}, images={}, unknown={}", + fallbacks.diagram, + fallbacks.image_unavailable + fallbacks.image_over_budget, + fallbacks.unknown_element + ) +} + +/// The `outcome` label of `knot_export_total` for a response status. +fn outcome(status: StatusCode) -> &'static str { + if status == StatusCode::SERVICE_UNAVAILABLE { + "busy" + } else if status.is_server_error() { + "server_error" + } else if status.is_client_error() { + "client_error" + } else { + "ok" + } +} + +fn busy() -> Response { + let mut response = json_err( + StatusCode::SERVICE_UNAVAILABLE, + "export.busy", + "too many exports at once", + ); + response + .headers_mut() + .insert(header::RETRY_AFTER, RETRY_AFTER_SECS); + response +} + +fn internal() -> Response { + json_err(StatusCode::INTERNAL_SERVER_ERROR, "internal", "") +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn fallbacks_header_sums_the_image_kinds() { + let fallbacks = Fallbacks { + diagram: 2, + image_unavailable: 1, + image_over_budget: 3, + unknown_element: 4, + }; + assert_eq!( + fallbacks_header(&fallbacks), + "diagrams=2, images=4, unknown=4" + ); + } + + /// Remembers the labels of every counter registered with it. + #[derive(Default)] + struct Counters(std::sync::Mutex>>); + + impl metrics::Recorder for Counters { + fn describe_counter( + &self, + _: metrics::KeyName, + _: Option, + _: metrics::SharedString, + ) { + } + fn describe_gauge( + &self, + _: metrics::KeyName, + _: Option, + _: metrics::SharedString, + ) { + } + fn describe_histogram( + &self, + _: metrics::KeyName, + _: Option, + _: metrics::SharedString, + ) { + } + fn register_counter( + &self, + key: &metrics::Key, + _: &metrics::Metadata<'_>, + ) -> metrics::Counter { + let labels = key + .labels() + .map(|l| (l.key().to_owned(), l.value().to_owned())) + .collect(); + self.0.lock().unwrap().push(labels); + metrics::Counter::noop() + } + fn register_gauge(&self, _: &metrics::Key, _: &metrics::Metadata<'_>) -> metrics::Gauge { + metrics::Gauge::noop() + } + fn register_histogram( + &self, + _: &metrics::Key, + _: &metrics::Metadata<'_>, + ) -> metrics::Histogram { + metrics::Histogram::noop() + } + } + + #[test] + fn a_request_dropped_before_its_response_still_counts() { + let counters = Counters::default(); + metrics::with_local_recorder(&counters, || { + // The request timeout drops the handler mid-export. + drop(Recorded::start(Some(Format::Docx))); + let mut done = Recorded::start(Some(Format::Odt)); + done.status = Some(StatusCode::OK); + drop(done); + }); + let label = |format: &str, outcome: &str| { + vec![ + ("format".to_owned(), format.to_owned()), + ("outcome".to_owned(), outcome.to_owned()), + ] + }; + assert_eq!( + *counters.0.lock().unwrap(), + [label("docx", "client_error"), label("odt", "ok")] + ); + } + + #[test] + fn outcome_follows_the_status() { + for (status, label) in [ + (StatusCode::OK, "ok"), + (StatusCode::BAD_REQUEST, "client_error"), + (StatusCode::PAYLOAD_TOO_LARGE, "client_error"), + (StatusCode::NOT_FOUND, "client_error"), + (StatusCode::SERVICE_UNAVAILABLE, "busy"), + (StatusCode::INTERNAL_SERVER_ERROR, "server_error"), + ] { + assert_eq!(outcome(status), label, "{status}"); + } + } +} diff --git a/crates/knot-server/src/routes/api/markdown.rs b/crates/knot-server/src/routes/api/markdown.rs index 8fde96d4..bdad13b1 100644 --- a/crates/knot-server/src/routes/api/markdown.rs +++ b/crates/knot-server/src/routes/api/markdown.rs @@ -54,6 +54,73 @@ pub enum RefreshError { Serialise(String), } +/// Why [`live_snapshot`] could not copy a page's live state. +#[derive(Debug, thiserror::Error)] +pub(crate) enum SnapshotError { + #[error("server state missing rooms registry")] + NoRooms, + #[error("room acquire failed for {doc_id}: {source}")] + Acquire { + doc_id: Uuid, + source: knot_crdt::AcquireError, + }, + #[error("room actor unreachable for {0}")] + RoomUnreachable(Uuid), + #[error("room actor: {0}")] + Actor(knot_crdt::EngineError), + #[error(transparent)] + Apply(knot_crdt::EngineError), +} + +/// Keeps `RefreshError`'s variants and messages as they were before the +/// snapshot step was shared (Debug-formatted engine errors, no acquire cause). +impl From for RefreshError { + fn from(e: SnapshotError) -> Self { + match e { + SnapshotError::NoRooms => Self::NoRooms, + SnapshotError::Acquire { doc_id, .. } => Self::Acquire(doc_id), + SnapshotError::RoomUnreachable(doc_id) => Self::RoomUnreachable(doc_id), + SnapshotError::Actor(e) => Self::Actor(format!("{e:?}")), + SnapshotError::Apply(e) => Self::Apply(format!("{e:?}")), + } + } +} + +/// Copy the page's live state into a transient doc, with the room's seq at +/// the time of the copy. The room actor only encodes its state +/// (`Event::ExportState`), so it keeps serving editors while the caller +/// reads the copy. Shared by the markdown refresh and the DOCX/ODT export. +pub(crate) async fn live_snapshot( + state: &AppState, + doc_id: Uuid, +) -> Result<(knot_crdt::DocHandle, i64), SnapshotError> { + let rooms = state.rooms_v2.clone().ok_or(SnapshotError::NoRooms)?; + let room = rooms + .acquire(doc_id) + .await + .map_err(|source| SnapshotError::Acquire { doc_id, source })?; + let (tx, rx) = tokio::sync::oneshot::channel(); + if room + .tx + .send(knot_crdt::Event::ExportState(tx)) + .await + .is_err() + { + return Err(SnapshotError::RoomUnreachable(doc_id)); + } + let (state_bytes, seq) = match rx.await { + Ok(Ok(v)) => v, + Ok(Err(e)) => return Err(SnapshotError::Actor(e)), + Err(_) => return Err(SnapshotError::RoomUnreachable(doc_id)), + }; + let engine = YrsEngine; + let transient = engine.new_doc(); + engine + .apply_update(&transient, &state_bytes) + .map_err(SnapshotError::Apply)?; + Ok((transient, seq)) +} + /// Re-render markdown from the live doc state, write to the cache, and /// re-run the task indexer. Used after any mutation that should be /// reflected on `/tasks` (markdown export, full-doc import via @@ -82,31 +149,8 @@ async fn refresh_markdown_inner( doc_id: Uuid, reindex_tasks: bool, ) -> Result { - let rooms = state.rooms_v2.clone().ok_or(RefreshError::NoRooms)?; - let room = rooms - .acquire(doc_id) - .await - .map_err(|_| RefreshError::Acquire(doc_id))?; - let (tx, rx) = tokio::sync::oneshot::channel(); - if room - .tx - .send(knot_crdt::Event::ExportState(tx)) - .await - .is_err() - { - return Err(RefreshError::RoomUnreachable(doc_id)); - } - let (state_bytes, seq) = match rx.await { - Ok(Ok(v)) => v, - Ok(Err(e)) => return Err(RefreshError::Actor(format!("{e:?}"))), - Err(_) => return Err(RefreshError::RoomUnreachable(doc_id)), - }; - let engine = YrsEngine; - let transient = engine.new_doc(); - if let Err(e) = engine.apply_update(&transient, &state_bytes) { - return Err(RefreshError::Apply(format!("{e:?}"))); - } - let text = match knot_markdown::to_markdown::serialise(&engine, &transient) { + let (transient, seq) = live_snapshot(state, doc_id).await?; + let text = match knot_markdown::to_markdown::serialise(&YrsEngine, &transient) { Ok(md) => md, Err(e) => return Err(RefreshError::Serialise(format!("{e:?}"))), }; diff --git a/crates/knot-server/src/routes/api/mod.rs b/crates/knot-server/src/routes/api/mod.rs index 725f4a69..acb836b6 100644 --- a/crates/knot-server/src/routes/api/mod.rs +++ b/crates/knot-server/src/routes/api/mod.rs @@ -10,6 +10,7 @@ pub mod boards; pub mod comments; pub mod contributors; pub mod docs; +pub mod export; pub mod export_import; pub mod grants; pub mod history; diff --git a/crates/knot-server/tests/export_check_integration.rs b/crates/knot-server/tests/export_check_integration.rs new file mode 100644 index 00000000..8c0bad4a --- /dev/null +++ b/crates/knot-server/tests/export_check_integration.rs @@ -0,0 +1,93 @@ +//! `export-check` over a seeded database: counts pages, failures and fallbacks +//! without touching the schema. + +use knot_server::export_check::check_all; +use knot_storage::{ + DocStore, PgDocStore, PgSnapshotStore, PgUserStore, PgWorkspaceStore, SnapshotStore, UserStore, + WorkspaceStore, +}; +use yrs::updates::decoder::Decode; +use yrs::updates::encoder::Encode; +use yrs::{ + Doc, ReadTxn, StateVector, Transact, Update, Xml, XmlElementPrelim, XmlFragment, XmlTextPrelim, +}; + +/// Full-state bytes and state vector for a doc built by `fill`. The doc is +/// rebuilt from a hand-built update, as a stored page would be. +fn page_state(fill: impl FnOnce(&Doc)) -> (Vec, Vec) { + let source = Doc::new(); + fill(&source); + let update = source + .transact() + .encode_state_as_update_v1(&StateVector::default()); + let doc = Doc::new(); + doc.transact_mut() + .apply_update(Update::decode_v1(&update).unwrap()) + .unwrap(); + let txn = doc.transact(); + ( + txn.encode_state_as_update_v1(&StateVector::default()), + txn.state_vector().encode_v1(), + ) +} + +async fn migration_count(pool: &sqlx::PgPool) -> i64 { + sqlx::query_scalar("SELECT count(*) FROM _sqlx_migrations") + .fetch_one(pool) + .await + .unwrap() +} + +#[tokio::test(flavor = "multi_thread")] +async fn check_all_reports_pages_failures_and_fallbacks() { + let db = knot_test_support::fresh_db().await; + let pool = db.pool.clone(); + let ws = PgWorkspaceStore::new(pool.clone()) + .create("default", "Workspace") + .await + .unwrap(); + let hash = knot_auth::Hasher::fast_for_tests() + .hash("hunter22") + .unwrap(); + let user = PgUserStore::new(pool.clone()) + .create_local("alice@example.com", "Alice", &hash) + .await + .unwrap(); + let docs = PgDocStore::new(pool.clone()); + let a = docs.create(ws.id, None, "A", "m", user.id).await.unwrap(); + let b = docs.create(ws.id, None, "B", "n", user.id).await.unwrap(); + let snaps = PgSnapshotStore::new(pool.clone()); + + // Page A: a mermaid block and an element the walker does not know. + let (bytes, sv) = page_state(|doc| { + let frag = doc.get_or_insert_xml_fragment("default"); + let mut txn = doc.transact_mut(); + let code = frag.push_back(&mut txn, XmlElementPrelim::empty("code_block")); + code.insert_attribute(&mut txn, "language", "mermaid"); + code.push_back(&mut txn, XmlTextPrelim::new("graph TD; A-->B")); + let callout = frag.push_back(&mut txn, XmlElementPrelim::empty("callout")); + let p = callout.push_back(&mut txn, XmlElementPrelim::empty("paragraph")); + p.push_back(&mut txn, XmlTextPrelim::new("careful")); + }); + snaps.insert(a.id, 1, &bytes, &sv).await.unwrap(); + + // Page B: plain text. + let (bytes, sv) = page_state(|doc| { + let frag = doc.get_or_insert_xml_fragment("default"); + let mut txn = doc.transact_mut(); + let p = frag.push_back(&mut txn, XmlElementPrelim::empty("paragraph")); + p.push_back(&mut txn, XmlTextPrelim::new("hello")); + }); + snaps.insert(b.id, 1, &bytes, &sv).await.unwrap(); + + let migrations = migration_count(&pool).await; + + let report = check_all(&pool, "http://localhost").await.unwrap(); + + assert_eq!(report.pages, 2); + assert!(report.failed.is_empty(), "{:?}", report.failed); + assert_eq!(report.fallbacks.diagram, 2, "one per format"); + assert_eq!(report.fallbacks.unknown_element, 2, "one per format"); + assert_eq!(report.fallbacks.image_unavailable, 0); + assert_eq!(migration_count(&pool).await, migrations); +} diff --git a/crates/knot-server/tests/export_integration.rs b/crates/knot-server/tests/export_integration.rs new file mode 100644 index 00000000..41299548 --- /dev/null +++ b/crates/knot-server/tests/export_integration.rs @@ -0,0 +1,1167 @@ +//! Page export to DOCX/ODT: the export service (snapshot, asset resolution, +//! build) and its routes, against a fresh database per test, with a real +//! rooms registry. + +use std::collections::HashMap; +use std::io::Read; +use std::sync::Arc; +use std::time::Duration; + +use axum::Router; +use axum::body::Body; +use axum::http::{Request, StatusCode}; +use axum::response::Response; +use http_body_util::BodyExt; +use knot_auth::{Hasher, Throttle}; +use knot_crdt::{Rooms, SnapshotPolicy, YrsEngine}; +use knot_export::{ + Block, ExportDoc, Fallbacks, ImageAsset, ImageSource, Inline, LinkTarget, Marks, Run, +}; +use knot_server::export::{self, Format}; +use knot_server::{AppState, router_with_state}; +use knot_storage::{ + BlobStore, BlobStoreError, PgSnapshotStore, PgUpdatesStore, SnapshotStore, UpdatesStore, + WorkspaceRole, +}; +use serde_json::{Value, json}; +use sha2::{Digest, Sha256}; +use tokio::sync::{Notify, Semaphore}; +use tower::ServiceExt; +use uuid::Uuid; +use yrs::{Any, ReadTxn, Transact, Xml, XmlElementPrelim, XmlFragment, XmlTextPrelim}; + +// --------------------------------------------------------------------------- +// Harness +// --------------------------------------------------------------------------- + +/// A logged-in user: cookie header, CSRF token, user id. +#[derive(Clone)] +struct Session { + cookie: String, + csrf: String, + user_id: Uuid, +} + +struct World { + state: AppState, + app: Router, + owner: Session, + workspace_id: Uuid, +} + +const PASSWORD: &str = "hunter22hunter22"; + +/// Minimal valid 1×1 PNG. +const TINY_PNG: &[u8] = &[ + 0x89, 0x50, 0x4E, 0x47, 0x0D, 0x0A, 0x1A, 0x0A, 0x00, 0x00, 0x00, 0x0D, 0x49, 0x48, 0x44, 0x52, + 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, 0x01, 0x08, 0x06, 0x00, 0x00, 0x00, 0x1F, 0x15, 0xC4, + 0x89, 0x00, 0x00, 0x00, 0x0D, 0x49, 0x44, 0x41, 0x54, 0x78, 0xDA, 0x63, 0x60, 0x60, 0x00, 0x00, + 0x00, 0x00, 0x04, 0x00, 0x01, 0x5C, 0x5B, 0x66, 0xE3, 0x00, 0x00, 0x00, 0x00, 0x49, 0x45, 0x4E, + 0x44, 0xAE, 0x42, 0x60, 0x82, +]; + +/// Valid 3×2 RGB PNG; the distinct sides show width and height are read. +const PNG_3X2: &[u8] = &[ + 0x89, 0x50, 0x4E, 0x47, 0x0D, 0x0A, 0x1A, 0x0A, 0x00, 0x00, 0x00, 0x0D, 0x49, 0x48, 0x44, 0x52, + 0x00, 0x00, 0x00, 0x03, 0x00, 0x00, 0x00, 0x02, 0x08, 0x02, 0x00, 0x00, 0x00, 0x12, 0x16, 0xF1, + 0x4D, 0x00, 0x00, 0x00, 0x10, 0x49, 0x44, 0x41, 0x54, 0x78, 0xDA, 0x63, 0x38, 0x21, 0x27, 0x07, + 0x41, 0x0C, 0x70, 0x16, 0x00, 0x41, 0x00, 0x06, 0x19, 0xD4, 0x44, 0xEE, 0x19, 0x00, 0x00, 0x00, + 0x00, 0x49, 0x45, 0x4E, 0x44, 0xAE, 0x42, 0x60, 0x82, +]; + +const WEBP_2X2: &[u8] = include_bytes!("../../knot-export/tests/fixtures/2x2.webp"); + +async fn world() -> World { + world_with(|_| {}).await +} + +/// A workspace with one owner ("Olive Owner"), a real rooms registry, and +/// `configure` applied to the state before the router is built. +async fn world_with(configure: impl FnOnce(&mut AppState)) -> World { + let db = knot_test_support::fresh_db().await; + let mut state = AppState::with_pool(db.pool.clone()); + state.hasher = Arc::new(Hasher::fast_for_tests()); + state.throttle = Arc::new(Throttle::new()); + state.session_key = b"test-key-32-bytes-aaaaaaaaaaaaaa".to_vec(); + let workspace = state + .workspaces + .as_ref() + .unwrap() + .create("default", "W") + .await + .unwrap(); + let hash = state.hasher.hash(PASSWORD).unwrap(); + let owner = state + .users + .as_ref() + .unwrap() + .create_local("owner@export.test", "Olive Owner", &hash) + .await + .unwrap(); + state + .workspaces + .as_ref() + .unwrap() + .add_member(workspace.id, owner.id, WorkspaceRole::Owner) + .await + .unwrap(); + wire_rooms(&mut state, &db.url).await; + configure(&mut state); + let app = router_with_state(state.clone()); + let owner = login(&app, "owner@export.test", owner.id).await; + World { + state, + app, + owner, + workspace_id: workspace.id, + } +} + +/// A real rooms registry on the test database (copied from +/// `markdown_import_integration.rs`): export snapshots the live room. +async fn wire_rooms(state: &mut AppState, db_url: &str) { + let pool = state.pool.as_ref().unwrap().clone(); + let bus = Arc::new(knot_crdt::PgBus::connect(db_url).await.unwrap()); + let updates: Arc = Arc::new(PgUpdatesStore::new(pool.clone())); + let snaps: Arc = Arc::new(PgSnapshotStore::new(pool.clone())); + let policy = SnapshotPolicy { + every_n: 1000, + idle: Duration::from_secs(3600), + }; + let rooms = Arc::new(Rooms::new( + Arc::new(YrsEngine), + bus.clone(), + updates, + snaps, + policy, + Duration::from_secs(3600), + )); + state.bus = Some(bus); + state.rooms_v2 = Some(rooms); +} + +async fn login(app: &Router, email: &str, user_id: Uuid) -> Session { + let response = send_raw( + app, + None, + "POST", + "/auth/login", + Some("application/json"), + json!({"email": email, "password": PASSWORD}) + .to_string() + .into_bytes(), + ) + .await; + assert_eq!(response.status(), StatusCode::NO_CONTENT, "login {email}"); + let cookies: Vec = response + .headers() + .get_all("set-cookie") + .iter() + .map(|v| v.to_str().unwrap().split(';').next().unwrap().to_string()) + .collect(); + let sid = cookies.iter().find(|c| c.starts_with("sid=")).unwrap(); + let csrf = cookies.iter().find(|c| c.starts_with("csrf=")).unwrap(); + Session { + cookie: format!("{sid}; {csrf}"), + csrf: csrf.trim_start_matches("csrf=").to_string(), + user_id, + } +} + +/// One request through `app`, returning the whole response. `None` sends +/// it anonymously; the CSRF header goes only with a session. +async fn send_raw( + app: &Router, + who: Option<&Session>, + method: &str, + uri: &str, + content_type: Option<&str>, + body: Vec, +) -> Response { + let mut request = Request::builder().method(method).uri(uri); + if let Some(who) = who { + request = request + .header("cookie", &who.cookie) + .header("x-csrf-token", &who.csrf); + } + if let Some(content_type) = content_type { + request = request.header("content-type", content_type); + } + app.clone() + .oneshot(request.body(Body::from(body)).unwrap()) + .await + .unwrap() +} + +async fn body_bytes(response: Response) -> Vec { + response + .into_body() + .collect() + .await + .unwrap() + .to_bytes() + .to_vec() +} + +async fn body_json(response: Response) -> Value { + serde_json::from_slice(&body_bytes(response).await).unwrap() +} + +impl World { + async fn create_doc(&self, title: &str) -> Uuid { + let response = send_raw( + &self.app, + Some(&self.owner), + "POST", + "/api/docs", + Some("application/json"), + json!({"title": title, "parent_id": null}) + .to_string() + .into_bytes(), + ) + .await; + assert_eq!(response.status(), StatusCode::CREATED, "create {title}"); + let body = body_json(response).await; + body["id"].as_str().unwrap().parse().unwrap() + } + + /// Replace the page's content through the room, as the import does. + async fn set_markdown(&self, doc_id: Uuid, markdown: &str) { + let response = send_raw( + &self.app, + Some(&self.owner), + "POST", + &format!("/api/docs/{doc_id}/markdown?mode=replace"), + Some("text/markdown"), + markdown.as_bytes().to_vec(), + ) + .await; + assert_eq!(response.status(), StatusCode::NO_CONTENT, "markdown import"); + } + + /// Replace the page's content with `content`'s state through the room, + /// for shapes the markdown import cannot produce (unknown elements). + async fn set_content(&self, doc_id: Uuid, content: &yrs::Doc) { + let update_bytes = content + .transact() + .encode_state_as_update_v1(&yrs::StateVector::default()); + let room = self + .state + .rooms_v2 + .as_ref() + .unwrap() + .acquire(doc_id) + .await + .unwrap(); + let (reply, done) = tokio::sync::oneshot::channel(); + room.tx + .send(knot_crdt::Event::ReplaceWithMarkdown { + update_bytes, + by_user: Some(self.owner.user_id), + reply, + }) + .await + .unwrap(); + done.await.unwrap().expect("replace page content"); + } + + /// Upload `bytes` as a blob of `doc_id`; returns its `/api/blobs/` url. + async fn upload(&self, doc_id: Uuid, content_type: &str, bytes: &[u8]) -> String { + let (multipart_type, body) = multipart(&[("file", bytes)], content_type); + let response = send_raw( + &self.app, + Some(&self.owner), + "POST", + &format!("/api/docs/{doc_id}/blobs"), + Some(&multipart_type), + body, + ) + .await; + assert_eq!(response.status(), StatusCode::CREATED, "upload"); + body_json(response).await["url"] + .as_str() + .unwrap() + .to_string() + } + + /// Store a blob directly, bypassing the upload route's ACL. + async fn insert_blob(&self, workspace_id: Uuid, doc_id: Uuid, bytes: &[u8]) -> Uuid { + let id = Uuid::new_v4(); + self.state + .blob_meta + .as_ref() + .unwrap() + .insert(&knot_storage::BlobMetadata { + id, + workspace_id, + doc_id, + content_type: "image/png".into(), + byte_size: bytes.len() as i64, + sha256: vec![0; 32], + original_name: None, + created_by: self.owner.user_id, + created_at: chrono::Utc::now(), + }) + .await + .unwrap(); + self.state + .blob_store + .as_ref() + .unwrap() + .put(id, bytes, "image/png") + .await + .unwrap(); + id + } + + /// A page in a workspace the owner is not a member of. + async fn foreign_doc(&self) -> (Uuid, Uuid) { + let state = &self.state; + let other_ws = state + .workspaces + .as_ref() + .unwrap() + .create("other", "Other") + .await + .unwrap(); + let outsider = state + .users + .as_ref() + .unwrap() + .create_local("outsider@export.test", "Outsider", "$h$") + .await + .unwrap(); + let doc = state + .docs + .as_ref() + .unwrap() + .create(other_ws.id, None, "Secret", "m", outsider.id) + .await + .unwrap(); + (other_ws.id, doc.id) + } + + /// Invite a member with `role` ("viewer" or "editor") and log them in. + async fn member(&self, email: &str, role: &str) -> Session { + let response = send_raw( + &self.app, + Some(&self.owner), + "POST", + "/api/workspace/members", + Some("application/json"), + json!({"email": email, "role": role, "password": PASSWORD, "display_name": role}) + .to_string() + .into_bytes(), + ) + .await; + assert_eq!(response.status(), StatusCode::CREATED, "invite {email}"); + let user = self + .state + .users + .as_ref() + .unwrap() + .find_by_email(email) + .await + .unwrap() + .unwrap(); + login(&self.app, email, user.id).await + } + + /// Move `doc_id` to the archive as the owner. + async fn archive(&self, doc_id: Uuid) { + let response = send_raw( + &self.app, + Some(&self.owner), + "DELETE", + &format!("/api/docs/{doc_id}"), + None, + Vec::new(), + ) + .await; + assert_eq!(response.status(), StatusCode::NO_CONTENT, "archive"); + } + + /// `GET /api/docs/{doc_id}/export/diagrams` as `who`. + async fn diagrams(&self, who: &Session, doc_id: Uuid) -> Response { + send_raw( + &self.app, + Some(who), + "GET", + &format!("/api/docs/{doc_id}/export/diagrams"), + None, + Vec::new(), + ) + .await + } + + /// The diagram keys the listing gives for `doc_id`, in its order. + async fn diagram_keys(&self, doc_id: Uuid) -> Vec { + let response = self.diagrams(&self.owner, doc_id).await; + assert_eq!(response.status(), StatusCode::OK, "diagrams listing"); + body_json(response).await["diagrams"] + .as_array() + .unwrap() + .iter() + .map(|d| d["key"].as_str().unwrap().to_string()) + .collect() + } + + /// `POST /api/docs/{doc_id}/export/{format}` as `who`, with one PNG part + /// per `(name, bytes)`, or an empty body without a content type when + /// there are none. + async fn export( + &self, + who: &Session, + doc_id: Uuid, + format: &str, + pictures: &[(&str, &[u8])], + ) -> Response { + let (content_type, body) = if pictures.is_empty() { + (None, Vec::new()) + } else { + let (content_type, body) = multipart(pictures, "image/png"); + (Some(content_type), body) + }; + send_raw( + &self.app, + Some(who), + "POST", + &format!("/api/docs/{doc_id}/export/{format}"), + content_type.as_deref(), + body, + ) + .await + } +} + +/// A blob store whose reads wait for the gate: each `get` tells `entered` +/// it started, then waits until `open` has a permit. Holds an export in the +/// middle of its build for as long as a test needs. +struct GatedBlobs { + inner: Arc, + entered: Arc, + open: Arc, +} + +#[async_trait::async_trait] +impl BlobStore for GatedBlobs { + async fn put(&self, id: Uuid, bytes: &[u8], content_type: &str) -> Result<(), BlobStoreError> { + self.inner.put(id, bytes, content_type).await + } + + async fn get(&self, id: Uuid) -> Result, BlobStoreError> { + self.entered.notify_one(); + drop(self.open.acquire().await.expect("gate")); + self.inner.get(id).await + } + + async fn delete(&self, id: Uuid) -> Result<(), BlobStoreError> { + self.inner.delete(id).await + } + + fn bytes_deleted_with_metadata(&self) -> bool { + self.inner.bytes_deleted_with_metadata() + } +} + +/// A `multipart/form-data` body with one `content_type` part per +/// `(name, bytes)`; returns the request's content type and the body. +fn multipart(parts: &[(&str, &[u8])], content_type: &str) -> (String, Vec) { + const BOUNDARY: &str = "----ExportTestBoundary"; + let mut body = Vec::new(); + for (name, bytes) in parts { + body.extend_from_slice(format!("--{BOUNDARY}\r\n").as_bytes()); + body.extend_from_slice( + format!("Content-Disposition: form-data; name=\"{name}\"; filename=\"picture\"\r\n") + .as_bytes(), + ); + body.extend_from_slice(format!("Content-Type: {content_type}\r\n\r\n").as_bytes()); + body.extend_from_slice(bytes); + body.extend_from_slice(b"\r\n"); + } + body.extend_from_slice(format!("--{BOUNDARY}--\r\n").as_bytes()); + (format!("multipart/form-data; boundary={BOUNDARY}"), body) +} + +/// A PNG of exactly `len` bytes: `TINY_PNG` padded after its end chunk, so +/// it still reads as a 1×1 picture. +fn padded_png(len: usize) -> Vec { + let mut png = TINY_PNG.to_vec(); + png.resize(len, 0); + png +} + +fn header<'a>(response: &'a Response, name: &str) -> &'a str { + response + .headers() + .get(name) + .unwrap_or_else(|| panic!("{name} missing")) + .to_str() + .unwrap() +} + +fn fallbacks_header(response: &Response) -> &str { + header(response, "x-knot-export-fallbacks") +} + +async fn error_code(response: Response) -> String { + body_json(response).await["error"]["code"] + .as_str() + .unwrap_or_default() + .to_string() +} + +/// One zip entry's bytes, or `None` when absent. +fn zip_entry(zip: &[u8], name: &str) -> Option> { + let mut archive = zip::ZipArchive::new(std::io::Cursor::new(zip)).unwrap(); + let mut file = archive.by_name(name).ok()?; + let mut out = Vec::new(); + file.read_to_end(&mut out).unwrap(); + Some(out) +} + +/// The names of all zip entries starting with `prefix`. +fn zip_names(zip: &[u8], prefix: &str) -> Vec { + let archive = zip::ZipArchive::new(std::io::Cursor::new(zip)).unwrap(); + archive + .file_names() + .filter(|name| name.starts_with(prefix)) + .map(str::to_string) + .collect() +} + +fn zip_text(zip: &[u8], name: &str) -> String { + String::from_utf8(zip_entry(zip, name).unwrap_or_else(|| panic!("{name} missing"))).unwrap() +} + +fn image(id: Uuid) -> Block { + Block::Image { + source: ImageSource::Blob(id.to_string()), + alt: String::new(), + } +} + +// --------------------------------------------------------------------------- +// Tests +// --------------------------------------------------------------------------- + +#[tokio::test(flavor = "multi_thread")] +async fn resolve_assets_checks_blob_access_and_budget() { + let world = world().await; + let page = world.create_doc("Page").await; + let (other_ws, other_doc) = world.foreign_doc().await; + let foreign = world.insert_blob(other_ws, other_doc, PNG_3X2).await; + let readable = world.insert_blob(world.workspace_id, page, PNG_3X2).await; + let missing = Uuid::new_v4(); + let doc = ExportDoc { + blocks: vec![image(foreign), image(readable), image(missing)], + }; + let user = world.owner.user_id; + + let assets = export::resolve_assets(&world.state, user, &doc, HashMap::new()) + .await + .unwrap(); + assert!(matches!( + assets.images[&foreign.to_string()], + ImageAsset::Unavailable + )); + assert!(matches!( + assets.images[&missing.to_string()], + ImageAsset::Unavailable + )); + match &assets.images[&readable.to_string()] { + ImageAsset::Embedded(picture) => { + assert_eq!((picture.width_px, picture.height_px), (3, 2)); + assert_eq!(picture.bytes, PNG_3X2); + } + other => panic!("readable PNG: {other:?}"), + } + + let budget = PNG_3X2.len() as i64 - 1; + let assets = + export::resolve_assets_with_budget(&world.state, user, &doc, HashMap::new(), budget) + .await + .unwrap(); + assert!(matches!( + assets.images[&readable.to_string()], + ImageAsset::OverBudget + )); +} + +/// The budget is a running total over the page: once earlier images have +/// spent it, an image that alone would fit is still refused. A smaller one +/// after it that fits the remainder is embedded, so the budget skips images +/// rather than stopping at the first refusal. +#[tokio::test(flavor = "multi_thread")] +async fn image_budget_is_a_running_total() { + let world = world().await; + let page = world.create_doc("Pictures").await; + let first = world.insert_blob(world.workspace_id, page, PNG_3X2).await; + let second = world.insert_blob(world.workspace_id, page, PNG_3X2).await; + let smaller = world.insert_blob(world.workspace_id, page, TINY_PNG).await; + let doc = ExportDoc { + blocks: vec![image(first), image(second), image(smaller)], + }; + // Room for one PNG_3X2 plus TINY_PNG, but not for two PNG_3X2. + let budget = 2 * PNG_3X2.len() as i64 - 1; + assert!((PNG_3X2.len() + TINY_PNG.len()) as i64 <= budget); + + let user = world.owner.user_id; + let assets = + export::resolve_assets_with_budget(&world.state, user, &doc, HashMap::new(), budget) + .await + .unwrap(); + assert!( + matches!(assets.images[&first.to_string()], ImageAsset::Embedded(_)), + "first: {:?}", + assets.images[&first.to_string()] + ); + assert!( + matches!(assets.images[&second.to_string()], ImageAsset::OverBudget), + "second: {:?}", + assets.images[&second.to_string()] + ); + assert!( + matches!(assets.images[&smaller.to_string()], ImageAsset::Embedded(_)), + "smaller: {:?}", + assets.images[&smaller.to_string()] + ); +} + +#[tokio::test(flavor = "multi_thread")] +async fn build_counts_each_fallback_kind() { + let world = world().await; + let page = world.create_doc("Fallbacks").await; + let owner = world.owner.user_id; + let (other_ws, other_doc) = world.foreign_doc().await; + let foreign = world.insert_blob(other_ws, other_doc, PNG_3X2).await; + + // An unreadable image, a board nobody supplied a picture for, and an + // element the walker does not know. + let content = yrs::Doc::new(); + let fragment = content.get_or_insert_xml_fragment("default"); + { + let mut txn = content.transact_mut(); + let picture = fragment.push_back(&mut txn, XmlElementPrelim::empty("image")); + picture.insert_attribute(&mut txn, "src", Any::from(format!("/api/blobs/{foreign}"))); + let board = fragment.push_back(&mut txn, XmlElementPrelim::empty("excalidraw_board")); + board.insert_attribute(&mut txn, "board_id", Any::from(Uuid::new_v4().to_string())); + let callout = fragment.push_back(&mut txn, XmlElementPrelim::empty("callout")); + callout.push_back(&mut txn, XmlTextPrelim::new("Heads up")); + } + world.set_content(page, &content).await; + + let snapshot = export::snapshot(&world.state, page).await.unwrap(); + let walked = knot_export::walk(snapshot.inner()); + assert_eq!(walked.unknown_elements, ["callout"]); + + for format in [Format::Docx, Format::Odt] { + let built = export::build(&world.state, owner, page, format, HashMap::new()) + .await + .unwrap(); + let fallbacks = built.fallbacks; + assert_eq!(fallbacks.image_unavailable, 1, "{format:?}: {fallbacks:?}"); + assert!(fallbacks.diagram >= 1, "{format:?}: {fallbacks:?}"); + assert_eq!(fallbacks.image_over_budget, 0, "{format:?}: {fallbacks:?}"); + assert_eq!( + fallbacks.unknown_element, + walked.unknown_elements.len() as u32, + "{format:?}: {fallbacks:?}" + ); + } +} + +#[tokio::test(flavor = "multi_thread")] +async fn resolve_assets_keeps_page_diagrams_and_names_mentions() { + let world = world().await; + let owner = world.owner.user_id; + let gone = Uuid::new_v4(); + let mention = |id: Uuid| Inline::Link { + target: LinkTarget::User(id.to_string()), + runs: vec![Run { + text: "@Old".into(), + marks: Marks::default(), + }], + }; + let doc = ExportDoc { + blocks: vec![ + Block::Mermaid { + key: "mermaid-abc".into(), + source: "graph TD; A-->B".into(), + }, + Block::Mermaid { + key: "mermaid-not-png".into(), + source: "graph TD; B-->C".into(), + }, + Block::Paragraph(vec![mention(owner), mention(gone)]), + ], + }; + let pictures = HashMap::from([ + ("mermaid-abc".to_string(), TINY_PNG.to_vec()), + ("mermaid-not-png".to_string(), b"GIF89a".to_vec()), + ("mermaid-not-on-page".to_string(), TINY_PNG.to_vec()), + ]); + + let assets = export::resolve_assets(&world.state, owner, &doc, pictures) + .await + .unwrap(); + let keys: Vec<&String> = assets.diagrams.keys().collect(); + assert_eq!(keys, ["mermaid-abc"]); + assert_eq!(assets.diagrams["mermaid-abc"].scale, 2); + assert_eq!( + assets.user_names, + HashMap::from([(owner.to_string(), "Olive Owner".to_string())]) + ); +} + +#[tokio::test(flavor = "multi_thread")] +async fn converted_webp_is_charged_at_its_prepared_size() { + let world = world().await; + let page = world.create_doc("Pictures").await; + let url = world.upload(page, "image/webp", WEBP_2X2).await; + let id = url.trim_start_matches("/api/blobs/").to_string(); + let doc = ExportDoc { + blocks: vec![Block::Image { + source: ImageSource::Blob(id.clone()), + alt: String::new(), + }], + }; + let prepared = knot_export::prepare_image(WEBP_2X2).unwrap(); + assert!( + prepared.bytes.len() > WEBP_2X2.len(), + "the PNG outgrows the WebP" + ); + + // The stored size fits; the converted PNG does not. + let budget = WEBP_2X2.len() as i64; + let user = world.owner.user_id; + let assets = + export::resolve_assets_with_budget(&world.state, user, &doc, HashMap::new(), budget) + .await + .unwrap(); + assert!(matches!(assets.images[&id], ImageAsset::OverBudget)); + + let budget = prepared.bytes.len() as i64; + let assets = + export::resolve_assets_with_budget(&world.state, user, &doc, HashMap::new(), budget) + .await + .unwrap(); + assert!(matches!(assets.images[&id], ImageAsset::Embedded(_))); +} + +#[tokio::test(flavor = "multi_thread")] +async fn webp_blob_is_converted_to_png() { + let world = world().await; + let page = world.create_doc("Pictures").await; + let url = world.upload(page, "image/webp", WEBP_2X2).await; + world.set_markdown(page, &format!("![w]({url})\n")).await; + + let built = export::build( + &world.state, + world.owner.user_id, + page, + Format::Docx, + HashMap::new(), + ) + .await + .unwrap(); + assert_eq!(built.fallbacks, Fallbacks::default()); + let png = zip_entry(&built.bytes, "word/media/image1.png").expect("word/media/image1.png"); + assert!(png.starts_with(b"\x89PNG\r\n\x1a\n")); +} + +#[tokio::test(flavor = "multi_thread")] +async fn page_links_use_base_url() { + let world = world_with(|s| s.base_url = "https://knot.example.test".into()).await; + let page = world.create_doc("Links").await; + let other = world.create_doc("Other").await; + world + .set_markdown(page, &format!("See [x](knot://doc/{other}).\n")) + .await; + + let built = export::build( + &world.state, + world.owner.user_id, + page, + Format::Docx, + HashMap::new(), + ) + .await + .unwrap(); + let rels = zip_text(&built.bytes, "word/_rels/document.xml.rels"); + let target = format!(r#"Target="{}/doc/{other}""#, world.state.base_url); + assert!(rels.contains(&target), "{target} in {rels}"); +} + +#[tokio::test(flavor = "multi_thread")] +async fn build_uses_live_state_and_current_display_names() { + let world = world().await; + let page = world.create_doc("Live page").await; + let owner = world.owner.user_id; + world + .set_markdown( + page, + &format!("# Title\n\nHi [@Old](knot://user/{owner})\n"), + ) + .await; + + for format in [Format::Docx, Format::Odt] { + let built = export::build(&world.state, owner, page, format, HashMap::new()) + .await + .unwrap(); + assert!(built.bytes.starts_with(b"PK"), "{format:?}"); + assert_eq!(built.title, "Live page"); + let body = match format { + Format::Docx => zip_text(&built.bytes, "word/document.xml"), + Format::Odt => zip_text(&built.bytes, "content.xml"), + }; + assert!(body.contains("Hi "), "{format:?}: {body}"); + assert!(body.contains("@Olive Owner"), "{format:?}: {body}"); + assert!(!body.contains("@Old"), "{format:?}: {body}"); + } +} + +// --------------------------------------------------------------------------- +// Routes +// --------------------------------------------------------------------------- + +const DOCX_TYPE: &str = "application/vnd.openxmlformats-officedocument.wordprocessingml.document"; +const ODT_TYPE: &str = "application/vnd.oasis.opendocument.text"; +const NO_FALLBACKS: &str = "diagrams=0, images=0, unknown=0"; + +#[tokio::test(flavor = "multi_thread")] +async fn viewer_can_export_docx_and_odt() { + let world = world().await; + let viewer = world.member("viewer@export.test", "viewer").await; + let page = world.create_doc("Plan").await; + world.set_markdown(page, "# Plan\n\nHello.\n").await; + + for (format, content_type) in [("docx", DOCX_TYPE), ("odt", ODT_TYPE)] { + let response = world.export(&viewer, page, format, &[]).await; + assert_eq!(response.status(), StatusCode::OK, "{format}"); + assert_eq!(header(&response, "content-type"), content_type); + let disposition = header(&response, "content-disposition"); + assert!(disposition.starts_with("attachment; "), "{disposition}"); + assert!( + disposition.contains(&format!("filename*=UTF-8''Plan.{format}")), + "{disposition}" + ); + assert_eq!(fallbacks_header(&response), NO_FALLBACKS); + assert!(body_bytes(response).await.starts_with(b"PK"), "{format}"); + } +} + +#[tokio::test(flavor = "multi_thread")] +async fn template_pages_export() { + let world = world().await; + let page = world.create_doc("Meeting notes").await; + world + .state + .docs + .as_ref() + .unwrap() + .set_template(world.workspace_id, page, world.owner.user_id, true) + .await + .unwrap(); + + let response = world.export(&world.owner, page, "odt", &[]).await; + assert_eq!(response.status(), StatusCode::OK); + assert!(body_bytes(response).await.starts_with(b"PK")); +} + +#[tokio::test(flavor = "multi_thread")] +async fn archived_pages_are_refused() { + let world = world().await; + let viewer = world.member("viewer@export.test", "viewer").await; + let editor = world.member("editor@export.test", "editor").await; + let page = world.create_doc("Old plan").await; + world.archive(page).await; + + for response in [ + world.diagrams(&viewer, page).await, + world.export(&viewer, page, "docx", &[]).await, + ] { + assert_eq!(response.status(), StatusCode::NOT_FOUND); + assert_eq!(error_code(response).await, "doc.not_found"); + } + for response in [ + world.diagrams(&editor, page).await, + world.export(&editor, page, "docx", &[]).await, + ] { + assert_eq!(response.status(), StatusCode::CONFLICT); + assert_eq!(error_code(response).await, "doc.archived"); + } +} + +#[tokio::test(flavor = "multi_thread")] +async fn request_errors() { + let world = world().await; + let page = world.create_doc("Page").await; + let uri = format!("/api/docs/{page}/export/docx"); + + // The session cookie alone, without the CSRF header. + let response = world + .app + .clone() + .oneshot( + Request::builder() + .method("POST") + .uri(&uri) + .header("cookie", &world.owner.cookie) + .body(Body::empty()) + .unwrap(), + ) + .await + .unwrap(); + assert_eq!(response.status(), StatusCode::FORBIDDEN); + assert_eq!(error_code(response).await, "auth.csrf"); + + let response = world.export(&world.owner, page, "pdf", &[]).await; + assert_eq!(response.status(), StatusCode::BAD_REQUEST); + assert_eq!(error_code(response).await, "export.bad_format"); + + let response = send_raw(&world.app, None, "POST", &uri, None, Vec::new()).await; + assert_eq!(response.status(), StatusCode::UNAUTHORIZED); + assert_eq!(error_code(response).await, "auth.session_required"); + + let (_, foreign) = world.foreign_doc().await; + for response in [ + world.diagrams(&world.owner, foreign).await, + world.export(&world.owner, foreign, "docx", &[]).await, + ] { + assert_eq!(response.status(), StatusCode::FORBIDDEN); + assert_eq!(error_code(response).await, "acl.no_grant"); + } +} + +#[tokio::test(flavor = "multi_thread")] +async fn body_over_limit_is_413_but_big_part_falls_back() { + let world = world().await; + let page = world.create_doc("Diagrams").await; + world + .set_markdown( + page, + "```mermaid\ngraph TD; A-->B\n```\n\n```mermaid\ngraph TD; B-->C\n```\n", + ) + .await; + let keys = world.diagram_keys(page).await; + assert_eq!(keys.len(), 2, "{keys:?}"); + + let huge = padded_png(11 * 1024 * 1024); + let response = world + .export(&world.owner, page, "docx", &[(&keys[0], &huge)]) + .await; + assert_eq!(response.status(), StatusCode::PAYLOAD_TOO_LARGE); + assert_eq!(error_code(response).await, "export.too_large"); + + // A readable picture past the 2 MiB part cap is dropped; the part after + // it is still read. + let big = padded_png(3 * 1024 * 1024); + let response = world + .export( + &world.owner, + page, + "docx", + &[(&keys[0], &big), (&keys[1], TINY_PNG)], + ) + .await; + assert_eq!(response.status(), StatusCode::OK); + assert_eq!( + fallbacks_header(&response), + "diagrams=1, images=0, unknown=0" + ); + + // A part of exactly 2 MiB is kept. + let at_cap = padded_png(2 * 1024 * 1024); + let response = world + .export( + &world.owner, + page, + "docx", + &[(&keys[0], &at_cap), (&keys[1], TINY_PNG)], + ) + .await; + assert_eq!(response.status(), StatusCode::OK); + assert_eq!(fallbacks_header(&response), NO_FALLBACKS); +} + +#[tokio::test(flavor = "multi_thread")] +async fn diagrams_listing_matches_writer_keys() { + let world = world().await; + let page = world.create_doc("Flow").await; + world + .set_markdown(page, "```mermaid\ngraph TD; A-->B\n```\n") + .await; + let key = format!("mermaid-{:x}", Sha256::digest(b"graph TD; A-->B")); + + let response = world.diagrams(&world.owner, page).await; + assert_eq!(response.status(), StatusCode::OK); + assert_eq!( + body_json(response).await, + json!({ + "limits": { + "max_body_bytes": 10485760, + "max_part_bytes": 2097152, + "max_parts": 100, + "scale": 2, + }, + "diagrams": [{"key": key, "kind": "mermaid", "source": "graph TD; A-->B"}], + }) + ); + + // A part named after nothing on the page is ignored. + let response = world + .export( + &world.owner, + page, + "docx", + &[("mermaid-not-on-page", TINY_PNG), (&key, TINY_PNG)], + ) + .await; + assert_eq!(response.status(), StatusCode::OK); + assert_eq!(fallbacks_header(&response), NO_FALLBACKS); + let docx = body_bytes(response).await; + assert_eq!(zip_names(&docx, "word/media/").len(), 1); + + // Boards are listed with their id and label. + let board_page = world.create_doc("Board").await; + let board_id = Uuid::new_v4().to_string(); + let content = yrs::Doc::new(); + let fragment = content.get_or_insert_xml_fragment("default"); + { + let mut txn = content.transact_mut(); + let board = fragment.push_back(&mut txn, XmlElementPrelim::empty("excalidraw_board")); + board.insert_attribute(&mut txn, "board_id", Any::from(board_id.clone())); + board.insert_attribute(&mut txn, "label", Any::from("Roadmap")); + } + world.set_content(board_page, &content).await; + let response = world.diagrams(&world.owner, board_page).await; + assert_eq!(response.status(), StatusCode::OK); + assert_eq!( + body_json(response).await["diagrams"], + json!([{ + "key": format!("board-{board_id}"), + "kind": "board", + "board_id": board_id, + "label": "Roadmap", + }]) + ); +} + +#[tokio::test(flavor = "multi_thread")] +async fn third_concurrent_export_gets_503() { + let world = world_with(|s| { + s.export_permits = Arc::new(Semaphore::new(0)); + s.export_busy_wait = Duration::from_millis(50); + }) + .await; + let page = world.create_doc("Busy").await; + + let response = world.export(&world.owner, page, "docx", &[]).await; + assert_eq!(response.status(), StatusCode::SERVICE_UNAVAILABLE); + assert_eq!(header(&response, "retry-after"), "5"); + assert_eq!(error_code(response).await, "export.busy"); + + // One free slot serves exports one after the other: each gives its + // permit back when it is done. + world.state.export_permits.add_permits(1); + for _ in 0..2 { + let response = world.export(&world.owner, page, "docx", &[]).await; + assert_eq!(response.status(), StatusCode::OK); + } +} + +/// A request dropped mid-build (the 30 s request timeout, or the client +/// going away) must not hand its permit back while the build still runs and +/// holds the page's images, pictures and output: the permit goes back when +/// the build ends, so no more than the permitted builds are ever in memory. +#[tokio::test(flavor = "multi_thread")] +async fn dropped_request_keeps_its_permit_until_the_build_ends() { + let entered = Arc::new(Notify::new()); + let open = Arc::new(Semaphore::new(0)); + let world = world_with(|s| { + s.blob_store = Some(Arc::new(GatedBlobs { + inner: s.blob_store.take().unwrap(), + entered: entered.clone(), + open: open.clone(), + })); + s.export_permits = Arc::new(Semaphore::new(1)); + s.export_busy_wait = Duration::from_millis(50); + }) + .await; + let page = world.create_doc("Dropped").await; + let url = world.upload(page, "image/png", PNG_3X2).await; + world.set_markdown(page, &format!("![p]({url})\n")).await; + + // Start the export and drop the request once its build is fetching the + // image. + { + let export = world.export(&world.owner, page, "docx", &[]); + tokio::pin!(export); + tokio::select! { + _ = &mut export => panic!("the export finished while its image read was held"), + () = entered.notified() => {} + } + } + + assert_eq!( + world.state.export_permits.available_permits(), + 0, + "the dropped request's build still holds the permit" + ); + let response = world.export(&world.owner, page, "docx", &[]).await; + assert_eq!(response.status(), StatusCode::SERVICE_UNAVAILABLE); + assert_eq!(error_code(response).await, "export.busy"); + + // Let the build finish: the permit comes back, and the next export runs. + open.add_permits(1); + let permit = tokio::time::timeout( + Duration::from_secs(30), + world.state.export_permits.acquire(), + ) + .await + .expect("the build gives its permit back when it ends") + .unwrap(); + drop(permit); + let response = world.export(&world.owner, page, "docx", &[]).await; + assert_eq!(response.status(), StatusCode::OK); + assert_eq!(fallbacks_header(&response), NO_FALLBACKS); +} + +#[tokio::test(flavor = "multi_thread")] +async fn parts_beyond_one_hundred_fall_back() { + let world = world().await; + let page = world.create_doc("Many diagrams").await; + let markdown: String = (0..101) + .map(|i| format!("```mermaid\ngraph TD; A-->N{i}\n```\n\n")) + .collect(); + world.set_markdown(page, &markdown).await; + let keys = world.diagram_keys(page).await; + assert_eq!(keys.len(), 101); + + let parts: Vec<(&str, &[u8])> = keys.iter().map(|key| (key.as_str(), TINY_PNG)).collect(); + let response = world.export(&world.owner, page, "docx", &parts).await; + assert_eq!(response.status(), StatusCode::OK); + assert_eq!( + fallbacks_header(&response), + "diagrams=1, images=0, unknown=0" + ); +} + +#[tokio::test(flavor = "multi_thread")] +async fn filename_is_sanitized_on_both_sides() { + let world = world().await; + let page = world.create_doc("Q4 / Plan").await; + + let response = world.export(&world.owner, page, "docx", &[]).await; + assert_eq!(response.status(), StatusCode::OK); + let disposition = header(&response, "content-disposition"); + assert!( + disposition.contains("filename*=UTF-8''Q4%20-%20Plan.docx"), + "{disposition}" + ); + assert!( + disposition.contains(r#"filename="Q4 - Plan.docx""#), + "{disposition}" + ); +} diff --git a/deploy/helm/knot/README.md b/deploy/helm/knot/README.md index 1a1c19f2..c6533bba 100644 --- a/deploy/helm/knot/README.md +++ b/deploy/helm/knot/README.md @@ -174,12 +174,15 @@ helm upgrade knot ./deploy/helm/knot -n knot -f my-values.yaml The `pre-upgrade` Job re-runs `knot-server migrate` before the new pods land, so any new migrations in the release are applied first. Failed migrations halt the upgrade — fix the cause and re-run. +Before rolling out a release that changes page export, run `knot-server export-check` against a staging copy of the production database: it exports every page to DOCX and ODT offline (no migration, no writes), prints the fallback counts, and exits 1 if any page fails. + ## Operational notes - **Healthchecks** — readiness `/api/readyz`, liveness `/api/healthz`. Disable with `probes.enabled=false` if you want to debug a wedged pod. - **Image size** — the runtime image is `scratch` + a static musl binary (mimalloc allocator). Expect ~20 MB. - **Resources** — defaults `100m/128Mi` requests, `1000m/512Mi` limits. Right-size based on workspace volume. - **Replicas** — knot's CRDT room actor is per-pod and discovers peers via Postgres `LISTEN/NOTIFY`, so `replicaCount > 1` works without sticky sessions. Each room is hosted by exactly one pod at a time; failover takes one TCP roundtrip. +- **Request body size** — image uploads and Word/OpenDocument exports (which upload the page's diagram pictures) send up to 10 MiB. A reverse proxy or ingress in front of knot must allow request bodies of at least 10 MiB. With ingress-nginx (default 1 MiB), set `nginx.ingress.kubernetes.io/proxy-body-size: 10m` in `ingress.annotations`. ## Uninstall diff --git a/docs/superpowers/plans/2026-10-23-page-export-docx-odt.md b/docs/superpowers/plans/2026-10-23-page-export-docx-odt.md new file mode 100644 index 00000000..b8127f35 --- /dev/null +++ b/docs/superpowers/plans/2026-10-23-page-export-docx-odt.md @@ -0,0 +1,1088 @@ +# Page Export to DOCX and ODT Implementation Plan + +> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. + +**Goal:** Export the open knot page to a neutral, well-structured .docx or .odt, with diagrams included as pictures and every element degrading gracefully. + +**Architecture:** +- A new pure crate `knot-export` walks the page's yrs XmlFragment into an export model. +- Two hand-written writers (OOXML, ODF) turn that model plus resolved assets into zip bytes. +- `knot-server` adds two routes behind `require_doc_role_mw`: + - one lists the diagrams on the page; + - one takes PNG pictures as multipart and returns the file. +- It also adds an offline `export-check` CLI. +- The SPA replaces the download icon with a menu. It renders Mermaid and board previews to PNG and saves the response through a `blob:` link. + +**Tech Stack:** +- Server: Rust 2024 (toolchain 1.97.1), yrs 0.27, zip 8, sha2 0.10, image-webp 0.2, png 0.18, imagesize 0.15, axum 0.8, multer 3. +- Web: React + TanStack Query + valibot, Mermaid 11.16. +- Tests: Vitest, Playwright (Chromium). + +**Spec:** `docs/superpowers/specs/2026-10-23-page-export-docx-odt-design.md` +Read it before any task, including its "Planning clarifications" section at +the end. The mockup is `docs/superpowers/specs/2026-10-23-page-export-mockup.html`. + +## Global Constraints + +- **Footprint:** one static binary, no new runtime process or service, no outbound network calls during export, everything in memory. +- **Dependency policy:** `cargo deny check` gains no new license or advisory exception. Don't call `zip::ZipArchive::extract`. Keep `zip = { version = "8", default-features = false, features = ["deflate"] }`. +- **Limits** (binary MB, as named constants): + - body 10 MiB; + - per picture part 2 MiB; + - at most 100 parts; + - image budget 40 MiB per export; + - 2 concurrent exports per process; + - busy wait 5 s, then `503 export.busy` with `Retry-After: 5`; + - diagram scale 2; + - diagram PNG long side ≤ 2000 px; + - WebP pixel cap 16,777,216 (4096×4096). The image budget counts the bytes after preparation too (see Task 12). +- **Page:** A4 (DOCX `w:w="11906" w:h="16838"`), margins 2.5 cm (`1417` twips), text width 16 cm (9072 twips, 5,760,720 EMU). Footer `Page X of Y`, centred. +- **Fonts:** body and headings "Calibri", code "Consolas". Named, never embedded. +- **Error codes:** `export.bad_format` (400), `export.too_large` (413), `export.busy` (503), plus the existing `doc.archived` (409), `doc.not_found` (404), `acl.no_grant` (403), `auth.csrf` (403) and `auth.session_required` (401). +- **Response header:** `X-Knot-Export-Fallbacks: diagrams=, images=, unknown=`. The client reads `diagrams` and `images` and ignores other keys. `unknown` counts unknown elements and is not shown to users. +- **Placeholders and labels (exact copy):** + - "[Image not available]" + - "[Image too large to include]" + - "Mermaid diagram (not included)" + - "Diagram: + revoke +``` + +- **`file`:** a hand-rolled `fetch` with a `FormData` body: one field per key, with the blob named `.png`. It sends the `X-CSRF-Token` header from `readCookie("csrf")` and `credentials: "include"`. It parses the error envelope like `blobsApi.upload`. A network error gives `{ code: "network", status: 0 }`. + +- [ ] **Step 1: Write the failing tests:** + +```ts +it("exportFileName follows the shared rule", () => { + expect(exportFileName("Q4 / Plan: ?", "docx")).toBe("Q4 - Plan- -draft--.docx"); + expect(exportFileName(" .. ", "odt")).toBe("Untitled.odt"); + expect(exportFileName("x".repeat(200), "docx")).toHaveLength(125); + expect(exportFileName("Über 😀", "docx")).toBe("Über 😀.docx"); + expect(exportFileName("a".repeat(119) + "😀😀", "docx")).toBe("a".repeat(119) + "😀.docx"); // cap counts code points +}); +it("parseFallbacks", ...) // "diagrams=2, images=1, unknown=3" → {diagrams:2, images:1}; null → zeros +it("fallbackMessage copy", () => { + expect(fallbackMessage({diagrams:2, images:1}, false)).toBe("Exported. 2 diagrams and 1 image couldn't be included."); + expect(fallbackMessage({diagrams:1, images:0}, false)).toBe("Exported. 1 diagram couldn't be included."); + expect(fallbackMessage({diagrams:0, images:0}, true)).toBe("Exported. Your latest edits weren't saved yet and may be missing."); + expect(fallbackMessage({diagrams:0, images:0}, false)).toBeNull(); +}); +it("file() posts multipart with CSRF and returns blob + fallbacks", ...) // fetch spy: method POST, X-CSRF-Token, FormData has key; headers X-Knot-Export-Fallbacks parsed +it("file() maps the error envelope and network errors", ...) // 503 export.busy → error.code; reject → code "network" +``` + +- [ ] **Step 2: Run, expect FAIL.** + - Run: `cd web && pnpm test src/features/export` +- [ ] **Step 3: Implement the client and helpers** with the signatures above. +- [ ] **Step 4: Run, expect PASS.** + - Run: `cd web && pnpm test && pnpm tsc && pnpm lint` +- [ ] **Step 5: Commit.** + - `git commit -am "feat(export): export api client and file helpers"` + +### Task 17: Web: the `useExport` flow + +**Files:** +- Create: `web/src/features/export/useExport.ts`, `useExport.test.ts` + +**Interfaces:** +- Consumes: `exportApi`, `renderDiagrams`, `exportFileName`, `fallbackMessage`, `saveBlob`, and `useUi().notify`. +- Produces: + - `export function useExport(opts: { docId: string; title: string; status: ConnStatus; pendingBytes: number; deps?: Partial }): { exporting: boolean; run: (format: "docx" | "odt") => Promise }` + - `type ExportDeps = { exportApi; renderDiagrams; exportFileName; fallbackMessage; saveBlob; notify }` (the types of those functions), injectable in tests. +- **`run`** is guarded by a ref that is set synchronously (`busyRef.current`). React state would be stale for two calls in the same tick. The ref is mirrored into `exporting` state for the UI. While the ref is set, `run` returns at once. Otherwise: + 1. wait up to 5 s for `status === "connected" && pendingBytes === 0`, mirrored into a ref inside an effect (the React Compiler rules forbid reading a ref during render); `unsynced` is whether that wait timed out; + 2. call `exportApi.diagrams`, then `renderDiagrams`; + 3. call `exportApi.file`; + 4. on error `413`, `408` or `0`, retry `file` once with an empty map; + 5. on success, `saveBlob(blob, exportFileName(title, format))`, and if `fallbackMessage` is non-null, `notify("warn", msg)`; + 6. on `export.busy`, `notify("warn", "Export is busy, try again in a moment.")`; + 7. on any other error, `notify("error", "Couldn't export this page. Try again, or export as Markdown.")`; + 8. finally, clear `exporting`. + +- [ ] **Step 1: Write the failing tests** (`renderHook`, fake timers, injected deps): + +```ts +it("waits for pending edits, then exports and saves with the sanitized name", ...) +it("exports anyway after 5 s of unsynced edits and appends the warning", ...) // status "offline" for 5 s +it("retries once without pictures on 413", ...) // file called twice; second with empty map +it("shows the busy toast on export.busy", ...) +it("second_click_while_exporting_does_nothing", ...) // two run() calls in the same tick → diagrams called once +it("shows the generic error toast for other failures", ...) // 500 internal → notify("error", "Couldn't export this page. Try again, or export as Markdown.") +``` + +- [ ] **Step 2: Run, expect FAIL.** + - Run: `cd web && pnpm test src/features/export/useExport.test.ts` +- [ ] **Step 3: Implement `useExport`.** +- [ ] **Step 4: Run, expect PASS.** + - Run: `cd web && pnpm test && pnpm tsc && pnpm lint` +- [ ] **Step 5: Commit.** + - `git commit -am "feat(export): export flow hook"` + +### Task 18: Web: `ExportMenu` in the page header + +**Files:** +- Create: `web/src/features/export/ExportMenu.tsx`, `ExportMenu.test.tsx` +- Modify: `web/src/features/docs/DocPage.tsx`. Replace only the `doc-export` `IconButton` (lines 328–343) with ``. Keep the `{!archived && (` guard (lines 327 and 344) and update the comment on line 326. + +**Interfaces:** +- Consumes: `useExport`, `IconButton`. +- **Trigger:** `IconButton` with `data-testid="doc-export"`, label "Export", `aria-haspopup="menu"` and `aria-expanded`. It shows the `Download` icon, or `LoaderCircle` with `animate-spin` while exporting. +- **Popover:** `role="menu"`, using the DocByline outside-click pattern (it ignores the trigger, and Escape returns focus to the trigger), right-aligned under the trigger, `max-w-[calc(100vw-2rem)]`. +- **Items** (`role="menuitem"`; ArrowUp/Down, Home/End and Enter/Space work): + + | testid | Label | Sub-label | + | --- | --- | --- | + | `export-docx` | "Word document" | ".docx · opens in Word, LibreOffice, Google Docs" | + | `export-odt` | "OpenDocument text" | ".odt · LibreOffice's own format" | + + Then a separator (`role="separator"`), then, **only when `!isTemplate`**: + + | testid | Label | Action | + | --- | --- | --- | + | `export-md` | "Markdown (zip)" | today's anchor download, `descendants=false` | + | `export-md-subtree` | "Markdown with subpages (zip)" | `descendants=true` | + + The shift-click shortcut is removed. +- Choosing a Word or OpenDocument item closes the menu and calls `run(format)`. The items are disabled while exporting. + +- [ ] **Step 1: Write the failing tests:** + +```ts +it("opens a menu with Word, OpenDocument and both Markdown items", ...) +it("hides the Markdown items on template pages", ...) +it("supports arrow keys, Home/End and Escape returning focus to the trigger", ...) +it("disables export items and spins the icon while exporting", ...) +it("runs docx export on select", ...) // useExport mocked via vi.mock +``` + + Also, in `DocPage.archived.test.tsx`, the existing assertion that `doc-export` is absent stays green unchanged. Add `it("readers see the export menu")` using the viewer server from `DocPage.byline.test.tsx`. +- [ ] **Step 2: Run, expect FAIL.** + - Run: `cd web && pnpm test src/features/export/ExportMenu.test.tsx` +- [ ] **Step 3: Implement `ExportMenu`** and wire it into DocPage. +- [ ] **Step 4: Run, expect PASS.** + - Run: `cd web && pnpm test && pnpm tsc && pnpm lint` +- [ ] **Step 5: Commit.** + - `git commit -am "feat(export): export menu in the page header"` + +### Task 19: e2e: export flow + +**Files:** +- Create: `e2e/support/zip.ts`, `e2e/flows/export.spec.ts` + +**Interfaces:** +- Produces `export function zipEntries(buf: Buffer): Map`. It reads the central directory and inflates `deflate` entries with `zlib.inflateRawSync`, and returns stored entries as they are. No new dependency. + +- [ ] **Step 1: Write the failing spec.** + - **Seed:** + - sign in (pattern `mobile.spec.ts`); + - create a page through `POST /api/docs` with body `{"title": "Export test"}` (the Markdown import doesn't set the title); + - `POST /api/docs/{id}/markdown?mode=replace` with: + + ````markdown + # Export test + + - [x] done + - [ ] todo + + | A | B | + |---|---| + | 1 | 2 | + + ```mermaid + graph TD; A-->B + ``` + ```` + + - **DOCX:** open the page, wait for `status-dot` to be connected, click `doc-export` then `export-docx`, and take the file from `page.waitForEvent("download")`. Assert: + - `zipEntries` has `word/document.xml`, which contains `Export test` and `w:val="Heading1"`; + - an entry matches `word/media/image1.png`; + - the download's `suggestedFilename()` is `Export test.docx`. + - **ODT:** the same for ODT: `content.xml` has `text:outline-level="1"`, and there is a `Pictures/` entry. +- [ ] **Step 2: Run, expect FAIL.** + - Run: `cd e2e && pnpm playwright test flows/export.spec.ts` + - Expected: FAIL until the full stack is built. Follow CONTRIBUTING.md; for the local Apple-container stack, Dex must run. +- [ ] **Step 3: Fix anything the e2e surfaces** in the owning task's files. +- [ ] **Step 4: Run, expect PASS.** + - Run: `cd e2e && pnpm playwright test flows/export.spec.ts flows/mobile.spec.ts` + - Expected: all pass. +- [ ] **Step 5: Commit.** + - `git add e2e && git commit -m "test(export): e2e export to docx and odt"` + +### Task 20: Whole-branch verification and acceptance files + +**Files:** none new. + +- [ ] **Step 1: Run every repo check** from Global Constraints. Expected: all green. The only advisory finding allowed is the pre-existing `ctutils` RUSTSEC-2026-0385. +- [ ] **Step 2: Build the acceptance files.** + - Run `cargo run -p knot-export --example kitchen_sink -- target/export-samples`. + - Also export a real seeded page through the UI as `.docx` and `.odt`. + - Attach all four to the PR. The project owner runs the manual acceptance checklist from the spec, including a rotated JPEG phone photo: EXIF orientation is not honoured, so record what the photo looks like. +- [ ] **Step 3: Run the manual checks.** + - Do one pass of the diagram step in Firefox and Safari, exporting a page with a Mermaid flowchart and a board (spec §8). + - If a staging copy or restored backup exists, run `knot-server export-check` against it and paste the summary into the PR. + - Add an SLO follow-up note to the PR, or an SLO line in `docs/SLO.md` if there's a measured target (spec §11). +- [ ] **Step 4: Check the binary size.** Run `cargo build --release --bin knot-server` and compare `ls -l target/release/knot-server` against main. Expected: growth ≤ 3 MB. Note the number in the PR. diff --git a/docs/superpowers/specs/2026-10-23-page-export-docx-odt-design.md b/docs/superpowers/specs/2026-10-23-page-export-docx-odt-design.md new file mode 100644 index 00000000..b8d588d2 --- /dev/null +++ b/docs/superpowers/specs/2026-10-23-page-export-docx-odt-design.md @@ -0,0 +1,651 @@ +# Export a page to Word (DOCX) and OpenDocument (ODT) — design + +**Date:** 2026-10-23 +**Status:** Draft (brainstorm), builder-checked, awaiting approval +**Mockup (approved in the brainstorm):** +[`2026-10-23-page-export-mockup.html`](2026-10-23-page-export-mockup.html). +It is the visual reference for the menu and the overall look, not a pixel spec. + +## Intent + +The company that uses knot works in Word and LibreOffice documents. People +want to take a knot page, export it, and send the file around internally +without reformatting it by hand. This feature covers that and nothing more: +one page in, one tidy document out, which opens cleanly in Word, LibreOffice +and Google Docs. + +PDF was considered and dropped for now. A company look (letterhead, house +fonts, a Word template) is a planned follow-up. This export is built so a +template can slot in later without redoing it (§10). + +## Goals + +- Export the page you are looking at to **.docx** and **.odt** from the page + header. +- Bring everything on the page along: text and formatting, headings, lists, + checklists, tables (column widths, merged cells, lists and images inside + cells), images, code, links, and **Mermaid diagrams and Excalidraw boards + as pictures**. People use diagrams, so they are in scope. +- Use real document structure: Word heading styles with outline levels, so + the navigation pane and a generated table of contents work. In ODT, the + equivalent outline-level headings. +- Look neutral and tidy. Nothing in the file should look broken or + half-formatted. +- Tell the user when something couldn't be included. + +## Non-goals + +- PDF output. +- Company templates or branding (follow-up, §10). +- Classification or sensitivity markings (e.g. "VS-NfD", Microsoft + sensitivity labels). The user confirmed that marking a document is up to + whoever sends it. This may change with company templates. +- Exporting several pages, or a page with its subpages, into one document. + The Markdown zip keeps doing that. +- Importing DOCX/ODT back into knot, or round-tripping edits. +- Comments, history, tracked changes, syntax highlighting, embedded fonts. +- Exporting from public share links (`/p/:token`). +- Export UI in the mobile app. The API should not stand in its way (§3). +- Git sync. That is a separate project, brainstormed later. +- Fixing the existing Markdown export, including its single-page zip + filename bug (`export_import.rs:428`), and the zip export refusing template + pages. + +## Anti-goals (failure even if it technically works) + +- **One odd element sinks the whole export.** The Markdown serializer fails + the entire page on an unknown node (`SerError::UnsupportedNode`, + `crates/knot-markdown/src/to_markdown.rs`). This export must degrade per + element and always produce a file. +- **Fake headings.** Big bold text that looks like a heading but isn't a + heading style breaks navigation, TOCs and any later template. +- **Leaking content.** An image pasted from a page the exporter cannot read + must not end up in their file. +- **Taking the server down.** An export of an image-heavy page must not run + the pod out of memory and drop everyone's live editing sessions (§3, + limits). +- **Silent omissions.** If a diagram or image couldn't be included, the user + is told before they send the file on. +- **A sidecar service.** knot ships as one static binary. Requiring + LibreOffice, Pandoc, Chromium or Gotenberg next to it is out. + +## What the user decided + +- DOCX and ODT now. PDF later, if at all. +- Neutral styling now. Company templates as a follow-up, with the export + structured so templating fits in later. +- Diagrams (Mermaid, Excalidraw boards) must be included as pictures. +- No classification marking in the export: that is up to whoever sends the + file. It may come back with templates. +- The menu and output in the mockup are right: + - the download button becomes a menu with Word, OpenDocument, and the + Markdown zip; + - the file opens with the page title, an "Exported from knot · date" line, + real headings, and page numbers in the footer; + - mentions, dates, page links, checklists, tables, attachments and comments + are handled as listed in §6. + +Calls I made that the user did not weigh in on are marked **(my call)**. + +## Current state (verified in the code) + +- **Content lives in a yrs Y.Doc** (`"default"` XmlFragment, snake_case + element tags). The schema is `tools/schema.json`. There is no ProseMirror + JSON on the server. The main server-side reader is the Markdown serializer, + which walks the XmlFragment directly. The room actor's + `patch_task_checked` (`crates/knot-crdt/src/room.rs:675`) also walks it, + looking for list items. +- **Markdown loses information an export needs:** table column widths, + `colspan`/`rowspan`, block content inside cells, and attachment metadata. + **This export must walk the Y.Doc, not the Markdown.** +- **Table column widths:** + - `colwidth` is a per-cell array with one entry per spanned column. It is + stored in the Y.Doc, though missing from `tools/schema.json`. + - Values are normalised weights summing to 1000 once a column has been + dragged, but older pixel widths and null or 0 entries are also valid. + - The editor works out the final shares in `resolveShares` + (`web/src/features/editor/columnWidths.ts`). The export must port that + logic so widths match what users see. +- **Live state:** `refresh_markdown_inner` + (`crates/knot-server/src/routes/api/markdown.rs:85-108`) asks the room actor + for a snapshot (`Event::ExportState`) and applies it to a temporary doc. The + export copies that snapshot step. It must not call `refresh_markdown_and_index` + or `export_markdown_only`, which also write `doc_markdown_cache` and + re-index tasks, and it must not read `doc_markdown_cache`, which lags. +- **Existing export:** `GET /api/docs/{id}/export?descendants=` returns a + Markdown zip (`routes/api/export_import.rs`). + - Access goes through `acl.effective_role`. Viewers can't see archived + pages, so they get 403. Editors and owners get 404 on archived *and* + template pages, because `list_alive` filters both. + - The page header's download icon (`data-testid="doc-export"`, + `web/src/features/docs/DocPage.tsx`) is hidden on archived pages. + Shift-click exports the subtree. + - Tests click it: `e2e/flows/mobile.spec.ts:81` and + `web/src/features/docs/DocPage.archived.test.tsx:127`. +- **Per-page routes** mount in `routes/api/docs.rs` `doc_id_routes` behind + `require_doc_role_mw`. For non-GET requests from editors and owners, that + middleware returns 409 `doc.archived`. Viewers get 404 on archived pages. +- **Request bodies:** `DefaultBodyLimit` (2 MB, `lib.rs:207`) only affects + extractors that opt in. Handlers that read the raw request set their own + caps: blob upload uses `multer` at 10 MiB (`routes/api/blobs.rs:23`), board + SVG 1 MiB, zip import 50 MiB. Every API route has a 30 s timeout + (`lib.rs:208`), and it includes reading the request body. +- **Special inline content is link marks with sentinel hrefs:** + `knot://doc/` (page links), `knot://user/` (mentions, text + `@Name` frozen at insertion), `knot://time/` (dates, text in the + author's local format at insertion). +- **Images:** + - `image` nodes carry `src`, `alt` and `title`, with no stored size. + Uploaded images have `src="/api/blobs/"`. Images imported from + Markdown can have any `src`: external `http(s)`, relative paths, `data:`. + - Blobs live in Postgres or S3 behind `BlobStore`. + - Access is checked against **the page the blob was uploaded to** + (`blobs.doc_id`), not the page that shows it. + - Uploads are capped at 10 MB. Nothing limits how many images a page has. +- **Attachments** are `attachment` nodes (`url`, `name`, `size`, + `contentType`). +- **Diagrams exist only in the browser:** + - Mermaid is a `code_block` with `language="mermaid"`, rendered + client-side (`web/src/features/editor/nodes/MermaidCodeBlock.tsx`). + - Mermaid is a global singleton, initialised once with + `securityLevel: "strict"`. + - Its output goes through `sanitizeSvg`, which strips `foreignObject`. + - Mermaid's default HTML labels use `foreignObject`, so flowchart labels + may already be missing in the editor today. This is unverified and + outside this feature; see Out of scope. + - An Excalidraw board is an `excalidraw_board` node (`board_id`, `label`) + with its own Y.Doc. The browser renders a preview SVG with `exportToSvg` + when the board editor closes and uploads it (`PUT /api/boards/{id}/svg`, + 1 MiB cap, stored in `boards.svg_cached`). The preview can be missing if + nobody has opened the board yet. It can also be stale. +- **Runtime:** static musl binary in a `scratch` image with a read-only root + filesystem and no fonts or external binaries. The Helm default is 1 + replica with a 512 Mi memory limit and 1 CPU, shared with the live + collaboration rooms. +- **CSP** allows `img-src data: blob:` and `worker-src blob:`, so browser-side + rasterizing through a canvas and downloading through a `blob:` link work. + There is no `wasm-unsafe-eval`. +- **Dependency policy** (`deny.toml`): permissive licenses plus MPL-2.0. + Unmaintained advisories fail the check. `zip` 8 and `multer` are already + dependencies. Probes during the brainstorm found `docx-rs` passes; typst, + krilla and resvg don't. +- **Error codes** follow `.` (`import.not_zip`, + `blob.too_large`). This feature adds the `export` area. + +## Design + +### 1. UI + +- The page header's download icon becomes a small menu (my call: same spot, + same icon, `data-testid="doc-export"` on the trigger) with these items: + - **Word document (.docx)** + - **OpenDocument text (.odt)** + - a separator + - **Markdown (zip)**: today's single-page zip + - **Markdown with subpages (zip)**: today's shift-click, now a visible + item for keyboard and touch users (my call: the shift-click shortcut goes + away). The tests at `mobile.spec.ts:81` and + `DocPage.archived.test.tsx:127` are updated to match. +- **Visibility:** anyone who can read the page (viewer, editor, owner) sees + the menu, on regular and template pages. It stays hidden on archived + pages. +- **Progress:** the trigger shows "Preparing…" or a spinner while diagrams + render and the server builds the file. A second export can't start while + one is running. +- **Unsaved edits** (my call): the file is built from the server's copy. If + the page still has unsynced local edits (the `pendingBytes` the page + already tracks), the export first waits up to 5 s for them to sync. If + they're still pending, it exports anyway and the success toast adds + "Your latest edits weren't saved yet and may be missing." +- **Outcome:** + - Success: the file downloads as `.docx` / `.odt`. Characters + that file systems reject are replaced, and an empty title becomes + "Untitled". + - If anything fell back (§4), a toast says what: "Exported. 2 diagrams + and 1 image couldn't be included." + - Failures use the existing toast (`useUi().notify`) with an actionable + message. +- The menu is keyboard-accessible (opens with Enter/Space, arrow keys move, + Escape closes) and works at phone width. + +### 2. Flow + +```text +browser knot-server + │ GET /api/docs/{id}/export/diagrams ───────► walk live Y.Doc, list diagrams + │ ◄─── {limits, diagrams:[{key, kind:"mermaid", source} | {key, kind:"board", board_id, label}]} + │ render each to PNG at 2× (§8), fit within limits + │ POST /api/docs/{id}/export/{docx|odt} ────► walk live Y.Doc → export model + │ multipart: one PNG per diagram key resolve images, mentions, links + │ write DOCX or ODT (in memory) + │ ◄─── 200 file bytes + Content-Disposition + fallback counts + │ save via blob: link, toast about fallbacks +``` + +The server says which diagrams the page has (my call), so the browser +doesn't depend on the editor's local state. That keeps the flow usable from +places without a loaded editor: the doc tree later, or the mobile app. A +diagram whose picture doesn't arrive, because it failed to render, was added +between the two calls, or was dropped to fit the limits, falls back (§6). It +never fails the export. + +### 3. API + +- **Mounting:** both routes mount in `routes/api/docs.rs` `doc_id_routes` + behind `require_doc_role_mw`, so access matches the rest of the page API. + Viewer or higher can use them, on regular and template pages. +- **Archived pages:** refused on both routes with the middleware's semantics, + so viewers get 404 and editors and owners get 409 `doc.archived`. The GET + route checks this itself, because the middleware lets GETs through. +- `GET /api/docs/{id}/export/diagrams` → `200` with: + - `limits`: max total body bytes, max bytes per part, max parts, and the + scale (2). + - `diagrams`, each with: + - `key`: Mermaid is `"mermaid-" + hex(sha256(source))` over the code + block's text exactly as stored; a board is `"board-" + board_id`. + Identical sources collapse to one entry. + - `kind`, plus `source` for Mermaid, or `board_id` and `label` for boards. +- `POST /api/docs/{id}/export/{format}`, where `format` is `docx` or `odt`: + - **Body:** `multipart/form-data`. Each part is named by a diagram key and + holds an `image/png` rendered at 2× CSS pixels. An empty body is valid + and means no diagram pictures. + - **Auth:** session plus CSRF, like every POST. + - **Response:** `200` with + `application/vnd.openxmlformats-officedocument.wordprocessingml.document` + or `application/vnd.oasis.opendocument.text`, + `Content-Disposition: attachment` with an RFC 6266 `filename*` UTF-8 + name, and a header `X-Knot-Export-Fallbacks` (e.g. + `diagrams=2, images=1`) for the toast. + - POST, not GET, because it carries the pictures (my call). +- **Limits** (my call; values live as named constants and are returned by + the diagrams endpoint): + - **Diagram upload:** + - The total body is capped at **10 MB**, the same as blob uploads, so any + reverse proxy that already allows image uploads also allows exports. + - Each part is capped at 2 MB, with at most 100 parts. + - The client downsizes pictures to fit. + - The server treats a part over the per-part cap, or past the count, as + missing (fallback). It returns `413 export.too_large` only when the + whole body exceeds 10 MB. + - On `413` or `408` the client retries once with no pictures, and the + toast reports the diagrams as not included. + - **Embedded images:** at most **40 MB** of image bytes per export. Images + past the budget become the placeholder "[Image too large to include]", + linking to the image in knot, and count as fallbacks. + - **Concurrency:** at most **2 exports at a time per server process** + (a semaphore). A further request waits up to 5 s, then gets + `503 export.busy` with `Retry-After`. The toast says "Export is busy, + try again in a moment." + - With these numbers, worst-case export memory stays around 200 MB + (two exports, each holding its images, its pictures and the output), + well inside 512 Mi. The builder may stream the zip into the response to + lower that, but doesn't have to. + - Everything has to finish inside the 30 s timeout, including the upload. +- **Errors:** unknown format → `400 export.bad_format`. Other `export.*` + codes as above. +- **Pictures:** unknown part names are ignored. Pictures are only ever used + for diagram keys that are actually on the page, so a client can't smuggle + in arbitrary images. + +### 4. Server architecture + +- **New pure crate `knot-export`** (my call), sitting next to + `knot-markdown`. It does not touch the database or the network. + 1. **Walk** the XmlFragment into an export document model: blocks and + inlines with resolved attributes, the table grid with spans and resolved + column shares (the port of `resolveShares`), and list nesting. + 2. **Write** that model with two writers, `docx` and `odt`. Both share the + model and one style vocabulary (§7). +- **knot-server** owns the routes. It: + - takes the live snapshot (copying the step in `markdown.rs:85-108`); + - passes the crate a **resolver** for things that need IO: + - image bytes through `BlobStore` and blob metadata, checking read + access on each blob's own page, within the image budget; + - mention display names from `users`; + - page links as `{base_url}/doc/`; + - diagram PNGs from the request; + - returns the bytes along with the fallback counts. +- **Hand-write OOXML and ODF** as zip plus XML (my call; no `docx-rs`): + - template support later means taking control of `styles.xml`, numbering + and headers/footers anyway (§10); + - no ODT crate is mature; + - one walker feeding two thin writers keeps the formats consistent; + - `zip` is already a dependency. + + The builder may use an XML writer crate if it passes `deny.toml`. +- **Degrade per element:** + - An unknown element renders its text content as plain paragraphs. + - An unknown mark is dropped and its text kept. + - A malformed structure, such as a text node directly under `list_item` + (a known shape from Markdown imports), is repaired or flattened, never + rejected. + - Every fallback (unknown element, diagram without a picture, unavailable + or over-budget image) is counted by kind, logged at `warn` with the doc + id, and included in the response's fallback counts. + +### 5. Export check over real pages + +A CLI subcommand, `knot-server export-check` (my call), next to the existing +`migrate` and admin commands. + +- It exports every live page in the database to both formats, in memory and + without diagram pictures. +- It reports pages that failed, plus fallback counts by kind. +- It changes nothing. + +Run it against a staging copy or a restored backup before release. This is +the strongest check of the "one odd element" anti-goal, since hand-built +fixtures only cover shapes we already know about. + +### 6. Content mapping + +| Page content | In the document | +| --- | --- | +| Page title | Title style, first paragraph | +| (export line) | Subtitle style: `Exported from knot · 23 Oct 2026` (date in `D Mon YYYY`, server's UTC date; my call) | +| `heading` 1–6 | Heading 1–6 styles with outline levels 1–6 | +| `paragraph`, empty paragraph | Normal; empty ones kept as empty paragraphs | +| bold / italic / underline / strike | Direct run formatting | +| inline `code` | "Inline Code" character style (monospace) | +| `link` (http/https/mailto) | Hyperlink (Hyperlink character style) | +| `knot://doc/` | Hyperlink to `{base_url}/doc/`, link text unchanged | +| `knot://user/` | Plain text `@`; the stored text if the user is gone (my call) | +| `knot://time/` | Plain text as stored (what readers saw in the page) (my call) | +| any other `knot://` href, or other schemes | Plain text, no link. A `knot://` URL never reaches the file. | +| `hard_break` | Line break | +| `bullet_list` / `ordered_list` (nested, `start`) | Real list numbering, bullets or decimal per level, nesting kept, `start` honored | +| `list_item` with `checked` | No bullet; `☐` / `☑` (U+2610 / U+2611) before the text, at the item's nesting level (my call: characters, not Word content controls, so Word and LibreOffice look the same) | +| `blockquote` | Quote style (indented, left rule) | +| `code_block` (not Mermaid) | "Code" paragraph style: monospace, light shading, whitespace, tabs and line breaks preserved; no highlighting | +| Mermaid `code_block` | The picture for its key; if missing, the source as a Code block with an italic note "Mermaid diagram (not included)" (my call) | +| `excalidraw_board` | The picture for its key; if missing, an italic placeholder "Diagram: [board label] (not included)" (my call) | +| `horizontal_rule` | Empty paragraph with a bottom border | +| `image`, `/api/blobs/`, readable, within budget | Embedded (§9), scaled to fit the text width and never enlarged past natural size at 96 dpi; `alt` used as the picture's description | +| `image`, blob unreadable or missing | Italic placeholder "[Image not available]" with the alt text | +| `image`, over the image budget | Italic placeholder "[Image too large to include]", linked to the image in knot | +| `image`, external `http(s)` URL | Not fetched (no outbound requests, no SSRF); a hyperlink with the alt text, or the URL (my call) | +| `image`, any other `src` (relative path, `data:`, other schemes) | Italic placeholder "[Image not available]" with the alt text; never a link (my call) | +| diagram pictures (Mermaid, boards) | Placed at half their pixel size (they arrive at 2×), i.e. their on-screen size at 96 dpi, scaled down to the text width if larger | +| `attachment` | `📎 name (size)`, hyperlinked to `{base_url}/api/blobs/`, which needs a login to open | +| `table` | Real table. Column widths from the resolved shares as percentages of the text width. `colspan`/`rowspan` become real merges. Block content in cells (lists, images, several paragraphs) is kept. Header cells form a repeating header row in a Table Header style. Cell `align` honored. Thin grey borders. | +| comments, presence, history | Left out | + +### 7. The neutral look (template-ready) + +- **Page:** A4 portrait (my call: German company context), 2.5 cm margins, + footer `Page X of Y` centered (DOCX `PAGE`/`NUMPAGES` fields; ODT + `text:page-number` / `text:page-count`). No header. +- **Fonts:** named, not embedded (my call): body and headings + Calibri-compatible ("Calibri", which LibreOffice maps to the + metric-compatible Carlito), code "Consolas" with fallbacks. The builder may + pick another widely available pair if it renders better in both suites. +- **DOCX compatibility:** the file declares Word 2013+ compatibility mode + (`w:compat` with `compatibilityMode` 15), so Word doesn't open it in + Compatibility Mode. +- **Styles:** every piece of formatting goes through a **named style**. Body + text gets no direct font or size formatting. The style set is fixed and + small: + - Normal + - Title and Subtitle + - Heading 1–6 + - Quote + - Code, plus the Inline Code character style + - List Paragraph, plus one bullet and one numbered definition + - Table Grid, plus Table Header + - Hyperlink + - Caption / placeholder + - Footer + + In DOCX, built-in styles use Word's built-in names ("heading 1", "Title", + …) with `w:outlineLvl`, so Word treats them as its own. In ODT, the names + are LibreOffice's (`Heading_20_1` …) with `text:outline-level`. +- **Metadata:** document title = page title. Creating application = "knot". + No author field (my call: keeps the exporter's name out of forwarded files; + revisit with templates). + +### 8. Diagrams in the browser + +- **Mermaid:** + - Render the source and rasterize the SVG to PNG on a canvas at 2× CSS + pixels. + - HTML labels (`foreignObject`) taint the canvas in some browsers and + can't be drawn by Word. Export renders therefore use SVG text labels + (`htmlLabels: false`, which currently also needs `flowchart.htmlLabels: + false`), with `securityLevel: "strict"`. + - Mermaid is a global singleton. The builder must make sure the export + config doesn't leak into editor renders, for example by re-initialising + around export renders and restoring afterwards. +- **Boards:** fetch the stored preview (`GET /api/boards/{id}/svg`) and + rasterize it the same way. With no preview (`board.no_preview`), send + nothing, and the placeholder applies. Rendering from the board's Y.Doc with + Excalidraw's `exportToBlob` is allowed if it proves simple, but isn't + required. +- **Running the renders:** in parallel, with a small concurrency cap and a + per-diagram timeout. A failure skips that diagram, not the export. +- **Size:** PNGs are capped at 2000 px on the long side, then downsized + further as needed to fit the limits the diagrams endpoint returned. +- **Browsers:** automated e2e runs in Chromium, as CI does today. Before + release there is one manual pass of the diagram step in Firefox and Safari + (my call). Canvas tainting and SVG-as-image font loading differ by engine. + +### 9. Image formats + +- PNG, JPEG and GIF embed as they are. +- **WebP is always converted to PNG on the server** (my call). Older Word + versions can't show WebP, and converting removes the question. Use a + pure-Rust decoder and PNG encoder that pass `deny.toml` (e.g. the + `image-webp` and `png` crates). The builder confirms the license check and + the binary-size cost. +- Other types render as the "[Image not available]" placeholder. SVG uploads + are already `attachment` nodes. +- Picture dimensions are read from the image header to size the frame. + +### 10. Ready for company templates (follow-up, not built now) + +The follow-up lets a workspace owner upload a Word template (.docx/.dotx) and +an ODT template (.ott). The export then: + +- keeps the template's styles, theme, numbering, header/footer and page + setup; +- replaces the body; +- maps our style vocabulary onto the template's styles by built-in name. + Word localizes style IDs (a German template has `berschrift1` where English + has `Heading1`) but keeps the built-in `w:name`, so mapping goes by name. + +This design prepares for that by: + +1. keeping all formatting in named styles (§7); +2. keeping style definitions, numbering definitions, footer and section setup + each in one place in each writer, so they can be swapped; +3. making the writers take their "package defaults" (styles.xml and friends) + as an input with built-in defaults, rather than hard-wiring them through + the code. + +### 11. Observability + +- A counter `knot_export_total{format, outcome}` (outcome `ok`, + `client_error`, `busy` or `server_error`) and a duration histogram, + registered and described in `knot-obs` like the existing metrics. +- `knot_export_fallback_total{kind}`, with kind one of `diagram`, + `image_unavailable`, `image_over_budget` or `unknown_element`. +- Add an SLO line to `docs/SLO.md` only if the builder can state a target + backed by measurement. Otherwise note it as a follow-up. + +## Testing + +Write tests first where practical (repo norm: TDD). + +- **`knot-export` unit tests:** + - **Fixture docs:** built in code with a `DocBuilder` like the one in + `crates/knot-markdown/tests/round_trip.rs`. Cover every node and mark in + §6, deep nesting, mixed checklists, and merged cells whose grid needs + computing. + - **Column widths:** pixel, null and normalised `colwidth` values resolve + like the editor's `resolveShares`. + - **Degradation cases:** an unknown element, a text node under + `list_item`, an empty table. +- **Writer tests:** unzip and parse the output, then assert: + - **Package integrity:** every relationship id resolves, every media part + has a content type, every style and numbering id referenced exists. ODT: + `mimetype` is the first entry and stored uncompressed, and the manifest + lists every file. + - **Structure:** headings use heading styles with outline levels, table + grids add up, spans are encoded, list nesting is right, the + compatibility mode is declared. + - **Well-formed XML everywhere.** +- **Server integration tests** (`crates/knot-server/tests`, `fresh_db`): + - **Access:** viewer gets 200; template pages export; non-member and + anonymous are refused. + - **Archived pages:** viewer gets 404; editor gets 409 on both routes. + - **Request errors:** missing CSRF returns 403; bad format returns 400; + a body over 10 MB returns 413; an oversized part or extra parts fall + back instead. + - **Busy:** with the semaphore held, a request returns 503. + - **Content:** + - an image from a page the user can't read becomes the placeholder; + - the image budget is enforced, with a placeholder; + - WebP is converted; + - mentions use the current display name; + - page links use `base_url`; + - diagram parts are used only for keys on the page; + - the diagrams endpoint lists keys that match what the writer looks up; + - fallback counts appear in the header. +- **`export-check` CLI:** a test over a small seeded workspace, including a + malformed page. +- **Web (Vitest):** + - **Menu:** items and visibility (readers, templates, hidden when + archived), keyboard behavior. + - **Diagram step:** with Mermaid mocked, a render failure skips that + diagram; downsizing respects the limits; a 413 triggers a retry without + pictures. + - **Unsaved edits:** the export waits for pending edits, then warns. + - **Results:** the fallback toast wording; the busy and error toasts; the + file is named after the page title. +- **e2e** (required for new user-visible surface, Chromium): export a page + containing a heading, a checklist, a table and a Mermaid block to DOCX and + ODT. Assert the download is a zip containing `word/document.xml` / + `content.xml` with the title, the heading style, and a picture part for + the diagram. Update `mobile.spec.ts:81` for the menu. +- **Manual acceptance before merge** (my call; knot has no feature flags, and + releases are cut from main): + - Export a "kitchen sink" page to both formats and attach the files to + the PR. + - The project owner opens them on company machines in: + - Word for Microsoft 365 (Windows and macOS); + - Word 2019 or newer; + - Word for the web (Teams/SharePoint preview); + - LibreOffice 7.6 or newer. + + Google Docs is best-effort, and only with non-sensitive sample content. + - **Pass means:** + - the file opens without a repair prompt or a Compatibility Mode banner; + - the navigation pane lists the headings; + - an inserted TOC lists the headings; + - "Page X of Y" is right; + - tables keep their widths and merges; + - the pictures are present; + - checklists show their boxes. + - Run `export-check` against a staging copy or restored backup if one is + available. + +## Constraints + +- One static binary; no new runtime processes or services; works within 512 + Mi memory and the 30 s timeout. Exports are built in memory; no disk. +- `cargo deny check` must not gain new license or advisory exceptions for + this feature. +- No outbound network calls during export. +- No change to the existing Markdown export's server behavior or to the CSP. +- The binary should grow by low single-digit MB at most. The image-size + target in the foundation spec stands. +- Documentation: + - a CHANGELOG entry under Unreleased (Added); + - the knot-api skill (`.claude/skills/knot-api/SKILL.md`) lists the new + routes; + - `ARCHITECTURE.md`'s crate table gains `knot-export`; + - the Helm README notes that exports upload up to 10 MB, the same as + images. + +## Left to the builder + +- Crate layout and naming inside `knot-export`. The XML writing approach. + Whether DOCX and ODT share more code than the model. +- Exact style metrics (sizes, spacing, colors), within "neutral and tidy" + and the mockup's look. +- Whether the zip is streamed into the response. +- The multipart parsing details (`multer`, as blob upload uses). +- How the Mermaid export config is applied and restored; the rasterizing + details. +- The image-header parsing approach and the WebP crates (within §9). +- Whether the menu is a new small component or reuses an existing + menu/popover pattern in `web/src/components`. + +## Out of scope / follow-ups + +- **Company templates** (§10). Next in line for export. Revisit + classification markings there (header/footer text or sensitivity labels). +- **PDF.** Options from the brainstorm research: + - typst in-process: about +35 MB binary, plus advisory exceptions; + - browser print; + - an optional sidecar. +- **Page plus subpages** in one document. +- **Mobile app export** through the same endpoints. +- **The zip export's filename bug and template 404** (Current state). +- **Mermaid flowchart labels possibly missing in the editor**, because + `sanitizeSvg` strips `foreignObject` (Current state). Worth checking + separately. +- **Git sync** of Markdown files, as a separate brainstorm. + +## Planning clarifications + +These came out of writing the implementation plan +(`docs/superpowers/plans/2026-10-23-page-export-docx-odt.md`). Each one +settles a detail the sections above left open or got slightly wrong. + +- **`knot-export` dependencies.** It depends on `yrs` directly and takes + `&yrs::Doc`, not `knot-crdt`, which would pull in tokio, sqlx and S3. The + server passes the snapshot handle's inner doc. +- **Mermaid config.** Mermaid resets its config on every `render()`, so export + renders append a per-render directive + `%%{init: {"htmlLabels": false}}%%` after the source. No re-initialise or + restore is needed (§8). +- **Mermaid keys** hash the code block's plain text content (the concatenated + text runs), matching the browser's `textContent`. +- **Attribute types.** The editor stores numeric attributes as numbers and + marks as `{}` maps. Markdown import stores strings. The walker accepts both, + and treats a mark as present by its key, whatever the value. +- **Column widths** port both `resolveShares` and `readColumnWeights` (first + grid row, colspan-aware). +- **Multipart parsing.** `multer` can't recover from a per-field limit, so only + the 10 MiB whole-body limit is enforced by multer (413). The 2 MiB + per-picture cap and the 100-part cap are enforced by hand, and those parts + fall back. An empty body, or a non-multipart content type, means "no + pictures". +- **Unsaved edits** count as present while the editor isn't connected, as + well as while `pendingBytes > 0`. Offline edits don't show up in + `pendingBytes`. +- **Markdown items** in the menu are hidden on template pages, because the + existing zip export refuses templates (out of scope to fix). +- **File naming.** The browser names the downloaded file itself (a `blob:` + download ignores `Content-Disposition`). The server still sends + `filename*` for API users. Both apply the same rule. +- **Busy wait.** The concurrency permit count and the busy wait live in + `AppState`, so tests can shorten them. +- **`export-check`**: + - connects without running migrations (it must change nothing); + - loads page state from snapshots and updates, without rooms; + - runs offline: images are not fetched and count as unavailable. Its job + is to catch content that breaks the writers. +- **Board previews.** The browser treats any non-200 board preview response + as "no picture". A board copied from a page the reader can't access + answers 403 or 404. +- **e2e unzip.** The e2e suite reads the downloaded zip with a small + zlib-based helper instead of a new dependency. +- **Memory and CPU safety.** + - WebP images are refused above 4096×4096 pixels. + - The image budget also counts the size after conversion, since a + converted WebP can grow. + - Image preparation and the walk-and-write step run on blocking threads, + so the single-CPU pod's live rooms keep running. +- **Fallback header.** It also carries `unknown=` (the count of unknown + elements), which the client doesn't show. diff --git a/docs/superpowers/specs/2026-10-23-page-export-mockup.html b/docs/superpowers/specs/2026-10-23-page-export-mockup.html new file mode 100644 index 00000000..16ec37e3 --- /dev/null +++ b/docs/superpowers/specs/2026-10-23-page-export-mockup.html @@ -0,0 +1,227 @@ + + + + + +knot Page Export + + + +
+
+

Exporting a page to Word or LibreOffice

+

Sketch for discussion: where the export lives, and what a colleague sees when they open the file. Styling is deliberately neutral; company templates come later.

+
+ +
+

1 · The download button becomes a small menu

+

Same place as today's export icon in the page header. Markdown export stays as the third entry.

+
+
+ Team handbook / Q4 planning + + ⇔✎⟲M↓ + ⤓ +
+
Q4 planning
+
+ +
+
+ +
+

2 · What the file looks like when opened

+

Real Word headings, so the navigation pane and a table of contents work. Diagrams arrive as pictures.

+
+
Q4 planning.docxWord
+
+
+
Navigation
+
Goals
+
Milestones
+
Process
+
Open questions
+
+
+
+
Q4 planning
+
Exported from knot · 23 Oct 2026
+ +

Goals

+

Ship the export feature before the planning review on 14 Nov 2026. Owner: @Christian Hüning. Background in Team handbook.

+
    +
  • Agree scope with the platform team
    • +
  • Draft the migration guide
    • +
  • Book the review slot
    • +
+ +

Milestones

+ + + + + +
WhenWhatWho
Week 44Spec signed offPlatform
Week 46Beta on staging
  • DOCX first
  • ODT second
Platform, QA
+ +

Process

+
+ + + Draft + + + + Review + + + + Publish + +
Mermaid diagram, exported as a picture
+
+ +

Open questions

+ 
knot export --format docx --page q4-planning
+

Attachment: budget-2027.xlsx (48 KB)

+ +
Q4 planningPage 1 of 2
+
+
+
+
+
+ +
+

3 · How knot-specific things come out

+
    +
  • MentionsPlain text with the person's name, e.g. "@Christian Hüning".
    • +
  • DatesWritten out as a date, e.g. "14 Nov 2026".
    • +
  • Links to pagesClickable links back to the page in knot (colleagues need access to open them).
    • +
  • Checklists☐ / ☑ characters, which look the same in Word and LibreOffice.
    • +
  • TablesKeep their column widths, merged cells, and lists or images inside cells.
    • +
  • DiagramsMermaid diagrams and boards as pictures; if one can't be drawn, its source or name appears instead.
    • +
  • AttachmentsFile name and size, linking back to knot.
    • +
  • CommentsLeft out.
    • +
  • PaperA4, page numbers in the footer.
    • +
+
+ +
Reply in the chat with what's wrong or missing.
+
diff --git a/e2e/flows/export.spec.ts b/e2e/flows/export.spec.ts new file mode 100644 index 00000000..5c49e021 --- /dev/null +++ b/e2e/flows/export.spec.ts @@ -0,0 +1,124 @@ +// Page export to Word and OpenDocument: the header menu renders the Mermaid +// diagram in the browser, posts it with the export request, and saves the +// server's file. The download is read back as a zip to check the document +// body, the heading style and the diagram's picture part. +import { readFileSync } from "node:fs"; + +import { expect, test, type Page } from "@playwright/test"; + +import { reset } from "../support/reset"; +import { zipEntries } from "../support/zip"; + +const EMAIL = "o@e.com"; +const PASSWORD = "owner-hunter22"; +const TITLE = "Export test"; + +const MARKDOWN = `# Export test + +- [x] done +- [ ] todo + +| A | B | +|---|---| +| 1 | 2 | + +\`\`\`mermaid +graph LR; A[Collect the requirements] --> B[Draft the design] --> C[Review it with the team] --> D[Build it] --> E[Ship it] +\`\`\` +`; + +/** The width of a PNG, from its IHDR chunk. */ +function pngWidth(png: Buffer | undefined): number { + if (!png || png.subarray(12, 16).toString("latin1") !== "IHDR") throw new Error("not a PNG"); + return png.readUInt32BE(16); +} + +/** + * The wide flowchart at its on-screen size, rendered at 2×: well over 600 px + * wide. Sized from the browser's default 300×150 box instead, it came out + * 600×32 px and unreadable. + */ +const MIN_DIAGRAM_PNG_WIDTH = 1200; + +test.beforeAll(async () => { + reset(); + const r = await fetch("http://localhost:3000/auth/setup", { + method: "POST", + headers: { "content-type": "application/json" }, + body: JSON.stringify({ email: EMAIL, display_name: "O", password: PASSWORD }), + }); + if (!r.ok) throw new Error(`setup failed: ${r.status} ${await r.text()}`); +}); + +async function signIn(page: Page) { + await page.goto("/login"); + await page.getByTestId("login-email").fill(EMAIL); + await page.getByTestId("login-password").fill(PASSWORD); + await page.getByTestId("login-submit").click(); + await page.waitForURL(/\/(?:doc\/.+)?$/); +} + +async function writeHeaders(page: Page) { + const csrf = (await page.context().cookies()).find((c) => c.name === "csrf"); + if (!csrf) throw new Error("csrf cookie not found"); + return { "X-CSRF-Token": csrf.value }; +} + +/** Creates the page and fills it; the Markdown import doesn't set the title. */ +async function seedPage(page: Page): Promise { + const headers = await writeHeaders(page); + const created = await page.request.post("/api/docs", { headers, data: { title: TITLE } }); + expect(created.status()).toBe(201); + const id = (await created.json()).id as string; + const imported = await page.request.post(`/api/docs/${id}/markdown?mode=replace`, { + headers: { ...headers, "Content-Type": "text/markdown" }, + data: MARKDOWN, + }); + expect(imported.status()).toBe(204); + return id; +} + +/** Signs in, seeds the page, exports it from the header menu and unzips the download. */ +async function exportPage(page: Page, item: "export-docx" | "export-odt") { + await signIn(page); + const id = await seedPage(page); + await page.goto(`/doc/${id}`); + await expect(page.getByTestId("status-dot")).toHaveAttribute("data-status", "connected", { + timeout: 10_000, + }); + + await page.getByTestId("doc-export").click(); + const downloaded = page.waitForEvent("download"); + await page.getByTestId(item).click(); + const download = await downloaded; + const path = await download.path(); + return { name: download.suggestedFilename(), entries: zipEntries(readFileSync(path)) }; +} + +test("export: a page downloads as a Word document with its diagram", async ({ page }) => { + const { name, entries } = await exportPage(page, "export-docx"); + + expect(name).toBe("Export test.docx"); + const body = entries.get("word/document.xml")?.toString("utf8"); + expect(body).toBeDefined(); + expect(body).toContain(TITLE); + expect(body).toContain('w:val="Heading1"'); + expect([...entries.keys()]).toContain("word/media/image1.png"); + expect(pngWidth(entries.get("word/media/image1.png"))).toBeGreaterThanOrEqual(MIN_DIAGRAM_PNG_WIDTH); + // Placed at least 11 cm wide (it is capped at the 16 cm text width). + const extent = //.exec(body ?? ""); + expect(Number(extent?.[1])).toBeGreaterThanOrEqual(4_000_000); +}); + +test("export: a page downloads as an OpenDocument text with its diagram", async ({ page }) => { + const { name, entries } = await exportPage(page, "export-odt"); + + expect(name).toBe("Export test.odt"); + const body = entries.get("content.xml")?.toString("utf8"); + expect(body).toBeDefined(); + expect(body).toContain(TITLE); + expect(body).toContain('text:outline-level="1"'); + const picture = [...entries.keys()].find((k) => /^Pictures\/.+\.png$/.test(k)); + expect(picture).toBeDefined(); + expect(pngWidth(entries.get(picture ?? ""))).toBeGreaterThanOrEqual(MIN_DIAGRAM_PNG_WIDTH); +}); diff --git a/e2e/flows/mobile.spec.ts b/e2e/flows/mobile.spec.ts index 5c9f6c40..d4788145 100644 --- a/e2e/flows/mobile.spec.ts +++ b/e2e/flows/mobile.spec.ts @@ -81,4 +81,17 @@ test("mobile: the document header fits the viewport", async ({ page }) => { for (const id of ["toggle-edit-mode", "toggle-markdown", "doc-export", "toggle-template", "open-comments"]) { await expect(page.getByTestId(id)).toBeInViewport(); } + + // The wrapped row puts the export trigger near the left edge; its menu + // still opens fully on screen. The drawer, still open over the left of + // the page, goes first. + const backdrop = page.getByTestId("sidebar-backdrop"); + if ((await backdrop.count()) > 0) await backdrop.click({ position: { x: 350, y: 300 } }); + await expect(backdrop).toHaveCount(0); + await page.getByTestId("doc-export").click(); + for (const id of ["export-docx", "export-odt", "export-md", "export-md-subtree"]) { + await expect(page.getByTestId(id)).toBeInViewport({ ratio: 1 }); + } + await page.keyboard.press("Escape"); + await expect(page.getByRole("menu")).toHaveCount(0); }); diff --git a/e2e/support/zip.ts b/e2e/support/zip.ts new file mode 100644 index 00000000..5d76f6f9 --- /dev/null +++ b/e2e/support/zip.ts @@ -0,0 +1,53 @@ +import { inflateRawSync } from "node:zlib"; + +const EOCD_SIG = 0x06054b50; +const CENTRAL_SIG = 0x02014b50; +const LOCAL_SIG = 0x04034b50; +const EOCD_MIN = 22; +const STORED = 0; +const DEFLATE = 8; + +/** + * Read a zip archive into name → bytes, without a zip dependency. + * + * Walks the central directory (the authoritative entry list), then reads each + * entry's data after its local header. Handles the two methods the exporters + * write: stored (returned as is) and deflate (inflated with zlib). Zip64 and + * other methods throw, since the exporters never produce them. + */ +export function zipEntries(buf: Buffer): Map { + const eocd = findEocd(buf); + const count = buf.readUInt16LE(eocd + 10); + let p = buf.readUInt32LE(eocd + 16); + const out = new Map(); + for (let i = 0; i < count; i++) { + if (buf.readUInt32LE(p) !== CENTRAL_SIG) throw new Error(`zip: bad central header at ${p}`); + const method = buf.readUInt16LE(p + 10); + const compressed = buf.readUInt32LE(p + 20); + const nameLen = buf.readUInt16LE(p + 28); + const extraLen = buf.readUInt16LE(p + 30); + const commentLen = buf.readUInt16LE(p + 32); + const local = buf.readUInt32LE(p + 42); + const name = buf.toString("utf8", p + 46, p + 46 + nameLen); + if (compressed === 0xffffffff || local === 0xffffffff) throw new Error(`zip: zip64 entry ${name}`); + + if (buf.readUInt32LE(local) !== LOCAL_SIG) throw new Error(`zip: bad local header for ${name}`); + const start = local + 30 + buf.readUInt16LE(local + 26) + buf.readUInt16LE(local + 28); + const data = buf.subarray(start, start + compressed); + if (method === STORED) out.set(name, Buffer.from(data)); + else if (method === DEFLATE) out.set(name, inflateRawSync(data)); + else throw new Error(`zip: unsupported method ${method} for ${name}`); + + p += 46 + nameLen + extraLen + commentLen; + } + return out; +} + +/** The end-of-central-directory record sits at the end, before a comment of up to 64 KiB. */ +function findEocd(buf: Buffer): number { + const floor = Math.max(0, buf.length - EOCD_MIN - 0xffff); + for (let i = buf.length - EOCD_MIN; i >= floor; i--) { + if (buf.readUInt32LE(i) === EOCD_SIG) return i; + } + throw new Error("zip: end of central directory not found"); +} diff --git a/web/src/features/docs/DocPage.archived.test.tsx b/web/src/features/docs/DocPage.archived.test.tsx index 5d074dd5..094f9638 100644 --- a/web/src/features/docs/DocPage.archived.test.tsx +++ b/web/src/features/docs/DocPage.archived.test.tsx @@ -40,7 +40,7 @@ function json(body: unknown, status = 200) { ); } -type Role = "owner" | "editor"; +type Role = "owner" | "editor" | "viewer"; function docBody(id: string, title: string, role: Role, archive: Record | null) { return { @@ -209,3 +209,12 @@ describe("DocPage — revoked connection", () => { expect(await screen.findByText("home")).toBeInTheDocument(); }); }); + +describe("DocPage — export menu", () => { + it("readers see the export menu", async () => { + vi.stubGlobal("fetch", server("viewer", { d1: [docBody("d1", "Runbook", "viewer", null)] })); + renderDoc("d1"); + + expect(await screen.findByTestId("doc-export")).toHaveAttribute("aria-haspopup", "menu"); + }); +}); diff --git a/web/src/features/docs/DocPage.tsx b/web/src/features/docs/DocPage.tsx index 2903cf52..8f8dd606 100644 --- a/web/src/features/docs/DocPage.tsx +++ b/web/src/features/docs/DocPage.tsx @@ -1,5 +1,5 @@ import { useMutation, useQuery, useQueryClient } from "@tanstack/react-query"; -import { Download, Eye, FileCode, History, LayoutTemplate, MessageSquare, Pencil, Share2 } from "lucide-react"; +import { Eye, FileCode, History, LayoutTemplate, MessageSquare, Pencil, Share2 } from "lucide-react"; import { lazy, Suspense, useEffect, useRef, useState, type CSSProperties } from "react"; import { Link, Outlet, useNavigate, useParams } from "react-router-dom"; @@ -10,6 +10,7 @@ import { useUi } from "../../stores/ui"; import { CommentSidebar } from "../comments/CommentSidebar"; import { MarkdownView } from "../editor/MarkdownView"; +import { ExportMenu } from "../export/ExportMenu"; import { purgeConfirmText } from "./archive"; import { ArchivedBanner } from "./ArchivedBanner"; import { Breadcrumb } from "./Breadcrumb"; @@ -323,24 +324,15 @@ export default function DocPage() { {canEditDoc && ( )} - {/* The zip export reads live docs only; restore to export. */} + {/* Export reads live docs only; restore to export. */} {!archived && ( - { - // Default: just this doc. Shift-click pulls the whole - // subtree — a power-user shortcut without cluttering the - // header with a separate button. - const descendants = e.shiftKey; - const a = document.createElement("a"); - a.href = `/api/docs/${id}/export?descendants=${descendants ? "true" : "false"}`; - a.download = `${meta.title || "doc"}.zip`; - a.click(); - }} - > - - + )} {isDocOwner && ( ) => void; - render: (id: string, src: string) => Promise<{ svg: string }>; -}; - -let cached: MermaidApi | null = null; -async function getMermaid(): Promise { - if (cached) return cached; - const mod = await import("mermaid"); - const m = (mod.default ?? mod) as MermaidApi; - m.initialize({ - startOnLoad: false, - securityLevel: "strict", - theme: "default", - fontFamily: "'Inter Variable', Inter, system-ui, sans-serif", - }); - cached = m; - return m; -} - function MermaidNodeView(props: ReactNodeViewProps) { const { node } = props; const lang = (node.attrs.language as string | null) ?? ""; diff --git a/web/src/features/export/ExportMenu.test.tsx b/web/src/features/export/ExportMenu.test.tsx new file mode 100644 index 00000000..49414cb9 --- /dev/null +++ b/web/src/features/export/ExportMenu.test.tsx @@ -0,0 +1,162 @@ +import { cleanup, fireEvent, render, screen } from "@testing-library/react"; +import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; + +const hook = vi.hoisted(() => ({ exporting: false, run: vi.fn() })); +vi.mock("./useExport", () => ({ + useExport: () => ({ exporting: hook.exporting, run: hook.run }), +})); + +import { ExportMenu } from "./ExportMenu"; + +beforeEach(() => { + hook.exporting = false; + hook.run.mockReset(); +}); +afterEach(cleanup); + +function mount(isTemplate = false) { + render( + , + ); +} + +function open() { + fireEvent.click(screen.getByTestId("doc-export")); +} + +describe("ExportMenu", () => { + it("opens a menu with Word, OpenDocument and both Markdown items", () => { + mount(); + const trigger = screen.getByTestId("doc-export"); + expect(trigger).toHaveAttribute("aria-haspopup", "menu"); + expect(trigger).toHaveAttribute("aria-expanded", "false"); + expect(screen.queryByRole("menu")).toBeNull(); + open(); + expect(trigger).toHaveAttribute("aria-expanded", "true"); + expect(screen.getByRole("menu")).toBeInTheDocument(); + expect(screen.getAllByRole("menuitem")).toHaveLength(4); + expect(screen.getByRole("separator")).toBeInTheDocument(); + expect(screen.getByTestId("export-docx")).toHaveTextContent("Word document"); + expect(screen.getByTestId("export-docx")).toHaveTextContent( + ".docx · opens in Word, LibreOffice, Google Docs", + ); + expect(screen.getByTestId("export-odt")).toHaveTextContent("OpenDocument text"); + expect(screen.getByTestId("export-odt")).toHaveTextContent(".odt · LibreOffice's own format"); + expect(screen.getByTestId("export-md")).toHaveTextContent("Markdown (zip)"); + expect(screen.getByTestId("export-md-subtree")).toHaveTextContent("Markdown with subpages (zip)"); + }); + + it("hides the Markdown items on template pages", () => { + mount(true); + open(); + expect(screen.getAllByRole("menuitem")).toHaveLength(2); + expect(screen.queryByRole("separator")).toBeNull(); + expect(screen.queryByTestId("export-md")).toBeNull(); + expect(screen.queryByTestId("export-md-subtree")).toBeNull(); + }); + + it("supports arrow keys, Home/End and Escape returning focus to the trigger", () => { + mount(); + open(); + const items = screen.getAllByRole("menuitem"); + expect(items[0]).toHaveFocus(); + fireEvent.keyDown(items[0]!, { key: "ArrowDown" }); + expect(items[1]).toHaveFocus(); + fireEvent.keyDown(items[1]!, { key: "End" }); + expect(items[3]).toHaveFocus(); + fireEvent.keyDown(items[3]!, { key: "ArrowDown" }); + expect(items[0]).toHaveFocus(); + fireEvent.keyDown(items[0]!, { key: "ArrowUp" }); + expect(items[3]).toHaveFocus(); + fireEvent.keyDown(items[3]!, { key: "Home" }); + expect(items[0]).toHaveFocus(); + fireEvent.keyDown(items[0]!, { key: "Escape" }); + expect(screen.queryByRole("menu")).toBeNull(); + expect(screen.getByTestId("doc-export")).toHaveFocus(); + }); + + describe("at phone width", () => { + /** Lays the menu out at `left`..`right` in a 375 px wide viewport. */ + function layOut(left: number, right: number) { + vi.spyOn(document.documentElement, "clientWidth", "get").mockReturnValue(375); + vi.spyOn(HTMLElement.prototype, "getBoundingClientRect").mockImplementation(function (this: HTMLElement) { + const [l, r] = this.getAttribute("role") === "menu" ? [left, right] : [0, 0]; + return { left: l, right: r, width: r - l, top: 0, bottom: 0, height: 0, x: l, y: 0, toJSON: () => ({}) }; + }); + } + afterEach(() => vi.restoreAllMocks()); + + it("shifts a menu that would open past the left edge back into view", () => { + // A trigger near the left edge of a wrapped header row. + layOut(-190, 100); + mount(); + open(); + expect(screen.getByRole("menu").style.transform).toBe("translateX(206px)"); + }); + + it("shifts a menu that would open past the right edge back into view", () => { + layOut(300, 590); + mount(); + open(); + expect(screen.getByRole("menu").style.transform).toBe("translateX(-231px)"); + }); + + it("leaves a menu that fits where it is", () => { + layOut(40, 330); + mount(); + open(); + expect(screen.getByRole("menu").style.transform).toBe(""); + }); + }); + + it("closes on an outside mousedown", () => { + mount(); + open(); + fireEvent.mouseDown(document.body); + expect(screen.queryByRole("menu")).toBeNull(); + }); + + it("disables export items and spins the icon while exporting", () => { + hook.exporting = true; + mount(); + open(); + expect(screen.getByTestId("export-docx")).toBeDisabled(); + expect(screen.getByTestId("export-odt")).toBeDisabled(); + expect(screen.getByTestId("doc-export").querySelector(".animate-spin")).not.toBeNull(); + }); + + it("runs docx export on select and closes the menu", () => { + mount(); + open(); + fireEvent.click(screen.getByTestId("export-docx")); + expect(hook.run).toHaveBeenCalledWith("docx"); + expect(screen.queryByRole("menu")).toBeNull(); + }); + + it("runs odt export on select", () => { + mount(); + open(); + fireEvent.click(screen.getByTestId("export-odt")); + expect(hook.run).toHaveBeenCalledWith("odt"); + }); + + it("downloads Markdown zips with and without subpages", () => { + const hrefs: string[] = []; + const click = vi + .spyOn(HTMLAnchorElement.prototype, "click") + .mockImplementation(function (this: HTMLAnchorElement) { + hrefs.push(`${this.getAttribute("href")}|${this.download}`); + }); + mount(); + open(); + fireEvent.click(screen.getByTestId("export-md")); + expect(screen.queryByRole("menu")).toBeNull(); + open(); + fireEvent.click(screen.getByTestId("export-md-subtree")); + expect(hrefs).toEqual([ + "/api/docs/d1/export?descendants=false|Runbook.zip", + "/api/docs/d1/export?descendants=true|Runbook.zip", + ]); + click.mockRestore(); + }); +}); diff --git a/web/src/features/export/ExportMenu.tsx b/web/src/features/export/ExportMenu.tsx new file mode 100644 index 00000000..0b9d8566 --- /dev/null +++ b/web/src/features/export/ExportMenu.tsx @@ -0,0 +1,174 @@ +import { Download, LoaderCircle } from "lucide-react"; +import { useEffect, useId, useLayoutEffect, useRef, useState, type KeyboardEvent } from "react"; + +import type { ConnStatus } from "../../components/StatusDot"; +import { IconButton } from "../../components/ui/IconButton"; +import { useExport } from "./useExport"; + +type Props = { + docId: string; + title: string; + isTemplate: boolean; + status: ConnStatus; + pendingBytes: number; +}; + +const ITEM_CLS = + "flex w-full flex-col items-start gap-0.5 px-3 py-1.5 text-left text-[13px] text-fg hover:bg-muted focus-visible:bg-muted focus-visible:outline-none disabled:opacity-40 disabled:cursor-not-allowed"; +/** Space the menu keeps from the viewport's edges, in px (as its `max-w`). */ +const VIEWPORT_GUTTER_PX = 16; + +/** Download-as menu in the page header: Word, OpenDocument, Markdown zips. */ +export function ExportMenu({ docId, title, isTemplate, status, pendingBytes }: Props) { + const [open, setOpen] = useState(false); + const buttonRef = useRef(null); + const menuRef = useRef(null); + const menuId = useId(); + const { exporting, run } = useExport({ docId, title, status, pendingBytes }); + + const enabledItems = () => + Array.from(menuRef.current?.querySelectorAll('[role="menuitem"]:not(:disabled)') ?? []); + + useEffect(() => { + if (!open) return; + enabledItems()[0]?.focus(); + const onDown = (e: MouseEvent) => { + const t = e.target as Node; + // The trigger is excluded so its own click can toggle the menu shut; + // otherwise mousedown would close it and the click would reopen it. + if (menuRef.current?.contains(t) || buttonRef.current?.contains(t)) return; + setOpen(false); + }; + const onKey = (e: globalThis.KeyboardEvent) => { + if (e.key !== "Escape") return; + setOpen(false); + buttonRef.current?.focus(); + }; + document.addEventListener("mousedown", onDown); + document.addEventListener("keydown", onKey); + return () => { + document.removeEventListener("mousedown", onDown); + document.removeEventListener("keydown", onKey); + }; + }, [open]); + + // The menu hangs from the trigger's right edge. At phone width the header + // row wraps and can put the trigger near the left edge, so shift the menu + // back into the viewport before it paints. Each open renders a new menu. + useLayoutEffect(() => { + const menu = menuRef.current; + if (!open || !menu) return; + const { left, right, width } = menu.getBoundingClientRect(); + if (width === 0) return; + const max = document.documentElement.clientWidth - VIEWPORT_GUTTER_PX; + const dx = left < VIEWPORT_GUTTER_PX ? VIEWPORT_GUTTER_PX - left : right > max ? max - right : 0; + if (dx !== 0) menu.style.transform = `translateX(${dx}px)`; + }, [open]); + + function onMenuKeyDown(e: KeyboardEvent) { + if (e.key === "Tab") { + setOpen(false); + return; + } + const items = enabledItems(); + if (items.length === 0) return; + const i = items.indexOf(document.activeElement as HTMLElement); + let next: number; + if (e.key === "ArrowDown") next = i < 0 ? 0 : (i + 1) % items.length; + else if (e.key === "ArrowUp") next = i <= 0 ? items.length - 1 : i - 1; + else if (e.key === "Home") next = 0; + else if (e.key === "End") next = items.length - 1; + else return; + e.preventDefault(); + items[next]?.focus(); + } + + function choose(action: () => void) { + setOpen(false); + buttonRef.current?.focus(); + action(); + } + + function downloadZip(descendants: boolean) { + const a = document.createElement("a"); + a.href = `/api/docs/${docId}/export?descendants=${descendants ? "true" : "false"}`; + a.download = `${title || "doc"}.zip`; + a.click(); + } + + return ( +
+ setOpen((v) => !v)} + > + {exporting ? ( + + ) : ( + + )} + + {open && ( + + ); +} diff --git a/web/src/features/export/export.api.test.ts b/web/src/features/export/export.api.test.ts new file mode 100644 index 00000000..c380dbf9 --- /dev/null +++ b/web/src/features/export/export.api.test.ts @@ -0,0 +1,65 @@ +import { afterEach, describe, expect, it, vi } from "vitest"; + +import { exportApi } from "./export.api"; + +afterEach(() => { + vi.restoreAllMocks(); + document.cookie = "csrf=; expires=Thu, 01 Jan 1970 00:00:00 GMT"; +}); + +describe("exportApi.file", () => { + it("posts multipart with CSRF and returns blob + fallbacks", async () => { + document.cookie = "csrf=tok123"; + const spy = vi.spyOn(globalThis, "fetch").mockResolvedValue( + new Response("DOCX", { + status: 200, + headers: { "X-Knot-Export-Fallbacks": "diagrams=2, images=1, unknown=0" }, + }), + ); + const r = await exportApi.file("d 1", "docx", new Map([["m1", new Blob(["png"])]])); + expect(spy).toHaveBeenCalledTimes(1); + const [url, init] = spy.mock.calls[0]!; + expect(url).toBe("/api/docs/d%201/export/docx"); + expect(init?.method).toBe("POST"); + expect(init?.credentials).toBe("include"); + expect((init?.headers as Record)["X-CSRF-Token"]).toBe("tok123"); + const fd = init?.body as FormData; + expect((fd.get("m1") as File).name).toBe("m1.png"); + if (!("ok" in r)) throw new Error("expected ok"); + expect(r.ok.fallbacks).toEqual({ diagrams: 2, images: 1 }); + expect(await r.ok.blob.text()).toBe("DOCX"); + }); + + it("maps the error envelope", async () => { + vi.spyOn(globalThis, "fetch").mockResolvedValue( + new Response(JSON.stringify({ error: { code: "export.busy", message: "Busy" } }), { + status: 503, + }), + ); + const r = await exportApi.file("d", "odt", new Map()); + expect(r).toMatchObject({ error: { code: "export.busy", message: "Busy", status: 503 } }); + }); + + it("maps network errors", async () => { + vi.spyOn(globalThis, "fetch").mockRejectedValue(new TypeError("down")); + const r = await exportApi.file("d", "odt", new Map()); + expect(r).toMatchObject({ error: { code: "network", status: 0 } }); + }); +}); + +describe("exportApi.diagrams", () => { + it("parses the list", async () => { + vi.spyOn(globalThis, "fetch").mockResolvedValue( + new Response( + JSON.stringify({ + limits: { max_body_bytes: 1, max_part_bytes: 2, max_parts: 3, scale: 2 }, + diagrams: [{ kind: "mermaid", key: "m1", source: "graph TD" }], + }), + { status: 200, headers: { "Content-Type": "application/json" } }, + ), + ); + const r = await exportApi.diagrams("d"); + if (!("ok" in r)) throw new Error("expected ok"); + expect(r.ok.diagrams).toHaveLength(1); + }); +}); diff --git a/web/src/features/export/export.api.ts b/web/src/features/export/export.api.ts new file mode 100644 index 00000000..4261c33f --- /dev/null +++ b/web/src/features/export/export.api.ts @@ -0,0 +1,64 @@ +import { type ApiError, type ApiResult, apiFetch } from "../../lib/api"; +import { readCookie } from "../../lib/csrf"; +import { DiagramList, parse } from "../../lib/validators"; +import { type Fallbacks, parseFallbacks } from "./exportFile"; +import type { DiagramLimits, DiagramRef } from "./renderDiagrams"; + +export type { Fallbacks }; + +function failure(code: string, message: string, status: number, details = {}): { error: ApiError } { + return { error: { code, message, details, status } }; +} + +export const exportApi = { + async diagrams(docId: string): Promise> { + const r = await apiFetch(`/api/docs/${encodeURIComponent(docId)}/export/diagrams`); + if ("error" in r) return r; + return { ok: parse(DiagramList, r.ok) }; + }, + + /** apiFetch cannot return a blob or response headers, so this is a plain fetch. */ + async file( + docId: string, + format: "docx" | "odt", + parts: Map, + ): Promise> { + const fd = new FormData(); + for (const [key, blob] of parts) fd.append(key, blob, `${key}.png`); + const headers: Record = {}; + const csrf = readCookie("csrf"); + if (csrf) headers["X-CSRF-Token"] = csrf; + let res: Response; + try { + res = await fetch(`/api/docs/${encodeURIComponent(docId)}/export/${format}`, { + method: "POST", + credentials: "include", + headers, + body: fd, + }); + } catch { + return failure("network", "Network error", 0); + } + if (!res.ok) { + const text = await res.text().catch(() => ""); + try { + const env = JSON.parse(text) as { error?: Partial }; + return failure( + env.error?.code ?? "http_error", + env.error?.message ?? `HTTP ${res.status}`, + res.status, + env.error?.details ?? {}, + ); + } catch { + return failure("http_error", `HTTP ${res.status}`, res.status); + } + } + let blob: Blob; + try { + blob = await res.blob(); + } catch { + return failure("network", "Network error", 0); + } + return { ok: { blob, fallbacks: parseFallbacks(res.headers.get("X-Knot-Export-Fallbacks")) } }; + }, +}; diff --git a/web/src/features/export/exportFile.test.ts b/web/src/features/export/exportFile.test.ts new file mode 100644 index 00000000..eb49ee58 --- /dev/null +++ b/web/src/features/export/exportFile.test.ts @@ -0,0 +1,40 @@ +import { describe, expect, it } from "vitest"; + +import { exportFileName, fallbackMessage, parseFallbacks } from "./exportFile"; + +describe("exportFileName", () => { + it("follows the shared rule", () => { + expect(exportFileName("Q4 / Plan: ?", "docx")).toBe("Q4 - Plan- -draft--.docx"); + expect(exportFileName(" .. ", "odt")).toBe("Untitled.odt"); + expect(exportFileName("x".repeat(200), "docx")).toHaveLength(125); + expect(exportFileName("Über 😀", "docx")).toBe("Über 😀.docx"); + expect(exportFileName("a".repeat(119) + "😀😀", "docx")).toBe("a".repeat(119) + "😀.docx"); + expect(exportFileName("a\u0001b\u009fc", "odt")).toBe("a-b-c.odt"); + expect(exportFileName(" Notes. ", "odt")).toBe("Notes.odt"); + }); +}); + +describe("parseFallbacks", () => { + it("reads diagrams and images, ignores the rest", () => { + expect(parseFallbacks("diagrams=2, images=1, unknown=3")).toEqual({ diagrams: 2, images: 1 }); + }); + it("returns zeros for null and garbage", () => { + expect(parseFallbacks(null)).toEqual({ diagrams: 0, images: 0 }); + expect(parseFallbacks("nonsense, diagrams=x")).toEqual({ diagrams: 0, images: 0 }); + }); +}); + +describe("fallbackMessage", () => { + it("builds the toast copy", () => { + expect(fallbackMessage({ diagrams: 2, images: 1 }, false)).toBe( + "Exported. 2 diagrams and 1 image couldn't be included.", + ); + expect(fallbackMessage({ diagrams: 1, images: 0 }, false)).toBe( + "Exported. 1 diagram couldn't be included.", + ); + expect(fallbackMessage({ diagrams: 0, images: 0 }, true)).toBe( + "Exported. Your latest edits weren't saved yet and may be missing.", + ); + expect(fallbackMessage({ diagrams: 0, images: 0 }, false)).toBeNull(); + }); +}); diff --git a/web/src/features/export/exportFile.ts b/web/src/features/export/exportFile.ts new file mode 100644 index 00000000..c3eae741 --- /dev/null +++ b/web/src/features/export/exportFile.ts @@ -0,0 +1,59 @@ +export type Fallbacks = { diagrams: number; images: number }; + +/** Must match `file_name` in crates/knot-server/src/export.rs. */ +const MAX_STEM_CODE_POINTS = 120; + +function trimDotsAndSpaces(s: string): string { + return s.replace(/^[ .]+|[ .]+$/g, ""); +} + +export function exportFileName(title: string, ext: "docx" | "odt"): string { + const replaced = Array.from(title, (c) => { + if (/[/\\:*?"<>|]/.test(c)) return "-"; + // Unicode Cc (C0, DEL, C1), as Rust's char::is_control. + if (/\p{Cc}/u.test(c)) return "-"; + return c; + }).join(""); + const capped = Array.from(trimDotsAndSpaces(replaced)).slice(0, MAX_STEM_CODE_POINTS).join(""); + const stem = trimDotsAndSpaces(capped) || "Untitled"; + return `${stem}.${ext}`; +} + +/** Reads `X-Knot-Export-Fallbacks: diagrams=, images=, unknown=`. */ +export function parseFallbacks(header: string | null): Fallbacks { + const out: Fallbacks = { diagrams: 0, images: 0 }; + if (!header) return out; + for (const part of header.split(",")) { + const m = /^\s*(diagrams|images)=(\d+)\s*$/.exec(part); + if (m) out[m[1] as keyof Fallbacks] = Number(m[2]); + } + return out; +} + +function count(n: number, singular: string): string { + return `${n} ${singular}${n === 1 ? "" : "s"}`; +} + +export function fallbackMessage(f: Fallbacks, unsynced: boolean): string | null { + const parts: string[] = []; + if (f.diagrams > 0) parts.push(count(f.diagrams, "diagram")); + if (f.images > 0) parts.push(count(f.images, "image")); + const messages: string[] = []; + if (parts.length > 0) messages.push(`${parts.join(" and ")} couldn't be included.`); + if (unsynced) messages.push("Your latest edits weren't saved yet and may be missing."); + if (messages.length === 0) return null; + return `Exported. ${messages.join(" ")}`; +} + +export function saveBlob(blob: Blob, fileName: string): void { + const url = URL.createObjectURL(blob); + const a = document.createElement("a"); + a.href = url; + a.download = fileName; + a.style.display = "none"; + document.body.appendChild(a); + a.click(); + a.remove(); + // Revoke after the click has been handled by the browser. + setTimeout(() => URL.revokeObjectURL(url), 0); +} diff --git a/web/src/features/export/renderDiagrams.test.ts b/web/src/features/export/renderDiagrams.test.ts new file mode 100644 index 00000000..7e2cf6ef --- /dev/null +++ b/web/src/features/export/renderDiagrams.test.ts @@ -0,0 +1,177 @@ +import { afterEach, describe, expect, it, vi } from "vitest"; + +import { + renderDiagrams, + sizeSvg, + type DiagramLimits, + type DiagramRef, + type RenderDeps, + type SizedSvg, +} from "./renderDiagrams"; + +const limits: DiagramLimits = { max_body_bytes: 10_485_760, max_part_bytes: 2_097_152, max_parts: 100, scale: 2 }; +const png = (n: number) => new Blob([new Uint8Array(n)], { type: "image/png" }); +const mer = (key: string): DiagramRef => ({ key, kind: "mermaid", source: `graph TD; ${key}` }); +const board = (key: string): DiagramRef => ({ key, kind: "board", board_id: `b-${key}`, label: key }); +const SVG_NS = "http://www.w3.org/2000/svg"; +/** An SVG shaped like Mermaid's default output: relative width, a max-width, a viewBox, no height. */ +const mermaidSvg = (w: number, h: number, body = "") => + `${body}`; + +function deps(over: Partial = {}): RenderDeps { + return { + renderMermaid: (s) => Promise.resolve(mermaidSvg(100, 50, `${s}`)), + fetchBoardSvg: (id) => Promise.resolve(`${id}`), + rasterize: () => Promise.resolve(png(10)), + ...over, + }; +} + +function root(sized: SizedSvg | null): Element { + if (!sized) throw new Error("not sized"); + return new DOMParser().parseFromString(sized.markup, "image/svg+xml").documentElement; +} + +afterEach(() => { + vi.useRealTimers(); +}); + +describe("sizeSvg", () => { + it("sizes Mermaid's width=100% output from its viewBox and drops the max-width", () => { + const sized = sizeSvg(mermaidSvg(1492, 82, "A")); + expect(sized).toMatchObject({ width: 1492, height: 82 }); + const svg = root(sized); + expect(svg.getAttribute("width")).toBe("1492"); + expect(svg.getAttribute("height")).toBe("82"); + expect(svg.getAttribute("viewBox")).toBe("0 0 1492 82"); + expect(svg.getAttribute("style")).toBe("background-color: white"); + expect(svg.querySelector("text")?.textContent).toBe("A"); + }); + + it("keeps a tall diagram's height", () => { + const sized = sizeSvg( + ``, + ); + expect(sized).toMatchObject({ width: 468, height: 1321 }); + expect(root(sized).hasAttribute("style")).toBe(false); + }); + + it("keeps absolute width and height, as a board preview has", () => { + expect(sizeSvg(``)).toMatchObject({ + width: 400, + height: 300, + }); + }); + + it("takes the missing side from the viewBox's aspect ratio", () => { + expect(sizeSvg(``)).toMatchObject({ + width: 200, + height: 100, + }); + expect(sizeSvg(``)).toMatchObject({ + width: 20, + height: 10, + }); + }); + + it("gives up on an SVG without a usable size, or that isn't one", () => { + expect(sizeSvg(``)).toBeNull(); + expect(sizeSvg(``)).toBeNull(); + expect(sizeSvg(``)).toBeNull(); + expect(sizeSvg(``)).toBeNull(); + expect(sizeSvg("not svg")).toBeNull(); + }); +}); + +describe("renderDiagrams", () => { + it("rasterizes each diagram at the size its SVG declares", async () => { + const rasterize = vi.fn((_svg: SizedSvg, _scale: number, _max: number) => Promise.resolve(png(10))); + const d = deps({ renderMermaid: () => Promise.resolve(mermaidSvg(1492, 82)), rasterize }); + await renderDiagrams([mer("m1"), board("b1")], limits, d); + expect(rasterize.mock.calls.map(([svg, scale]) => [svg.width, svg.height, scale])).toEqual([ + [1492, 82, 2], + [80, 40, 2], + ]); + expect(root(rasterize.mock.calls[0]![0]).getAttribute("width")).toBe("1492"); + }); + + it("treats an SVG without a size as no picture", async () => { + const rasterize = vi.fn(() => Promise.resolve(png(10))); + const d = deps({ renderMermaid: () => Promise.resolve(``), rasterize }); + const out = await renderDiagrams([mer("m1"), board("b1")], limits, d); + expect([...out.keys()]).toEqual(["b1"]); + expect(rasterize).toHaveBeenCalledTimes(1); + }); + + it("renders mermaid and boards into keyed PNG blobs", async () => { + const out = await renderDiagrams([mer("m1"), board("b1")], limits, deps()); + expect([...out.keys()]).toEqual(["m1", "b1"]); + expect(out.get("m1")?.type).toBe("image/png"); + }); + + it("skips a diagram whose render throws or times out", async () => { + vi.useFakeTimers(); + const d = deps({ + renderMermaid: (s) => { + if (s.includes("bad")) return Promise.reject(new Error("boom")); + if (s.includes("hang")) return new Promise(() => {}); + return Promise.resolve(mermaidSvg(10, 10)); + }, + }); + const p = renderDiagrams([mer("bad"), mer("hang"), mer("ok")], limits, d); + await vi.advanceTimersByTimeAsync(10_001); + const out = await p; + expect([...out.keys()]).toEqual(["ok"]); + }); + + it("treats a missing board preview as no picture", async () => { + const out = await renderDiagrams([board("b1"), mer("m1")], limits, deps({ fetchBoardSvg: () => Promise.resolve(null) })); + expect([...out.keys()]).toEqual(["m1"]); + }); + + it("re-rasterizes once at half size when a part is too big, then drops it", async () => { + const rasterize = vi.fn((_svg: SizedSvg, _scale: number, _max: number) => Promise.resolve(png(100))); + const out = await renderDiagrams([mer("m1")], { ...limits, max_part_bytes: 50 }, deps({ rasterize })); + expect(out.size).toBe(0); + expect(rasterize.mock.calls.map((c) => c[2])).toEqual([2000, 1000]); + }); + + it("keeps the half-size retry when it fits", async () => { + const rasterize = (_s: SizedSvg, _k: number, max: number) => Promise.resolve(png(max === 2000 ? 100 : 10)); + const out = await renderDiagrams([mer("m1")], { ...limits, max_part_bytes: 50 }, deps({ rasterize })); + expect([...out.keys()]).toEqual(["m1"]); + }); + + it("stops adding parts at the count limit", async () => { + const out = await renderDiagrams([mer("a"), mer("b"), mer("c")], { ...limits, max_parts: 2 }, deps()); + expect([...out.keys()]).toEqual(["a", "b"]); + }); + + it("stops adding parts at the body limit", async () => { + const out = await renderDiagrams( + [mer("a"), mer("b"), mer("c")], + { ...limits, max_body_bytes: 64 * 1024 + 25 }, + deps({ rasterize: () => Promise.resolve(png(10)) }), + ); + expect([...out.keys()]).toEqual(["a", "b"]); + }); + + it("never runs more than three renders at once", async () => { + let active = 0; + let peak = 0; + const renderMermaid = async () => { + active++; + peak = Math.max(peak, active); + await new Promise((r) => setTimeout(r, 5)); + active--; + return mermaidSvg(10, 10); + }; + const out = await renderDiagrams( + Array.from({ length: 8 }, (_, i) => mer(`m${i}`)), + limits, + deps({ renderMermaid }), + ); + expect(out.size).toBe(8); + expect(peak).toBe(3); + }); +}); diff --git a/web/src/features/export/renderDiagrams.ts b/web/src/features/export/renderDiagrams.ts new file mode 100644 index 00000000..96261b19 --- /dev/null +++ b/web/src/features/export/renderDiagrams.ts @@ -0,0 +1,191 @@ +import { boardsApi } from "../../lib/boards.api"; +import { getMermaid } from "../../lib/mermaid"; + +export type DiagramLimits = { + max_body_bytes: number; + max_part_bytes: number; + max_parts: number; + scale: number; +}; + +export type DiagramRef = + | { key: string; kind: "mermaid"; source: string } + | { key: string; kind: "board"; board_id: string; label: string }; + +/** An SVG document with an explicit size in CSS px. */ +export type SizedSvg = { markup: string; width: number; height: number }; + +export type RenderDeps = { + /** Returns SVG markup. */ + renderMermaid: (source: string) => Promise; + /** Returns SVG markup, or null on any non-200. */ + fetchBoardSvg: (boardId: string) => Promise; + /** Returns an image/png blob of `svg` at `scale` times its size, at most `maxSide` px on the long side. */ + rasterize: (svg: SizedSvg, scale: number, maxSide: number) => Promise; +}; + +const MAX_SIDE = 2000; +const CONCURRENCY = 3; +const TIMEOUT_MS = 10_000; +const BODY_HEADROOM = 64 * 1024; +const SVG_NS = "http://www.w3.org/2000/svg"; + +let renderSeq = 0; + +/** An absolute length in CSS px ("120", "120.5px"); null for a relative or other one ("100%", "auto", "3cm"). */ +function pxLength(value: string | null): number | null { + const m = value === null ? null : /^\s*(\d*\.?\d+)\s*(?:px)?\s*$/i.exec(value); + const n = m ? Number(m[1]) : NaN; + return Number.isFinite(n) && n > 0 ? n : null; +} + +/** The width and height of a `viewBox`, when it has a positive size. */ +function viewBoxSize(value: string | null): { width: number; height: number } | null { + const parts = (value ?? "").trim().split(/[\s,]+/).map(Number); + const [, , width = NaN, height = NaN] = parts; + return parts.length === 4 && width > 0 && height > 0 && Number.isFinite(width + height) ? { width, height } : null; +} + +/** + * `svg` with explicit px `width` and `height` on its root, and those sizes. + * + * An `` gives an SVG its natural size only from absolute ones. Mermaid's + * default output (`width="100%"`, a `max-width` style, a viewBox and no + * height) has none: Chromium then draws it into a 300×150 box and Firefox and + * Safari report 0×0, so the diagram comes out unreadably small. Absolute + * width and height win; a missing or relative side comes from the viewBox. + * The `max-width` goes, since it would cap the root against the image's box. + * Null when there is no size to find, or the markup isn't an SVG document. + */ +export function sizeSvg(svg: string): SizedSvg | null { + const doc = new DOMParser().parseFromString(svg, "image/svg+xml"); + const root = doc.documentElement; + if (root.namespaceURI !== SVG_NS || root.localName !== "svg") return null; + if (doc.getElementsByTagName("parsererror").length > 0) return null; + + let width = pxLength(root.getAttribute("width")); + let height = pxLength(root.getAttribute("height")); + if (width === null || height === null) { + const box = viewBoxSize(root.getAttribute("viewBox")); + if (!box) return null; + if (width !== null) height = (width * box.height) / box.width; + else if (height !== null) width = (height * box.width) / box.height; + else ({ width, height } = box); + } + root.setAttribute("width", String(width)); + root.setAttribute("height", String(height)); + const style = root.getAttribute("style"); + if (style !== null) { + const kept = style + .split(";") + .map((d) => d.trim()) + .filter((d) => d && !/^max-width\s*:/i.test(d)) + .join("; "); + if (kept) root.setAttribute("style", kept); + else root.removeAttribute("style"); + } + return { markup: new XMLSerializer().serializeToString(doc), width, height }; +} + +function rasterizeToPng(svg: SizedSvg, scale: number, maxSide: number): Promise { + const url = URL.createObjectURL(new Blob([svg.markup], { type: "image/svg+xml;charset=utf-8" })); + return new Promise((resolve, reject) => { + const img = document.createElement("img"); + img.onload = () => resolve(img); + img.onerror = () => reject(new Error("svg image load failed")); + img.src = url; + }) + .then( + (img) => + new Promise((resolve, reject) => { + // The size sizeSvg found, never the image's natural size (see there). + const k = Math.min(scale, maxSide / Math.max(svg.width, svg.height)); + const canvas = document.createElement("canvas"); + canvas.width = Math.max(1, Math.round(svg.width * k)); + canvas.height = Math.max(1, Math.round(svg.height * k)); + const ctx = canvas.getContext("2d"); + if (!ctx) { + reject(new Error("no 2d context")); + return; + } + ctx.fillStyle = "#ffffff"; + ctx.fillRect(0, 0, canvas.width, canvas.height); + ctx.drawImage(img, 0, 0, canvas.width, canvas.height); + canvas.toBlob((b) => { + if (b) resolve(b); + else reject(new Error("toBlob returned null")); + }, "image/png"); + }), + ) + .finally(() => URL.revokeObjectURL(url)); +} + +export const defaultRenderDeps: RenderDeps = { + async renderMermaid(source) { + const m = await getMermaid(); + const id = `knot-export-mermaid-${++renderSeq}`; + return (await m.render(id, source + '\n%%{init: {"htmlLabels": false}}%%')).svg; + }, + async fetchBoardSvg(boardId) { + const res = await boardsApi.getSvg(boardId); + return "ok" in res ? res.ok : null; + }, + rasterize: rasterizeToPng, +}; + +async function renderOne(d: DiagramRef, limits: DiagramLimits, deps: RenderDeps): Promise { + const markup = d.kind === "mermaid" ? await deps.renderMermaid(d.source) : await deps.fetchBoardSvg(d.board_id); + // Without a size there is nothing legible to draw: let it fall back. + const svg = markup === null ? null : sizeSvg(markup); + if (svg === null) return null; + let png = await deps.rasterize(svg, limits.scale, MAX_SIDE); + if (png.size > limits.max_part_bytes) { + png = await deps.rasterize(svg, limits.scale, MAX_SIDE / 2); + if (png.size > limits.max_part_bytes) return null; + } + return png; +} + +async function renderGuarded(d: DiagramRef, limits: DiagramLimits, deps: RenderDeps): Promise { + let timer: ReturnType | undefined; + const timeout = new Promise((resolve) => { + timer = setTimeout(() => resolve(null), TIMEOUT_MS); + }); + try { + return await Promise.race([renderOne(d, limits, deps), timeout]); + } catch { + return null; + } finally { + clearTimeout(timer); + } +} + +export async function renderDiagrams( + diagrams: DiagramRef[], + limits: DiagramLimits, + deps: RenderDeps = defaultRenderDeps, +): Promise> { + const results: (Blob | null)[] = new Array(diagrams.length).fill(null); + let next = 0; + const worker = async (): Promise => { + while (next < diagrams.length) { + const i = next++; + const d = diagrams[i]; + if (d) results[i] = await renderGuarded(d, limits, deps); + } + }; + await Promise.all(Array.from({ length: Math.min(CONCURRENCY, diagrams.length) }, worker)); + + const out = new Map(); + const budget = limits.max_body_bytes - BODY_HEADROOM; + let total = 0; + for (let i = 0; i < diagrams.length; i++) { + const blob = results[i]; + const d = diagrams[i]; + if (!blob || !d) continue; + if (out.size >= limits.max_parts || total + blob.size > budget) break; + out.set(d.key, blob); + total += blob.size; + } + return out; +} diff --git a/web/src/features/export/useExport.test.ts b/web/src/features/export/useExport.test.ts new file mode 100644 index 00000000..6eac8709 --- /dev/null +++ b/web/src/features/export/useExport.test.ts @@ -0,0 +1,130 @@ +import { act, renderHook } from "@testing-library/react"; +import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; + +import type { ConnStatus } from "../../components/StatusDot"; +import { type ExportDeps, useExport } from "./useExport"; + +const limits = { max_body_bytes: 1000, max_part_bytes: 500, max_parts: 5, scale: 2 }; +const png = new Blob(["x"], { type: "image/png" }); +const doc = new Blob(["d"]); +const okFile = { ok: { blob: doc, fallbacks: { diagrams: 0, images: 0 } } }; +const err = (status: number, code: string) => ({ error: { code, message: code, details: {}, status } }); + +function makeDeps(file: ReturnType = vi.fn(() => Promise.resolve(okFile)), over: Partial = {}) { + const diagrams = vi.fn(() => Promise.resolve({ ok: { limits, diagrams: [] } })); + const deps = { + exportApi: { diagrams, file }, + renderDiagrams: vi.fn(() => Promise.resolve(new Map([["d1", png]]))), + exportFileName: vi.fn((t: string, f: string) => `${t}.${f}`), + fallbackMessage: vi.fn(() => null as string | null), + saveBlob: vi.fn(), + notify: vi.fn(), + ...over, + }; + return { deps: deps as unknown as ExportDeps, diagrams, file, mocks: deps }; +} + +type Props = { status: ConnStatus; pendingBytes: number }; + +function setup(deps: ExportDeps, initial: Props = { status: "connected", pendingBytes: 0 }) { + return renderHook((p: Props) => useExport({ docId: "d", title: "T", ...p, deps }), { initialProps: initial }); +} + +describe("useExport", () => { + beforeEach(() => vi.useFakeTimers()); + afterEach(() => vi.useRealTimers()); + + it("waits for pending edits, then exports and saves with the sanitized name", async () => { + const { deps, diagrams, file, mocks } = makeDeps(); + const { result, rerender } = setup(deps, { status: "connected", pendingBytes: 10 }); + let p!: Promise; + act(() => { + p = result.current.run("docx"); + }); + await act(async () => { + await vi.advanceTimersByTimeAsync(1000); + }); + expect(diagrams).not.toHaveBeenCalled(); + expect(result.current.exporting).toBe(true); + rerender({ status: "connected", pendingBytes: 0 }); + await act(async () => { + await vi.advanceTimersByTimeAsync(500); + await p; + }); + expect(file).toHaveBeenCalledWith("d", "docx", new Map([["d1", png]])); + expect(mocks.fallbackMessage).toHaveBeenCalledWith({ diagrams: 0, images: 0 }, false); + expect(mocks.saveBlob).toHaveBeenCalledWith(doc, "T.docx"); + expect(mocks.notify).not.toHaveBeenCalled(); + expect(result.current.exporting).toBe(false); + }); + + it("exports anyway after 5 s of unsynced edits and appends the warning", async () => { + const msg = "Exported. Your latest edits weren't saved yet and may be missing."; + const { deps, diagrams, mocks } = makeDeps(undefined, { fallbackMessage: vi.fn(() => msg) }); + const { result } = setup(deps, { status: "offline", pendingBytes: 0 }); + let p!: Promise; + act(() => { + p = result.current.run("odt"); + }); + await act(async () => { + await vi.advanceTimersByTimeAsync(4000); + }); + expect(diagrams).not.toHaveBeenCalled(); + await act(async () => { + await vi.advanceTimersByTimeAsync(1500); + await p; + }); + expect(deps.fallbackMessage).toHaveBeenCalledWith({ diagrams: 0, images: 0 }, true); + expect(mocks.saveBlob).toHaveBeenCalledWith(doc, "T.odt"); + expect(mocks.notify).toHaveBeenCalledWith("warn", msg); + }); + + it("retries once without pictures on 413", async () => { + const file = vi.fn().mockResolvedValueOnce(err(413, "too_large")).mockResolvedValueOnce(okFile); + const { deps, mocks } = makeDeps(file); + const { result } = setup(deps); + await act(async () => { + await result.current.run("docx"); + }); + expect(file).toHaveBeenCalledTimes(2); + expect((file.mock.calls[0] as [string, string, Map])[2].size).toBe(1); + expect((file.mock.calls[1] as [string, string, Map])[2].size).toBe(0); + expect(mocks.saveBlob).toHaveBeenCalledTimes(1); + }); + + it("shows the busy toast on export.busy", async () => { + const { deps, mocks } = makeDeps(vi.fn(() => Promise.resolve(err(503, "export.busy")))); + const { result } = setup(deps); + await act(async () => { + await result.current.run("docx"); + }); + expect(mocks.notify).toHaveBeenCalledWith("warn", "Export is busy, try again in a moment."); + expect(mocks.saveBlob).not.toHaveBeenCalled(); + expect(result.current.exporting).toBe(false); + }); + + it("second_click_while_exporting_does_nothing", async () => { + const { deps, diagrams, mocks } = makeDeps(); + const { result } = setup(deps); + await act(async () => { + const a = result.current.run("docx"); + const b = result.current.run("docx"); + await Promise.all([a, b]); + }); + expect(diagrams).toHaveBeenCalledTimes(1); + expect(mocks.saveBlob).toHaveBeenCalledTimes(1); + }); + + it("shows the generic error toast for other failures", async () => { + const { deps, mocks } = makeDeps(vi.fn(() => Promise.resolve(err(500, "internal")))); + const { result } = setup(deps); + await act(async () => { + await result.current.run("docx"); + }); + expect(mocks.notify).toHaveBeenCalledWith( + "error", + "Couldn't export this page. Try again, or export as Markdown.", + ); + expect(result.current.exporting).toBe(false); + }); +}); diff --git a/web/src/features/export/useExport.ts b/web/src/features/export/useExport.ts new file mode 100644 index 00000000..e39de30e --- /dev/null +++ b/web/src/features/export/useExport.ts @@ -0,0 +1,102 @@ +import { useCallback, useEffect, useRef, useState } from "react"; + +import type { ConnStatus } from "../../components/StatusDot"; +import { useUi } from "../../stores/ui"; +import { exportApi } from "./export.api"; +import { exportFileName, fallbackMessage, saveBlob } from "./exportFile"; +import { renderDiagrams } from "./renderDiagrams"; + +export type ExportDeps = { + exportApi: typeof exportApi; + renderDiagrams: typeof renderDiagrams; + exportFileName: typeof exportFileName; + fallbackMessage: typeof fallbackMessage; + saveBlob: typeof saveBlob; + notify: (kind: "info" | "warn" | "error", text: string) => void; +}; + +const SYNC_WAIT_MS = 5000; +const SYNC_POLL_MS = 100; +const RETRY_STATUSES = new Set([413, 408, 0]); + +export function useExport(opts: { + docId: string; + title: string; + status: ConnStatus; + pendingBytes: number; + deps?: Partial; +}): { exporting: boolean; run: (format: "docx" | "odt") => Promise } { + const { docId, title, status, pendingBytes } = opts; + const storeNotify = useUi((s) => s.notify); + const deps: ExportDeps = { + exportApi, + renderDiagrams, + exportFileName, + fallbackMessage, + saveBlob, + notify: storeNotify, + ...opts.deps, + }; + + const [exporting, setExporting] = useState(false); + const busyRef = useRef(false); + const syncedRef = useRef(false); + const depsRef = useRef(deps); + + useEffect(() => { + syncedRef.current = status === "connected" && pendingBytes === 0; + }, [status, pendingBytes]); + useEffect(() => { + depsRef.current = deps; + }); + + const run = useCallback( + async (format: "docx" | "odt"): Promise => { + if (busyRef.current) return; + busyRef.current = true; + setExporting(true); + const d = depsRef.current; + try { + // Give in-flight edits a moment to reach the server. + let waited = 0; + while (!syncedRef.current && waited < SYNC_WAIT_MS) { + await new Promise((r) => setTimeout(r, SYNC_POLL_MS)); + waited += SYNC_POLL_MS; + } + const unsynced = !syncedRef.current; + + const list = await d.exportApi.diagrams(docId); + if ("error" in list) { + fail(d, list.error.code); + return; + } + const parts = await d.renderDiagrams(list.ok.diagrams, list.ok.limits); + + let res = await d.exportApi.file(docId, format, parts); + if ("error" in res && RETRY_STATUSES.has(res.error.status)) { + res = await d.exportApi.file(docId, format, new Map()); + } + if ("error" in res) { + fail(d, res.error.code); + return; + } + d.saveBlob(res.ok.blob, d.exportFileName(title, format)); + const msg = d.fallbackMessage(res.ok.fallbacks, unsynced); + if (msg !== null) d.notify("warn", msg); + } catch { + fail(d, ""); + } finally { + busyRef.current = false; + setExporting(false); + } + }, + [docId, title], + ); + + return { exporting, run }; +} + +function fail(d: ExportDeps, code: string): void { + if (code === "export.busy") d.notify("warn", "Export is busy, try again in a moment."); + else d.notify("error", "Couldn't export this page. Try again, or export as Markdown."); +} diff --git a/web/src/lib/mermaid.ts b/web/src/lib/mermaid.ts new file mode 100644 index 00000000..beb4ed12 --- /dev/null +++ b/web/src/lib/mermaid.ts @@ -0,0 +1,19 @@ +export type MermaidApi = { + initialize: (opts: Record) => void; + render: (id: string, src: string) => Promise<{ svg: string }>; +}; + +let cached: MermaidApi | null = null; +export async function getMermaid(): Promise { + if (cached) return cached; + const mod = await import("mermaid"); + const m = (mod.default ?? mod) as MermaidApi; + m.initialize({ + startOnLoad: false, + securityLevel: "strict", + theme: "default", + fontFamily: "'Inter Variable', Inter, system-ui, sans-serif", + }); + cached = m; + return m; +} diff --git a/web/src/lib/validators.ts b/web/src/lib/validators.ts index cd0f759f..3c506417 100644 --- a/web/src/lib/validators.ts +++ b/web/src/lib/validators.ts @@ -160,6 +160,28 @@ const releaseNotesSchema = v.object({ export type ReleaseNotes = v.InferOutput; export const ReleaseNotes = releaseNotesSchema; +const diagramListSchema = v.object({ + limits: v.object({ + max_body_bytes: v.number(), + max_part_bytes: v.number(), + max_parts: v.number(), + scale: v.number(), + }), + diagrams: v.array( + v.variant("kind", [ + v.object({ kind: v.literal("mermaid"), key: v.string(), source: v.string() }), + v.object({ + kind: v.literal("board"), + key: v.string(), + board_id: v.string(), + label: v.string(), + }), + ]), + ), +}); +export type DiagramList = v.InferOutput; +export const DiagramList = diagramListSchema; + export function parse(schema: v.GenericSchema, data: unknown): T { return v.parse(schema, data); }