diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml new file mode 100644 index 0000000..15b3e90 --- /dev/null +++ b/.github/workflows/ci.yml @@ -0,0 +1,47 @@ +name: CI + +on: + push: + branches: [main] + pull_request: + +jobs: + test: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - uses: pnpm/action-setup@v4 + - uses: actions/setup-node@v4 + with: + node-version: 22 + cache: pnpm + - run: pnpm install --frozen-lockfile + # Formatting (oxfmt) and linting (oxlint) through Vite+. + - run: pnpm check + - run: pnpm typecheck + - run: pnpm coverage + - uses: actions/upload-artifact@v4 + with: + name: coverage + path: coverage + if-no-files-found: ignore + - run: pnpm exec playwright install --with-deps chromium firefox webkit + # Includes the visual suite: its Linux Chromium baselines are committed (re-record them with + # record-baselines.yml after an intentional rendering change). + - run: pnpm vp test --run --project browser + env: + BROWSERS: chromium,firefox,webkit + - run: pnpm build + - run: pnpm size + # `pnpm docs` is forwarded to npm's built-in `docs` (opens a browser); run the script. + - run: pnpm run docs + - uses: actions/upload-artifact@v4 + with: + name: api-docs + path: docs/api + - uses: actions/upload-artifact@v4 + if: failure() + with: + name: vitest-attachments + path: .vitest-attachments + if-no-files-found: ignore diff --git a/.github/workflows/record-baselines.yml b/.github/workflows/record-baselines.yml new file mode 100644 index 0000000..f141450 --- /dev/null +++ b/.github/workflows/record-baselines.yml @@ -0,0 +1,30 @@ +name: Record visual baselines + +# Records the Chromium screenshot baselines on Linux (the CI platform) and uploads them as an +# artifact. Trigger manually after an intentional rendering change, then: +# +# gh run download -n visual-baselines -D tests/browser/__screenshots__ +# +# and commit the result. Baselines are per browser and platform, so the macOS ones recorded locally +# stay untouched. +on: + workflow_dispatch: + +jobs: + record: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - uses: pnpm/action-setup@v4 + - uses: actions/setup-node@v4 + with: + node-version: 22 + cache: pnpm + - run: pnpm install --frozen-lockfile + - run: pnpm exec playwright install --with-deps chromium + - run: pnpm vp test --run --project browser tests/browser/visual.test.ts --update + - uses: actions/upload-artifact@v4 + with: + name: visual-baselines + path: tests/browser/__screenshots__/visual.test.ts/*-linux.png + if-no-files-found: error diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml new file mode 100644 index 0000000..967d291 --- /dev/null +++ b/.github/workflows/release.yml @@ -0,0 +1,38 @@ +name: Release + +# Publishes to npm with provenance when a version tag is pushed. Requires the `NPM_TOKEN` secret +# (or configure npm trusted publishing for this repository and drop the token). +on: + push: + tags: ['v*'] + +jobs: + publish: + runs-on: ubuntu-latest + permissions: + contents: write + id-token: write + steps: + - uses: actions/checkout@v4 + with: + fetch-depth: 0 + - uses: pnpm/action-setup@v4 + - uses: actions/setup-node@v4 + with: + node-version: 22 + cache: pnpm + registry-url: https://registry.npmjs.org + - run: pnpm install --frozen-lockfile + - run: pnpm check + - run: pnpm typecheck + - run: pnpm test:unit + - run: pnpm build + - run: pnpm size + - run: pnpm changelog + - run: npm publish --provenance --access public --tag ${{ contains(github.ref_name, '-') && 'next' || 'latest' }} + env: + NODE_AUTH_TOKEN: ${{ secrets.NPM_TOKEN }} + - uses: softprops/action-gh-release@v2 + with: + body_path: CHANGELOG.md + generate_release_notes: true diff --git a/.gitignore b/.gitignore index 7a2166a..7d523d8 100644 --- a/.gitignore +++ b/.gitignore @@ -11,3 +11,8 @@ yarn-debug.log* yarn-error.log* pnpm-debug.log* + +.vitest-attachments/ + +docs/api/ +coverage/ diff --git a/.prettierrc.cjs b/.prettierrc.cjs deleted file mode 100644 index 15b577b..0000000 --- a/.prettierrc.cjs +++ /dev/null @@ -1,14 +0,0 @@ -module.exports = { - useTabs: false, - singleQuote: true, - trailingComma: 'all', - printWidth: 100, - tabWidth: 2, - plugins: [ - require('@ianvs/prettier-plugin-sort-imports'), - ], - importOrder: ['.css$', '^node:', '', '^[$]', '^[../]', '^[./]'], - importOrderSeparation: true, - importOrderSortSpecifiers: true, - importOrderCaseInsensitive: true, -}; diff --git a/.sandbox/index.html b/.sandbox/index.html index b6d66b7..87d2e55 100644 --- a/.sandbox/index.html +++ b/.sandbox/index.html @@ -1,42 +1,91 @@ + - Media Captions + Media Captions Sandbox - - + -
-
+
+
+ +
- - diff --git a/.sandbox/launch.js b/.sandbox/launch.js index 8a53ba7..53d92a9 100644 --- a/.sandbox/launch.js +++ b/.sandbox/launch.js @@ -1,23 +1,3 @@ import { execSync } from 'child_process'; -import fs from 'fs'; -import path from 'path'; -const SANDBOX_TEMPLATE = path.resolve(process.cwd(), '.sandbox'), - SANDBOX_DIR = path.resolve(process.cwd(), 'sandbox'), - IGNORED_FILES = new Set(['launch.js']); - -// Copy files from .sandbox template directory to sandbox. -if (!fs.existsSync(SANDBOX_DIR)) { - fs.mkdirSync(SANDBOX_DIR); - const files = fs.readdirSync(SANDBOX_TEMPLATE); - for (const file of files) { - if (IGNORED_FILES.has(file)) continue; - const from = path.resolve(SANDBOX_TEMPLATE, file); - const to = path.resolve(SANDBOX_DIR, file); - fs.writeFileSync(to, fs.readFileSync(from, 'utf-8')); - } -} - -execSync('vite --open=/sandbox/index.html --port=3100 --host', { - stdio: 'inherit', -}); +execSync('vp dev --open=/.sandbox/index.html --port=3100 --host', { stdio: 'inherit' }); diff --git a/.sandbox/main.ts b/.sandbox/main.ts index 2280064..dbafd0b 100644 --- a/.sandbox/main.ts +++ b/.sandbox/main.ts @@ -1,23 +1,266 @@ -import { CaptionsRenderer, VTTCue, VTTRegion } from '../src'; +import { CaptionsRenderer, CueTrack, parseText, VTTCue, VTTRegion } from '../src'; +import { defineMediaCaptionsElement } from '../src/element'; -const overlay = document.getElementById('overlay')!; -const renderer = new CaptionsRenderer(overlay); +type Scenario = ( + mount: (label?: string, small?: boolean) => CaptionsRenderer, +) => Promise | void; -const cues = [ - new VTTCue(0, 10, 'Cue A...'), - new VTTCue(10, 20, 'Cue B...'), - new VTTCue(20, 30, 'Cue C...'), -]; +const root = document.getElementById('root')!, + timeInput = document.getElementById('current-time') as HTMLInputElement, + renderers: CaptionsRenderer[] = []; -const region = new VTTRegion(); -cues[1].region = region; +function mount(label?: string, small = false) { + const viewport = document.createElement('div'); + viewport.className = 'viewport' + (small ? ' small' : ''); + if (label) { + const tag = document.createElement('div'); + tag.className = 'label'; + tag.textContent = label; + viewport.append(tag); + } + const overlay = document.createElement('div'); + viewport.append(overlay); + root.append(viewport); + const renderer = new CaptionsRenderer(overlay); + renderers.push(renderer); + return renderer; +} -renderer.changeTrack({ - regions: [region], - cues, -}); +function cue(start: number, end: number, text: string, settings: Partial = {}) { + const result = new VTTCue(start, end, text); + Object.assign(result, settings); + return result; +} -const input = document.getElementById('current-time')! as HTMLInputElement; -input.addEventListener('change', () => { - renderer.currentTime = input.valueAsNumber; -}); +const scenarios: Record = { + cues(attach) { + const renderer = attach(); + renderer.changeTrack({ + cues: [ + cue(0, 10, 'line:0 (snap to first line)', { line: 0 }), + cue(0, 10, 'line:50% position:10% size:35% align:start', { + snapToLines: false, + line: 50, + position: 10, + size: 35, + align: 'start', + }), + cue(0, 10, 'position:90% size:35% align:end', { + position: 90, + size: 35, + align: 'end', + line: -4, + }), + cue( + 0, + 10, + 'Default cue with bold, italic, and colour', + ), + cue(0, 10, 'Karaoke <00:00:01.000>timed <00:00:02.000>text <00:00:04.000>updates', { + line: -6, + }), + ], + }); + }, + + regions(attach) { + const renderer = attach(); + const top = new VTTRegion(); + Object.assign(top, { + id: 'top', + width: 45, + lines: 2, + regionAnchorX: 0, + regionAnchorY: 0, + viewportAnchorX: 5, + viewportAnchorY: 8, + }); + const bottom = new VTTRegion(); + Object.assign(bottom, { + id: 'bottom', + width: 60, + lines: 3, + regionAnchorX: 100, + regionAnchorY: 100, + viewportAnchorX: 95, + viewportAnchorY: 92, + scroll: 'up', + }); + const cues = [ + cue(0, 10, 'REGION top: width 45%, anchored top-left'), + cue(0, 10, 'Second line in the top region'), + cue(0, 10, 'REGION bottom: width 60%, 3 lines, scroll:up'), + cue(1, 10, 'Roll-up captions push older lines'), + cue(2, 10, 'upwards as new cues arrive'), + ]; + cues[0].region = cues[1].region = top; + cues[2].region = cues[3].region = cues[4].region = bottom; + renderer.changeTrack({ regions: [top, bottom], cues }); + }, + + 'region-scroll'(attach) { + const renderer = attach(); + const region = new VTTRegion(); + Object.assign(region, { id: 'rollup', width: 70, lines: 3, viewportAnchorX: 15, scroll: 'up' }); + region.regionAnchorX = 0; + const lines = [ + 'Roll-up caption line one', + 'Roll-up caption line two', + 'Roll-up caption line three', + 'Line four scrolls the first line out', + 'Line five keeps rolling', + ]; + const cues = lines.map((text, i) => cue(i * 0.8, 20, text)); + for (const c of cues) c.region = region; + renderer.changeTrack({ regions: [region], cues }); + }, + + async ssa(attach) { + const renderer = attach(); + const ass = `[Script Info] +ScriptType: v4.00+ +PlayResX: 1280 +PlayResY: 720 + +[V4+ Styles] +Format: Name, Fontname, Fontsize, PrimaryColour, SecondaryColour, OutlineColour, BackColour, Bold, Italic, Underline, StrikeOut, ScaleX, ScaleY, Spacing, Angle, BorderStyle, Outline, Shadow, Alignment, MarginL, MarginR, MarginV, Encoding +Style: Default,Arial,52,&H00FFFFFF,&H000000FF,&H00000000,&H80000000,0,0,0,0,100,100,0,0,1,3,2,2,40,40,40,1 +Style: Sign,Arial,40,&H0000FFFF,&H000000FF,&H00203040,&H00000000,-1,0,0,0,100,100,0,0,3,4,0,8,40,40,30,1 +Style: Note,Arial,34,&H00FFD0A0,&H000000FF,&H00000000,&H00000000,0,-1,0,0,100,100,1,0,1,2,0,7,40,40,120,1 + +[Events] +Format: Layer, Start, End, Style, Name, MarginL, MarginR, MarginV, Effect, Text +Dialogue: 0,0:00:00.00,0:00:10.00,Default,Rin,0,0,0,,Outlined dialogue with {\\i1}italics{\\i0} and {\\c&H00A5FF&}colour{\\r} tags +Dialogue: 1,0:00:00.00,0:00:10.00,Sign,,0,0,0,,BorderStyle 3 opaque box, top centre +Dialogue: 0,0:00:00.00,0:00:10.00,Note,,0,0,0,,Top-left italic note with letter spacing +Dialogue: 0,0:00:00.00,0:00:10.00,Default,,0,0,0,,{\\pos(640,400)\\an5}\\pos(640,400) centred anchor +Dialogue: 0,0:00:00.00,0:00:10.00,Default,,0,0,0,,{\\an1}{\\k60}Ka{\\k60}ra{\\k60}o{\\k60}ke +`; + const result = await parseText(ass, { type: 'ass' }); + renderer.changeTrack(result); + }, + + 'edge-styles'(attach) { + root.classList.add('grid'); + for (const style of ['uniform', 'drop-shadow', 'raised', 'depressed'] as const) { + const renderer = attach(`data-edge-style="${style}"`, true); + renderer.overlay.setAttribute('data-edge-style', style); + renderer.overlay.style.setProperty('--cue-bg-color', 'transparent'); + renderer.changeTrack({ cues: [cue(0, 10, `${style} edge style`)] }); + } + }, + + layout(attach) { + // The structured cue model: parsers of positioned formats emit these instead of CSS. + const renderer = attach(); + const centered = cue(0, 10, 'layout: left 50%, translate x -0.5, max-content'); + centered.layout = { left: 50, bottom: 6, width: 'max-content', translate: { x: -0.5 } }; + centered.textStyle = { + color: '#ffd166', + textStroke: '0.08em #1c1f2b', + backgroundColor: 'transparent', + }; + + const fixed = cue(0, 10, 'layout.fixed: pinned at 70% / 35%, never moved by collisions'); + fixed.layout = { + left: 70, + top: 35, + width: 'max-content', + translate: { x: -0.5, y: -0.5 }, + fixed: true, + }; + fixed.textStyle = { fontSize: '4cqh', opacity: '0.9' }; + + const boxed = cue(0, 10, 'textStyle: outline box, bold, letter spacing'); + boxed.layout = { left: 4, top: 8, width: 'max-content', maxWidth: 40 }; + boxed.textStyle = { + backgroundColor: 'rgba(20, 30, 60, 0.95)', + outline: '0.15em solid #9cc4ff', + fontWeight: 'bold', + letterSpacing: '0.05em', + textAlign: 'left', + }; + + const faded = cue(0, 10, 'textStyle.animation: fade in'); + faded.layout = { right: 4, top: 8, width: 'max-content' }; + faded.textStyle = { animation: 'media-captions-fade-in 1.5s' }; + + renderer.changeTrack({ cues: [centered, fixed, boxed, faded] }); + }, + + live(attach) { + // A CueTrack fed incrementally, the way CEA-608/708 stream decoders do in live mode. + const renderer = attach('CueTrack live: open-ended cues updated in place'); + const track = new CueTrack(undefined, { retention: 30 }); + renderer.changeTrack({ cues: track }); + + const first = cue(0, Infinity, 'Live caption arrives with endTime = Infinity'); + track.add(first); + + // Later the decoder learns when it ended and updates the same object. + first.endTime = 2; + track.update(first); + + const second = cue(2, Infinity, 'Next caption is on screen right now'); + track.add(second); + second.text = 'Next caption is on screen right now (text edited in place)'; + track.update(second); + }, + + element(attach) { + // The custom element in light DOM (page stylesheets apply). + defineMediaCaptionsElement(); + const viewport = document.createElement('div'); + viewport.className = 'viewport'; + const tag = document.createElement('div'); + tag.className = 'label'; + tag.textContent = ''; + viewport.append(tag); + const el = document.createElement('media-captions'); + el.setAttribute('edge-style', 'uniform'); + viewport.append(el); + root.append(viewport); + el.load({ + cues: [ + cue(0, 10, 'Rendered by the element'), + cue(0, 10, 'Attributes: src, type, for, dir, edge-style, shadow', { line: 1 }), + ], + }); + renderers.push(el.renderer); + void attach; // handled manually above + }, + + collisions(attach) { + const renderer = attach(); + renderer.changeTrack({ + cues: [ + cue(0, 10, 'Three cues share the same line setting'), + cue(0.5, 10, 'so collision avoidance stacks them'), + cue(1, 10, 'without overlap, in cue order'), + cue(0, 10, 'line:1 top cue', { line: 1 }), + cue(0.5, 10, 'another line:1 cue pushed down', { line: 1 }), + ], + }); + }, +}; + +const params = new URLSearchParams(location.search), + name = params.get('scenario') ?? 'cues', + scenario = scenarios[name] ?? scenarios.cues; + +document.getElementById('links')!.innerHTML = Object.keys(scenarios) + .map((key) => `${key}`) + .join(' ยท '); + +function setTime(time: number) { + for (const renderer of renderers) renderer.currentTime = time; +} + +await scenario(mount); +setTime(Number(params.get('time') ?? timeInput.value)); +timeInput.addEventListener('change', () => setTime(timeInput.valueAsNumber)); + +// Signal to screenshot tooling that cues are rendered. +requestAnimationFrame(() => + requestAnimationFrame(() => document.body.setAttribute('data-ready', '')), +); diff --git a/README.md b/README.md index cfcf0e4..6133982 100644 --- a/README.md +++ b/README.md @@ -9,26 +9,31 @@ Captions parsing and rendering library built for the modern web. - ๐Ÿšฏ 0 dependencies. - ๐Ÿ’ช Built with TypeScript (TS 5 bundle mode ready). -- ๐Ÿชถ 5kB total + modular (parser/renderer split) + tree-shaking support. -- ๐Ÿ’ค Parsers are lazy loaded on-demand. +- ๐Ÿชถ ~13kB core (gzipped) + modular (parser/renderer split) + tree-shaking support. +- ๐Ÿ’ค Parsers are lazy loaded on-demand (each format is its own chunk). - ๐Ÿš„ Efficiently load and apply styles in parallel via CSS files. -- ๐Ÿ—‚๏ธ Supports VTT, SRT, and SSA/ASS. +- ๐Ÿ—‚๏ธ Supports VTT, SRT, SSA/ASS, TTML/IMSC1/DFXP, SCC, LRC, SBV, SAMI, MicroDVD, CEA-608/708 from + video streams, and fMP4 `wvtt`/`stpp` tracks. - โฌ†๏ธ Roll-up captions via VTT regions. - ๐Ÿงฐ Modern `fetch` and `ReadableStream` APIs. -- ๐Ÿ“ก Chunked text and response streaming support. -- ๐Ÿ“ WebVTT spec-compliant settings and rendering. -- ๐ŸŽค Timed text-tracks for karaoke-style captions. +- ๐Ÿ“ก Chunked text and response streaming support (including HLS `X-TIMESTAMP-MAP`). +- ๐Ÿ“ WebVTT spec-compliant parsing and rendering (including `STYLE` blocks), verified against + the web-platform-tests suite, conformance suites, and real-browser layout tests. +- ๐ŸŽค Timed text-tracks for karaoke-style captions (VTT, LRC, and ASS `\k` tags). +- ๐ŸŽž๏ธ Frame-accurate cue timing via `requestVideoFrameCallback`. - ๐Ÿ› ๏ธ Supports custom captions parser and cue renderer. +- ๐Ÿ”’ Cue text is rendered as DOM nodes, never HTML strings, so untrusted files can not inject + markup and strict CSP / Trusted Types policies are satisfied. +- ๐Ÿ“ก Live-ready: a `CueTrack` with incremental updates, open-ended cues, and eviction feeds the + renderer from CEA-608/708 stream decoders. +- ๐Ÿงฉ Structured `cue.layout` / `cue.textStyle` model shared by SSA, TTML, and 708, with JSON + round-tripping for Workers. +- ๐Ÿงฑ Drop-in `` custom element, plus `media-captions/parsers/*` entries. - ๐Ÿ’ฅ Collision detection to avoid overlapping or out-of-bounds cues. - ๐Ÿ—๏ธ Fixed and in-order cue rendering (including on font or overlay size changes). - ๐Ÿ›‘ Adjustable parsing error-tolerance with strict and non-strict modes. - ๐Ÿ–ฅ๏ธ Works in the browser and server-side (string renderer). -- ๐ŸŽจ Easy customization via CSS. - -โž• Planning to also add a TTML, CEA-608, and CEA-708 parser that will map to VTT and render -correctly. In addition, custom font loading and text codes support is planned for SSA/ASS captions. -We don't have an exact date but most likely after the [Vidstack Player][vidstack-player] 1.0. If -urgent and you're willing to sponsor, feel free to email me at rahim.alwer@gmail.com. +- ๐ŸŽจ Easy customization via CSS, including FCC edge-style presets. ๐Ÿ”— **Quicklinks** @@ -96,6 +101,9 @@ The library is old, outdated, and unmaintained. ## Installation +> Coming from media-captions 1.x? See [docs/MIGRATION.md](./docs/MIGRATION.md) for every +> breaking change and its replacement. + First, install the NPM package: ```bash @@ -129,6 +137,7 @@ like so: - [`parseTextStream`](#parsetextstream) - [`parseResponse`](#parseresponse) - [`parseByteStream`](#parsebytestream) + - [`inferCaptionsFormat`](#infercaptionsformat) - [`CaptionsParser`](#captionsparser) - **Rendering** - [`createVTTCueTemplate`](#createvttcuetemplate) @@ -137,12 +146,28 @@ like so: - [`renderVTTTokensString`](#rendervtttokensstring) - [`updateTimedVTTCueNodes`](#updatetimedvttcuenodes) - [`CaptionsRenderer`](#captionsrenderer) + - [Composable renderer (`createRenderer`)](#composable-renderer-createrenderer) + - [Canvas renderer (`media-captions/canvas`)](#canvas-renderer-media-captionscanvas) + - [`CueTrack`](#cuetrack) + - [``](#media-captions) + - [`syncCaptionsRenderer`](#synccaptionsrenderer) + - [`loadEmbeddedFonts`](#loadembeddedfonts) - [Styling](#styling) - **Formats** - [VTT](#vtt) - [SRT](#srt) - [SSA/ASS](#ssaass) + - [TTML](#ttml) + - [SCC (CEA-608)](#scc-cea-608) + - [CEA-608/708 from video streams](#cea-608708-from-video-streams) + - [LRC](#lrc) + - [SBV](#sbv) + - [SAMI](#sami) + - [MicroDVD](#microdvd) - [Streaming](#streaming) +- [HLS Segments](#hls-segments) +- [fMP4 subtitle tracks](#fmp4-subtitle-tracks) +- [Player integration (hls.js)](#player-integration-hlsjs) - [Types](#types) ## Parse Options @@ -150,16 +175,25 @@ like so: All parsing functions exported from this package accept the following options: - `strict`: Whether strict mode is enabled. In strict mode parsing errors will throw and cancel - the parsing process. + the parsing process, and the WebVTT grammar is enforced exactly (signature, timestamp digits, + `%` on percentages). Outside strict mode the parser is deliberately tolerant of common real-world + deviations (missing signature, comma millisecond separators, bare percentages, legacy + `align:middle`) while still reporting them as errors when `errors` is enabled. +- `lenient`: Whether those real-world deviations are accepted (default `true`). Set to `false` for + browser-exact WebVTT parsing that still recovers: invalid cues are dropped and reported rather + than thrown. This is the mode the web-platform-tests suite runs in. Ignored when `strict` is set. - `errors`: Whether errors should be collected and reported in the final [parser result](#parse-result). By default, this value will be true in dev mode or if `strict` mode is true. If set to true and `strict` mode is false, the `onError` callback will be invoked. Do note, setting this to true will dynamically load error builders which will slightly increase bundle size (~1kB). - `type`: The type of the captions file format so the correct parser is loaded. Options - include `vtt`, `srt`, `ssa`, `ass`, or a custom [`CaptionsParser`](#captionsparser) object. + include `vtt`, `srt`, `ssa`, `ass`, `ttml` (also `dfxp`/`xml`), `scc`, `lrc`, `sbv`, or a + custom [`CaptionsParser`](#captionsparser) object. +- `channel`: CEA-608 data channel to decode for SCC files (`1` or `2`). - `onHeaderMetadata`: Callback that is invoked when the metadata from the header block has been parsed. +- `onStyle`: Invoked with the CSS text of each WebVTT `STYLE` block. - `onCue`: Invoked when parsing a VTT cue block has finished parsing and a `VTTCue` has been created. Do note, regardless of which captions file format is provided a `VTTCue` will be created. @@ -199,6 +233,10 @@ All parsing functions exported from this package return a `Promise` which will r - `errors`: An array containing `ParseError` objects. Do note, errors will only be collected if in development mode, if `strict` parsing option is set to true, or the `errors` parsing option is set to true. +- `fonts`: Fonts embedded in the file (SSA/ASS `[Fonts]` section), see + [`loadEmbeddedFonts`](#loadembeddedfonts). +- `styles`: CSS text from WebVTT `STYLE` blocks, applied by the renderer (see + [Styling](#styling)). ```ts import { parseText } from 'media-captions'; @@ -318,8 +356,13 @@ const result = await parseResponse(fetch('/media/subs/english.vtt'), { }); ``` -The captions type will inferred from the response header `content-type` field. You can specify -the specific captions format like so: +Every parser is also published as an explicit entry (`media-captions/parsers/vtt`, `srt`, `ssa`, +`ttml`, `scc`, `lrc`, `sbv`) for bundlers or runtimes that can not follow dynamic imports; pass the +default export as `type`. + +The captions type is inferred from the response `content-type` header (e.g., `text/vtt`, +`application/x-subrip`, `application/ttml+xml`) and falls back to the URL file extension for +generic types like `text/plain`. You can specify the specific captions format like so: ```ts parseResponse(..., { type: 'vtt' }); @@ -353,6 +396,18 @@ const result = await parseByteStream(byteStream, { }); ``` +## `inferCaptionsFormat` + +Returns the captions format for a `content-type` header value and optional URL, or `undefined` +when it can not be determined. This is what [`parseResponse`](#parseresponse) uses internally. + +```ts +import { inferCaptionsFormat } from 'media-captions'; + +inferCaptionsFormat('application/ttml+xml'); // 'ttml' +inferCaptionsFormat('text/plain', '/subs/en.srt?token=1'); // 'srt' +``` + ## `CaptionsParser` You can create a custom caption parser and provide it to the `type` option on any parse function. @@ -418,7 +473,9 @@ const cueHTML = template.content.cloneNode(true); ## `renderVTTCueString` -This function takes a `VTTCue` and renders the cue text string into a HTML string. This +This function takes a `VTTCue` and renders the cue text string into a HTML string. All text and +attribute values are escaped, so the result is safe to assign to `innerHTML` even when the cue +came from an untrusted captions file. Class names are restricted to `[A-Za-z0-9_-]`. This function can be used server-side to render cue content like so: ```ts @@ -474,8 +531,17 @@ const tokens = tokenizeVTTCue(cue); ``` Nodes can be a `VTTBlockNode` which can have children (i.e., class, italic, bold, underline, -ruby, ruby text, voice, lang, timestamp) or a `VTTLeafNode` (i.e., text nodes). The tokens -can be used for custom rendering like so: +ruby, ruby text, voice, lang, timestamp) or a `VTTLeafNode` (i.e., text nodes). Text data and +annotations are entity-decoded (decimal and hex references plus the Latin-1 subset of named +references, including legacy forms without `;`), so escape them yourself if you render to HTML. +For the complete HTML table (2,125 names, about 12 kB gzipped) register the optional entry once: + +````ts +import { registerFullHTMLEntities } from 'media-captions/entities'; +registerFullHTMLEntities(); +``` Unknown or mismatched end tags are ignored and never corrupt nesting. As an +extension, `` and `` classes are treated as colours so other formats can +carry arbitrary colours through cue text. The tokens can be used for custom rendering like so: ```ts function renderTokens(tokens: VTTNode[]) { @@ -496,7 +562,7 @@ function renderTokens(tokens: VTTNode[]) { } } } -``` +```` All token types are listed below for use in TypeScript: @@ -573,12 +639,18 @@ and cues should be visually rendered. It includes: - Collision detection to avoid overlapping cues. - Updating timed text nodes with `data-past` and `data-future` attributes. - Updating when the overlay is resized. -- Applying SSA/ASS styles. +- Applying SSA/ASS styles and layers (z-order). +- Setting the overlay `lang` attribute from the track `Language` header. +- Finding active cues in O(log n) using a sorted index, so large tracks stay cheap. +- Rendering in three phases (measure, pure layout, write) so a render forces at most two layouts. +- Dispatching `enter`/`exit` events on cues and optionally announcing them to screen readers. - Accepts native `VTTCue` objects. > **Warning** > The [styles files](#installation) need to be included for the overlay renderer to work correctly! +Simultaneous cues stacked without overlapping + ```html
@@ -600,25 +672,301 @@ parseResponse(fetch('/media/subs/english.vtt')).then((result) => { renderer.changeTrack(result); }); +// Or use `syncCaptionsRenderer(renderer, video)` for frame-accurate updates. video.addEventListener('timeupdate', () => { renderer.currentTime = video.currentTime; }); ``` +**Init options** + +- `features`: the renderer features to install. Defaults to every feature; see + [Composable renderer](#composable-renderer-createrenderer) to pick a smaller set. +- `dir`: Text direction (`ltr` or `rtl`). +- `retention`: Seconds to keep ended cues before evicting them from the track (for live streams). +- `announce`: `true` / `'polite'` / `'assertive'` adds a visually hidden `aria-live` region after + the overlay that receives the plain text of cues as they appear. The visual overlay itself stays + `aria-live="off"` because sighted users read it and the audio already carries the words. +- `stacking`: how simultaneous cues stack. `'reading-order'` (default) keeps the newest cue in the + default slot so lines read top-down in cue order; `'spec'` follows the WebVTT rule (and SSA + `Collisions: Normal`) where the earliest cue keeps its slot. A track's `Collisions` metadata + selects this automatically when the option is omitted. +- `lineStep`: `'line-height'` (spec) or `'box'`, which snaps lines by the padded cue box height so + stacked lines never overlap. +- `safeArea`: inset from the overlay edges in percent (broadcast title-safe is about 10). +- `reducedMotion`: `true`, `false`, or `'auto'` (default, follows `prefers-reduced-motion`). When + on, cue animations hold their final state and transitions are disabled. Also a live property. + **Props** - `dir`: Sets the text direction (i.e., `ltr` or `rtl`). - `currentTime`: Updates the current playback time and schedules a re-render. +- `activeCues`: The cues currently displayed, in render order (read-only). +- `track`: The [`CueTrack`](#cuetrack) being rendered. Add, update, or remove cues on it for live + content. **Methods** -- `changeTrack(track: CaptionsRendererTrack)`: Resets the renderer and prepares new regions and cues. +- `changeTrack(track: CaptionsRendererTrack)`: Resets the renderer and prepares new regions and + cues. Pass the parse result directly; its `metadata.Language` is applied as the overlay `lang` + and its `styles` (WebVTT `STYLE` blocks) are injected scoped to the overlay. `cues` may also be a + `CueTrack` to follow. +- `attachTrack(track: CueTrack)`: Renders from an existing track and follows its changes. - `addCue(cue: VTTCue)`: Add a new cue to the renderer. - `removeCue(cue: VTTCue)`: Remove a cue from the renderer. - `update(forceUpdate: boolean)`: Schedules a re-render to happen. - `reset()`: Reset the renderer and clear all internal state including region and cue DOM nodes. - `destroy()`: Reset the renderer and destroy internal observers and event listeners. +**Cue events** + +Every cue dispatches `enter` when it starts showing and `exit` when it stops, matching the native +`TextTrackCue` events, so analytics or custom effects can hook individual cues: + +```ts +cue.addEventListener('enter', () => console.log('showing', cue.text)); +``` + +## Composable renderer (`createRenderer`) + +`CaptionsRenderer` is the batteries-included build. Underneath it is a small core plus a set of +**features**, each a separate import, so a player that only shows SRT never ships region +handling, SSA typesetting, or animation code: + +```ts +import { + createRenderer, + regions, + typesetting, + animations, + announcer, + vttStyles, +} from 'media-captions/renderer'; + +// Core only: WebVTT positioning, collision avoidance, timed text, cue events. ~8.5 KB min+gz. +const renderer = createRenderer(overlay); + +// Pick what your content needs. +const renderer = createRenderer(overlay, { + features: [regions(), typesetting(), animations()], + dir: 'rtl', +}); +``` + +`createRenderer` returns a `CaptionsRendererCore` with the same API as `CaptionsRenderer` +(`changeTrack`, `currentTime`, `track`, `activeCues`, `reset`, `destroy`, ...) and works with +`syncCaptionsRenderer`. `CaptionsRenderer` itself is `createRenderer` with `defaultFeatures()` +installed, plus `announcer()` when `announce` is set; pass `features` to it to override the set. + +| Feature | Renders | Needed by | +| ----------------- | --------------------------------------------------------------------------------------------- | ----------------------------------------------------- | +| `regions()` | WebVTT regions: region elements, anchors, roll-up scrolling (needs `regions.css`) | VTT with `REGION` blocks, CEA-608 roll-up via regions | +| `typesetting()` | `cue.layout`, `cue.textStyle`, `cue.layer`: absolute boxes, clips, colours, fonts, transforms | SSA/ASS, TTML/IMSC, CEA-708 | +| `animations()` | `cue.animations` as media-synced Web Animations; holds the final state under reduced motion | SSA `\fad`/`\move`/`\t`/karaoke, TTML `set` | +| `vttStyles()` | WebVTT `STYLE` blocks, rewritten and scoped to the overlay | VTT with `STYLE` blocks | +| `announcer(mode)` | A hidden `aria-live` region receiving the plain text of entering cues | Accessibility, opt-in | + +Cue text spans (``, ``, ``, per-span styles) are always rendered by the core, since they +are content rather than layout. + +**What happens without a feature.** A cue that needs one still renders as plain WebVTT text in the +default slot. In development builds the renderer logs one warning per missing capability, naming +the feature to add, so a parser/renderer mismatch is visible instead of silent. Parsers do not +depend on features; they only fill the cue model. + +**Writing a feature.** A feature is an object with a `name`, optional `capabilities`, and hooks +that run in phase order (never array order): `setup`, `changeTrack`, `createCue`/`disposeCue`, +`containerFor`, `beforeMeasure`/`measureContainer`/`writeContainer`, `writeCue`, `update`, +`reset`, `resize`, `destroy`. Features with the same `name` replace each other, so a preset can be +overridden by appending. See `RendererFeature` in `media-captions/renderer` for the contract. + +```ts +import type { RendererFeature } from 'media-captions/renderer'; + +// Tag every cue element with its start time for styling or analytics. +export function cueTiming(): RendererFeature { + return { + name: 'cue-timing', + createCue: (_, cue, { display }) => (display.dataset.start = cue.startTime.toFixed(3)), + }; +} +``` + +## Canvas renderer (`media-captions/canvas`) + +> Experimental. Same cue model, track, and layout engine as the DOM renderer; a different writer. + +```ts +import { parseResponse, syncCaptionsRenderer } from 'media-captions'; +import { CanvasCaptionsRenderer, paintCaptions } from 'media-captions/canvas'; + +const renderer = new CanvasCaptionsRenderer(canvas, { edgeStyle: 'uniform', fontSize: 0.05 }); +renderer.changeTrack(await parseResponse(fetch('/subs.ass'))); +syncCaptionsRenderer(renderer, video); + +// Or stateless, for a WebCodecs / OffscreenCanvas pipeline or a thumbnail: +paintCaptions(ctx, cues, time, { width: 1920, height: 1080 }); +``` + +Painting into a canvas is what makes captions possible where the DOM overlay is not shown: iOS +Safari fullscreen and picture-in-picture (paint, `captureStream()`, composite), burn-in and +export through WebCodecs, server or Worker rendering with `OffscreenCanvas`, thumbnails, and +DOM-less runtimes. Everything is a fraction of the frame size, so a device-pixel-ratio canvas +simply renders sharper. + +What it renders: WebVTT positioning (line snapping, percentage lines, position/size/align, stacking +and collision avoidance shared with the DOM renderer), inline tags and class colours, regions with +roll-up, the SSA/TTML/CEA-708 layout and text style model (boxes, anchors, colours, fonts, +strokes, shadows, rotation and scale about the alignment anchor or `\org`, clip rectangles and +polygons, `\p` drawings via `Path2D`, image cues), and `cue.animations` sampled at media time. + +Vertical writing modes flow into columns (`text-orientation: mixed`: CJK upright, other scripts +sideways). Ruby annotations sit over their base (beside it in vertical text) at half size, and +3D rotations project orthographically like CSS without `perspective`. Not yet: WebVTT `STYLE` +blocks. Line breaking is a greedy wrap with a balance pass, so long lines may break differently +from the browser. + +**Options** (`CanvasCaptionsOptions`): `fontFamily`, `fontSize` (fraction of the height, default +`0.05`), `lineHeight` (`1.2`), `paddingX`/`paddingY` (em), `safeArea` (fraction of the width, +`0.01`), `color`, `backgroundColor`, `edgeStyle` + `edgeColor`, `classColors`, `dir`, `stacking`, +`lineStep`, `reducedMotion`. These mirror the stylesheet defaults so both writers agree on +geometry; the browser suite checks the DOM and canvas boxes land within a couple of pixels. + +**Picture-in-picture and iOS fullscreen.** Those surfaces show video pixels only, so composite the +frame and the captions into one canvas and stream it: + +```ts +const frame = document.createElement('canvas'), + ctx = frame.getContext('2d')!, + out = document.createElement('video'); +out.muted = true; +out.playsInline = true; +out.srcObject = frame.captureStream(30); + +function tick() { + frame.width = video.videoWidth; + frame.height = video.videoHeight; + ctx.drawImage(video, 0, 0); // or a WebCodecs VideoFrame + paintCaptions(ctx, renderer.activeCues, video.currentTime, { edgeStyle: 'uniform' }); + video.requestVideoFrameCallback(tick); +} +video.requestVideoFrameCallback(tick); + +await out.play(); +await out.requestPictureInPicture(); // or out.webkitEnterFullscreen() on iOS +``` + +The playground's Canvas tab has both buttons wired up against its mock video. + +**Headless pieces** are exported for other writers and tests: `measureCue` (text flow to +`CueLayoutInput` without a DOM), `layoutCaptions` (measure + layout, no painting), `flowCue`, +`sampleAnimation`, and `TextMeasurer` with `canvasTextMeasurer` / `monospaceTextMeasurer`. + +See `docs/design/canvas-and-style-model.md` for what this exploration showed and the proposed +typed style model. + +## `CueTrack` + +A sorted, incrementally maintained list of cues with O(log n) active-cue lookup, change events, +mutable end times, and eviction. The renderer uses one internally; use it directly for live +content where cues arrive continuously and end times are only known later: + +```ts +import { CaptionsRenderer, CueTrack } from 'media-captions'; +import { CEA708Decoder } from 'media-captions/cea'; + +const track = new CueTrack(undefined, { retention: 30 }), + renderer = new CaptionsRenderer(overlay, { retention: 30 }); + +renderer.changeTrack({ cues: track }); + +const decoder = new CEA708Decoder({ + live: true, + onCue: (cue) => track.add(cue), // endTime is Infinity while the caption is on screen + onCueUpdate: (cue) => track.update(cue), // end time is now known; re-indexed in place +}); +``` + +- Open-ended cues use `endTime = Infinity`. This works on top of the native `VTTCue` too (a finite + sentinel is stored internally), and serialises as `null` in JSON. +- `add(cue)`, `addAll(cues)`, `remove(cue)`, `update(cue)`, `clear()`, `has(cue)`, `size`, `cues`. +- `activeAt(time)`: cues active at a time, in start order. +- `evict(time)`: drops cues that ended more than `retention` seconds ago; `maxCues` caps the total. +- `dedupe: true` ignores cues that duplicate an existing one (same times, id, and text), which HLS + and DASH segments produce whenever a cue straddles a segment boundary or a segment is re-fetched. +- `on(listener)`: subscribe to `add`, `remove`, `update`, and `clear` events; returns an unsubscribe. + +## `` + +A framework-agnostic custom element that loads, syncs, and renders captions over a media element: + +```html + + +
+ + +
+ + +``` + +- Attributes: `src`, `type` (format, inferred when omitted), `for` (media element id) or the + `media` property, `dir`, `edge-style`, `frame-accurate` (`"false"` for event-driven sync), and + `shadow` to render in a shadow root with stylesheets from the `styles` attribute (defaults to + the jsDelivr CSS). In light DOM (the default) the page includes the stylesheets itself. +- Properties/methods: `renderer`, `track`, `load(result)`, `clear()`, `destroy()`. +- Events: `load` (detail: parse result), `error` (detail: `Error` or `ParseError[]`), and + `cuechange` (detail: `{ activeCues }`). +- Importing the module has no side effects, so it is safe to import during server rendering. + +## `syncCaptionsRenderer` + +The media `timeupdate` event only fires a few times per second, which makes short cues and +karaoke timed text visibly late. This helper drives a [`CaptionsRenderer`](#captionsrenderer) +from `requestVideoFrameCallback` while playing (falling back to `requestAnimationFrame`) and +only relies on events while paused or seeking. It returns a function that stops syncing. + +```ts +import { CaptionsRenderer, syncCaptionsRenderer } from 'media-captions'; + +const renderer = new CaptionsRenderer(captions), + stop = syncCaptionsRenderer(renderer, video); + +// Later... +stop(); +``` + +For a low-power, event-driven mode, mirror the cues into a hidden native text track (our `VTTCue` +extends the native class) and let the browser fire `cuechange` exactly at cue boundaries: + +```ts +const track = video.addTextTrack('metadata'); +for (const cue of cues) track.addCue(cue); + +syncCaptionsRenderer(renderer, video, { frameAccurate: false, track }); +``` + +## `loadEmbeddedFonts` + +SSA/ASS files can embed fonts in a `[Fonts]` section. The parser decodes them into the `fonts` +array on the parse result, and this helper registers them with the document using the +`FontFace` API so styled cues render with the intended typeface. + +```ts +import { loadEmbeddedFonts, parseResponse } from 'media-captions'; + +const result = await parseResponse(fetch('/subs/english.ass')); +const faces = await loadEmbeddedFonts(result.fonts ?? []); + +// Remove them later if needed. +for (const face of faces) document.fonts.delete(face); +``` + ## Styling Captions rendered with the [`CaptionOverlayRenderer`](#captionsoverlayrenderer) can be @@ -631,10 +979,14 @@ easily customized with CSS. Here are all the parts you can select and customize: --overlay-padding: 1%; --cue-color: white; --cue-bg-color: rgba(0, 0, 0, 0.8); - --cue-font-size: calc(var(--overlay-height) / 100 * 5); + --cue-font-size: 5cqh; /* 5% of the overlay height via container query units */ --cue-line-height: calc(var(--cue-font-size) * 1.2); --cue-padding-x: calc(var(--cue-font-size) * 0.6); --cue-padding-y: calc(var(--cue-font-size) * 0.4); + --cue-edge-color: black; + /* uniform outline drawn with `paint-order: stroke fill` (cheaper and cleaner than shadows) */ + --cue-text-stroke: 0.08em black; + --cue-text-shadow: none; } #captions [data-part='region'] { @@ -671,6 +1023,126 @@ easily customized with CSS. Here are all the parts you can select and customize: } ``` +Every part also exposes a matching `part` attribute, so the overlay can be styled from outside a +shadow root with `::part(cue)`, `::part(region)`, and so on. + +### WebVTT `STYLE` blocks + +`STYLE` blocks in a VTT file are parsed into `result.styles` and applied by +[`CaptionsRenderer`](#captionsrenderer). Selectors are rewritten to the rendered DOM and scoped +to the overlay, so several renderers on one page never leak styles into each other: + +```text +WEBVTT + +STYLE +::cue { color: papayawhip; } +::cue(b) { color: peachpuff; } +::cue(v[voice="Bob"]) { color: lime; } +::cue(:past) { color: gray; } +::cue-region(#top) { opacity: 0.8; } +``` + +Declarations are restricted to presentational properties (colour, background, font, text +decoration and shadow, outline, opacity, visibility, and similar) and anything that would load an +external resource such as `url()` or `@import` is dropped, so untrusted files can style captions +but never the page. `transformVTTStyle(css, scope)` is exported if you want to apply the same +rewriting yourself. + +### Cue layout and text style model + +Formats with absolute positioning (SSA/ASS, TTML, CEA-708) express placement and styling through +typed fields on the cue, never CSS strings. Each writer serialises them: the DOM renderer to CSS +variables and properties, the canvas renderer to pixels. Custom renderers can read them directly, +and cues survive `structuredClone` / `postMessage` and `toJSON`: + +```ts +// Lengths: pixels, or relative to the overlay (`vw`/`vh`), the font (`em`), or the box (`%`). +type CueLength = number | { unit: 'vw' | 'vh' | 'em' | '%'; value: number }; + +cue.layout = { + left: 50, // percentages of the overlay + bottom: 5, + width: 'max-content', // or 'auto' or a percentage + maxWidth: 90, + translate: { x: -0.5 }, // fraction of the cue box; centres the box on `left` + fixed: false, // true: never moved by collision avoidance (SSA \pos) + clip: { rect: [0, 0, 100, 50] }, // screen-fixed, overlay %; or { polygon }, or a box { inset } +}; + +cue.textStyle = { + color: 'rgba(255,255,255,1)', + fontSize: { unit: 'vh', value: 6.67 }, + stroke: { width: { unit: 'vh', value: 0.5 }, color: 'black' }, // painted behind the glyphs + shadow: { x: { unit: 'em', value: 0.06 }, y: { unit: 'em', value: 0.06 }, color: 'black' }, + transform: { rotate: -15, scaleX: 1.1, origin: [50, 0] }, // pivot in box %, or `originAt` on the overlay + wrap: 'nowrap', + padding: { y: 0 }, + textAlign: 'center', +}; + +// Per-run typography: cue text references `cue.spans` with . +cue.text = 'Normal bigger and a '; +cue.spans = { + big: { fontSize: { unit: 'em', value: 1.5 }, stroke: { width: 2, color: 'black' } }, + shape: { drawing: { path: 'M0 0 L10 0 L10 10 Z', viewBox: [0, 0, 10, 10], width: 5, height: 8 } }, + sung: { sweep: { sung: 'white', unsung: 'orange' } }, // karaoke fill, driven by a `sweep` keyframe +}; + +// Media-synchronised animations, sampled from `currentTime` so they scrub and pause with the +// video. Keyframes carry the same typed values: `opacity`, `color`, `strokeColor`, `strokeWidth`, +// `fontSize`, `letterSpacing`, `shadow`, `blur`, `transform`, `left`/`top` (overlay %), +// `translate` (box fraction), `clip`, and `sweep` (0..1). +cue.animations = [ + { duration: 0.5, keyframes: [{ opacity: 0 }, { opacity: 1 }] }, // fade in + { + target: { span: 'big' }, + delay: 1, + duration: 2, + keyframes: [{ color: 'white' }, { color: 'red' }], + }, +]; + +cue.style = { '--cue-padding-x': '0' }; // raw CSS escape hatch for the DOM writer, applied last + +JSON.stringify(cue); // plain object, region referenced by id +VTTCue.from(JSON.parse(json), regions); // rebuilds the cue +``` + +The stylesheet defaults live in `@layer media-captions`, so any unlayered author rule overrides +them regardless of specificity or order, and the colour and overlay size variables are registered +with `@property` so they are typed, have fallbacks, and can be transitioned. + +Cue text uses `text-wrap: balance`, which is what the WebVTT rendering rules ask for and which +browsers now support natively, and region (roll-up) cues use `text-wrap: stable` so earlier lines +never reflow. Japanese cues get `word-break: auto-phrase`. + +### Caption settings presets + +Players are expected to offer viewer controls for caption appearance. Set these attributes on the +overlay element (or the `` element's overlay via `renderer.overlay`): + +| Attribute | Values | +| --------------------- | -------------------------------------------- | +| `data-text-size` | `small`, `medium`, `large`, `x-large` | +| `data-contrast` | `high` | +| `data-background` | `none`, `translucent`, `opaque` | +| `data-font` | `sans`, `serif`, `mono`, `casual` | +| `data-edge-style` | see below | +| `data-reduced-motion` | set by the renderer's `reducedMotion` option | + +### Edge Styles + +FCC/CVAA guidelines require user-selectable character edge styles. Set `data-edge-style` on the +overlay element to `uniform`, `drop-shadow`, `raised`, `depressed`, or `none`, and customize the +colour with `--cue-edge-color`: + +```html +
+``` + +The uniform, drop-shadow, raised, and depressed edge styles + ## VTT Web Video Text Tracks (WebVTT) is the natively supported captions format supported @@ -708,17 +1180,9 @@ parseResponse(fetch('/subs/english.vtt'), { type: 'vtt' }); WebVTT supports regions for bounding/positioning cues and implementing roll up captions by setting `scroll:up`. -Visual explanation of VTT regions +Two VTT regions anchored top-left and bottom-right -Visual explanation of VTT region scroll up setting for roll up captions +Roll-up captions in a three line VTT region with scroll:up ### VTT Cues @@ -743,16 +1207,18 @@ cue.align = 'end'; cue.lineAlign = 'end'; ``` -Visual explanation of VTT cues +VTT cues positioned with line, position, size, and align settings ## SRT SubRip Subtitle (SRT) is a simple captions format that only contains cues. There are no -regions or positioning settings as found in [VTT](#vtt). +regions as found in [VTT](#vtt), but the parser understands the common extensions: + +- ``, ``, `` tags, and `` which maps to a WebVTT colour class (named + WebVTT colours, hex values, and common HTML colour names). +- `{\anN}` numpad alignment tags left over from ASS conversions (e.g., `{\an8}` for top placement). +- Extended `X1: X2: Y1: Y2:` coordinates on the timing line are ignored instead of rendered. +- `-->` without surrounding whitespace, and `.` or `,` as the milliseconds separator. SRT is a plain-text file that looks like this: @@ -801,29 +1267,195 @@ Continue dialogue on a new line. parseResponse(fetch('/subs/english.ssa'), { type: 'ssa' }); ``` +SSA/ASS styles rendered with outlines, an opaque box, colour and karaoke tags + The following features are supported: -- Multiple styles blocks and all format fields (e.g., PrimaryColour, Bold, ScaleX, etc.). -- Multiple events blocks and associating them with styles. +- `[Script Info]` (`PlayResX`/`PlayResY`, `WrapStyle`, `ScriptType`) with all sizes, margins, + outlines, and positions scaled to the play resolution so they track the overlay size. +- Multiple styles blocks and all format fields (colours with alpha, bold/italic/underline/strike, + scale, spacing, angle, border style, outline, shadow, alignment, margins), including legacy SSA + v4.00 alignment values. +- Multiple events blocks, `Layer` (rendered as z-order), `Name` (rendered as a voice span), and + per-dialogue margins. +- Override tags, mapped onto the structured cue model: + - Formatting: `\i`, `\b`, `\u`, `\s`, `\c`/`\1c`, `\2c`, `\3c`, `\4c`, `\alpha`/`\1a`, `\3a`, `\4a`, `\r` and + `\rStyle`. + - Per-run typography via `cue.spans`: `\fs`, `\fn`, `\fsp`, `\fscx`/`\fscy`, `\frx`/`\fry`/`\frz`, `\bord`, + `\xbord`/`\ybord`, `\shad`, `\xshad`/`\yshad`, `\blur`, `\be`. + - Placement: `\an`/`\a`, `\pos`, `\org` (rotation pivots on the alignment anchor by default), `\move` + (media-synced position animation), `\clip` (rectangles + anywhere, resolved against the final box; drawings on positioned cues as exact polygon clip + paths), `\q`. + - Animation: `\fad`, `\fade`, and `\t` (colours, alpha, scale, rotation, border, blur, font size, + spacing, shadow; acceleration is sampled; chained `\t` blocks compose). + - Karaoke: `\k` timestamps, `\kf`/`\K` fill sweeps, `\ko` outline highlights. + - Drawings: `\p` vector drawings (`m n l b s p c`, b-splines converted to cubics) rendered as inline SVG, + with `\bord` strokes. +- `Effect` events: `Scroll up`/`Scroll down` (clipped band animation) and `Banner`. +- `WrapStyle`, `Collisions` (drives the renderer's stacking mode), `ScaledBorderAndShadow`. +- Embedded fonts in `[Fonts]`, see [`loadEmbeddedFonts`](#loadembeddedfonts). + +All animations are Web Animations driven from media time, so they pause, seek, and scrub with the +video rather than running on the wall clock. + +The following are approximated or not supported: + +- `\iclip`, `\fax`/`\fay` shear, `\pbo`, `\kt`, `\fe`. +- Vector `\clip` drawings on non-positioned cues (rectangular clips work everywhere; on `\move` + cues the clip travels with the box). +- Karaoke sweeps use a text-clipped gradient, so strokes and shadows inside the syllable can show + through. +- Movie, Picture, Sound, and Command events. + +For pixel-exact libass parity on heavy typesetting (thousands of animated drawings per frame), +[SubtitlesOctopus](https://github.com/libass/JavascriptSubtitlesOctopus) remains an option. You'll +need to fall back to this implementation on iOS Safari (iPhone) as custom captions are not +supported there. + +## TTML + +Timed Text Markup Language (TTML) and its profiles IMSC1, DFXP, EBU-TT-D, and SMPTE-TT are XML +based and widely used in broadcast and DASH. The parser is a small tolerant XML tokenizer that +works server-side (no `DOMParser`). + +```ts +parseResponse(fetch('/subs/english.ttml'), { type: 'ttml' }); +``` + +Supported: clock and offset time expressions (including frames and ticks), time inheritance +across `body`/`div`/`p`/`span` (with the next paragraph's start used as a missing end), referential +and inline styling, regions (`tts:origin`, `tts:extent`, `tts:displayAlign`, `tts:textAlign`) in +percentages, pixels, or cells (`ttp:cellResolution`) mapped to cue positioning, `tts:fontSize` +mapped to a scaled cue font size, vertical writing modes (`tbrl`, `tblr`), italics, bold, +underline, colours, `xml:lang`, ruby, `
`, `xml:space`, and timed spans mapped to WebVTT +timestamp tags, `` animations on paragraphs, spans, regions, and containers (paragraphs are +split into styled slices), `seq` time containers, span and region timing, `tts:visibility`, +`tts:display`, `tts:opacity`, `tts:position`, `tts:padding`, `tts:lineHeight`, `tts:textOutline`, +`tts:textShadow`, span-level typography and colours (via `cue.spans`), `itts:forcedDisplay` (a +`forced` class plus `HasForcedCues` metadata), SMPTE-TT / IMSC image cues, wall-clock time bases +(made relative to the earliest cue, with `ClockStart` metadata so you can re-offset with +`shiftVTTCues`), and `ttp:dropMode` `dropNTSC` and `dropPAL`. 62 documents from the W3C IMSC test +suite parse under `tests/imsc`. Not supported: `rubyPosition`, `tts:showBackground`, bidi +overrides, and external image URLs (never fetched). + +## SCC (CEA-608) + +Scenarist Closed Caption (SCC) files carry raw CEA-608 byte pairs with SMPTE timecodes and are the +standard interchange format for broadcast captions in the US. + +```text +Scenarist_SCC V1.0 + +00:00:01:15 9420 9420 94ae 94ae 9452 9452 97a1 97a1 c8e5 ecec ef2e 942f 942f +00:00:03:00 942c 942c +``` + +```ts +parseResponse(fetch('/subs/english.scc'), { type: 'scc' }); +``` + +The parser decodes pop-on, roll-up, and paint-on captions, including special and extended +characters, colours, italics, underline, and the 15x32 row/column grid which is mapped to cue +`line`/`position`. Drop-frame (`;`) and non-drop timecodes are supported. CC1 is decoded by +default; pass `channel: 2` to decode CC2 instead. SCC files only carry field 1, so CC3/CC4 need +the stream decoders below. Text mode and XDS are not supported. + +## CEA-608/708 from video streams + +Broadcast and HLS/DASH streams carry captions inside the video as `cc_data` (ATSC A/53 user +data in MPEG-2, or SEI messages in H.264/H.265). Players such as hls.js and mux.js surface these +as byte triplets. Two stream decoders turn them into `VTTCue` objects that the renderer can show +like any other track: + +```ts +import { CEA608Decoder, CEA708Decoder, parseCCData } from 'media-captions/cea'; + +// CEA-608: channels 1/2 are on field 1, channels 3/4 on field 2. +const cc608 = new CEA608Decoder({ channel: 1, onCue: (cue) => renderer.addCue(cue) }); + +// CEA-708 (DTVCC): pick a service (1 is the primary caption service). +const cc708 = new CEA708Decoder({ service: 1, onCue: (cue) => renderer.addCue(cue) }); + +// `sei` is the payload of a user_data_registered_itu_t_t35 SEI message starting at "GA94", +// or a raw cc_data() structure. `pts` is the presentation time in seconds. +const triplets = parseCCData(sei); +cc608.decodeCCData(triplets, pts); +cc708.decodeCCData(triplets, pts); + +// When the stream ends, close any open cues. +cc608.flush(); +cc708.flush(); +``` + +The decoders live in the separate `media-captions/cea` entry so the core bundle stays small. +Both expose `cues` (everything emitted so far), `reset()`, and `flush(endTime?)`. The +608 decoder also accepts raw byte pairs via `decodePair(byte1, byte2, time, field)`, and is the +engine behind the SCC parser. The 708 decoder assembles DTVCC packets and service blocks, models +the eight caption windows with pen attributes (sizes, edges, colours), window fill and borders, +print and scroll directions, display effects (fade and wipe via `cue.textStyle.animation`), and +word wrapping, and maps window anchors to cue `line`/`position`, so positioned captions land where +the broadcaster placed them. Both decoders support `live: true`, which emits open-ended cues and +updates them in place through `onCueUpdate` (see [`CueTrack`](#cuetrack)). -The following features are not supported yet: +## LRC -- Layers -- Movie -- Picture -- Sound -- Command -- Font Loading -- Text Codes (stripped out for now) +LRC is the lyrics format used by music players. Enhanced LRC word timings are mapped to WebVTT +timestamp tags so karaoke styling works out of the box. -It is very likely we will implement custom font loading, layers, and text codes in the -near future. The rest is unlikely for now. You can always try and implement custom transitions -or animations using CSS (see [Styling](#styling)). +```text +[ti:Song Title] +[offset:-200] +[00:12.00]Line one +[00:17.20]<00:17.20>Word <00:17.80>by <00:18.40>word +``` + +```ts +parseResponse(fetch('/lyrics/song.lrc'), { type: 'lrc' }); +``` -We recommend using [SubtitlesOctopus](https://github.com/libass/JavascriptSubtitlesOctopus) for -SSA/ASS captions as it supports most features and is a performant WASM wrapper of -[libass](https://github.com/libass/libass). You'll need to fall back to this implementation on -iOS Safari (iPhone) as custom captions are not supported there. +ID tags are returned as metadata, `offset` is applied, and each cue ends when the next begins. + +## SBV + +SubViewer (SBV) is the simple format exported by YouTube: + +```text +0:00:00.000,0:00:02.000 +Hello, Joe! + +0:00:02.000,0:00:04.000 +Hello, [br]Jane! +``` + +```ts +parseResponse(fetch('/subs/english.sbv'), { type: 'sbv' }); +``` + +## SAMI + +Synchronized Accessible Media Interchange (`.smi`) is the HTML-like format from Windows Media +Player, still common in archives and in Korean subtitle distribution: + +```ts +parseResponse(fetch('/subs/movie.smi'), { type: 'smi' }); +``` + +Every language class in the file is emitted. Each cue's `id` is its class name and its text is +wrapped in `` when the class declares a language, so hosts can filter by either; +`metadata.Languages` lists the classes. Unclosed tags, ` ` clears, ``, and +out-of-order `SYNC` blocks are handled. + +## MicroDVD + +Frame-based `.sub` files (`{start}{end}Text|line`). The optional `{1}{1}25.000` header sets the +frame rate (default 23.976, reported as `metadata.FrameRate`). Formatting codes (`{y:i}`, +`{Y:b}`, `{c:$bbggrr}`, `{f:}`, `{s:}`) map to WebVTT tags and span styles; `{P:x,y}` positions map +to a fixed layout on an assumed 640x480 canvas. + +```ts +parseResponse(fetch('/subs/movie.sub'), { type: 'sub' }); +``` ## Streaming @@ -858,6 +1490,103 @@ async function handle() { } ``` +## HLS Segments + +WebVTT segments served over HLS carry an `X-TIMESTAMP-MAP` header that maps the segment's local +time to the MPEG-TS timeline. The header is preserved in the parse result metadata and can be +applied like so: + +```ts +import { parseResponse, parseVTTTimestampMap, shiftVTTCues } from 'media-captions'; + +const { metadata, cues } = await parseResponse(fetch(segmentURL)); +const map = parseVTTTimestampMap(metadata); + +if (map) { + // `initialPTS` is the first video PTS (90kHz) of the stream, as exposed by your HLS client. + shiftVTTCues(cues, map.offset - initialPTS / 90000); +} +``` + +## fMP4 subtitle tracks + +DASH and modern HLS deliver subtitles as ISOBMFF samples: `wvtt` (WebVTT) or `stpp` (TTML, +including IMSC images). The `media-captions/mp4` entry demuxes init and media segments straight +into cues, so a player does not have to unwrap the boxes itself: + +```ts +import { CaptionsRenderer, CueTrack } from 'media-captions'; +import { MP4SubtitleDemuxer } from 'media-captions/mp4'; + +const track = new CueTrack(undefined, { dedupe: true, retention: 60 }), + renderer = new CaptionsRenderer(overlay); +renderer.changeTrack({ cues: track }); + +const demuxer = new MP4SubtitleDemuxer({ onCue: (cue) => track.add(cue) }); +const tracks = demuxer.init(initSegmentBytes); // [{ id, type: 'wvtt' | 'stpp', timescale, language }] +demuxer.push(mediaSegmentBytes, { timeOffset: periodStart }); +renderer.changeTrack({ cues: track, regions: demuxer.regions, styles: demuxer.styles }); +``` + +`parseMP4Subtitles(init, segments)` is the one-shot equivalent that returns a normal parse result. + +## Player integration (hls.js) + +Wiring the pieces together for an HLS player with both WebVTT subtitle segments and embedded +CEA-608/708 captions: + +```ts +import Hls from 'hls.js'; +import { + CaptionsRenderer, + CueTrack, + parseText, + parseVTTTimestampMap, + shiftVTTCues, + syncCaptionsRenderer, +} from 'media-captions'; +import { CEA608Decoder, CEA708Decoder, parseCCData } from 'media-captions/cea'; + +const track = new CueTrack(undefined, { dedupe: true, retention: 60 }), + renderer = new CaptionsRenderer(overlay, { retention: 60 }); +renderer.changeTrack({ cues: track }); +syncCaptionsRenderer(renderer, video); + +// Embedded captions: hls.js exposes SEI user data per fragment. +const cc608 = new CEA608Decoder({ + channel: 1, + live: true, + onCue: (c) => track.add(c), + onCueUpdate: (c) => track.update(c), + }), + cc708 = new CEA708Decoder({ + service: 1, + live: true, + onCue: (c) => track.add(c), + onCueUpdate: (c) => track.update(c), + }); + +hls.on(Hls.Events.FRAG_PARSING_USERDATA, (_, data) => { + for (const sample of data.samples) { + const triplets = parseCCData(sample.bytes); + cc608.decodeCCData(triplets, sample.pts); + cc708.decodeCCData(triplets, sample.pts); + } +}); + +// WebVTT subtitle segments: parse each fragment and align it with X-TIMESTAMP-MAP. +hls.on(Hls.Events.FRAG_LOADED, async (_, data) => { + if (data.frag.type !== 'subtitle') return; + const { metadata, cues } = await parseText(new TextDecoder().decode(data.payload)); + const map = parseVTTTimestampMap(metadata); + if (map) shiftVTTCues(cues, map.offset - hls.initPTS / 90000); + track.addAll(cues); // dedupe drops the cues repeated across segment boundaries +}); +``` + +Disable hls.js's own subtitle rendering (`renderTextTracksNatively: false`, `enableCEA708Captions: false`) +so cues are not drawn twice. + ## Types Here's the types that are available from this package for use in TypeScript: @@ -869,20 +1598,76 @@ import type { CaptionsParserInit, CaptionsRenderer, CaptionsRendererTrack, + EmbeddedFont, ParseByteStreamOptions, ParseCaptionsOptions, ParsedCaptionsResult, ParseError, ParseErrorCode, ParseErrorInit, + SyncCaptionsRendererOptions, TextCue, VTTCue, VTTCueTemplate, VTTHeaderMetadata, VTTRegion, + VTTTimestampMap, } from 'media-captions'; ``` +## Development + +```bash +pnpm install +pnpm check # oxfmt + oxlint via Vite+ (`vp check`) +pnpm typecheck # tsc +pnpm test # unit suites (node + jsdom) and real-browser layout suites (Playwright) +pnpm test:unit +pnpm test:browser # Chromium; BROWSERS=chromium,firefox,webkit widens it (playwright install once) +pnpm build # vp pack (tsdown + publint + attw) -> dist/prod.js and the cea, element, entities, parsers/* entries +pnpm size # gzipped size budgets: published entries + tree-shaken usage probes (scripts/size-check.mjs) +pnpm coverage # unit suites with V8 coverage +pnpm run docs # TypeDoc API reference into docs/api (`pnpm docs` is npm's browser opener) +pnpm playground # interactive playground at http://localhost:3200/playground/index.html +pnpm sandbox # interactive scenarios at http://localhost:3100/.sandbox/index.html +pnpm screenshots # regenerates the README images from the sandbox scenarios +``` + +The whole toolchain is [Vite+](https://viteplus.dev): Vite, Vitest, Oxlint, Oxfmt, and tsdown are +configured together in `vite.config.ts` (`test`, `lint`, `fmt`, `pack`). + +### Playground + +`pnpm playground` opens an interactive playground at `http://localhost:3200/playground/index.html` +with a mocked media clock (play, scrub, rate, frame stepping, loop, jump to cue), built-in samples +for every parser plus a synthesised CEA-608/708 live stream, an editable source panel with file +loading, live renderer options (stacking, line step, safe area, announcer, reduced motion, edge +styles, presets, colours, shadow DOM), a debug overlay of layout boxes, an inspector (active cues, +all cues, metadata, errors, events, timeline), a `` element view, and a gallery +that renders every sample at once. Views are shareable via the URL. `playground/README.md` has the +details and `pnpm playground:screenshots` regenerates `playground/screenshots/`. + +Sandbox scenarios (used for the README images): `cues`, `regions`, `region-scroll`, `collisions`, +`ssa`, `edge-styles`, `layout` (the cue layout/text style model), `live` (a `CueTrack` fed +incrementally), and `element` (``). + +CI (`.github/workflows/ci.yml`) runs formatting, linting, type-checking, the unit, WPT, IMSC, +corpus, and fuzz suites with coverage, the layout suites in Chromium, Firefox, and WebKit, the build with package checks, +the size budgets, and publishes the API docs as an artifact. `release.yml` publishes to npm with +provenance when a `v*` tag is pushed. Screenshot baselines are per platform; the Linux Chromium +baselines are committed so the visual suite runs in CI, and the manual `record-baselines.yml` +workflow re-records them (download its artifact) after an intentional rendering change. + +Parsing is covered by the vendored web-platform-tests WebVTT suites under `tests/wpt` (118 +tests, all passing with `lenient: false`; `tests/wpt/KNOWN_DIVERGENCES.md` lists the lenient-mode +tolerances and the bugs the suite found), by the W3C IMSC test documents +under `tests/imsc`, by hand-written conformance suites under `tests/conformance` (WebVTT file +structure, cue text, SSA/ASS incl. typesetting), and by per-format suites. Rendering is measured in Chromium, Firefox, and WebKit under +`tests/browser` (stacking, line snapping, percentage lines, position/size/align, vertical text, +RTL, regions, resize, SSA layout, transforms). `tests/browser/visual.test.ts` adds screenshot +comparisons with a small pixel tolerance; baselines live in `tests/browser/__screenshots__` per +browser and platform, so the first run on a new platform records them and later runs compare. + ## ๐Ÿ“ License Media Captions is [MIT licensed](./LICENSE). diff --git a/assets/collisions.png b/assets/collisions.png new file mode 100644 index 0000000..dbe44f3 Binary files /dev/null and b/assets/collisions.png differ diff --git a/assets/edge-styles.png b/assets/edge-styles.png new file mode 100644 index 0000000..f24d7e1 Binary files /dev/null and b/assets/edge-styles.png differ diff --git a/assets/layout-model.png b/assets/layout-model.png new file mode 100644 index 0000000..9da4717 Binary files /dev/null and b/assets/layout-model.png differ diff --git a/assets/live-track.png b/assets/live-track.png new file mode 100644 index 0000000..557c283 Binary files /dev/null and b/assets/live-track.png differ diff --git a/assets/ssa.png b/assets/ssa.png new file mode 100644 index 0000000..6ccc01b Binary files /dev/null and b/assets/ssa.png differ diff --git a/assets/vtt-cues.png b/assets/vtt-cues.png index ab11e33..b3817c2 100644 Binary files a/assets/vtt-cues.png and b/assets/vtt-cues.png differ diff --git a/assets/vtt-region-scroll.png b/assets/vtt-region-scroll.png index f1bdf4e..e1f49a3 100644 Binary files a/assets/vtt-region-scroll.png and b/assets/vtt-region-scroll.png differ diff --git a/assets/vtt-regions.png b/assets/vtt-regions.png index 0d8db17..d90b4d6 100644 Binary files a/assets/vtt-regions.png and b/assets/vtt-regions.png differ diff --git a/cliff.toml b/cliff.toml index 86450f7..286e9e2 100644 --- a/cliff.toml +++ b/cliff.toml @@ -31,9 +31,9 @@ commit_parsers = [ { message = "^feat", group = "โœจ Features" }, { message = "^fix", group = "๐Ÿ› Bug Fixes" }, { message = "^perf", group = "๐ŸŽ๏ธ Performance" }, - { message = "^refactor", group = "Refactor", skip=true }, - { message = "^style", group = "Styling", skip=true }, - { message = "^test", group = "Testing", skip=true }, + { message = "^refactor", group = "Refactor", skip = true }, + { message = "^style", group = "Styling", skip = true }, + { message = "^test", group = "Testing", skip = true }, { message = "^chore", skip = true }, { message = "^docs", skip = true }, { body = ".*security", group = "๐Ÿ”’ Security" }, diff --git a/docs/MIGRATION.md b/docs/MIGRATION.md new file mode 100644 index 0000000..5ac4409 --- /dev/null +++ b/docs/MIGRATION.md @@ -0,0 +1,104 @@ +# Migrating from media-captions 1.x + +This release is a new package: the parsers were rewritten for spec conformance, the renderer was +rebuilt as a composable core with features, a canvas writer was added, and the cue style model is +typed. Most call sites keep working unchanged: `parseText`, `parseResponse`, `parseByteStream`, +`CaptionsRenderer`, `renderVTTCueString`, `tokenizeVTTCue`, `VTTCue`, `VTTRegion`, and the parse +options are all still there. What follows is everything that can break. + +## Package and entry points + +- **ESM only.** No CommonJS build. `main`/`exports` point at ESM files. +- **New entry points.** `media-captions/renderer` (composable core and features), + `media-captions/canvas` (canvas writer), `media-captions/cea` (CEA-608/708 from video streams), + `media-captions/mp4` (fMP4 `wvtt`/`stpp` demux), `media-captions/element` + (``), `media-captions/entities` (full HTML entity table), and + `media-captions/parsers/` for bundlers that can not follow dynamic imports. The main + entry still lazy-loads parsers by `type`. +- **New formats** need a `type` or a recognisable file: `ttml`, `scc`, `lrc`, `sbv`, `smi`, `sub` + join `vtt`, `srt`, `ssa`/`ass`. `inferCaptionsFormat(text)` sniffs a string. + +## Parsing + +- **WebVTT is spec-strict about structure.** The signature must be `WEBVTT` followed by a space, + tab, or line end (a BOM is allowed). A missing signature is reported and tolerated outside + `strict` mode. `strict` also enforces the timestamp grammar and requires `%` on percentages; + lenient mode still accepts comma separators, bare numbers, and legacy `align:middle`. +- **Cue settings are only read from the timing line.** The 1.x tolerance for settings on the line + after the timing line is gone; it swallowed real caption text that began with `line:`, + `position:`, or `align:`. +- **Numeric settings must match the spec grammar** (no exponents, prefixes, or negative zero), + compound `line`/`position` values are atomic, `REGION` lines are digits only, and a `-->` line + discards a `REGION` block. Cues whose end is not after their start are kept and reported, as + browsers do. +- **Cue text is escaped.** Rendered HTML never contains raw markup from the file. Class names are + restricted to word characters and hyphens, unknown end tags do not corrupt nesting, and + timestamps render as sibling `` elements. +- **SSA/ASS is a full typesetting parser.** Sizes, margins, and outlines scale with `PlayResX`/ + `PlayResY` (default 384x288). Override tags, layers, `\pos`/`\move`/`\fad`/`\t`, karaoke, `\p` + drawings, `\clip`, `Effect` fields, and embedded fonts (`loadEmbeddedFonts`) are supported. + Output is expressed through `cue.layout`, `cue.textStyle`, `cue.spans`, and `cue.animations` + rather than `--cue-*` custom properties. +- **Hostile input is bounded.** Regexes are linear, nesting depth is capped, and malformed lines + never throw outside `strict`. +- **`lenient: false`** is new: the spec grammar (what `strict` enforces) without the throw, so + invalid cues are dropped and reported the way a browser would. The default stays lenient. +- **Mismatched end tags follow the spec.** `` while `` is the current node is ignored + instead of closing the `` ancestor, so `x y` keeps ` y` bold italic (as browsers + render it). Unknown end tags are still ignored. + +## Cue model + +`VTTCue` gained structured fields that parsers of positioned formats fill in. If you read cues +directly (custom renderers, analytics, caches), note: + +- `cue.layout`, `cue.textStyle`, `cue.spans`, `cue.animations`, `cue.layer` are new. + `cue.style` (raw CSS custom properties) remains as an escape hatch and is applied last. +- **Values, not CSS strings.** Lengths are `number` (px) or `{ unit: 'vw' | 'vh' | 'em' | '%', +value }`. Transforms are `{ scaleX, scaleY, rotate, rotateX, rotateY, origin | originAt }`. + Strokes are `{ width, color }`, shadows `{ x, y, blur?, color }`. Font weight is a number, + italic/underline/strike are booleans, opacity is a number. Karaoke sweeps are + `span.sweep = { sung, unsung }` plus a `sweep` keyframe (0..1). Images are + `textStyle.image = { url, fit? }`. Clips are `layout.clip`: `{ rect }` or `{ polygon }` in + overlay percentages (screen-fixed) or `{ inset }` in box percentages. +- **Keyframes are typed** the same way: `opacity`, `color`, `strokeColor`, `strokeWidth`, + `fontSize`, `letterSpacing`, `shadow`, `blur`, `transform`, `left`/`top` (overlay %), + `translate` (box fraction), `clip`, `sweep`. +- `toJSON()` / `VTTCue.from(json, regions)` round-trip everything; regions are referenced by id. +- Open-ended live cues use `endTime: Infinity` (`null` in JSON). +- `cue.region` is our own property on the native base class in Firefox and WebKit (their native + setter rejected our `VTTRegion`). + +## Rendering + +- **Stylesheet.** Defaults live in `@layer media-captions`, so any unlayered author rule wins + regardless of specificity. Colour and overlay size variables are registered with `@property`. + Cue boxes use `contain: layout style` (paint containment clipped transformed text). Image cues + are `background-size: contain`. `regions.css` is still separate. +- **Renderer init options** grew: `retention`, `announce`, `stacking`, `lineStep`, `safeArea`, + `reducedMotion`, `features`. `changeTrack` also takes `metadata` and `styles`, and `cues` may be a + `CueTrack` for live content. `activeCues`, `track`, `attachTrack`, and cue `enter`/`exit` events + are new. `dir` and `currentTime` are unchanged. +- **Composable renderer.** `CaptionsRenderer` is now the core with every feature installed. + `createRenderer(overlay, { features })` from `media-captions/renderer` lets you pick `regions()`, + `typesetting()`, `animations()`, `vttStyles()`, `announcer()`. Without a feature the matching + cue fields are ignored (with a development warning), not errors. +- **Canvas renderer.** `CanvasCaptionsRenderer` and `paintCaptions` in `media-captions/canvas` + paint the same cue model into a 2D context; `syncCaptionsRenderer` accepts either renderer. +- **Token renderers.** `renderVTTTokensString(tokens, currentTime, layout?)`, + `renderVTTTokensDOM(tokens, currentTime, doc?, layout?)`, and + `getVTTTokenAttributes(token, currentTime, layout?)` take the cue layout as a trailing optional + argument (only needed for SSA `\org` pivots on spans). `renderVTTTokensDOM` builds DOM nodes + without `innerHTML`, so it works under strict CSP. + +## Quick fixes + +| 1.x | Now | +| ----------------------------------------------------- | ---------------------------------------------------------------------------------------------- | +| `require('media-captions')` | `import ... from 'media-captions'` | +| Settings on the line after the timing line | Move them onto the timing line | +| `cue.style['--cue-left']` etc. from the SSA parser | `cue.layout.left` (overlay %) | +| Reading `textStyle.fontSize` as a CSS string | `lengthToCSS(textStyle.fontSize)` from the main entry, or resolve the typed value | +| `layout.clipPath` / `layout.clipRect` | `layout.clip` | +| `video.addEventListener('timeupdate', ...)` sync loop | `syncCaptionsRenderer(renderer, video)` (frame accurate) | +| Custom overlay markup styling by element | `data-part` / `part` attributes (`captions`, `cue-display`, `cue`, `region`, `timed`, `voice`) | diff --git a/docs/design/canvas-and-style-model.md b/docs/design/canvas-and-style-model.md new file mode 100644 index 0000000..710ac85 --- /dev/null +++ b/docs/design/canvas-and-style-model.md @@ -0,0 +1,165 @@ +# Canvas rendering and the cue style model + +Status: September 2026. `media-captions/canvas` ships as an experimental writer, and the typed +style model proposed below has been implemented (see "Outcome"). This note records what the canvas +work showed about the renderer architecture and the parser to renderer contract. + +## What we have + +The pipeline is parser -> cue model -> renderer. The cue model is `VTTCue` plus four extension +fields the typesetting formats fill in: + +- `layout`: box placement as overlay percentages, a translation as a fraction of the box, and a + `clipPath` **CSS string**. +- `textStyle` / `spans`: presentation as **CSS value strings** (`fontSize: +'calc(var(--overlay-height) * 0.0667)'`, `transform: 'scaleX(1.2) rotate(-10deg)'`, + `textStroke: 'calc(...) rgba(...)'`, `backgroundImage: 'linear-gradient(...)'`). +- `animations`: Web Animations keyframes with CSS property names and string values. +- `drawing`: SVG path data. + +The DOM renderer hands these strings to the browser, which is why they are CSS. It works, but it +leaks the DOM writer's needs into the model and forces a set of workarounds: + +| Workaround | Why it exists | +| -------------------------------------------------------- | ------------------------------------------------------------------ | +| `calc(var(--overlay-width) * K + T%)` strings | Parsers do not know the pixel size; CSS resolves it at paint time | +| `display: inline-block` on transformed spans | CSS transforms do not apply to inline boxes | +| `--cue-transform-origin` variable on two elements | The `\t` target and the box must share a pivot | +| `data-fixed` attribute read back during measure | Layout learns "do not move me" from the DOM instead of the model | +| `LAYOUT_CACHE` symbol on elements | Measuring the DOM is expensive, so results hide on the node | +| `requestAnimationFrame` before `data-active` | Region scroll transitions need a resting frame first | +| `clipRect` resolved at write time into `--cue-clip-path` | A screen-fixed clip on a layout-positioned box needs the final box | +| `contain: layout style` (not `paint`) | Paint containment clipped rotated/scaled text | +| Sweep gradients with `background-clip: text` | Karaoke fill needs a glyph mask; CSS only offers this trick | + +None of these are bugs. They are the cost of expressing a typesetting model in CSS. + +## What the canvas writer showed + +`src/canvas` re-implements the write phase (and the measure phase) against a 2D context, reusing +the cue model, `CueTrack`, `orderForPositioning`, and the pure layout engine unchanged. + +**The layout engine was already the right boundary.** `CueLayoutInput` is the whole contract +between measurement and layout. The headless measurer (`src/canvas/measure.ts`) produces it from +text metrics, and the browser tests show the boxes land within a couple of pixels of the DOM +renderer for the WebVTT tier. Positioning, stacking, and collision avoidance are shared code. + +**Things that were hacks in the DOM are one line on a canvas.** + +- Screen-fixed clips: `ctx.rect(); ctx.clip()` in overlay pixels. No write-time resolution. +- Transform pivots: `translate(origin); rotate(); scale(); translate(-origin)`. No variable + plumbing between two elements, no inline-block. +- Drawings: `new Path2D(svgPath)`. No SVG element construction. +- Fixed cues and region activation: plain data, no attributes or animation frames. +- Animations: sampled at media time from the keyframes (`src/canvas/animate.ts`). No paused Web + Animation objects to create, track, and seek. +- Measurement cache: a `Map` keyed by frame size. + +**What became harder.** Line breaking, balancing, font fallback, bidi, and accessibility all come +for free in the DOM and had to be written or given up. The canvas flow is greedy wrapping with a +balance pass and character-level overflow; vertical text and ruby annotations had to be written by +hand; there is no selection and no screen reader access (the announcer feature covers the last +one). + +**The bridge that should not exist.** `src/canvas/css-values.ts` parses the CSS strings back into +numbers: `calc()` lengths, transform lists, shadows, strokes, clip polygons, colours. It works +because parsers only emit a small dialect, but it is a second interpreter for our own output, and +every new string form needs updating in two places. + +## Proposal: a typed style model + +Make the cue model carry values, and let each writer serialise them. + +```ts +type Length = + | number // pixels (rare; images with known sizes) + | { unit: 'vw' | 'vh' | 'em' | '%'; value: number }; // of the overlay, the font, or the box + +type Color = string; // normalised `#rrggbbaa` + +interface Transform { scaleX?: number; scaleY?: number; rotate?: number; rotateX?: number; rotateY?: number; origin?: [Length, Length] } +interface Stroke { width: Length; color: Color } +interface Shadow { x: Length; y: Length; blur?: Length; color: Color } +interface Fill { color: Color } | { sweep: { from: Color; to: Color; progress: number } } + +interface CueTextStyle { + color?: Color; background?: Color; fontFamily?: string; fontSize?: Length; fontWeight?: number; + italic?: boolean; decoration?: ('underline' | 'line-through')[]; letterSpacing?: Length; + lineHeight?: Length | 'normal'; opacity?: number; textAlign?: ...; wrap?: 'normal' | 'none'; + stroke?: Stroke; shadow?: Shadow; outline?: Stroke; padding?: { x?: Length; y?: Length }; + transform?: Transform; image?: { url: string; fit: 'contain' | 'cover' | 'fill' }; +} + +interface CueLayout { + left?: Length; top?: Length; right?: Length; bottom?: Length; + width?: Length | 'auto' | 'max-content'; maxWidth?: Length; height?: Length; + anchor?: { x: number; y: number }; // was `translate`: fraction of the box + fixed?: boolean; + clip?: { rect: [Length, Length, Length, Length] } | { polygon: [Length, Length][] } // overlay-relative +} + +interface CueAnimation { + target?: 'display' | 'cue' | { span: string }; + delay?: number; duration: number; easing?: ...; + keyframes: { offset?: number; opacity?: number; color?: Color; left?: Length; top?: Length; transform?: Transform; stroke?: Partial; fill?: Fill }[]; +} +``` + +**Writers become serialisers.** The DOM writer turns `{ unit: 'vh', value: 6.67 }` into +`calc(var(--overlay-height) * 0.0667)` and `Transform` into a `transform` string with its origin; +the canvas writer turns the same values into pixels and context calls; the string renderer +(`renderVTTCueString`) keeps emitting inline CSS. `toJSON` output becomes stable and readable. + +**Parsers get simpler.** `ssa-parser.ts` currently formats CSS in a dozen places (`_lenY`, +`toRGBA`, transform joins, gradient strings, `calc()` clip polygons). With typed values it emits +`{ unit: 'vh', value: bord * 2 / playResY * 100 }` and is done. Karaoke sweeps become a `Fill` +with progress driven by the animation, and each writer decides how to paint a partial fill (a +gradient mask in CSS, a clipped double fill on canvas). + +**Migration.** Additive, in three steps, no flag day: + +1. Add the typed fields alongside the strings (`fontSize` stays; `fontSizeValue` or a `v2` + namespace), with a `toCSS()` helper both writers can use. Parsers emit both for one release. +2. Switch the DOM writer to serialise from typed values; the CSS strings become derived and + deprecated in `toJSON`. +3. Drop the strings and `css-values.ts`. `renderVTTCueString` serialises typed spans. + +Since this ships as a new package with breaking changes allowed, steps 1 and 2 can collapse into +one release and step 3 can follow once TTML and CEA-708 are converted (SSA is the bulk). + +## Outcome + +The typed model landed in one step, since the package ships with breaking changes allowed: + +- `CueLength`, `CueTransform` (with `origin` in box percentages or `originAt` on the overlay), + `CueStroke`, `CueShadow`, `CueSweep`, `CueClip` (`rect`/`polygon` overlay-relative, `inset` + box-relative), and typed `CueKeyframe`s replaced every CSS string in `CueTextStyle`, + `CueSpanStyle`, `CueLayout`, and `CueAnimation`. +- `src/vtt/style-css.ts` is the DOM/string writers' serialiser; `src/canvas/values.ts` is the + canvas writer's resolver. `src/canvas/css-values.ts` (the bridge that should not exist) is gone. +- SSA, TTML, CEA-708, and MicroDVD parsers emit values. `_lenY` returns a `vh` length; clips are + overlay percentages for positioned and layout-positioned cues alike; karaoke is a `sweep` fill + with a 0..1 keyframe; `\org` is an overlay point. +- Clips are resolved once per layout in the DOM write phase with pixel arithmetic; only animated + clips (scroll bands) still go through the `calc(var(--overlay-*))` form, generated by the writer + per keyframe. +- Bundle effect: the canvas renderer lost ~1.1 KB (no CSS parsing), the DOM renderer gained + ~1.2 KB (it now serialises), and the SSA/TTML parser chunks shrank. + +## What canvas unlocks next + +- **Burn-in and export.** `paintCaptions(ctx, cues, time)` onto `VideoFrame`s in a WebCodecs + pipeline, or in a Worker with `OffscreenCanvas`. The API is already stateless. +- **iOS fullscreen and picture-in-picture.** Paint into a canvas, `captureStream()` it, and + composite with the video; the only route to custom captions in those surfaces. +- **Thumbnails and previews.** One call per time; no DOM, no layout thrash. +- **Deterministic visual tests.** Rasterised goldens do not drift with browser text stacks. +- **DOM-less runtimes.** Smart TVs with a canvas but a weak DOM, Node with a canvas binding. + +## Known gaps in the canvas writer + +Perspective (3D rotations project orthographically, as CSS does without `perspective`), karaoke +sweep gradients (final colour), blur filters, `\move` easing beyond linear, +STYLE blocks (no CSS engine), and the `text-wrap: balance` heuristic is an approximation of the +browser's. Fonts are limited to what the canvas can resolve by name; `loadEmbeddedFonts` still +works since it registers `FontFace`s document-wide. diff --git a/package.json b/package.json index a70e65c..b6339f5 100644 --- a/package.json +++ b/package.json @@ -2,51 +2,43 @@ "name": "media-captions", "version": "1.0.4", "description": "Media captions parser and renderer.", - "license": "MIT", - "type": "module", - "main": "./dist/prod.js", - "types": "dist/types/index.d.ts", - "sideEffects": false, - "jsdelivr": "./dist/prod.js", - "engines": { - "node": ">=16" - }, - "files": [ - "*.d.ts", - "dist/", - "styles/" + "keywords": [ + "ass", + "captions", + "cea-608", + "cea-708", + "cues", + "custom-element", + "dash", + "engine", + "fast", + "fmp4", + "hls", + "lightweight", + "media", + "parser", + "player", + "regions", + "scc", + "srt", + "ssa", + "stpp", + "streaming", + "text-tracks", + "ts", + "ttml", + "typescript", + "video", + "vidstack", + "vtt", + "web", + "web-component", + "wvtt" ], - "scripts": { - "dev": "pnpm clean && rollup -c -w & pnpm run types -w", - "build": "pnpm clean && rollup -c && pnpm types", - "types": "tsc -p tsconfig.build.json", - "clean": "rimraf dist", - "format": "prettier src --write --loglevel warn", - "changelog": "pnpm exec git cliff --output CHANGELOG.md", - "test": "vitest --run", - "test:watch": "vitest --watch --single-thread", - "sandbox": "node ./.sandbox/launch.js", - "release": "pnpm validate && pnpm changelog && git push --follow-tags origin main && npm publish --tag next", - "validate": "pnpm test && pnpm build" - }, - "devDependencies": { - "@ianvs/prettier-plugin-sort-imports": "^3.7.0", - "acorn": "^8.8.2", - "acorn-walk": "^8.2.0", - "conventional-changelog-cli": "^4.1.0", - "esbuild": "^0.18.1", - "git-cliff": "^1.4.0", - "magic-string": "^0.30.0", - "prettier": "^2.8.4", - "rimraf": "^4.4.1", - "rollup": "^3.25.1", - "rollup-plugin-esbuild": "^5.0.0", - "tslib": "^2.5.0", - "typescript": "^5.0.0", - "undici": "^5.21.0", - "vite": "^4.2.0", - "vitest": "^0.29.0" + "bugs": { + "url": "https://github.com/vidstack/media-captions/issues" }, + "license": "MIT", "contributors": [ "Rahim Alwer " ], @@ -54,42 +46,172 @@ "type": "git", "url": "https://github.com/vidstack/media-captions.git" }, - "bugs": { - "url": "https://github.com/vidstack/media-captions/issues" - }, + "files": [ + "dist/", + "styles/" + ], + "type": "module", + "sideEffects": false, + "main": "./dist/prod.js", + "types": "./dist/prod.d.ts", + "jsdelivr": "./dist/prod.js", "exports": { ".": { - "types": "./dist/types/index.d.ts", + "types": "./dist/prod.d.ts", "test": "./dist/dev.js", "development": "./dist/dev.js", "default": "./dist/prod.js" }, + "./cea": { + "types": "./dist/prod-cea.d.ts", + "test": "./dist/dev-cea.js", + "development": "./dist/dev-cea.js", + "default": "./dist/prod-cea.js" + }, + "./element": { + "types": "./dist/prod-element.d.ts", + "test": "./dist/dev-element.js", + "development": "./dist/dev-element.js", + "default": "./dist/prod-element.js" + }, + "./entities": { + "types": "./dist/prod-entities.d.ts", + "test": "./dist/dev-entities.js", + "development": "./dist/dev-entities.js", + "default": "./dist/prod-entities.js" + }, + "./mp4": { + "types": "./dist/prod-mp4.d.ts", + "test": "./dist/dev-mp4.js", + "development": "./dist/dev-mp4.js", + "default": "./dist/prod-mp4.js" + }, + "./renderer": { + "types": "./dist/prod-renderer.d.ts", + "test": "./dist/dev-renderer.js", + "development": "./dist/dev-renderer.js", + "default": "./dist/prod-renderer.js" + }, + "./canvas": { + "types": "./dist/prod-canvas.d.ts", + "test": "./dist/dev-canvas.js", + "development": "./dist/dev-canvas.js", + "default": "./dist/prod-canvas.js" + }, + "./parsers/vtt": { + "types": "./dist/prod-parser-vtt.d.ts", + "test": "./dist/dev-parser-vtt.js", + "development": "./dist/dev-parser-vtt.js", + "default": "./dist/prod-parser-vtt.js" + }, + "./parsers/srt": { + "types": "./dist/prod-parser-srt.d.ts", + "test": "./dist/dev-parser-srt.js", + "development": "./dist/dev-parser-srt.js", + "default": "./dist/prod-parser-srt.js" + }, + "./parsers/ssa": { + "types": "./dist/prod-parser-ssa.d.ts", + "test": "./dist/dev-parser-ssa.js", + "development": "./dist/dev-parser-ssa.js", + "default": "./dist/prod-parser-ssa.js" + }, + "./parsers/ttml": { + "types": "./dist/prod-parser-ttml.d.ts", + "test": "./dist/dev-parser-ttml.js", + "development": "./dist/dev-parser-ttml.js", + "default": "./dist/prod-parser-ttml.js" + }, + "./parsers/scc": { + "types": "./dist/prod-parser-scc.d.ts", + "test": "./dist/dev-parser-scc.js", + "development": "./dist/dev-parser-scc.js", + "default": "./dist/prod-parser-scc.js" + }, + "./parsers/lrc": { + "types": "./dist/prod-parser-lrc.d.ts", + "test": "./dist/dev-parser-lrc.js", + "development": "./dist/dev-parser-lrc.js", + "default": "./dist/prod-parser-lrc.js" + }, + "./parsers/sbv": { + "types": "./dist/prod-parser-sbv.d.ts", + "test": "./dist/dev-parser-sbv.js", + "development": "./dist/dev-parser-sbv.js", + "default": "./dist/prod-parser-sbv.js" + }, + "./parsers/sami": { + "types": "./dist/prod-parser-sami.d.ts", + "test": "./dist/dev-parser-sami.js", + "development": "./dist/dev-parser-sami.js", + "default": "./dist/prod-parser-sami.js" + }, + "./parsers/microdvd": { + "types": "./dist/prod-parser-microdvd.d.ts", + "test": "./dist/dev-parser-microdvd.js", + "development": "./dist/dev-parser-microdvd.js", + "default": "./dist/prod-parser-microdvd.js" + }, "./styles/*": "./styles/*", "./package.json": "./package.json" }, "publishConfig": { "access": "public" }, - "keywords": [ - "ass", - "captions", - "cues", - "engine", - "fast", - "lightweight", - "media", - "parser", - "player", - "regions", - "srt", - "ssa", - "streaming", - "text-tracks", - "ts", - "typescript", - "video", - "vidstack", - "vtt", - "web" - ] + "scripts": { + "dev": "vp pack --watch", + "build": "vp pack", + "clean": "rimraf dist", + "check": "vp check", + "lint": "vp lint", + "format": "vp fmt", + "format:check": "vp fmt --check", + "typecheck": "tsc -p tsconfig.json --noEmit", + "test": "vp test --run", + "test:unit": "vp test --run --project unit", + "test:browser": "vp test --run --project browser", + "test:watch": "vp test --watch", + "sandbox": "node ./.sandbox/launch.js", + "screenshots": "node ./scripts/screenshots.mjs", + "changelog": "pnpm exec git cliff --output CHANGELOG.md", + "validate": "pnpm check && pnpm typecheck && pnpm test && pnpm build && pnpm size", + "release": "pnpm validate && pnpm changelog && git push --follow-tags origin main && npm publish --tag next", + "size": "node scripts/size-check.mjs", + "docs": "typedoc", + "coverage": "vp test --run --project unit --coverage", + "playground": "vp dev --open=/playground/index.html --port=3200", + "bench": "vp test bench --run --project unit", + "playground:screenshots": "node playground/screenshot.mjs" + }, + "devDependencies": { + "@arethetypeswrong/core": "^0.18.5", + "@types/node": "^26.4.1", + "@vitest/browser-playwright": "^4.1.11", + "@vitest/coverage-v8": "4.1.11", + "git-cliff": "^2.13.1", + "jsdom": "^30.0.1", + "playwright": "^1.62.1", + "publint": "^0.3.24", + "rimraf": "^6.1.3", + "rolldown": "^1.2.7", + "tslib": "^2.8.1", + "typedoc": "^0.28.20", + "typescript": "^5.9.3", + "undici": "^8.10.1", + "vite-plus": "^0.3.0", + "vitest": "4.1.11" + }, + "engines": { + "node": ">=18" + }, + "packageManager": "pnpm@10.5.2", + "pnpm": { + "overrides": { + "vite": "npm:@voidzero-dev/vite-plus-core@0.3.0", + "vitest": "4.1.11" + }, + "onlyBuiltDependencies": [ + "esbuild" + ] + } } diff --git a/playground/README.md b/playground/README.md new file mode 100644 index 0000000..0e12654 --- /dev/null +++ b/playground/README.md @@ -0,0 +1,131 @@ +# Playground + +An interactive visual playground for every parser and renderer feature in `media-captions`. It +imports the library straight from `../src`, so it always reflects the working tree. + +```bash +pnpm playground # vp dev --open=/playground/index.html --port=3200 +# or, without pnpm: +./node_modules/.bin/vp dev --open=/playground/index.html --port=3200 +``` + +Vanilla TypeScript + CSS, no framework, no dependencies. Everything lives under `playground/`: + +| Path | What it is | +| ----------------------- | ----------------------------------------------------------------------- | +| `index.html`, `main.ts` | Entry point; wires the panels together and runs the frame loop | +| `styles.css` | Dark UI styling (the library's own `styles/*.css` are loaded alongside) | +| `ui/media.ts` | `FakeMediaElement`: the mock clock / media element | +| `ui/stage.ts` | The mock video surface with a moving background and layout-box overlay | +| `ui/transport.ts` | Play/pause, scrub, rate, stepping, loop, jump to cue | +| `ui/timeline.ts` | Cue bars under the scrub bar | +| `ui/sources.ts` | Format dropdown, editable source, Apply, file picker | +| `ui/options.ts` | Renderer and styling controls | +| `ui/inspector.ts` | Active cues, cue table, metadata, errors, events | +| `ui/gallery.ts` | All samples at once | +| `ui/live-cea.ts` | Feeds synthesised `cc_data` into the live CEA-608/708 decoders | +| `ui/state.ts` | URL state (`?format=ass&t=12.5&rate=1&...`) | +| `samples/*.ts` | One built-in sample per format (`{ name, type, text }`) or a generator | +| `samples/cea-encode.ts` | CEA-608 pair and CEA-708 packet encoders (adapted from `tests/cea`) | +| `screenshot.mjs` | Headless walk through every scenario; asserts zero console errors | +| `screenshots/*.png` | Output of `screenshot.mjs` | + +## Mock media + +There is no real video. The "video" is a 16:9 stage (switchable to 4:3 / 9:16, 320-1280px wide) +with a moving pattern so playback, scrubbing, and pausing are visible. `FakeMediaElement` +(`ui/media.ts`) is an `EventTarget` with `currentTime`, `paused`, `duration`, `playbackRate`, +`play()`/`pause()`, the `timeupdate` / `playing` / `pause` / `seeking` / `seeked` / `ended` events, +and `requestVideoFrameCallback`. The clock is advanced from a `requestAnimationFrame` loop that +accumulates `performance.now()` deltas ร— rate. + +Two driving modes (Renderer options โ†’ Driving): + +- **`renderer.currentTime` each frame**: the loop assigns the time directly. +- **`syncCaptionsRenderer(fakeMedia)`**: the library helper drives the renderer from the fake + element's events and frame callbacks, exactly as it would with a real `