From e517d6e0b570e52f7f8f836176bee4a5e7fdfd22 Mon Sep 17 00:00:00 2001 From: han4wluc Date: Tue, 13 Oct 2026 02:25:32 +0800 Subject: [PATCH 1/3] fix: count auto-forward reading units with the bundled segmenter MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Reading units came from the host Intl.Segmenter whenever one existed, so the auto-forward delay of the same line depended on the browser's or runtime's ICU data. Node 22.12 (ICU 76.1, Unicode 16.0) segments 10 of the 766 official Unicode 17.0 GraphemeBreakTest cases and 2,976 of 15,568,896 code-point probes differently from Node 24.21 (ICU 78.3, Unicode 17.0). Khmer, Myanmar, Gujarati and Balinese conjuncts and U+2701 U+200D U+2701 change their unit counts; "ភាសាខ្មែរ" counts 5 units there and 4 on Unicode 17.0 hosts. Always use the bundled unicode-segmenter (Unicode 17.0), which already served hosts without Intl.Segmenter. On Unicode 17.0 hosts (Node 24.21, Node 22.22 with ICU 78.2, Bun 1.4.2) it matches Intl.Segmenter on all 766 official cases, a 44-text multilingual corpus and every probe, so results on those hosts do not change. --- docs/Concepts.md | 2 +- docs/RouteEngine.md | 4 +++ spec/autoForwardTiming.test.js | 66 +++++++++++++++++++++++++++++++++- src/autoForwardTiming.js | 25 ++++--------- 4 files changed, 76 insertions(+), 21 deletions(-) diff --git a/docs/Concepts.md b/docs/Concepts.md index 70349b8b..76856484 100644 --- a/docs/Concepts.md +++ b/docs/Concepts.md @@ -251,7 +251,7 @@ Global playback modes use a different timing model: - Global `autoMode` starts its delay after the current line is completed. - In practice, completion is driven by Route Graphics `renderComplete`, so text reveal and other tracked render work finish first. -- Its delay is the persisted `autoForwardDelay` minimum plus length-derived reading time adjusted by `autoForwardSpeed`. The speed setting ranges from `0` (half speed) through `50` (normal) to `100` (double speed). Reading time starts at 60ms per weighted grapheme in the current line's resolved dialogue text. Han, Hiragana, Katakana, and Hangul graphemes have a weight of 3, whitespace has a weight of 0.5, and other graphemes have a weight of 1. The result is capped at 20 seconds unless the configured minimum is already higher. +- Its delay is the persisted `autoForwardDelay` minimum plus length-derived reading time adjusted by `autoForwardSpeed`. The speed setting ranges from `0` (half speed) through `50` (normal) to `100` (double speed). Reading time starts at 60ms per weighted grapheme in the current line's resolved dialogue text, counted with the bundled Unicode 17.0 grapheme rules on every host. Han, Hiragana, Katakana, and Hangul graphemes have a weight of 3, whitespace has a weight of 0.5, and other graphemes have a weight of 1. The result is capped at 20 seconds unless the configured minimum is already higher. - ADV append and NVL timing only measure visible dialogue introduced by the current line; prior accumulated dialogue, character names, and content discarded by `dialogue.clear` are not counted. - Global `skipMode` does not wait for completion; it advances aggressively on its own short timer. - `nextLineConfig.auto` is the only built-in auto-like behavior that can intentionally start from line start via `trigger: "fromStart"`. diff --git a/docs/RouteEngine.md b/docs/RouteEngine.md index 264ca639..11851550 100644 --- a/docs/RouteEngine.md +++ b/docs/RouteEngine.md @@ -2001,6 +2001,10 @@ double-speed range and applies only to the length-derived reading time. The persisted `runtime.autoForwardDelay` minimum is still added unchanged, subject to the auto-mode delay cap. +Reading units always use the bundled Unicode 17.0 grapheme segmenter rather +than the host's `Intl.Segmenter`, whose Unicode data depends on the browser or +runtime version. + ### Dialogue Append Reveal In ADV mode, `dialogue.append: true` appends the line content to the current diff --git a/spec/autoForwardTiming.test.js b/spec/autoForwardTiming.test.js index 9f105559..220d5310 100644 --- a/spec/autoForwardTiming.test.js +++ b/spec/autoForwardTiming.test.js @@ -1,4 +1,4 @@ -import { describe, expect, it } from "vitest"; +import { describe, expect, it, vi } from "vitest"; import { AUTO_FORWARD_MAX_DELAY_MS, estimateAutoForwardDelay, @@ -6,6 +6,29 @@ import { getAutoForwardReadingUnits, } from "../src/autoForwardTiming.js"; +// Reading units under the bundled Unicode 17.0 grapheme rules. Rows marked +// "Unicode 17" counted differently through Intl.Segmenter on Unicode 16 hosts. +const READING_UNIT_CORPUS = [ + ["Latin", "Hello, world.", 12.5], + ["combining marks", "cafe\u0301 A\u030Angstro\u0308m", 12.5], + ["stacked combining marks", "Z\u0351\u036B\u0343\u036A\u0302a", 2], + ["Han and kana", "漢字とカタカナ", 21], + ["Hangul syllables", "한국어", 9], + ["Hangul conjoining jamo", "\u1100\u1161\u11A8\u1112\u1161\u11AB", 6], + ["emoji ZWJ family", "👨‍👩‍👧‍👦", 1], + ["emoji ZWJ with skin tone", "🧑🏿‍🦱", 1], + ["regional indicator flags", "🇯🇵🇰🇷🇺🇳🇳", 4], + ["emoji tag sequence", "🏴󠁧󠁢󠁥󠁮󠁧󠁿", 1], + ["emoji keycaps", "1️⃣#️⃣", 2], + ["Thai tone marks and sara am", "ภาษาไทย น้ำ กำลัง", 12], + ["Devanagari conjuncts", "क्षत्रिय श्री", 4.5], + ["Khmer conjuncts (Unicode 17)", "ភាសាខ្មែរ", 4], + ["Myanmar kinzi (Unicode 17)", "\u1004\u103A\u1039\u1011\u1039\u1011", 1], + ["pictograph ZWJ pictograph (Unicode 17)", "\u2701\u200D\u2701", 2], + ["CRLF", "a\r\nb", 2.5], + ["mixed dialogue", "Hi 世界 e\u0301 👨‍👩‍👧‍👦 สวัสดี 🇯🇵!", 18.5], +]; + describe("auto-forward timing", () => { it("weights Latin text and whitespace", () => { expect(getAutoForwardReadingUnits("abcd")).toBe(4); @@ -49,6 +72,47 @@ describe("auto-forward timing", () => { } }); + it.each(READING_UNIT_CORPUS)( + "counts %s with the bundled Unicode 17 grapheme rules", + (_label, text, units) => { + expect(getAutoForwardReadingUnits(text)).toBe(units); + }, + ); + + it("ignores the host Intl.Segmenter and its Unicode data", async () => { + const segmenterDescriptor = Object.getOwnPropertyDescriptor( + Intl, + "Segmenter", + ); + let hostSegmenterCount = 0; + // Stands in for a host with outdated grapheme data: one code point each. + class OutdatedHostSegmenter { + constructor() { + hostSegmenterCount += 1; + } + + segment(text) { + return Array.from(text, (segment) => ({ segment })); + } + } + Object.defineProperty(Intl, "Segmenter", { + configurable: true, + value: OutdatedHostSegmenter, + }); + + try { + vi.resetModules(); + const timing = await import("../src/autoForwardTiming.js"); + for (const [, text, units] of READING_UNIT_CORPUS) { + expect(timing.getAutoForwardReadingUnits(text)).toBe(units); + } + expect(hostSegmenterCount).toBe(0); + } finally { + Object.defineProperty(Intl, "Segmenter", segmenterDescriptor); + vi.resetModules(); + } + }); + it("uses the base delay for empty content", () => { expect( estimateAutoForwardDelay({ diff --git a/src/autoForwardTiming.js b/src/autoForwardTiming.js index 2f2d9755..549215ed 100644 --- a/src/autoForwardTiming.js +++ b/src/autoForwardTiming.js @@ -15,25 +15,12 @@ const CJK_GRAPHEME_RE = /[\p{Script=Han}\p{Script=Hiragana}\p{Script=Katakana}\p{Script=Hangul}]/u; const WHITESPACE_GRAPHEME_RE = /^\s+$/u; -let graphemeSegmenter; - -const segmentGraphemes = (text) => { - const normalizedText = `${text ?? ""}`.normalize("NFC"); - const Segmenter = globalThis.Intl?.Segmenter; - - if (typeof Segmenter !== "function") { - return Array.from(splitGraphemes(normalizedText)); - } - - graphemeSegmenter ??= new Segmenter(undefined, { - granularity: "grapheme", - }); - - return Array.from( - graphemeSegmenter.segment(normalizedText), - ({ segment }) => segment, - ); -}; +// Graphemes always come from the bundled UAX #29 segmenter (Unicode 17.0). +// Intl.Segmenter follows the host's ICU data, so the same line counted +// differently on hosts with older Unicode tables (for example Khmer and +// Myanmar conjuncts, or U+2701 U+200D U+2701). +const segmentGraphemes = (text) => + Array.from(splitGraphemes(`${text ?? ""}`.normalize("NFC"))); export const getAutoForwardReadingUnits = (text) => segmentGraphemes(text).reduce((units, grapheme) => { From 6460503fa6e2e42ea258b8be1b19d92d25360238 Mon Sep 17 00:00:00 2001 From: han4wluc Date: Tue, 13 Oct 2026 02:26:05 +0800 Subject: [PATCH 2/3] fix: compute the auto-forward speed multiplier exactly The multiplier was `2 ** ((speed - 50) / 50)`. ECMA-262 leaves `**` and Math.pow implementation-approximated, and hosts disagree in the last bit: V8 13.6 calls the C library's pow (glibc 2.43 on Linux), its own fdlibm port (--no-use-std-math-pow) differs at 9 of the 101 integer speeds, JavaScriptCore returns 0.7071067811865475 instead of 0.7071067811865476 at speed 25, and glibc itself misses the nearest double for about 0.08% of fractional speeds (5.39, 44.62, 55.39, 61.09 and 76.42 among the hundredths). The playback scheduler compares delays as exact quanta, so a one-ulp difference can move the tick on which auto-forward fires. Return the correctly rounded value instead. The clamp, subtraction and division stay IEEE-754 basic operations, and 2 ** x is evaluated with exact BigInt fixed-point arithmetic under a proven error bound, doubling the precision whenever the bracket straddles a rounding boundary. The result is unique, so any correctly rounded implementation, including non-JavaScript ports, reproduces it bit for bit. All 101 integer speeds keep the values V8 returns on Linux today; 832 of 1,021,864 distinct measured speeds move by one unit in the last place to the nearest double. A call takes about 7 microseconds. --- docs/RouteEngine.md | 4 ++ spec/autoForwardTiming.test.js | 98 ++++++++++++++++++++++++++++++++++ src/autoForwardTiming.js | 98 ++++++++++++++++++++++++++++++++-- 3 files changed, 196 insertions(+), 4 deletions(-) diff --git a/docs/RouteEngine.md b/docs/RouteEngine.md index 11851550..e3246626 100644 --- a/docs/RouteEngine.md +++ b/docs/RouteEngine.md @@ -2001,6 +2001,10 @@ double-speed range and applies only to the length-derived reading time. The persisted `runtime.autoForwardDelay` minimum is still added unchanged, subject to the auto-mode delay cap. +The speed multiplier is `2 ** ((speed - 50) / 50)` rounded to the nearest +double (correctly rounded), so it is the same on every JavaScript host; the +host's `Math.pow` differs in the last bit between engines and C libraries. + Reading units always use the bundled Unicode 17.0 grapheme segmenter rather than the host's `Intl.Segmenter`, whose Unicode data depends on the browser or runtime version. diff --git a/spec/autoForwardTiming.test.js b/spec/autoForwardTiming.test.js index 220d5310..6c3a0f87 100644 --- a/spec/autoForwardTiming.test.js +++ b/spec/autoForwardTiming.test.js @@ -6,6 +6,62 @@ import { getAutoForwardReadingUnits, } from "../src/autoForwardTiming.js"; +// 2 ** ((speed - 50) / 50) rounded to the nearest binary64, indexed by speed. +const INTEGER_SPEED_MULTIPLIERS = [ + 0.5, 0.5069797398950145, 0.5140569133280333, 0.5212328804205607, + 0.5285090202806901, 0.5358867312681466, 0.543367431263029, 0.5509525579383053, + 0.55864356903611, 0.5664419426478994, 0.5743491774985174, 0.5823667932342279, + 0.5904963307147651, 0.5987393523094643, 0.6070974421975235, + 0.6155722066724582, 0.6241652744508059, 0.63287829698514, 0.6417129487814521, + 0.6506709277209668, 0.6597539553864471, 0.668963777393056, 0.6783021637238359, + 0.6877709090698718, 0.6973718331752027, 0.7071067811865476, + 0.7169776240079136, 0.7269862586601553, 0.7371346086455506, + 0.7474246243174693, 0.757858283255199, 0.7684375906440062, 0.7791645796604999, + 0.7900413118633771, 0.8010698775896221, 0.8122523963562356, + 0.8235910172675731, 0.8350879194283694, 0.8467453123625271, + 0.8585654364377537, 0.8705505632961241, 0.8827029962906549, + 0.8950250709279725, 0.9075191553171609, 0.9201876506248751, + 0.9330329915368074, 0.9460576467255959, 0.9592641193252643, + 0.9726549474122855, 0.9862327044933592, 1, 1.013959479790029, + 1.0281138266560665, 1.0424657608411214, 1.0570180405613803, + 1.0717734625362931, 1.086734862526058, 1.1019051158766107, 1.11728713807222, + 1.1328838852957985, 1.148698354997035, 1.1647335864684558, 1.1809926614295303, + 1.1974787046189286, 1.214194884395047, 1.2311444133449163, 1.2483305489016119, + 1.26575659397028, 1.2834258975629043, 1.3013418554419336, 1.3195079107728942, + 1.337927554786112, 1.3566043274476718, 1.3755418181397439, 1.3947436663504054, + 1.4142135623730951, 1.4339552480158273, 1.4539725173203106, + 1.4742692172911012, 1.4948492486349383, 1.515716566510398, 1.5368751812880124, + 1.5583291593209998, 1.5800826237267545, 1.6021397551792442, 1.624504792712471, + 1.6471820345351462, 1.6701758388567387, 1.6934906247250543, + 1.7171308728755075, 1.7411011265922482, 1.7654059925813097, 1.790050141855945, + 1.8150383106343217, 1.8403753012497501, 1.8660659830736148, + 1.8921152934511918, 1.9185282386505287, 1.945309894824571, 1.9724654089867184, + 2, +]; + +// Fractional speeds where glibc 2.43 pow (which V8 calls on Linux) returns +// the neighbouring binary64 instead of the nearest one. +const FRACTIONAL_SPEED_MULTIPLIERS = [ + [5.39, 0.5387918749305093], + [44.62, 0.9281307732922005], + [55.39, 1.0775837498610186], + [61.09, 1.1661876906400275], + [76.42, 1.4423287144604011], + [1.6218108357861638, 0.5113688606410209], + [11.278214515186846, 0.5846172445536842], + [15.85592566115398, 0.6229198767484339], + [23.260451527312398, 0.6902586802976548], + [32.64301389745219, 0.7861411522977813], + [41.55971207655966, 0.8895787565530576], + [48.62368886824697, 0.9811011421990845], + [56.852721643664715, 1.0996576386064774], + [77.03163006808609, 1.4546102034938393], + [79.53354422934353, 1.5059468826790383], + [89.62236507454348, 1.732010050185167], + [93.5848910594359, 1.829815033321451], + [96.70571486931294, 1.9107172431722372], +]; + // Reading units under the bundled Unicode 17.0 grapheme rules. Rows marked // "Unicode 17" counted differently through Intl.Segmenter on Unicode 16 hosts. const READING_UNIT_CORPUS = [ @@ -128,6 +184,48 @@ describe("auto-forward timing", () => { expect(getAutoForwardSpeedMultiplier(100)).toBe(2); }); + it("returns the correctly rounded multiplier for every integer speed", () => { + expect( + Array.from({ length: 101 }, (_, speed) => + getAutoForwardSpeedMultiplier(speed), + ), + ).toEqual(INTEGER_SPEED_MULTIPLIERS); + // JavaScriptCore and fdlibm-based pow implementations return + // 0.7071067811865475 here. + expect(getAutoForwardSpeedMultiplier(25)).toBe(Math.SQRT1_2); + }); + + it.each(FRACTIONAL_SPEED_MULTIPLIERS)( + "returns the correctly rounded multiplier for speed %s", + (speed, multiplier) => { + expect(getAutoForwardSpeedMultiplier(speed)).toBe(multiplier); + }, + ); + + it("keeps fractional speeds monotonic between half and double speed", () => { + let previous = getAutoForwardSpeedMultiplier(0); + for (let step = 1; step <= 10_000; step += 1) { + const multiplier = getAutoForwardSpeedMultiplier(step / 100); + expect(multiplier).toBeGreaterThanOrEqual(previous); + previous = multiplier; + } + expect(previous).toBe(2); + }); + + it("derives the reading delay from the exact multiplier", () => { + const text = "Hi 世界 e\u0301 👨‍👩‍👧‍👦"; + expect(getAutoForwardReadingUnits(text)).toBe(11.5); + expect(estimateAutoForwardDelay({ text, baseDelay: 0, speed: 25 })).toBe( + 975.8073580374355, + ); + expect(estimateAutoForwardDelay({ text, baseDelay: 0, speed: 55.39 })).toBe( + 640.3214600155142, + ); + expect( + estimateAutoForwardDelay({ text, baseDelay: 0, speed: 93.5848910594359 }), + ).toBe(377.08729430838866); + }); + it("scales only length-derived reading time with the speed setting", () => { expect( estimateAutoForwardDelay({ diff --git a/src/autoForwardTiming.js b/src/autoForwardTiming.js index 549215ed..ff606a1f 100644 --- a/src/autoForwardTiming.js +++ b/src/autoForwardTiming.js @@ -35,15 +35,105 @@ export const getAutoForwardReadingUnits = (text) => return units + 1; }, 0); +const binary64View = new DataView(new ArrayBuffer(8)); +const EXP2_INITIAL_PRECISION_BITS = 128; +const LN2_GUARD_BITS = 64n; +const scaledLn2Cache = new Map(); + +// ln(2) * 2 ** bits to within 2, from ln(2) = sum of 1 / (k * 2 ** k), k >= 1. +const getScaledLn2 = (bits) => { + let scaledLn2 = scaledLn2Cache.get(bits); + if (scaledLn2 === undefined) { + const scale = 1n << (BigInt(bits) + LN2_GUARD_BITS); + let sum = 0n; + for (let k = 1n; scale >> k !== 0n; k += 1n) { + sum += (scale >> k) / k; + } + scaledLn2 = sum >> LN2_GUARD_BITS; + scaledLn2Cache.set(bits, scaledLn2); + } + return scaledLn2; +}; + +// Nearest binary64 (ties to even) to scaled / 2 ** bits, for normal results. +const roundScaledToBinary64 = (scaled, bits) => { + const shift = BigInt(scaled.toString(2).length - 53); + let significand = scaled >> shift; + const remainder = scaled - (significand << shift); + const half = 1n << (shift - 1n); + if (remainder > half || (remainder === half && (significand & 1n) === 1n)) { + significand += 1n; + } + let exponent = shift - BigInt(bits); + if (significand === 1n << 53n) { + significand >>= 1n; + exponent += 1n; + } + binary64View.setBigUint64( + 0, + ((exponent + 1075n) << 52n) | (significand - (1n << 52n)), + ); + return binary64View.getFloat64(0); +}; + +// 2 ** x rounded to the nearest binary64, for binary64 x in [-1, 1]. +// +// `2 ** x` and Math.pow are implementation-approximated in ECMA-262 and differ +// in the last bit between engines and C libraries (V8 uses the host C pow; +// JavaScriptCore returns 0.7071067811865475 for 2 ** -0.5, V8 on glibc +// 0.7071067811865476). The correctly rounded value has exactly one answer, so +// it is computed with exact BigInt fixed-point arithmetic. With 2 ** -bits as +// the unit, x * ln(2) is off by less than 3 units, each truncated series term +// adds less than 4, and the bracket [sum - error, sum + error] must round to +// a single binary64; otherwise the precision doubles. 2 ** x is irrational +// for every non-integer rational x, so it never lies exactly halfway and the +// loop ends (in practice after the first pass). +const exp2CorrectlyRounded = (x) => { + if (x === 0) return 1; + if (x === 1) return 2; + if (x === -1) return 0.5; + + binary64View.setFloat64(0, x); + const bitsOfX = binary64View.getBigUint64(0); + const exponentField = Number((bitsOfX >> 52n) & 0x7ffn); + const fraction = bitsOfX & ((1n << 52n) - 1n); + const magnitude = exponentField === 0 ? fraction : fraction | (1n << 52n); + const significandOfX = bitsOfX >> 63n === 1n ? -magnitude : magnitude; + const shiftOfX = 1075 - Math.max(exponentField, 1); + + for (let bits = EXP2_INITIAL_PRECISION_BITS; ; bits *= 2) { + const scale = 1n << BigInt(bits); + const product = significandOfX * getScaledLn2(bits); + const t = + shiftOfX >= 0 + ? product >> BigInt(shiftOfX) + : product << BigInt(-shiftOfX); + let term = scale; + let sum = scale; + let termCount = 1; + for (let n = 1n; term !== 0n; n += 1n) { + term = (term * t) / (scale * n); + sum += term; + termCount += 1; + } + const error = BigInt(4 * termCount + 16); + const lower = roundScaledToBinary64(sum - error, bits); + const upper = roundScaledToBinary64(sum + error, bits); + if (lower === upper) return lower; + } +}; + +// 2 ** ((speed - 50) / 50): 0.5 at speed 0, 1 at 50, 2 at 100. The clamp, +// subtraction and division are IEEE-754 basic operations and the power is +// correctly rounded, so every host returns the same binary64. export const getAutoForwardSpeedMultiplier = (speed) => { const normalizedSpeed = Number.isFinite(speed) ? Math.min(MAX_AUTO_FORWARD_SPEED, Math.max(MIN_AUTO_FORWARD_SPEED, speed)) : DEFAULT_AUTO_FORWARD_SPEED; - return ( - 2 ** - ((normalizedSpeed - NORMAL_AUTO_FORWARD_SPEED) / - (MAX_AUTO_FORWARD_SPEED - NORMAL_AUTO_FORWARD_SPEED)) + return exp2CorrectlyRounded( + (normalizedSpeed - NORMAL_AUTO_FORWARD_SPEED) / + (MAX_AUTO_FORWARD_SPEED - NORMAL_AUTO_FORWARD_SPEED), ); }; From 92cab875567f5d2fc916a28e5cec5e91cc711ec7 Mon Sep 17 00:00:00 2001 From: Luciano Hanyon Wu Date: Mon, 19 Oct 2026 01:28:19 +0800 Subject: [PATCH 3/3] fix: weight auto-forward reading units with bundled script data Graphemes already came from the bundled Unicode 17.0 segmenter, but their weights still used the host: \p{Script=Han|Hiragana|Katakana|Hangul} and \s follow the engine's regex Unicode tables, and normalize("NFC") the host's normalization data. CJK Extension J (U+323B0..U+33479, new in 17.0) weighed 3 per character on Node 24 but 1 in Chromium 143, whose regex data is older. Weigh graphemes with bundled tables instead: the Han, Hiragana, Katakana and Hangul ranges from Unicode 17.0 Scripts.txt (115,792 code points, an exact match), and the ECMAScript WhiteSpace and LineTerminator code points. Drop the NFC step: the bundled segmenter keeps bases and marks together, and over every decomposable code point in conjunct, Hangul and emoji contexts NFC only changed counts for U+2194 U+0338 next to a ZWJ emoji sequence. Pin unicode-segmenter to 0.17.0 so the documented Unicode version cannot move with a patch release. Also cache the last speed multiplier, since timers re-read it on every reconcile; correct the comment on V8's pow; spell out in the docs that (speed - 50) / 50 is ordinary double arithmetic; restore Intl.Segmenter correctly in tests on hosts without it; drop the test made redundant by the host-segmenter stub; and escape invisible code points in the corpus. --- bun.lock | 2 +- docs/Concepts.md | 2 +- docs/RouteEngine.md | 17 ++-- package.json | 2 +- spec/autoForwardTiming.test.js | 100 +++++++++++++++------- src/autoForwardTiming.js | 150 ++++++++++++++++++++++++++++++--- 6 files changed, 219 insertions(+), 54 deletions(-) diff --git a/bun.lock b/bun.lock index 8153d3e8..13640513 100644 --- a/bun.lock +++ b/bun.lock @@ -8,7 +8,7 @@ "jempl": "1.1.2", "js-yaml": "^4.1.0", "puty": "^0.1.1", - "unicode-segmenter": "^0.17.0", + "unicode-segmenter": "0.17.0", }, "devDependencies": { "@vitest/coverage-v8": "4.0.16", diff --git a/docs/Concepts.md b/docs/Concepts.md index 76856484..0e7c2a03 100644 --- a/docs/Concepts.md +++ b/docs/Concepts.md @@ -251,7 +251,7 @@ Global playback modes use a different timing model: - Global `autoMode` starts its delay after the current line is completed. - In practice, completion is driven by Route Graphics `renderComplete`, so text reveal and other tracked render work finish first. -- Its delay is the persisted `autoForwardDelay` minimum plus length-derived reading time adjusted by `autoForwardSpeed`. The speed setting ranges from `0` (half speed) through `50` (normal) to `100` (double speed). Reading time starts at 60ms per weighted grapheme in the current line's resolved dialogue text, counted with the bundled Unicode 17.0 grapheme rules on every host. Han, Hiragana, Katakana, and Hangul graphemes have a weight of 3, whitespace has a weight of 0.5, and other graphemes have a weight of 1. The result is capped at 20 seconds unless the configured minimum is already higher. +- Its delay is the persisted `autoForwardDelay` minimum plus length-derived reading time adjusted by `autoForwardSpeed`. The speed setting ranges from `0` (half speed) through `50` (normal) to `100` (double speed). Reading time starts at 60ms per weighted grapheme in the current line's resolved dialogue text, counted with the bundled Unicode 17.0 grapheme rules and script data on every host. Han, Hiragana, Katakana, and Hangul graphemes have a weight of 3, whitespace has a weight of 0.5, and other graphemes have a weight of 1. The result is capped at 20 seconds unless the configured minimum is already higher. - ADV append and NVL timing only measure visible dialogue introduced by the current line; prior accumulated dialogue, character names, and content discarded by `dialogue.clear` are not counted. - Global `skipMode` does not wait for completion; it advances aggressively on its own short timer. - `nextLineConfig.auto` is the only built-in auto-like behavior that can intentionally start from line start via `trigger: "fromStart"`. diff --git a/docs/RouteEngine.md b/docs/RouteEngine.md index e3246626..a6ad23e3 100644 --- a/docs/RouteEngine.md +++ b/docs/RouteEngine.md @@ -2001,13 +2001,16 @@ double-speed range and applies only to the length-derived reading time. The persisted `runtime.autoForwardDelay` minimum is still added unchanged, subject to the auto-mode delay cap. -The speed multiplier is `2 ** ((speed - 50) / 50)` rounded to the nearest -double (correctly rounded), so it is the same on every JavaScript host; the -host's `Math.pow` differs in the last bit between engines and C libraries. - -Reading units always use the bundled Unicode 17.0 grapheme segmenter rather -than the host's `Intl.Segmenter`, whose Unicode data depends on the browser or -runtime version. +The speed multiplier is `2 ** x`, where `x = (speed - 50) / 50` is evaluated in +ordinary double arithmetic and 2 raised to that double `x` is then rounded to +the nearest double (correctly rounded). It is the same on every JavaScript host; +the host's `Math.pow` differs in the last bit between engines and C libraries. + +Reading units always use the bundled Unicode 17.0 grapheme segmenter and Unicode +17.0 Han, Hiragana, Katakana, and Hangul code point ranges, rather than the +host's `Intl.Segmenter` and regular-expression script data, whose Unicode +version depends on the browser or runtime. The text is not normalized first, +because `String.prototype.normalize` also uses the host's Unicode data. ### Dialogue Append Reveal diff --git a/package.json b/package.json index 5557b6ca..1d4fc15a 100644 --- a/package.json +++ b/package.json @@ -55,7 +55,7 @@ "jempl": "1.1.2", "js-yaml": "^4.1.0", "puty": "^0.1.1", - "unicode-segmenter": "^0.17.0" + "unicode-segmenter": "0.17.0" }, "devDependencies": { "@vitest/coverage-v8": "4.0.16", diff --git a/spec/autoForwardTiming.test.js b/spec/autoForwardTiming.test.js index 6c3a0f87..3660b5bd 100644 --- a/spec/autoForwardTiming.test.js +++ b/spec/autoForwardTiming.test.js @@ -62,8 +62,11 @@ const FRACTIONAL_SPEED_MULTIPLIERS = [ [96.70571486931294, 1.9107172431722372], ]; -// Reading units under the bundled Unicode 17.0 grapheme rules. Rows marked -// "Unicode 17" counted differently through Intl.Segmenter on Unicode 16 hosts. +// Reading units under the bundled Unicode 17.0 grapheme rules and script +// ranges. Rows marked "Unicode 17" counted differently on Unicode 16 hosts, +// through Intl.Segmenter or \p{Script=...}. Text with joiners, variation +// selectors, tags, regional indicators or combining marks is escaped so every +// code point shows in review. const READING_UNIT_CORPUS = [ ["Latin", "Hello, world.", 12.5], ["combining marks", "cafe\u0301 A\u030Angstro\u0308m", 12.5], @@ -71,18 +74,48 @@ const READING_UNIT_CORPUS = [ ["Han and kana", "漢字とカタカナ", 21], ["Hangul syllables", "한국어", 9], ["Hangul conjoining jamo", "\u1100\u1161\u11A8\u1112\u1161\u11AB", 6], - ["emoji ZWJ family", "👨‍👩‍👧‍👦", 1], - ["emoji ZWJ with skin tone", "🧑🏿‍🦱", 1], - ["regional indicator flags", "🇯🇵🇰🇷🇺🇳🇳", 4], - ["emoji tag sequence", "🏴󠁧󠁢󠁥󠁮󠁧󠁿", 1], - ["emoji keycaps", "1️⃣#️⃣", 2], - ["Thai tone marks and sara am", "ภาษาไทย น้ำ กำลัง", 12], - ["Devanagari conjuncts", "क्षत्रिय श्री", 4.5], - ["Khmer conjuncts (Unicode 17)", "ភាសាខ្មែរ", 4], + ["CJK Extension J (Unicode 17)", "\u{323B0}\u{33479}", 6], + ["Unicode whitespace", "\u3000\u00A0\t\u2028", 2], + [ + "emoji ZWJ family", + "\u{1F468}\u200D\u{1F469}\u200D\u{1F467}\u200D\u{1F466}", + 1, + ], + ["emoji ZWJ with skin tone", "\u{1F9D1}\u{1F3FF}\u200D\u{1F9B1}", 1], + [ + "regional indicator flags", + "\u{1F1EF}\u{1F1F5}\u{1F1F0}\u{1F1F7}\u{1F1FA}\u{1F1F3}\u{1F1F3}", + 4, + ], + [ + "emoji tag sequence", + "\u{1F3F4}\u{E0067}\u{E0062}\u{E0065}\u{E006E}\u{E0067}\u{E007F}", + 1, + ], + ["emoji keycaps", "1\uFE0F\u20E3#\uFE0F\u20E3", 2], + [ + "Thai tone marks and sara am", + "\u0E20\u0E32\u0E29\u0E32\u0E44\u0E17\u0E22 \u0E19\u0E49\u0E33 \u0E01\u0E33\u0E25\u0E31\u0E07", + 12, + ], + [ + "Devanagari conjuncts", + "\u0915\u094D\u0937\u0924\u094D\u0930\u093F\u092F \u0936\u094D\u0930\u0940", + 4.5, + ], + [ + "Khmer conjuncts (Unicode 17)", + "\u1797\u17B6\u179F\u17B6\u1781\u17D2\u1798\u17C2\u179A", + 4, + ], ["Myanmar kinzi (Unicode 17)", "\u1004\u103A\u1039\u1011\u1039\u1011", 1], ["pictograph ZWJ pictograph (Unicode 17)", "\u2701\u200D\u2701", 2], ["CRLF", "a\r\nb", 2.5], - ["mixed dialogue", "Hi 世界 e\u0301 👨‍👩‍👧‍👦 สวัสดี 🇯🇵!", 18.5], + [ + "mixed dialogue", + "Hi \u4E16\u754C e\u0301 \u{1F468}\u200D\u{1F469}\u200D\u{1F467}\u200D\u{1F466} \u0E2A\u0E27\u0E31\u0E2A\u0E14\u0E35 \u{1F1EF}\u{1F1F5}!", + 18.5, + ], ]; describe("auto-forward timing", () => { @@ -107,24 +140,22 @@ describe("auto-forward timing", () => { it("counts composed characters and emoji sequences as graphemes", () => { expect(getAutoForwardReadingUnits("e\u0301")).toBe(1); - expect(getAutoForwardReadingUnits("👨‍👩‍👧‍👦")).toBe(1); + expect( + getAutoForwardReadingUnits( + "\u{1F468}\u200D\u{1F469}\u200D\u{1F467}\u200D\u{1F466}", + ), + ).toBe(1); }); - it("keeps emoji sequences intact without Intl.Segmenter", () => { - const segmenterDescriptor = Object.getOwnPropertyDescriptor( - Intl, - "Segmenter", - ); - Object.defineProperty(Intl, "Segmenter", { - configurable: true, - value: undefined, - }); - - try { - expect(getAutoForwardReadingUnits("👨‍👩‍👧‍👦")).toBe(1); - expect(getAutoForwardReadingUnits("🇨🇳")).toBe(1); - } finally { - Object.defineProperty(Intl, "Segmenter", segmenterDescriptor); + it("counts composed and decomposed text alike without normalizing it", () => { + for (const [composed, decomposed, units] of [ + ["\u00E9", "e\u0301", 1], + ["\uAC01", "\u1100\u1161\u11A8", 3], + ["\u0929\u094D\u0915", "\u0928\u093C\u094D\u0915", 1], + ["\u1EC7", "e\u0323\u0302", 1], + ]) { + expect(getAutoForwardReadingUnits(composed)).toBe(units); + expect(getAutoForwardReadingUnits(decomposed)).toBe(units); } }); @@ -135,11 +166,12 @@ describe("auto-forward timing", () => { }, ); - it("ignores the host Intl.Segmenter and its Unicode data", async () => { + it("ignores the host Intl.Segmenter and String.prototype.normalize", async () => { const segmenterDescriptor = Object.getOwnPropertyDescriptor( Intl, "Segmenter", ); + const normalizeSpy = vi.spyOn(String.prototype, "normalize"); let hostSegmenterCount = 0; // Stands in for a host with outdated grapheme data: one code point each. class OutdatedHostSegmenter { @@ -163,8 +195,15 @@ describe("auto-forward timing", () => { expect(timing.getAutoForwardReadingUnits(text)).toBe(units); } expect(hostSegmenterCount).toBe(0); + expect(normalizeSpy).not.toHaveBeenCalled(); } finally { - Object.defineProperty(Intl, "Segmenter", segmenterDescriptor); + normalizeSpy.mockRestore(); + // Hosts without Intl.Segmenter have no descriptor to restore. + if (segmenterDescriptor) { + Object.defineProperty(Intl, "Segmenter", segmenterDescriptor); + } else { + delete Intl.Segmenter; + } vi.resetModules(); } }); @@ -213,7 +252,8 @@ describe("auto-forward timing", () => { }); it("derives the reading delay from the exact multiplier", () => { - const text = "Hi 世界 e\u0301 👨‍👩‍👧‍👦"; + const text = + "Hi \u4E16\u754C e\u0301 \u{1F468}\u200D\u{1F469}\u200D\u{1F467}\u200D\u{1F466}"; expect(getAutoForwardReadingUnits(text)).toBe(11.5); expect(estimateAutoForwardDelay({ text, baseDelay: 0, speed: 25 })).toBe( 975.8073580374355, diff --git a/src/autoForwardTiming.js b/src/autoForwardTiming.js index ff606a1f..7cbfab8a 100644 --- a/src/autoForwardTiming.js +++ b/src/autoForwardTiming.js @@ -11,24 +11,136 @@ export const AUTO_FORWARD_CJK_READING_WEIGHT = 3; export const AUTO_FORWARD_WHITESPACE_READING_WEIGHT = 0.5; export const AUTO_FORWARD_MAX_DELAY_MS = 20_000; -const CJK_GRAPHEME_RE = - /[\p{Script=Han}\p{Script=Hiragana}\p{Script=Katakana}\p{Script=Hangul}]/u; -const WHITESPACE_GRAPHEME_RE = /^\s+$/u; +// Han, Hiragana, Katakana and Hangul code points from Unicode 17.0 +// Scripts.txt, as sorted inclusive [first, last] ranges. \p{Script=...} in a +// regular expression follows the host's Unicode tables, so a character added +// to these scripts in a newer Unicode version (for example CJK Extension J in +// 17.0) weighed 1 instead of 3 on older hosts. Update this table together with +// unicode-segmenter. +const CJK_CODE_POINT_RANGES = [ + [0x1100, 0x11ff], + [0x2e80, 0x2e99], + [0x2e9b, 0x2ef3], + [0x2f00, 0x2fd5], + [0x3005, 0x3005], + [0x3007, 0x3007], + [0x3021, 0x3029], + [0x302e, 0x302f], + [0x3038, 0x303b], + [0x3041, 0x3096], + [0x309d, 0x309f], + [0x30a1, 0x30fa], + [0x30fd, 0x30ff], + [0x3131, 0x318e], + [0x31f0, 0x321e], + [0x3260, 0x327e], + [0x32d0, 0x32fe], + [0x3300, 0x3357], + [0x3400, 0x4dbf], + [0x4e00, 0x9fff], + [0xa960, 0xa97c], + [0xac00, 0xd7a3], + [0xd7b0, 0xd7c6], + [0xd7cb, 0xd7fb], + [0xf900, 0xfa6d], + [0xfa70, 0xfad9], + [0xff66, 0xff6f], + [0xff71, 0xff9d], + [0xffa0, 0xffbe], + [0xffc2, 0xffc7], + [0xffca, 0xffcf], + [0xffd2, 0xffd7], + [0xffda, 0xffdc], + [0x16fe2, 0x16fe3], + [0x16ff0, 0x16ff6], + [0x1aff0, 0x1aff3], + [0x1aff5, 0x1affb], + [0x1affd, 0x1affe], + [0x1b000, 0x1b122], + [0x1b132, 0x1b132], + [0x1b150, 0x1b152], + [0x1b155, 0x1b155], + [0x1b164, 0x1b167], + [0x1f200, 0x1f200], + [0x20000, 0x2a6df], + [0x2a700, 0x2b81d], + [0x2b820, 0x2cead], + [0x2ceb0, 0x2ebe0], + [0x2ebf0, 0x2ee5d], + [0x2f800, 0x2fa1d], + [0x30000, 0x3134a], + [0x31350, 0x33479], +]; + +// ECMAScript WhiteSpace and LineTerminator code points, which \s matches. +const WHITESPACE_CODE_POINT_RANGES = [ + [0x0009, 0x000d], + [0x0020, 0x0020], + [0x00a0, 0x00a0], + [0x1680, 0x1680], + [0x2000, 0x200a], + [0x2028, 0x2029], + [0x202f, 0x202f], + [0x205f, 0x205f], + [0x3000, 0x3000], + [0xfeff, 0xfeff], +]; + +const isInCodePointRanges = (ranges, codePoint) => { + let low = 0; + let high = ranges.length - 1; + while (low <= high) { + const middle = (low + high) >> 1; + const [first, last] = ranges[middle]; + if (codePoint < first) { + high = middle - 1; + } else if (codePoint > last) { + low = middle + 1; + } else { + return true; + } + } + return false; +}; + +const isWhitespaceGrapheme = (grapheme) => { + for (const character of grapheme) { + if ( + !isInCodePointRanges( + WHITESPACE_CODE_POINT_RANGES, + character.codePointAt(0), + ) + ) { + return false; + } + } + return true; +}; + +const isCjkGrapheme = (grapheme) => { + for (const character of grapheme) { + if (isInCodePointRanges(CJK_CODE_POINT_RANGES, character.codePointAt(0))) { + return true; + } + } + return false; +}; // Graphemes always come from the bundled UAX #29 segmenter (Unicode 17.0). // Intl.Segmenter follows the host's ICU data, so the same line counted // differently on hosts with older Unicode tables (for example Khmer and -// Myanmar conjuncts, or U+2701 U+200D U+2701). -const segmentGraphemes = (text) => - Array.from(splitGraphemes(`${text ?? ""}`.normalize("NFC"))); +// Myanmar conjuncts, or U+2701 U+200D U+2701). The text is not normalized: +// String.prototype.normalize also follows the host's Unicode tables, and the +// bundled segmenter already keeps a base and its combining marks together. +const segmentGraphemes = (text) => Array.from(splitGraphemes(`${text ?? ""}`)); export const getAutoForwardReadingUnits = (text) => segmentGraphemes(text).reduce((units, grapheme) => { - if (WHITESPACE_GRAPHEME_RE.test(grapheme)) { + if (isWhitespaceGrapheme(grapheme)) { return units + AUTO_FORWARD_WHITESPACE_READING_WEIGHT; } - if (CJK_GRAPHEME_RE.test(grapheme)) { + if (isCjkGrapheme(grapheme)) { return units + AUTO_FORWARD_CJK_READING_WEIGHT; } @@ -79,7 +191,8 @@ const roundScaledToBinary64 = (scaled, bits) => { // 2 ** x rounded to the nearest binary64, for binary64 x in [-1, 1]. // // `2 ** x` and Math.pow are implementation-approximated in ECMA-262 and differ -// in the last bit between engines and C libraries (V8 uses the host C pow; +// in the last bit between engines and C libraries (V8 calls the C library pow +// or its own fdlibm port, depending on the build and --use-std-math-pow; // JavaScriptCore returns 0.7071067811865475 for 2 ** -0.5, V8 on glibc // 0.7071067811865476). The correctly rounded value has exactly one answer, so // it is computed with exact BigInt fixed-point arithmetic. With 2 ** -bits as @@ -125,16 +238,25 @@ const exp2CorrectlyRounded = (x) => { // 2 ** ((speed - 50) / 50): 0.5 at speed 0, 1 at 50, 2 at 100. The clamp, // subtraction and division are IEEE-754 basic operations and the power is -// correctly rounded, so every host returns the same binary64. +// correctly rounded, so every host returns the same binary64. The last result +// is cached: the speed setting rarely changes, while playback timers re-read +// the multiplier on every reconcile. +let cachedSpeedExponent = 0; +let cachedSpeedMultiplier = 1; + export const getAutoForwardSpeedMultiplier = (speed) => { const normalizedSpeed = Number.isFinite(speed) ? Math.min(MAX_AUTO_FORWARD_SPEED, Math.max(MIN_AUTO_FORWARD_SPEED, speed)) : DEFAULT_AUTO_FORWARD_SPEED; - - return exp2CorrectlyRounded( + const exponent = (normalizedSpeed - NORMAL_AUTO_FORWARD_SPEED) / - (MAX_AUTO_FORWARD_SPEED - NORMAL_AUTO_FORWARD_SPEED), - ); + (MAX_AUTO_FORWARD_SPEED - NORMAL_AUTO_FORWARD_SPEED); + + if (exponent !== cachedSpeedExponent) { + cachedSpeedMultiplier = exp2CorrectlyRounded(exponent); + cachedSpeedExponent = exponent; + } + return cachedSpeedMultiplier; }; export const estimateAutoForwardDelay = ({