From 94b5a6cff40b2f737c5623a77498ea62b6e8e41d Mon Sep 17 00:00:00 2001 From: palmoni5 Date: Tue, 10 Nov 2026 00:24:51 +0200 Subject: [PATCH 1/8] fix: preserve Hebrew niqqud and combining marks in visual RTL output --- lib/src/paragraph.dart | 52 ++++++++++++++++++++++++++++++++++++++---- 1 file changed, 47 insertions(+), 5 deletions(-) diff --git a/lib/src/paragraph.dart b/lib/src/paragraph.dart index c91d2bb..e0fa77d 100644 --- a/lib/src/paragraph.dart +++ b/lib/src/paragraph.dart @@ -216,7 +216,7 @@ class Paragraph { start = limit; } - _reorderString(textData, el); + _reorderString(textData, el, n.hasNonspacingMark); _fixMirroredCharacters(textData); List indexes = []; @@ -283,6 +283,11 @@ class Normalization { for (int j = 0; j < buffer.length; ++j) { final character = buffer[j]; final chClass = _getCanonicalClass(character); + // Some precomposed characters decompose into combining marks even + // when their original bidi class is a strong letter. + if (!hasNSMs && getCharacterType(character) == CharacterType.nonspacingMark) { + hasNSMs = true; + } int k = text.length; // insertion point if (chClass != _CanonicalClass.notReordered) { // bubble-sort combining marks as necessary @@ -782,7 +787,11 @@ void _resolveImplicitTypes( } /// 3.4 Reordering Resolved Levels -void _reorderString(List<_CharData> textData, int embeddingLevel) { +void _reorderString( + List<_CharData> textData, + int embeddingLevel, + bool hasNonspacingMark, +) { //L1. On each line, reset the embedding level of the following characters to the paragraph embedding level: // 1. Segment separators, // 2. Paragraph separators, @@ -843,9 +852,42 @@ void _reorderString(List<_CharData> textData, int embeddingLevel) { } } - // TODO - L3. Combining marks applied to a right-to-left base character will at this point precede their base - // character. If the rendering engine expects them to follow the base characters in the final display process, - // then the ordering of the marks and the base character must be reversed. + // L3. L2 reverses characters inside odd-level runs, leaving combining + // marks before their base character and reversing the order of adjacent + // marks. Restore each reversed base+mark cluster in-place. Skipping the pass + // for unpointed text keeps the common case unchanged. + if (!hasNonspacingMark) { + return; + } + + var i = 0; + while (i < textData.length) { + final level = textData[i].embeddingLevel; + if ((level & 1) == 0 || + getCharacterType(textData[i].char) != CharacterType.nonspacingMark) { + ++i; + continue; + } + + final start = i; + do { + ++i; + } while (i < textData.length && + textData[i].embeddingLevel == level && + getCharacterType(textData[i].char) == CharacterType.nonspacingMark); + + // A free-standing mark or a level boundary has no base to restore. + if (i >= textData.length || textData[i].embeddingLevel != level) { + continue; + } + + for (var left = start, right = i; left < right; ++left, --right) { + final temp = textData[left]; + textData[left] = textData[right]; + textData[right] = temp; + } + ++i; + } } /// L4. A character is depicted by a mirrored glyph if and only if (a) the resolved directionality of that character is R, and (b) the Bidi_Mirrored property value of that character is true. From 6af210a1b1f112b088578dc38f92d0dc8ee3174c Mon Sep 17 00:00:00 2001 From: palmoni5 Date: Tue, 10 Nov 2026 00:25:12 +0200 Subject: [PATCH 2/8] test: cover combining_marks_test.dart for RTL combining mark changes --- test/combining_marks_test.dart | 39 ++++++++++++++++++++++++++++++++++ 1 file changed, 39 insertions(+) create mode 100644 test/combining_marks_test.dart diff --git a/test/combining_marks_test.dart b/test/combining_marks_test.dart new file mode 100644 index 0000000..b812f4b --- /dev/null +++ b/test/combining_marks_test.dart @@ -0,0 +1,39 @@ +import 'package:bidi/bidi.dart' as bidi; +import 'package:test/test.dart'; + +void main() { + test('Hebrew niqqud stays after its base when RTL text is reordered', () { + const input = 'א\u05B0ב\u05B1ג'; + final paragraph = bidi.BidiString.fromLogical(input).paragraphs.single; + expect(String.fromCharCodes(paragraph.bidiText), 'גב\u05B1א\u05B0'); + expect(paragraph.indices, [4, 2, 3, 0, 1]); + }); + + test('multiple niqqud marks remain in their original cluster order', () { + const input = 'ש\u05B8\u05C1לו\u05B9ם'; + final paragraph = bidi.BidiString.fromLogical(input).paragraphs.single; + expect( + String.fromCharCodes(paragraph.bidiText), + 'םו\u05B9לש\u05B8\u05C1', + ); + expect(paragraph.indices, [6, 4, 5, 3, 0, 1, 2]); + }); + + test('RTL combining marks inside a Latin paragraph remain attached', () { + const input = 'abc א\u05B0ב def'; + final paragraph = bidi.BidiString.fromLogical(input).paragraphs.single; + expect( + String.fromCharCodes(paragraph.bidiText), + 'abc בא\u05B0 def', + ); + expect(paragraph.indices, [0, 1, 2, 3, 6, 4, 5, 7, 8, 9, 10]); + }); + + test('left-to-right accent marks are not moved', () { + const input = 'e\u0301cole'; + final paragraph = bidi.BidiString.fromLogical(input).paragraphs.single; + // Canonical normalization may compose the first accent. + expect(String.fromCharCodes(paragraph.bidiText), 'école'); + expect(paragraph.indices, hasLength(5)); + }); +} From 7ffa603f5e03a68d6e02ebd9d1700689c688efe7 Mon Sep 17 00:00:00 2001 From: palmoni5 Date: Tue, 10 Nov 2026 00:25:14 +0200 Subject: [PATCH 3/8] test: cover dart-ci.yml for RTL combining mark changes --- .github/workflows/dart-ci.yml | 23 +++++++++++++++++++++++ 1 file changed, 23 insertions(+) create mode 100644 .github/workflows/dart-ci.yml diff --git a/.github/workflows/dart-ci.yml b/.github/workflows/dart-ci.yml new file mode 100644 index 0000000..aa7c918 --- /dev/null +++ b/.github/workflows/dart-ci.yml @@ -0,0 +1,23 @@ +name: Dart checks + +on: + pull_request: + push: + branches: [main] + +jobs: + dart: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - uses: subosito/flutter-action@v2 + with: + channel: stable + - name: Resolve dependencies (including Flutter example) + run: flutter pub get + - name: Static analysis + run: dart analyze lib test + - name: Regression tests + run: dart test --reporter expanded + - name: Benchmark smoke run + run: dart run benchmark/bidi_benchmark.dart From 442206ee055ddfdc138b2ff85e58120e91a074dc Mon Sep 17 00:00:00 2001 From: palmoni5 Date: Tue, 10 Nov 2026 00:25:17 +0200 Subject: [PATCH 4/8] test: cover bidi_benchmark.dart for RTL combining mark changes --- benchmark/bidi_benchmark.dart | 32 ++++++++++++++++++++++++++++++++ 1 file changed, 32 insertions(+) create mode 100644 benchmark/bidi_benchmark.dart diff --git a/benchmark/bidi_benchmark.dart b/benchmark/bidi_benchmark.dart new file mode 100644 index 0000000..42e14c3 --- /dev/null +++ b/benchmark/bidi_benchmark.dart @@ -0,0 +1,32 @@ +/// A small, reproducible smoke benchmark. It prints timings rather than +/// imposing machine-dependent performance thresholds. +import 'package:bidi/bidi.dart' as bidi; + +void main() { + final cases = { + 'mixed Hebrew/Latin': + 'זהו טקסט עברי עם PDF, Windows, מספרים 1234, וסימני פיסוק.', + 'vowelled Hebrew': + 'בְּרֵאשִׁית בָּרָא אֱלֹקִים אֵת הַשָּׁמַיִם וְאֵת הָאָרֶץ.', + 'marker-heavy': + '${List.filled(200, '\u200E').join()}Hello שלום', + 'long paragraph': + List.filled(100, 'שלום, World 2026;').join(' '), + }; + + const warmup = 50; + const iterations = 300; + for (final entry in cases.entries) { + var checksum = 0; + for (var i = 0; i < warmup; ++i) { + checksum += bidi.logicalToVisual(entry.value).length; + } + final watch = Stopwatch()..start(); + for (var i = 0; i < iterations; ++i) { + checksum += bidi.logicalToVisual(entry.value).length; + } + watch.stop(); + final micros = watch.elapsedMicroseconds / iterations; + print('${entry.key}: ${micros.toStringAsFixed(1)} us/op (checksum=$checksum)'); + } +} From eb422a84ae823205d9f46d4a366c6430cf7fba8b Mon Sep 17 00:00:00 2001 From: palmoni5 Date: Tue, 10 Nov 2026 00:29:32 +0200 Subject: [PATCH 5/8] test: isolate RTL cluster ordering from legacy Hebrew presentation-form shaping --- test/combining_marks_test.dart | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/test/combining_marks_test.dart b/test/combining_marks_test.dart index b812f4b..2540502 100644 --- a/test/combining_marks_test.dart +++ b/test/combining_marks_test.dart @@ -10,13 +10,12 @@ void main() { }); test('multiple niqqud marks remain in their original cluster order', () { - const input = 'ש\u05B8\u05C1לו\u05B9ם'; + // Avoid presentation-form composition (e.g. Shin + Shin Dot) so this + // regression isolates visual mark attachment rather than normalization. + const input = 'א\u05B8\u05BCב'; final paragraph = bidi.BidiString.fromLogical(input).paragraphs.single; - expect( - String.fromCharCodes(paragraph.bidiText), - 'םו\u05B9לש\u05B8\u05C1', - ); - expect(paragraph.indices, [6, 4, 5, 3, 0, 1, 2]); + expect(String.fromCharCodes(paragraph.bidiText), 'בא\u05B8\u05BC'); + expect(paragraph.indices, [3, 0, 1, 2]); }); test('RTL combining marks inside a Latin paragraph remain attached', () { From e7a39dcf53459a9a940a7269f90680b20c8da53f Mon Sep 17 00:00:00 2001 From: palmoni5 Date: Tue, 10 Nov 2026 00:37:42 +0200 Subject: [PATCH 6/8] test: use two distinct noncomposing niqqud marks for cluster-order fixture --- test/combining_marks_test.dart | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/test/combining_marks_test.dart b/test/combining_marks_test.dart index 2540502..be26450 100644 --- a/test/combining_marks_test.dart +++ b/test/combining_marks_test.dart @@ -12,9 +12,9 @@ void main() { test('multiple niqqud marks remain in their original cluster order', () { // Avoid presentation-form composition (e.g. Shin + Shin Dot) so this // regression isolates visual mark attachment rather than normalization. - const input = 'א\u05B8\u05BCב'; + const input = 'א\u05B0\u05B1ב'; final paragraph = bidi.BidiString.fromLogical(input).paragraphs.single; - expect(String.fromCharCodes(paragraph.bidiText), 'בא\u05B8\u05BC'); + expect(String.fromCharCodes(paragraph.bidiText), 'בא\u05B0\u05B1'); expect(paragraph.indices, [3, 0, 1, 2]); }); From ea25e656bb680004ae63706e291cb42e5feab75d Mon Sep 17 00:00:00 2001 From: palmoni5 Date: Tue, 10 Nov 2026 10:43:36 +0200 Subject: [PATCH 7/8] fix: keep benchmark comments compatible with full analysis --- benchmark/bidi_benchmark.dart | 12 +++++------- lib/src/paragraph.dart | 8 +++----- 2 files changed, 8 insertions(+), 12 deletions(-) diff --git a/benchmark/bidi_benchmark.dart b/benchmark/bidi_benchmark.dart index 42e14c3..acd80c5 100644 --- a/benchmark/bidi_benchmark.dart +++ b/benchmark/bidi_benchmark.dart @@ -1,5 +1,4 @@ -/// A small, reproducible smoke benchmark. It prints timings rather than -/// imposing machine-dependent performance thresholds. +// Prints timings without imposing machine-dependent thresholds. import 'package:bidi/bidi.dart' as bidi; void main() { @@ -8,10 +7,8 @@ void main() { 'זהו טקסט עברי עם PDF, Windows, מספרים 1234, וסימני פיסוק.', 'vowelled Hebrew': 'בְּרֵאשִׁית בָּרָא אֱלֹקִים אֵת הַשָּׁמַיִם וְאֵת הָאָרֶץ.', - 'marker-heavy': - '${List.filled(200, '\u200E').join()}Hello שלום', - 'long paragraph': - List.filled(100, 'שלום, World 2026;').join(' '), + 'marker-heavy': '${List.filled(200, '\u200E').join()}Hello שלום', + 'long paragraph': List.filled(100, 'שלום, World 2026;').join(' '), }; const warmup = 50; @@ -27,6 +24,7 @@ void main() { } watch.stop(); final micros = watch.elapsedMicroseconds / iterations; - print('${entry.key}: ${micros.toStringAsFixed(1)} us/op (checksum=$checksum)'); + print( + '${entry.key}: ${micros.toStringAsFixed(1)} us/op (checksum=$checksum)'); } } diff --git a/lib/src/paragraph.dart b/lib/src/paragraph.dart index e0fa77d..db8e3f5 100644 --- a/lib/src/paragraph.dart +++ b/lib/src/paragraph.dart @@ -285,7 +285,8 @@ class Normalization { final chClass = _getCanonicalClass(character); // Some precomposed characters decompose into combining marks even // when their original bidi class is a strong letter. - if (!hasNSMs && getCharacterType(character) == CharacterType.nonspacingMark) { + if (!hasNSMs && + getCharacterType(character) == CharacterType.nonspacingMark) { hasNSMs = true; } int k = text.length; // insertion point @@ -852,10 +853,7 @@ void _reorderString( } } - // L3. L2 reverses characters inside odd-level runs, leaving combining - // marks before their base character and reversing the order of adjacent - // marks. Restore each reversed base+mark cluster in-place. Skipping the pass - // for unpointed text keeps the common case unchanged. + // L3 restores base+mark clusters reversed inside odd-level runs. if (!hasNonspacingMark) { return; } From 483bffe6153553c46df61c6c048c48cbfc726a67 Mon Sep 17 00:00:00 2001 From: palmoni5 Date: Tue, 10 Nov 2026 11:00:11 +0200 Subject: [PATCH 8/8] ci: analyze the complete combining-mark package --- .github/workflows/dart-ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/dart-ci.yml b/.github/workflows/dart-ci.yml index aa7c918..ab3786b 100644 --- a/.github/workflows/dart-ci.yml +++ b/.github/workflows/dart-ci.yml @@ -16,7 +16,7 @@ jobs: - name: Resolve dependencies (including Flutter example) run: flutter pub get - name: Static analysis - run: dart analyze lib test + run: flutter analyze - name: Regression tests run: dart test --reporter expanded - name: Benchmark smoke run