Dynamic font loading

Data/system/fonts/MaterialSymbolsRounded.codepoints

@@ -53,6 +53,7 @@ network_wifi_1_bar ebe4
 network_wifi_2_bar ebd6
 network_wifi_3_bar ebe1
 note_add e89c
+palette e40a
 power_settings_new f8c7
 refresh e5d5
 sd_card e623

Modules/binfont-module/include/binfont/generator.h

@@ -17,7 +17,10 @@ typedef struct BinFontGeneratorConfig {
     uint16_t size;
     /** Bits per pixel: 1, 2, 3 or 4 */
     uint8_t bpp;
-    /** Codepoints to include. Codepoints that the TTF doesn't contain are skipped. */
+    /**
+     * Codepoints to include. Codepoints that the TTF doesn't contain are skipped.
+     * NULL (with codepoint_count 0) includes every codepoint in the TTF's character map.
+     */
     const uint32_t* codepoints;
     size_t codepoint_count;
 } BinFontGeneratorConfig;
@@ -30,11 +33,23 @@ typedef struct BinFontGeneratorConfig {
  * @retval ERROR_NONE on success
  * @retval ERROR_INVALID_ARGUMENT when the configuration is invalid
  * @retval ERROR_NOT_FOUND when the TTF file can't be read
- * @retval ERROR_NOT_SUPPORTED when the TTF file can't be parsed or contains none of the codepoints
+ * @retval ERROR_NOT_SUPPORTED when the TTF file or its character map can't be parsed, or it contains none of the codepoints
  * @retval ERROR_OUT_OF_MEMORY when memory allocation failed
  */
 error_t binfont_generate(const BinFontGeneratorConfig* config, uint8_t** out_data, size_t* out_size);
 
+/**
+ * @brief Lists the codepoints in the character map of a TrueType font.
+ * @param[in] ttf_path path to a TrueType (.ttf) file
+ * @param[out] out_codepoints the codepoints, to be released with memory_free()
+ * @param[out] out_count the number of codepoints
+ * @retval ERROR_NONE on success
+ * @retval ERROR_NOT_FOUND when the TTF file can't be read
+ * @retval ERROR_NOT_SUPPORTED when the TTF file or its character map can't be parsed
+ * @retval ERROR_OUT_OF_MEMORY when memory allocation failed
+ */
+error_t binfont_get_ttf_codepoints(const char* ttf_path, uint32_t** out_codepoints, size_t* out_count);
+
 #ifdef __cplusplus
 }
 #endif

Modules/binfont-module/private/stb_truetype.h

@@ -1,6 +1,9 @@
 // stb_truetype.h - v1.26 - public domain
 // authored from 2009-2021 by Sean Barrett / RAD Game Tools
 //
+// Modified for Tactility: stbtt__hheap_alloc() allocates smaller chunks, so rasterizing works
+// with a fragmented heap on devices without PSRAM.
+//
 // =======================================================================
 //
 //    NO SECURITY GUARANTEE -- DO NOT USE THIS ON UNTRUSTED FONT FILES
@@ -2775,7 +2778,7 @@ static void *stbtt__hheap_alloc(stbtt__hheap *hh, size_t size, void *userdata)
       return p;
    } else {
       if (hh->num_remaining_in_head_chunk == 0) {
-         int count = (size < 32 ? 2000 : size < 128 ? 800 : 100);
+         int count = (size < 32 ? 100 : size < 128 ? 50 : 10);
          stbtt__hheap_chunk *c = (stbtt__hheap_chunk *) STBTT_malloc(sizeof(stbtt__hheap_chunk) + size * count, userdata);
          if (c == NULL)
             return NULL;

Modules/binfont-module/source/generator.cpp

@@ -9,10 +9,29 @@
 #include <cstdio>
 #include <cstring>
 
+namespace {
+
+/** Passed to stb_truetype as user data, to detect allocation failures it otherwise ignores */
+struct AllocationState {
+    bool failed;
+};
+
+void* stbtt_allocate(size_t size, void* user) {
+    void* pointer = memory_alloc(size);
+    if (pointer == nullptr && user != nullptr) {
+        static_cast<AllocationState*>(user)->failed = true;
+    }
+    return pointer;
+}
+
+} // namespace
+
 #define STBTT_STATIC
 #define STB_TRUETYPE_IMPLEMENTATION
-#define STBTT_malloc(size, user) ((void)(user), memory_alloc(size))
+#define STBTT_malloc(size, user) stbtt_allocate(size, user)
 #define STBTT_free(pointer, user) ((void)(user), memory_free(pointer))
+// stb_truetype skips what it can't allocate, allocation failures are detected via AllocationState
+#define STBTT_assert(condition) ((void)0)
 #include "stb_truetype.h"
 
 namespace {
@@ -253,15 +272,125 @@ void write_cmap(ByteBuffer& out, const GlyphInfo* glyphs, size_t count) {
     out.endRecord(record);
 }
 
-error_t generate(const BinFontGeneratorConfig* config, const stbtt_fontinfo& info, GlyphInfo* glyphs, ByteBuffer& out) {
+/** Appends codepoints to a growable array */
+class CodepointList {
+    uint32_t* data = nullptr;
+    size_t count = 0;
+    size_t capacity = 0;
+    bool failed = false;
+
+public:
+    ~CodepointList() { memory_free(data); }
+
+    void addRange(uint32_t first, uint32_t last) {
+        for (uint32_t codepoint = first; codepoint <= last && !failed; codepoint++) {
+            if (count == capacity) {
+                const size_t new_capacity = capacity ? capacity * 2 : 256;
+                auto* new_data = static_cast<uint32_t*>(memory_realloc(data, new_capacity * sizeof(uint32_t)));
+                if (new_data == nullptr) {
+                    failed = true;
+                    return;
+                }
+                data = new_data;
+                capacity = new_capacity;
+            }
+            data[count++] = codepoint;
+        }
+    }
+
+    const uint32_t* getData() const { return data; }
+    size_t getCount() const { return count; }
+    bool hasFailed() const { return failed; }
+
+    uint32_t* release() {
+        uint32_t* result = data;
+        data = nullptr;
+        count = 0;
+        capacity = 0;
+        return result;
+    }
+};
+
+/**
+ * Collects the codepoints of the character map that stb_truetype selected (formats 0, 4, 6 and 12).
+ * @return false when the format is not supported or the table is out of bounds
+ */
+bool collect_codepoints(const stbtt_fontinfo& info, size_t ttfSize, CodepointList& list) {
+    stbtt_uint8* data = info.data;
+    const auto map = static_cast<size_t>(info.index_map);
+    auto in_bounds = [ttfSize](size_t offset, size_t size) { return offset <= ttfSize && size <= ttfSize - offset; };
+    if (!in_bounds(map, 2)) {
+        return false;
+    }
+
+    switch (ttUSHORT(data + map)) {
+        case 0:
+            list.addRange(0, 255);
+            break;
+        case 4: {
+            if (!in_bounds(map, 14)) {
+                return false;
+            }
+            const size_t segment_count = ttUSHORT(data + map + 6) / 2;
+            const size_t end_codes = map + 14;
+            const size_t start_codes = end_codes + segment_count * 2 + 2;
+            if (!in_bounds(start_codes, segment_count * 2)) {
+                return false;
+            }
+            for (size_t i = 0; i < segment_count; i++) {
+                const uint32_t start = ttUSHORT(data + start_codes + i * 2);
+                const uint32_t end = ttUSHORT(data + end_codes + i * 2);
+                // The last segment maps 0xFFFF to the missing glyph
+                if (start <= end && start != 0xFFFF) {
+                    list.addRange(start, end == 0xFFFF ? 0xFFFE : end);
+                }
+            }
+            break;
+        }
+        case 6: {
+            if (!in_bounds(map, 10)) {
+                return false;
+            }
+            const uint32_t first = ttUSHORT(data + map + 6);
+            const uint32_t entry_count = ttUSHORT(data + map + 8);
+            if (entry_count > 0) {
+                list.addRange(first, first + entry_count - 1);
+            }
+            break;
+        }
+        case 12: {
+            if (!in_bounds(map, 16)) {
+                return false;
+            }
+            const size_t group_count = ttULONG(data + map + 12);
+            if (!in_bounds(map + 16, group_count * 12)) {
+                return false;
+            }
+            for (size_t i = 0; i < group_count; i++) {
+                stbtt_uint8* group = data + map + 16 + i * 12;
+                const uint32_t start = ttULONG(group);
+                const uint32_t end = ttULONG(group + 4);
+                if (start <= end && end <= 0x10FFFF) {
+                    list.addRange(start, end);
+                }
+            }
+            break;
+        }
+        default:
+            return false;
+    }
+    return !list.hasFailed();
+}
+
+error_t generate(const BinFontGeneratorConfig* config, const uint32_t* codepoints, size_t codepointCount, const stbtt_fontinfo& info, GlyphInfo* glyphs, ByteBuffer& out) {
     const float scale = stbtt_ScaleForMappingEmToPixels(&info, config->size);
 
     // Collect glyphs, sorted by codepoint and without duplicates or missing glyphs
     size_t count = 0;
-    for (size_t i = 0; i < config->codepoint_count; i++) {
-        const int index = stbtt_FindGlyphIndex(&info, static_cast<int>(config->codepoints[i]));
+    for (size_t i = 0; i < codepointCount; i++) {
+        const int index = stbtt_FindGlyphIndex(&info, static_cast<int>(codepoints[i]));
         if (index != 0) {
-            glyphs[count].codepoint = config->codepoints[i];
+            glyphs[count].codepoint = codepoints[i];
             glyphs[count].ttfIndex = index;
             count++;
         }
@@ -386,6 +515,10 @@ error_t generate(const BinFontGeneratorConfig* config, const stbtt_fontinfo& inf
                 bitmap_capacity = pixel_count;
             }
             stbtt_MakeGlyphBitmap(&info, bitmap, width, height, width, scale, scale, glyph.ttfIndex);
+            if (static_cast<const AllocationState*>(info.userdata)->failed) {
+                memory_free(bitmap);
+                return ERROR_OUT_OF_MEMORY;
+            }
             for (size_t i = 0; i < pixel_count; i++) {
                 writer.write((bitmap[i] * max_value + 127) / 255, config->bpp);
             }
@@ -404,7 +537,7 @@ error_t generate(const BinFontGeneratorConfig* config, const stbtt_fontinfo& inf
 extern "C" {
 
 error_t binfont_generate(const BinFontGeneratorConfig* config, uint8_t** out_data, size_t* out_size) {
-    if (config->ttf_path == nullptr || config->size == 0 || config->bpp < 1 || config->bpp > 4 || config->codepoint_count == 0) {
+    if (config->ttf_path == nullptr || config->size == 0 || config->bpp < 1 || config->bpp > 4 || (config->codepoints == nullptr) != (config->codepoint_count == 0)) {
         return ERROR_INVALID_ARGUMENT;
     }
 
@@ -420,15 +553,29 @@ error_t binfont_generate(const BinFontGeneratorConfig* config, uint8_t** out_dat
         memory_free(ttf);
         return ERROR_NOT_SUPPORTED;
     }
+    AllocationState allocation = { .failed = false };
+    info.userdata = &allocation;
+
+    const uint32_t* codepoints = config->codepoints;
+    size_t codepoint_count = config->codepoint_count;
+    CodepointList all_codepoints;
+    if (codepoints == nullptr) {
+        if (!collect_codepoints(info, ttf_size, all_codepoints)) {
+            memory_free(ttf);
+            return all_codepoints.hasFailed() ? ERROR_OUT_OF_MEMORY : ERROR_NOT_SUPPORTED;
+        }
+        codepoints = all_codepoints.getData();
+        codepoint_count = all_codepoints.getCount();
+    }
 
-    auto* glyphs = static_cast<GlyphInfo*>(memory_alloc(config->codepoint_count * sizeof(GlyphInfo)));
+    auto* glyphs = static_cast<GlyphInfo*>(memory_alloc(codepoint_count * sizeof(GlyphInfo)));
     if (glyphs == nullptr) {
         memory_free(ttf);
         return ERROR_OUT_OF_MEMORY;
     }
 
     ByteBuffer out;
-    const error_t error = generate(config, info, glyphs, out);
+    const error_t error = generate(config, codepoints, codepoint_count, info, glyphs, out);
     memory_free(glyphs);
     memory_free(ttf);
     if (error != ERROR_NONE) {
@@ -440,4 +587,30 @@ error_t binfont_generate(const BinFontGeneratorConfig* config, uint8_t** out_dat
     return ERROR_NONE;
 }
 
+error_t binfont_get_ttf_codepoints(const char* ttf_path, uint32_t** out_codepoints, size_t* out_count) {
+    size_t ttf_size = 0;
+    uint8_t* ttf = read_file(ttf_path, ttf_size);
+    if (ttf == nullptr) {
+        return ERROR_NOT_FOUND;
+    }
+
+    stbtt_fontinfo info;
+    const int font_offset = stbtt_GetFontOffsetForIndex(ttf, 0);
+    CodepointList codepoints;
+    error_t error = ERROR_NONE;
+    if (font_offset < 0 || stbtt_InitFont(&info, ttf, font_offset) == 0) {
+        error = ERROR_NOT_SUPPORTED;
+    } else if (!collect_codepoints(info, ttf_size, codepoints)) {
+        error = codepoints.hasFailed() ? ERROR_OUT_OF_MEMORY : ERROR_NOT_SUPPORTED;
+    }
+    memory_free(ttf);
+    if (error != ERROR_NONE) {
+        return error;
+    }
+
+    *out_count = codepoints.getCount();
+    *out_codepoints = codepoints.release();
+    return ERROR_NONE;
+}
+
 }

Modules/binfont-module/source/module.c

@@ -17,6 +17,7 @@ static const struct ModuleSymbol SYMBOLS[] = {
     DEFINE_MODULE_SYMBOL(binfont_get_glyph_bitmap),
     // generator
     DEFINE_MODULE_SYMBOL(binfont_generate),
+    DEFINE_MODULE_SYMBOL(binfont_get_ttf_codepoints),
     // render
     DEFINE_MODULE_SYMBOL(binfont_draw_glyph_rgb565),
     DEFINE_MODULE_SYMBOL(binfont_draw_text_rgb565),

Modules/binfont-module/tests/CMakeLists.txt

@@ -10,6 +10,7 @@ target_include_directories(BinFontModuleTests PRIVATE ${DOCTESTINC} ${PROJECT_SO
 target_compile_definitions(BinFontModuleTests PRIVATE
     "BINFONT_TEST_FIXTURES_DIR=\"${PROJECT_SOURCE_DIR}/fixtures\""
     "BINFONT_TEST_TEMP_DIR=\"${CMAKE_CURRENT_BINARY_DIR}\""
+    "BINFONT_TEST_TEXT_TTF=\"${CMAKE_SOURCE_DIR}/Data/system/fonts/AdwaitaSans.ttf\""
     "BINFONT_TEST_ICON_TTF=\"${CMAKE_SOURCE_DIR}/Data/system/fonts/MaterialSymbolsRounded.ttf\""
 )
 

Modules/binfont-module/tests/source/binfont_test.cpp

@@ -7,6 +7,7 @@
 #include <graphics/pixel_buffer.h>
 #include <tactility/memory.h>
 
+#include <algorithm>
 #include <cstdio>
 #include <cstdlib>
 #include <string>
@@ -300,6 +301,58 @@ TEST_CASE("binfont_generate rasterizes a TTF close to lv_font_conv") {
     binfont_close(reference);
 }
 
+TEST_CASE("binfont_generate includes every codepoint of the TTF without a codepoint list") {
+    const BinFontGeneratorConfig config = {
+        .ttf_path = BINFONT_TEST_TEXT_TTF,
+        .size = 14,
+        .bpp = 2,
+        .codepoints = nullptr,
+        .codepoint_count = 0,
+    };
+    uint8_t* data = nullptr;
+    size_t size = 0;
+    REQUIRE(binfont_generate(&config, &data, &size) == ERROR_NONE);
+    BinFont* font = nullptr;
+    REQUIRE(binfont_open_memory(data, size, true, &font) == ERROR_NONE);
+
+    BinFontGlyph glyph;
+    for (uint32_t codepoint : { 0x20u, 0x41u, 0x7Eu, 0xE9u, 0xFFu, 0x2026u, 0x20ACu }) {
+        CAPTURE(codepoint);
+        CHECK(binfont_get_glyph(font, codepoint, &glyph));
+    }
+    // Not in the subset
+    CHECK_FALSE(binfont_get_glyph(font, 0x0100, &glyph));
+    CHECK_FALSE(binfont_get_glyph(font, 0x0416, &glyph));
+
+    binfont_close(font);
+}
+
+TEST_CASE("binfont_get_ttf_codepoints lists the character map of a TTF") {
+    uint32_t* codepoints = nullptr;
+    size_t count = 0;
+    REQUIRE(binfont_get_ttf_codepoints(BINFONT_TEST_TEXT_TTF, &codepoints, &count) == ERROR_NONE);
+    CHECK(count > 190);
+    CHECK(std::find(codepoints, codepoints + count, 0x41u) != codepoints + count);
+    CHECK(std::find(codepoints, codepoints + count, 0x20ACu) != codepoints + count);
+    CHECK(std::find(codepoints, codepoints + count, 0x0416u) == codepoints + count);
+    memory_free(codepoints);
+
+    CHECK_EQ(binfont_get_ttf_codepoints(BINFONT_TEST_FIXTURES_DIR "/missing.ttf", &codepoints, &count), ERROR_NOT_FOUND);
+}
+
+TEST_CASE("binfont_generate rejects a codepoint count without codepoints") {
+    const BinFontGeneratorConfig config = {
+        .ttf_path = BINFONT_TEST_TEXT_TTF,
+        .size = 14,
+        .bpp = 2,
+        .codepoints = nullptr,
+        .codepoint_count = 5,
+    };
+    uint8_t* data = nullptr;
+    size_t size = 0;
+    CHECK_EQ(binfont_generate(&config, &data, &size), ERROR_INVALID_ARGUMENT);
+}
+
 TEST_CASE("binfont_generate fails for a missing TTF") {
     const uint32_t codepoint = 0xE5C3;
     const BinFontGeneratorConfig config = {