fix(perf): Remove FontSources.masks as they were consuming large amounts of memory even when no font sources were set

Cargo.toml

@@ -43,7 +43,6 @@ anyhow = "1.0"
 approx = "0.5.1"
 async-trait = "0.1"
 base64 = "0.22.1"
-bit-set = "0.8"
 brotli = ">=5, <9"
 bytes = "1"
 clap = { version = "4", features = ["derive", "unstable-markdown", "wrap_help"] }

martin-core/Cargo.toml

@@ -36,7 +36,6 @@ postgres = [
 unstable-cog = ["dep:png", "dep:tiff", "_tiles"]
 unstable-rendering = ["styles", "dep:maplibre_native", "dep:image"]
 fonts = [
-    "dep:bit-set",
     "dep:pbf_font_tools",
     "dep:regex",
     "dep:itertools",
@@ -63,7 +62,6 @@ maplibre_native = { workspace = true, optional = true }
 actix-web = { workspace = true, optional = true }
 async-trait.workspace = true
 base64  = { workspace = true, optional = true }
-bit-set = { workspace = true, optional = true }
 dashmap = { workspace = true, optional = true }
 deadpool-postgres = { workspace = true, optional = true }
 futures = { workspace = true, optional = true }

martin-core/src/resources/fonts/mod.rs

@@ -16,13 +16,12 @@
 //! let font_data = sources.get_font_range("Arial,Helvetica", 0, 255).unwrap();
 //! ```
 
-use std::collections::HashMap;
+use std::collections::{BTreeSet, HashMap};
 use std::ffi::OsStr;
 use std::fmt::Debug;
 use std::path::PathBuf;
-use std::sync::LazyLock;
+use std::sync::{Arc, LazyLock};
 
-use bit_set::BitSet;
 use dashmap::{DashMap, Entry};
 use itertools::Itertools as _;
 use pbf_font_tools::freetype::{Face, Library};
@@ -53,7 +52,7 @@ const RADIUS: usize = 8;
 const CUTOFF: f64 = 0.25_f64;
 /// Maximum Unicode codepoint range ID.
 ///
-/// Each range is 256 codepoints long, so the highest range ID is 0xFFFF / 256 = 255.
+/// Each range is 256 codepoints long, so the highest range ID is 0x10FFFF / 256 = 4351.
 const MAX_UNICODE_CP_RANGE_ID: usize = MAX_UNICODE_CP / CP_RANGE_SIZE;
 
 mod error;
@@ -63,36 +62,37 @@ mod cache;
 pub use cache::{FontCache, NO_FONT_CACHE, OptFontCache};
 
 /// Glyph information: (codepoints, count, ranges, first, last).
-type GetGlyphInfo = (BitSet, usize, Vec<(usize, usize)>, usize, usize);
+type GetGlyphInfo = (BTreeSet<usize>, usize, Vec<(usize, usize)>, usize, usize);
 
 /// Extracts available codepoints from a font face.
 ///
 /// Returns `None` if the font contains no usable glyphs.
 fn get_available_codepoints(face: &mut Face) -> Option<GetGlyphInfo> {
-    let mut codepoints = BitSet::with_capacity(MAX_UNICODE_CP);
+    let mut codepoints = BTreeSet::new();
     let mut spans = Vec::new();
     let mut first: Option<usize> = None;
-    let mut count = 0;
+    let mut last = 0;
 
-    for cp in 0..=MAX_UNICODE_CP {
-        if face.get_char_index(cp).is_ok() {
-            codepoints.insert(cp);
-            count += 1;
-            if first.is_none() {
+    for (cp, _) in face.chars() {
+        codepoints.insert(cp);
+        if let Some(start) = first {
+            if cp != last + 1 {
+                spans.push((start, last));
                 first = Some(cp);
             }
-        } else if let Some(start) = first {
-            spans.push((start, cp - 1));
-            first = None;
+        } else {
+            first = Some(cp);
         }
+        last = cp;
     }
 
-    if count == 0 {
-        None
-    } else {
+    if let Some(first) = first {
+        spans.push((first, last));
+        let count = usize::try_from(face.num_glyphs()).unwrap();
         let start = spans[0].0;
-        let end = spans[spans.len() - 1].1;
-        Some((codepoints, count, spans, start, end))
+        Some((codepoints, count, spans, start, last))
+    } else {
+        None
     }
 }
 
@@ -123,25 +123,25 @@ pub struct FontSources {
     /// Map of font name to font source data.
     fonts: DashMap<String, FontSource>,
     /// Pre-computed bitmasks for each 256-character Unicode range.
-    masks: Vec<BitSet>,
+    masks: Arc<Vec<BTreeSet<usize>>>,
 }
 
 impl Default for FontSources {
     fn default() -> Self {
         let mut masks = Vec::with_capacity(MAX_UNICODE_CP_RANGE_ID + 1);
 
-        let mut bs = BitSet::with_capacity(CP_RANGE_SIZE);
+        let mut set = BTreeSet::new();
         for v in 0..=MAX_UNICODE_CP {
-            bs.insert(v);
-            if v % CP_RANGE_SIZE == (CP_RANGE_SIZE - 1) {
-                masks.push(bs);
-                bs = BitSet::with_capacity(CP_RANGE_SIZE);
+            set.insert(v);
+            if (v % CP_RANGE_SIZE) == (CP_RANGE_SIZE - 1) {
+                masks.push(set);
+                set = BTreeSet::new();
             }
         }
 
         Self {
             fonts: DashMap::new(),
-            masks,
+            masks: Arc::new(masks),
         }
     }
 }
@@ -181,19 +181,17 @@ impl FontSources {
             return Err(FontError::InvalidFontRange(start, end));
         }
 
-        let mut needed = self.masks[(start as usize) / CP_RANGE_SIZE].clone();
+        let needed: &BTreeSet<usize> = &self.masks[(start as usize) / CP_RANGE_SIZE];
         let fonts = ids
             .split(',')
             .filter_map(|id| match self.fonts.get(id) {
                 None => Some(Err(FontError::FontNotFound(id.to_string()))),
                 Some(v) => {
-                    let mut ds = needed.clone();
-                    ds.intersect_with(&v.codepoints);
-                    if ds.is_empty() {
+                    if needed.intersection(&v.codepoints).count() == 0 {
                         None
                     } else {
-                        needed.difference_with(&v.codepoints);
-                        Some(Ok((id, v, ds)))
+                        let diff: Vec<usize> = v.codepoints.difference(needed).copied().collect();
+                        Some(Ok((id, v, diff)))
                     }
                 }
             })
@@ -225,7 +223,7 @@ impl FontSources {
             // and https://www.freetype.org/freetype2/docs/tutorial/step1.html for details.
             face.set_char_size(0, CHAR_HEIGHT, 0, 0)?;
 
-            for cp in &ds {
+            for cp in ds {
                 let glyph = render_sdf_glyph(&face, cp as u32, BUFFER_SIZE, RADIUS, CUTOFF)?;
                 stack.glyphs.push(glyph);
             }
@@ -247,7 +245,7 @@ pub struct FontSource {
     /// Face index within the font file (for .ttc collections).
     face_index: isize,
     /// Unicode codepoints this font supports.
-    codepoints: BitSet,
+    codepoints: BTreeSet<usize>,
     /// Font metadata for the catalog.
     catalog_entry: CatalogFontEntry,
 }
@@ -383,14 +381,13 @@ mod tests {
         // U+3320: SQUARE SANTIIMU, U+1F60A: SMILING FACE WITH SMILING EYES
         for codepoint in [0x3320, 0x1f60a] {
             let font_path = PathBuf::from(env!("CARGO_MANIFEST_DIR"))
-                .join(format!("src/resources/fonts/tests/u+{codepoint:x}.ttf"));
-            assert!(font_path.is_file()); // make sure the file path is correct
-
+                .join(format!("../tests/fixtures/fonts2/u+{codepoint:x}.ttf"));
+            assert!(font_path.is_file(), "{}", font_path.display());
             let mut face = lib.new_face(&font_path, 0).unwrap();
 
             let (_codepoints, count, _ranges, first, last) =
                 get_available_codepoints(&mut face).unwrap();
-            assert_eq!(count, 1);
+            assert_eq!(count, 2);
             assert_eq!(format!("U+{first:X}"), format!("U+{codepoint:X}"));
             assert_eq!(format!("U+{last:X}"), format!("U+{codepoint:X}"));
         }