perf: lexer + recognizer micro-optimizations on the C# hot path

Cargo.toml

@@ -19,6 +19,7 @@ path = "src/lib.rs"
 [features]
 default = []
 std = []
+perf-counters = []
 
 [dependencies]
 thiserror = "2"

src/atn/lexer.rs

@@ -1,4 +1,5 @@
-use std::collections::BTreeSet;
+use std::collections::{BTreeSet, HashSet};
+use std::hash::BuildHasherDefault;
 
 use crate::atn::{Atn, AtnStateKind, LexerAction, Transition};
 use crate::char_stream::{CharStream, TextInterval};
@@ -7,12 +8,16 @@ use crate::lexer::{
     BaseLexer, Lexer, LexerCustomAction, LexerDfaActionKey, LexerDfaCachedAccept,
     LexerDfaCachedState, LexerDfaCachedTransition, LexerDfaConfigKey, LexerDfaKey, LexerPredicate,
 };
+use crate::prediction::PredictionFxHasher;
 use crate::token::{CommonToken, DEFAULT_CHANNEL, INVALID_TOKEN_TYPE, TokenFactory};
 
+#[allow(clippy::disallowed_types)]
+type FxHashSet<K> = HashSet<K, BuildHasherDefault<PredictionFxHasher>>;
+
 const MIN_CHAR_VALUE: i32 = 0;
 const MAX_CHAR_VALUE: i32 = 0x0010_FFFF;
 
-#[derive(Clone, Debug, Eq, Ord, PartialEq, PartialOrd)]
+#[derive(Clone, Debug, Eq, Hash, Ord, PartialEq, PartialOrd)]
 struct LexerConfig {
     state: usize,
     position: usize,
@@ -23,7 +28,7 @@ struct LexerConfig {
     actions: Vec<LexerActionTrace>,
 }
 
-#[derive(Clone, Copy, Debug, Eq, Ord, PartialEq, PartialOrd)]
+#[derive(Clone, Copy, Debug, Eq, Hash, Ord, PartialEq, PartialOrd)]
 struct LexerActionTrace {
     action_index: usize,
     position: usize,
@@ -100,7 +105,7 @@ impl LexerActionResult {
 /// Accumulates one epsilon-closure expansion, including whether predicate
 /// evaluation made the closure input-position-sensitive.
 struct ClosureState {
-    seen: BTreeSet<LexerConfig>,
+    seen: FxHashSet<LexerConfig>,
     closed: Vec<LexerConfig>,
     has_semantic_context: bool,
 }
@@ -723,7 +728,7 @@ where
     P: FnMut(&BaseLexer<I, F>, LexerPredicate) -> bool,
 {
     let mut state = ClosureState {
-        seen: BTreeSet::new(),
+        seen: FxHashSet::default(),
         closed: Vec::new(),
         has_semantic_context: false,
     };

src/atn/mod.rs

@@ -359,9 +359,19 @@ impl IntervalSet {
 
     /// Returns true when `value` falls inside any stored interval.
     pub fn contains(&self, value: i32) -> bool {
-        self.ranges
-            .iter()
-            .any(|(start, stop)| (*start..=*stop).contains(&value))
+        // Ranges are kept sorted and coalesced by `normalize`, so the first
+        // range whose `start > value` cannot contain `value` and neither can
+        // any range after it. Binary searching for that boundary turns
+        // membership lookup from O(n) to O(log n), which matters because
+        // parser/lexer hot paths call this once per `Set`/`NotSet`/`Wildcard`
+        // transition probe.
+        match self
+            .ranges
+            .binary_search_by(|(start, _)| start.cmp(&value))
+        {
+            Ok(_) => true,
+            Err(pos) => pos > 0 && self.ranges[pos - 1].1 >= value,
+        }
     }
 
     pub fn ranges(&self) -> &[(i32, i32)] {

src/lexer.rs

@@ -1,10 +1,16 @@
-use std::collections::{BTreeMap, BTreeSet};
+use std::collections::{BTreeSet, HashMap};
+use std::hash::BuildHasherDefault;
+use std::rc::Rc;
 
 use crate::char_stream::{CharStream, TextInterval};
 use crate::int_stream::EOF;
+use crate::prediction::PredictionFxHasher;
 use crate::recognizer::{Recognizer, RecognizerData};
 use crate::token::{CommonToken, CommonTokenFactory, TokenFactory, TokenSourceError, TokenSpec};
 
+#[allow(clippy::disallowed_types)]
+type FxHashMap<K, V> = HashMap<K, V, BuildHasherDefault<PredictionFxHasher>>;
+
 pub const SKIP: i32 = -3;
 pub const MORE: i32 = -2;
 pub const DEFAULT_MODE: i32 = 0;
@@ -113,29 +119,29 @@ pub struct BaseLexer<I, F = CommonTokenFactory> {
 /// runtime-suite descriptors.
 #[derive(Clone, Debug, Default)]
 struct LexerDfaTrace {
-    state_numbers: BTreeMap<LexerDfaKey, usize>,
-    accept_predictions: BTreeMap<usize, i32>,
+    state_numbers: FxHashMap<LexerDfaKey, usize>,
+    accept_predictions: FxHashMap<usize, i32>,
     edges: BTreeSet<LexerDfaEdge>,
-    cached_states: BTreeMap<usize, LexerDfaCachedState>,
-    transitions: BTreeMap<(usize, i32), LexerDfaCachedTransition>,
-    mode_starts: BTreeMap<i32, usize>,
+    cached_states: FxHashMap<usize, Rc<LexerDfaCachedState>>,
+    transitions: FxHashMap<(usize, i32), LexerDfaCachedTransition>,
+    mode_starts: FxHashMap<i32, usize>,
 }
 
 impl LexerDfaTrace {
-    const fn new() -> Self {
+    fn new() -> Self {
         Self {
-            state_numbers: BTreeMap::new(),
-            accept_predictions: BTreeMap::new(),
+            state_numbers: FxHashMap::default(),
+            accept_predictions: FxHashMap::default(),
             edges: BTreeSet::new(),
-            cached_states: BTreeMap::new(),
-            transitions: BTreeMap::new(),
-            mode_starts: BTreeMap::new(),
+            cached_states: FxHashMap::default(),
+            transitions: FxHashMap::default(),
+            mode_starts: FxHashMap::default(),
         }
     }
 }
 
 /// Normalized lexer ATN config-set identity used for observed DFA traces.
-#[derive(Clone, Debug, Eq, Ord, PartialEq, PartialOrd)]
+#[derive(Clone, Debug, Eq, Hash, Ord, PartialEq, PartialOrd)]
 pub(crate) struct LexerDfaKey {
     configs: Vec<LexerDfaConfigKey>,
 }
@@ -148,7 +154,7 @@ impl LexerDfaKey {
 }
 
 /// One lexer ATN config identity with the absolute input position removed.
-#[derive(Clone, Debug, Eq, Ord, PartialEq, PartialOrd)]
+#[derive(Clone, Debug, Eq, Hash, Ord, PartialEq, PartialOrd)]
 pub(crate) struct LexerDfaConfigKey {
     pub(crate) state: usize,
     pub(crate) alt_rule_index: Option<usize>,
@@ -158,7 +164,7 @@ pub(crate) struct LexerDfaConfigKey {
     pub(crate) actions: Vec<LexerDfaActionKey>,
 }
 
-#[derive(Clone, Copy, Debug, Eq, Ord, PartialEq, PartialOrd)]
+#[derive(Clone, Copy, Debug, Eq, Hash, Ord, PartialEq, PartialOrd)]
 pub(crate) struct LexerDfaActionKey {
     pub(crate) action_index: usize,
     pub(crate) position_delta: usize,
@@ -220,7 +226,7 @@ where
     I: CharStream,
 {
     /// Creates a lexer base using `CommonTokenFactory`.
-    pub const fn new(input: I, data: RecognizerData) -> Self {
+    pub fn new(input: I, data: RecognizerData) -> Self {
         Self::with_factory(input, data, CommonTokenFactory)
     }
 }
@@ -231,7 +237,7 @@ where
     F: TokenFactory,
 {
     /// Creates a lexer base with a custom token factory.
-    pub const fn with_factory(input: I, data: RecognizerData, factory: F) -> Self {
+    pub fn with_factory(input: I, data: RecognizerData, factory: F) -> Self {
         Self {
             input,
             data,
@@ -530,7 +536,7 @@ where
             .or_insert(transition);
     }
 
-    pub(crate) fn cached_lexer_dfa_state(&self, state: usize) -> Option<LexerDfaCachedState> {
+    pub(crate) fn cached_lexer_dfa_state(&self, state: usize) -> Option<Rc<LexerDfaCachedState>> {
         self.lexer_dfa.cached_states.get(&state).cloned()
     }
 
@@ -542,7 +548,7 @@ where
         self.lexer_dfa
             .cached_states
             .entry(state)
-            .or_insert(cached_state);
+            .or_insert_with(|| Rc::new(cached_state));
     }
 
     pub(crate) fn cached_lexer_mode_start(&self, mode: i32) -> Option<usize> {