Accelerate lexer fallback and parser deduplication - #111
Conversation
Copy/Paste DetectionFound 25 duplication(s) across 3 changed Rust file(s) (threshold: 100 tokens). Show duplicationsFound a 54 line (320 tokens) duplication in the following files:
atn: &LexerAtn,
hooks: &mut H,
mut generated_action: A,
mut generated_predicate: P,
unknown_policy: UnknownSemanticPolicy,
mut accept_adjuster: E,
) -> Result<TokenId, TokenStoreError>
where
I: CharStream,
H: SemanticHooks,
A: FnMut(&mut BaseLexer<I>, LexerCustomAction) -> bool,
P: FnMut(&BaseLexer<I>, LexerPredicate) -> Option<bool>,
E: FnMut(&mut BaseLexer<I>, i32, usize),
{
let hooks = RefCell::new(hooks);
let token = next_token_with_hooks_impl(
lexer,
sink,
atn,
&mut |lexer, action| {
if !generated_action(lexer, action)
&& !dispatch_lexer_action_hook(&hooks, lexer, action)
&& unknown_policy == UnknownSemanticPolicy::Error
&& let (Ok(rule), Ok(index)) = (
usize::try_from(action.rule_index()),
usize::try_from(action.action_index()),
)
{
lexer.record_semantic_error(true, rule, index);
}
},
&mut |lexer, predicate| {
generated_predicate(lexer, predicate)
.or_else(|| dispatch_lexer_predicate_hook(&hooks, lexer, predicate))
.unwrap_or_else(|| match unknown_policy {
UnknownSemanticPolicy::AssumeTrue => true,
UnknownSemanticPolicy::AssumeFalse => false,
UnknownSemanticPolicy::Error => {
lexer.record_semantic_error(
false,
predicate.rule_index(),
predicate.pred_index(),
);
false
}
})
},
&mut |lexer| dispatch_lexer_before_token_hook(&hooks, lexer),
&mut accept_adjuster,
&mut |lexer, accept_position| {
dispatch_lexer_after_accept_hook(&hooks, lexer, accept_position);
},
LexerMatchStrategy {
compiled: None,Found a 26 line (153 tokens) duplication in the following files:
pub fn next_token_with_hooks<I, A, P, E>(
lexer: &mut BaseLexer<I>,
sink: &mut TokenSink<'_>,
atn: &LexerAtn,
mut custom_action: A,
mut semantic_predicate: P,
mut accept_adjuster: E,
) -> Result<TokenId, TokenStoreError>
where
I: CharStream,
A: FnMut(&mut BaseLexer<I>, LexerCustomAction),
P: FnMut(&BaseLexer<I>, LexerPredicate) -> bool,
E: FnMut(&mut BaseLexer<I>, i32, usize),
{
next_token_with_hooks_impl(
lexer,
sink,
atn,
&mut custom_action,
&mut semantic_predicate,
&mut |_| {},
&mut accept_adjuster,
&mut |_, _| {},
LexerMatchStrategy {
compiled: None,
use_cache: false,Found a 27 line (145 tokens) duplication in the following files:
fn generated_match_token_recovers_missing_token_from_context_follow() {
let atn = generated_match_recovery_atn();
let data = RecognizerData::new(
"Mini.g4",
Vocabulary::new(
[None, Some("'X'"), Some("'Y'")],
[None, Some("X"), Some("Y")],
[None::<&str>, None, None],
),
);
let mut parser = BaseParser::new(
CommonTokenStream::new(Source {
tokens: vec![TestToken::eof("parser-test", 3, 1, 3)],
index: 0,
}),
data,
);
parser.rule_context_stack = vec![
RuleContextFrame {
rule_index: 0,
invoking_state: 0,
},
RuleContextFrame {
rule_index: 1,
invoking_state: 1,
},
];Found a 25 line (142 tokens) duplication in the following files:
atn: &LexerAtn,
hooks: &mut H,
) -> Result<TokenId, TokenStoreError>
where
I: CharStream,
H: SemanticHooks,
{
let hooks = RefCell::new(hooks);
let token = next_token_with_hooks_impl(
lexer,
sink,
atn,
&mut |lexer, action| {
let _ = dispatch_lexer_action_hook(&hooks, lexer, action);
},
&mut |lexer, predicate| {
dispatch_lexer_predicate_hook(&hooks, lexer, predicate).unwrap_or(true)
},
&mut |lexer| dispatch_lexer_before_token_hook(&hooks, lexer),
&mut |_, _, _| {},
&mut |lexer, accept_position| {
dispatch_lexer_after_accept_hook(&hooks, lexer, accept_position);
},
LexerMatchStrategy {
compiled: None,Found a 23 line (138 tokens) duplication in the following files:
let mut next = Vec::new();
for config in active {
let Some(state) = atn.state(config.state) else {
continue;
};
for transition in &state.transitions {
if !transition.matches(symbol, MIN_CHAR_VALUE, MAX_CHAR_VALUE) {
continue;
}
let mut advanced = config.clone();
set_config_state(atn, &mut advanced, transition.target());
if symbol == EOF {
advanced.consumed_eof = true;
} else {
advanced.position += 1;
}
next.push(advanced);
}
}
let closure = epsilon_closure(atn, next, &mut |predicate| {
semantic_predicate(lexer, predicate)
});Found a 24 line (136 tokens) duplication in the following files:
let source_has_semantic_context = dfa_state_has_semantic_context;
for config in active {
let Some(state) = atn.state(config.state) else {
continue;
};
for transition in &state.transitions {
if !transition.matches(symbol, MIN_CHAR_VALUE, MAX_CHAR_VALUE) {
continue;
}
let mut advanced = config.clone();
set_config_state(atn, &mut advanced, transition.target());
if symbol == EOF {
advanced.consumed_eof = true;
} else {
advanced.position += 1;
}
next.push(advanced);
}
}
let closure = epsilon_closure(atn, next, &mut |predicate| {
semantic_predicate(lexer, predicate)
});
let target_has_semantic_context = closure.has_semantic_context;Found a 24 line (134 tokens) duplication in the following files:
let atn = AtnDeserializer::new(&SerializedAtn::from_i32(&[
4, 0, 2, // version, lexer, max token type
9, // states
6, -1, // 0 token start
2, 0, // 1 rule 0 start
1, 0, // 2
1, 0, // 3
7, 0, // 4 rule 0 stop
2, 1, // 5 rule 1 start
1, 1, // 6
1, 1, // 7
7, 1, // 8 rule 1 stop
0, // non-greedy
0, // precedence
2, // rules
1, 1, // rule 0 starts at 1, token type 1
5, 2, // rule 1 starts at 5, token type 2
1, // modes
0, // default mode starts at 0
0, // sets
8, // edges
0, 1, 1, 0, 0, 0, // start -> rule 0
0, 5, 1, 0, 0, 0, // start -> rule 1
1, 2, 5, 'a' as i32, 0, 0, 2, 3, 5, 'b' as i32, 0, 0, 3, 4, 1, 0, 0, 0, 5, 6, 5,Found a 23 line (129 tokens) duplication in the following files:
let source_has_semantic_context = dfa_state_has_semantic_context;
for config in active {
let Some(state) = atn.state(config.state) else {
continue;
};
for transition in &state.transitions {
if !transition.matches(symbol, MIN_CHAR_VALUE, MAX_CHAR_VALUE) {
continue;
}
let mut advanced = config.clone();
set_config_state(atn, &mut advanced, transition.target());
if symbol == EOF {
advanced.consumed_eof = true;
} else {
advanced.position += 1;
}
next.push(advanced);
}
}
let closure = epsilon_closure(atn, next, &mut |predicate| {
semantic_predicate(lexer, predicate)
});Found a 27 line (127 tokens) duplication in the following files:
let mut atn = ParserAtnBuilder::new(2);
assert_eq!(
atn.add_state(AtnStateKind::RuleStart, Some(0))
.expect("state")
.index(),
0
);
assert_eq!(
atn.add_state(AtnStateKind::BlockStart, Some(0))
.expect("state")
.index(),
1
);
assert_eq!(
atn.add_state(AtnStateKind::Basic, Some(0))
.expect("state")
.index(),
2
);
assert_eq!(
atn.add_state(AtnStateKind::Basic, Some(0))
.expect("state")
.index(),
3
);
assert_eq!(
atn.add_state(AtnStateKind::BlockEnd, Some(0))Found a 33 line (126 tokens) duplication in the following files:
if self.fast_parser_predicate_matches(predicate_context, transition, index) {
let boundary = left_recursive_boundary(atn, state, target);
outcomes.extend(
self.recognize_state_fast(
atn,
FastRecognizeRequest {
state_number: target,
stop_state,
index,
rule_start_index,
decision_start_index: next_decision_start_index,
precedence,
depth: depth + 1,
recovery_symbols: Rc::clone(&epsilon_recovery_symbols),
recovery_state: epsilon_recovery_state,
},
FastRecognizeScratch {
predicate_context,
visiting,
memo,
expected,
},
)
.into_iter()
.map(|mut outcome| {
if let Some(rule_index) = boundary {
let boundary = self.arena_boundary_node(rule_index);
self.defer_fast_outcome_node(&mut outcome, boundary);
}
outcome
}),
);
} else {Found a 32 line (125 tokens) duplication in the following files:
let boundary = left_recursive_boundary(atn, state, target);
outcomes.extend(
self.recognize_state_fast(
atn,
FastRecognizeRequest {
state_number: target,
stop_state,
index,
rule_start_index,
decision_start_index: next_decision_start_index,
precedence,
depth: depth + 1,
recovery_symbols: Rc::clone(&epsilon_recovery_symbols),
recovery_state: epsilon_recovery_state,
},
FastRecognizeScratch {
predicate_context,
visiting,
memo,
expected,
},
)
.into_iter()
.map(|mut outcome| {
if let Some(rule_index) = boundary {
let boundary = self.arena_boundary_node(rule_index);
self.defer_fast_outcome_node(&mut outcome, boundary);
}
outcome
}),
);
}Found a 22 line (125 tokens) duplication in the following files:
atn.add_state(AtnStateKind::BlockEnd, Some(0))
.expect("state")
.index(),
4
);
assert_eq!(
atn.add_state(AtnStateKind::RuleStop, Some(0))
.expect("state")
.index(),
5
);
atn.set_rule_to_start_state(vec![0])
.expect("rule start states");
atn.set_rule_to_stop_state(vec![5])
.expect("rule stop states");
atn.add_decision_state(1).expect("decision state");
atn.add_transition(0, ParserTransitionSpec::Epsilon { target: 1 })
.expect("transition");
atn.add_transition(
1,
ParserTransitionSpec::Atom {
target: 2,Found a 22 line (117 tokens) duplication in the following files:
atn: &LexerAtn,
mut custom_action: A,
mut semantic_predicate: P,
mut accept_adjuster: E,
) -> Result<TokenId, TokenStoreError>
where
I: CharStream,
A: FnMut(&mut BaseLexer<I>, LexerCustomAction),
P: FnMut(&BaseLexer<I>, LexerPredicate) -> bool,
E: FnMut(&mut BaseLexer<I>, i32, usize),
{
next_token_with_hooks_impl(
lexer,
sink,
atn,
&mut custom_action,
&mut semantic_predicate,
&mut |_| {},
&mut accept_adjuster,
&mut |_, _| {},
LexerMatchStrategy {
compiled: None,Found a 33 line (115 tokens) duplication in the following files:
outcomes.extend(
self.recognize_state(
atn,
RecognizeRequest {
state_number: *target,
stop_state,
index,
rule_start_index,
decision_start_index: next_decision_start_index,
init_action_rules,
predicates,
semantics,
rule_args,
member_actions,
return_actions,
local_int_arg,
member_values: member_values.clone(),
return_values: return_values.clone(),
rule_alt_number: next_alt_number,
track_alt_numbers,
consumed_eof,
precedence,
depth: depth + 1,
recovery_symbols: epsilon_recovery_symbols.clone(),
recovery_state: epsilon_recovery_state,
},
visiting,
memo,
expected,
)
.into_iter()
.map(|mut outcome| {
prepend_decision(&mut outcome, decision);Found a 15 line (113 tokens) duplication in the following files:
fn generated_match_token_counts_single_token_deletion_recovery() {
let atn = generated_match_recovery_atn();
let data = RecognizerData::new(
"Mini.g4",
Vocabulary::new(
[None, Some("'X'"), Some("'Y'"), Some("'Z'")],
[None, Some("X"), Some("Y"), Some("Z")],
[None::<&str>, None, None, None],
),
);
let mut parser = BaseParser::new(
CommonTokenStream::new(Source {
tokens: vec![
TestToken::new(3).with_text("z"),
TestToken::new(2).with_text("y"),Found a 12 line (112 tokens) duplication in the following files:
let mut atn = ParserAtnBuilder::new(1);
for (state, kind, rule) in [
(0, AtnStateKind::RuleStart, 0),
(1, AtnStateKind::StarLoopEntry, 0),
(2, AtnStateKind::Basic, 0), // ops hub
(3, AtnStateKind::Basic, 0), // shift prec
(4, AtnStateKind::Basic, 0), // shift first >
(5, AtnStateKind::Basic, 0), // shift second >
(6, AtnStateKind::Basic, 0), // rel prec
(7, AtnStateKind::Basic, 0), // rel >
(8, AtnStateKind::LoopEnd, 0),
(9, AtnStateKind::RuleStop, 0),Found a 22 line (112 tokens) duplication in the following files:
fn predicate_after_token_atn() -> Atn {
let mut atn = ParserAtnBuilder::new(2);
assert_eq!(
atn.add_state(AtnStateKind::RuleStart, Some(0))
.expect("state")
.index(),
0
);
assert_eq!(
atn.add_state(AtnStateKind::Basic, Some(0))
.expect("state")
.index(),
1
);
assert_eq!(
atn.add_state(AtnStateKind::Basic, Some(0))
.expect("state")
.index(),
2
);
assert_eq!(
atn.add_state(AtnStateKind::Basic, Some(0))Found a 22 line (111 tokens) duplication in the following files:
fn left_recursive_loop_with_caller_follow_atn(caller_symbol: i32) -> Atn {
let mut atn = ParserAtnBuilder::new(2);
assert_eq!(
atn.add_state(AtnStateKind::RuleStart, Some(0))
.expect("state")
.index(),
0
);
assert_eq!(
atn.add_state(AtnStateKind::Basic, Some(0))
.expect("state")
.index(),
1
);
assert_eq!(
atn.add_state(AtnStateKind::Basic, Some(0))
.expect("state")
.index(),
2
);
assert_eq!(
atn.add_state(AtnStateKind::RuleStart, Some(1))Found a 14 line (110 tokens) duplication in the following files:
fn parser_matches_token_and_reports_mismatch() {
let source = Source {
tokens: vec![
TestToken::new(1).with_text("x"),
TestToken::eof("parser-test", 1, 1, 1),
],
index: 0,
};
let data = RecognizerData::new(
"Mini.g4",
Vocabulary::new([None, Some("'x'")], [None, Some("X")], [None::<&str>, None]),
);
let mut parser = BaseParser::new(CommonTokenStream::new(source), data);
let matched = parser.match_token(1).expect("token 1 should match");Found a 13 line (109 tokens) duplication in the following files:
fn parser_matches_token_and_reports_mismatch() {
let source = Source {
tokens: vec![
TestToken::new(1).with_text("x"),
TestToken::eof("parser-test", 1, 1, 1),
],
index: 0,
};
let data = RecognizerData::new(
"Mini.g4",
Vocabulary::new([None, Some("'x'")], [None, Some("X")], [None::<&str>, None]),
);
let mut parser = BaseParser::new(CommonTokenStream::new(source), data);Found a 22 line (108 tokens) duplication in the following files:
) -> Option<RecognizeOutcome> {
let (error_index, message) = self.expected_error_message(rule_index, start_index, expected);
let diagnostic = diagnostic_for_token(self.token_at(error_index), message);
let mut next_index = error_index;
loop {
let symbol = self.token_type_at(next_index);
if sync_symbols.contains(&symbol) {
if next_index == error_index {
return None;
}
break;
}
if symbol == TOKEN_EOF {
break;
}
let after = self.consume_index(next_index, symbol);
if after == next_index {
break;
}
next_index = after;
}
let mut nodes = NodeSeqId::EMPTY;Found a 15 line (108 tokens) duplication in the following files:
fn outcome_ties_keep_later_non_recursive_alternative() {
let arena = RecognitionArena::default();
let first = RecognizeOutcome {
index: 1,
consumed_eof: false,
alt_number: 0,
member_values: BTreeMap::new(),
return_values: BTreeMap::new(),
diagnostics: DiagnosticSeqId::EMPTY,
decisions: Vec::new(),
actions: vec![ParserAction::new(1, 0, 0, None)],
nodes: NodeSeqId::EMPTY,
};
let second = RecognizeOutcome {
actions: vec![ParserAction::new(2, 0, 0, None)],Found a 13 line (107 tokens) duplication in the following files:
let mut hooks = LifecycleRecordingHooks::default();
let mut store = TokenStore::new(lexer.source_text(), lexer.source_name());
let mut sink = TokenSink::new(&mut store);
let mut ids = Vec::new();
for _ in 0..3 {
let id = if compiled {
next_token_compiled_with_semantic_hooks(
&mut lexer, &mut sink, &atn, &dfa, &mut hooks,
)
} else {
next_token_with_semantic_hooks(&mut lexer, &mut sink, &atn, &mut hooks)
}
.expect("lifecycle token should fit");Found a 16 line (105 tokens) duplication in the following files:
let start_state = atn.rule_to_start_state().get(rule_index).ok_or_else(|| {
AntlrError::Unsupported(format!("rule {rule_index} has no start state"))
})?;
let stop_state = atn
.rule_to_stop_state()
.get(rule_index)
.filter(|state| *state != usize::MAX)
.ok_or_else(|| {
AntlrError::Unsupported(format!("rule {rule_index} has no stop state"))
})?;
let start_index = self.current_visible_index();
self.clear_prediction_diagnostics();
self.reset_per_parse_caches();
self.reset_recognition_arena();
let caller_follow_state = self.pending_invoking_follow_state(atn);Found a 13 line (100 tokens) duplication in the following files:
let mut expected = BTreeSet::new();
for index in (1..self.rule_context_stack.len()).rev() {
let invoking_state = self.rule_context_stack[index].invoking_state;
let Ok(state_number) = usize::try_from(invoking_state) else {
continue;
};
let Some(Transition::Rule { follow_state, .. }) = atn
.state(state_number)
.and_then(|state| state.transitions().first())
.map(ParserTransition::data)
else {
continue;
}; |
|
Warning Review limit reached
Next review available in: 46 minutes Enable usage-based reviews in Billing to review now. Otherwise, wait until the next included review is available. How can I continue?After more reviews become available, a review can be triggered using the To avoid repeated limits, reduce automatic review volume by pausing incremental auto-reviews earlier, using label-based review opt-in, excluding WIP or generated PR titles, or requesting reviews manually when the PR is ready. If your team needs uninterrupted high-volume reviews, an organization admin can enable usage-based reviews. How do review limits work?CodeRabbit enforces per-developer PR review limits for each organization. Most developers receive the normal plan review availability. For paid Pro and Pro+ PR reviews, CodeRabbit uses adaptive limits for sustained high-volume activity. When a developer's recent PR review activity reaches the 95th percentile or higher among CodeRabbit users, additional reviews become available more gradually as earlier reviews age out of the rolling window. Please refer docs for additional details. Review details⚙️ Run configurationConfiguration used: Organization UI Review profile: ASSERTIVE Plan: Pro Run ID: 📒 Files selected for processing (3)
WalkthroughThe lexer DFA now records escape edges with serialized continuation configurations and can resume ATN interpretation from narrowed states. Compiled matching distinguishes completed matches, continuation resumes, and budget-only restarts. Serialization validation and predicate-related tests cover the new data. Parser fast outcome deduplication now selects inline, dense, or sparse strategies, reuses scratch storage, and reports strategy-specific performance counters. Poem
🚥 Pre-merge checks | ✅ 4 | ❌ 1❌ Failed checks (1 warning)
✅ Passed checks (4 passed)
Thanks for using CodeRabbit! It's free for OSS, and your support helps us grow. If you like it, consider giving us a shout-out. Comment |
![gemini-code-assist[bot]](https://avatars.githubusercontent.com/in/956858?s=60&v=4){kind=small}
There was a problem hiding this comment.
Code Review
This pull request enhances the lexer DFA compilation by allowing the interpreter to resume from narrowed pre-closure configurations (continuations) upon reaching an escape edge, rather than restarting from the token boundary. This is achieved by introducing CompiledMatch, CompiledResume, and CompiledLexerContinuation structures, alongside updating the matching logic and serialization format. Feedback on these changes focuses on improving safety and robustness, specifically by avoiding direct indexing on collections to prevent runtime panics, replacing direct as casts with checked conversions (try_from) to avoid silent truncation, and using checked_add to prevent potential integer overflow panics during range merging.
Important
The consumer version of Gemini Code Assist on GitHub is being sunset. Starting June 18, 2026, new organization installations will be blocked, and all code review activity will officially cease on July 17, 2026.
For more details on the timeline and next steps, please review the Help Documentation.
tinovyatkin
changed the title
tinovyatkin
changed the title
|
@codex review |
|
@codex review |
![coderabbitai[bot]](https://avatars.githubusercontent.com/in/347564?s=60&v=4){kind=small}
There was a problem hiding this comment.
Actionable comments posted: 2
Caution
Some comments are outside the diff and can’t be posted inline due to platform limitations.
⚠️ Outside diff range comments (1)
src/atn/lexer_dfa.rs (1)
1413-1488: 📐 Maintainability & Code Quality | 🔵 Trivial | ⚡ Quick winExercise EOF and wide-character continuation routes.
This test only covers an ASCII character escape. Add compiled-vs-interpreted cases for an EOF escape and a non-ASCII escape range; both use distinct routing and position semantics.
🤖 Prompt for AI Agents
Verify each finding against current code. Fix only still-valid issues, skip the rest with a brief reason, keep changes minimal, and validate. In `@src/atn/lexer_dfa.rs` around lines 1413 - 1488, Extend predicate_edge_resumes_the_interpreter_for_true_and_false_outcomes with compiled-vs-interpreted cases covering an EOF escape and a non-ASCII escape range, using appropriate inputs and predicate outcomes. Assert token results and drained error messages match for both implementations, while preserving the existing ASCII coverage and exercising each route’s distinct position semantics.
🤖 Prompt for all review comments with AI agents
Verify each finding against current code. Fix only still-valid issues, skip the
rest with a brief reason, keep changes minimal, and validate.
Inline comments:
In `@src/atn/lexer.rs`:
- Around line 669-675: Validate the continuation from CompiledMatch::Resume
against the ATN before calling match_token_from_continuation, including config,
stack, rule/action indices, non-empty states, and valid behind offsets; if any
check fails, treat it like CompiledMatch::Restart and fall through to
cached/interpreted matching instead of resuming unchecked payloads.
In `@src/parser.rs`:
- Around line 11699-11713: Update the Sparse branch in fast_outcome_dedup to
release scratch.sparse_keys storage when its capacity exceeds a defined
retention threshold after deduplication, while retaining the existing allocation
for normal sizes. Ensure the oversized set is cleared and shrunk so reused
parsers do not permanently retain capacity.
---
Outside diff comments:
In `@src/atn/lexer_dfa.rs`:
- Around line 1413-1488: Extend
predicate_edge_resumes_the_interpreter_for_true_and_false_outcomes with
compiled-vs-interpreted cases covering an EOF escape and a non-ASCII escape
range, using appropriate inputs and predicate outcomes. Assert token results and
drained error messages match for both implementations, while preserving the
existing ASCII coverage and exercising each route’s distinct position semantics.
🪄 Autofix (Beta)
Fix all unresolved CodeRabbit comments on this PR:
- Push a commit to this branch (recommended)
- Create a new PR with the fixes
ℹ️ Review info
⚙️ Run configuration
Configuration used: Organization UI
Review profile: ASSERTIVE
Plan: Pro
Run ID: 6c9f93a0-de6f-4ecf-89a8-4a5bc888d129
📒 Files selected for processing (3)
src/atn/lexer.rssrc/atn/lexer_dfa.rssrc/parser.rs
|
Codex Review: Didn't find any major issues. Hooray! Reviewed commit: ℹ️ About Codex in GitHubYour team has set up Codex to review pull requests in this repo. Reviews are triggered when you
If Codex has suggestions, it will comment; otherwise it will react with 👍. Codex can also answer questions or update the PR. Try commenting "@codex address that feedback". |
|
@codex review |
|
Codex Review: Didn't find any major issues. 🎉 Reviewed commit: ℹ️ About Codex in GitHubYour team has set up Codex to review pull requests in this repo. Reviews are triggered when you
If Codex has suggestions, it will comment; otherwise it will react with 👍. Codex can also answer questions or update the PR. Try commenting "@codex address that feedback". |
1 participant
Summary
Closes #108.
Root causes
Compiled lexer fallback
The compiled lexer escaped at predicate-sensitive edges, then rematched the token from the mode start. For the MySQL lexer, that rebuilt a closure spanning roughly 900 rules for each dynamic edge.
The lexer now captures configs immediately after the escaped consuming transition and before its dynamic epsilon closure. The interpreter resumes from those narrowed configs while preserving semantic predicates, actions, recursive rule stacks, longest-match accepts, and recognition-error spans.
Clean parser outcome deduplication
Large
VALUESlists produced compact, ordered endpoint ranges. The clean fast recognizer stored the first eight endpoints inline, then linearly scanned an overflowVecfor every subsequent endpoint. Statement 44 spent 27.7% of total sampled time in that quadratic scan even though almost every endpoint was unique.The replacement chooses from the observed data shape:
FxHashSet<(usize, bool)>Vec::retainpreserves first-discovered greedy ordering, and the key keeps ordinary and EOF-consuming outcomes distinct. Bitmap words are cleared through a touched-word list, so scratch reuse does not scan the full retained capacity.Both runtime and codegen changes remain grammar-agnostic.
Performance
Exact same-host parser A/B,
d57174505versusbc9ee21ad:Synthetic bulk-
VALUESparse scaling:The 12k payment fixture recorded 2.33 million clean endpoint inputs: 2.35 million calls stayed inline, 15 large lists used the dense bitmap, and none required sparse hashing. A three-run statement-44 profile reduced clean deduplication from 27.7% to 1.36% of total self samples.
The previously recorded C++ workload total on this host is 2.96 s; current Rust is 3.19-3.29 s.
Validation
cargo test --locked --all-targets --all-featurescargo clippy --locked --all-targets --all-features -- -D warningscargo run --release --quiet --bin antlr4-runtime-testsuite(357 passed, 0 failed, 0 skipped)git diff --check