diff --git a/docs/issue-80-lexer-range-benchmark.md b/docs/issue-80-lexer-range-benchmark.md new file mode 100644 index 00000000..fabf7e2b --- /dev/null +++ b/docs/issue-80-lexer-range-benchmark.md @@ -0,0 +1,77 @@ +# Issue 80 ASCII range-scan benchmark record + +Measurements were taken on 2026-07-19 on an Apple M3 Pro with Rust 1.96.0. +The baseline was `origin/main` at `5c7affddb`; generated lexers were rebuilt +with the matching generator/runtime for each revision. The grammars-v4 +checkout was `284602b3f23ca54dc30778204ab7ae9e969145e9`. + +## Retained scope + +The compiled lexer DFA now represents an exact ASCII self-loop set as up to +four canonical, coalesced inclusive ranges when the existing `Until1` through +`Until3` forms do not apply. Unsupported shapes keep the ordinary DFA walk. +The retained scanner is a portable scalar implementation. Range descriptors +also participate in compiled-DFA serialization and malformed-stream +validation. + +The final implementation contains no architecture-specific code or `unsafe`. +AVX2 and NEON candidates were developed and tested, then removed under issue +#80's decision gate because the measured backends did not justify their +dispatch, threshold, and maintenance costs. + +## Scalar results + +The Kotlin, C#, Java, and Trino lex-only suite used 20 warmups and 200 measured +lexes for each fixture. Ratios below one are faster. + +| Fixtures | Count | Scalar ranges vs main | +|---|---:|---:| +| Kotlin | 6 | 0.9843x | +| C# | 4 | 0.9702x | +| Java | 4 | 0.9610x | +| Trino SQL | 5 | 0.9969x | +| **All shared fixtures** | **19** | **0.9796x** | + +The aggregate-time ratio was `0.9854x`. The largest per-fixture ratio was +`1.0275x` on the short Unicode fallback fixture, below the repository's +`1.15x` regression guard. + +Two Java fixtures isolate the range-heavy paths. Already-built baseline and +current runners were measured with 100 warmups and 2,000 timed lexes: + +| Fixture | Main | Scalar ranges | Ratio | +|---|---:|---:|---:| +| Long identifiers and mixed ASCII ranges | 23.865 us | 21.411 us | 0.8972x | +| Long numbers and whitespace | 20.077 us | 17.196 us | 0.8565x | + +## Architecture-backend decision + +Before removal, the same generated runner was built with a forced scalar +scanner, normal AArch64 runtime dispatch with a 32-byte NEON threshold, and a +64-byte NEON threshold. Each process used 200 warmups and 5,000 timed lexes. +The table reports candidate/scalar ratios, so values above one are slower. + +| Fixture | NEON at 32 bytes | NEON at 64 bytes | +|---|---:|---:| +| Long identifiers and mixed ASCII ranges | 1.0010x | 1.0196x | +| Long numbers and whitespace | 1.0200x | 1.0061x | +| Bazel Java | 1.0088x | 1.0129x | +| Trino Java | 1.0215x | 1.0312x | + +Neither threshold improved a representative real fixture. The AVX2 candidate +cross-built and passed the x86_64 test suite under Rosetta, but that +environment does not provide representative native AVX2 benchmark evidence. +Shipping either backend would therefore fail the issue's decision gate. + +## Diagnostics + +An untimed `perf-counters` run over the long-identifier fixture generated 37 +range descriptors: 32 one-range, one two-range, three three-range, and one +four-range descriptor. One lex recorded 43 scalar range scans and 705 skipped +bytes, attributed as 360 identifier, 288 number, and 57 whitespace bytes. +The small inventory did not justify a pooled descriptor indirection. + +Correctness validation included randomized descriptor/scanner differential +tests, accelerated-versus-ordinary DFA token-stream comparisons, malformed +serialization tests, all nine Kotlin parity snippets, and the rendered ANTLR +runtime testsuite (`357 passed, 0 failed, 0 skipped`). diff --git a/src/atn/ascii_range.rs b/src/atn/ascii_range.rs new file mode 100644 index 00000000..5e1d8528 --- /dev/null +++ b/src/atn/ascii_range.rs @@ -0,0 +1,292 @@ +//! Exact ASCII range descriptors and range-prefix scanners. + +const ASCII_SYMBOLS: usize = 128; +pub(super) const MAX_RANGES: usize = 4; + +#[derive(Clone, Copy, Debug, Default, Eq, Hash, PartialEq)] +pub(super) struct AsciiRange { + pub(super) low: u8, + pub(super) high: u8, +} + +#[derive(Clone, Copy, Debug, Eq, Hash, PartialEq)] +pub(super) struct AsciiRanges { + ranges: [AsciiRange; MAX_RANGES], + count: u8, +} + +impl AsciiRanges { + pub(super) fn from_self_loops(row: &[u16; ASCII_SYMBOLS], state: u16) -> Option { + let mut ranges = [AsciiRange::default(); MAX_RANGES]; + let mut count = 0; + let mut start = None; + + for (symbol, &target) in row.iter().enumerate() { + if target == state { + start.get_or_insert(symbol); + continue; + } + if let Some(low) = start.take() { + if count == MAX_RANGES { + return None; + } + ranges[count] = AsciiRange { + low: u8::try_from(low).expect("ASCII range start overflow"), + high: u8::try_from(symbol - 1).expect("ASCII range end overflow"), + }; + count += 1; + } + } + if let Some(low) = start { + if count == MAX_RANGES { + return None; + } + ranges[count] = AsciiRange { + low: u8::try_from(low).expect("ASCII range start overflow"), + high: 127, + }; + count += 1; + } + + Self::new(u8::try_from(count).expect("range count overflow"), ranges) + } + + pub(super) fn new(count: u8, ranges: [AsciiRange; MAX_RANGES]) -> Option { + let count = usize::from(count); + if !(1..=MAX_RANGES).contains(&count) { + return None; + } + if ranges[..count] + .iter() + .any(|range| range.low > range.high || !range.high.is_ascii()) + { + return None; + } + if ranges[..count] + .windows(2) + .any(|pair| pair[0].high + 1 >= pair[1].low) + { + return None; + } + if ranges[count..] + .iter() + .any(|range| *range != AsciiRange::default()) + { + return None; + } + Some(Self { + ranges, + count: u8::try_from(count).expect("validated range count fits u8"), + }) + } + + pub(super) const fn count(self) -> u8 { + self.count + } + + pub(super) fn as_slice(&self) -> &[AsciiRange] { + &self.ranges[..usize::from(self.count)] + } + + pub(super) fn packed_words(self) -> [u32; 2] { + let mut words = [0_u32; 2]; + for (index, range) in self.as_slice().iter().enumerate() { + let shift = (index % 2) * 16; + words[index / 2] |= u32::from(range.low) << shift; + words[index / 2] |= u32::from(range.high) << (shift + 8); + } + words + } + + pub(super) fn from_packed(count: u8, words: [u32; 2]) -> Option { + let mut ranges = [AsciiRange::default(); MAX_RANGES]; + for (index, range) in ranges.iter_mut().enumerate() { + let shift = (index % 2) * 16; + range.low = (words[index / 2] >> shift) as u8; + range.high = (words[index / 2] >> (shift + 8)) as u8; + } + Self::new(count, ranges) + } + + #[cfg(test)] + fn contains(self, byte: u8) -> bool { + self.as_slice() + .iter() + .any(|range| byte >= range.low && byte <= range.high) + } + + #[cfg(any(feature = "perf-counters", test))] + pub(super) fn class(self) -> AsciiRangeClass { + let mut any_digit = false; + let mut any_alpha = false; + let mut all_whitespace = true; + let mut all_number = true; + let mut all_identifier = true; + for range in self.as_slice() { + for byte in range.low..=range.high { + any_digit |= byte.is_ascii_digit(); + any_alpha |= byte.is_ascii_alphabetic(); + all_whitespace &= byte.is_ascii_whitespace(); + all_number &= byte.is_ascii_hexdigit() || matches!(byte, b'_' | b'\''); + all_identifier &= byte.is_ascii_alphanumeric() || matches!(byte, b'_' | b'$'); + } + } + if all_whitespace { + return AsciiRangeClass::Whitespace; + } + if any_digit && all_number { + return AsciiRangeClass::Number; + } + if any_alpha && all_identifier { + return AsciiRangeClass::Identifier; + } + AsciiRangeClass::Other + } +} + +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +#[cfg(any(feature = "perf-counters", test))] +pub(crate) enum AsciiRangeClass { + Identifier, + Number, + Whitespace, + Other, +} + +#[inline] +pub(super) fn scan_scalar(ranges: AsciiRanges, input: &[u8]) -> usize { + let active_ranges = ranges.as_slice(); + input + .iter() + .position(|&byte| { + !active_ranges + .iter() + .any(|range| byte >= range.low && byte <= range.high) + }) + .unwrap_or(input.len()) +} + +#[cfg(test)] +mod tests { + use super::*; + + fn ranges(items: &[(u8, u8)]) -> AsciiRanges { + let mut ranges = [AsciiRange::default(); MAX_RANGES]; + for (slot, &(low, high)) in ranges.iter_mut().zip(items) { + *slot = AsciiRange { low, high }; + } + AsciiRanges::new(items.len() as u8, ranges).expect("test ranges should be canonical") + } + + #[test] + fn descriptors_are_canonical_and_round_trip_packed_words() { + let descriptor = ranges(&[(b'0', b'9'), (b'A', b'Z'), (b'_', b'_'), (b'a', b'z')]); + assert_eq!( + AsciiRanges::from_packed(descriptor.count(), descriptor.packed_words()), + Some(descriptor) + ); + + let mut adjacent = [AsciiRange::default(); MAX_RANGES]; + adjacent[0] = AsciiRange { + low: b'a', + high: b'm', + }; + adjacent[1] = AsciiRange { + low: b'n', + high: b'z', + }; + assert_eq!(AsciiRanges::new(2, adjacent), None); + + let mut non_ascii = [AsciiRange::default(); MAX_RANGES]; + non_ascii[0] = AsciiRange { + low: 127, + high: 128, + }; + assert_eq!(AsciiRanges::new(1, non_ascii), None); + } + + #[test] + fn randomized_range_scans_match_dfa_rows() { + let mut random = 0x80A5_5EED_u32; + for count in 1..=MAX_RANGES { + for _ in 0..64 { + let mut row = [0_u16; ASCII_SYMBOLS]; + let state = 7; + let mut cursor = (next_random(&mut random) & 7) as usize; + for _ in 0..count { + let width = (next_random(&mut random) % 12) as usize; + let high = cursor + width; + row[cursor..=high].fill(state); + cursor = high + 2 + (next_random(&mut random) % 8) as usize; + } + let descriptor = AsciiRanges::from_self_loops(&row, state) + .expect("generated row should fit the descriptor bound"); + assert_eq!(usize::from(descriptor.count()), count); + for byte in 0_u8..=127 { + assert_eq!(descriptor.contains(byte), row[usize::from(byte)] == state); + } + + let accepted: Vec = (0_u8..=127) + .filter(|&byte| descriptor.contains(byte)) + .collect(); + let rejected: Vec = (0_u8..=u8::MAX) + .filter(|&byte| !descriptor.contains(byte)) + .collect(); + for len in [0, 1, 7, 15, 31, 32, 47, 63, 64, 95, 127, 255] { + let mut input = Vec::with_capacity(len + 1); + for _ in 0..len { + let index = next_random(&mut random) as usize % accepted.len(); + input.push(accepted[index]); + } + if next_random(&mut random) & 1 != 0 { + let index = next_random(&mut random) as usize % rejected.len(); + input.push(rejected[index]); + } + assert_scan_matches_row(descriptor, &row, state, &input); + } + + for len in [1, 17, 65, 260] { + let mut input = Vec::with_capacity(len); + for _ in 0..len { + input.push(next_random(&mut random).to_le_bytes()[0]); + } + assert_scan_matches_row(descriptor, &row, state, &input); + } + } + } + } + + fn next_random(random: &mut u32) -> u32 { + *random = random.wrapping_mul(1_664_525).wrapping_add(1_013_904_223); + *random + } + + fn assert_scan_matches_row( + descriptor: AsciiRanges, + row: &[u16; ASCII_SYMBOLS], + state: u16, + input: &[u8], + ) { + let expected = input + .iter() + .position(|&byte| !byte.is_ascii() || row[usize::from(byte)] != state) + .unwrap_or(input.len()); + assert_eq!(scan_scalar(descriptor, input), expected); + } + + #[test] + fn classifies_common_range_shapes_without_grammar_names() { + assert_eq!( + ranges(&[(b'0', b'9'), (b'A', b'Z'), (b'_', b'_'), (b'a', b'z')]).class(), + AsciiRangeClass::Identifier + ); + assert_eq!( + ranges(&[(b'0', b'9'), (b'A', b'F'), (b'a', b'f')]).class(), + AsciiRangeClass::Number + ); + assert_eq!( + ranges(&[(b'\t', b'\n'), (b'\x0c', b'\r'), (b' ', b' ')]).class(), + AsciiRangeClass::Whitespace + ); + } +} diff --git a/src/atn/lexer.rs b/src/atn/lexer.rs index 73355f0d..bccb6803 100644 --- a/src/atn/lexer.rs +++ b/src/atn/lexer.rs @@ -1162,11 +1162,14 @@ fn match_token_compiled_ascii<'a>( record_compiled_accept(accept, position, &mut best); } if position < input.len() && self_loop_prefix == MIN_RUN_SCAN_PREFIX { - if let Some(scan) = dfa.ascii_run(state).scan(&input[position..]) { + if let Some(scan) = dfa.scan_ascii_run(state, &input[position..]) { #[cfg(feature = "perf-counters")] { direct_chars += scan.bytes; crate::perf::record_lexer_run_scan(scan.bytes, scan.found_exit); + if let Some(range) = scan.range { + crate::perf::record_lexer_range_scan(range.class(), scan.bytes); + } } position += scan.bytes; error_stop = error_stop.max(position); diff --git a/src/atn/lexer_dfa.rs b/src/atn/lexer_dfa.rs index 43b8096d..7b9395c1 100644 --- a/src/atn/lexer_dfa.rs +++ b/src/atn/lexer_dfa.rs @@ -22,6 +22,9 @@ use std::hash::BuildHasherDefault; use memchr::{memchr, memchr2, memchr3}; +#[cfg(feature = "perf-counters")] +use crate::atn::ascii_range::AsciiRangeClass; +use crate::atn::ascii_range::{self, AsciiRanges}; use crate::atn::lexer::{ LexerConfig, best_accept, epsilon_closure, lexer_action_belongs_to_accept, prune_after_accepts, set_config_state, @@ -69,6 +72,7 @@ pub(super) enum AsciiRun { Until1(u8), Until2(u8, u8), Until3(u8, u8, u8), + Ranges(AsciiRanges), } impl AsciiRun { @@ -79,10 +83,9 @@ impl AsciiRun { if target == state { continue; } - if count == exits.len() { - return Self::None; + if count < exits.len() { + exits[count] = u8::try_from(symbol).expect("symbol overflow"); } -exits[count] = u8::try_from(symbol).expect("symbol overflow"); count += 1; } match count { @@ -90,37 +93,63 @@ exits[count] = u8::try_from(symbol).expect("symbol overflow"); 1 => Self::Until1(exits[0]), 2 => Self::Until2(exits[0], exits[1]), 3 => Self::Until3(exits[0], exits[1], exits[2]), - _ => unreachable!("exit array bounds the descriptor"), + _ => AsciiRanges::from_self_loops(row, state).map_or(Self::None, Self::Ranges), } } - pub(super) fn scan(self, input: &[u8]) -> Option { + fn scan(self, input: &[u8]) -> Option { let exit = match self { Self::None => return None, Self::Any => None, Self::Until1(a) => memchr(a, input), Self::Until2(a, b) => memchr2(a, b, input), Self::Until3(a, b, c) => memchr3(a, b, c, input), + Self::Ranges(ranges) => { + let bytes = ascii_range::scan_scalar(ranges, input); + return Some(AsciiRunScan { + bytes, + found_exit: bytes != input.len(), + #[cfg(any(feature = "perf-counters", test))] + range: Some(AsciiRangeScan { ranges }), + }); + } }; Some(AsciiRunScan { bytes: exit.unwrap_or(input.len()), found_exit: exit.is_some(), + #[cfg(any(feature = "perf-counters", test))] + range: None, }) } - const fn pack(self) -> u32 { + const fn serialized_words(self) -> usize { + if matches!(self, Self::Ranges(_)) { + 3 + } else { + 1 + } + } + + fn write_serialized(self, out: &mut Vec) { match self { - Self::None => 0, - Self::Any => 1, - Self::Until1(a) => 2 | ((a as u32) << 8), - Self::Until2(a, b) => 3 | ((a as u32) << 8) | ((b as u32) << 16), + Self::None => out.push(0), + Self::Any => out.push(1), + Self::Until1(a) => out.push(2 | (u32::from(a) << 8)), + Self::Until2(a, b) => { + out.push(3 | (u32::from(a) << 8) | (u32::from(b) << 16)); + } Self::Until3(a, b, c) => { - 4 | ((a as u32) << 8) | ((b as u32) << 16) | ((c as u32) << 24) + out.push(4 | (u32::from(a) << 8) | (u32::from(b) << 16) | (u32::from(c) << 24)); + } + Self::Ranges(ranges) => { + out.push(5 | (u32::from(ranges.count()) << 8)); + out.extend(ranges.packed_words()); } } } - const fn unpack(word: u32) -> Option { + fn read_serialized(reader: &mut SerializedReader<'_>) -> Option { + let word = reader.next()?; match word.to_le_bytes() { [0, 0, 0, 0] => Some(Self::None), [1, 0, 0, 0] => Some(Self::Any), @@ -129,15 +158,56 @@ exits[count] = u8::try_from(symbol).expect("symbol overflow"); [4, a, b, c] if a.is_ascii() && b.is_ascii() && c.is_ascii() => { Some(Self::Until3(a, b, c)) } + [5, count, 0, 0] => Some(Self::Ranges(AsciiRanges::from_packed( + count, + [reader.next()?, reader.next()?], + )?)), _ => None, } } + + #[cfg(feature = "perf-counters")] + fn descriptor_kind(self) -> AsciiRunDescriptorKind { + match self { + Self::None => AsciiRunDescriptorKind::None, + Self::Any => AsciiRunDescriptorKind::Any, + Self::Until1(_) | Self::Until2(..) | Self::Until3(..) => AsciiRunDescriptorKind::Until, + Self::Ranges(ranges) => AsciiRunDescriptorKind::Ranges { + count: ranges.count(), + class: ranges.class(), + }, + } + } } #[derive(Clone, Copy, Debug, Eq, PartialEq)] pub(super) struct AsciiRunScan { pub(super) bytes: usize, pub(super) found_exit: bool, + #[cfg(any(feature = "perf-counters", test))] + pub(super) range: Option, +} + +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +#[cfg(any(feature = "perf-counters", test))] +pub(super) struct AsciiRangeScan { + ranges: AsciiRanges, +} + +#[cfg(feature = "perf-counters")] +impl AsciiRangeScan { + pub(super) fn class(self) -> AsciiRangeClass { + self.ranges.class() + } +} + +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +#[cfg(feature = "perf-counters")] +pub(crate) enum AsciiRunDescriptorKind { + None, + Any, + Until, + Ranges { count: u8, class: AsciiRangeClass }, } /// A lexer DFA compiled ahead of time from a lexer ATN. @@ -248,6 +318,8 @@ impl CompiledLexerDfa { let start = build_mode(atn, mode, &mut dfa, &mut pools); dfa.mode_starts.push(start); } + #[cfg(feature = "perf-counters")] + dfa.record_ascii_run_descriptors(); dfa } @@ -301,6 +373,10 @@ impl CompiledLexerDfa { self.ascii_runs[usize::from(state)] } + pub(super) fn scan_ascii_run(&self, state: u16, input: &[u8]) -> Option { + self.ascii_run(state).scan(input) + } + /// `LexerTransition` target for a non-EOF symbol, or [`DEAD_STATE`]. pub(super) fn char_target(&self, state: u16, symbol: i32) -> u16 { let compiled = &self.states[usize::from(state)]; @@ -359,9 +435,14 @@ impl CompiledLexerDfa { /// to [`Self::compile`]. pub fn serialize(&self) -> Vec { // Exact word count: the tag, eight section-length words, and each - // section's payload (states are 4 words, runs are 1 word, ASCII rows - // pack 2 targets per word, and wide ranges/action traces are 3 words - // each behind their per-row/per-accept length words). + // section's payload (states are 4 words, compact runs are 1 word, + // range runs are 3 words, ASCII rows pack 2 targets per word, and wide + // ranges/action traces are 3 words each behind their row length). + let ascii_run_words: usize = self + .ascii_runs + .iter() + .map(|run| run.serialized_words()) + .sum(); let wide_words: usize = self.wide_rows.iter().map(|row| 1 + row.len() * 3).sum(); let accept_words: usize = self .accepts @@ -387,7 +468,7 @@ impl CompiledLexerDfa { let capacity = 9 + self.mode_starts.len() + self.states.len() * 4 - + self.ascii_runs.len() + + ascii_run_words + self.ascii_rows.len() * (ASCII_EDGE_SYMBOLS / 2) + wide_words + accept_words @@ -406,9 +487,9 @@ impl CompiledLexerDfa { out.push(u32::from(state.eof_target)); out.push(state.accept); } -out.push(u32::try_from(self.ascii_runs.len()).expect("ascii_runs length overflow")); + out.push(u32::try_from(self.ascii_runs.len()).expect("ascii_runs length overflow")); for &run in &self.ascii_runs { - out.push(run.pack()); + run.write_serialized(&mut out); } out.push(self.ascii_rows.len() as u32); for row in &self.ascii_rows { @@ -512,7 +593,12 @@ out.push(u32::try_from(self.ascii_runs.len()).expect("ascii_runs length overflow escape_rows, continuations, }; - dfa.table_indexes_are_valid().then_some(dfa) + if !dfa.table_indexes_are_valid() { + return None; + } + #[cfg(feature = "perf-counters")] + dfa.record_ascii_run_descriptors(); + Some(dfa) } /// Cheap structural validation so a corrupted embedded stream degrades to @@ -563,6 +649,13 @@ out.push(u32::try_from(self.ascii_runs.len()).expect("ascii_runs length overflow .all(|range| continuation_ok(range.continuation)) }) } + + #[cfg(feature = "perf-counters")] + fn record_ascii_run_descriptors(&self) { + for &run in &self.ascii_runs { + crate::perf::record_lexer_run_descriptor(run.descriptor_kind()); + } + } } /// Wide rows must hold well-formed, sorted, disjoint ranges for @@ -580,7 +673,7 @@ fn escape_row_is_searchable(row: &[CompiledLexerEscapeRange]) -> bool { /// Version tag guarding embedded tables against format or construction-semantic /// drift. -const SERIALIZED_TAG: u32 = 0x4C58_4404; +const SERIALIZED_TAG: u32 = 0x4C58_4405; /// Cursor over a serialized DFA stream. struct SerializedReader<'a> { @@ -621,7 +714,7 @@ impl SerializedReader<'_> { let count = self.next_len()?; let mut runs = Vec::with_capacity(count.min(self.data.len())); for _ in 0..count { - runs.push(AsciiRun::unpack(self.next()?)?); + runs.push(AsciiRun::read_serialized(self)?); } Some(runs) } @@ -1273,7 +1366,7 @@ fn commit_mode( (dfa.accepts.len() - 1) as u32 }); let (ascii_row, wide_row) = split_rows(&state_rows.segments); -let state_id = u16::try_from(dfa.states.len()).expect("state ID overflow"); + let state_id = u16::try_from(dfa.states.len()).expect("state ID overflow"); let ascii_run = AsciiRun::classify(&ascii_row, state_id); dfa.states.push(CompiledLexerState { ascii_row: pools.intern_ascii(&mut dfa.ascii_rows, ascii_row), @@ -1345,7 +1438,7 @@ mod tests { use crate::atn::serialized::{AtnDeserializer, SerializedAtn}; use crate::char_stream::{CharStream, InputStream, TextInterval}; use crate::int_stream::IntStream; - use crate::lexer::BaseLexer; + use crate::lexer::{BaseLexer, Lexer}; use crate::recognizer::RecognizerData; use crate::token::{TOKEN_EOF, Token, TokenSink, TokenStore}; use crate::vocabulary::Vocabulary; @@ -1363,6 +1456,14 @@ mod tests { column: usize, } + #[derive(Debug, Eq, PartialEq)] + struct StreamSnapshot { + tokens: Vec, + errors: Vec, + final_mode: i32, + popped_modes: Vec, + } + fn compiled_token( lexer: &mut BaseLexer, atn: &LexerAtn, @@ -1570,11 +1671,75 @@ mod tests { .expect("artificial complement-loop lexer ATN should deserialize") } - fn compiled_stream( - input: &str, - atn: &LexerAtn, - dfa: &CompiledLexerDfa, - ) -> (Vec, Vec) { + /// Three `+` rules for decimal digits, ASCII identifier continuations, and + /// whitespace. Their loop states exercise one-, three-, and four-range + /// descriptors; trailing built-in actions cover channel changes, mode + /// stack changes, and skipped tokens. + #[rustfmt::skip] + fn range_loop_atn() -> LexerAtn { + AtnDeserializer::new(&SerializedAtn::from_i32(&[ + 4, 0, 3, // version, lexer, max token type + 13, // states + 6, -1, // 0 token start + 2, 0, // 1 number start + 1, 0, // 2 + 1, 0, // 3 + 7, 0, // 4 number stop + 2, 1, // 5 identifier start + 1, 1, // 6 + 1, 1, // 7 + 7, 1, // 8 identifier stop + 2, 2, // 9 whitespace start + 1, 2, // 10 + 1, 2, // 11 + 7, 2, // 12 whitespace stop + 0, // non-greedy + 0, // precedence + 3, // rules + 1, 1, // number starts at 1, token type 1 + 5, 2, // identifier starts at 5, token type 2 + 9, 3, // whitespace starts at 9, token type 3 + 1, // modes + 0, // default mode starts at 0 + 3, // sets + 1, 0, // set 0: decimal digits + '0' as i32, '9' as i32, + 4, 0, // set 1: ASCII identifier continuation + '0' as i32, '9' as i32, + 'A' as i32, 'Z' as i32, + '_' as i32, '_' as i32, + 'a' as i32, 'z' as i32, + 3, 0, // set 2: tab/newline, carriage return, space + '\t' as i32, '\n' as i32, + '\r' as i32, '\r' as i32, + ' ' as i32, ' ' as i32, + 15, // edges + 0, 1, 1, 0, 0, 0, // start -> number + 0, 5, 1, 0, 0, 0, // start -> identifier + 0, 9, 1, 0, 0, 0, // start -> whitespace + 1, 2, 1, 0, 0, 0, // + 2, 3, 7, 0, 0, 0, // number set + 3, 2, 1, 0, 0, 0, // loop + 3, 4, 6, 0, 0, 0, // channel action, then stop + 5, 6, 1, 0, 0, 0, // + 6, 7, 7, 1, 0, 0, // identifier set + 7, 6, 1, 0, 0, 0, // loop + 7, 8, 6, 1, 1, 0, // push-mode action, then stop + 9, 10, 1, 0, 0, 0, // + 10, 11, 7, 2, 0, 0, // whitespace set + 11, 10, 1, 0, 0, 0, // loop + 11, 12, 6, 2, 2, 0, // skip action, then stop + 0, // decisions + 3, // lexer actions + 0, 7, 0, // channel 7 + 5, 0, 0, // push mode 0 + 6, 0, 0, // skip + ])) + .deserialize() + .expect("artificial range-loop lexer ATN should deserialize") + } + + fn compiled_stream(input: &str, atn: &LexerAtn, dfa: &CompiledLexerDfa) -> StreamSnapshot { let mut lexer = BaseLexer::new(InputStream::new(input), recognizer_data()); let mut tokens = Vec::new(); loop { @@ -1590,7 +1755,35 @@ mod tests { .into_iter() .map(|error| error.message) .collect(); - (tokens, errors) + let final_mode = lexer.mode(); + let mut popped_modes = Vec::new(); + while let Some(mode) = lexer.pop_mode() { + popped_modes.push(mode); + } + StreamSnapshot { + tokens, + errors, + final_mode, + popped_modes, + } + } + + fn serialized_run_offset(dfa: &CompiledLexerDfa, state: usize) -> usize { + 4 + dfa.mode_starts.len() + + dfa.states.len() * 4 + + dfa.ascii_runs[..state] + .iter() + .map(|run| run.serialized_words()) + .sum::() + } + + fn first_serialized_range_offset(dfa: &CompiledLexerDfa) -> usize { + let state = dfa + .ascii_runs + .iter() + .position(|run| matches!(run, AsciiRun::Ranges(_))) + .expect("test DFA should contain a range descriptor"); + serialized_run_offset(dfa, state) } #[test] @@ -1603,6 +1796,7 @@ mod tests { Some(AsciiRunScan { bytes: 4, found_exit: false, + range: None, }) ); @@ -1616,12 +1810,42 @@ mod tests { Some(AsciiRunScan { bytes: 4, found_exit: true, + range: None, }) ); row[usize::from(b'\r')] = DEAD_STATE; assert_eq!(AsciiRun::classify(&row, state), AsciiRun::None); assert_eq!(AsciiRun::None.scan(b"body"), None); + + let mut identifier = [DEAD_STATE; ASCII_EDGE_SYMBOLS]; + for byte in b'0'..=b'9' { + identifier[usize::from(byte)] = state; + } + for byte in b'A'..=b'Z' { + identifier[usize::from(byte)] = state; + } + identifier[usize::from(b'_')] = state; + for byte in b'a'..=b'z' { + identifier[usize::from(byte)] = state; + } + let AsciiRun::Ranges(ranges) = AsciiRun::classify(&identifier, state) else { + panic!("identifier row should produce a range descriptor"); + }; + assert_eq!(ranges.count(), 4); + assert_eq!( + AsciiRun::Ranges(ranges) + .scan(b"abc_123!") + .expect("range descriptor should scan") + .bytes, + 7 + ); + + let mut unsupported = [DEAD_STATE; ASCII_EDGE_SYMBOLS]; + for byte in *b"acegi" { + unsupported[usize::from(byte)] = state; + } + assert_eq!(AsciiRun::classify(&unsupported, state), AsciiRun::None); } #[test] @@ -1670,6 +1894,63 @@ mod tests { } } + #[test] + fn range_scans_match_scalar_compiled_token_streams() { + let atn = range_loop_atn(); + let accelerated = CompiledLexerDfa::compile(&atn); + let counts = accelerated + .ascii_runs + .iter() + .filter_map(|run| match run { + AsciiRun::Ranges(ranges) => Some(ranges.count()), + _ => None, + }) + .collect::>(); + assert!(counts.contains(&1), "{counts:?}"); + assert!(counts.contains(&3), "{counts:?}"); + assert!(counts.contains(&4), "{counts:?}"); + + let mut scalar = accelerated.clone(); + for run in &mut scalar.ascii_runs { + if matches!(run, AsciiRun::Ranges(_)) { + *run = AsciiRun::None; + } + } + let input = format!( + "{}!\t{}\r\n{} {}", + "identifier_0123456789".repeat(12), + "1234567890".repeat(20), + "Another_identifier_9876543210".repeat(10), + "short_123" + ); + + let accelerated = compiled_stream(&input, &atn, &accelerated); + let scalar = compiled_stream(&input, &atn, &scalar); + assert_eq!(accelerated, scalar); + assert_eq!(accelerated.final_mode, 0); + assert_eq!(accelerated.popped_modes, [0, 0, 0]); + assert!( + accelerated.tokens.iter().any(|token| token.channel == 7), + "{accelerated:?}" + ); + assert!( + accelerated.tokens.iter().any(|token| token.line > 1), + "{accelerated:?}" + ); + assert_eq!( + accelerated + .tokens + .last() + .expect("stream includes EOF") + .token_type, + TOKEN_EOF + ); + assert_eq!( + accelerated.errors, + ["token recognition error at: '!'".to_owned()] + ); + } + #[test] fn compiled_dfa_matches_longest_token_and_skips() { let atn = two_rule_atn(false); @@ -1848,6 +2129,37 @@ mod tests { assert_eq!(rejected, 0); } + #[cfg(feature = "perf-counters")] + #[test] + fn lexer_counters_report_range_descriptor_and_scan_coverage() { + crate::perf::reset(); + let before = crate::perf::lexer_range_descriptor_snapshot(); + let atn = range_loop_atn(); + let dfa = CompiledLexerDfa::compile(&atn); + let descriptors = crate::perf::lexer_range_descriptor_snapshot(); + assert!(descriptors[3] > before[3], "{before:?} -> {descriptors:?}"); + assert!(descriptors[5] > before[5], "{before:?} -> {descriptors:?}"); + assert!(descriptors[6] > before[6], "{before:?} -> {descriptors:?}"); + assert!(descriptors[7] > before[7], "{before:?} -> {descriptors:?}"); + assert!(descriptors[8] > before[8], "{before:?} -> {descriptors:?}"); + assert!(descriptors[9] > before[9], "{before:?} -> {descriptors:?}"); + + crate::perf::reset(); + let input = format!( + "{} {} {}", + "long_identifier_0123456789".repeat(20), + "1234567890".repeat(40), + " \t\r\n".repeat(80) + ); + let _ = compiled_stream(&input, &atn, &dfa); + let scans = crate::perf::lexer_range_scan_snapshot(); + assert!(scans[0] > 0, "{scans:?}"); + assert!(scans[1] > 0, "{scans:?}"); + assert!(scans[2] > 0, "{scans:?}"); + assert!(scans[3] > 0, "{scans:?}"); + assert!(scans[4] > 0, "{scans:?}"); + } + #[test] fn compiled_dfa_reports_recognition_errors_like_the_interpreter() { let atn = wide_range_atn(); @@ -1879,7 +2191,7 @@ mod tests { #[test] fn serialization_round_trips() { - let atn = two_rule_atn(true); + let atn = range_loop_atn(); let dfa = CompiledLexerDfa::compile(&atn); let stream = dfa.serialize(); @@ -1887,12 +2199,12 @@ mod tests { CompiledLexerDfa::from_serialized(&stream).expect("stream should deserialize"); assert_eq!(restored.serialize(), stream); assert_eq!(restored.continuations.len(), dfa.continuations.len()); - assert_eq!(restored.ascii_runs, dfa.ascii_runs,); + assert_eq!(restored.ascii_runs, dfa.ascii_runs); - let mut lexer = BaseLexer::new(InputStream::new(" ab"), recognizer_data()); + let mut lexer = BaseLexer::new(InputStream::new("identifier_123"), recognizer_data()); let token = compiled_token(&mut lexer, &atn, &restored); - assert_eq!(token.token_type, 1); - assert_eq!(token.text, "ab"); + assert_eq!(token.token_type, 2); + assert_eq!(token.text, "identifier_123"); // A stream from a different runtime version is rejected, not trusted. let mut wrong_tag = stream; @@ -1909,12 +2221,55 @@ mod tests { .position(|&run| run != AsciiRun::Any) .expect("test grammar should contain a state that is not Any"); let mut stream = dfa.serialize(); - let first_run = 4 + dfa.mode_starts.len() + dfa.states.len() * 4; - stream[first_run + state] = AsciiRun::Any.pack(); + stream[serialized_run_offset(&dfa, state)] = 1; assert!(CompiledLexerDfa::from_serialized(&stream).is_none()); } + #[test] + fn malformed_serialized_range_descriptors_are_rejected() { + let dfa = CompiledLexerDfa::compile(&range_loop_atn()); + let stream = dfa.serialize(); + let range = first_serialized_range_offset(&dfa); + assert_eq!(stream[range].to_le_bytes()[0], 5); + + let mut zero_ranges = stream.clone(); + zero_ranges[range] = 5; + assert!(CompiledLexerDfa::from_serialized(&zero_ranges).is_none()); + + let mut too_many_ranges = stream.clone(); + too_many_ranges[range] = 5 | (5 << 8); + assert!(CompiledLexerDfa::from_serialized(&too_many_ranges).is_none()); + + let mut adjacent_ranges = stream.clone(); + adjacent_ranges[range] = 5 | (2 << 8); + adjacent_ranges[range + 1] = u32::from(b'a') + | (u32::from(b'm') << 8) + | (u32::from(b'n') << 16) + | (u32::from(b'z') << 24); + adjacent_ranges[range + 2] = 0; + assert!(CompiledLexerDfa::from_serialized(&adjacent_ranges).is_none()); + + let mut non_ascii_range = stream.clone(); + non_ascii_range[range] = 5 | (1 << 8); + non_ascii_range[range + 1] = u32::from(b'a') | (128 << 8); + non_ascii_range[range + 2] = 0; + assert!(CompiledLexerDfa::from_serialized(&non_ascii_range).is_none()); + + let mut nonzero_padding = stream.clone(); + nonzero_padding[range] = 5 | (1 << 8); + nonzero_padding[range + 1] = u32::from(b'0') + | (u32::from(b'9') << 8) + | (u32::from(b'A') << 16) + | (u32::from(b'Z') << 24); + nonzero_padding[range + 2] = 0; + assert!(CompiledLexerDfa::from_serialized(&nonzero_padding).is_none()); + + let mut truncated = stream; + truncated.truncate(range + 2); + assert!(CompiledLexerDfa::from_serialized(&truncated).is_none()); + } + #[test] fn malformed_wide_rows_are_rejected() { let atn = wide_range_atn(); diff --git a/src/atn/mod.rs b/src/atn/mod.rs index dfadf7d9..99629ccf 100644 --- a/src/atn/mod.rs +++ b/src/atn/mod.rs @@ -5,6 +5,7 @@ //! still mutates and inspects that shape. Parsers use the packed, //! index-addressed [`parser_atn::ParserAtn`] representation instead. +pub(crate) mod ascii_range; pub mod lexer; pub mod lexer_dfa; pub mod parser; diff --git a/src/perf.rs b/src/perf.rs index f6569452..4d445377 100644 --- a/src/perf.rs +++ b/src/perf.rs @@ -57,6 +57,17 @@ struct Counters { lexer_run_scan_exits: u64, lexer_run_scan_ends: u64, lexer_run_rejected_states: u64, + lexer_run_descriptors_none: u64, + lexer_run_descriptors_any: u64, + lexer_run_descriptors_until: u64, + lexer_range_descriptors: [u64; 4], + lexer_range_descriptor_classes: [u64; 4], + lexer_range_scalar_calls: u64, + lexer_range_scalar_bytes: u64, + lexer_range_identifier_bytes: u64, + lexer_range_number_bytes: u64, + lexer_range_whitespace_bytes: u64, + lexer_range_other_bytes: u64, decisions: BTreeMap, } @@ -338,8 +349,80 @@ pub(crate) fn record_lexer_run_rejected_state() { }); } +pub(crate) fn record_lexer_run_descriptor(kind: crate::atn::lexer_dfa::AsciiRunDescriptorKind) { + use crate::atn::lexer_dfa::AsciiRunDescriptorKind; + + with_counters(|counters| match kind { + AsciiRunDescriptorKind::None => { + counters.lexer_run_descriptors_none = + counters.lexer_run_descriptors_none.saturating_add(1); + } + AsciiRunDescriptorKind::Any => { + counters.lexer_run_descriptors_any = + counters.lexer_run_descriptors_any.saturating_add(1); + } + AsciiRunDescriptorKind::Until => { + counters.lexer_run_descriptors_until = + counters.lexer_run_descriptors_until.saturating_add(1); + } + AsciiRunDescriptorKind::Ranges { count, class } => { + if let Some(counter) = count + .checked_sub(1) + .and_then(|index| counters.lexer_range_descriptors.get_mut(usize::from(index))) + { + *counter = counter.saturating_add(1); + } + let class_index = match class { + crate::atn::ascii_range::AsciiRangeClass::Identifier => 0, + crate::atn::ascii_range::AsciiRangeClass::Number => 1, + crate::atn::ascii_range::AsciiRangeClass::Whitespace => 2, + crate::atn::ascii_range::AsciiRangeClass::Other => 3, + }; + counters.lexer_range_descriptor_classes[class_index] = + counters.lexer_range_descriptor_classes[class_index].saturating_add(1); + } + }); +} + +pub(crate) fn record_lexer_range_scan( + class: crate::atn::ascii_range::AsciiRangeClass, + bytes: usize, +) { + use crate::atn::ascii_range::AsciiRangeClass; + + let bytes = u64::try_from(bytes).unwrap_or(u64::MAX); + with_counters(|counters| { + counters.lexer_range_scalar_calls = counters.lexer_range_scalar_calls.saturating_add(1); + counters.lexer_range_scalar_bytes = counters.lexer_range_scalar_bytes.saturating_add(bytes); + let class_bytes = match class { + AsciiRangeClass::Identifier => &mut counters.lexer_range_identifier_bytes, + AsciiRangeClass::Number => &mut counters.lexer_range_number_bytes, + AsciiRangeClass::Whitespace => &mut counters.lexer_range_whitespace_bytes, + AsciiRangeClass::Other => &mut counters.lexer_range_other_bytes, + }; + *class_bytes = class_bytes.saturating_add(bytes); + }); +} + pub fn reset() { - COUNTERS.with(|counters| *counters.borrow_mut() = Counters::default()); + COUNTERS.with(|counters| { + let mut counters = counters.borrow_mut(); + // Compiled DFAs are usually initialized during warmup and then reused. + // Keep their static descriptor inventory while clearing timed counters. + let descriptors = ( + counters.lexer_run_descriptors_none, + counters.lexer_run_descriptors_any, + counters.lexer_run_descriptors_until, + counters.lexer_range_descriptors, + counters.lexer_range_descriptor_classes, + ); + *counters = Counters::default(); + counters.lexer_run_descriptors_none = descriptors.0; + counters.lexer_run_descriptors_any = descriptors.1; + counters.lexer_run_descriptors_until = descriptors.2; + counters.lexer_range_descriptors = descriptors.3; + counters.lexer_range_descriptor_classes = descriptors.4; + }); } pub fn dump() { @@ -373,7 +456,7 @@ fn dump_decisions(counters: &Counters) { } } -const fn totals(counters: &Counters) -> [(&'static str, u64); 50] { +const fn totals(counters: &Counters) -> [(&'static str, u64); 67] { [ ("prediction.adaptive_calls", counters.adaptive_calls), ( @@ -470,6 +553,71 @@ const fn totals(counters: &Counters) -> [(&'static str, u64); 50] { "lexer.run_rejected_states", counters.lexer_run_rejected_states, ), + ( + "lexer.run_descriptors.none", + counters.lexer_run_descriptors_none, + ), + ( + "lexer.run_descriptors.any", + counters.lexer_run_descriptors_any, + ), + ( + "lexer.run_descriptors.until", + counters.lexer_run_descriptors_until, + ), + ( + "lexer.range_descriptors.1", + counters.lexer_range_descriptors[0], + ), + ( + "lexer.range_descriptors.2", + counters.lexer_range_descriptors[1], + ), + ( + "lexer.range_descriptors.3", + counters.lexer_range_descriptors[2], + ), + ( + "lexer.range_descriptors.4", + counters.lexer_range_descriptors[3], + ), + ( + "lexer.range_descriptor_classes.identifier", + counters.lexer_range_descriptor_classes[0], + ), + ( + "lexer.range_descriptor_classes.number", + counters.lexer_range_descriptor_classes[1], + ), + ( + "lexer.range_descriptor_classes.whitespace", + counters.lexer_range_descriptor_classes[2], + ), + ( + "lexer.range_descriptor_classes.other", + counters.lexer_range_descriptor_classes[3], + ), + ( + "lexer.range_scalar_calls", + counters.lexer_range_scalar_calls, + ), + ( + "lexer.range_scalar_bytes", + counters.lexer_range_scalar_bytes, + ), + ( + "lexer.range_identifier_bytes", + counters.lexer_range_identifier_bytes, + ), + ( + "lexer.range_number_bytes", + counters.lexer_range_number_bytes, + ), + ( + "lexer.range_whitespace_bytes", + counters.lexer_range_whitespace_bytes, + ), + ("lexer.range_other_bytes", counters.lexer_range_other_bytes), ] } @@ -501,6 +649,41 @@ pub(crate) fn lexer_run_snapshot() -> [u64; 6] { }) } +#[cfg(test)] +pub(crate) fn lexer_range_descriptor_snapshot() -> [u64; 11] { + COUNTERS.with(|counters| { + let counters = counters.borrow(); + [ + counters.lexer_run_descriptors_none, + counters.lexer_run_descriptors_any, + counters.lexer_run_descriptors_until, + counters.lexer_range_descriptors[0], + counters.lexer_range_descriptors[1], + counters.lexer_range_descriptors[2], + counters.lexer_range_descriptors[3], + counters.lexer_range_descriptor_classes[0], + counters.lexer_range_descriptor_classes[1], + counters.lexer_range_descriptor_classes[2], + counters.lexer_range_descriptor_classes[3], + ] + }) +} + +#[cfg(test)] +pub(crate) fn lexer_range_scan_snapshot() -> [u64; 6] { + COUNTERS.with(|counters| { + let counters = counters.borrow(); + [ + counters.lexer_range_scalar_calls, + counters.lexer_range_scalar_bytes, + counters.lexer_range_identifier_bytes, + counters.lexer_range_number_bytes, + counters.lexer_range_whitespace_bytes, + counters.lexer_range_other_bytes, + ] + }) +} + fn print_counter(name: &str, value: u64) { #[allow(clippy::print_stderr)] { diff --git a/tools/parse-bench/README.md b/tools/parse-bench/README.md index eff08b9e..80c02562 100644 --- a/tools/parse-bench/README.md +++ b/tools/parse-bench/README.md @@ -116,7 +116,8 @@ python3 tools/parse-bench/run.py \ The source-derived fixtures cover ordinary Kotlin, C#, Java, and Trino input. Two lex-only Kotlin fixtures add concentrated ASCII coverage for long identifiers, strings, comments, whitespace, and punctuation, plus mixed-script -coverage for the Unicode fallback. +coverage for the Unicode fallback. Two lex-only Java fixtures isolate long +ASCII range classes for identifiers, numbers, and whitespace. Use a detached checkout for same-machine baseline comparisons, and select the compiler-level configurations explicitly: diff --git a/tools/parse-bench/fixtures/java/lexer-number-whitespace-stress.java b/tools/parse-bench/fixtures/java/lexer-number-whitespace-stress.java new file mode 100644 index 00000000..afc0ba04 --- /dev/null +++ b/tools/parse-bench/fixtures/java/lexer-number-whitespace-stress.java @@ -0,0 +1,29 @@ +package lexerbench; + +final class NumberWhitespaceStress { + static final long decimalValue = + 123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890L; + static final long hexadecimalValue = + 0xabcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789L; + + + + + + + + + + + + + + + + + static final int horizontallySeparatedValue = 7; + + long punctuationHeavy(long value) { + return (((value + decimalValue) * (hexadecimalValue - value)) / ((value + 1) | 1)) % 97; + } +} diff --git a/tools/parse-bench/fixtures/java/lexer-range-stress.java b/tools/parse-bench/fixtures/java/lexer-range-stress.java new file mode 100644 index 00000000..45a8b596 --- /dev/null +++ b/tools/parse-bench/fixtures/java/lexer-range-stress.java @@ -0,0 +1,16 @@ +package lexerbench; + +final class RangeStress { + static final int identifier_segment_0001_identifier_segment_0002_identifier_segment_0003_identifier_segment_0004_identifier_segment_0005_identifier_segment_0006_identifier_segment_0007_identifier_segment_0008_identifier_segment_0009_identifier_segment_0010_identifier_segment_0011_identifier_segment_0012_identifier_segment_0013_identifier_segment_0014_identifier_segment_0015_identifier_segment_0016 = + 123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890; + + static final long hexadecimal_identifier_segment_0001_identifier_segment_0002_identifier_segment_0003_identifier_segment_0004_identifier_segment_0005_identifier_segment_0006_identifier_segment_0007_identifier_segment_0008 = + 0xabcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789L; + + static final int whitespaceSeparatedValue = 7; + + int mix(int a, int mediumIdentifier, int identifier_segment_0001_identifier_segment_0002_identifier_segment_0003_identifier_segment_0004) { + int[] compactPunctuation = {1,2,3,4,5,6,7,8,9,10}; + return (((a + mediumIdentifier) * (compactPunctuation[3] - compactPunctuation[7])) / 2) % 97; + } +} diff --git a/tools/parse-bench/fixtures/manifest.json b/tools/parse-bench/fixtures/manifest.json index 5d52322a..6cb70133 100644 --- a/tools/parse-bench/fixtures/manifest.json +++ b/tools/parse-bench/fixtures/manifest.json @@ -72,6 +72,26 @@ "license": "MIT", "description": "Mono compiler statement binder and emitter code with deep statement forms, nested classes, switch paths, and exception-flow constructs." }, + { + "language": "java", + "path": "java/lexer-range-stress.java", + "source": "repository synthetic benchmark", + "license": "BSD-3-Clause", + "description": "Lex-only ASCII range stress input with long identifiers, decimal and hexadecimal numbers, whitespace, mixed identifier lengths, and punctuation-heavy expressions.", + "phases": [ + "lex" + ] + }, + { + "language": "java", + "path": "java/lexer-number-whitespace-stress.java", + "source": "repository synthetic benchmark", + "license": "BSD-3-Clause", + "description": "Lex-only ASCII range stress input isolating long decimal and hexadecimal literals plus horizontal and newline whitespace runs.", + "phases": [ + "lex" + ] + }, { "language": "java", "path": "java/mojang-data-result.java",