diff --git a/crates/oxc_parser/src/lexer/token.rs b/crates/oxc_parser/src/lexer/token.rs index 072e85b2adbe7..aae47ee034906 100644 --- a/crates/oxc_parser/src/lexer/token.rs +++ b/crates/oxc_parser/src/lexer/token.rs @@ -6,7 +6,7 @@ use oxc_span::Span; use super::kind::Kind; -// Bit layout for u128: +// Bit layout for `u128`: // - Bits 0-31 (32 bits): `start` // - Bits 32-63 (32 bits): `end` // - Bits 64-71 (8 bits): `kind` (as u8) @@ -24,8 +24,8 @@ const LONE_SURROGATES_SHIFT: u32 = 74; const HAS_SEPARATOR_SHIFT: u32 = 75; const START_MASK: u128 = 0xFFFF_FFFF; // 32 bits -const KIND_MASK: u128 = 0xFF; // 8 bits const END_MASK: u128 = 0xFFFF_FFFF; // 32 bits +const KIND_MASK: u128 = 0xFF; // 8 bits const IS_ON_NEW_LINE_FLAG: u128 = 1 << IS_ON_NEW_LINE_SHIFT; const ESCAPED_FLAG: u128 = 1 << ESCAPED_SHIFT; @@ -60,13 +60,13 @@ impl Token { token.0 |= IS_ON_NEW_LINE_FLAG; token } +} +// Getters and setters +impl Token { #[inline] - pub fn kind(&self) -> Kind { - // SAFETY: `Kind` is `#[repr(u8)]`. Only `Token::default` and `Token::set_kind` set these bits, - // and they set them to the `u8` value of an existing `Kind`. - // So transmuting these bits back to `Kind` must produce a valid `Kind`. - unsafe { mem::transmute::(((self.0 >> KIND_SHIFT) & KIND_MASK) as u8) } + pub fn span(&self) -> Span { + Span::new(self.start(), self.end()) } #[inline] @@ -75,38 +75,30 @@ impl Token { } #[inline] - pub fn end(&self) -> u32 { - ((self.0 >> END_SHIFT) & END_MASK) as u32 - } - - #[inline] - pub fn is_on_new_line(&self) -> bool { - (self.0 & IS_ON_NEW_LINE_FLAG) != 0 - } - - #[inline] - pub fn escaped(&self) -> bool { - (self.0 & ESCAPED_FLAG) != 0 - } - - #[inline] - pub fn lone_surrogates(&self) -> bool { - (self.0 & LONE_SURROGATES_FLAG) != 0 + pub(crate) fn set_start(&mut self, start: u32) { + self.0 &= !(START_MASK << START_SHIFT); // Clear current `start` bits + self.0 |= u128::from(start) << START_SHIFT; } #[inline] - pub fn span(&self) -> Span { - Span::new(self.start(), self.end()) + pub fn end(&self) -> u32 { + ((self.0 >> END_SHIFT) & END_MASK) as u32 } #[inline] - pub fn has_separator(&self) -> bool { - (self.0 & HAS_SEPARATOR_FLAG) != 0 + pub(crate) fn set_end(&mut self, end: u32) { + let start = self.start(); + debug_assert!(end >= start, "Token end ({end}) cannot be less than start ({start})"); + self.0 &= !(END_MASK << END_SHIFT); // Clear current `end` bits + self.0 |= u128::from(end) << END_SHIFT; } #[inline] - pub(crate) fn set_has_separator(&mut self) { - self.0 |= HAS_SEPARATOR_FLAG; + pub fn kind(&self) -> Kind { + // SAFETY: `Kind` is `#[repr(u8)]`. Only `Token::default` and `Token::set_kind` set these bits, + // and they set them to the `u8` value of an existing `Kind`. + // So transmuting these bits back to `Kind` must produce a valid `Kind`. + unsafe { mem::transmute::(((self.0 >> KIND_SHIFT) & KIND_MASK) as u8) } } #[inline] @@ -116,9 +108,8 @@ impl Token { } #[inline] - pub(crate) fn set_start(&mut self, start: u32) { - self.0 &= !(START_MASK << START_SHIFT); // Clear current `start` bits - self.0 |= u128::from(start) << START_SHIFT; + pub fn is_on_new_line(&self) -> bool { + (self.0 & IS_ON_NEW_LINE_FLAG) != 0 } #[inline] @@ -126,22 +117,34 @@ impl Token { self.0 = (self.0 & !IS_ON_NEW_LINE_FLAG) | (u128::from(value) * IS_ON_NEW_LINE_FLAG); } + #[inline] + pub fn escaped(&self) -> bool { + (self.0 & ESCAPED_FLAG) != 0 + } + #[inline] pub(crate) fn set_escaped(&mut self, escaped: bool) { self.0 = (self.0 & !ESCAPED_FLAG) | (u128::from(escaped) * ESCAPED_FLAG); } + #[inline] + pub fn lone_surrogates(&self) -> bool { + (self.0 & LONE_SURROGATES_FLAG) != 0 + } + #[inline] pub(crate) fn set_lone_surrogates(&mut self, value: bool) { self.0 = (self.0 & !LONE_SURROGATES_FLAG) | (u128::from(value) * LONE_SURROGATES_FLAG); } #[inline] - pub(crate) fn set_end(&mut self, end: u32) { - let start = self.start(); - debug_assert!(end >= start, "Token end ({end}) cannot be less than start ({start})"); - self.0 &= !(END_MASK << END_SHIFT); // Clear current `end` bits - self.0 |= u128::from(end) << END_SHIFT; + pub fn has_separator(&self) -> bool { + (self.0 & HAS_SEPARATOR_FLAG) != 0 + } + + #[inline] + pub(crate) fn set_has_separator(&mut self) { + self.0 |= HAS_SEPARATOR_FLAG; } }