Skip to content
Merged
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
79 changes: 41 additions & 38 deletions crates/oxc_parser/src/lexer/token.rs
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,7 @@ use oxc_span::Span;

use super::kind::Kind;

// Bit layout for u128:
// Bit layout for `u128`:
// - Bits 0-31 (32 bits): `start`
// - Bits 32-63 (32 bits): `end`
// - Bits 64-71 (8 bits): `kind` (as u8)
Expand All @@ -24,8 +24,8 @@ const LONE_SURROGATES_SHIFT: u32 = 74;
const HAS_SEPARATOR_SHIFT: u32 = 75;

const START_MASK: u128 = 0xFFFF_FFFF; // 32 bits
const KIND_MASK: u128 = 0xFF; // 8 bits
const END_MASK: u128 = 0xFFFF_FFFF; // 32 bits
const KIND_MASK: u128 = 0xFF; // 8 bits

const IS_ON_NEW_LINE_FLAG: u128 = 1 << IS_ON_NEW_LINE_SHIFT;
const ESCAPED_FLAG: u128 = 1 << ESCAPED_SHIFT;
Expand Down Expand Up @@ -60,13 +60,13 @@ impl Token {
token.0 |= IS_ON_NEW_LINE_FLAG;
token
}
}

// Getters and setters
impl Token {
#[inline]
pub fn kind(&self) -> Kind {
// SAFETY: `Kind` is `#[repr(u8)]`. Only `Token::default` and `Token::set_kind` set these bits,
// and they set them to the `u8` value of an existing `Kind`.
// So transmuting these bits back to `Kind` must produce a valid `Kind`.
unsafe { mem::transmute::<u8, Kind>(((self.0 >> KIND_SHIFT) & KIND_MASK) as u8) }
pub fn span(&self) -> Span {
Span::new(self.start(), self.end())
}

#[inline]
Expand All @@ -75,38 +75,30 @@ impl Token {
}

#[inline]
pub fn end(&self) -> u32 {
((self.0 >> END_SHIFT) & END_MASK) as u32
}

#[inline]
pub fn is_on_new_line(&self) -> bool {
(self.0 & IS_ON_NEW_LINE_FLAG) != 0
}

#[inline]
pub fn escaped(&self) -> bool {
(self.0 & ESCAPED_FLAG) != 0
}

#[inline]
pub fn lone_surrogates(&self) -> bool {
(self.0 & LONE_SURROGATES_FLAG) != 0
pub(crate) fn set_start(&mut self, start: u32) {
self.0 &= !(START_MASK << START_SHIFT); // Clear current `start` bits
self.0 |= u128::from(start) << START_SHIFT;
}

#[inline]
pub fn span(&self) -> Span {
Span::new(self.start(), self.end())
pub fn end(&self) -> u32 {
((self.0 >> END_SHIFT) & END_MASK) as u32
}

#[inline]
pub fn has_separator(&self) -> bool {
(self.0 & HAS_SEPARATOR_FLAG) != 0
pub(crate) fn set_end(&mut self, end: u32) {
let start = self.start();
debug_assert!(end >= start, "Token end ({end}) cannot be less than start ({start})");
self.0 &= !(END_MASK << END_SHIFT); // Clear current `end` bits
self.0 |= u128::from(end) << END_SHIFT;
}

#[inline]
pub(crate) fn set_has_separator(&mut self) {
self.0 |= HAS_SEPARATOR_FLAG;
pub fn kind(&self) -> Kind {
// SAFETY: `Kind` is `#[repr(u8)]`. Only `Token::default` and `Token::set_kind` set these bits,
// and they set them to the `u8` value of an existing `Kind`.
// So transmuting these bits back to `Kind` must produce a valid `Kind`.
unsafe { mem::transmute::<u8, Kind>(((self.0 >> KIND_SHIFT) & KIND_MASK) as u8) }
}

#[inline]
Expand All @@ -116,32 +108,43 @@ impl Token {
}

#[inline]
pub(crate) fn set_start(&mut self, start: u32) {
self.0 &= !(START_MASK << START_SHIFT); // Clear current `start` bits
self.0 |= u128::from(start) << START_SHIFT;
pub fn is_on_new_line(&self) -> bool {
(self.0 & IS_ON_NEW_LINE_FLAG) != 0
}

#[inline]
pub(crate) fn set_is_on_new_line(&mut self, value: bool) {
self.0 = (self.0 & !IS_ON_NEW_LINE_FLAG) | (u128::from(value) * IS_ON_NEW_LINE_FLAG);
}

#[inline]
pub fn escaped(&self) -> bool {
(self.0 & ESCAPED_FLAG) != 0
}

#[inline]
pub(crate) fn set_escaped(&mut self, escaped: bool) {
self.0 = (self.0 & !ESCAPED_FLAG) | (u128::from(escaped) * ESCAPED_FLAG);
}

#[inline]
pub fn lone_surrogates(&self) -> bool {
(self.0 & LONE_SURROGATES_FLAG) != 0
}

#[inline]
pub(crate) fn set_lone_surrogates(&mut self, value: bool) {
self.0 = (self.0 & !LONE_SURROGATES_FLAG) | (u128::from(value) * LONE_SURROGATES_FLAG);
}

#[inline]
pub(crate) fn set_end(&mut self, end: u32) {
let start = self.start();
debug_assert!(end >= start, "Token end ({end}) cannot be less than start ({start})");
self.0 &= !(END_MASK << END_SHIFT); // Clear current `end` bits
self.0 |= u128::from(end) << END_SHIFT;
pub fn has_separator(&self) -> bool {
(self.0 & HAS_SEPARATOR_FLAG) != 0
}

#[inline]
pub(crate) fn set_has_separator(&mut self) {
self.0 |= HAS_SEPARATOR_FLAG;
}
}

Expand Down
Loading