Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 2 additions & 1 deletion crates/oxc_parser/src/lexer/byte_handlers.rs
Original file line number Diff line number Diff line change
@@ -1,6 +1,7 @@
use super::{Kind, Lexer};
use crate::diagnostics;

use super::{Kind, Lexer};

/// Handle next byte of source.
///
/// # SAFETY
Expand Down
4 changes: 3 additions & 1 deletion crates/oxc_parser/src/lexer/comment.rs
Original file line number Diff line number Diff line change
@@ -1,13 +1,15 @@
use memchr::memmem::Finder;

use oxc_syntax::identifier::is_line_terminator;

use crate::diagnostics;

use super::{
cold_branch,
search::{byte_search, safe_byte_match_table, SafeByteMatchTable},
source::SourcePosition,
Kind, Lexer,
};
use crate::diagnostics;

// Irregular line breaks - '\u{2028}' (LS) and '\u{2029}' (PS)
const LS_OR_PS_FIRST: u8 = 0xE2;
Expand Down
3 changes: 2 additions & 1 deletion crates/oxc_parser/src/lexer/identifier.rs
Original file line number Diff line number Diff line change
Expand Up @@ -6,12 +6,13 @@ use oxc_syntax::identifier::{
is_identifier_part, is_identifier_part_unicode, is_identifier_start_unicode,
};

use crate::diagnostics;

use super::{
cold_branch,
search::{byte_search, safe_byte_match_table, SafeByteMatchTable},
Kind, Lexer, SourcePosition,
};
use crate::diagnostics;

const MIN_ESCAPED_STR_LEN: usize = 16;

Expand Down
4 changes: 3 additions & 1 deletion crates/oxc_parser/src/lexer/jsx.rs
Original file line number Diff line number Diff line change
@@ -1,12 +1,14 @@
use memchr::{memchr, memchr2};

use oxc_syntax::identifier::is_identifier_part;

use crate::diagnostics;

use super::{
cold_branch,
search::{byte_search, safe_byte_match_table, SafeByteMatchTable},
Kind, Lexer, Token,
};
use crate::diagnostics;

static NOT_ASCII_JSX_ID_CONTINUE_TABLE: SafeByteMatchTable =
safe_byte_match_table!(|b| !(b.is_ascii_alphanumeric() || matches!(b, b'_' | b'$' | b'-')));
Expand Down
2 changes: 1 addition & 1 deletion crates/oxc_parser/src/lexer/kind.rs
Original file line number Diff line number Diff line change
Expand Up @@ -197,7 +197,7 @@ pub enum Kind {
}

#[allow(clippy::enum_glob_use)]
use self::Kind::*;
use Kind::*;

impl Kind {
#[inline]
Expand Down
35 changes: 17 additions & 18 deletions crates/oxc_parser/src/lexer/mod.rs
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,17 @@
//! * [rustc](https://github.com/rust-lang/rust/blob/1.82.0/compiler/rustc_lexer/src)
//! * [v8](https://v8.dev/blog/scanner)

use std::collections::VecDeque;

use rustc_hash::FxHashMap;

use oxc_allocator::Allocator;
use oxc_ast::ast::RegExpFlags;
use oxc_diagnostics::OxcDiagnostic;
use oxc_span::{SourceType, Span};

use crate::{diagnostics, UniquePromise};

mod byte_handlers;
mod comment;
mod identifier;
Expand All @@ -24,25 +35,13 @@ mod typescript;
mod unicode;
mod whitespace;

use std::collections::VecDeque;
pub use kind::Kind;
pub use number::{parse_big_int, parse_float, parse_int};
pub use token::Token;

use oxc_allocator::Allocator;
use oxc_ast::ast::RegExpFlags;
use oxc_diagnostics::OxcDiagnostic;
use oxc_span::{SourceType, Span};
use rustc_hash::FxHashMap;

use self::{
byte_handlers::handle_byte,
source::{Source, SourcePosition},
trivia_builder::TriviaBuilder,
};
pub use self::{
kind::Kind,
number::{parse_big_int, parse_float, parse_int},
token::Token,
};
use crate::{diagnostics, UniquePromise};
use byte_handlers::handle_byte;
use source::{Source, SourcePosition};
use trivia_builder::TriviaBuilder;

#[derive(Debug, Clone, Copy)]
pub struct LexerCheckpoint<'a> {
Expand Down
3 changes: 2 additions & 1 deletion crates/oxc_parser/src/lexer/numeric.rs
Original file line number Diff line number Diff line change
@@ -1,8 +1,9 @@
use oxc_syntax::identifier::{is_identifier_part_ascii, is_identifier_start};

use super::{Kind, Lexer, Span};
use crate::diagnostics;

use super::{Kind, Lexer, Span};

impl Lexer<'_> {
/// 12.9.3 Numeric Literals with `0` prefix
pub(super) fn read_zero(&mut self) -> Kind {
Expand Down
3 changes: 2 additions & 1 deletion crates/oxc_parser/src/lexer/regex.rs
Original file line number Diff line number Diff line change
@@ -1,9 +1,10 @@
use oxc_diagnostics::Result;
use oxc_syntax::identifier::is_line_terminator;

use super::{Kind, Lexer, RegExpFlags, Token};
use crate::diagnostics;

use super::{Kind, Lexer, RegExpFlags, Token};

impl Lexer<'_> {
/// Re-tokenize the current `/` or `/=` and return `RegExp`
/// See Section 12:
Expand Down
3 changes: 2 additions & 1 deletion crates/oxc_parser/src/lexer/source.rs
Original file line number Diff line number Diff line change
Expand Up @@ -2,9 +2,10 @@

use std::{marker::PhantomData, slice, str};

use super::search::SEARCH_BATCH_SIZE;
use crate::{UniquePromise, MAX_LEN};

use super::search::SEARCH_BATCH_SIZE;

/// `Source` holds the source text for the lexer, and provides APIs to read it.
///
/// It provides a cursor which allows consuming source text either as `char`s, or as bytes.
Expand Down
3 changes: 2 additions & 1 deletion crates/oxc_parser/src/lexer/string.rs
Original file line number Diff line number Diff line change
Expand Up @@ -2,12 +2,13 @@ use std::cmp::max;

use oxc_allocator::String;

use crate::diagnostics;

use super::{
cold_branch,
search::{byte_search, safe_byte_match_table, SafeByteMatchTable},
Kind, Lexer, LexerContext, Span, Token,
};
use crate::diagnostics;

const MIN_ESCAPED_STR_LEN: usize = 16;

Expand Down
3 changes: 2 additions & 1 deletion crates/oxc_parser/src/lexer/template.rs
Original file line number Diff line number Diff line change
Expand Up @@ -2,12 +2,13 @@ use std::cmp::max;

use oxc_allocator::String;

use crate::diagnostics;

use super::{
cold_branch,
search::{byte_search, safe_byte_match_table, SafeByteMatchTable},
Kind, Lexer, SourcePosition, Token,
};
use crate::diagnostics;

const MIN_ESCAPED_TEMPLATE_LIT_LEN: usize = 16;

Expand Down
3 changes: 2 additions & 1 deletion crates/oxc_parser/src/lexer/unicode.rs
Original file line number Diff line number Diff line change
Expand Up @@ -4,9 +4,10 @@ use oxc_syntax::identifier::{
is_irregular_line_terminator, is_irregular_whitespace, CR, FF, LF, LS, PS, TAB, VT,
};

use super::{Kind, Lexer, Span};
use crate::diagnostics;

use super::{Kind, Lexer, Span};

enum SurrogatePair {
// valid \u Hex4Digits \u Hex4Digits
Astral(u32),
Expand Down
Loading