diff --git a/apps/oxlint/src/js_plugins/parse.rs b/apps/oxlint/src/js_plugins/parse.rs index da16ea9009ca7..bbce63c434a27 100644 --- a/apps/oxlint/src/js_plugins/parse.rs +++ b/apps/oxlint/src/js_plugins/parse.rs @@ -8,7 +8,7 @@ use napi_derive::napi; use oxc_allocator::Allocator; use oxc_ast_visit::utf8_to_utf16::Utf8ToUtf16; -use oxc_estree_tokens::{EstreeTokenOptions, collect_token_context, to_estree_tokens_json}; +use oxc_estree_tokens::{EstreeTokenOptions, to_estree_tokens_json}; use oxc_linter::RawTransferMetadata2 as RawTransferMetadata; use oxc_napi::get_source_type; use oxc_parser::{ParseOptions, Parser, ParserReturn, config::RuntimeParserConfig}; @@ -220,14 +220,11 @@ unsafe fn parse_raw_impl( // Fallback to TypeScript token parsing in JS for BOM files. (0, 0) } else { - let token_options = EstreeTokenOptions::linter(); - let token_context = collect_token_context(program, token_options); let tokens_json = to_estree_tokens_json( - &allocator, - source_text, &tokens, - &token_context, - token_options, + program, + EstreeTokenOptions::linter(), + &allocator, ); let tokens_json = allocator.alloc_str(&tokens_json); let tokens_offset = tokens_json.as_ptr() as u32; diff --git a/crates/oxc_estree_tokens/src/lib.rs b/crates/oxc_estree_tokens/src/lib.rs index c51747ffae714..58b90cf7d74f2 100644 --- a/crates/oxc_estree_tokens/src/lib.rs +++ b/crates/oxc_estree_tokens/src/lib.rs @@ -68,41 +68,39 @@ impl EstreeTokenOptions { } } -pub fn collect_token_context( - program: &Program<'_>, - options: EstreeTokenOptions, -) -> EstreeTokenContext { - let mut context = EstreeTokenContext { - jsx_namespace_jsx_identifiers: options.jsx_namespace_jsx_identifiers, - member_expr_in_jsx_expression_jsx_identifiers: options - .member_expr_in_jsx_expression_jsx_identifiers, - ..Default::default() - }; - context.visit_program(program); - context -} - +/// Serialize tokens to JSON. pub fn to_estree_tokens_json( - allocator: &Allocator, - source_text: &str, tokens: &[Token], - context: &EstreeTokenContext, + program: &Program<'_>, options: EstreeTokenOptions, + allocator: &Allocator, ) -> String { - let estree_tokens = to_estree_tokens(allocator, source_text, tokens, context, options); + let estree_tokens = to_estree_tokens(tokens, program, options, allocator); serde_json::to_string_pretty(&estree_tokens).unwrap_or_default() } -pub fn to_estree_tokens<'a>( - allocator: &'a Allocator, - source_text: &'a str, +/// Convert `Token`s to `EstreeToken`s. +fn to_estree_tokens<'a>( tokens: &[Token], - context: &EstreeTokenContext, + program: &Program<'a>, options: EstreeTokenOptions, + allocator: &'a Allocator, ) -> ArenaVec<'a, EstreeToken<'a>> { + // Traverse AST to collect details of tokens requiring correction, depending on provided options + let mut context = EstreeTokenContext { + jsx_namespace_jsx_identifiers: options.jsx_namespace_jsx_identifiers, + member_expr_in_jsx_expression_jsx_identifiers: options + .member_expr_in_jsx_expression_jsx_identifiers, + ..Default::default() + }; + context.visit_program(program); + + // Create UTF-8 to UTF-16 conversion table + let source_text = program.source_text; let utf8_to_utf16 = Utf8ToUtf16::new(source_text); let mut converter = utf8_to_utf16.converter(); + // Convert tokens to `EstreeToken`s let mut estree_tokens = ArenaVec::with_capacity_in(tokens.len(), allocator); for token in tokens { let kind = token.kind(); diff --git a/crates/oxc_linter/src/lib.rs b/crates/oxc_linter/src/lib.rs index e46e3ed6ea3c1..54aa546f5e574 100644 --- a/crates/oxc_linter/src/lib.rs +++ b/crates/oxc_linter/src/lib.rs @@ -20,7 +20,7 @@ use oxc_ast_macros::ast; use oxc_ast_visit::utf8_to_utf16::Utf8ToUtf16; use oxc_data_structures::box_macros::boxed_array; use oxc_diagnostics::OxcDiagnostic; -use oxc_estree_tokens::{EstreeTokenOptions, collect_token_context, to_estree_tokens_json}; +use oxc_estree_tokens::{EstreeTokenOptions, to_estree_tokens_json}; use oxc_semantic::AstNode; use oxc_span::Span; @@ -576,14 +576,11 @@ impl Linter { // Keep JS fallback path for BOM sources. (0, 0) } else if let Some(parser_tokens) = ctx_host.current_sub_host().parser_tokens() { - let token_options = EstreeTokenOptions::linter(); - let token_context = collect_token_context(program, token_options); let tokens_json = to_estree_tokens_json( - allocator, - source_text, parser_tokens, - &token_context, - token_options, + program, + EstreeTokenOptions::linter(), + allocator, ); if tokens_json.is_empty() { (0, 0) diff --git a/tasks/benchmark/benches/parser.rs b/tasks/benchmark/benches/parser.rs index 8a7027756850b..b5d7cb7681045 100644 --- a/tasks/benchmark/benches/parser.rs +++ b/tasks/benchmark/benches/parser.rs @@ -1,7 +1,7 @@ use oxc_allocator::Allocator; use oxc_ast_visit::utf8_to_utf16::Utf8ToUtf16; use oxc_benchmark::{BenchmarkId, Criterion, black_box, criterion_group, criterion_main}; -use oxc_estree_tokens::{EstreeTokenOptions, collect_token_context, to_estree_tokens_json}; +use oxc_estree_tokens::{EstreeTokenOptions, to_estree_tokens_json}; use oxc_parser::{ParseOptions, Parser, ParserReturn, config::RuntimeParserConfig}; use oxc_tasks_common::TestFiles; @@ -140,14 +140,11 @@ fn bench_estree_tokens(criterion: &mut Criterion) { span_converter.convert_program(&mut program); runner.run(|| { - let token_options = EstreeTokenOptions::test262(); - let token_context = collect_token_context(&program, token_options); let tokens_json = to_estree_tokens_json( - &allocator, - source_text, &tokens, - &token_context, - token_options, + &program, + EstreeTokenOptions::test262(), + &allocator, ); let tokens_json = black_box(tokens_json); // Allocate into tokens JSON into arena, same as linter and NAPI parser package do diff --git a/tasks/coverage/src/tools.rs b/tasks/coverage/src/tools.rs index ac4c53372686b..500a3fa18ef10 100644 --- a/tasks/coverage/src/tools.rs +++ b/tasks/coverage/src/tools.rs @@ -11,7 +11,7 @@ use oxc::{ span::{ModuleKind, SourceType, Span}, transformer::{JsxOptions, JsxRuntime, TransformOptions}, }; -use oxc_estree_tokens::{EstreeTokenOptions, collect_token_context, to_estree_tokens_json}; +use oxc_estree_tokens::{EstreeTokenOptions, to_estree_tokens_json}; use oxc_formatter::{ ArrowParentheses, AttributePosition, BracketSameLine, BracketSpacing, Expand, FormatOptions, Formatter, IndentStyle, IndentWidth, LineEnding, LineWidth, QuoteProperties, QuoteStyle, @@ -854,15 +854,8 @@ pub fn run_estree_test262_tokens(files: &[Test262File]) -> Vec { let span_converter = Utf8ToUtf16::new(source_text); span_converter.convert_program_with_ascending_order_checks(&mut program); - let token_options = EstreeTokenOptions::test262(); - let token_context = collect_token_context(&program, token_options); - let oxc_tokens_json = to_estree_tokens_json( - &allocator, - source_text, - &tokens, - &token_context, - token_options, - ); + let oxc_tokens_json = + to_estree_tokens_json(&tokens, &program, EstreeTokenOptions::test262(), &allocator); let token_path = workspace_root() .join("estree-conformance/tests/test262-tokens") @@ -905,15 +898,8 @@ pub fn run_estree_acorn_jsx_tokens(files: &[AcornJsxFile]) -> Vec Vec