Skip to content

refactor(estree/tokens): combine collect_token_context into to_estree_tokens_json #19534

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
graphite-app merged 1 commit into from
Feb 24, 2026
Merged

refactor(estree/tokens): combine collect_token_context into to_estree_tokens_json #19534

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
11 changes: 4 additions & 7 deletions apps/oxlint/src/js_plugins/parse.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -8,7 +8,7 @@ use napi_derive::napi;

use oxc_allocator::Allocator;
use oxc_ast_visit::utf8_to_utf16::Utf8ToUtf16;
use oxc_estree_tokens::{EstreeTokenOptions, collect_token_context, to_estree_tokens_json};
use oxc_estree_tokens::{EstreeTokenOptions, to_estree_tokens_json};
use oxc_linter::RawTransferMetadata2 as RawTransferMetadata;
use oxc_napi::get_source_type;
use oxc_parser::{ParseOptions, Parser, ParserReturn, config::RuntimeParserConfig};
Expand Down Expand Up @@ -220,14 +220,11 @@ unsafe fn parse_raw_impl(
// Fallback to TypeScript token parsing in JS for BOM files.
(0, 0)
} else {
let token_options = EstreeTokenOptions::linter();
let token_context = collect_token_context(program, token_options);
let tokens_json = to_estree_tokens_json(
&allocator,
source_text,
&tokens,
&token_context,
token_options,
program,
EstreeTokenOptions::linter(),
&allocator,
);
let tokens_json = allocator.alloc_str(&tokens_json);
let tokens_offset = tokens_json.as_ptr() as u32;
Expand Down
42 changes: 20 additions & 22 deletions crates/oxc_estree_tokens/src/lib.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -68,41 +68,39 @@ impl EstreeTokenOptions {
}
}

pub fn collect_token_context(
program: &Program<'_>,
options: EstreeTokenOptions,
) -> EstreeTokenContext {
let mut context = EstreeTokenContext {
jsx_namespace_jsx_identifiers: options.jsx_namespace_jsx_identifiers,
member_expr_in_jsx_expression_jsx_identifiers: options
.member_expr_in_jsx_expression_jsx_identifiers,
..Default::default()
};
context.visit_program(program);
context
}

/// Serialize tokens to JSON.
pub fn to_estree_tokens_json(
allocator: &Allocator,
source_text: &str,
tokens: &[Token],
context: &EstreeTokenContext,
program: &Program<'_>,
options: EstreeTokenOptions,
allocator: &Allocator,
) -> String {
let estree_tokens = to_estree_tokens(allocator, source_text, tokens, context, options);
let estree_tokens = to_estree_tokens(tokens, program, options, allocator);
serde_json::to_string_pretty(&estree_tokens).unwrap_or_default()
}

pub fn to_estree_tokens<'a>(
allocator: &'a Allocator,
source_text: &'a str,
/// Convert `Token`s to `EstreeToken`s.
fn to_estree_tokens<'a>(
tokens: &[Token],
context: &EstreeTokenContext,
program: &Program<'a>,
options: EstreeTokenOptions,
allocator: &'a Allocator,
) -> ArenaVec<'a, EstreeToken<'a>> {
// Traverse AST to collect details of tokens requiring correction, depending on provided options
let mut context = EstreeTokenContext {
jsx_namespace_jsx_identifiers: options.jsx_namespace_jsx_identifiers,
member_expr_in_jsx_expression_jsx_identifiers: options
.member_expr_in_jsx_expression_jsx_identifiers,
..Default::default()
};
context.visit_program(program);

// Create UTF-8 to UTF-16 conversion table
let source_text = program.source_text;
let utf8_to_utf16 = Utf8ToUtf16::new(source_text);
let mut converter = utf8_to_utf16.converter();

// Convert tokens to `EstreeToken`s
let mut estree_tokens = ArenaVec::with_capacity_in(tokens.len(), allocator);
for token in tokens {
let kind = token.kind();
Expand Down
11 changes: 4 additions & 7 deletions crates/oxc_linter/src/lib.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -20,7 +20,7 @@ use oxc_ast_macros::ast;
use oxc_ast_visit::utf8_to_utf16::Utf8ToUtf16;
use oxc_data_structures::box_macros::boxed_array;
use oxc_diagnostics::OxcDiagnostic;
use oxc_estree_tokens::{EstreeTokenOptions, collect_token_context, to_estree_tokens_json};
use oxc_estree_tokens::{EstreeTokenOptions, to_estree_tokens_json};
use oxc_semantic::AstNode;
use oxc_span::Span;

Expand Down Expand Up @@ -576,14 +576,11 @@ impl Linter {
// Keep JS fallback path for BOM sources.
(0, 0)
} else if let Some(parser_tokens) = ctx_host.current_sub_host().parser_tokens() {
let token_options = EstreeTokenOptions::linter();
let token_context = collect_token_context(program, token_options);
let tokens_json = to_estree_tokens_json(
allocator,
source_text,
parser_tokens,
&token_context,
token_options,
program,
EstreeTokenOptions::linter(),
allocator,
);
if tokens_json.is_empty() {
(0, 0)
Expand Down
11 changes: 4 additions & 7 deletions tasks/benchmark/benches/parser.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
use oxc_allocator::Allocator;
use oxc_ast_visit::utf8_to_utf16::Utf8ToUtf16;
use oxc_benchmark::{BenchmarkId, Criterion, black_box, criterion_group, criterion_main};
use oxc_estree_tokens::{EstreeTokenOptions, collect_token_context, to_estree_tokens_json};
use oxc_estree_tokens::{EstreeTokenOptions, to_estree_tokens_json};
use oxc_parser::{ParseOptions, Parser, ParserReturn, config::RuntimeParserConfig};
use oxc_tasks_common::TestFiles;

Expand Down Expand Up @@ -140,14 +140,11 @@ fn bench_estree_tokens(criterion: &mut Criterion) {
span_converter.convert_program(&mut program);

runner.run(|| {
let token_options = EstreeTokenOptions::test262();
let token_context = collect_token_context(&program, token_options);
let tokens_json = to_estree_tokens_json(
&allocator,
source_text,
&tokens,
&token_context,
token_options,
&program,
EstreeTokenOptions::test262(),
&allocator,
);
let tokens_json = black_box(tokens_json);
// Allocate into tokens JSON into arena, same as linter and NAPI parser package do
Expand Down
33 changes: 8 additions & 25 deletions tasks/coverage/src/tools.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -11,7 +11,7 @@ use oxc::{
span::{ModuleKind, SourceType, Span},
transformer::{JsxOptions, JsxRuntime, TransformOptions},
};
use oxc_estree_tokens::{EstreeTokenOptions, collect_token_context, to_estree_tokens_json};
use oxc_estree_tokens::{EstreeTokenOptions, to_estree_tokens_json};
use oxc_formatter::{
ArrowParentheses, AttributePosition, BracketSameLine, BracketSpacing, Expand, FormatOptions,
Formatter, IndentStyle, IndentWidth, LineEnding, LineWidth, QuoteProperties, QuoteStyle,
Expand Down Expand Up @@ -854,15 +854,8 @@ pub fn run_estree_test262_tokens(files: &[Test262File]) -> Vec<CoverageResult> {
let span_converter = Utf8ToUtf16::new(source_text);
span_converter.convert_program_with_ascending_order_checks(&mut program);

let token_options = EstreeTokenOptions::test262();
let token_context = collect_token_context(&program, token_options);
let oxc_tokens_json = to_estree_tokens_json(
&allocator,
source_text,
&tokens,
&token_context,
token_options,
);
let oxc_tokens_json =
to_estree_tokens_json(&tokens, &program, EstreeTokenOptions::test262(), &allocator);

let token_path = workspace_root()
.join("estree-conformance/tests/test262-tokens")
Expand Down Expand Up @@ -905,15 +898,8 @@ pub fn run_estree_acorn_jsx_tokens(files: &[AcornJsxFile]) -> Vec<CoverageResult
let span_converter = Utf8ToUtf16::new(source_text);
span_converter.convert_program_with_ascending_order_checks(&mut program);

let token_options = EstreeTokenOptions::test262();
let token_context = collect_token_context(&program, token_options);
let oxc_tokens_json = to_estree_tokens_json(
&allocator,
source_text,
&tokens,
&token_context,
token_options,
);
let oxc_tokens_json =
to_estree_tokens_json(&tokens, &program, EstreeTokenOptions::test262(), &allocator);

let token_path = workspace_root().join(f.path.with_extension("tokens.json"));
let expected_tokens_json = fs::read_to_string(&token_path).unwrap_or_default();
Expand Down Expand Up @@ -1089,14 +1075,11 @@ pub fn run_estree_typescript_tokens(files: &[TypeScriptFile]) -> Vec<CoverageRes
let span_converter = Utf8ToUtf16::new(source_text);
span_converter.convert_program_with_ascending_order_checks(&mut program);

let token_options = EstreeTokenOptions::typescript();
let token_context = collect_token_context(&program, token_options);
let oxc_tokens_json = to_estree_tokens_json(
&allocator,
source_text,
&tokens,
&token_context,
token_options,
&program,
EstreeTokenOptions::typescript(),
&allocator,
);

if oxc_tokens_json != *expected_tokens {
Expand Down
Loading