refactor(estree/tokens): convert tokens spans before AST spans

apps/oxlint/src/js_plugins/parse.rs

@@ -214,9 +214,6 @@ unsafe fn parse_raw_impl(
                 Utf8ToUtf16::new(source_text)
             };
 
-            span_converter.convert_program(program);
-            span_converter.convert_comments(&mut program.comments);
-
             let tokens_json = to_estree_tokens_json(
                 &tokens,
                 program,
@@ -225,6 +222,10 @@ unsafe fn parse_raw_impl(
                 EstreeTokenOptions::linter(),
                 &allocator,
             );
+
+            span_converter.convert_program(program);
+            span_converter.convert_comments(&mut program.comments);
+
             let tokens_json = allocator.alloc_str(&tokens_json);
             let tokens_offset = tokens_json.as_ptr() as u32;
             #[expect(clippy::cast_possible_truncation)]

crates/oxc_estree_tokens/src/lib.rs

@@ -62,6 +62,9 @@ impl EstreeTokenOptions {
 
 /// Serialize tokens to JSON.
 ///
+/// `program` must have unconverted UTF-8 byte offset spans (as returned by the parser).
+/// Token span conversion to UTF-16 is handled internally.
+///
 /// `source_text` must be the original source text, prior to BOM removal.
 /// i.e. BOM must be present on start of `source_text`, if the file has a BOM.
 pub fn to_estree_tokens_json(
@@ -79,6 +82,9 @@ pub fn to_estree_tokens_json(
 
 /// Serialize tokens to pretty-printed JSON.
 ///
+/// `program` must have unconverted UTF-8 byte offset spans (as returned by the parser).
+/// Token span conversion to UTF-16 is handled internally.
+///
 /// `source_text` must be the original source text, prior to BOM removal.
 /// i.e. BOM must be present on start of `source_text`, if the file has a BOM.
 pub fn to_estree_tokens_pretty_json(
@@ -126,15 +132,10 @@ fn to_estree_tokens<'a>(
 
     for token in tokens {
         let kind = token.kind();
-        let mut value = &source_text[token.start() as usize..token.end() as usize];
-
         let mut start = token.start();
         let mut end = token.end();
-        if let Some(span_converter) = span_converter.as_mut() {
-            span_converter.convert_offset(&mut start);
-            span_converter.convert_offset(&mut end);
-        }
 
+        // Compare override start against original UTF-8 token start (before span conversion)
         let token_type = if next_override_start == start {
             let token_type = match next_override_kind {
                 TokenKindOverride::Identifier => "Identifier",
@@ -151,6 +152,8 @@ fn to_estree_tokens<'a>(
             get_token_type(kind)
         };
 
+        let mut value = &source_text[start as usize..end as usize];
+
         if kind == Kind::PrivateIdentifier {
             value = &value[1..];
         }
@@ -168,6 +171,12 @@ fn to_estree_tokens<'a>(
             None
         };
 
+        // Convert offsets to UTF-16
+        if let Some(span_converter) = span_converter.as_mut() {
+            span_converter.convert_offset(&mut start);
+            span_converter.convert_offset(&mut end);
+        }
+
         estree_tokens.push(EstreeToken { token_type, value, regex, start, end });
     }
 

crates/oxc_linter/src/lib.rs

@@ -569,9 +569,6 @@ impl Linter {
             Utf8ToUtf16::new(source_text)
         };
 
-        span_converter.convert_program(program);
-        span_converter.convert_comments(&mut program.comments);
-
         let (tokens_offset, tokens_len) =
             if let Some(tokens) = ctx_host.current_sub_host().parser_tokens() {
                 let tokens_json = to_estree_tokens_json(
@@ -591,6 +588,9 @@ impl Linter {
                 (0, 0)
             };
 
+        span_converter.convert_program(program);
+        span_converter.convert_comments(&mut program.comments);
+
         // Get offset of `Program` within buffer (bottom 32 bits of pointer)
         let program_offset = ptr::from_ref(program) as u32;
 

tasks/benchmark/benches/parser.rs

@@ -133,11 +133,11 @@ fn bench_estree_tokens(criterion: &mut Criterion) {
                     })
                     .with_config(config)
                     .parse();
-                let ParserReturn { mut program, tokens, .. } = ret;
+                let ParserReturn { program, tokens, .. } = ret;
 
-                // Span conversion of AST is not performed in measured section, as we only want to measure tokens
+                // Creating span converter is not performed in measured section, as we only want to measure tokens.
+                // Span converter needs to be created anyway for serializing AST.
                 let span_converter = Utf8ToUtf16::new(program.source_text);
-                span_converter.convert_program(&mut program);
 
                 runner.run(|| {
                     let tokens_json = to_estree_tokens_json(

tasks/coverage/src/tools.rs

@@ -848,7 +848,6 @@ pub fn run_estree_test262_tokens(files: &[Test262File]) -> Vec<CoverageResult> {
 
             let ParserReturn { mut program, tokens, .. } = ret;
             let span_converter = Utf8ToUtf16::new(source_text);
-            span_converter.convert_program_with_ascending_order_checks(&mut program);
 
             let oxc_tokens_json = to_estree_tokens_pretty_json(
                 &tokens,
@@ -859,6 +858,8 @@ pub fn run_estree_test262_tokens(files: &[Test262File]) -> Vec<CoverageResult> {
                 &allocator,
             );
 
+            span_converter.convert_program_with_ascending_order_checks(&mut program);
+
             let token_path = workspace_root()
                 .join("estree-conformance/tests/test262-tokens")
                 .join(f.path.strip_prefix("test262/").unwrap_or(&f.path))
@@ -898,7 +899,6 @@ pub fn run_estree_acorn_jsx_tokens(files: &[AcornJsxFile]) -> Vec<CoverageResult
 
             let ParserReturn { mut program, tokens, .. } = ret;
             let span_converter = Utf8ToUtf16::new(source_text);
-            span_converter.convert_program_with_ascending_order_checks(&mut program);
 
             let oxc_tokens_json = to_estree_tokens_pretty_json(
                 &tokens,
@@ -909,6 +909,8 @@ pub fn run_estree_acorn_jsx_tokens(files: &[AcornJsxFile]) -> Vec<CoverageResult
                 &allocator,
             );
 
+            span_converter.convert_program_with_ascending_order_checks(&mut program);
+
             let token_path = workspace_root().join(f.path.with_extension("tokens.json"));
             let expected_tokens_json = fs::read_to_string(&token_path).unwrap_or_default();
 
@@ -1074,7 +1076,6 @@ pub fn run_estree_typescript_tokens(files: &[TypeScriptFile]) -> Vec<CoverageRes
 
                 let ParserReturn { mut program, tokens, .. } = ret;
                 let span_converter = Utf8ToUtf16::new(source_text);
-                span_converter.convert_program_with_ascending_order_checks(&mut program);
 
                 let oxc_tokens_json = to_estree_tokens_pretty_json(
                     &tokens,
@@ -1085,6 +1086,8 @@ pub fn run_estree_typescript_tokens(files: &[TypeScriptFile]) -> Vec<CoverageRes
                     &allocator,
                 );
 
+                span_converter.convert_program_with_ascending_order_checks(&mut program);
+
                 if oxc_tokens_json != *expected_tokens {
                     return CoverageResult {
                         path: f.path.clone(),