perf(linter/plugins): collect tokens even when not needed

apps/oxlint/src/js_plugins/parse.rs

@@ -11,7 +11,7 @@ use oxc_ast_visit::utf8_to_utf16::Utf8ToUtf16;
 use oxc_estree_tokens::{ESTreeTokenOptionsJS, update_tokens};
 use oxc_linter::RawTransferMetadata2 as RawTransferMetadata;
 use oxc_napi::get_source_type;
-use oxc_parser::{ParseOptions, Parser, ParserReturn, config::RuntimeParserConfig};
+use oxc_parser::{ParseOptions, Parser, ParserReturn, config::TokensParserConfig};
 use oxc_semantic::SemanticBuilder;
 
 use crate::generated::raw_transfer_constants::{BLOCK_ALIGN as BUFFER_ALIGN, BUFFER_SIZE};
@@ -166,16 +166,14 @@ unsafe fn parse_raw_impl(
             str::from_utf8_unchecked(source_bytes)
         };
 
-        // Parse with same options as linter.
-        // We use `RuntimeParserConfig` even though we always pass `true` here, to avoid compiling the parser twice.
-        // The linter itself uses `RuntimeParserConfig`.
+        // Parse with same options as linter
         let parser_ret = Parser::new(&allocator, source_text, source_type)
             .with_options(ParseOptions {
                 parse_regular_expression: true,
                 allow_return_outside_function: true,
                 ..ParseOptions::default()
             })
-            .with_config(RuntimeParserConfig::new(true))
+            .with_config(TokensParserConfig)
             .parse();
         let ParserReturn { program: parsed_program, errors, mut tokens, panicked, .. } = parser_ret;
         let program = allocator.alloc(parsed_program);

crates/oxc_linter/src/service/runtime.rs

@@ -21,7 +21,7 @@ use smallvec::SmallVec;
 
 use oxc_allocator::{Allocator, AllocatorGuard, AllocatorPool, Box as ArenaBox};
 use oxc_diagnostics::{DiagnosticSender, DiagnosticService, Error, OxcDiagnostic};
-use oxc_parser::{ParseOptions, Parser, Token, config::RuntimeParserConfig};
+use oxc_parser::{ParseOptions, Parser, Token, config::TokensParserConfig};
 use oxc_resolver::Resolver;
 use oxc_semantic::{Semantic, SemanticBuilder};
 use oxc_span::{CompactStr, SourceType, VALID_EXTENSIONS};
@@ -1028,14 +1028,16 @@ impl Runtime {
         check_syntax_errors: bool,
     ) -> Result<(ResolvedModuleRecord, Semantic<'a>, ArenaBox<'a, [Token]>), Vec<OxcDiagnostic>>
     {
-        let collect_tokens = self.linter.has_external_linter();
+        // Always collect tokens, even if we don't need them (files where no JS plugins are enabled).
+        // Removing the "are tokens enabled?" branch by using `TokensParserConfig` static config
+        // is a bigger perf win than skipping storing the tokens.
         let ret = Parser::new(allocator, source_text, source_type)
             .with_options(ParseOptions {
                 parse_regular_expression: true,
                 allow_return_outside_function: true,
                 ..ParseOptions::default()
             })
-            .with_config(RuntimeParserConfig::new(collect_tokens))
+            .with_config(TokensParserConfig)
             .parse();
 
         if !ret.errors.is_empty() {

tasks/benchmark/benches/parser.rs

@@ -2,7 +2,7 @@ use oxc_allocator::Allocator;
 use oxc_ast_visit::utf8_to_utf16::Utf8ToUtf16;
 use oxc_benchmark::{BenchmarkId, Criterion, black_box, criterion_group, criterion_main};
 use oxc_estree_tokens::{ESTreeTokenOptionsJS, to_estree_tokens_json, update_tokens};
-use oxc_parser::{ParseOptions, Parser, ParserReturn, config::RuntimeParserConfig};
+use oxc_parser::{ParseOptions, Parser, ParserReturn, config::TokensParserConfig};
 use oxc_tasks_common::TestFiles;
 
 fn bench_parser(criterion: &mut Criterion) {
@@ -49,18 +49,16 @@ fn bench_parser_tokens(criterion: &mut Criterion) {
             let mut allocator = Allocator::default();
 
             b.iter(|| {
-                // Use `RuntimeParserConfig` (runtime config), same as NAPI parser package will.
+                // Use `TokensParserConfig` (tokens enabled), same as linter does.
                 // `bench_parser` uses `NoTokensParserConfig` (implicitly as default).
                 // Usually it's inadvisable to use 2 different configs in the same application,
                 // but this is just a benchmark, and it's better if we don't entwine this benchmark with `bench_parser`.
-                let config = RuntimeParserConfig::new(true);
-
                 Parser::new(&allocator, source_text, source_type)
                     .with_options(ParseOptions {
                         parse_regular_expression: true,
                         ..ParseOptions::default()
                     })
-                    .with_config(config)
+                    .with_config(TokensParserConfig)
                     .parse();
 
                 allocator.reset();
@@ -120,18 +118,16 @@ fn bench_estree_tokens(criterion: &mut Criterion) {
             b.iter_with_setup_wrapper(|runner| {
                 allocator.reset();
 
-                // Use `RuntimeParserConfig` (runtime config), same as NAPI parser package will.
+                // Use `TokensParserConfig` (tokens enabled), same as linter does.
                 // `bench_estree` uses `NoTokensParserConfig` (implicitly as default).
                 // Usually it's inadvisable to use 2 different configs in the same application,
                 // but this is just a benchmark, and it's better if we don't entwine this benchmark with `bench_estree`.
-                let config = RuntimeParserConfig::new(true);
-
                 let ret = Parser::new(&allocator, source_text, source_type)
                     .with_options(ParseOptions {
                         parse_regular_expression: true,
                         ..ParseOptions::default()
                     })
-                    .with_config(config)
+                    .with_config(TokensParserConfig)
                     .parse();
                 let ParserReturn { program, tokens, .. } = ret;
 
@@ -173,18 +169,16 @@ fn bench_estree_tokens_raw(criterion: &mut Criterion) {
             b.iter_with_setup_wrapper(|runner| {
                 allocator.reset();
 
-                // Use `RuntimeParserConfig` (runtime config), same as NAPI parser package will.
+                // Use `TokensParserConfig` (tokens enabled), same as linter does.
                 // `bench_estree` uses `NoTokensParserConfig` (implicitly as default).
                 // Usually it's inadvisable to use 2 different configs in the same application,
                 // but this is just a benchmark, and it's better if we don't entwine this benchmark with `bench_estree`.
-                let config = RuntimeParserConfig::new(true);
-
                 let ret = Parser::new(&allocator, source_text, source_type)
                     .with_options(ParseOptions {
                         parse_regular_expression: true,
                         ..ParseOptions::default()
                     })
-                    .with_config(config)
+                    .with_config(TokensParserConfig)
                     .parse();
                 let ParserReturn { program, mut tokens, .. } = ret;