From 5ca0d9a385b2eeaeddcfa09a83e5f4a5b43e054b Mon Sep 17 00:00:00 2001
From: overlookmotel <557937+overlookmotel@users.noreply.github.com>
Date: Tue, 24 Feb 2026 11:31:05 +0000
Subject: [PATCH] refactor(linter/plugins): combine functions for getting
 tokens (#19529)

Refactor. Logic related to tokens was spread across 2 files. Move it all into `tokens.ts`.

Also, fix one small bug where there'd be an error when linting an empty file.
---
 apps/oxlint/src-js/plugins/source_code.ts | 18 +--------
 apps/oxlint/src-js/plugins/tokens.ts      | 45 +++++++++++++----------
 2 files changed, 27 insertions(+), 36 deletions(-)

diff --git a/apps/oxlint/src-js/plugins/source_code.ts b/apps/oxlint/src-js/plugins/source_code.ts
index 2754d1b2b1f79..b8575cc469252 100644
--- a/apps/oxlint/src-js/plugins/source_code.ts
+++ b/apps/oxlint/src-js/plugins/source_code.ts
@@ -4,8 +4,6 @@ import {
   SOURCE_LEN_OFFSET,
   IS_JSX_FLAG_POS,
   IS_TS_FLAG_POS,
-  TOKENS_OFFSET_POS_32,
-  TOKENS_LEN_POS_32,
 } from "../generated/constants.ts";
 
 // We use the deserializer which removes `ParenthesizedExpression`s from AST,
@@ -31,10 +29,10 @@ import type { BufferWithArrays, Comment, Node } from "./types.ts";
 import type { ScopeManager } from "./scope.ts";
 
 // Text decoder, for decoding source text from buffer
-const textDecoder = new TextDecoder("utf-8", { ignoreBOM: true });
+export const textDecoder = new TextDecoder("utf-8", { ignoreBOM: true });
 
 // Buffer containing AST. Set before linting a file by `setupSourceForFile`.
-let buffer: BufferWithArrays | null = null;
+export let buffer: BufferWithArrays | null = null;
 
 // Indicates if the original source text has a BOM. Set before linting a file by `setupSourceForFile`.
 let hasBOM = false;
@@ -161,18 +159,6 @@ export function fileIsTs(): boolean {
   return buffer[IS_TS_FLAG_POS] === 1;
 }
 
-/**
- * Get serialized ESTree tokens JSON from buffer, if present.
- */
-export function getSerializedTokensJSON(): string | null {
-  debugAssertIsNonNull(buffer);
-  const tokensLen = buffer.uint32[TOKENS_LEN_POS_32];
-  if (tokensLen === 0) return null;
-
-  const tokensOffset = buffer.uint32[TOKENS_OFFSET_POS_32];
-  return textDecoder.decode(buffer.subarray(tokensOffset, tokensOffset + tokensLen));
-}
-
 // `SourceCode` object.
 //
 // Only one file is linted at a time, so we can reuse a single object for all files.
diff --git a/apps/oxlint/src-js/plugins/tokens.ts b/apps/oxlint/src-js/plugins/tokens.ts
index 86d2f57bad32e..407982dd48a09 100644
--- a/apps/oxlint/src-js/plugins/tokens.ts
+++ b/apps/oxlint/src-js/plugins/tokens.ts
@@ -3,8 +3,9 @@
  */
 
 import { ast, initAst } from "./source_code.ts";
-import { getSerializedTokensJSON } from "./source_code.ts";
+import { buffer, textDecoder } from "./source_code.ts";
 import { getNodeLoc } from "./location.ts";
+import { TOKENS_OFFSET_POS_32, TOKENS_LEN_POS_32 } from "../generated/constants.ts";
 import { debugAssert, debugAssertIsNonNull } from "../utils/asserts.ts";
 
 import type { Comment, Node, NodeOrToken } from "./types.ts";
@@ -149,35 +150,39 @@ export let tokensAndComments: TokenOrComment[] | null = null;
  * Caller must ensure `filePath` and `sourceText` are initialized before calling this function.
  */
 export function initTokens() {
-  const serializedTokens = getSerializedTokensJSON();
-  debugAssertIsNonNull(
-    serializedTokens,
-    "Serialized tokens should be provided by Rust parser transfer",
-  );
-  tokens = parseSerializedTokens(serializedTokens);
+  debugAssert(tokens === null, "Tokens already initialized");
 
-  // Check `tokens` have valid ranges and are in ascending order
-  debugCheckValidRanges(tokens, "token");
-}
+  // Get tokens JSON from buffer, and deserialize it
+  debugAssertIsNonNull(buffer);
 
-/**
- * Deserialize serialized ESTree tokens from Rust and hydrate ESLint-compatible token fields.
- */
-function parseSerializedTokens(serializedTokens: string): Token[] {
-  const parsed = JSON.parse(serializedTokens) as Token[];
-  for (const token of parsed) {
+  const { uint32 } = buffer;
+  const tokensJsonLen = uint32[TOKENS_LEN_POS_32];
+  if (tokensJsonLen === 0) {
+    tokens = [];
+    return;
+  }
+
+  const tokensJsonOffset = uint32[TOKENS_OFFSET_POS_32];
+  const tokensJson = textDecoder.decode(
+    buffer.subarray(tokensJsonOffset, tokensJsonOffset + tokensJsonLen),
+  );
+  tokens = JSON.parse(tokensJson) as Token[];
+
+  // Add `range` property to each token, and set prototype of each to `TokenProto` which provides getter for `loc`
+  for (const token of tokens) {
     const { start, end } = token;
     debugAssert(
       typeof start === "number" && typeof end === "number",
       "Precomputed tokens should include `start` and `end`",
     );
 
-    // `TokenProto` provides getter for `loc`
-    // @ts-expect-error - TS doesn't understand `__proto__`
-    token.__proto__ = TokenProto;
     token.range = [start, end];
+    // `TokenProto` provides getter for `loc`
+    Object.setPrototypeOf(token, TokenProto);
   }
-  return parsed;
+
+  // Check `tokens` have valid ranges and are in ascending order
+  debugCheckValidRanges(tokens, "token");
 }
 
 /**