feat(napi/oxlint): pass AST in buffer to JS

.github/generated/ast_changes_watch_list.yml

@@ -65,6 +65,8 @@ src:
   - 'crates/oxc_syntax/src/serialize.rs'
   - 'crates/oxc_syntax/src/symbol.rs'
   - 'crates/oxc_traverse/src/generated/scopes_collector.rs'
+  - 'napi/oxlint2/src/generated/constants.cjs'
+  - 'napi/oxlint2/src/generated/raw_transfer_constants.rs'
   - 'napi/parser/generated/constants.js'
   - 'napi/parser/generated/deserialize/js.js'
   - 'napi/parser/generated/deserialize/ts.js'

crates/oxc_allocator/src/lib.rs

@@ -88,9 +88,58 @@ use pool_fixed_size as pool;
     target_endian = "little"
 ))]
 use pool_fixed_size::FixedSizeAllocatorMetadata;
+// Export so can be used in `napi/oxlint2`
+#[cfg(all(
+    feature = "fixed_size",
+    not(feature = "disable_fixed_size"),
+    target_pointer_width = "64",
+    target_endian = "little"
+))]
+pub use pool_fixed_size::free_fixed_size_allocator;
 
 pub use pool::{AllocatorGuard, AllocatorPool};
 
+// Dummy implementations of interfaces from `pool_fixed_size`, just to stop clippy complaining.
+// Seems to be necessary due to feature unification.
+#[cfg(not(all(
+    feature = "fixed_size",
+    not(feature = "disable_fixed_size"),
+    target_pointer_width = "64",
+    target_endian = "little"
+)))]
+#[allow(missing_docs, clippy::missing_safety_doc, clippy::unused_self, clippy::allow_attributes)]
+mod dummies {
+    use std::{ptr::NonNull, sync::atomic::AtomicBool};
+
+    use super::Allocator;
+
+    #[doc(hidden)]
+    pub struct FixedSizeAllocatorMetadata {
+        pub id: u32,
+        pub alloc_ptr: NonNull<u8>,
+        pub is_double_owned: AtomicBool,
+    }
+
+    #[doc(hidden)]
+    pub unsafe fn free_fixed_size_allocator(_metadata_ptr: NonNull<FixedSizeAllocatorMetadata>) {
+        unreachable!();
+    }
+
+    #[doc(hidden)]
+    impl Allocator {
+        pub unsafe fn fixed_size_metadata_ptr(&self) -> NonNull<FixedSizeAllocatorMetadata> {
+            unreachable!();
+        }
+    }
+}
+#[cfg(not(all(
+    feature = "fixed_size",
+    not(feature = "disable_fixed_size"),
+    target_pointer_width = "64",
+    target_endian = "little"
+)))]
+pub use dummies::*;
+
 #[cfg(all(
     feature = "fixed_size",
     not(feature = "disable_fixed_size"),

crates/oxc_allocator/src/pool_fixed_size.rs

@@ -142,7 +142,8 @@ pub struct FixedSizeAllocatorMetadata {
 const ALLOC_SIZE: usize = BLOCK_SIZE + TWO_GIB;
 const ALLOC_ALIGN: usize = TWO_GIB;
 
-const ALLOC_LAYOUT: Layout = match Layout::from_size_align(ALLOC_SIZE, ALLOC_ALIGN) {
+/// Layout of backing allocations for fixed-size allocators.
+pub const ALLOC_LAYOUT: Layout = match Layout::from_size_align(ALLOC_SIZE, ALLOC_ALIGN) {
     Ok(layout) => layout,
     Err(_) => unreachable!(),
 };
@@ -287,7 +288,7 @@ impl Drop for FixedSizeAllocator {
     }
 }
 
-/// Deallocate memory backing a [`FixedSizeAllocator`] if it's not double-owned
+/// Deallocate memory backing a `FixedSizeAllocator` if it's not double-owned
 /// (both owned by a `FixedSizeAllocator` on Rust side *and* held as a buffer on JS side).
 ///
 /// If it is double-owned, don't deallocate the memory but set the flag that it's no longer double-owned
@@ -302,7 +303,7 @@ impl Drop for FixedSizeAllocator {
 /// Calling this function in any other circumstances would result in a double-free.
 ///
 /// `metadata_ptr` must point to a valid `FixedSizeAllocatorMetadata`.
-unsafe fn free_fixed_size_allocator(metadata_ptr: NonNull<FixedSizeAllocatorMetadata>) {
+pub unsafe fn free_fixed_size_allocator(metadata_ptr: NonNull<FixedSizeAllocatorMetadata>) {
     // Get pointer to start of original allocation from `FixedSizeAllocatorMetadata`
     let alloc_ptr = {
         // SAFETY: This `Allocator` was created by the `FixedSizeAllocator`.
@@ -341,13 +342,13 @@ unsafe fn free_fixed_size_allocator(metadata_ptr: NonNull<FixedSizeAllocatorMeta
 unsafe impl Send for FixedSizeAllocator {}
 
 impl Allocator {
-    /// Get pointer to the [`FixedSizeAllocatorMetadata`] for this [`Allocator`].
+    /// Get pointer to the `FixedSizeAllocatorMetadata` for this [`Allocator`].
     ///
     /// # SAFETY
     /// * This `Allocator` must have been created by a `FixedSizeAllocator`.
     /// * This pointer must not be used to create a mutable reference to the `FixedSizeAllocatorMetadata`,
     ///   only immutable references.
-    unsafe fn fixed_size_metadata_ptr(&self) -> NonNull<FixedSizeAllocatorMetadata> {
+    pub unsafe fn fixed_size_metadata_ptr(&self) -> NonNull<FixedSizeAllocatorMetadata> {
         // SAFETY: Caller guarantees this `Allocator` was created by a `FixedSizeAllocator`.
         //
         // `FixedSizeAllocator::new` writes `FixedSizeAllocatorMetadata` after the end of

crates/oxc_linter/src/external_linter.rs

@@ -2,6 +2,8 @@ use std::{fmt::Debug, pin::Pin, sync::Arc};
 
 use serde::{Deserialize, Serialize};
 
+use oxc_allocator::Allocator;
+
 pub type ExternalLinterLoadPluginCb = Arc<
     dyn Fn(
             String,
@@ -17,7 +19,11 @@ pub type ExternalLinterLoadPluginCb = Arc<
 >;
 
 pub type ExternalLinterCb = Arc<
-    dyn Fn(String, Vec<u32>) -> Result<Vec<LintResult>, Box<dyn std::error::Error + Send + Sync>>
+    dyn Fn(
+            String,
+            Vec<u32>,
+            &Allocator,
+        ) -> Result<Vec<LintResult>, Box<dyn std::error::Error + Send + Sync>>
         + Sync
         + Send,
 >;

crates/oxc_linter/src/lib.rs

@@ -213,7 +213,7 @@ impl Linter {
         }
 
         #[cfg(all(feature = "oxlint2", not(feature = "disable_oxlint2")))]
-        self.run_external_rules(&external_rules, path, &ctx_host, allocator);
+        self.run_external_rules(&external_rules, path, semantic, &ctx_host, allocator);
 
         // Stop clippy complaining about unused vars
         #[cfg(not(all(feature = "oxlint2", not(feature = "disable_oxlint2"))))]
@@ -233,9 +233,12 @@ impl Linter {
         &self,
         external_rules: &[(ExternalRuleId, AllowWarnDeny)],
         path: &Path,
+        semantic: &Semantic<'_>,
         ctx_host: &ContextHost,
-        _allocator: &Allocator,
+        allocator: &Allocator,
     ) {
+        use std::ptr;
+
         use oxc_diagnostics::OxcDiagnostic;
         use oxc_span::Span;
 
@@ -248,9 +251,23 @@ impl Linter {
         // `external_linter` always exists when `oxlint2` feature is enabled
         let external_linter = self.external_linter.as_ref().unwrap();
 
+        // Write offset of `Program` and source text length in metadata at end of buffer
+        let program = semantic.nodes().program().unwrap();
+        let program_offset = ptr::from_ref(program) as u32;
+        #[expect(clippy::cast_possible_truncation)]
+        let source_len = program.source_text.len() as u32;
+
+        let metadata = RawTransferMetadata::new(program_offset, source_len);
+        let metadata_ptr = allocator.end_ptr().cast::<RawTransferMetadata>();
+        // SAFETY: `Allocator` was created by `FixedSizeAllocator` which reserved space after `end_ptr`
+        // for a `RawTransferMetadata`. `end_ptr` is aligned for `FixedSizeAllocator`.
+        unsafe { metadata_ptr.write(metadata) };
+
+        // Pass AST and rule IDs to JS
         let result = (external_linter.run)(
             path.to_str().unwrap().to_string(),
             external_rules.iter().map(|(rule_id, _)| rule_id.raw()).collect(),
+            allocator,
         );
         match result {
             Ok(diagnostics) => {
@@ -304,10 +321,9 @@ struct RawTransferMetadata2 {
 use RawTransferMetadata2 as RawTransferMetadata;
 
 #[cfg(all(feature = "oxlint2", not(feature = "disable_oxlint2")))]
-#[expect(dead_code)]
 impl RawTransferMetadata {
-    pub fn new(data_offset: u32, is_ts: bool) -> Self {
-        Self { data_offset, is_ts, source_len: 0, _padding: 0 }
+    pub fn new(data_offset: u32, source_len: u32) -> Self {
+        Self { data_offset, is_ts: false, source_len, _padding: 0 }
     }
 }
 

napi/oxlint2/Cargo.toml

@@ -22,6 +22,7 @@ test = false
 doctest = false
 
 [dependencies]
+oxc_allocator = { workspace = true, features = ["fixed_size"] }
 oxlint = { workspace = true, features = ["oxlint2", "allocator"] }
 
 napi = { workspace = true, features = ["async"] }

napi/oxlint2/src/bindings.d.ts

@@ -4,6 +4,6 @@ export type JsLoadPluginCb =
   ((arg: string) => Promise<string>)
 
 export type JsRunCb =
-  ((arg0: string, arg1: Array<number>) => string)
+  ((arg0: string, arg1: number, arg2: Uint8Array | undefined | null, arg3: Array<number>) => string)
 
 export declare function lint(loadPlugin: JsLoadPluginCb, run: JsRunCb): Promise<boolean>

napi/oxlint2/src/generated/constants.cjs

@@ -0,0 +1,18 @@
+// Auto-generated code, DO NOT EDIT DIRECTLY!
+// To edit this generated file you have to edit `tasks/ast_tools/src/generators/raw_transfer.rs`.
+
+const BUFFER_SIZE = 2147483616,
+  BUFFER_ALIGN = 4294967296,
+  DATA_POINTER_POS_32 = 536870900,
+  IS_TS_FLAG_POS = 2147483612,
+  SOURCE_LEN_POS_32 = 536870901,
+  PROGRAM_OFFSET = 0;
+
+module.exports = {
+  BUFFER_SIZE,
+  BUFFER_ALIGN,
+  DATA_POINTER_POS_32,
+  IS_TS_FLAG_POS,
+  SOURCE_LEN_POS_32,
+  PROGRAM_OFFSET,
+};

napi/oxlint2/src/generated/raw_transfer_constants.rs

@@ -0,0 +1,10 @@
+// Auto-generated code, DO NOT EDIT DIRECTLY!
+// To edit this generated file you have to edit `tasks/ast_tools/src/generators/raw_transfer.rs`.
+
+#![expect(clippy::unreadable_literal)]
+#![allow(dead_code)]
+
+pub const BLOCK_SIZE: usize = 2147483632;
+pub const BLOCK_ALIGN: usize = 4294967296;
+pub const BUFFER_SIZE: usize = 2147483616;
+pub const RAW_METADATA_SIZE: usize = 16;

napi/oxlint2/src/index.js

@@ -1,4 +1,17 @@
+import { createRequire } from 'node:module';
 import { lint } from './bindings.js';
+import { DATA_POINTER_POS_32, SOURCE_LEN_POS_32 } from './generated/constants.cjs';
+
+// Import lazy visitor from `oxc-parser`.
+// Use `require` not `import` as `oxc-parser` uses `require` internally,
+// and need to make sure get same instance of modules as it uses internally,
+// otherwise `TOKEN` here won't be same `TOKEN` as used within `oxc-parser`.
+const require = createRequire(import.meta.url);
+const { TOKEN } = require('../../parser/raw-transfer/lazy-common.js'),
+  { Visitor, getVisitorsArr } = require('../../parser/raw-transfer/visitor.js'),
+  walkProgram = require('../../parser/generated/lazy/walk.js');
+
+const textDecoder = new TextDecoder('utf-8', { ignoreBOM: true });
 
 class PluginRegistry {
   registeredPluginPaths = new Set();
@@ -33,6 +46,9 @@ class PluginRegistry {
   }
 }
 
+// Buffers cache
+const buffers = [];
+
 class Linter {
   pluginRegistry = new PluginRegistry();
 
@@ -63,15 +79,34 @@ class Linter {
   }
 
   // TODO(camc314): why do we have to destructure here?
-  // In `./bindings.d.ts`, it doesn't indicate that we have to (typed as `(filePath: string, ruleIds: number[]))`
-  lint([filePath, ruleIds]) {
+  // In `./bindings.d.ts`, it doesn't indicate that we have to
+  // (typed as `(filePath: string, bufferId: number, buffer: Uint8Array | undefined | null, ruleIds: number[])`).
+  lint([filePath, bufferId, buffer, ruleIds]) {
+    // If new buffer, add it to `buffers` array. Otherwise, get existing buffer from array.
+    // Do this before checks below, to make sure buffer doesn't get garbage collected when not expected
+    // if there's an error.
+    // TODO: Is this enough to guarantee soundness?
+    if (buffer !== null) {
+      const { buffer: arrayBuffer, byteOffset } = buffer;
+      buffer.uint32 = new Uint32Array(arrayBuffer, byteOffset);
+      buffer.float64 = new Float64Array(arrayBuffer, byteOffset);
+
+      while (buffers.length <= bufferId) {
+        buffers.push(null);
+      }
+      buffers[bufferId] = buffer;
+    } else {
+      buffer = buffers[bufferId];
+    }
+
     if (typeof filePath !== 'string' || filePath.length === 0) {
       throw new Error('expected filePath to be a non-zero length string');
     }
     if (!Array.isArray(ruleIds) || ruleIds.length === 0) {
       throw new Error('Expected `ruleIds` to be a non-zero len array');
     }
 
+    // Get visitors for this file from all rules
     const diagnostics = [];
 
     const createContext = (ruleId) => ({
@@ -85,13 +120,25 @@ class Linter {
       },
     });
 
-    const rules = [];
+    const visitors = [];
     for (const { rule, ruleId } of this.pluginRegistry.getRules(ruleIds)) {
-      rules.push(rule.create(createContext(ruleId)));
+      visitors.push(rule.create(createContext(ruleId)));
     }
 
-    // TODO: walk the AST
+    // TODO: Combine visitors for multiple rules
+    const visitor = new Visitor(visitors[0]);
+
+    // Visit AST
+    const programPos = buffer.uint32[DATA_POINTER_POS_32],
+      sourceByteLen = buffer.uint32[SOURCE_LEN_POS_32];
+
+    const sourceText = textDecoder.decode(buffer.subarray(0, sourceByteLen));
+    const sourceIsAscii = sourceText.length === sourceByteLen;
+    const ast = { buffer, sourceText, sourceByteLen, sourceIsAscii, nodes: new Map(), token: TOKEN };
+
+    walkProgram(programPos, ast, getVisitorsArr(visitor));
 
+    // Send diagnostics back to Rust
     return JSON.stringify(diagnostics);
   }
 }