refactor(napi/parser): raw transfer: store offsets as consts

.github/generated/ast_changes_watch_list.yml

@@ -60,6 +60,7 @@ src:
   - 'crates/oxc_syntax/src/serialize.rs'
   - 'crates/oxc_syntax/src/symbol.rs'
   - 'crates/oxc_traverse/src/generated/scopes_collector.rs'
+  - 'napi/parser/generated/constants.js'
   - 'napi/parser/generated/deserialize/js.js'
   - 'napi/parser/generated/deserialize/ts.js'
   - 'napi/parser/generated/lazy/constructors.js'

napi/parser/bench.bench.mjs

@@ -8,6 +8,7 @@ import { experimentalGetLazyVisitor, parseAsync, parseSync } from './index.js';
 // Use `require` not `import` to load these internal modules, to avoid evaluating the modules
 // twice as ESM and CJS
 const require = createRequire(import.meta.filename);
+const { DATA_POINTER_POS_32, PROGRAM_OFFSET } = require('./generated/constants.js');
 const deserializeJS = require('./generated/deserialize/js.js');
 const deserializeTS = require('./generated/deserialize/ts.js');
 const { isJsAst, prepareRaw, returnBufferToCache } = require('./raw-transfer/common.js');
@@ -180,9 +181,7 @@ for (const { filename, code } of fixtures) {
       token: TOKEN,
     };
 
-    // (2 * 1024 * 1024 * 1024 - 16) >> 2
-    const metadataPos32 = 536870908;
-    const programPos = buffer.uint32[metadataPos32];
+    const programPos = buffer.uint32[DATA_POINTER_POS_32] + PROGRAM_OFFSET;
 
     benchRaw('parser_napi_raw_lazy_visit_only(debugger)', () => {
       ast.nodes = new Map();

napi/parser/generated/constants.js

@@ -0,0 +1,8 @@
+// Auto-generated code, DO NOT EDIT DIRECTLY!
+// To edit this generated file you have to edit `tasks/ast_tools/src/generators/raw_transfer.rs`.
+
+const DATA_POINTER_POS_32 = 536870908,
+  IS_TS_FLAG_POS = 2147483636,
+  PROGRAM_OFFSET = 0;
+
+module.exports = { DATA_POINTER_POS_32, IS_TS_FLAG_POS, PROGRAM_OFFSET };

napi/parser/generated/deserialize/js.js

@@ -20,10 +20,7 @@ function deserialize(buffer, sourceTextInput, sourceLenInput) {
   sourceLen = sourceLenInput;
   sourceIsAscii = sourceText.length === sourceLen;
 
-  // (2 * 1024 * 1024 * 1024 - 16) >> 2
-  const metadataPos32 = 536870908;
-
-  const data = deserializeRawTransferData(uint32[metadataPos32]);
+  const data = deserializeRawTransferData(uint32[536870908]);
 
   uint8 =
     uint32 =

napi/parser/generated/deserialize/ts.js

@@ -20,10 +20,7 @@ function deserialize(buffer, sourceTextInput, sourceLenInput) {
   sourceLen = sourceLenInput;
   sourceIsAscii = sourceText.length === sourceLen;
 
-  // (2 * 1024 * 1024 * 1024 - 16) >> 2
-  const metadataPos32 = 536870908;
-
-  const data = deserializeRawTransferData(uint32[metadataPos32]);
+  const data = deserializeRawTransferData(uint32[536870908]);
 
   uint8 =
     uint32 =

napi/parser/package.json

@@ -44,6 +44,7 @@
     "wasm.mjs",
     "bindings.js",
     "webcontainer-fallback.js",
+    "generated/constants.js",
     "generated/deserialize/js.js",
     "generated/deserialize/ts.js",
     "generated/lazy/constructors.js",

napi/parser/raw-transfer/common.js

@@ -7,6 +7,7 @@ const {
   parseAsyncRaw: parseAsyncRawBinding,
   getBufferOffset,
 } = require('../bindings.js');
+const { IS_TS_FLAG_POS } = require('../generated/constants.js');
 
 module.exports = {
   parseSyncRawImpl,
@@ -227,9 +228,7 @@ function prepareRaw(sourceText) {
  * @returns {boolean} - `true` if AST is JS, `false` if TS
  */
 function isJsAst(buffer) {
-  // 2147483636 = (2 * 1024 * 1024 * 1024) - 12
-  // i.e. 12 bytes from end of 2 GiB buffer
-  return buffer[2147483636] === 0;
+  return buffer[IS_TS_FLAG_POS] === 0;
 }
 
 /**

napi/parser/raw-transfer/lazy.js

@@ -2,6 +2,7 @@
 
 const { parseSyncRawImpl, parseAsyncRawImpl, returnBufferToCache } = require('./common.js'),
   { TOKEN } = require('./lazy-common.js'),
+  { DATA_POINTER_POS_32, PROGRAM_OFFSET } = require('../generated/constants.js'),
   { RawTransferData } = require('../generated/lazy/constructors.js'),
   walkProgram = require('../generated/lazy/walk.js'),
   { Visitor, getVisitorsArr } = require('./visitor.js');
@@ -102,9 +103,7 @@ function construct(buffer, sourceText, sourceLen) {
   bufferRecycleRegistry.register(ast, buffer, ast);
 
   // Get root data class instance
-  // (2 * 1024 * 1024 * 1024 - 16) >> 2
-  const metadataPos32 = 536870908;
-  const rawDataPos = buffer.uint32[metadataPos32];
+  const rawDataPos = buffer.uint32[DATA_POINTER_POS_32];
   const data = new RawTransferData(rawDataPos, ast);
 
   return {
@@ -122,7 +121,7 @@ function construct(buffer, sourceText, sourceLen) {
     },
     dispose: dispose.bind(null, ast),
     visit(visitor) {
-      walkProgram(rawDataPos, ast, getVisitorsArr(visitor));
+      walkProgram(rawDataPos + PROGRAM_OFFSET, ast, getVisitorsArr(visitor));
     },
   };
 }

tasks/ast_tools/src/generators/raw_transfer.rs

@@ -24,6 +24,13 @@ use crate::{
 
 use super::define_generator;
 
+/// Size of raw transfer buffer
+const BUFFER_SIZE: u32 = 1 << 31; // 2 GiB
+/// Size of metadata written to end of buffer.
+const METADATA_SIZE: u32 = 16;
+/// Offset of flag for whether AST is TypeScript or not, relative to start of metadata.
+const IS_TS_FLAG_OFFSET: u32 = 4;
+
 // Offsets of `Vec`'s fields.
 // `Vec` is `#[repr(transparent)]` and `RawVec` is `#[repr(C)]`, so these offsets are fixed.
 pub(super) const VEC_PTR_FIELD_OFFSET: usize = 0;
@@ -36,7 +43,11 @@ define_generator!(RawTransferGenerator);
 
 impl Generator for RawTransferGenerator {
     fn generate_many(&self, schema: &Schema, codegen: &Codegen) -> Vec<Output> {
-        let Codes { js, ts, .. } = generate_deserializers(schema, codegen);
+        let consts = get_constants(schema);
+
+        let Codes { js, ts, .. } = generate_deserializers(consts, schema, codegen);
+        let constants = generate_constants(consts);
+
         vec![
             Output::Javascript {
                 path: format!("{NAPI_PARSER_PACKAGE_PATH}/generated/deserialize/js.js"),
@@ -46,43 +57,14 @@ impl Generator for RawTransferGenerator {
                 path: format!("{NAPI_PARSER_PACKAGE_PATH}/generated/deserialize/ts.js"),
                 code: ts,
             },
+            Output::Javascript {
+                path: format!("{NAPI_PARSER_PACKAGE_PATH}/generated/constants.js"),
+                code: constants,
+            },
         ]
     }
 }
 
-/// Prelude to generated deserializer.
-/// Defines the main `deserialize` function.
-static PRELUDE: &str = "
-    'use strict';
-
-    module.exports = deserialize;
-
-    let uint8, uint32, float64, sourceText, sourceIsAscii, sourceLen;
-
-    const textDecoder = new TextDecoder('utf-8', { ignoreBOM: true }),
-        decodeStr = textDecoder.decode.bind(textDecoder),
-        { fromCodePoint } = String;
-
-    function deserialize(buffer, sourceTextInput, sourceLenInput) {
-        uint8 = buffer;
-        uint32 = buffer.uint32;
-        float64 = buffer.float64;
-
-        sourceText = sourceTextInput;
-        sourceLen = sourceLenInput;
-        sourceIsAscii = sourceText.length === sourceLen;
-
-        // (2 * 1024 * 1024 * 1024 - 16) >> 2
-        const metadataPos32 = 536870908;
-
-        const data = deserializeRawTransferData(uint32[metadataPos32]);
-
-        uint8 = uint32 = float64 = sourceText = undefined;
-
-        return data;
-    }
-";
-
 /// Container for generated code.
 struct Codes {
     /// Code which is part of JS deserializer only
@@ -94,10 +76,43 @@ struct Codes {
 }
 
 /// Generate deserializer functions for all types.
-fn generate_deserializers(schema: &Schema, codegen: &Codegen) -> Codes {
+fn generate_deserializers(consts: Constants, schema: &Schema, codegen: &Codegen) -> Codes {
     let estree_derive_id = codegen.get_derive_id_by_name("ESTree");
 
-    let mut codes = Codes { js: PRELUDE.to_string(), ts: PRELUDE.to_string(), both: String::new() };
+    // Prelude to generated deserializer.
+    // Defines the main `deserialize` function.
+    let Constants { data_pointer_pos_32, .. } = consts;
+
+    #[rustfmt::skip]
+    let prelude = format!("
+        'use strict';
+
+        module.exports = deserialize;
+
+        let uint8, uint32, float64, sourceText, sourceIsAscii, sourceLen;
+
+        const textDecoder = new TextDecoder('utf-8', {{ ignoreBOM: true }}),
+            decodeStr = textDecoder.decode.bind(textDecoder),
+            {{ fromCodePoint }} = String;
+
+        function deserialize(buffer, sourceTextInput, sourceLenInput) {{
+            uint8 = buffer;
+            uint32 = buffer.uint32;
+            float64 = buffer.float64;
+
+            sourceText = sourceTextInput;
+            sourceLen = sourceLenInput;
+            sourceIsAscii = sourceText.length === sourceLen;
+
+            const data = deserializeRawTransferData(uint32[{data_pointer_pos_32}]);
+
+            uint8 = uint32 = float64 = sourceText = undefined;
+
+            return data;
+        }}
+    ");
+
+    let mut codes = Codes { js: prelude.clone(), ts: prelude, both: String::new() };
 
     for type_def in &schema.types {
         match type_def {
@@ -946,3 +961,42 @@ impl DeserializeFunctionName for CellDef {
         self.inner_type(schema).plain_name(schema)
     }
 }
+
+/// Constants for position of fields in buffer which deserialization starts from.
+#[derive(Clone, Copy)]
+struct Constants {
+    data_pointer_pos_32: u32,
+    is_ts_pos: u32,
+    program_offset: u32,
+}
+
+/// Generate constants file.
+fn generate_constants(consts: Constants) -> String {
+    let Constants { data_pointer_pos_32, is_ts_pos, program_offset } = consts;
+
+    #[rustfmt::skip]
+    let output = format!("
+        const DATA_POINTER_POS_32 = {data_pointer_pos_32},
+            IS_TS_FLAG_POS = {is_ts_pos},
+            PROGRAM_OFFSET = {program_offset};
+
+        module.exports = {{ DATA_POINTER_POS_32, IS_TS_FLAG_POS, PROGRAM_OFFSET }};
+    ");
+    output
+}
+
+/// Calculate constants.
+fn get_constants(schema: &Schema) -> Constants {
+    let metadata_pos = BUFFER_SIZE - METADATA_SIZE;
+    let data_pointer_pos_32 = metadata_pos / 4;
+    let is_ts_pos = metadata_pos + IS_TS_FLAG_OFFSET;
+
+    let program_offset = schema
+        .type_by_name("RawTransferData")
+        .as_struct()
+        .unwrap()
+        .field_by_name("program")
+        .offset_64();
+
+    Constants { data_pointer_pos_32, is_ts_pos, program_offset }
+}