Skip to content

perf(napi/parser, linter/plugins): use utf8Slice for decoding UTF-8 strings #21022

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
graphite-app merged 1 commit into from
Apr 3, 2026
Merged

perf(napi/parser, linter/plugins): use utf8Slice for decoding UTF-8 strings #21022

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
18 changes: 8 additions & 10 deletions apps/oxlint/src-js/generated/deserialize.js
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -14,10 +14,8 @@ let uint8,
parent = null,
getLoc;

const textDecoder = new TextDecoder("utf-8", { ignoreBOM: true }),
decodeStr = textDecoder.decode.bind(textDecoder),
{ fromCharCode } = String,
{ latin1Slice } = Buffer.prototype,
const { fromCharCode } = String,
{ utf8Slice, latin1Slice } = Buffer.prototype,
stringDecodeArrays = Array(65).fill(null);
for (let i = 0; i <= 64; i++) stringDecodeArrays[i] = Array(i).fill(0);

Expand Down Expand Up @@ -5892,22 +5890,22 @@ function deserializeStr(pos) {
isInSourceRegion = pos >= sourceStartPos;
if (isInSourceRegion && end <= firstNonAsciiPos)
return sourceTextLatin.substr(pos - sourceStartPos, len);
// Use `TextDecoder` for strings longer than 64 bytes
if (len > 64) return decodeStr(uint8.subarray(pos, end));
// Use `utf8Slice` for strings longer than 64 bytes
if (len > 64) return utf8Slice.call(uint8, pos, end);
// If string is in source region, use slice of `sourceTextLatin` if all ASCII
if (isInSourceRegion) {
// Check if all bytes are ASCII, use `TextDecoder` if not
for (let i = pos; i < end; i++) if (uint8[i] >= 128) return decodeStr(uint8.subarray(pos, end));
// Check if all bytes are ASCII, use `utf8Slice` if not
for (let i = pos; i < end; i++) if (uint8[i] >= 128) return utf8Slice.call(uint8, pos, end);
// String is all ASCII, so slice from `sourceTextLatin`
return sourceTextLatin.substr(pos - sourceStartPos, len);
}
// String is not in source region - use `fromCharCode.apply` with a temp array of correct length.
// Copy bytes into temp array.
// If any byte is non-ASCII, use `TextDecoder`.
// If any byte is non-ASCII, use `utf8Slice`.
let arr = stringDecodeArrays[len];
for (let i = 0; i < len; i++) {
let b = uint8[pos + i];
if (b >= 128) return decodeStr(uint8.subarray(pos, end));
if (b >= 128) return utf8Slice.call(uint8, pos, end);
arr[i] = b;
}
// Call `fromCharCode` with temp array
Expand Down
5 changes: 2 additions & 3 deletions apps/oxlint/src-js/plugins/source_code.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -30,8 +30,7 @@ import type { ScopeManager } from "./scope.ts";
import type { Token } from "./tokens.ts";
import type { BufferWithArrays, Node } from "./types.ts";

// Text decoder, for decoding source text from buffer
const textDecoder = new TextDecoder("utf-8", { ignoreBOM: true });
const { utf8Slice } = Buffer.prototype;

// Buffer containing AST. Set before linting a file by `setupSourceForFile`.
export let buffer: BufferWithArrays | null = null;
Expand Down Expand Up @@ -65,7 +64,7 @@ export function initSourceText(): void {
programPos = uint32[DATA_POINTER_POS_32];
sourceStartPos = uint32[(programPos + SOURCE_START_OFFSET) >> 2];
sourceByteLen = uint32[(programPos + SOURCE_LEN_OFFSET) >> 2];
sourceText = textDecoder.decode(buffer.subarray(sourceStartPos, sourceStartPos + sourceByteLen));
sourceText = utf8Slice.call(buffer, sourceStartPos, sourceStartPos + sourceByteLen);
}

/**
Expand Down
18 changes: 8 additions & 10 deletions napi/parser/src-js/generated/deserialize/js.js
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -9,10 +9,8 @@ let uint8,
sourceEndPos = 0,
firstNonAsciiPos = 0;

const textDecoder = new TextDecoder("utf-8", { ignoreBOM: true }),
decodeStr = textDecoder.decode.bind(textDecoder),
{ fromCharCode } = String,
{ latin1Slice } = Buffer.prototype,
const { fromCharCode } = String,
{ utf8Slice, latin1Slice } = Buffer.prototype,
stringDecodeArrays = Array(65).fill(null);
for (let i = 0; i <= 64; i++) stringDecodeArrays[i] = Array(i).fill(0);

Expand Down Expand Up @@ -4557,21 +4555,21 @@ function deserializeStr(pos) {
pos = uint32[pos32];
let end = pos + len;
if (end <= firstNonAsciiPos) return sourceTextLatin.substr(pos, len);
// Use `TextDecoder` for strings longer than 64 bytes
if (len > 64) return decodeStr(uint8.subarray(pos, end));
// Use `utf8Slice` for strings longer than 64 bytes
if (len > 64) return utf8Slice.call(uint8, pos, end);
if (pos < sourceEndPos) {
// Check if all bytes are ASCII, use `TextDecoder` if not
for (let i = pos; i < end; i++) if (uint8[i] >= 128) return decodeStr(uint8.subarray(pos, end));
// Check if all bytes are ASCII, use `utf8Slice` if not
for (let i = pos; i < end; i++) if (uint8[i] >= 128) return utf8Slice.call(uint8, pos, end);
// String is all ASCII, so slice from `sourceTextLatin`
return sourceTextLatin.substr(pos, len);
}
// String is not in source region - use `fromCharCode.apply` with a temp array of correct length.
// Copy bytes into temp array.
// If any byte is non-ASCII, use `TextDecoder`.
// If any byte is non-ASCII, use `utf8Slice`.
let arr = stringDecodeArrays[len];
for (let i = 0; i < len; i++) {
let b = uint8[pos + i];
if (b >= 128) return decodeStr(uint8.subarray(pos, end));
if (b >= 128) return utf8Slice.call(uint8, pos, end);
arr[i] = b;
}
// Call `fromCharCode` with temp array
Expand Down
18 changes: 8 additions & 10 deletions napi/parser/src-js/generated/deserialize/js_parent.js
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -10,10 +10,8 @@ let uint8,
firstNonAsciiPos = 0,
parent = null;

const textDecoder = new TextDecoder("utf-8", { ignoreBOM: true }),
decodeStr = textDecoder.decode.bind(textDecoder),
{ fromCharCode } = String,
{ latin1Slice } = Buffer.prototype,
const { fromCharCode } = String,
{ utf8Slice, latin1Slice } = Buffer.prototype,
stringDecodeArrays = Array(65).fill(null);
for (let i = 0; i <= 64; i++) stringDecodeArrays[i] = Array(i).fill(0);

Expand Down Expand Up @@ -5088,21 +5086,21 @@ function deserializeStr(pos) {
pos = uint32[pos32];
let end = pos + len;
if (end <= firstNonAsciiPos) return sourceTextLatin.substr(pos, len);
// Use `TextDecoder` for strings longer than 64 bytes
if (len > 64) return decodeStr(uint8.subarray(pos, end));
// Use `utf8Slice` for strings longer than 64 bytes
if (len > 64) return utf8Slice.call(uint8, pos, end);
if (pos < sourceEndPos) {
// Check if all bytes are ASCII, use `TextDecoder` if not
for (let i = pos; i < end; i++) if (uint8[i] >= 128) return decodeStr(uint8.subarray(pos, end));
// Check if all bytes are ASCII, use `utf8Slice` if not
for (let i = pos; i < end; i++) if (uint8[i] >= 128) return utf8Slice.call(uint8, pos, end);
// String is all ASCII, so slice from `sourceTextLatin`
return sourceTextLatin.substr(pos, len);
}
// String is not in source region - use `fromCharCode.apply` with a temp array of correct length.
// Copy bytes into temp array.
// If any byte is non-ASCII, use `TextDecoder`.
// If any byte is non-ASCII, use `utf8Slice`.
let arr = stringDecodeArrays[len];
for (let i = 0; i < len; i++) {
let b = uint8[pos + i];
if (b >= 128) return decodeStr(uint8.subarray(pos, end));
if (b >= 128) return utf8Slice.call(uint8, pos, end);
arr[i] = b;
}
// Call `fromCharCode` with temp array
Expand Down
18 changes: 8 additions & 10 deletions napi/parser/src-js/generated/deserialize/js_range.js
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -9,10 +9,8 @@ let uint8,
sourceEndPos = 0,
firstNonAsciiPos = 0;

const textDecoder = new TextDecoder("utf-8", { ignoreBOM: true }),
decodeStr = textDecoder.decode.bind(textDecoder),
{ fromCharCode } = String,
{ latin1Slice } = Buffer.prototype,
const { fromCharCode } = String,
{ utf8Slice, latin1Slice } = Buffer.prototype,
stringDecodeArrays = Array(65).fill(null);
for (let i = 0; i <= 64; i++) stringDecodeArrays[i] = Array(i).fill(0);

Expand Down Expand Up @@ -5099,21 +5097,21 @@ function deserializeStr(pos) {
pos = uint32[pos32];
let end = pos + len;
if (end <= firstNonAsciiPos) return sourceTextLatin.substr(pos, len);
// Use `TextDecoder` for strings longer than 64 bytes
if (len > 64) return decodeStr(uint8.subarray(pos, end));
// Use `utf8Slice` for strings longer than 64 bytes
if (len > 64) return utf8Slice.call(uint8, pos, end);
if (pos < sourceEndPos) {
// Check if all bytes are ASCII, use `TextDecoder` if not
for (let i = pos; i < end; i++) if (uint8[i] >= 128) return decodeStr(uint8.subarray(pos, end));
// Check if all bytes are ASCII, use `utf8Slice` if not
for (let i = pos; i < end; i++) if (uint8[i] >= 128) return utf8Slice.call(uint8, pos, end);
// String is all ASCII, so slice from `sourceTextLatin`
return sourceTextLatin.substr(pos, len);
}
// String is not in source region - use `fromCharCode.apply` with a temp array of correct length.
// Copy bytes into temp array.
// If any byte is non-ASCII, use `TextDecoder`.
// If any byte is non-ASCII, use `utf8Slice`.
let arr = stringDecodeArrays[len];
for (let i = 0; i < len; i++) {
let b = uint8[pos + i];
if (b >= 128) return decodeStr(uint8.subarray(pos, end));
if (b >= 128) return utf8Slice.call(uint8, pos, end);
arr[i] = b;
}
// Call `fromCharCode` with temp array
Expand Down
18 changes: 8 additions & 10 deletions napi/parser/src-js/generated/deserialize/js_range_parent.js
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -10,10 +10,8 @@ let uint8,
firstNonAsciiPos = 0,
parent = null;

const textDecoder = new TextDecoder("utf-8", { ignoreBOM: true }),
decodeStr = textDecoder.decode.bind(textDecoder),
{ fromCharCode } = String,
{ latin1Slice } = Buffer.prototype,
const { fromCharCode } = String,
{ utf8Slice, latin1Slice } = Buffer.prototype,
stringDecodeArrays = Array(65).fill(null);
for (let i = 0; i <= 64; i++) stringDecodeArrays[i] = Array(i).fill(0);

Expand Down Expand Up @@ -5633,21 +5631,21 @@ function deserializeStr(pos) {
pos = uint32[pos32];
let end = pos + len;
if (end <= firstNonAsciiPos) return sourceTextLatin.substr(pos, len);
// Use `TextDecoder` for strings longer than 64 bytes
if (len > 64) return decodeStr(uint8.subarray(pos, end));
// Use `utf8Slice` for strings longer than 64 bytes
if (len > 64) return utf8Slice.call(uint8, pos, end);
if (pos < sourceEndPos) {
// Check if all bytes are ASCII, use `TextDecoder` if not
for (let i = pos; i < end; i++) if (uint8[i] >= 128) return decodeStr(uint8.subarray(pos, end));
// Check if all bytes are ASCII, use `utf8Slice` if not
for (let i = pos; i < end; i++) if (uint8[i] >= 128) return utf8Slice.call(uint8, pos, end);
// String is all ASCII, so slice from `sourceTextLatin`
return sourceTextLatin.substr(pos, len);
}
// String is not in source region - use `fromCharCode.apply` with a temp array of correct length.
// Copy bytes into temp array.
// If any byte is non-ASCII, use `TextDecoder`.
// If any byte is non-ASCII, use `utf8Slice`.
let arr = stringDecodeArrays[len];
for (let i = 0; i < len; i++) {
let b = uint8[pos + i];
if (b >= 128) return decodeStr(uint8.subarray(pos, end));
if (b >= 128) return utf8Slice.call(uint8, pos, end);
arr[i] = b;
}
// Call `fromCharCode` with temp array
Expand Down
18 changes: 8 additions & 10 deletions napi/parser/src-js/generated/deserialize/ts.js
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -9,10 +9,8 @@ let uint8,
sourceEndPos = 0,
firstNonAsciiPos = 0;

const textDecoder = new TextDecoder("utf-8", { ignoreBOM: true }),
decodeStr = textDecoder.decode.bind(textDecoder),
{ fromCharCode } = String,
{ latin1Slice } = Buffer.prototype,
const { fromCharCode } = String,
{ utf8Slice, latin1Slice } = Buffer.prototype,
stringDecodeArrays = Array(65).fill(null);
for (let i = 0; i <= 64; i++) stringDecodeArrays[i] = Array(i).fill(0);

Expand Down Expand Up @@ -4866,21 +4864,21 @@ function deserializeStr(pos) {
pos = uint32[pos32];
let end = pos + len;
if (end <= firstNonAsciiPos) return sourceTextLatin.substr(pos, len);
// Use `TextDecoder` for strings longer than 64 bytes
if (len > 64) return decodeStr(uint8.subarray(pos, end));
// Use `utf8Slice` for strings longer than 64 bytes
if (len > 64) return utf8Slice.call(uint8, pos, end);
if (pos < sourceEndPos) {
// Check if all bytes are ASCII, use `TextDecoder` if not
for (let i = pos; i < end; i++) if (uint8[i] >= 128) return decodeStr(uint8.subarray(pos, end));
// Check if all bytes are ASCII, use `utf8Slice` if not
for (let i = pos; i < end; i++) if (uint8[i] >= 128) return utf8Slice.call(uint8, pos, end);
// String is all ASCII, so slice from `sourceTextLatin`
return sourceTextLatin.substr(pos, len);
}
// String is not in source region - use `fromCharCode.apply` with a temp array of correct length.
// Copy bytes into temp array.
// If any byte is non-ASCII, use `TextDecoder`.
// If any byte is non-ASCII, use `utf8Slice`.
let arr = stringDecodeArrays[len];
for (let i = 0; i < len; i++) {
let b = uint8[pos + i];
if (b >= 128) return decodeStr(uint8.subarray(pos, end));
if (b >= 128) return utf8Slice.call(uint8, pos, end);
arr[i] = b;
}
// Call `fromCharCode` with temp array
Expand Down
18 changes: 8 additions & 10 deletions napi/parser/src-js/generated/deserialize/ts_parent.js
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -10,10 +10,8 @@ let uint8,
firstNonAsciiPos = 0,
parent = null;

const textDecoder = new TextDecoder("utf-8", { ignoreBOM: true }),
decodeStr = textDecoder.decode.bind(textDecoder),
{ fromCharCode } = String,
{ latin1Slice } = Buffer.prototype,
const { fromCharCode } = String,
{ utf8Slice, latin1Slice } = Buffer.prototype,
stringDecodeArrays = Array(65).fill(null);
for (let i = 0; i <= 64; i++) stringDecodeArrays[i] = Array(i).fill(0);

Expand Down Expand Up @@ -5424,21 +5422,21 @@ function deserializeStr(pos) {
pos = uint32[pos32];
let end = pos + len;
if (end <= firstNonAsciiPos) return sourceTextLatin.substr(pos, len);
// Use `TextDecoder` for strings longer than 64 bytes
if (len > 64) return decodeStr(uint8.subarray(pos, end));
// Use `utf8Slice` for strings longer than 64 bytes
if (len > 64) return utf8Slice.call(uint8, pos, end);
if (pos < sourceEndPos) {
// Check if all bytes are ASCII, use `TextDecoder` if not
for (let i = pos; i < end; i++) if (uint8[i] >= 128) return decodeStr(uint8.subarray(pos, end));
// Check if all bytes are ASCII, use `utf8Slice` if not
for (let i = pos; i < end; i++) if (uint8[i] >= 128) return utf8Slice.call(uint8, pos, end);
// String is all ASCII, so slice from `sourceTextLatin`
return sourceTextLatin.substr(pos, len);
}
// String is not in source region - use `fromCharCode.apply` with a temp array of correct length.
// Copy bytes into temp array.
// If any byte is non-ASCII, use `TextDecoder`.
// If any byte is non-ASCII, use `utf8Slice`.
let arr = stringDecodeArrays[len];
for (let i = 0; i < len; i++) {
let b = uint8[pos + i];
if (b >= 128) return decodeStr(uint8.subarray(pos, end));
if (b >= 128) return utf8Slice.call(uint8, pos, end);
arr[i] = b;
}
// Call `fromCharCode` with temp array
Expand Down
18 changes: 8 additions & 10 deletions napi/parser/src-js/generated/deserialize/ts_range.js
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -9,10 +9,8 @@ let uint8,
sourceEndPos = 0,
firstNonAsciiPos = 0;

const textDecoder = new TextDecoder("utf-8", { ignoreBOM: true }),
decodeStr = textDecoder.decode.bind(textDecoder),
{ fromCharCode } = String,
{ latin1Slice } = Buffer.prototype,
const { fromCharCode } = String,
{ utf8Slice, latin1Slice } = Buffer.prototype,
stringDecodeArrays = Array(65).fill(null);
for (let i = 0; i <= 64; i++) stringDecodeArrays[i] = Array(i).fill(0);

Expand Down Expand Up @@ -5439,21 +5437,21 @@ function deserializeStr(pos) {
pos = uint32[pos32];
let end = pos + len;
if (end <= firstNonAsciiPos) return sourceTextLatin.substr(pos, len);
// Use `TextDecoder` for strings longer than 64 bytes
if (len > 64) return decodeStr(uint8.subarray(pos, end));
// Use `utf8Slice` for strings longer than 64 bytes
if (len > 64) return utf8Slice.call(uint8, pos, end);
if (pos < sourceEndPos) {
// Check if all bytes are ASCII, use `TextDecoder` if not
for (let i = pos; i < end; i++) if (uint8[i] >= 128) return decodeStr(uint8.subarray(pos, end));
// Check if all bytes are ASCII, use `utf8Slice` if not
for (let i = pos; i < end; i++) if (uint8[i] >= 128) return utf8Slice.call(uint8, pos, end);
// String is all ASCII, so slice from `sourceTextLatin`
return sourceTextLatin.substr(pos, len);
}
// String is not in source region - use `fromCharCode.apply` with a temp array of correct length.
// Copy bytes into temp array.
// If any byte is non-ASCII, use `TextDecoder`.
// If any byte is non-ASCII, use `utf8Slice`.
let arr = stringDecodeArrays[len];
for (let i = 0; i < len; i++) {
let b = uint8[pos + i];
if (b >= 128) return decodeStr(uint8.subarray(pos, end));
if (b >= 128) return utf8Slice.call(uint8, pos, end);
arr[i] = b;
}
// Call `fromCharCode` with temp array
Expand Down
18 changes: 8 additions & 10 deletions napi/parser/src-js/generated/deserialize/ts_range_parent.js
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -10,10 +10,8 @@ let uint8,
firstNonAsciiPos = 0,
parent = null;

const textDecoder = new TextDecoder("utf-8", { ignoreBOM: true }),
decodeStr = textDecoder.decode.bind(textDecoder),
{ fromCharCode } = String,
{ latin1Slice } = Buffer.prototype,
const { fromCharCode } = String,
{ utf8Slice, latin1Slice } = Buffer.prototype,
stringDecodeArrays = Array(65).fill(null);
for (let i = 0; i <= 64; i++) stringDecodeArrays[i] = Array(i).fill(0);

Expand Down Expand Up @@ -5997,21 +5995,21 @@ function deserializeStr(pos) {
pos = uint32[pos32];
let end = pos + len;
if (end <= firstNonAsciiPos) return sourceTextLatin.substr(pos, len);
// Use `TextDecoder` for strings longer than 64 bytes
if (len > 64) return decodeStr(uint8.subarray(pos, end));
// Use `utf8Slice` for strings longer than 64 bytes
if (len > 64) return utf8Slice.call(uint8, pos, end);
if (pos < sourceEndPos) {
// Check if all bytes are ASCII, use `TextDecoder` if not
for (let i = pos; i < end; i++) if (uint8[i] >= 128) return decodeStr(uint8.subarray(pos, end));
// Check if all bytes are ASCII, use `utf8Slice` if not
for (let i = pos; i < end; i++) if (uint8[i] >= 128) return utf8Slice.call(uint8, pos, end);
// String is all ASCII, so slice from `sourceTextLatin`
return sourceTextLatin.substr(pos, len);
}
// String is not in source region - use `fromCharCode.apply` with a temp array of correct length.
// Copy bytes into temp array.
// If any byte is non-ASCII, use `TextDecoder`.
// If any byte is non-ASCII, use `utf8Slice`.
let arr = stringDecodeArrays[len];
for (let i = 0; i < len; i++) {
let b = uint8[pos + i];
if (b >= 128) return decodeStr(uint8.subarray(pos, end));
if (b >= 128) return utf8Slice.call(uint8, pos, end);
arr[i] = b;
}
// Call `fromCharCode` with temp array
Expand Down
Loading
Loading