feat(linter/plugins): handle BOMs

apps/oxlint/conformance/snapshot.md

@@ -7,8 +7,8 @@
 | Status            | Count | %      |
 | ----------------- | ----- | ------ |
 | Total rules       |   292 | 100.0% |
-| Fully passing     |   289 |  99.0% |
-| Partially passing |     3 |   1.0% |
+| Fully passing     |   291 |  99.7% |
+| Partially passing |     1 |   0.3% |
 | Fully failing     |     0 |   0.0% |
 | Load errors       |     0 |   0.0% |
 | No tests run      |     0 |   0.0% |
@@ -18,8 +18,8 @@
 | Status      | Count | %      |
 | ----------- | ----- | ------ |
 | Total tests | 33090 | 100.0% |
-| Passing     | 32803 |  99.1% |
-| Failing     |     5 |   0.0% |
+| Passing     | 32807 |  99.1% |
+| Failing     |     1 |   0.0% |
 | Skipped     |   282 |   0.9% |
 
 ## Fully Passing Rules
@@ -156,6 +156,7 @@
 - `no-inner-declarations` (68 tests)
 - `no-invalid-regexp` (108 tests)
 - `no-invalid-this` (562 tests) (4 skipped)
+- `no-irregular-whitespace` (280 tests)
 - `no-iterator` (9 tests)
 - `no-label-var` (5 tests)
 - `no-labels` (29 tests)
@@ -306,6 +307,7 @@
 - `symbol-description` (8 tests)
 - `template-curly-spacing` (57 tests)
 - `template-tag-spacing` (63 tests)
+- `unicode-bom` (7 tests)
 - `use-isnan` (214 tests)
 - `valid-typeof` (54 tests)
 - `vars-on-top` (61 tests)
@@ -317,8 +319,6 @@
 ## Rules with Failures
 
 - `no-eval` - 100 / 101 (99.0%)
-- `no-irregular-whitespace` - 279 / 280 (99.6%)
-- `unicode-bom` - 4 / 7 (57.1%)
 
 ## Rules with Failures Detail
 
@@ -359,146 +359,3 @@ AssertionError [ERR_ASSERTION]: Should have 1 error but had 0: []
     at runInvalidTestCase (apps/oxlint/dist/index.js)
     at apps/oxlint/dist/index.js
 
-
-### `no-irregular-whitespace`
-
-Pass: 279 / 280 (99.6%)
-Fail: 1 / 280 (0.4%)
-Skip: 0 / 280 (0.0%)
-
-#### no-irregular-whitespace > valid
-
-```js
-console.log('hello BOM');
-```
-
-```json
-{}
-```
-
-AssertionError [ERR_ASSERTION]: Should have no errors but had 1: [
-  {
-    ruleId: 'rule-to-test/no-irregular-whitespace',
-    message: 'Irregular whitespace not allowed.',
-    messageId: 'noIrregularWhitespace',
-    severity: 1,
-    nodeType: null,
-    line: 1,
-    column: 0,
-    endLine: 1,
-    endColumn: 1,
-    suggestions: null
-  }
-]
-
-1 !== 0
-
-    at assertErrorCountIsCorrect (apps/oxlint/dist/index.js)
-    at assertValidTestCasePasses (apps/oxlint/dist/index.js)
-    at runValidTestCase (apps/oxlint/dist/index.js)
-    at apps/oxlint/dist/index.js
-
-
-### `unicode-bom`
-
-Pass: 4 / 7 (57.1%)
-Fail: 3 / 7 (42.9%)
-Skip: 0 / 7 (0.0%)
-
-#### unicode-bom > valid
-
-```js
- var a = 123;
-```
-
-```json
-{
-  "options": [
-    "always"
-  ]
-}
-```
-
-AssertionError [ERR_ASSERTION]: Should have no errors but had 1: [
-  {
-    ruleId: 'rule-to-test/unicode-bom',
-    message: 'Expected Unicode BOM (Byte Order Mark).',
-    messageId: 'expected',
-    severity: 1,
-    nodeType: null,
-    line: 1,
-    column: 0,
-    endLine: 1,
-    endColumn: 0,
-    suggestions: null
-  }
-]
-
-1 !== 0
-
-    at assertErrorCountIsCorrect (apps/oxlint/dist/index.js)
-    at assertValidTestCasePasses (apps/oxlint/dist/index.js)
-    at runValidTestCase (apps/oxlint/dist/index.js)
-    at apps/oxlint/dist/index.js
-
-
-#### unicode-bom > invalid
-
-```js
- var a = 123;
-```
-
-```json
-{
-  "output": " var a = 123;",
-  "errors": [
-    {
-      "messageId": "unexpected",
-      "line": 1,
-      "column": 1
-    }
-  ]
-}
-```
-
-AssertionError [ERR_ASSERTION]: Should have 1 error but had 0: []
-
-0 !== 1
-
-    at assertErrorCountIsCorrect (apps/oxlint/dist/index.js)
-    at assertInvalidTestCasePasses (apps/oxlint/dist/index.js)
-    at runInvalidTestCase (apps/oxlint/dist/index.js)
-    at apps/oxlint/dist/index.js
-
-
-#### unicode-bom > invalid
-
-```js
- var a = 123;
-```
-
-```json
-{
-  "output": " var a = 123;",
-  "options": [
-    "never"
-  ],
-  "errors": [
-    {
-      "messageId": "unexpected",
-      "line": 1,
-      "column": 1
-    }
-  ]
-}
-```
-
-AssertionError [ERR_ASSERTION]: Should have 1 error but had 0: []
-
-0 !== 1
-
-    at assertErrorCountIsCorrect (apps/oxlint/dist/index.js)
-    at assertInvalidTestCasePasses (apps/oxlint/dist/index.js)
-    at runInvalidTestCase (apps/oxlint/dist/index.js)
-    at apps/oxlint/dist/index.js
-

apps/oxlint/src-js/generated/constants.ts

@@ -6,6 +6,7 @@ export const BUFFER_ALIGN = 4294967296;
 export const DATA_POINTER_POS_32 = 536870902;
 export const IS_TS_FLAG_POS = 2147483612;
 export const IS_JSX_FLAG_POS = 2147483613;
+export const HAS_BOM_FLAG_POS = 2147483614;
 export const PROGRAM_OFFSET = 0;
 export const SOURCE_START_OFFSET = 8;
 export const SOURCE_LEN_OFFSET = 16;

apps/oxlint/src-js/plugins/lint.ts

@@ -5,6 +5,7 @@ import { allOptions, DEFAULT_OPTIONS_ID } from "./options.ts";
 import { diagnostics } from "./report.ts";
 import { setSettingsForFile, resetSettings } from "./settings.ts";
 import { ast, initAst, resetSourceAndAst, setupSourceForFile } from "./source_code.ts";
+import { HAS_BOM_FLAG_POS } from "../generated/constants.ts";
 import { typeAssertIs, debugAssert, debugAssertIsNonNull } from "../utils/asserts.ts";
 import { getErrorMessage } from "../utils/utils.ts";
 import { setGlobalsForFile, resetGlobals } from "./globals.ts";
@@ -154,7 +155,7 @@ export function lintFileImpl(
   //
   // But... source text and AST can be accessed in body of `create` method, or `before` hook, via `context.sourceCode`.
   // So we pass the buffer to source code module here, so it can decode source text / deserialize AST on demand.
-  const hasBOM = false; // TODO: Set this correctly
+  const hasBOM = buffer[HAS_BOM_FLAG_POS] === 1;
   const parserServices = PARSER_SERVICES_DEFAULT; // TODO: Set this correctly
   setupSourceForFile(buffer, hasBOM, parserServices);
 

apps/oxlint/src/js_plugins/parse.rs

@@ -151,7 +151,7 @@ unsafe fn parse_raw_impl(
 
     // Parse source.
     // Enclose parsing logic in a scope to make 100% sure no references to within `Allocator` exist after this.
-    let program_offset = {
+    let (program_offset, has_bom) = {
         // SAFETY: We checked above that `source_len` does not exceed length of buffer
         let source_text = unsafe { buffer.get_unchecked(..source_len) };
         // SAFETY: Caller guarantees source occupies this region of the buffer and is valid UTF-8
@@ -179,22 +179,46 @@ unsafe fn parse_raw_impl(
 
         if parsing_failed {
             // Use sentinel value for program offset to indicate that parsing failed
-            PARSE_FAIL_SENTINEL
+            (PARSE_FAIL_SENTINEL, false)
         } else {
-            // Convert spans to UTF-16
-            let span_converter = Utf8ToUtf16::new(source_text);
+            // If has BOM, remove it
+            const BOM: &str = "\u{feff}";
+            const BOM_LEN: usize = BOM.len();
+
+            let mut source_text = program.source_text;
+            let has_bom = source_text.starts_with(BOM);
+            if has_bom {
+                source_text = &source_text[BOM_LEN..];
+                program.source_text = source_text;
+            }
+
+            // Convert spans to UTF-16.
+            // If source starts with BOM, create converter which ignores the BOM.
+            let span_converter = if has_bom {
+                #[expect(clippy::cast_possible_truncation)]
+                Utf8ToUtf16::new_with_offset(source_text, BOM_LEN as u32)
+            } else {
+                Utf8ToUtf16::new(source_text)
+            };
+
             span_converter.convert_program(program);
             span_converter.convert_comments(&mut program.comments);
 
             // Return offset of `Program` within buffer (bottom 32 bits of pointer)
-            ptr::from_ref(program) as u32
+            let program_offset = ptr::from_ref(program) as u32;
+
+            (program_offset, has_bom)
         }
     };
 
     // Write metadata into end of buffer
     #[allow(clippy::cast_possible_truncation)]
-    let metadata =
-        RawTransferMetadata::new(program_offset, source_type.is_typescript(), source_type.is_jsx());
+    let metadata = RawTransferMetadata::new(
+        program_offset,
+        source_type.is_typescript(),
+        source_type.is_jsx(),
+        has_bom,
+    );
     const RAW_METADATA_OFFSET: usize = BUFFER_SIZE - RAW_METADATA_SIZE;
     const _: () = assert!(RAW_METADATA_OFFSET.is_multiple_of(BUMP_ALIGN));
     // SAFETY: `RAW_METADATA_OFFSET` is less than length of `buffer`.

apps/oxlint/test/fixtures/bom/.oxlintrc.json

@@ -0,0 +1,7 @@
+{
+  "categories": { "correctness": "off" },
+  "jsPlugins": ["./plugin.ts"],
+  "rules": {
+    "bom-plugin/bom": "error"
+  }
+}

apps/oxlint/test/fixtures/bom/files/bom.js

@@ -0,0 +1,3 @@
+debugger;
+debugger;
+debugger;

apps/oxlint/test/fixtures/bom/files/bom_unicode.js

@@ -0,0 +1,4 @@
+debugger;
+// 😀🤪😆😎🤮
+debugger;
+debugger;

apps/oxlint/test/fixtures/bom/files/no_bom.js

@@ -0,0 +1,3 @@
+debugger;
+debugger;
+debugger;

apps/oxlint/test/fixtures/bom/files/no_bom_unicode.js

@@ -0,0 +1,4 @@
+debugger;
+// 😀🤪😆😎🤮
+debugger;
+debugger;