test(linter/plugins): conformance tests include config in snapshot

apps/oxlint/conformance/snapshot.md

@@ -336,6 +336,10 @@ function foo() { ('use strict'); this.eval; }
 
 ```json
 {
+  "languageOptions": {
+    "ecmaVersion": 5,
+    "sourceType": "script"
+  },
   "errors": [
     {
       "messageId": "unexpected",
@@ -365,7 +369,8 @@ function foo() { 'use strict'; this.eval(); }
 ```json
 {
   "languageOptions": {
-    "ecmaVersion": 3
+    "ecmaVersion": 3,
+    "sourceType": "script"
   },
   "errors": [
     {
@@ -398,7 +403,16 @@ Skip: 1 / 1072 (0.1%)
 ```
 
 ```json
-{}
+{
+  "languageOptions": {
+    "sourceType": "script",
+    "parserOptions": {
+      "ecmaFeatures": {
+        "jsx": true
+      }
+    }
+  }
+}
 ```
 
 AssertionError [ERR_ASSERTION]: Should have no errors but had 1: [
@@ -432,7 +446,16 @@ foo
 ```
 
 ```json
-{}
+{
+  "languageOptions": {
+    "sourceType": "script",
+    "parserOptions": {
+      "ecmaFeatures": {
+        "jsx": true
+      }
+    }
+  }
+}
 ```
 
 AssertionError [ERR_ASSERTION]: Should have no errors but had 1: [
@@ -465,7 +488,16 @@ AssertionError [ERR_ASSERTION]: Should have no errors but had 1: [
 ```
 
 ```json
-{}
+{
+  "languageOptions": {
+    "sourceType": "script",
+    "parserOptions": {
+      "ecmaFeatures": {
+        "jsx": true
+      }
+    }
+  }
+}
 ```
 
 AssertionError [ERR_ASSERTION]: Should have no errors but had 1: [
@@ -498,7 +530,16 @@ AssertionError [ERR_ASSERTION]: Should have no errors but had 1: [
 ```
 
 ```json
-{}
+{
+  "languageOptions": {
+    "sourceType": "script",
+    "parserOptions": {
+      "ecmaFeatures": {
+        "jsx": true
+      }
+    }
+  }
+}
 ```
 
 AssertionError [ERR_ASSERTION]: Should have no errors but had 1: [
@@ -539,7 +580,8 @@ function foo() { 'use strict'; this.eval(); }
 ```json
 {
   "languageOptions": {
-    "ecmaVersion": 3
+    "ecmaVersion": 3,
+    "sourceType": "script"
   }
 }
 ```
@@ -576,7 +618,8 @@ AssertionError [ERR_ASSERTION]: Should have no errors but had 1: [
 ```json
 {
   "languageOptions": {
-    "ecmaVersion": 3
+    "ecmaVersion": 3,
+    "sourceType": "script"
   }
 }
 ```
@@ -614,6 +657,7 @@ AssertionError [ERR_ASSERTION]: Should have no errors but had 1: [
 {
   "languageOptions": {
     "ecmaVersion": 3,
+    "sourceType": "script",
     "parserOptions": {
       "ecmaFeatures": {
         "impliedStrict": true
@@ -657,7 +701,11 @@ AssertionError [ERR_ASSERTION]: Should have no errors but had 1: [
 ```
 
 ```json
-{}
+{
+  "languageOptions": {
+    "parser": {}
+  }
+}
 ```
 
 Error: Parsing failed
@@ -852,6 +900,10 @@ Skip: 0 / 347 (0.0%)
 
 ```json
 {
+  "languageOptions": {
+    "ecmaVersion": 5,
+    "sourceType": "script"
+  },
   "errors": [
     {
       "messageId": "usedBeforeDefined",
@@ -892,10 +944,11 @@ const test = Object.assign({ ...bar }, {
 
 ```json
 {
-  "output": "const test = {...bar, <!-- html comment\n                foo: 'bar',\n                baz: \"cats\"\n                --> weird\n            }",
   "languageOptions": {
+    "ecmaVersion": 2018,
     "sourceType": "script"
   },
+  "output": "const test = {...bar, <!-- html comment\n                foo: 'bar',\n                baz: \"cats\"\n                --> weird\n            }",
   "errors": [
     {
       "messageId": "useSpreadMessage",

apps/oxlint/conformance/src/capture.ts

@@ -49,6 +49,18 @@ export function resetCurrentRule(): void {
   currentRule = null;
 }
 
+// Current test case being tested
+let currentTest: TestCase | null = null;
+
+/**
+ * Set the current test being tested.
+ * Call before running linter for a test case (in `modifyTestCase` hook).
+ * @param test - `TestCase` object
+ */
+export function setCurrentTest(test: TestCase): void {
+  currentTest = test;
+}
+
 /**
  * `describe` function that tracks the test hierarchy.
  * @param name - Name of the test group
@@ -80,32 +92,56 @@ export function it(code: string, fn: () => void): void {
 
   try {
     fn();
+
+    // Check that the test case was actually run
+    if (currentTest === null) throw new Error("Test case was not run with `RuleTester`");
+
     testResult.isPassed = true;
   } catch (err) {
-    if (err instanceof Error && err.message === "Custom parsers are not supported") {
-      testResult.isSkipped = true;
-    }
+    testResult.testCase = currentTest;
 
-    // Skip test cases which start with `/* global */`, `/* globals */`, `/* exported */`, or `/* eslint */` comments.
-    // Oxlint does not support defining globals inline.
-    // `RuleTester` does not support enabling other rules beyond the rule under test.
-    if (code.match(/^\s*\/\*\s*(globals?|exported|eslint)\s/)) {
-      testResult.isSkipped = true;
-    }
+    if (!(err instanceof Error)) {
+      testResult.error = new Error("Unknown error");
+    } else if (currentTest === null) {
+      testResult.error = new Error("Test case was not run with `RuleTester`");
+    } else {
+      testResult.error = err;
 
-    // Skip test cases which include `// eslint-disable` comments.
-    // These are not handled by `RuleTester`.
-    if (code.match(/\/\/\s*eslint-disable((-next)?-line)?(\s|$)/)) {
-      testResult.isSkipped = true;
+      const ruleName = describeStack[0];
+      if (shouldSkipTest(ruleName, currentTest, code, err)) testResult.isSkipped = true;
     }
-
-    testResult.error = err as Error;
-    testResult.testCase = err?.__testCase ?? null;
+  } finally {
+    // Reset current test
+    currentTest = null;
   }
 
   currentRule!.tests.push(testResult);
 }
 
+/**
+ * Determine if failing test case should be skipped.
+ * @param ruleName - Rule name
+ * @param test - Test case
+ * @param code - Code for test case
+ * @param err - Error thrown during test case
+ * @returns `true` if test should be skipped
+ */
+function shouldSkipTest(ruleName: string, test: TestCase, code: string, err: Error): boolean {
+  // We cannot support custom parsers
+  if (err.message === "Custom parsers are not supported") return true;
+
+  // Skip test cases which start with `/* global */`, `/* globals */`, `/* exported */`, or `/* eslint */` comments.
+  // Oxlint does not support defining globals inline.
+  // `RuleTester` does not support enabling other rules beyond the rule under test.
+  if (code.match(/^\s*\/\*\s*(globals?|exported|eslint)\s/)) return true;
+
+  // Skip test cases which include `// eslint-disable` comments.
+  // These are not handled by `RuleTester`.
+  if (code.match(/\/\/\s*eslint-disable((-next)?-line)?(\s|$)/)) return true;
+
+  return false;
+}
+
 // Add `it.only` property for compatibility.
 // `it.only` behaves the same as `it`.
 it.only = (name: string, fn: () => void): void => it(name, fn);

apps/oxlint/conformance/src/rule_tester.ts

@@ -6,7 +6,7 @@
 import eslintGlobals from "../submodules/eslint/conf/globals.js";
 import { createRequire } from "node:module";
 import { RuleTester } from "#oxlint";
-import { describe, it } from "./capture.ts";
+import { describe, it, setCurrentTest } from "./capture.ts";
 import { ESLINT_RULES_TESTS_DIR_PATH } from "./run.ts";
 import { FILTER_ONLY_CODE } from "./filter.ts";
 
@@ -89,6 +89,10 @@ class RuleTesterShim extends RuleTester {
 (RuleTester as any).registerModifyTestCaseHook(modifyTestCase);
 
 function modifyTestCase(test: TestCase): void {
+  // Record current test case.
+  // Clone it to avoid including the changes to the original test case made below.
+  setCurrentTest({ ...test });
+
   // Enable ESLint compat mode.
   // This makes `RuleTester` adjust column indexes in diagnostics to match ESLint's behavior.
   test.eslintCompat = true;

apps/oxlint/src-js/package/rule_tester.ts

@@ -815,32 +815,6 @@ function getMessagePlaceholders(message: string): string[] {
   return Array.from(message.matchAll(PLACEHOLDER_REGEX), ([, name]) => name.trim());
 }
 
-// In conformance build, wrap `runValidTestCase` and `runInvalidTestCase` to add test case to error object.
-// This is used in conformance tests.
-type RunFunction<T> = (test: T, plugin: Plugin, config: Config, seenTestCases: Set<string>) => void;
-
-function wrapRunTestCaseFunction<T extends ValidTestCase | InvalidTestCase>(
-  run: RunFunction<T>,
-): RunFunction<T> {
-  return function (test, plugin, config, seenTestCases) {
-    try {
-      run(test, plugin, config, seenTestCases);
-    } catch (err) {
-      // oxlint-disable-next-line no-ex-assign
-      if (typeof err !== "object" || err === null) err = new Error("Unknown error");
-      err.__testCase = test;
-      throw err;
-    }
-  };
-}
-
-if (CONFORMANCE) {
-  // oxlint-disable-next-line no-func-assign
-  (runValidTestCase as any) = wrapRunTestCaseFunction(runValidTestCase);
-  // oxlint-disable-next-line no-func-assign
-  (runInvalidTestCase as any) = wrapRunTestCaseFunction(runInvalidTestCase);
-}
-
 /**
  * Create config for a test run.
  * Merges config from `RuleTester` instance on top of shared config.