diff --git a/apps/oxlint/conformance/tester.ts b/apps/oxlint/conformance/tester.ts new file mode 100644 index 0000000000000..61098b0a16069 --- /dev/null +++ b/apps/oxlint/conformance/tester.ts @@ -0,0 +1,215 @@ +/* + * A script for running a test case with both ESLint and Oxlint. + * Purpose is to be able to "spot the difference" between the two. + */ + +// oxlint-disable no-console + +// Use the conformance testing version of `RuleTester`, +// which modifies test cases before they run +import { RuleTester as OxlintRuleTester } from "./src/rule_tester.ts"; +// @ts-expect-error - internal module of ESLint with no types +import { RuleTester as ESLintRuleTester } from "./submodules/eslint/lib/rule-tester/index.js"; +// @ts-expect-error - internal module of ESLint with no types +import { builtinRules } from "./submodules/eslint/lib/unsupported-api.js"; + +import type { Rule } from "#oxlint"; +import type { RuleTester as OxlintRuleTesterTypes } from "#oxlint"; + +type ValidTestCase = OxlintRuleTesterTypes.ValidTestCase; +type InvalidTestCase = OxlintRuleTesterTypes.InvalidTestCase; + +type ValidTestCaseWithoutCode = Omit & { code?: string }; +type InvalidTestCaseWithoutCode = Omit & { code?: string }; +type TestCaseWithoutCode = ValidTestCaseWithoutCode | InvalidTestCaseWithoutCode; + +// Reset `describe` + `it` to simple pass-through functions. +// Importing from `rule_tester.ts` sets up `RuleTester` to use `capture.ts`'s `describe` / `it`, +// which we don't want for manual testing. +// `OxlintRuleTesterOriginal` is the original `RuleTester` exported by `oxlint`. +const OxlintRuleTesterOriginal = Object.getPrototypeOf(OxlintRuleTester); + +const simpleDescribe = (name: string, fn: () => void) => fn(); +const simpleIt = (name: string, fn: () => void) => fn(); + +OxlintRuleTesterOriginal.describe = simpleDescribe; +OxlintRuleTesterOriginal.it = simpleIt; + +// Set global `describe` and `it` for ESLint's `RuleTester` +(globalThis as any).describe = simpleDescribe; +(globalThis as any).it = simpleIt; + +/* ----------------------------------------------------------------------------- + +(Obviously these instructions are aimed at AI, hence the rather rude and demanding tone. +Sorry humans! In fact, sorry to you too AI.) + +# HOW TO USE THIS SCRIPT + +DO NOT edit this file except the section between the bottom of this comment and the start of the next comment +"Run test case with both ESLint and Oxlint". + +## 1. Select a test + +Select a failing test case to investigate from `snapshot.md`. + +## 2. Copy details + +Copy details from `snapshot.md` for the case into the variables in this script, below this comment. + +e.g.: + +Details of test case from `snapshot.md`: + +------------- Snapshot extract begins ------------- + +#### block-scoped-var > invalid + +```js +for (var a = 0;;) {} a; +``` + +```json +{ + "errors": [ + { + "messageId": "outOfScope", + "data": { + "name": "a", + "definitionLine": 1, + "definitionColumn": 10 + }, + "line": 1, + "column": 22 + } + ] +} +``` + +------------- Snapshot extract ends ------------- + +Fill in the variables below as follows: + +------------- Script setup begins ------------- + +const ruleName = "block-scoped-var"; + +const code = `for (var a = 0;;) {} a;`; + +const isInvalid = true; + +const testCase: TestCaseWithoutCode = { + "errors": [ + { + "messageId": "outOfScope", + "data": { + "name": "a", + "definitionLine": 1, + "definitionColumn": 10 + }, + "line": 1, + "column": 22 + } + ] +}; + +const config = {}; + +------------- Script setup ends ------------- + +## 3. Run the test + +Run this script with `node conformance/tester.ts`. + +Observe the output. ESLint should not produce any errors, but Oxlint will. + +## 4. Discover the cause + +Add `console.log` statements to the rule's code in `submodules/eslint/lib/rules`. + +Run this script again. Find out what is happening differently between ESLint and Oxlint. + +## 5. Clean up + +`submodules/eslint` is a git repo. +Use git to throw away the changes you made to the rule's code (`console.log`). +Leave the eslint repo as it was before you started, ready to investigate another case. + +-----------------------------------------------------------------------------*/ + +const ruleName = "block-scoped-var"; + +const code = `for (var a = 0;;) {} a;`; + +const isInvalid = true; + +const testCase: TestCaseWithoutCode = { + errors: [ + { + messageId: "outOfScope", + data: { + name: "a", + definitionLine: 1, + definitionColumn: 10, + }, + line: 1, + column: 22, + }, + ], +}; + +const config = {}; + +// ----------------------------------------------------------------------------- +// Run test case with both ESLint and Oxlint +// ----------------------------------------------------------------------------- + +runBoth(ruleName, isInvalid, code, testCase, config); + +function runBoth( + ruleName: string, + isInvalid: boolean, + code: string, + testCase: TestCaseWithoutCode, + config?: Record | null, +) { + testCase = { code, ...testCase }; + + const valid: ValidTestCase[] = [], + invalid: InvalidTestCase[] = []; + if (isInvalid) { + invalid.push(testCase as InvalidTestCase); + } else { + valid.push(testCase as ValidTestCase); + } + + const rule = builtinRules.get(ruleName) as unknown as Rule; + + console.log("--------------------"); + console.log("ESLint"); + console.log("--------------------"); + runOne(ESLintRuleTester, rule, valid, invalid, config); + + console.log("\n--------------------"); + console.log("Oxlint"); + console.log("--------------------"); + config = { ...config, eslintCompat: true }; + runOne(OxlintRuleTester, rule, valid, invalid, config); +} + +function runOne( + RuleTester: any, + rule: Rule, + valid: ValidTestCase[], + invalid: InvalidTestCase[], + config?: Record | null, +) { + try { + const tester = new RuleTester(config); + tester.run("my-rule", rule, { valid, invalid }); + console.log("No errors"); + } catch (err) { + console.log("ERROR:"); + console.log(err); + } +}