refactor: update bench

Dockerfile.benchmark

@@ -14,4 +14,5 @@ WORKDIR /usr/src/benchmark
 COPY ./benchmark .
 
 RUN npm ci
-RUN node run.js
+RUN node bench.js formatter
+RUN node bench.js linter

benchmark/.prettierignore

@@ -0,0 +1,4 @@
+**/*.json
+**/*.css
+**/*.md
+**/*.htnl

benchmark/.prettierrc.json

@@ -0,0 +1 @@
+{}

benchmark/README.md

@@ -9,8 +9,11 @@ Run in the root directory, not `./benchmark`
 ## Running locally
 
 1. Install hyperfine: `cargo install hyperfine`
-2. Install node modules: `npm i`
-3. Run the benchmarks: `node run.js`
+2. Install node modules: `pnpm i`
+3. Run one of benchmark suites:
+    - the benchmarks of the formatters: `node bench.js formatter`
+    - the benchmarks of the linter: `node bench.js linter`
+
 
 ## Results
 
@@ -21,46 +24,39 @@ Setup: MacBook Pro (13-inch, M1, 2020)
 * Biome's ~25 times faster than Prettier
 * Biome's ~20 times faster than parallel-prettier
 * Biome's ~20 times faster than `xargs -P`[^1]
-* Biome's 1.5-2 times faster than `dprint`
+* Biome's 1.5 to 2 times faster than `dprint`
 * Biome single-threaded is ~7 times faster than Prettier.
 
+The speed-ups for the multithreaded benchmarks can vary significantly depending on the setup.
+For example, Biome is 100 times faster than Prettier on an M1 Max with 10 cores.
 
 [^1]: Run `time find lib/ examples declarations benchmark -name '*.js' -print0 | xargs -P8 -0 -n 200 npx prettier --write --loglevel=error` in the `target/webpack` directory. I manually tinkered with the `-n` parameter to get the fastest run.
 
 ### Linting
 
-* Biome's ~15x times faster than ESLint
+* Biome's ~15x times faster than ESLint (without any plugins)
 * Biome single-threaded is ~4 times faster than ESLint.
 
-The speed-ups for the multithreaded benchmarks can vary significantly depending on the setup. For example, Biome is 100 times faster than Prettier on an M1 Max with 10 cores.
-
-## Analysis
-
-### Formatter
-
-* Biome's formatter is fast :).
-* It should be possible to speed up Prettier. Biome's architecture isn't that different, and native has its advantages, but Prettier should be able to compete in single-threaded mode.
-
-### Linting
-
-* Biome's linter is fast but there is room for improvements
+* Biome's linter is fast, but there is room for improvements
 * Biome's linter spends significant time building the semantic model, the control flow graph, and matching queries. I'm convinced there's room for improvement ([3565](https://github.com/rome/tools/pull/3565), [3569](https://github.com/rome/tools/pull/3569)).
 * Computing the diff for code fixes is expensive. Biome can spend up to 3s computing diffs (not measured by these benchmarks, see explanations below)
 
+
 ## Notes
 
 We've been careful to create fair benchmarks. This section explains some of the decisions behind the benchmark setup and how these decisions affect the results. Please [let us know](https://github.com/rome/tools/issues) if you have ideas on how to make the benchmarks fairer or if there's a mistake with our setup.
 
 ### Formatting
 
-* Compares the wall time of Biome, Prettier, and dprint to format all files in a project where all files are correctly formatted.
+* Compares the wall time of Biome, Prettier, and dprint to format all files in a project where all files are correctly formatted. To ensure that files are correctly formatted we run twice a toll (warm-up) before benchmarking it.
 * dprint and Prettier support incremental formatting to only format changed files, whereas Biome does not. This benchmark does not measure incremental formatting as it measures cold formatting time. You may see significant speedups on subsequent formatting runs when enabling incremental formatting.
 * The benchmark limits Prettier to only format JavaScript and TypeScript files because Biome doesn't support other file types.
-* Biome only prints a summary with the number of formatted files. The prettier benchmark uses `--loglevel=error` for a fairer benchmark so that Prettier doesn't print every filename.
+* Biome only prints a summary with the number of formatted files. The prettier benchmark uses `--log-level=error` for a fairer benchmark so that Prettier doesn't print every filename.
 
 ### Linting
 
 * Compares the wall time of Biome and ESLint to check all files in a project.
 * Only enables rules that both Biome and ESLint support.
+* The comparison includes ESLint without plugin and ESLint with the TypeScript ESLint plugin.
+  To have a fair comparison we don't enable any rules that require type information because this is known to be very slow and Biome has not such capabilities yet.
 * Biome prints rich diffs for each lint diagnostic, whereas ESLint only shows the name and description of the lint rule. That's why the benchmark uses `--max-diagnostics=0` when running Biome because Biome then only counts diagnostics without generating the diffs. Overall, this results in a more accurate comparison but has the downside that Biome only prints the diagnostic count, whereas ESLint prints a line for each lint error.
-

benchmark/bench.js

@@ -0,0 +1,233 @@
+#!/usr/bin/node
+
+import { execSync } from "node:child_process";
+import * as fs from "node:fs";
+//import * as os from "node:os";
+import * as path from "node:path";
+import * as process from "node:process";
+import * as util from "node:util";
+
+const REPOSITORIES = {
+	eslint: {
+		repository: "https://github.com/eslint/eslint.git",
+		lintedDirs: [
+			"lib",
+			"messages",
+			"tests/lib",
+			"tests/performance",
+			"tools",
+		],
+	},
+	prettier: {
+		repository: "https://github.com/prettier/prettier.git",
+		formattedDirs: ["src", "scripts"],
+	},
+	webpack: {
+		repository: "https://github.com/webpack/webpack.git",
+		formattedDirs: ["lib", "examples", "benchmark"],
+		lintedDirs: ["lib"],
+	},
+}
+
+const TMP_DIR = path.resolve("./target");
+
+function benchmarkFormatter(biomeBin, options) {
+	// Run Dprint once to run the installer
+	execSync("npx dprint --version");
+
+	for (const [name, config] of Object.entries(REPOSITORIES)) {
+		if (!config.formattedDirs || ("repository" in options && options.repository !== name)) {
+			continue;
+		}
+		console.info(`\n⌛ repository: ${name}`);
+		const projectDirectory = cloneProject(name, config.repository, config.formattedDirs);
+		console.info("");
+
+		const dirs = config.formattedDirs.join(" ");
+		const dirGlobs = config.formattedDirs.map((dir) => `"${dir}/**"`).join(" ");
+		const biomeCommand = `${biomeBin} format --config-path=../../ --write --max-diagnostics=0 ${dirs}`;
+		const benchCommands = {
+			prettier: `../../node_modules/.bin/prettier --config=../../.prettierrc.json --ignore-path=../../.prettierignore  --write --log-level=error ${dirs}`,
+			// FIXME: Parallel Prettier is crashing on Node 22
+			//"parallel-prettier": `../../node_modules/.bin/pprettier --write --concurrency ${os.cpus().length} ${dirGlobs}`,
+			dprint: `../../node_modules/dprint/dprint fmt --config=../../dprint.json ${dirGlobs}`,
+			biome: biomeCommand,
+			"biome-1-thread": withEnvVariable("RAYON_NUM_THREADS", "1", biomeCommand),
+		};
+
+		let suite = "";
+		for (const benchName of Object.keys(benchCommands)) {
+			if ("bench" in options && !options.bench.includes(benchName)) {
+				continue;
+			}
+			suite += ` -n "${benchName}" '${benchCommands[benchName]}'`;
+		}
+		if (suite.length === 0) {
+			console.error(`Benchmark '${options.bench}' doesn't exist.`);
+			process.exit(1);
+		}
+
+		// Run 2 warmups to make sure the files are formatted correctly
+		let hyperfineCommand = `hyperfine --warmup 2 --shell=${shellOption()} ${suite}`;
+		if (options.verbose) {
+			hyperfineCommand += " --show-output";
+			console.info(`${hyperfineCommand}\n`)
+		}
+
+		execSync(hyperfineCommand, {
+			cwd: projectDirectory,
+			stdio: "inherit",
+		});
+	}
+}
+
+function benchmarkLinter(biomeBin, options) {
+	for (const [name, config] of Object.entries(REPOSITORIES)) {
+		if (!config.lintedDirs || ("repository" in options && options.repository !== name)) {
+			continue;
+		}
+		console.info(`\n⌛ repository: ${name}`);
+		const projectDirectory = cloneProject(name, config.repository, config.formattedDirs);
+		console.info("");
+
+		const dirs = config.lintedDirs.map((dir) => `"${dir}"`).join(" ");
+		const biomeCmd = `"${biomeBin}" lint --config-path=../../ --max-diagnostics=0 ${dirs}`;
+		const benchCommands = {
+			eslint: `../../node_modules/.bin/eslint --quiet --config=../../eslint.config.js --no-ignore ${dirs}`,
+			"ts-eslint": `../../node_modules/.bin/eslint --quiet --config=../../ts-eslint.config.js --no-ignore ${dirs}`,
+			biome: biomeCmd,
+			"biome-1-thread": withEnvVariable("RAYON_NUM_THREADS", "1", biomeCmd),
+			"ts-biome": `"${biomeBin}" lint --config-path=../../ts-biome.json --max-diagnostics=0 ${dirs}`,
+		};
+
+		let suite = "";
+		for (const benchName of Object.keys(benchCommands)) {
+			if ("bench" in options && !options.bench.includes(benchName)) {
+				continue;
+			}
+			suite += ` -n "${benchName}" '${benchCommands[benchName]}'`;
+		}
+		if (suite.length === 0) {
+			console.error(`Benchmark '${options.bench}' doesn't exist.`);
+			process.exit(1);
+		}
+
+		let hyperfineCommand = `hyperfine --ignore-failure --shell=${shellOption()} ${suite}`;
+		if (options.verbose) {
+			hyperfineCommand += " --show-output";
+			console.info(`${hyperfineCommand}\n`)
+		}
+
+		execSync(hyperfineCommand, {
+			cwd: projectDirectory,
+			stdio: "inherit",
+		});
+	}
+}
+
+function shellOption() {
+	if (process.platform == "win32") {
+		// Use Powershell so that it is possible to set an environment variable for a single command (ugh!)
+		return "powershell";
+	} else {
+		return "default";
+	}
+}
+
+function withEnvVariable(name, value, command) {
+	switch (process.platform) {
+		case "win32": {
+			return `$Env:${name}=${value}; ${command}`;
+		}
+		default:
+			return `${name}="${value}" ${command}`;
+	}
+}
+
+function withDirectory(cwd) {
+	return {
+		run(command, options) {
+			execSync(command, {
+				cwd,
+				...options,
+			});
+		},
+	};
+}
+
+function cloneProject(name, repository, dirs = []) {
+	const projectDirectory = path.join(TMP_DIR, name);
+
+	const inProjectDirectory = withDirectory(projectDirectory);
+
+	if (fs.existsSync(projectDirectory)) {
+		inProjectDirectory.run("git reset --hard @{u}");
+		inProjectDirectory.run("git clean -df");
+		inProjectDirectory.run("git pull --quiet --depth=1 --ff-only");
+	} else {
+		withDirectory(TMP_DIR).run(
+			`git clone --quiet --depth=1 ${dirs.length > 0 ? "--sparse" : ""} ${repository}`,
+			{
+				stdio: "inherit",
+			},
+		);
+	}
+
+	if (dirs.length > 0) {
+		console.info(
+			`Adding directories ${dirs.join()} to sparse checkout in ${projectDirectory}`,
+		);
+		inProjectDirectory.run(`git sparse-checkout add ${dirs.join(" ")}`);
+	}
+
+	return projectDirectory;
+}
+
+function buildBiome() {
+	console.info("Building Biome...");
+	execSync("cargo build --bin biome --release", {
+		stdio: "inherit",
+	});
+	return path.resolve("../target/release/biome");
+}
+
+function run({ positionals, values: options }) {
+	if (positionals.length !== 1) {
+		console.error(`A positional argument among 'formatter' and 'linter' must be passed.`);
+		process.exit(1);
+	}
+	fs.mkdirSync(TMP_DIR, { recursive: true });
+	let biomeBinPath;
+	if ("biome" in options) {
+		biomeBinPath = options.biome;
+	} else {
+		biomeBinPath = buildBiome()
+	}
+	switch (positionals[0]) {
+		case "formatter":
+			benchmarkFormatter(biomeBinPath, options);
+			break;
+		case "linter":
+			benchmarkLinter(biomeBinPath, options);
+			break;
+	}
+}
+
+run(util.parseArgs({
+	allowPositionals: true,
+	options: {
+		bench: {
+			multiple: true,
+			type: "string",
+		},
+		biome: {
+			type: "string",
+		},
+		repository: {
+			type: "string",
+		},
+		verbose: {
+			type: "boolean",
+		}
+	}
+}));