feat: add new experimental Rust compiler

.changeset/rust-compiler-experimental.md

@@ -0,0 +1,23 @@
+---
+'astro': minor
+---
+
+Adds a new `experimental.rustCompiler` flag to opt into the experimental Rust-based Astro compiler
+
+This experimental compiler is faster, provides better error messages, and generally has better support for modern JavaScript, TypeScript, and CSS features.
+
+After enabling in your Astro config, the `@astrojs/compiler-rs` package must also be installed into your project separately:
+
+```js
+import { defineConfig } from "astro/config";
+
+export default defineConfig({
+  experimental: {
+    rustCompiler: true
+  }
+});
+```
+
+This new compiler is still in early development and may exhibit some differences compared to the existing Go-based compiler. Notably, this compiler is generally more strict in regard to invalid  HTML syntax and may throw errors in cases where the Go-based compiler would have been more lenient. For example, unclosed tags (e.g. `<p>My paragraph`) will now result in errors.
+
+For more information about using this experimental feature in your project, especially regarding expected differences and limitations, please see the [experimental Rust compiler reference docs](https://v6.docs.astro.build/en/reference/experimental-flags/rust-compiler/). To give feedback on the compiler, or to keep up with its development, see the [RFC for a new compiler for Astro](https://github.com/withastro/roadmap/discussions/1306) for more information and discussion.

packages/astro/package.json

@@ -178,6 +178,7 @@
   },
   "devDependencies": {
     "@astrojs/check": "workspace:*",
+    "@astrojs/compiler-rs": "^0.1.1",
     "@playwright/test": "1.58.2",
     "@types/aria-query": "^5.0.4",
     "@types/cssesc": "^3.0.2",

packages/astro/src/core/compile/compile-rs.ts

@@ -0,0 +1,154 @@
+import { fileURLToPath } from 'node:url';
+import type { ResolvedConfig } from 'vite';
+import type { AstroConfig } from '../../types/public/config.js';
+import type { AstroError } from '../errors/errors.js';
+import { AggregateError, CompilerError } from '../errors/errors.js';
+import { AstroErrorData } from '../errors/index.js';
+import { normalizePath, resolvePath } from '../viteUtils.js';
+import { createStylePreprocessor, type PartialCompileCssResult } from './style.js';
+import type { CompileCssResult } from './types.js';
+
+export interface CompileProps {
+	astroConfig: AstroConfig;
+	viteConfig: ResolvedConfig;
+	toolbarEnabled: boolean;
+	filename: string;
+	source: string;
+}
+
+export interface CompileResult {
+	code: string;
+	map: string;
+	scope: string;
+	css: CompileCssResult[];
+	scripts: any[];
+	hydratedComponents: any[];
+	clientOnlyComponents: any[];
+	serverComponents: any[];
+	containsHead: boolean;
+	propagation: boolean;
+	styleError: string[];
+	diagnostics: any[];
+}
+
+export async function compile({
+	astroConfig,
+	viteConfig,
+	toolbarEnabled,
+	filename,
+	source,
+}: CompileProps): Promise<CompileResult> {
+	let preprocessStyles;
+	let transform;
+	try {
+		 ({ preprocessStyles, transform } = await import('@astrojs/compiler-rs'));
+	}
+	catch (err: unknown) {
+		throw new Error(`Failed to load @astrojs/compiler-rs. Make sure it is installed and up to date. Original error: ${err}`);
+	}
+
+	const cssPartialCompileResults: PartialCompileCssResult[] = [];
+	const cssTransformErrors: AstroError[] = [];
+	let transformResult: any;
+
+	try {
+		const preprocessedStyles = await preprocessStyles(
+			source,
+			createStylePreprocessor({
+				filename,
+				viteConfig,
+				astroConfig,
+				cssPartialCompileResults,
+				cssTransformErrors,
+			}),
+		);
+
+		transformResult = transform(source, {
+			compact: astroConfig.compressHTML,
+			filename,
+			normalizedFilename: normalizeFilename(filename, astroConfig.root),
+			sourcemap: 'both',
+			internalURL: 'astro/compiler-runtime',
+			// TODO: remove in Astro v7
+			astroGlobalArgs: JSON.stringify(astroConfig.site),
+			scopedStyleStrategy: astroConfig.scopedStyleStrategy,
+			resultScopedSlot: true,
+			transitionsAnimationURL: 'astro/components/viewtransitions.css',
+			annotateSourceFile:
+				viteConfig.command === 'serve' &&
+				astroConfig.devToolbar &&
+				astroConfig.devToolbar.enabled &&
+				toolbarEnabled,
+			preprocessedStyles,
+			resolvePath(specifier) {
+				return resolvePath(specifier, filename);
+			},
+		});
+	} catch (err: any) {
+		// The compiler should be able to handle errors by itself, however
+		// for the rare cases where it can't let's directly throw here with as much info as possible
+		throw new CompilerError({
+			...AstroErrorData.UnknownCompilerError,
+			message: err.message ?? 'Unknown compiler error',
+			stack: err.stack,
+			location: {
+				file: filename,
+			},
+		});
+	}
+
+	handleCompileResultErrors(filename, transformResult, cssTransformErrors);
+
+	return {
+		...transformResult,
+		css: transformResult.css.map((code: string, i: number) => ({
+			...cssPartialCompileResults[i],
+			code,
+		})),
+	};
+}
+
+function handleCompileResultErrors(
+	filename: string,
+	result: any,
+	cssTransformErrors: AstroError[],
+) {
+	const compilerError = result.diagnostics.find((diag: any) => diag.severity === 'error');
+
+	if (compilerError) {
+		throw new CompilerError({
+			name: 'CompilerError',
+			message: compilerError.text,
+			location: {
+				line: compilerError.labels[0].line,
+				column: compilerError.labels[0].column,
+				file: filename,
+			},
+			hint: compilerError.hint,
+		});
+	}
+
+	switch (cssTransformErrors.length) {
+		case 0:
+			break;
+		case 1: {
+			throw cssTransformErrors[0];
+		}
+		default: {
+			throw new AggregateError({
+				...cssTransformErrors[0],
+				errors: cssTransformErrors,
+			});
+		}
+	}
+}
+
+function normalizeFilename(filename: string, root: URL) {
+	const normalizedFilename = normalizePath(filename);
+	const normalizedRoot = normalizePath(fileURLToPath(root));
+	if (normalizedFilename.startsWith(normalizedRoot)) {
+		return normalizedFilename.slice(normalizedRoot.length - 1);
+	} else {
+		return normalizedFilename;
+	}
+}

packages/astro/src/core/compile/style.ts

@@ -1,5 +1,4 @@
 import fs from 'node:fs';
-import type { TransformOptions } from '@astrojs/compiler';
 import { preprocessCSS, type ResolvedConfig } from 'vite';
 import type { AstroConfig } from '../../types/public/config.js';
 import { AstroErrorData, CSSError, positionAt } from '../errors/index.js';
@@ -8,6 +7,20 @@ import type { CompileCssResult } from './types.js';
 
 export type PartialCompileCssResult = Pick<CompileCssResult, 'isGlobal' | 'dependencies'>;
 
+interface PreprocessorResult {
+	code: string;
+	map?: string;
+}
+
+interface PreprocessorError {
+	error: string;
+}
+
+export type PreprocessStyleFn = (
+	content: string,
+	attrs: Record<string, string>,
+) => Promise<PreprocessorResult | PreprocessorError>;
+
 /**
  * Rewrites absolute URLs in CSS to include the base path.
  *
@@ -90,7 +103,7 @@ export function createStylePreprocessor({
 	astroConfig: AstroConfig;
 	cssPartialCompileResults: Partial<CompileCssResult>[];
 	cssTransformErrors: Error[];
-}): TransformOptions['preprocessStyle'] {
+}): PreprocessStyleFn {
 	let processedStylesCount = 0;
 
 	return async (content, attrs) => {
@@ -105,7 +118,9 @@ export function createStylePreprocessor({
 			const rewrittenCode = rewriteCssUrls(result.code, astroConfig.base);
 
 			cssPartialCompileResults[index] = {
-				isGlobal: !!attrs['is:global'],
+				// Use `in` operator to handle both Go compiler (boolean `true`) and
+				// Rust compiler (empty string `""`) representations of boolean attributes.
+				isGlobal: 'is:global' in attrs,
 				dependencies: result.deps ? [...result.deps].map((dep) => normalizePath(dep)) : [],
 			};
 

packages/astro/src/core/config/schemas/base.ts

@@ -104,6 +104,7 @@ export const ASTRO_CONFIG_DEFAULTS = {
 		contentIntellisense: false,
 		chromeDevtoolsWorkspace: false,
 		svgo: false,
+		rustCompiler: false,
 	},
 } satisfies AstroUserConfig & { server: { open: boolean } };
 
@@ -493,6 +494,7 @@ export const AstroConfigSchema = z.object({
 				.union([z.boolean(), z.custom<SvgoConfig>((value) => value && typeof value === 'object')])
 				.optional()
 				.default(ASTRO_CONFIG_DEFAULTS.experimental.svgo),
+			rustCompiler: z.boolean().optional().default(ASTRO_CONFIG_DEFAULTS.experimental.rustCompiler),
 		})
 		.prefault({}),
 	legacy: z

packages/astro/src/types/public/config.ts

@@ -2763,6 +2763,30 @@ export interface AstroUserConfig<
 		 * See the [experimental SVGO optimization docs](https://docs.astro.build/en/reference/experimental-flags/svg-optimization/) for more information.
 		 */
 		svgo?: boolean | SvgoConfig;
+
+		/**
+		 * @name experimental.rustCompiler
+		 * @type {boolean}
+		 * @default `false`
+		 * @version 6.0.0
+		 * @description
+		 *
+		 * Enables the experimental Rust-based Astro compiler (`@astrojs/compiler-rs`) as a replacement to the current Go compiler.
+		 * 
+		 * This option requires installing the `@astrojs/compiler-rs` package manually in your project. This compiler is a work in progress and may not yet support all features of the current Go compiler, but it should offer improved performance and better error messages. This compiler is more strict than the previous Go compiler regarding invalid syntax. For instance, unclosed HTML tags or missing closing brackets will throw an error instead of being ignored.
+		 *
+		 * ```js
+		 * // astro.config.mjs
+		 * import { defineConfig } from 'astro/config';
+		 *
+		 * export default defineConfig({
+		 *   experimental: {
+		 *     rustCompiler: true,
+		 *   },
+		 * });
+		 * ```
+		 */
+		rustCompiler?: boolean;
 	};
 }
 

packages/astro/src/vite-plugin-astro/compile-rs.ts

@@ -0,0 +1,52 @@
+import type { SourceMapInput } from 'rollup';
+import { type CompileProps, type CompileResult, compile } from '../core/compile/compile-rs.js';
+import { getFileInfo } from '../vite-plugin-utils/index.js';
+import type { CompileMetadata } from './types.js';
+
+interface CompileAstroOption {
+	compileProps: CompileProps;
+	astroFileToCompileMetadata: Map<string, CompileMetadata>;
+}
+
+export interface CompileAstroResult extends Omit<CompileResult, 'map'> {
+	map: SourceMapInput;
+}
+
+export async function compileAstro({
+	compileProps,
+	astroFileToCompileMetadata,
+}: CompileAstroOption): Promise<CompileAstroResult> {
+	const transformResult = await compile(compileProps);
+
+	const { fileId: file, fileUrl: url } = getFileInfo(
+		compileProps.filename,
+		compileProps.astroConfig,
+	);
+
+	let SUFFIX = '';
+	SUFFIX += `\nconst $$file = ${JSON.stringify(file)};\nconst $$url = ${JSON.stringify(
+		url,
+	)};export { $$file as file, $$url as url };\n`;
+
+	// Add HMR handling in dev mode.
+	if (!compileProps.viteConfig.isProduction) {
+		let i = 0;
+		while (i < transformResult.scripts.length) {
+			SUFFIX += `import "${compileProps.filename}?astro&type=script&index=${i}&lang.ts";`;
+			i++;
+		}
+	}
+
+	// Attach compile metadata to map for use by virtual modules
+	astroFileToCompileMetadata.set(compileProps.filename, {
+		originalCode: compileProps.source,
+		css: transformResult.css,
+		scripts: transformResult.scripts,
+	});
+
+	return {
+		...transformResult,
+		code: transformResult.code + SUFFIX,
+		map: transformResult.map || null,
+	};
+}

packages/astro/src/vite-plugin-astro/compile.ts

@@ -5,6 +5,7 @@ import type { AstroConfig } from '../types/public/config.js';
 import { getFileInfo } from '../vite-plugin-utils/index.js';
 import type { CompileMetadata } from './types.js';
 import { frontmatterRE } from './utils.js';
+import type { SourceMapInput } from 'rollup';
 
 interface CompileAstroOption {
 	compileProps: CompileProps;
@@ -13,7 +14,7 @@ interface CompileAstroOption {
 }
 
 export interface CompileAstroResult extends Omit<CompileResult, 'map'> {
-	map: ESBuildTransformResult['map'];
+	map: SourceMapInput;
 }
 
 interface EnhanceCompilerErrorOptions {

packages/astro/src/vite-plugin-astro/index.ts

@@ -9,6 +9,7 @@ import type { AstroSettings } from '../types/astro.js';
 import type { AstroConfig } from '../types/public/config.js';
 import { normalizeFilename, specialQueriesRE } from '../vite-plugin-utils/index.js';
 import { type CompileAstroResult, compileAstro } from './compile.js';
+import { compileAstro as compileAstroRs } from './compile-rs.js';
 import { handleHotUpdate } from './hmr.js';
 import { parseAstroRequest } from './query.js';
 import type { PluginMetadata as AstroPluginMetadata, CompileMetadata } from './types.js';
@@ -90,14 +91,21 @@ export default function astro({ settings, logger }: AstroPluginOptions): vite.Pl
 				const toolbarEnabled = await settings.preferences.get('devToolbar.enabled');
 				// Initialize `compile` function to simplify usage later
 				compile = (code, filename) => {
+					const compileProps = {
+						astroConfig: config,
+						viteConfig,
+						toolbarEnabled,
+						filename,
+						source: code,
+					};
+					if (config.experimental.rustCompiler) {
+						return compileAstroRs({
+							compileProps,
+							astroFileToCompileMetadata,
+						});
+					}
 					return compileAstro({
-						compileProps: {
-							astroConfig: config,
-							viteConfig,
-							toolbarEnabled,
-							filename,
-							source: code,
-						},
+						compileProps,
 						astroFileToCompileMetadata,
 						logger,
 					});

packages/astro/test/astro-partial-html.test.js

@@ -26,8 +26,8 @@ describe('Partial HTML', async () => {
 		assert.match(html, /^<!DOCTYPE html/);
 
 		// test 2: correct CSS present
-		const allInjectedStyles = $('style').text();
-		assert.match(allInjectedStyles, /\[data-astro-cid-[^{]+\{color:red\}/);
+		const allInjectedStyles = $('style').text().replace(/\s*/g, '');
+		assert.match(allInjectedStyles, /\[data-astro-cid-[^{]+\{color:red;?\}/);
 	});
 
 	it('injects framework styles', async () => {