diff --git a/.changeset/internal-helpers-content-modules.md b/.changeset/internal-helpers-content-modules.md new file mode 100644 index 000000000000..76917f5a26cf --- /dev/null +++ b/.changeset/internal-helpers-content-modules.md @@ -0,0 +1,5 @@ +--- +'@astrojs/internal-helpers': minor +--- + +Adds `markdown`, `frontmatter`, and `shiki` helper modules, shared by Astro's content pipeline. diff --git a/.changeset/mdx-processor-option.md b/.changeset/mdx-processor-option.md new file mode 100644 index 000000000000..76985800467b --- /dev/null +++ b/.changeset/mdx-processor-option.md @@ -0,0 +1,17 @@ +--- +'@astrojs/mdx': major +--- + +`@astrojs/mdx` now picks up the processor from `markdown.processor` automatically, so a single Sätteri or unified configuration drives both `.md` and `.mdx` files. To run `.mdx` files through a different processor (or the same processor with different options) than your `.md` files, pass the `processor` option to the integration: + +```js +// astro.config.mjs +import { defineConfig, satteri } from 'astro/config'; +import mdx from '@astrojs/mdx'; +import { unified } from '@astrojs/markdown-remark'; + +export default defineConfig({ + markdown: { processor: satteri() }, + integrations: [mdx({ processor: unified({ remarkPlugins: [/* ... */] }) })], +}); +``` diff --git a/.changeset/seven-emus-knock.md b/.changeset/seven-emus-knock.md new file mode 100644 index 000000000000..7a5a2dfd8f4f --- /dev/null +++ b/.changeset/seven-emus-knock.md @@ -0,0 +1,58 @@ +--- +'astro': major +'@astrojs/markdown-remark': major +'@astrojs/markdown-satteri': major +--- + +Adds a `markdown.processor` config option to configure the Markdown processor to use. By default, [Sätteri](https://github.com/bruits/satteri) a fast Rust-based Markdown/MDX compiler is used. Both `.md` and `.mdx` files now render through Sätteri. + +To pass Sätteri plugins or enable additional parser features, use the `satteri()` explicitely: + +```js +// astro.config.mjs +import { defineConfig, satteri } from 'astro/config'; + +export default defineConfig({ + markdown: { + processor: satteri({ + hastPlugins: [myPlugin], + features: { directive: true, smartPunctuation: false }, + }), + }, +}); +``` + +To keep using your existing remark/rehype plugins, install `@astrojs/markdown-remark` into your project and use the `unified` export: + +```sh +pnpm add @astrojs/markdown-remark +``` + +```js +// astro.config.mjs +import { defineConfig } from 'astro/config'; +import { unified } from '@astrojs/markdown-remark'; +import remarkToc from 'remark-toc'; + +export default defineConfig({ + markdown: { + processor: unified({ remarkPlugins: [remarkToc] }), + }, +}); +``` + +The top-level `markdown.remarkPlugins`, `markdown.rehypePlugins`, and `markdown.remarkRehype` options are deprecated. They'll continue to work when `@astrojs/markdown-remark` is installed for now, but this will be removed in the next major. + +The top-level `markdown.gfm` and `markdown.smartypants` options are also deprecated. Move them onto your processor instead — `satteri({ features: { gfm: false, smartPunctuation: false } })`, or `unified({ gfm: false, smartypants: false })`: + +```js +// astro.config.mjs +import { defineConfig } from 'astro/config'; +import { unified } from '@astrojs/markdown-remark'; + +export default defineConfig({ + markdown: { + processor: unified({ gfm: false, smartypants: false }), + }, +}); +``` diff --git a/knip.js b/knip.js index 9e08e67c53d2..600e677d4d44 100644 --- a/knip.js +++ b/knip.js @@ -74,6 +74,11 @@ export default { // False positive because of cloudflare:workers ignoreDependencies: ['cloudflare'], }, + 'packages/integrations/mdx': { + entry: [srcEntry, dtsEntry, testEntry], + // Optional peer dep, only loaded when the user picks the unified processor + ignoreDependencies: ['@astrojs/markdown-remark'], + }, 'packages/integrations/netlify': { entry: [srcEntry, dtsEntry, testEntry], }, @@ -85,6 +90,9 @@ export default { 'packages/markdown/remark': { entry: [srcEntry, dtsEntry, testEntry], }, + 'packages/markdown/satteri': { + entry: [srcEntry, dtsEntry, testEntry], + }, 'packages/upgrade': { entry: ['src/index.ts', testEntry], }, diff --git a/packages/astro/components/Code.astro b/packages/astro/components/Code.astro index 846d54e4d14c..ee2e4589152d 100644 --- a/packages/astro/components/Code.astro +++ b/packages/astro/components/Code.astro @@ -1,5 +1,5 @@ --- -import { createShikiHighlighter, type ThemePresets } from '@astrojs/markdown-remark/shiki'; +import { createShikiHighlighter, type ThemePresets } from '@astrojs/internal-helpers/shiki'; import type { ShikiTransformer, ThemeRegistration, ThemeRegistrationRaw } from 'shiki'; import { bundledLanguages } from 'shiki/langs'; import type { CodeLanguage } from '../dist/types/public/common.js'; diff --git a/packages/astro/e2e/nested-in-svelte.test.ts b/packages/astro/e2e/nested-in-svelte.test.ts index 8c4e873fa19c..c57a8479612e 100644 --- a/packages/astro/e2e/nested-in-svelte.test.ts +++ b/packages/astro/e2e/nested-in-svelte.test.ts @@ -14,88 +14,88 @@ test.afterAll(async () => { }); test.describe('Nested Frameworks in Svelte', () => { - test('React counter', async ({ astro, page }) => { - await page.goto(astro.resolveUrl('/')); + test('React counter', async ({ astro, page }) => { + await page.goto(astro.resolveUrl('/')); - const counter = page.locator('#react-counter'); - await expect(counter, 'component is visible').toBeVisible(); + const counter = page.locator('#react-counter'); + await expect(counter, 'component is visible').toBeVisible(); - const count = counter.locator('#react-counter-count'); - await expect(count, 'initial count is 0').toHaveText('0'); + const count = counter.locator('#react-counter-count'); + await expect(count, 'initial count is 0').toHaveText('0'); - await waitForHydrate(page, counter); + await waitForHydrate(page, counter); - const increment = counter.locator('#react-counter-increment'); - await increment.click(); + const increment = counter.locator('#react-counter-increment'); + await increment.click(); - await expect(count, 'count incremented by 1').toHaveText('1'); - }); + await expect(count, 'count incremented by 1').toHaveText('1'); + }); - test('Preact counter', async ({ astro, page }) => { - await page.goto(astro.resolveUrl('/')); + test('Preact counter', async ({ astro, page }) => { + await page.goto(astro.resolveUrl('/')); - const counter = page.locator('#preact-counter'); - await expect(counter, 'component is visible').toBeVisible(); + const counter = page.locator('#preact-counter'); + await expect(counter, 'component is visible').toBeVisible(); - const count = counter.locator('#preact-counter-count'); - await expect(count, 'initial count is 0').toHaveText('0'); + const count = counter.locator('#preact-counter-count'); + await expect(count, 'initial count is 0').toHaveText('0'); - await waitForHydrate(page, counter); + await waitForHydrate(page, counter); - const increment = counter.locator('#preact-counter-increment'); - await increment.click(); + const increment = counter.locator('#preact-counter-increment'); + await increment.click(); - await expect(count, 'count incremented by 1').toHaveText('1'); - }); + await expect(count, 'count incremented by 1').toHaveText('1'); + }); - test('Solid counter', async ({ astro, page }) => { - await page.goto(astro.resolveUrl('/')); + test('Solid counter', async ({ astro, page }) => { + await page.goto(astro.resolveUrl('/')); - const counter = page.locator('#solid-counter'); - await expect(counter, 'component is visible').toBeVisible(); + const counter = page.locator('#solid-counter'); + await expect(counter, 'component is visible').toBeVisible(); - const count = counter.locator('#solid-counter-count'); - await expect(count, 'initial count is 0').toHaveText('0'); + const count = counter.locator('#solid-counter-count'); + await expect(count, 'initial count is 0').toHaveText('0'); - await waitForHydrate(page, counter); + await waitForHydrate(page, counter); - const increment = counter.locator('#solid-counter-increment'); - await increment.click(); + const increment = counter.locator('#solid-counter-increment'); + await increment.click(); - await expect(count, 'count incremented by 1').toHaveText('1'); - }); + await expect(count, 'count incremented by 1').toHaveText('1'); + }); - test('Vue counter', async ({ astro, page }) => { - await page.goto(astro.resolveUrl('/')); + test('Vue counter', async ({ astro, page }) => { + await page.goto(astro.resolveUrl('/')); - const counter = page.locator('#vue-counter'); - await expect(counter, 'component is visible').toBeVisible(); + const counter = page.locator('#vue-counter'); + await expect(counter, 'component is visible').toBeVisible(); - const count = counter.locator('#vue-counter-count'); - await expect(count, 'initial count is 0').toHaveText('0'); + const count = counter.locator('#vue-counter-count'); + await expect(count, 'initial count is 0').toHaveText('0'); - await waitForHydrate(page, counter); + await waitForHydrate(page, counter); - const increment = counter.locator('#vue-counter-increment'); - await increment.click(); + const increment = counter.locator('#vue-counter-increment'); + await increment.click(); - await expect(count, 'count incremented by 1').toHaveText('1'); - }); + await expect(count, 'count incremented by 1').toHaveText('1'); + }); - test('Svelte counter', async ({ astro, page }) => { - await page.goto(astro.resolveUrl('/')); + test('Svelte counter', async ({ astro, page }) => { + await page.goto(astro.resolveUrl('/')); - const counter = page.locator('#svelte-counter'); - await expect(counter, 'component is visible').toBeVisible(); + const counter = page.locator('#svelte-counter'); + await expect(counter, 'component is visible').toBeVisible(); - const count = counter.locator('#svelte-counter-count'); - await expect(count, 'initial count is 0').toHaveText('0'); + const count = counter.locator('#svelte-counter-count'); + await expect(count, 'initial count is 0').toHaveText('0'); - await waitForHydrate(page, counter); + await waitForHydrate(page, counter); - const increment = counter.locator('#svelte-counter-increment'); - await increment.click(); + const increment = counter.locator('#svelte-counter-increment'); + await increment.click(); - await expect(count, 'count incremented by 1').toHaveText('1'); - }); + await expect(count, 'count incremented by 1').toHaveText('1'); }); +}); diff --git a/packages/astro/package.json b/packages/astro/package.json index d3fea6989372..5aedb8997c18 100644 --- a/packages/astro/package.json +++ b/packages/astro/package.json @@ -31,6 +31,7 @@ }, "exports": { ".": "./dist/index.js", + "./markdown": "./dist/markdown/index.js", "./env": "./env.d.ts", "./env/runtime": "./dist/env/runtime.js", "./env/setup": "./dist/env/setup.js", @@ -39,7 +40,6 @@ "./astro-jsx": "./astro-jsx.d.ts", "./tsconfigs/*.json": "./tsconfigs/*", "./tsconfigs/*": "./tsconfigs/*.json", - "./jsx/rehype.js": "./dist/jsx/rehype.js", "./jsx-runtime": { "types": "./jsx-runtime.d.ts", "default": "./dist/jsx-runtime/index.js" @@ -135,7 +135,7 @@ "dependencies": { "@astrojs/compiler-rs": "^0.1.10", "@astrojs/internal-helpers": "workspace:*", - "@astrojs/markdown-remark": "workspace:*", + "@astrojs/markdown-satteri": "workspace:*", "@astrojs/telemetry": "workspace:*", "@capsizecss/unpack": "^4.0.0", "@clack/prompts": "^1.1.0", @@ -194,6 +194,7 @@ }, "devDependencies": { "@astrojs/check": "workspace:*", + "@astrojs/markdown-remark": "workspace:*", "@playwright/test": "1.58.2", "@types/aria-query": "^5.0.4", "@types/hast": "^3.0.4", @@ -210,8 +211,6 @@ "expect-type": "^1.3.0", "fs-fixture": "^2.13.0", "hono": "^4.12.14", - "mdast-util-mdx": "^3.0.0", - "mdast-util-mdx-jsx": "^3.2.0", "node-mocks-http": "^1.17.2", "parse-srcset": "^1.0.2", "rehype-autolink-headings": "^7.1.0", @@ -232,6 +231,7 @@ "publishConfig": { "exports": { ".": "./dist/index.js", + "./markdown": "./dist/markdown/index.js", "./env": "./env.d.ts", "./env/runtime": "./dist/env/runtime.js", "./env/setup": "./dist/env/setup.js", @@ -240,7 +240,6 @@ "./astro-jsx": "./astro-jsx.d.ts", "./tsconfigs/*.json": "./tsconfigs/*", "./tsconfigs/*": "./tsconfigs/*.json", - "./jsx/rehype.js": "./dist/jsx/rehype.js", "./jsx-runtime": { "types": "./jsx-runtime.d.ts", "default": "./dist/jsx-runtime/index.js" diff --git a/packages/astro/src/config/entrypoint.ts b/packages/astro/src/config/entrypoint.ts index cd252d2b6fa2..c2c9d9eda0f8 100644 --- a/packages/astro/src/config/entrypoint.ts +++ b/packages/astro/src/config/entrypoint.ts @@ -14,6 +14,7 @@ export { defineConfig, getViteConfig } from './index.js'; export { sessionDrivers } from '../core/session/drivers.js'; export { svgoOptimizer } from '../assets/svg/svgo.js'; export { logHandlers } from '../core/logger/handlers.js'; +export { satteri } from '@astrojs/markdown-satteri'; /** * Return the configuration needed to use the Sharp-based image service diff --git a/packages/astro/src/content/content-layer.ts b/packages/astro/src/content/content-layer.ts index d1eda4d9838f..da4aa710edc4 100644 --- a/packages/astro/src/content/content-layer.ts +++ b/packages/astro/src/content/content-layer.ts @@ -1,9 +1,6 @@ import { existsSync, promises as fs } from 'node:fs'; -import { - createMarkdownProcessor, - parseFrontmatter, - type MarkdownProcessor, -} from '@astrojs/markdown-remark'; +import { parseFrontmatter } from '@astrojs/internal-helpers/frontmatter'; +import type { MarkdownProcessor } from '@astrojs/internal-helpers/markdown'; import PQueue from 'p-queue'; import type { FSWatcher } from 'vite'; import xxhash from 'xxhash-wasm'; @@ -154,7 +151,16 @@ export class ContentLayer { content: string, options?: RenderMarkdownOptions, ): Promise { - this.#markdownProcessor ??= await createMarkdownProcessor(this.#settings.config.markdown); + if (!this.#markdownProcessor) { + const { markdown, image } = this.#settings.config; + this.#markdownProcessor = await markdown.processor.createRenderer({ + image, + syntaxHighlight: markdown.syntaxHighlight, + shikiConfig: markdown.shikiConfig, + gfm: markdown.gfm, + smartypants: markdown.smartypants, + }); + } const { frontmatter, content: body } = parseFrontmatter(content); const { code, metadata } = await this.#markdownProcessor.render(body, { frontmatter, diff --git a/packages/astro/src/content/data-store.ts b/packages/astro/src/content/data-store.ts index 1e89919feacb..35565df9a1a5 100644 --- a/packages/astro/src/content/data-store.ts +++ b/packages/astro/src/content/data-store.ts @@ -1,4 +1,4 @@ -import type { MarkdownHeading } from '@astrojs/markdown-remark'; +import type { MarkdownHeading } from '@astrojs/internal-helpers/markdown'; import * as devalue from 'devalue'; export interface RenderedContent { diff --git a/packages/astro/src/content/runtime.ts b/packages/astro/src/content/runtime.ts index 7d4fed72ddba..13beface7c30 100644 --- a/packages/astro/src/content/runtime.ts +++ b/packages/astro/src/content/runtime.ts @@ -1,4 +1,4 @@ -import type { MarkdownHeading } from '@astrojs/markdown-remark'; +import type { MarkdownHeading } from '@astrojs/internal-helpers/markdown'; import { escape } from 'html-escaper'; import { Traverse } from 'neotraverse/modern'; import * as z from 'zod/v4'; @@ -461,7 +461,9 @@ async function updateImageReferencesInBody(html: string, fileName: string) { // function because getImage is async. for (const [_full, imagePath] of html.matchAll(CONTENT_LAYER_IMAGE_REGEX)) { try { - const decodedImagePath = JSON.parse(imagePath.replaceAll('"', '"')); + const decodedImagePath = JSON.parse( + imagePath.replace(/&(?:#x22|quot);/g, '"').replace(/&(?:#x27|apos);/g, "'"), + ); let image: GetImageResult; if (URL.canParse(decodedImagePath.src)) { diff --git a/packages/astro/src/content/utils.ts b/packages/astro/src/content/utils.ts index 9139789070ab..39e77e2da96b 100644 --- a/packages/astro/src/content/utils.ts +++ b/packages/astro/src/content/utils.ts @@ -1,7 +1,7 @@ import fsMod from 'node:fs'; import path from 'node:path'; import { fileURLToPath, pathToFileURL } from 'node:url'; -import { parseFrontmatter } from '@astrojs/markdown-remark'; +import { parseFrontmatter } from '@astrojs/internal-helpers/frontmatter'; import { slug as githubSlug } from 'github-slugger'; import colors from 'piccolore'; import type { RunnableDevEnvironment, Rolldown } from 'vite'; diff --git a/packages/astro/src/core/config/merge.ts b/packages/astro/src/core/config/merge.ts index 5f6487147998..b51e540d8e45 100644 --- a/packages/astro/src/core/config/merge.ts +++ b/packages/astro/src/core/config/merge.ts @@ -58,6 +58,19 @@ function mergeConfigRecursively( merged[key] = value; continue; } + // Markdown processor descriptors (satteri(), unified(), third-party) are opaque + // objects with a `createRenderer` function — treat them as atomic so swapping + // processors via updateConfig replaces cleanly instead of merging fields from + // the previous descriptor into the new one. + if ( + key === 'processor' && + rootPath === 'markdown' && + isObject(value) && + typeof value.createRenderer === 'function' + ) { + merged[key] = value; + continue; + } if (isObject(existing) && isObject(value)) { merged[key] = mergeConfigRecursively(existing, value, rootPath ? `${rootPath}.${key}` : key); continue; diff --git a/packages/astro/src/core/config/schemas/base.ts b/packages/astro/src/core/config/schemas/base.ts index d5611f59d75d..9e78e1b79b45 100644 --- a/packages/astro/src/core/config/schemas/base.ts +++ b/packages/astro/src/core/config/schemas/base.ts @@ -1,11 +1,16 @@ +import { + markdownConfigDefaults, + type ShikiConfig, + type Smartypants as _Smartypants, + syntaxHighlightDefaults, +} from '@astrojs/internal-helpers/markdown'; +import { satteri } from '@astrojs/markdown-satteri'; +import type { MarkdownProcessorEntry } from '../../../markdown/index.js'; import type { RehypePlugin as _RehypePlugin, RemarkPlugin as _RemarkPlugin, RemarkRehype as _RemarkRehype, - Smartypants as _Smartypants, - ShikiConfig, -} from '@astrojs/markdown-remark'; -import { markdownConfigDefaults, syntaxHighlightDefaults } from '@astrojs/markdown-remark'; +} from '../../../types/public/config.js'; import type { OutgoingHttpHeaders } from 'node:http'; import { type BuiltinTheme, bundledThemes } from 'shiki'; import * as z from 'zod/v4'; @@ -417,14 +422,41 @@ export const AstroConfigSchema = z.object({ remarkRehype: z .custom((data) => data instanceof Object && !Array.isArray(data)) .default(ASTRO_CONFIG_DEFAULTS.markdown.remarkRehype), - gfm: z.boolean().default(ASTRO_CONFIG_DEFAULTS.markdown.gfm), + // Deprecated: left undefined unless the user explicitly sets them, so the + // deprecation warning only fires when actually used. The active processor + // (`satteri()` / `unified()`) supplies the real default (`gfm`/smart + // punctuation on) when these are absent. + gfm: z.boolean().optional(), smartypants: z .union([z.boolean(), smartypantsOptionsSchema]) .transform((val): false | Smartypants => { if (val === true) return smartypantsOptionsSchema.parse({}); return val; }) - .prefault(ASTRO_CONFIG_DEFAULTS.markdown.smartypants), + .optional(), + processor: z + .object({ + name: z.string(), + // `z.custom` preserves reference identity; `z.record` would clone, breaking + // the closure inside `createRenderer` that reads `descriptor.options.*`. + options: z + .custom>( + (v) => typeof v === 'object' && v !== null && !Array.isArray(v), + ) + .default(() => ({})), + createRenderer: z.custom( + (v) => typeof v === 'function', + ), + createMdxRenderer: z + .custom( + (v) => v === undefined || typeof v === 'function', + ) + .optional(), + }) + // A factory (not a shared value) so every config gets its own descriptor — + // integrations extend the pipeline by mutating `processor.options`, which + // would otherwise leak across configs built in the same process. + .default(() => satteri()), }) .prefault({}), vite: z diff --git a/packages/astro/src/core/config/validate.ts b/packages/astro/src/core/config/validate.ts index 50bd9bf522d8..91c1ad165735 100644 --- a/packages/astro/src/core/config/validate.ts +++ b/packages/astro/src/core/config/validate.ts @@ -1,4 +1,4 @@ -import type { AstroConfig } from '../../types/public/config.js'; +import type { AstroConfig, AstroUserConfig } from '../../types/public/config.js'; import { errorMap } from '../errors/index.js'; import { AstroConfigRefinedSchema, createRelativeSchema } from './schemas/index.js'; @@ -10,6 +10,9 @@ export async function validateConfig( ): Promise { const AstroConfigRelativeSchema = createRelativeSchema(cmd, root); + await coerceLegacyMarkdownPlugins(userConfig); + warnDeprecatedMarkdownOptions(userConfig); + // First-Pass Validation return await validateConfigRefined( await AstroConfigRelativeSchema.parseAsync(userConfig, { @@ -26,11 +29,103 @@ export async function validateConfig( ); } +let didWarnAboutDeprecatedMarkdownOptions = false; + +function warnDeprecatedMarkdownOptions( + config: Pick | undefined, +): void { + if (didWarnAboutDeprecatedMarkdownOptions) return; + const md = config?.markdown; + if (!md) return; + const deprecated = (['gfm', 'smartypants'] as const).filter((k) => md[k] !== undefined); + if (deprecated.length === 0) return; + didWarnAboutDeprecatedMarkdownOptions = true; + + const names = deprecated.map((key) => `\`markdown.${key}\``).join(' and '); + const isPlural = deprecated.length > 1; + console.warn( + `[astro] ${names} ${isPlural ? 'are' : 'is'} deprecated. Move ${isPlural ? 'them' : 'it'} onto your processor instead (e.g. \`unified({ gfm: false, smartypants: false })\` or \`satteri({ features: { gfm: false, smartPunctuation: false } })\`). Will be removed in a future major.`, + ); +} + +let didWarnAboutLegacyMarkdownPlugins = false; +let didWarnAboutProcessorSwap = false; + +/** + * Folds legacy `markdown.{remark,rehype}Plugins` / `remarkRehype` into a unified + * processor descriptor and clears the legacy keys. Mutates `config` in place. + * Runs on every validate pass so integration-added legacy plugins are picked up. + * Without this they'd land in `config.markdown.*` and be silently dropped. + */ +async function coerceLegacyMarkdownPlugins( + config: Pick, +): Promise { + const md = config?.markdown; + if (!md) return; + + const remarkPlugins = md.remarkPlugins ?? []; + const rehypePlugins = md.rehypePlugins ?? []; + const remarkRehype = md.remarkRehype ?? {}; + if ( + remarkPlugins.length === 0 && + rehypePlugins.length === 0 && + Object.keys(remarkRehype).length === 0 + ) { + return; + } + + let unified: typeof import('@astrojs/markdown-remark').unified; + let isUnifiedProcessor: typeof import('@astrojs/markdown-remark').isUnifiedProcessor; + try { + ({ unified, isUnifiedProcessor } = await import('@astrojs/markdown-remark')); + } catch { + throw new Error( + '`markdown.remarkPlugins`, `markdown.rehypePlugins`, and `markdown.remarkRehype` require `@astrojs/markdown-remark`. Install it with:\n npm install @astrojs/markdown-remark', + ); + } + + const current = md.processor; + if (current && isUnifiedProcessor(current)) { + // `createRenderer` reads `.options.*` at render time, so in-place mutation propagates. + current.options.remarkPlugins.push(...remarkPlugins); + current.options.rehypePlugins.push(...rehypePlugins); + Object.assign(current.options.remarkRehype, remarkRehype); + if (!didWarnAboutLegacyMarkdownPlugins) { + didWarnAboutLegacyMarkdownPlugins = true; + console.warn( + '[astro] `markdown.remarkPlugins`, `markdown.rehypePlugins`, and `markdown.remarkRehype` are deprecated. Pass them to `unified({...})` from `@astrojs/markdown-remark` directly instead.', + ); + } + } else { + // Sätteri and third-party processors can't run remark/rehype plugins, so legacy + // plugins force a swap to `unified()`. Warn loudly — this changes the active + // Markdown processor, e.g. away from the default native pipeline. + if (!didWarnAboutProcessorSwap) { + didWarnAboutProcessorSwap = true; + const replaced = current?.name ?? 'satteri'; + console.warn( + `[astro] Found deprecated \`markdown.remarkPlugins\`/\`rehypePlugins\`/\`remarkRehype\`, so the ` + + `Markdown processor was switched from \`${replaced}\`` + + `${replaced === 'satteri' ? ' (the default native pipeline)' : ''} to \`unified\`. ` + + `To keep \`${replaced}\`, remove those options; to silence this warning, set ` + + '`markdown.processor: unified({...})` from `@astrojs/markdown-remark` explicitly.', + ); + } + md.processor = unified({ remarkPlugins, rehypePlugins, remarkRehype }); + } + + delete md.remarkPlugins; + delete md.rehypePlugins; + delete md.remarkRehype; +} + /** * Used twice: * - To validate the user config * - To validate the config after all integrations (that may have updated it) */ export async function validateConfigRefined(updatedConfig: AstroConfig): Promise { + await coerceLegacyMarkdownPlugins(updatedConfig); + warnDeprecatedMarkdownOptions(updatedConfig); return await AstroConfigRefinedSchema.parseAsync(updatedConfig, { error: errorMap }); } diff --git a/packages/astro/src/markdown/index.ts b/packages/astro/src/markdown/index.ts new file mode 100644 index 000000000000..8b4d96a628ba --- /dev/null +++ b/packages/astro/src/markdown/index.ts @@ -0,0 +1,66 @@ +import type { + AstroMarkdownProcessorOptions, + AstroMetadata as MdxAstroMetadata, + MarkdownProcessor, +} from '@astrojs/internal-helpers/markdown'; +import type { PluggableList } from 'unified'; + +export { + extractFrontmatter, + isFrontmatterValid, + parseFrontmatter, + type ParseFrontmatterOptions, + type ParseFrontmatterResult, +} from '@astrojs/internal-helpers/frontmatter'; +export { resolvePath } from '../core/viteUtils.js'; +export type { AstroMarkdownProcessorOptions } from '@astrojs/internal-helpers/markdown'; +export type { MdxAstroMetadata }; + +/** + * Configuration that ends up on `markdown.processor`. Returned by factory functions like + * `satteri()` or `unified()`. Third-party processors implement this interface to plug in + * their own markdown rendering pipeline. + */ +export interface MarkdownProcessorEntry< + TOptions extends Record = Record, +> { + /** Identifier for this processor. Used by integrations to look up built-in MDX support. */ + readonly name: string; + /** + * Processor-specific options. + */ + options?: TOptions; + /** Create the runtime renderer for `.md` files. */ + createRenderer(shared: AstroMarkdownProcessorOptions): Promise; + /** + * Create the runtime renderer for `.mdx` files. Optional — when absent, `@astrojs/mdx` + * falls back to its built-in handling for the known `satteri` and `unified` processor + * names. Third-party processors should provide this to enable MDX support. + */ + createMdxRenderer?( + shared: AstroMarkdownProcessorOptions, + mdx: MdxRendererOptions, + ): Promise; +} + +/** Cross-cutting MDX options passed to `createMdxRenderer` regardless of processor. */ +export interface MdxRendererOptions { + optimize: boolean | { ignoreElementNames?: string[] }; + recmaPlugins: PluggableList; +} + +/** Runtime renderer for `.mdx` files returned by `createMdxRenderer`. */ +export interface MdxRenderer { + process( + content: string, + filePath: string, + frontmatter: Record, + ): Promise; +} + +export interface MdxRenderResult { + code: string; + /** Source map. Stringified to satisfy Vite's `SourceMapInput`. */ + map?: string | null; + astroMetadata: MdxAstroMetadata; +} diff --git a/packages/astro/src/types/public/config.ts b/packages/astro/src/types/public/config.ts index 0b579de20972..6c9646996379 100644 --- a/packages/astro/src/types/public/config.ts +++ b/packages/astro/src/types/public/config.ts @@ -7,15 +7,17 @@ import type { ShikiConfig, Smartypants, SyntaxHighlightConfigType, -} from '@astrojs/markdown-remark'; +} from '@astrojs/internal-helpers/markdown'; import type { UserConfig as OriginalViteUserConfig, SSROptions as ViteSSROptions } from 'vite'; import type { FontFamily, FontProvider } from '../../assets/fonts/types.js'; +import type { SvgOptimizer } from '../../assets/svg/types.js'; import type { ImageFit, ImageLayout } from '../../assets/types.js'; import type { AssetsPrefix } from '../../core/app/types.js'; import type { CacheProviderConfig, RouteRules } from '../../core/cache/types.js'; import type { AstroConfigType } from '../../core/config/schemas/index.js'; import type { REDIRECT_STATUS_CODES } from '../../core/constants.js'; import type { CspAlgorithm, CspDirective, CspHash } from '../../core/csp/config.js'; +import type { LoggerHandlerConfig } from '../../core/logger/config.js'; import type { AstroLogger, AstroLoggerLevel } from '../../core/logger/core.js'; import type { SessionConfig, @@ -23,9 +25,19 @@ import type { SessionDriverName, } from '../../core/session/types.js'; import type { EnvSchema } from '../../env/schema.js'; +import type { MarkdownProcessorEntry } from '../../markdown/index.js'; import type { AstroIntegration } from './integrations.js'; -import type { SvgOptimizer } from '../../assets/svg/types.js'; -import type { LoggerHandlerConfig } from '../../core/logger/config.js'; + +// Plugin/option shapes for the deprecated `markdown.remarkPlugins` / +// `rehypePlugins` / `remarkRehype` fields. Defined in `@astrojs/internal-helpers/markdown` +// alongside the rest of the markdown config contract. +export type { + RehypePlugin, + RehypePlugins, + RemarkPlugin, + RemarkPlugins, + RemarkRehype, +} from '@astrojs/internal-helpers/markdown'; export type Locales = (string | { codes: [string, ...string[]]; path: string })[]; @@ -2085,6 +2097,7 @@ export interface AstroUserConfig< * @docs * @name markdown.remarkPlugins * @type {RemarkPlugins} + * @deprecated Use `markdown.processor: unified({ remarkPlugins })` from `@astrojs/markdown-remark` instead. Will be removed in a future major. * @description * Pass [remark plugins](https://github.com/remarkjs/remark) to customize how your Markdown is built. You can import and apply the plugin function (recommended), or pass the plugin name as a string. * @@ -2102,6 +2115,7 @@ export interface AstroUserConfig< * @docs * @name markdown.rehypePlugins * @type {RehypePlugins} + * @deprecated Use `markdown.processor: unified({ rehypePlugins })` from `@astrojs/markdown-remark` instead. Will be removed in a future major. * @description * Pass [rehype plugins](https://github.com/remarkjs/remark-rehype) to customize how your Markdown's output HTML is processed. You can import and apply the plugin function (recommended), or pass the plugin name as a string. * @@ -2121,6 +2135,7 @@ export interface AstroUserConfig< * @type {boolean} * @default `true` * @version 2.0.0 + * @deprecated Configure via the processor instead: `unified({ gfm: false })` or `satteri({ features: { gfm: false } })`. Will be removed in a future major. * @description * Astro uses [GitHub-flavored Markdown](https://github.com/remarkjs/remark-gfm) by default. To disable this, set the `gfm` flag to `false`: * @@ -2140,6 +2155,7 @@ export interface AstroUserConfig< * @type {boolean | Smartypants} * @default `true` * @version 2.0.0 + * @deprecated Configure via the processor instead: `unified({ smartypants: false })` or `satteri({ features: { smartPunctuation: false } })`. Will be removed in a future major. * @description * Whether to use the [SmartyPants formatter](https://daringfireball.net/projects/smartypants/) to transform straight quotes into smart quotes, dashes into en/em dashes, and triple dots into ellipses. * @@ -2153,6 +2169,7 @@ export interface AstroUserConfig< * @docs * @name markdown.remarkRehype * @type {RemarkRehype} + * @deprecated Use `markdown.processor: unified({ remarkRehype })` from `@astrojs/markdown-remark` instead. Will be removed in a future major. * @description * Pass options to [remark-rehype](https://github.com/remarkjs/remark-rehype#api). * @@ -2166,6 +2183,47 @@ export interface AstroUserConfig< * ``` */ remarkRehype?: RemarkRehype; + + /** + * @docs + * @name markdown.processor + * @type {MarkdownProcessorEntry} + * @default `satteri()` + * @version 7.0.0 + * @description + * + * The markdown processor used to render `.md` and `.mdx` files. By default Astro uses [Sätteri](https://github.com/bruits/satteri), a Rust-based Markdown / MDX compiler. + * + * To opt into the legacy remark/rehype pipeline, use the `unified()` factory from `@astrojs/markdown-remark`: + * + * ```js + * import { defineConfig } from 'astro/config'; + * import { unified } from '@astrojs/markdown-remark'; + * import remarkToc from 'remark-toc'; + * + * export default defineConfig({ + * markdown: { + * processor: unified({ remarkPlugins: [remarkToc] }), + * }, + * }); + * ``` + * + * To pass Sätteri plugins or enable additional features, use the `satteri()` factory: + * + * ```js + * import { defineConfig, satteri } from 'astro/config'; + * + * export default defineConfig({ + * markdown: { + * processor: satteri({ + * hastPlugins: [myPlugin], + * features: { directive: true, definitionList: true }, + * }), + * }, + * }); + * ``` + */ + processor?: MarkdownProcessorEntry; }; /** diff --git a/packages/astro/src/types/public/content.ts b/packages/astro/src/types/public/content.ts index 6f2a06bdf551..12a43279d7e5 100644 --- a/packages/astro/src/types/public/content.ts +++ b/packages/astro/src/types/public/content.ts @@ -1,4 +1,4 @@ -import type { MarkdownHeading } from '@astrojs/markdown-remark'; +import type { MarkdownHeading } from '@astrojs/internal-helpers/markdown'; import type { DataEntry, RenderedContent } from '../../content/data-store.js'; import type { LiveCollectionError } from '../../content/loaders/errors.js'; import type { AstroComponentFactory } from '../../runtime/server/index.js'; diff --git a/packages/astro/src/types/public/index.ts b/packages/astro/src/types/public/index.ts index 16d5035806d8..c119501aa9e5 100644 --- a/packages/astro/src/types/public/index.ts +++ b/packages/astro/src/types/public/index.ts @@ -1,10 +1,6 @@ export type { RemotePattern } from '@astrojs/internal-helpers/remote'; -export type { - MarkdownHeading, - RehypePlugins, - RemarkPlugins, - ShikiConfig, -} from '@astrojs/markdown-remark'; +export type { MarkdownHeading, ShikiConfig } from '@astrojs/internal-helpers/markdown'; +export type { RehypePlugins, RemarkPlugins } from './config.js'; export type { ExternalImageService, ImageService, diff --git a/packages/astro/src/vite-plugin-html/transform/slots.ts b/packages/astro/src/vite-plugin-html/transform/slots.ts index a7dc54a4041e..39aded0ea21f 100644 --- a/packages/astro/src/vite-plugin-html/transform/slots.ts +++ b/packages/astro/src/vite-plugin-html/transform/slots.ts @@ -14,8 +14,10 @@ const rehypeSlots: Plugin<[{ s: MagicString }], Root> = ({ s }) => { const end = node.position?.end.offset ?? 0; const first = node.children.at(0) ?? node; const last = node.children.at(-1) ?? node; + const firstPos = 'position' in first ? first.position : undefined; + const lastPos = 'position' in last ? last.position : undefined; const text = file.value - .slice(first.position?.start.offset ?? 0, last.position?.end.offset ?? 0) + .slice(firstPos?.start.offset ?? 0, lastPos?.end.offset ?? 0) .toString(); s.overwrite( start, diff --git a/packages/astro/src/vite-plugin-markdown/content-entry-type.ts b/packages/astro/src/vite-plugin-markdown/content-entry-type.ts index 15c7e53e13b7..40b9caf147c1 100644 --- a/packages/astro/src/vite-plugin-markdown/content-entry-type.ts +++ b/packages/astro/src/vite-plugin-markdown/content-entry-type.ts @@ -1,5 +1,4 @@ import { fileURLToPath, pathToFileURL } from 'node:url'; -import { createMarkdownProcessor } from '@astrojs/markdown-remark'; import { safeParseFrontmatter } from '../content/utils.js'; import type { ContentEntryType } from '../types/public/content.js'; @@ -18,9 +17,13 @@ export const markdownContentEntryType: ContentEntryType = { handlePropagation: true, async getRenderFunction(config) { - const processor = await createMarkdownProcessor({ - image: config.image, - ...config.markdown, + const { markdown, image } = config; + const processor = await markdown.processor.createRenderer({ + image, + syntaxHighlight: markdown.syntaxHighlight, + shikiConfig: markdown.shikiConfig, + gfm: markdown.gfm, + smartypants: markdown.smartypants, }); return async function renderToString(entry) { // Process markdown even if it's empty as remark/rehype plugins may add content or frontmatter dynamically diff --git a/packages/astro/src/vite-plugin-markdown/images.ts b/packages/astro/src/vite-plugin-markdown/images.ts index c98d3c2ca04f..e494a0606ac0 100644 --- a/packages/astro/src/vite-plugin-markdown/images.ts +++ b/packages/astro/src/vite-plugin-markdown/images.ts @@ -25,7 +25,7 @@ export function getMarkdownCodeForImages( let occurrenceCounter = 0; while ((match = regex.exec(html)) !== null) { const matchKey = ${rawUrl} + '_' + occurrenceCounter; - const imageProps = JSON.parse(match[1].replace(/"/g, '"').replace(/'/g, "'")); + const imageProps = JSON.parse(match[1].replace(/&(?:#x22|quot);/g, '"').replace(/&(?:#x27|apos);/g, "'")); const { src, ...props } = imageProps; imageSources[matchKey] = await getImage({src: Astro__${entry.safeName}, ...props}); occurrenceCounter++; @@ -45,7 +45,7 @@ export function getMarkdownCodeForImages( let occurrenceCounter = 0; while ((match = regex.exec(html)) !== null) { const matchKey = ${rawUrl} + '_' + occurrenceCounter; - const props = JSON.parse(match[1].replace(/"/g, '"').replace(/'/g, "'")); + const props = JSON.parse(match[1].replace(/&(?:#x22|quot);/g, '"').replace(/&(?:#x27|apos);/g, "'")); imageSources[matchKey] = await getImage(props); occurrenceCounter++; } @@ -59,7 +59,7 @@ export function getMarkdownCodeForImages( const imageSources = await images(html); return html.replaceAll(/__ASTRO_IMAGE_="([^"]+)"/gm, (full, imagePath) => { - const decodedImagePath = JSON.parse(imagePath.replace(/"/g, '"')); + const decodedImagePath = JSON.parse(imagePath.replace(/&(?:#x22|quot);/g, '"')); // Use the 'index' property for each image occurrence const srcKey = decodedImagePath.src + '_' + decodedImagePath.index; diff --git a/packages/astro/src/vite-plugin-markdown/index.ts b/packages/astro/src/vite-plugin-markdown/index.ts index 373e6e43fe81..3a4c0e01cfee 100644 --- a/packages/astro/src/vite-plugin-markdown/index.ts +++ b/packages/astro/src/vite-plugin-markdown/index.ts @@ -1,10 +1,7 @@ import fs from 'node:fs'; import { fileURLToPath, pathToFileURL } from 'node:url'; -import { - createMarkdownProcessor, - isFrontmatterValid, - type MarkdownProcessor, -} from '@astrojs/markdown-remark'; +import { isFrontmatterValid } from '@astrojs/internal-helpers/frontmatter'; +import type { MarkdownProcessor } from '@astrojs/internal-helpers/markdown'; import type { Plugin } from 'vite'; import { safeParseFrontmatter } from '../content/utils.js'; import { AstroError, AstroErrorData } from '../core/errors/index.js'; @@ -81,9 +78,13 @@ export default function markdown({ settings, logger }: AstroPluginOptions): Plug // Lazily initialize the Markdown processor if (!processor) { - processor = createMarkdownProcessor({ - image: settings.config.image, - ...settings.config.markdown, + const { markdown: md, image } = settings.config; + processor = md.processor.createRenderer({ + image, + syntaxHighlight: md.syntaxHighlight, + shikiConfig: md.shikiConfig, + gfm: md.gfm, + smartypants: md.smartypants, }); } diff --git a/packages/astro/test/astro-markdown.test.ts b/packages/astro/test/astro-markdown.test.ts index 01dc73ab9f01..154c662ff90e 100644 --- a/packages/astro/test/astro-markdown.test.ts +++ b/packages/astro/test/astro-markdown.test.ts @@ -62,9 +62,11 @@ describe('Astro Markdown', () => { }); it('handles Prism', async () => { + const { unified } = await import('@astrojs/markdown-remark'); const prismFixture = await loadFixture({ root: FIXTURE_ROOT, markdown: { + processor: unified(), syntaxHighlight: 'prism', }, outDir: './dist/astro-markdown-syntax-highlighting/', diff --git a/packages/astro/test/markdown.test.ts b/packages/astro/test/markdown.test.ts index e1403862b899..ec13f7888f05 100644 --- a/packages/astro/test/markdown.test.ts +++ b/packages/astro/test/markdown.test.ts @@ -70,7 +70,10 @@ describe('Markdown tests', () => { it('Does not unescape entities', async () => { const html = await fixture.readFile('/entities/index.html'); - assert.match(html, /<i>This should NOT be italic<\/i>/); + assert.match( + html, + /(?:<|<)i(?:>|>)This should NOT be italic(?:<|<)\/i(?:>|>)/, + ); }); it('Resolves the image paths correctly', async () => { diff --git a/packages/astro/test/units/content-layer/test-helpers.ts b/packages/astro/test/units/content-layer/test-helpers.ts index df3beb649756..245f8912baf8 100644 --- a/packages/astro/test/units/content-layer/test-helpers.ts +++ b/packages/astro/test/units/content-layer/test-helpers.ts @@ -2,6 +2,7 @@ import path from 'node:path'; import { tmpdir } from 'node:os'; import { mkdtempSync } from 'node:fs'; import { pathToFileURL } from 'node:url'; +import { satteri } from '@astrojs/markdown-satteri'; /** * Creates a temporary directory for tests @@ -41,14 +42,16 @@ export function createMinimalSettings(root: URL, overrides: Record root, srcDir: new URL('./src/', root), cacheDir: new URL('./.cache/', root), - markdown: {}, + markdown: { processor: satteri() }, experimental: {}, }; + const overrideConfig = overrides.config || {}; const settings: Record = { config: { ...defaultConfig, - ...(overrides.config || {}), + ...overrideConfig, + markdown: { ...defaultConfig.markdown, ...(overrideConfig.markdown ?? {}) }, }, dotAstroDir: new URL('./.astro/', root), contentEntryTypes: [], diff --git a/packages/astro/tsconfig.build.json b/packages/astro/tsconfig.build.json index aef5245acc21..c1739174e508 100644 --- a/packages/astro/tsconfig.build.json +++ b/packages/astro/tsconfig.build.json @@ -7,6 +7,6 @@ "references": [ { "path": "../internal-helpers/tsconfig.json" }, { "path": "../telemetry/tsconfig.json" }, - { "path": "../markdown/remark/tsconfig.json" } + { "path": "../markdown/satteri/tsconfig.json" } ] } diff --git a/packages/integrations/markdoc/src/content-entry-type.ts b/packages/integrations/markdoc/src/content-entry-type.ts index 097e9dbb03b4..6b3b9d52e815 100644 --- a/packages/integrations/markdoc/src/content-entry-type.ts +++ b/packages/integrations/markdoc/src/content-entry-type.ts @@ -1,7 +1,7 @@ import fs from 'node:fs'; import path from 'node:path'; import { fileURLToPath, pathToFileURL } from 'node:url'; -import { parseFrontmatter } from '@astrojs/markdown-remark'; +import { parseFrontmatter } from '@astrojs/internal-helpers/frontmatter'; import type { Config as MarkdocConfig, Node } from '@markdoc/markdoc'; import Markdoc from '@markdoc/markdoc'; import type { AstroConfig, ContentEntryType } from 'astro'; diff --git a/packages/integrations/markdoc/src/extensions/shiki.ts b/packages/integrations/markdoc/src/extensions/shiki.ts index 1102242fdfe5..e68d52b5b0c5 100644 --- a/packages/integrations/markdoc/src/extensions/shiki.ts +++ b/packages/integrations/markdoc/src/extensions/shiki.ts @@ -1,4 +1,4 @@ -import { createShikiHighlighter } from '@astrojs/markdown-remark'; +import { createShikiHighlighter } from '@astrojs/internal-helpers/shiki'; import Markdoc from '@markdoc/markdoc'; import type { ShikiConfig } from 'astro'; import { unescapeHTML } from 'astro/runtime/server/index.js'; diff --git a/packages/integrations/mdx/package.json b/packages/integrations/mdx/package.json index 00f723e9c90a..391037a109d5 100644 --- a/packages/integrations/mdx/package.json +++ b/packages/integrations/mdx/package.json @@ -34,7 +34,8 @@ "test": "astro-scripts test --timeout 70000 \"test/**/*.test.ts\"" }, "dependencies": { - "@astrojs/markdown-remark": "workspace:*", + "@astrojs/internal-helpers": "workspace:*", + "@astrojs/markdown-satteri": "workspace:*", "@mdx-js/mdx": "^3.1.1", "acorn": "^8.16.0", "es-module-lexer": "^2.0.0", @@ -44,16 +45,24 @@ "rehype-raw": "^7.0.0", "remark-gfm": "^4.0.1", "remark-smartypants": "^3.0.2", + "satteri": "^0.6.3", "source-map": "^0.7.6", "unist-util-visit": "^5.1.0", "vfile": "^6.0.3" }, "peerDependencies": { + "@astrojs/markdown-remark": "workspace:*", "astro": "^7.0.0-alpha.0" }, + "peerDependenciesMeta": { + "@astrojs/markdown-remark": { + "optional": true + } + }, "devDependencies": { - "@shikijs/rehype": "^4.0.2", - "@shikijs/twoslash": "^4.0.2", + "@astrojs/markdown-remark": "workspace:*", + "@shikijs/rehype": "^4.0.0", + "@shikijs/twoslash": "^4.0.0", "@types/estree": "^1.0.8", "@types/hast": "^3.0.4", "@types/mdast": "^4.0.4", diff --git a/packages/integrations/mdx/src/image-constants.ts b/packages/integrations/mdx/src/image-constants.ts new file mode 100644 index 000000000000..643015ac683d --- /dev/null +++ b/packages/integrations/mdx/src/image-constants.ts @@ -0,0 +1,10 @@ +// Tag name we rewrite markdown-derived `` elements to. Lowercase + hyphenated +// so MDX routes the tag through the `_components` map. +export const ASTRO_IMAGE_ELEMENT = 'astro-image'; +// Module-level identifier bound to Astro's `Image` component (from `astro:assets`). +// Imported by every compiled MDX file that contains a rewritten image; used as the +// fallback when no `components.img` is provided. +export const ASTRO_IMAGE_IMPORT = '__AstroImage__'; +// Boolean export set on MDX modules that contain rewritten images. Read by +// `vite-plugin-mdx-postprocess` to decide whether to wire up the image component. +export const USES_ASTRO_IMAGE_FLAG = '__usesAstroImage'; diff --git a/packages/integrations/mdx/src/index.ts b/packages/integrations/mdx/src/index.ts index 10a43bc2cfe3..35cc5111b6f9 100644 --- a/packages/integrations/mdx/src/index.ts +++ b/packages/integrations/mdx/src/index.ts @@ -1,6 +1,12 @@ import fs from 'node:fs/promises'; import { fileURLToPath } from 'node:url'; -import { markdownConfigDefaults } from '@astrojs/markdown-remark'; +import type { UnifiedProcessorDescriptor } from '@astrojs/markdown-remark'; +import { + type AstroMarkdownProcessorOptions, + markdownConfigDefaults, +} from '@astrojs/internal-helpers/markdown'; +import type { SatteriProcessorDescriptor } from '@astrojs/markdown-satteri'; +import type { Features } from 'satteri'; import type { AstroIntegration, AstroIntegrationLogger, @@ -8,14 +14,29 @@ import type { ContentEntryType, HookParameters, } from 'astro'; +import type { MarkdownProcessorEntry } from 'astro/markdown'; import type { Options as RemarkRehypeOptions } from 'remark-rehype'; import type { PluggableList } from 'unified'; -import type { OptimizeOptions } from './rehype-optimize-static.js'; +import type { MdastPluginDefinition, HastPluginDefinition } from './satteri/index.js'; import { ignoreStringPlugins, safeParseFrontmatter } from './utils.js'; import { type VitePluginMdxOptions, vitePluginMdx } from './vite-plugin-mdx.js'; import { vitePluginMdxPostprocess } from './vite-plugin-mdx-postprocess.js'; -export type MdxOptions = Omit & { +// Inlined name-checks to avoid eagerly importing the unified/satteri runtime +// modules when they're not the active processor. +const isSatteriProcessor = (p: { name: string }): p is SatteriProcessorDescriptor => + p.name === 'satteri'; +const isUnifiedProcessor = (p: { name: string }): p is UnifiedProcessorDescriptor => + p.name === 'unified'; + +// `gfm`/`smartypants` are deprecated and stay unset unless the user opts in; the +// MDX pipelines treat an absent value as the default (on), like the `.md` processors. +type SharedMarkdownOptions = Required< + Pick +> & + Pick; + +export type MdxOptions = SharedMarkdownOptions & { extendMarkdownConfig: boolean; recmaPlugins: PluggableList; // Markdown allows strings as remark and rehype plugins. @@ -23,7 +44,16 @@ export type MdxOptions = Omit & { @@ -83,24 +113,74 @@ export default function mdx(partialMdxOptions: Partial = {}): AstroI }, }); }, - 'astro:config:done': ({ config, logger }) => { + 'astro:config:done': async ({ config, logger }) => { // We resolve the final MDX options here so that other integrations have a chance to modify // `config.markdown` before we access it const extendMarkdownConfig = partialMdxOptions.extendMarkdownConfig ?? defaultMdxOptions.extendMarkdownConfig; + const markdownConfig = extendMarkdownConfig ? config.markdown : markdownConfigDefaults; + const resolvedMdxOptions = applyDefaultOptions({ options: partialMdxOptions, - defaults: markdownConfigToMdxOptions( - extendMarkdownConfig ? config.markdown : markdownConfigDefaults, - logger, - ), + defaults: markdownConfigToMdxOptions(markdownConfig, logger), }); + const descriptor = partialMdxOptions.processor ?? config.markdown.processor; + + if (extendMarkdownConfig) { + // Per docs: when MDX provides its own plugin list, it REPLACES the + // markdown processor's plugins; when MDX omits it, MDX inherits. + // (Object-shaped options like `remarkRehype`/`features` still merge.) + if (isSatteriProcessor(descriptor)) { + if (partialMdxOptions.mdastPlugins === undefined) { + resolvedMdxOptions.mdastPlugins = [...descriptor.options.mdastPlugins]; + } + if (partialMdxOptions.hastPlugins === undefined) { + resolvedMdxOptions.hastPlugins = [...descriptor.options.hastPlugins]; + } + resolvedMdxOptions.features = { + ...descriptor.options.features, + ...resolvedMdxOptions.features, + }; + } else if (isUnifiedProcessor(descriptor)) { + if (partialMdxOptions.remarkPlugins === undefined) { + resolvedMdxOptions.remarkPlugins = ignoreStringPlugins( + descriptor.options.remarkPlugins, + logger, + ); + } + if (partialMdxOptions.rehypePlugins === undefined) { + resolvedMdxOptions.rehypePlugins = ignoreStringPlugins( + descriptor.options.rehypePlugins, + logger, + ); + } + resolvedMdxOptions.remarkRehype = { + ...descriptor.options.remarkRehype, + ...resolvedMdxOptions.remarkRehype, + }; + // `gfm`/`smartypants` from `unified({...})` apply to `.mdx` too, unless + // `mdx({...})` set its own. + if (partialMdxOptions.gfm === undefined && descriptor.options.gfm !== undefined) { + resolvedMdxOptions.gfm = descriptor.options.gfm; + } + if ( + partialMdxOptions.smartypants === undefined && + descriptor.options.smartypants !== undefined + ) { + resolvedMdxOptions.smartypants = descriptor.options.smartypants; + } + } + // Third-party processors don't expose their plugins to MDX's built-in option + // merging; they handle their own pipeline via `createMdxRenderer`. + } + // Mutate `mdxOptions` so that `vitePluginMdx` can reference the actual options Object.assign(vitePluginMdxOptions, { mdxOptions: resolvedMdxOptions, srcDir: config.srcDir, + processor: descriptor, }); // @ts-expect-error After we assign, we don't need to reference `mdxOptions` in this context anymore. // Re-assign it so that the garbage can be collected later. @@ -114,18 +194,21 @@ const defaultMdxOptions = { extendMarkdownConfig: true, recmaPlugins: [], optimize: false, + mdastPlugins: [], + hastPlugins: [], } satisfies Partial; function markdownConfigToMdxOptions( - markdownConfig: typeof markdownConfigDefaults, - logger: AstroIntegrationLogger, + markdownConfig: SharedMarkdownOptions, + _logger: AstroIntegrationLogger, ): MdxOptions { return { ...defaultMdxOptions, ...markdownConfig, - remarkPlugins: ignoreStringPlugins(markdownConfig.remarkPlugins, logger), - rehypePlugins: ignoreStringPlugins(markdownConfig.rehypePlugins, logger), - remarkRehype: (markdownConfig.remarkRehype as any) ?? {}, + // Plugins come from the processor descriptor — merged in astro:config:done. + remarkPlugins: [], + rehypePlugins: [], + remarkRehype: {}, }; } @@ -147,5 +230,8 @@ function applyDefaultOptions({ rehypePlugins: options.rehypePlugins ?? defaults.rehypePlugins, shikiConfig: options.shikiConfig ?? defaults.shikiConfig, optimize: options.optimize ?? defaults.optimize, + mdastPlugins: options.mdastPlugins ?? defaults.mdastPlugins, + hastPlugins: options.hastPlugins ?? defaults.hastPlugins, + features: options.features ?? defaults.features, }; } diff --git a/packages/integrations/mdx/src/satteri/charset.ts b/packages/integrations/mdx/src/satteri/charset.ts new file mode 100644 index 000000000000..c36cd70c629e --- /dev/null +++ b/packages/integrations/mdx/src/satteri/charset.ts @@ -0,0 +1,47 @@ +// TODO: This is a workaround around a missing API in Sätteri. The visitor architecture naturally does not provide +// a way to "visit" the root and prepend nodes (it'd kill performance), as such this file currently works around that +// in a clumsy way. Once Sätteri ship an actual API to prepend / append, this file could get removed. + +import * as path from 'node:path'; +import { fileURLToPath } from 'node:url'; + +const exportConstPartialTrueRe = /export\s+const\s+partial\s*=\s*true/; +const leadingComponentRe = /^\s*<\s*([A-Za-z][A-Za-z0-9]*)\b/; + +// Scans MDX source directly because Sätteri exposes no root-level visitor. +// Skips imports/exports/blank lines and bails when the first content line is a +// capitalized JSX element (treated as a wrapping layout). +export function shouldAddCharset(content: string, filePath: string, srcDir: URL): boolean { + const srcDirPath = fileURLToPath(srcDir).replace(/\\/g, '/'); + const pagesDir = path.posix.join(srcDirPath, 'pages'); + const normalizedFilePath = filePath.replace(/\\/g, '/'); + if (!normalizedFilePath.startsWith(pagesDir)) return false; + + const segments = normalizedFilePath.slice(pagesDir.length).split('/'); + if (segments.some((part) => part.startsWith('_'))) return false; + + if (exportConstPartialTrueRe.test(content)) return false; + + // Strip MDX JSX expression comments `{/* ... */}` so they don't trip the + // "first content" scan below — they're not the wrapping layout. + const stripped = content.replace(/\{\/\*[\s\S]*?\*\/\}/g, ''); + + for (const rawLine of stripped.split('\n')) { + const line = rawLine.trim(); + if (!line) continue; + if (line.startsWith('import ') || line.startsWith('export ')) continue; + if (line.startsWith('//') || line.startsWith('/*')) continue; + // Other JSX expressions (e.g. `{someVar}` at top level) aren't a wrapping layout. + if (line.startsWith('{')) continue; + + const match = leadingComponentRe.exec(line); + if (match) { + const tag = match[1]; + // Capitalized tag suggests a user-provided wrapping layout component + if (tag[0] >= 'A' && tag[0] <= 'Z') return false; + } + break; + } + + return true; +} diff --git a/packages/integrations/mdx/src/satteri/hast-astro-metadata.ts b/packages/integrations/mdx/src/satteri/hast-astro-metadata.ts new file mode 100644 index 000000000000..c6ddb95c0e98 --- /dev/null +++ b/packages/integrations/mdx/src/satteri/hast-astro-metadata.ts @@ -0,0 +1,183 @@ +import type { AstroMetadata } from '@astrojs/internal-helpers/markdown'; +import { resolvePath } from 'astro/markdown'; +import type { Identifier, Literal } from 'estree'; +import { + defineHastPlugin, + type EstreeProgram, + type HastPluginDefinition, + type HastVisitorContext, +} from 'satteri'; +import { findAttrValue, hasDirective, isComponent, type MdxJsxHastNode } from './jsx-utils.js'; + +export type { AstroMetadata }; + +type ImportSpecifier = { local: string; imported: string }; +type MatchedImport = { name: string; path: string }; + +function exportNameToString(name: Identifier | Literal): string { + return name.type === 'Identifier' ? name.name : String(name.value); +} + +function collectImportsFromEsm( + program: EstreeProgram, + imports: Map>, +): void { + for (const stmt of program.body) { + if (stmt.type !== 'ImportDeclaration') continue; + const source = String(stmt.source.value); + + let specSet = imports.get(source); + if (!specSet) { + specSet = new Set(); + imports.set(source, specSet); + } + + for (const spec of stmt.specifiers) { + switch (spec.type) { + case 'ImportDefaultSpecifier': + specSet.add({ local: spec.local.name, imported: 'default' }); + break; + case 'ImportNamespaceSpecifier': + specSet.add({ local: spec.local.name, imported: '*' }); + break; + case 'ImportSpecifier': + specSet.add({ + local: spec.local.name, + imported: exportNameToString(spec.imported), + }); + break; + } + } + } +} + +function findMatchingImport( + tagName: string, + imports: Map>, +): MatchedImport | undefined { + const tagSpecifier = tagName.split('.')[0]; + for (const [source, specs] of imports) { + for (const { imported, local } of specs) { + if (local === tagSpecifier) { + if (tagSpecifier !== tagName) { + switch (imported) { + case '*': { + const accessPath = tagName.slice(tagSpecifier.length + 1); + return { name: accessPath, path: source }; + } + case 'default': { + const accessPath = tagName.slice(tagSpecifier.length + 1); + return { name: `default.${accessPath}`, path: source }; + } + default: { + return { name: tagName, path: source }; + } + } + } + return { name: imported, path: source }; + } + } + } +} + +function processJsxNode( + node: MdxJsxHastNode, + ctx: HastVisitorContext, + metadata: AstroMetadata, + imports: Map>, + filePath: string, +) { + const tagName = node.name; + if (!tagName || !isComponent(tagName)) return; + + const hasClient = hasDirective(node, 'client:'); + const hasServerDefer = !hasClient && hasDirective(node, 'server:defer'); + if (!hasClient && !hasServerDefer) return; + + const matchedImport = findMatchingImport(tagName, imports); + if (!matchedImport) { + throw new Error( + `Expected a matching import for component \`${tagName}\`. Did you forget to import it?`, + ); + } + + if (matchedImport.path.endsWith('.astro') && hasClient) { + let clientAttr = 'client:*'; + for (const a of node.attributes) { + if (a.type === 'mdxJsxAttribute' && a.name.startsWith('client:')) { + clientAttr = a.name; + break; + } + } + console.warn( + `You are attempting to render <${tagName} ${clientAttr} />, but ${tagName} is an Astro component. Astro components do not render in the client and should not have a hydration directive. Please use a framework component for client rendering.`, + ); + } + + const resolvedPath = resolvePath(matchedImport.path, filePath); + const exportName = + matchedImport.name === '*' ? tagName.split('.').slice(1).join('.') : matchedImport.name; + + if (hasClient && findAttrValue(node, 'client:only') !== null) { + metadata.clientOnlyComponents.push({ + exportName: matchedImport.name, + localName: '', + specifier: tagName, + resolvedPath, + }); + ctx.setProperty(node, 'client:display-name', tagName); + ctx.setProperty(node, 'client:component-path', resolvedPath); + ctx.setProperty(node, 'client:component-export', exportName); + ctx.setProperty(node, 'client:component-hydration', ''); + } else if (hasClient) { + metadata.hydratedComponents.push({ + exportName: '*', + localName: '', + specifier: tagName, + resolvedPath, + }); + ctx.setProperty(node, 'client:component-path', resolvedPath); + ctx.setProperty(node, 'client:component-export', exportName); + ctx.setProperty(node, 'client:component-hydration', ''); + } else if (hasServerDefer) { + metadata.serverComponents.push({ + exportName: matchedImport.name, + localName: tagName, + specifier: matchedImport.path, + resolvedPath, + }); + ctx.setProperty(node, 'server:component-directive', 'server:defer'); + ctx.setProperty(node, 'server:component-path', resolvedPath); + ctx.setProperty(node, 'server:component-export', exportName); + } +} + +export function createAstroMetadataPlugin( + metadata: AstroMetadata, + filePath: string, +): HastPluginDefinition { + const imports = new Map>(); + return defineHastPlugin({ + name: 'astro-metadata', + mdxjsEsm(node) { + const program = node.parseExpression(); + if (program) { + collectImportsFromEsm(program, imports); + } + }, + + mdxJsxFlowElement: { + filter: [], + visit(node, ctx) { + processJsxNode(node, ctx, metadata, imports, filePath); + }, + }, + + mdxJsxTextElement: { + filter: [], + visit(node, ctx) { + processJsxNode(node, ctx, metadata, imports, filePath); + }, + }, + }); +} diff --git a/packages/integrations/mdx/src/satteri/hast-images-to-component.ts b/packages/integrations/mdx/src/satteri/hast-images-to-component.ts new file mode 100644 index 000000000000..ec074451e6b1 --- /dev/null +++ b/packages/integrations/mdx/src/satteri/hast-images-to-component.ts @@ -0,0 +1,82 @@ +import { + defineHastPlugin, + type HastNode, + type HastPluginDefinition, + type MdxJsxAttributeNode, +} from 'satteri'; +import { ASTRO_IMAGE_ELEMENT } from '../image-constants.js'; +import { makeJsxAttr, makeJsxExprAttr } from './jsx-utils.js'; + +export interface ImageImportInfo { + importedImages: Map; + hasImages: boolean; +} + +export function createImageToComponentPlugin( + localImagePaths: Set, + remoteImagePaths: Set, + imageImportInfo: ImageImportInfo, +): HastPluginDefinition { + return defineHastPlugin({ + name: 'image-to-component', + element: { + filter: ['img'], + visit(node) { + const src = node.properties?.src; + if (typeof src !== 'string') return; + + const decodedSrc = decodeURI(src); + const isLocal = localImagePaths.has(decodedSrc); + const isRemote = remoteImagePaths.has(decodedSrc); + if (!isLocal && !isRemote) return; + + const attrs: MdxJsxAttributeNode[] = []; + + if (node.properties) { + for (const [key, value] of Object.entries(node.properties)) { + if (key === 'src') continue; + if (value == null || value === false) continue; + + if (key === 'widths' || key === 'densities') { + attrs.push(makeJsxExprAttr(key, JSON.stringify(String(value).split(' ')))); + } else { + const attrName = key === 'className' ? 'class' : key === 'htmlFor' ? 'for' : key; + attrs.push(makeJsxAttr(attrName, String(value))); + } + } + } + + if (isLocal) { + let importName = imageImportInfo.importedImages.get(decodedSrc); + if (!importName) { + importName = `__${imageImportInfo.importedImages.size}_${decodedSrc.replace(/\W/g, '_')}__`; + imageImportInfo.importedImages.set(decodedSrc, importName); + } + attrs.push(makeJsxExprAttr('src', importName)); + } else { + const hasWidth = node.properties && 'width' in node.properties; + const hasHeight = node.properties && 'height' in node.properties; + if (!hasWidth || !hasHeight) { + attrs.push(makeJsxExprAttr('inferSize', 'true')); + } + attrs.push(makeJsxAttr('src', decodedSrc)); + } + + imageImportInfo.hasImages = true; + + // Emit `` (lowercase hyphenated) so MDX routes the tag through + // `_components['astro-image']`. The vite-plugin-mdx-postprocess wraps the + // final component to map `astro-image` → `components.img ?? __AstroImage__`, + // which lets a user's `export const components = { img: MyImage }` win. + // Match the unified pipeline, which emits a flow (block) element for + // markdown-derived images so MDX compiles them consistently. + return { + type: 'mdxJsxFlowElement', + name: ASTRO_IMAGE_ELEMENT, + attributes: attrs, + children: [], + } as unknown as HastNode; + }, + }, + }); +} diff --git a/packages/integrations/mdx/src/satteri/index.ts b/packages/integrations/mdx/src/satteri/index.ts new file mode 100644 index 000000000000..02860d9b6924 --- /dev/null +++ b/packages/integrations/mdx/src/satteri/index.ts @@ -0,0 +1,215 @@ +import { + createDefaultAstroMetadata, + type MarkdownHeading, +} from '@astrojs/internal-helpers/markdown'; +import { createShikiHighlighter } from '@astrojs/internal-helpers/shiki'; +import { + satteriCollectImagesPlugin, + satteriHeadingIdsPlugin, + satteriShikiPlugin, +} from '@astrojs/markdown-satteri'; +import { + mdxToJs, + type HastPluginDefinition, + type MdastPluginDefinition, + type MdxCompileOptions, +} from 'satteri'; +import { ASTRO_IMAGE_IMPORT, USES_ASTRO_IMAGE_FLAG } from '../image-constants.js'; +import type { MdxOptions } from '../index.js'; +import { shouldAddCharset } from './charset.js'; +import { type AstroMetadata, createAstroMetadataPlugin } from './hast-astro-metadata.js'; +import { createImageToComponentPlugin, type ImageImportInfo } from './hast-images-to-component.js'; + +type HighlightFn = (code: string, lang: string, meta?: string) => Promise; + +export type { MdastPluginDefinition, HastPluginDefinition, MarkdownHeading }; +export type { AstroMetadata } from './hast-astro-metadata.js'; + +declare module 'hast' { + interface ElementData { + lang?: string | null; + } +} + +export interface CompileMdxResult { + code: string; + astroMetadata: AstroMetadata; +} + +interface CreateMdxProcessorContext { + srcDir: URL; +} + +export function createMdxProcessor(mdxOptions: MdxOptions, ctx: CreateMdxProcessorContext) { + let highlightFn: HighlightFn | undefined; + let initPromise: Promise | undefined; + + function initShiki() { + const syntaxHighlight = mdxOptions.syntaxHighlight; + const syntaxHighlightType = + typeof syntaxHighlight === 'string' + ? syntaxHighlight + : syntaxHighlight + ? syntaxHighlight.type + : undefined; + + if (syntaxHighlightType === 'prism') { + throw new Error( + 'Prism syntax highlighting is not supported by the `satteri()` markdown processor. Use shiki instead, or switch to `markdown.processor: unified({...})`.', + ); + } + + if (syntaxHighlightType === 'shiki') { + const shikiConfig = mdxOptions.shikiConfig ?? {}; + initPromise = createShikiHighlighter({ + langs: shikiConfig.langs, + theme: shikiConfig.theme, + themes: shikiConfig.themes, + langAlias: shikiConfig.langAlias, + }).then((hl) => { + highlightFn = (code, lang, meta) => + hl.codeToHtml(code, lang, { + meta, + wrap: shikiConfig.wrap, + defaultColor: shikiConfig.defaultColor, + transformers: shikiConfig.transformers, + }); + }); + } + } + + return { + async process( + content: string, + filePath: string, + frontmatter: Record, + ): Promise { + if (!highlightFn && !initPromise) { + initShiki(); + } + if (initPromise) await initPromise; + + const headings: MarkdownHeading[] = []; + const localImagePaths = new Set(); + const remoteImagePaths = new Set(); + + const astroMetadata = createDefaultAstroMetadata(); + + const collectImages = satteriCollectImagesPlugin(localImagePaths, remoteImagePaths); + const headingIds = satteriHeadingIdsPlugin(headings, frontmatter); + const astroMeta = createAstroMetadataPlugin(astroMetadata, filePath); + const imageImportInfo: ImageImportInfo = { + importedImages: new Map(), + hasImages: false, + }; + const imageToComponent = createImageToComponentPlugin( + localImagePaths, + remoteImagePaths, + imageImportInfo, + ); + + const syntaxHighlight = mdxOptions.syntaxHighlight; + const excludeLangs = + typeof syntaxHighlight === 'object' ? syntaxHighlight.excludeLangs : undefined; + + const allMdastPlugins: MdastPluginDefinition[] = mdxOptions.mdastPlugins?.length + ? [collectImages, ...mdxOptions.mdastPlugins] + : [collectImages]; + + const hastPlugins: HastPluginDefinition[] = []; + if (highlightFn) { + // `mdx: true` wraps the highlighted HTML in a JSX `` node + // rather than a raw HTML node, since the Sätteri pipeline is compiling to JSX. + hastPlugins.push(satteriShikiPlugin(highlightFn, excludeLangs, { mdx: true })); + } + hastPlugins.push(headingIds, imageToComponent, astroMeta); + if (mdxOptions.hastPlugins?.length) { + hastPlugins.push(...mdxOptions.hastPlugins); + } + + let optimizeStatic: MdxCompileOptions['optimizeStatic']; + if (mdxOptions.optimize) { + const ignoreElements = + typeof mdxOptions.optimize === 'object' + ? mdxOptions.optimize.ignoreElementNames + : undefined; + + optimizeStatic = { + component: 'Fragment', + prop: 'set:html', + ...(ignoreElements && { ignoreElements }), + }; + } + + const mdxResult = await mdxToJs(content, { + mdastPlugins: allMdastPlugins, + hastPlugins, + optimizeStatic, + features: { + gfm: mdxOptions.gfm !== false, + smartPunctuation: mdxOptions.smartypants !== false, + ...mdxOptions.features, + }, + filename: filePath, + jsxImportSource: 'astro', + }); + let compiled = mdxResult.code; + + compiled = compiled.replace(/^export default MDXContent;\s*$/m, ''); + + if (imageImportInfo.hasImages) { + // `vite-plugin-mdx-postprocess` wraps Content to map + // `astro-image` → `components.img ?? __AstroImage__`, so a user's + // `export const components = { img: ... }` override is honored. + compiled += `\nimport { Image as ${ASTRO_IMAGE_IMPORT} } from "astro:assets";`; + for (const [src, importName] of imageImportInfo.importedImages) { + compiled += `\nimport ${importName} from ${JSON.stringify(src)};`; + } + compiled += `\nexport const ${USES_ASTRO_IMAGE_FLAG} = true;`; + } + + compiled += `\nexport const frontmatter = ${JSON.stringify(frontmatter)};`; + compiled += `\nexport function getHeadings() { return ${JSON.stringify(headings)}; }`; + + if (frontmatter.layout) { + compiled = compiled.replace(/^function MDXContent\(/m, 'function __OriginalMDXContent__('); + compiled += ` +import { jsx as __astro_layout_jsx__ } from 'astro/jsx-runtime'; +import __astro_layout_component__ from ${JSON.stringify(frontmatter.layout)}; +export default function MDXContent(props) { + const content = __OriginalMDXContent__(props); + const { layout, ...frontmatterContent } = frontmatter; + frontmatterContent.file = file; + frontmatterContent.url = url; + return __astro_layout_jsx__(__astro_layout_component__, { + file, + url, + content: frontmatterContent, + frontmatter: frontmatterContent, + headings: getHeadings(), + 'server:root': true, + children: content, + }); +}`; + } else if (shouldAddCharset(content, filePath, ctx.srcDir)) { + // Default MDX pages without a layout to UTF-8 so users don't have to think about it. + compiled = compiled.replace(/^function MDXContent\(/m, 'function __OriginalMDXContent__('); + compiled += ` +import { jsx as __astro_charset_jsx__, jsxs as __astro_charset_jsxs__, Fragment as __astro_charset_Fragment__ } from 'astro/jsx-runtime'; +export default function MDXContent(props) { + return __astro_charset_jsxs__(__astro_charset_Fragment__, { + children: [ + __astro_charset_jsx__('meta', { charset: 'utf-8' }), + __OriginalMDXContent__(props), + ], + }); +}`; + } + + return { + code: compiled, + astroMetadata, + }; + }, + }; +} diff --git a/packages/integrations/mdx/src/satteri/jsx-utils.ts b/packages/integrations/mdx/src/satteri/jsx-utils.ts new file mode 100644 index 000000000000..b82ad012fb3e --- /dev/null +++ b/packages/integrations/mdx/src/satteri/jsx-utils.ts @@ -0,0 +1,46 @@ +import type { MdxJsxAttributeNode, MdxJsxFlowElementHast, MdxJsxTextElementHast } from 'satteri'; + +// JSX-specific helpers used by the MDX pipeline. Non-MDX equivalents (e.g. +// `makeFragmentNode`, `collectHastText`) live in `@astrojs/markdown-satteri` +// because they're shared with the markdown render pipeline. + +// Sätteri hands plugin visitors readonly nodes; these helpers only read them. +export type MdxJsxHastNode = Readonly; + +const nonAlphaRe = /[^a-zA-Z]/; + +export function isComponent(tagName: string): boolean { + return ( + (tagName[0] && tagName[0].toLowerCase() !== tagName[0]) || + tagName.includes('.') || + nonAlphaRe.test(tagName[0]) + ); +} + +export function hasDirective(node: MdxJsxHastNode, prefix: string): boolean { + for (const a of node.attributes) { + if (a.type === 'mdxJsxAttribute' && a.name.startsWith(prefix)) return true; + } + return false; +} + +export function findAttrValue(node: MdxJsxHastNode, name: string): string | null { + for (const a of node.attributes) { + if (a.type === 'mdxJsxAttribute' && a.name === name) { + return typeof a.value === 'string' ? a.value : null; + } + } + return null; +} + +export function makeJsxAttr(name: string, value: string): MdxJsxAttributeNode { + return { type: 'mdxJsxAttribute', name, value }; +} + +export function makeJsxExprAttr(name: string, expression: string): MdxJsxAttributeNode { + return { + type: 'mdxJsxAttribute', + name, + value: { type: 'mdxJsxAttributeValueExpression', value: expression }, + }; +} diff --git a/packages/integrations/mdx/src/plugins.ts b/packages/integrations/mdx/src/unified/index.ts similarity index 92% rename from packages/integrations/mdx/src/plugins.ts rename to packages/integrations/mdx/src/unified/index.ts index b6584f8b7ff0..baf3fc405bb1 100644 --- a/packages/integrations/mdx/src/plugins.ts +++ b/packages/integrations/mdx/src/unified/index.ts @@ -5,19 +5,21 @@ import { remarkCollectImages, } from '@astrojs/markdown-remark'; import { createProcessor, nodeTypes } from '@mdx-js/mdx'; -import { rehypeAnalyzeAstroMetadata } from 'astro/jsx/rehype.js'; import rehypeRaw from 'rehype-raw'; import remarkGfm from 'remark-gfm'; import remarkSmartypants from 'remark-smartypants'; import { SourceMapGenerator } from 'source-map'; import type { PluggableList } from 'unified'; -import type { MdxOptions } from './index.js'; +import type { MdxOptions } from '../index.js'; +import { rehypeAnalyzeAstroMetadata } from './rehype-analyze-astro-metadata.js'; import { rehypeApplyFrontmatterExport } from './rehype-apply-frontmatter-export.js'; import { rehypeInjectHeadingsExport } from './rehype-collect-headings.js'; import { rehypeImageToComponent } from './rehype-images-to-component.js'; import rehypeMetaString from './rehype-meta-string.js'; import { rehypeOptimizeStatic } from './rehype-optimize-static.js'; +export { getAstroMetadata } from './rehype-analyze-astro-metadata.js'; + // Skip nonessential plugins during performance benchmark runs const isPerformanceBenchmark = Boolean(process.env.ASTRO_PERFORMANCE_BENCHMARK); @@ -44,7 +46,8 @@ function getRemarkPlugins(mdxOptions: MdxOptions): PluggableList { let remarkPlugins: PluggableList = []; if (!isPerformanceBenchmark) { - if (mdxOptions.gfm) { + // `undefined` means "use the default" — gfm is on unless explicitly disabled. + if (mdxOptions.gfm !== false) { remarkPlugins.push(remarkGfm); } if (mdxOptions.smartypants !== false) { diff --git a/packages/astro/src/jsx/rehype.ts b/packages/integrations/mdx/src/unified/rehype-analyze-astro-metadata.ts similarity index 88% rename from packages/astro/src/jsx/rehype.ts rename to packages/integrations/mdx/src/unified/rehype-analyze-astro-metadata.ts index 23ead4eb0149..dc78e033747c 100644 --- a/packages/astro/src/jsx/rehype.ts +++ b/packages/integrations/mdx/src/unified/rehype-analyze-astro-metadata.ts @@ -1,5 +1,8 @@ -import type { RehypePlugin } from '@astrojs/markdown-remark'; -import type { RootContent } from 'hast'; +import { createDefaultAstroMetadata, type AstroMetadata } from '@astrojs/internal-helpers/markdown'; +import { resolvePath } from 'astro/markdown'; +import type { Program } from 'estree'; +import type { RootContent, Root as HastRoot } from 'hast'; +import type { Plugin } from 'unified'; import type {} from 'mdast-util-mdx'; import type { MdxJsxAttribute, @@ -8,15 +11,12 @@ import type { } from 'mdast-util-mdx-jsx'; import { visit } from 'unist-util-visit'; import type { VFile } from 'vfile'; -import { AstroError } from '../core/errors/errors.js'; -import { AstroErrorData } from '../core/errors/index.js'; -import { resolvePath } from '../core/viteUtils.js'; -import { createDefaultAstroMetadata } from '../vite-plugin-astro/metadata.js'; -import type { PluginMetadata } from '../vite-plugin-astro/types.js'; const ClientOnlyPlaceholder = 'astro-client-only'; -export const rehypeAnalyzeAstroMetadata: RehypePlugin = () => { +export type { AstroMetadata }; + +export const rehypeAnalyzeAstroMetadata: Plugin<[], HastRoot> = () => { return (tree, file) => { // Initial metadata for this MDX file, it will be mutated as we traverse the tree const metadata = createDefaultAstroMetadata(); @@ -24,8 +24,13 @@ export const rehypeAnalyzeAstroMetadata: RehypePlugin = () => { // Parse imports in this file. This is used to match components with their import source const imports = parseImports(tree.children); - visit(tree, (node) => { - if (node.type !== 'mdxJsxFlowElement' && node.type !== 'mdxJsxTextElement') return; + visit(tree, (visitedNode) => { + if (visitedNode.type !== 'mdxJsxFlowElement' && visitedNode.type !== 'mdxJsxTextElement') + return; + + // `visit` returns nodes typed against widened hast augmentations (e.g. from satteri). + // Narrow back to the mdast-util-mdx-jsx types our helpers expect. + const node = visitedNode as unknown as MdxJsxFlowElementHast | MdxJsxTextElementHast; const tagName = node.name; if ( @@ -40,10 +45,9 @@ export const rehypeAnalyzeAstroMetadata: RehypePlugin = () => { // Match this component with its import source const matchedImport = findMatchingImport(tagName, imports); if (!matchedImport) { - throw new AstroError({ - ...AstroErrorData.NoMatchingImport, - message: AstroErrorData.NoMatchingImport.message(node.name!), - }); + throw new Error( + `Expected a matching import for component \`${node.name}\`. Did you forget to import it?`, + ); } // If this is an Astro component, that means the `client:` directive is misused as it doesn't @@ -101,7 +105,7 @@ export const rehypeAnalyzeAstroMetadata: RehypePlugin = () => { }; export function getAstroMetadata(file: VFile) { - return file.data.__astroMetadata as PluginMetadata['astro'] | undefined; + return file.data.__astroMetadata as AstroMetadata | undefined; } type ImportSpecifier = { local: string; imported: string }; @@ -130,7 +134,10 @@ function parseImports(children: RootContent[]) { for (const child of children) { if (child.type !== 'mdxjsEsm') continue; - const body = child.data?.estree?.body; + // satteri and `mdast-util-mdxjs-esm` both augment `hast` for `mdxjsEsm` with + // conflicting `data` types; the one that wins doesn't model `estree`, so type + // the access here. The unified pipeline's nodes always carry the program. + const body = (child.data as { estree?: Program } | undefined)?.estree?.body; if (!body) continue; for (const ast of body) { diff --git a/packages/integrations/mdx/src/rehype-apply-frontmatter-export.ts b/packages/integrations/mdx/src/unified/rehype-apply-frontmatter-export.ts similarity index 98% rename from packages/integrations/mdx/src/rehype-apply-frontmatter-export.ts rename to packages/integrations/mdx/src/unified/rehype-apply-frontmatter-export.ts index 5880c30b318b..168afc689cc5 100644 --- a/packages/integrations/mdx/src/rehype-apply-frontmatter-export.ts +++ b/packages/integrations/mdx/src/unified/rehype-apply-frontmatter-export.ts @@ -3,7 +3,7 @@ import { fileURLToPath } from 'node:url'; import { isFrontmatterValid } from '@astrojs/markdown-remark'; import type { Root, RootContent } from 'hast'; import type { VFile } from 'vfile'; -import { jsToTreeNode } from './utils.js'; +import { jsToTreeNode } from '../utils.js'; // Passed metadata to help determine adding charset utf8 by default declare module 'vfile' { diff --git a/packages/integrations/mdx/src/rehype-collect-headings.ts b/packages/integrations/mdx/src/unified/rehype-collect-headings.ts similarity index 87% rename from packages/integrations/mdx/src/rehype-collect-headings.ts rename to packages/integrations/mdx/src/unified/rehype-collect-headings.ts index a51e8e9f02d8..b75df4876b1a 100644 --- a/packages/integrations/mdx/src/rehype-collect-headings.ts +++ b/packages/integrations/mdx/src/unified/rehype-collect-headings.ts @@ -1,5 +1,5 @@ import type { VFile } from 'vfile'; -import { jsToTreeNode } from './utils.js'; +import { jsToTreeNode } from '../utils.js'; export function rehypeInjectHeadingsExport() { return function (tree: any, file: VFile) { diff --git a/packages/integrations/mdx/src/rehype-images-to-component.ts b/packages/integrations/mdx/src/unified/rehype-images-to-component.ts similarity index 86% rename from packages/integrations/mdx/src/rehype-images-to-component.ts rename to packages/integrations/mdx/src/unified/rehype-images-to-component.ts index 6676ee3230a6..6ae50a29a248 100644 --- a/packages/integrations/mdx/src/rehype-images-to-component.ts +++ b/packages/integrations/mdx/src/unified/rehype-images-to-component.ts @@ -3,11 +3,12 @@ import type { MdxJsxAttribute, MdxjsEsm } from 'mdast-util-mdx'; import type { MdxJsxFlowElementHast } from 'mdast-util-mdx-jsx'; import { visit } from 'unist-util-visit'; import type { VFile } from 'vfile'; -import { jsToTreeNode } from './utils.js'; - -export const ASTRO_IMAGE_ELEMENT = 'astro-image'; -export const ASTRO_IMAGE_IMPORT = '__AstroImage__'; -export const USES_ASTRO_IMAGE_FLAG = '__usesAstroImage'; +import { + ASTRO_IMAGE_ELEMENT, + ASTRO_IMAGE_IMPORT, + USES_ASTRO_IMAGE_FLAG, +} from '../image-constants.js'; +import { jsToTreeNode } from '../utils.js'; function createArrayAttribute(name: string, values: (string | number)[]): MdxJsxAttribute { return { @@ -40,23 +41,13 @@ function createArrayAttribute(name: string, values: (string | number)[]): MdxJsx }; } -/** - * Convert the element properties (except `src`) to MDX JSX attributes. - * - * @param {Properties} props - The element properties - * @returns {MdxJsxAttribute[]} The MDX attributes - */ function getImageComponentAttributes(props: Properties): MdxJsxAttribute[] { const attrs: MdxJsxAttribute[] = []; for (const [prop, value] of Object.entries(props)) { if (prop === 'src') continue; - /* - * component expects an array for those attributes but the - * received properties are sanitized as strings. So we need to convert them - * back to an array. - */ + // expects widths/densities as arrays; hast properties arrive as strings. if (prop === 'widths' || prop === 'densities') { attrs.push(createArrayAttribute(prop, String(value).split(' '))); } else { @@ -175,14 +166,13 @@ export function rehypeImageToComponent() { parent!.children.splice(index!, 1, element); }); - // Add all the import statements to the top of the file for the images tree.children.unshift(...importsStatements); tree.children.unshift( jsToTreeNode(`import { Image as ${ASTRO_IMAGE_IMPORT} } from "astro:assets";`), ); // Export `__usesAstroImage` to pick up `astro:assets` usage in the module graph. - // @see the '@astrojs/mdx-postprocess' plugin + // @see `vite-plugin-mdx-postprocess`. tree.children.push(jsToTreeNode(`export const ${USES_ASTRO_IMAGE_FLAG} = true`)); }; } diff --git a/packages/integrations/mdx/src/rehype-meta-string.ts b/packages/integrations/mdx/src/unified/rehype-meta-string.ts similarity index 100% rename from packages/integrations/mdx/src/rehype-meta-string.ts rename to packages/integrations/mdx/src/unified/rehype-meta-string.ts diff --git a/packages/integrations/mdx/src/rehype-optimize-static.ts b/packages/integrations/mdx/src/unified/rehype-optimize-static.ts similarity index 98% rename from packages/integrations/mdx/src/rehype-optimize-static.ts rename to packages/integrations/mdx/src/unified/rehype-optimize-static.ts index b6f772925cb0..81800f90a638 100644 --- a/packages/integrations/mdx/src/rehype-optimize-static.ts +++ b/packages/integrations/mdx/src/unified/rehype-optimize-static.ts @@ -1,5 +1,5 @@ import type { RehypePlugin } from '@astrojs/markdown-remark'; -import type { Expression } from 'estree'; +import type { Expression, Program } from 'estree'; import { SKIP, visit } from 'estree-util-visit'; import type { Element, RootContent, RootContentMap } from 'hast'; import { toHtml } from 'hast-util-to-html'; @@ -12,7 +12,7 @@ type ParentNode = Element | MdxJsxFlowElementHast | MdxJsxTextElementHast; // Nodes that can have its children optimized as a single HTML string type OptimizableNode = Element | MdxJsxFlowElementHast | MdxJsxTextElementHast; -export interface OptimizeOptions { +interface OptimizeOptions { ignoreElementNames?: string[]; } @@ -255,7 +255,11 @@ function getExportConstComponentObjectKeys(node: RootContentMap['mdxjsEsm']) { let variableInit: Expression | undefined | null; // Find the initial value of `components` in the AST. - for (const part of node.data?.estree?.body || []) { + for (const part of ( + node.data as { + estree: Program; + } + )?.estree?.body || []) { if ( part.type !== 'ExportNamedDeclaration' || part.declaration?.type !== 'VariableDeclaration' diff --git a/packages/integrations/mdx/src/utils.ts b/packages/integrations/mdx/src/utils.ts index b2e798212d34..2bde89e43a57 100644 --- a/packages/integrations/mdx/src/utils.ts +++ b/packages/integrations/mdx/src/utils.ts @@ -1,4 +1,4 @@ -import { parseFrontmatter } from '@astrojs/markdown-remark'; +import { parseFrontmatter } from '@astrojs/internal-helpers/frontmatter'; import type { Options as AcornOpts } from 'acorn'; import { parse } from 'acorn'; import type { AstroConfig, AstroIntegrationLogger, SSRError } from 'astro'; diff --git a/packages/integrations/mdx/src/vite-plugin-mdx-postprocess.ts b/packages/integrations/mdx/src/vite-plugin-mdx-postprocess.ts index c401f25e5abf..6a5115796e67 100644 --- a/packages/integrations/mdx/src/vite-plugin-mdx-postprocess.ts +++ b/packages/integrations/mdx/src/vite-plugin-mdx-postprocess.ts @@ -5,7 +5,7 @@ import { ASTRO_IMAGE_ELEMENT, ASTRO_IMAGE_IMPORT, USES_ASTRO_IMAGE_FLAG, -} from './rehype-images-to-component.js'; +} from './image-constants.js'; import { type FileInfo, getFileInfo } from './utils.js'; const underscoreFragmentImportRegex = /[\s,{]_Fragment[\s,}]/; diff --git a/packages/integrations/mdx/src/vite-plugin-mdx.ts b/packages/integrations/mdx/src/vite-plugin-mdx.ts index 5c79a6bf669f..d690567c2cdf 100644 --- a/packages/integrations/mdx/src/vite-plugin-mdx.ts +++ b/packages/integrations/mdx/src/vite-plugin-mdx.ts @@ -1,26 +1,32 @@ import type { SSRError } from 'astro'; -import { getAstroMetadata } from 'astro/jsx/rehype.js'; +import type { MarkdownProcessorEntry, MdxRenderer } from 'astro/markdown'; import { VFile } from 'vfile'; import type { Plugin } from 'vite'; import type { MdxOptions } from './index.js'; -import { createMdxProcessor } from './plugins.js'; import { safeParseFrontmatter } from './utils.js'; +// Inlined name-checks to avoid eagerly importing the unified/satteri runtime +// modules (each pulls in shiki + WASM bindings). The pipeline-specific factory +// is loaded lazily inside `resolveMdxRenderer`. +const isSatteriProcessor = (p: { name: string }) => p.name === 'satteri'; +const isUnifiedProcessor = (p: { name: string }) => p.name === 'unified'; + export interface VitePluginMdxOptions { mdxOptions: MdxOptions; srcDir: URL; + processor: MarkdownProcessorEntry; } // NOTE: Do not destructure `opts` as we're assigning a reference that will be mutated later export function vitePluginMdx(opts: VitePluginMdxOptions): Plugin { - let processor: ReturnType | undefined; + let mdxRenderer: MdxRenderer | undefined; let sourcemapEnabled: boolean; return { name: '@mdx-js/rolldown', enforce: 'pre', buildEnd() { - processor = undefined; + mdxRenderer = undefined; }, configResolved(resolved) { sourcemapEnabled = !!resolved.build.sourcemap; @@ -45,8 +51,6 @@ export function vitePluginMdx(opts: VitePluginMdxOptions): Plugin { } }, }, - // Override transform to alter code before MDX compilation - // ex. inject layouts transform: { filter: { id: /\.mdx$/, @@ -54,44 +58,28 @@ export function vitePluginMdx(opts: VitePluginMdxOptions): Plugin { async handler(code, id) { const { frontmatter, content } = safeParseFrontmatter(code, id); - const vfile = new VFile({ - value: content, - path: id, - data: { - astro: { - frontmatter, - }, - applyFrontmatterExport: { - srcDir: opts.srcDir, - }, - }, - }); - - // Lazily initialize the MDX processor - if (!processor) { - processor = createMdxProcessor(opts.mdxOptions, { - sourcemap: sourcemapEnabled, - }); - } - try { - const compiled = await processor.process(vfile); - + if (!mdxRenderer) { + mdxRenderer = await resolveMdxRenderer(opts, sourcemapEnabled); + } + const result = await mdxRenderer.process(content, id, frontmatter); return { - code: String(compiled.value), - map: compiled.map, - meta: getMdxMeta(vfile), + code: result.code, + map: result.map ?? null, + meta: { + astro: result.astroMetadata, + // `lang: 'ts'` makes Vite resolve `.js` import specifiers to `.ts` files. + vite: { lang: 'ts' }, + }, }; } catch (e: any) { const err: SSRError = e; - - // For some reason MDX puts the error location in the error's name, not very useful for us. + // Surface compile failures as a dedicated MDX error with a source + // location so the dev overlay can point at the offending file. err.name = 'MDXError'; err.loc = { file: id, line: e.line, column: e.column }; - - // For another some reason, MDX doesn't include a stack trace. Weird + // Compiler errors may arrive without a JS stack; capture one here. Error.captureStackTrace(err); - throw err; } }, @@ -99,19 +87,68 @@ export function vitePluginMdx(opts: VitePluginMdxOptions): Plugin { }; } -function getMdxMeta(vfile: VFile): Record { - const astroMetadata = getAstroMetadata(vfile); - if (!astroMetadata) { - throw new Error( - 'Internal MDX error: Astro metadata is not set by rehype-analyze-astro-metadata', +async function resolveMdxRenderer( + opts: VitePluginMdxOptions, + sourcemap: boolean, +): Promise { + const { processor } = opts; + + // Third-party processors opt into MDX support by implementing createMdxRenderer themselves. + if (processor.createMdxRenderer) { + return processor.createMdxRenderer( + { + syntaxHighlight: opts.mdxOptions.syntaxHighlight, + shikiConfig: opts.mdxOptions.shikiConfig, + gfm: opts.mdxOptions.gfm, + smartypants: opts.mdxOptions.smartypants, + }, + { optimize: opts.mdxOptions.optimize, recmaPlugins: opts.mdxOptions.recmaPlugins }, ); } - return { - astro: astroMetadata, - vite: { - // Setting this vite metadata to `ts` causes Vite to resolve .js - // extensions to .ts files. - lang: 'ts', - }, - }; + + if (isSatteriProcessor(processor)) { + const { createMdxProcessor: createSatteriMdxProcessor } = await import('./satteri/index.js'); + const satteriProcessor = createSatteriMdxProcessor(opts.mdxOptions, { srcDir: opts.srcDir }); + return { + async process(content, filePath, frontmatter) { + const result = await satteriProcessor.process(content, filePath, frontmatter); + return { code: result.code, map: null, astroMetadata: result.astroMetadata }; + }, + }; + } + if (isUnifiedProcessor(processor)) { + const { createMdxProcessor: createRemarkMdxProcessor, getAstroMetadata } = await import( + './unified/index.js' + ); + const remarkProcessor = createRemarkMdxProcessor(opts.mdxOptions, { sourcemap }); + return { + async process(content, filePath, frontmatter) { + const vfile = new VFile({ + value: content, + path: filePath, + data: { + astro: { frontmatter }, + applyFrontmatterExport: { srcDir: opts.srcDir }, + }, + }); + const compiled = await remarkProcessor.process(vfile); + const astroMetadata = getAstroMetadata(vfile); + if (!astroMetadata) { + throw new Error( + 'Internal MDX error: Astro metadata is not set by rehype-analyze-astro-metadata', + ); + } + return { + code: String(compiled.value), + map: compiled.map ? JSON.stringify(compiled.map) : null, + astroMetadata, + }; + }, + }; + } + + throw new Error( + `The markdown processor "${processor.name}" does not provide MDX support. ` + + `Implement \`createMdxRenderer\` on the processor descriptor to enable MDX rendering.`, + ); } diff --git a/packages/integrations/mdx/test/fixtures/image-remark-imgattr/astro.config.mjs b/packages/integrations/mdx/test/fixtures/image-remark-imgattr/astro.config.mjs index 7cd97f8b14a6..681653342969 100644 --- a/packages/integrations/mdx/test/fixtures/image-remark-imgattr/astro.config.mjs +++ b/packages/integrations/mdx/test/fixtures/image-remark-imgattr/astro.config.mjs @@ -1,8 +1,9 @@ import mdx from '@astrojs/mdx'; +import { unified } from '@astrojs/markdown-remark'; import { defineConfig } from 'astro/config'; import plugin from "./remarkPlugin" // https://astro.build/config export default defineConfig({ - integrations: [mdx({remarkPlugins:[plugin]})], + integrations: [mdx({ processor: unified({ remarkPlugins: [plugin] }) })], }); diff --git a/packages/integrations/mdx/test/fixtures/mdx-frontmatter-injection/astro.config.mjs b/packages/integrations/mdx/test/fixtures/mdx-frontmatter-injection/astro.config.mjs index 00d8e38f5397..232cea4b2e8f 100644 --- a/packages/integrations/mdx/test/fixtures/mdx-frontmatter-injection/astro.config.mjs +++ b/packages/integrations/mdx/test/fixtures/mdx-frontmatter-injection/astro.config.mjs @@ -1,12 +1,16 @@ import mdx from '@astrojs/mdx'; +import { unified } from '@astrojs/markdown-remark'; import { defineConfig } from 'astro/config'; import { rehypeReadingTime, remarkDescription, remarkTitle } from './src/markdown-plugins.mjs'; // https://astro.build/config export default defineConfig({ site: 'https://astro.build/', - integrations: [mdx({ - remarkPlugins: [remarkTitle, remarkDescription], - rehypePlugins: [rehypeReadingTime], - })], + integrations: [mdx()], + markdown: { + processor: unified({ + remarkPlugins: [remarkTitle, remarkDescription], + rehypePlugins: [rehypeReadingTime], + }), + }, }); diff --git a/packages/integrations/mdx/test/fixtures/mdx-frontmatter-injection/package.json b/packages/integrations/mdx/test/fixtures/mdx-frontmatter-injection/package.json index 0a193d6e2f65..0e8fc96f6527 100644 --- a/packages/integrations/mdx/test/fixtures/mdx-frontmatter-injection/package.json +++ b/packages/integrations/mdx/test/fixtures/mdx-frontmatter-injection/package.json @@ -3,6 +3,7 @@ "version": "0.0.0", "private": true, "dependencies": { + "@astrojs/markdown-remark": "workspace:*", "@astrojs/mdx": "workspace:*", "astro": "workspace:*", "mdast-util-to-string": "^4.0.0", diff --git a/packages/integrations/mdx/test/fixtures/mdx-vite-env-vars/astro.config.mjs b/packages/integrations/mdx/test/fixtures/mdx-vite-env-vars/astro.config.mjs index 717d379c01cd..863caf393602 100644 --- a/packages/integrations/mdx/test/fixtures/mdx-vite-env-vars/astro.config.mjs +++ b/packages/integrations/mdx/test/fixtures/mdx-vite-env-vars/astro.config.mjs @@ -1,9 +1,12 @@ import mdx from '@astrojs/mdx'; +import { satteri } from 'astro/config'; export default { site: 'https://mdx-is-neat.com/', markdown: { syntaxHighlight: false, + // Keep straight quotes so assertions can match `import.meta.env` output literally + processor: satteri({ features: { smartPunctuation: false } }), }, integrations: [mdx()], vite: { diff --git a/packages/integrations/mdx/test/mdx-astro-markdown-remarkRehype.test.ts b/packages/integrations/mdx/test/mdx-astro-markdown-remarkRehype.test.ts index 6e82a89f938a..56a7b8f66ede 100644 --- a/packages/integrations/mdx/test/mdx-astro-markdown-remarkRehype.test.ts +++ b/packages/integrations/mdx/test/mdx-astro-markdown-remarkRehype.test.ts @@ -1,5 +1,6 @@ import * as assert from 'node:assert/strict'; import { describe, it } from 'node:test'; +import { unified } from '@astrojs/markdown-remark'; import mdx from '@astrojs/mdx'; import { parseHTML } from 'linkedom'; import { loadFixture } from './test-utils.ts'; @@ -57,7 +58,7 @@ describe('MDX with Astro Markdown remark-rehype config', () => { ); }); - it('Renders footnotes with values from custom configuration without extending the default', async () => { + it('Renders footnotes by merging mdx remarkRehype on top of the markdown processor remarkRehype', async () => { const fixture = await loadFixture({ root: new URL('./fixtures/mdx-astro-markdown-remarkRehype/', import.meta.url), integrations: [ @@ -68,9 +69,11 @@ describe('MDX with Astro Markdown remark-rehype config', () => { }), ], markdown: { - remarkRehype: { - footnoteBackLabel: 'Kembali ke konten', - }, + processor: unified({ + remarkRehype: { + footnoteBackLabel: 'Kembali ke konten', + }, + }), }, }); @@ -81,7 +84,7 @@ describe('MDX with Astro Markdown remark-rehype config', () => { assert.equal(document.querySelector('#footnote-label')!.textContent, 'Catatan kaki'); assert.equal( document.querySelector('.data-footnote-backref')!.getAttribute('aria-label'), - 'Back to reference 1', + 'Kembali ke konten', ); }); }); diff --git a/packages/integrations/mdx/test/mdx-get-headings.test.ts b/packages/integrations/mdx/test/mdx-get-headings.test.ts index dec741a661af..bc2974a52d03 100644 --- a/packages/integrations/mdx/test/mdx-get-headings.test.ts +++ b/packages/integrations/mdx/test/mdx-get-headings.test.ts @@ -1,6 +1,6 @@ import * as assert from 'node:assert/strict'; import { before, describe, it } from 'node:test'; -import { rehypeHeadingIds } from '@astrojs/markdown-remark'; +import { rehypeHeadingIds, unified } from '@astrojs/markdown-remark'; import mdx from '@astrojs/mdx'; import { parseHTML } from 'linkedom'; import { visit } from 'unist-util-visit'; @@ -108,7 +108,7 @@ describe('MDX heading IDs can be customized by user plugins', () => { integrations: [mdx()], markdown: { rehypePlugins: [ - () => (tree) => { + () => (tree: any) => { let count = 0; visit(tree, 'element', (node) => { if (!/^h\d$/.test(node.tagName)) return; @@ -162,8 +162,9 @@ describe('MDX heading IDs can be injected before user plugins', () => { before(async () => { fixture = await loadFixture({ root: new URL('./fixtures/mdx-get-headings/', import.meta.url), - integrations: [ - mdx({ + integrations: [mdx()], + markdown: { + processor: unified({ rehypePlugins: [ rehypeHeadingIds, () => (tree) => { @@ -176,7 +177,7 @@ describe('MDX heading IDs can be injected before user plugins', () => { }, ], }), - ], + }, }); await fixture.build(); diff --git a/packages/integrations/mdx/test/mdx-plugins.test.ts b/packages/integrations/mdx/test/mdx-plugins.test.ts index 6eb7b1b8f2d1..ac4166a7dd53 100644 --- a/packages/integrations/mdx/test/mdx-plugins.test.ts +++ b/packages/integrations/mdx/test/mdx-plugins.test.ts @@ -1,5 +1,6 @@ import * as assert from 'node:assert/strict'; import { before, describe, it } from 'node:test'; +import { unified } from '@astrojs/markdown-remark'; import mdx from '@astrojs/mdx'; import { parseHTML } from 'linkedom'; import remarkToc from 'remark-toc'; @@ -25,7 +26,9 @@ describe('MDX plugins - Astro config integration', () => { 'astro:config:setup': ({ updateConfig }) => { updateConfig({ markdown: { - rehypePlugins: [rehypeExamplePlugin], + processor: unified({ + rehypePlugins: [rehypeExamplePlugin], + }), }, }); }, @@ -81,17 +84,30 @@ describe('MDX plugins - Astro config integration', () => { const html = await fixture.readFile(FILE); const { document } = parseHTML(html); - assert.notEqual(selectRemarkExample(document), null, 'MDX remark plugins not applied.'); - assert.notEqual(selectRehypeExample(document), null, 'MDX rehype plugins not applied.'); + // Coerce element/null to boolean before asserting — assert.equal on a raw + // linkedom Element vs null pulls util.inspect through the DOM's circular + // refs and can take down the test process on failure. + assert.equal( + selectRemarkExample(document) !== null, + true, + 'MDX remark plugins not applied.', + ); + assert.equal( + selectRehypeExample(document) !== null, + true, + 'MDX rehype plugins not applied.', + ); }); it('Handles Markdown plugins', async () => { const html = await fixture.readFile(FILE); const { document } = parseHTML(html); + // MDX provides its own `remarkPlugins`, so per docs it replaces + // markdown's `remarkToc` regardless of `extendMarkdownConfig`. assert.equal( - selectTocLink(document), - null, + selectTocLink(document) !== null, + false, '`remarkToc` plugin applied unexpectedly. Should override Markdown config.', ); }); @@ -100,10 +116,11 @@ describe('MDX plugins - Astro config integration', () => { const html = await fixture.readFile(FILE); const { document } = parseHTML(html); + const gfmLinkPresent = selectGfmLink(document) !== null; if (extendMarkdownConfig === true) { - assert.equal(selectGfmLink(document), null, 'Does not respect `markdown.gfm` option.'); + assert.equal(gfmLinkPresent, false, 'Does not respect `markdown.gfm` option.'); } else { - assert.notEqual(selectGfmLink(document), null, 'Respects `markdown.gfm` unexpectedly.'); + assert.equal(gfmLinkPresent, true, 'Respects `markdown.gfm` unexpectedly.'); } }); diff --git a/packages/integrations/mdx/test/mdx-syntax-highlighting.test.ts b/packages/integrations/mdx/test/mdx-syntax-highlighting.test.ts index 7058f8e45826..c199f588d03c 100644 --- a/packages/integrations/mdx/test/mdx-syntax-highlighting.test.ts +++ b/packages/integrations/mdx/test/mdx-syntax-highlighting.test.ts @@ -1,5 +1,6 @@ import * as assert from 'node:assert/strict'; import { describe, it } from 'node:test'; +import { unified } from '@astrojs/markdown-remark'; import mdx from '@astrojs/mdx'; import rehypeShiki from '@shikijs/rehype'; import { transformerTwoslash } from '@shikijs/twoslash'; @@ -63,6 +64,7 @@ describe('MDX syntax highlighting', () => { root: FIXTURE_ROOT, markdown: { syntaxHighlight: 'prism', + processor: unified(), }, integrations: [mdx()], }); @@ -81,6 +83,7 @@ describe('MDX syntax highlighting', () => { root: FIXTURE_ROOT, markdown: { syntaxHighlight: 'shiki', + processor: unified(), }, integrations: [ mdx({ @@ -107,6 +110,7 @@ describe('MDX syntax highlighting', () => { root: FIXTURE_ROOT, markdown: { syntaxHighlight: false, + processor: unified(), }, integrations: [ mdx({ @@ -139,6 +143,7 @@ describe('MDX syntax highlighting', () => { root: FIXTURE_ROOT, markdown: { syntaxHighlight: false, + processor: unified(), }, integrations: [ mdx({ diff --git a/packages/integrations/mdx/test/mdx-vite-env-vars.test.ts b/packages/integrations/mdx/test/mdx-vite-env-vars.test.ts index f2f105a8e3c3..576d1cb505e0 100644 --- a/packages/integrations/mdx/test/mdx-vite-env-vars.test.ts +++ b/packages/integrations/mdx/test/mdx-vite-env-vars.test.ts @@ -46,7 +46,7 @@ describe('MDX - Vite env vars', () => { assert.equal( document .querySelector('[data-env-variable-exports-unknown]') - ?.innerHTML.includes('exports: ""'), + ?.textContent?.includes('exports: ""'), true, ); }); diff --git a/packages/integrations/mdx/test/units/rehype-optimize-static.test.ts b/packages/integrations/mdx/test/units/rehype-optimize-static.test.ts index f93b8d3f3338..9063256e38b4 100644 --- a/packages/integrations/mdx/test/units/rehype-optimize-static.test.ts +++ b/packages/integrations/mdx/test/units/rehype-optimize-static.test.ts @@ -1,7 +1,7 @@ import assert from 'node:assert/strict'; import { describe, it } from 'node:test'; import { compile as _compile, type CompileOptions } from '@mdx-js/mdx'; -import { rehypeOptimizeStatic } from '../../dist/rehype-optimize-static.js'; +import { rehypeOptimizeStatic } from '../../dist/unified/rehype-optimize-static.js'; async function compile(mdxCode: string, options?: Readonly) { const result = await _compile(mdxCode, { diff --git a/packages/integrations/mdx/test/units/rehype-plugins.test.ts b/packages/integrations/mdx/test/units/rehype-plugins.test.ts index 89e65acaa53b..e3b80f1a9982 100644 --- a/packages/integrations/mdx/test/units/rehype-plugins.test.ts +++ b/packages/integrations/mdx/test/units/rehype-plugins.test.ts @@ -2,8 +2,8 @@ import assert from 'node:assert/strict'; import { describe, it } from 'node:test'; import type * as hast from 'hast'; import { VFile } from 'vfile'; -import { rehypeInjectHeadingsExport } from '../../dist/rehype-collect-headings.js'; -import rehypeMetaString from '../../dist/rehype-meta-string.js'; +import { rehypeInjectHeadingsExport } from '../../dist/unified/rehype-collect-headings.js'; +import rehypeMetaString from '../../dist/unified/rehype-meta-string.js'; describe('rehypeMetaString', () => { function createCodeNode(meta: string | undefined): hast.Element { diff --git a/packages/integrations/solid/src/index.ts b/packages/integrations/solid/src/index.ts index 3ee9e391a870..91712fc8ccec 100644 --- a/packages/integrations/solid/src/index.ts +++ b/packages/integrations/solid/src/index.ts @@ -44,7 +44,10 @@ function getViteConfiguration( { include, exclude }: Options, devtoolsPlugin: DevtoolsPlugin | null, ) { - const plugins: PluginOption[] = [solid({ include, exclude, ssr: true }), configEnvironmentPlugin()]; + const plugins: PluginOption[] = [ + solid({ include, exclude, ssr: true }), + configEnvironmentPlugin(), + ]; if (devtoolsPlugin) { plugins.push(devtoolsPlugin({ autoname: true })); diff --git a/packages/integrations/vue/src/index.ts b/packages/integrations/vue/src/index.ts index 2e86b0d4cba7..6e8cbe3a4d63 100644 --- a/packages/integrations/vue/src/index.ts +++ b/packages/integrations/vue/src/index.ts @@ -121,7 +121,11 @@ async function getViteConfiguration( // The vue vite plugin may not manage to resolve it automatically vueOptions.compiler ??= await import('vue/compiler-sfc'); - const plugins: PluginOption[] = [vue(vueOptions), virtualAppEntrypoint(vueOptions), configEnvironmentPlugin()]; + const plugins: PluginOption[] = [ + vue(vueOptions), + virtualAppEntrypoint(vueOptions), + configEnvironmentPlugin(), + ]; if (options?.jsx) { const vueJsx = (await import('@vitejs/plugin-vue-jsx')).default; diff --git a/packages/internal-helpers/package.json b/packages/internal-helpers/package.json index b37c25b0d32d..2a7a88871126 100644 --- a/packages/internal-helpers/package.json +++ b/packages/internal-helpers/package.json @@ -18,7 +18,16 @@ "./cli": "./dist/cli.js", "./create-filter": "./dist/create-filter.js", "./request": "./dist/request.js", - "./object": "./dist/object.js" + "./object": "./dist/object.js", + "./markdown": "./dist/markdown.js", + "./frontmatter": "./dist/frontmatter.js", + "./shiki": "./dist/shiki.js" + }, + "imports": { + "#shiki-engine": { + "workerd": "./dist/shiki-engine-workerd.js", + "default": "./dist/shiki-engine-default.js" + } }, "typesVersions": { "*": { @@ -28,6 +37,15 @@ "remote": [ "./dist/remote.d.ts" ], + "markdown": [ + "./dist/markdown.d.ts" + ], + "frontmatter": [ + "./dist/frontmatter.d.ts" + ], + "shiki": [ + "./dist/shiki.d.ts" + ], "fs": [ "./dist/fs.d.ts" ], @@ -53,9 +71,17 @@ "test": "astro-scripts test \"test/**/*.test.ts\"" }, "dependencies": { - "picomatch": "^4.0.4" + "@types/hast": "^3.0.4", + "@types/mdast": "^4.0.4", + "js-yaml": "^4.1.1", + "picomatch": "^4.0.4", + "retext-smartypants": "^6.2.0", + "shiki": "^4.0.2", + "smol-toml": "^1.6.0", + "unified": "^11.0.5" }, "devDependencies": { + "@types/js-yaml": "^4.0.9", "@types/picomatch": "^4.0.2", "astro-scripts": "workspace:*" }, diff --git a/packages/markdown/remark/src/frontmatter.ts b/packages/internal-helpers/src/frontmatter.ts similarity index 88% rename from packages/markdown/remark/src/frontmatter.ts rename to packages/internal-helpers/src/frontmatter.ts index 16c103f73598..9a26b7e3bba0 100644 --- a/packages/markdown/remark/src/frontmatter.ts +++ b/packages/internal-helpers/src/frontmatter.ts @@ -11,12 +11,12 @@ export function isFrontmatterValid(frontmatter: Record) { return typeof frontmatter === 'object' && frontmatter !== null; } -// Capture frontmatter wrapped with `---` or `+++`, including any characters and new lines within it. -// Only capture if `---` or `+++` exists near the top of the file, including: -// 1. Start of file (including if has BOM encoding) -// 2. Start of file with any whitespace (but `---` or `+++` must still start on a new line) +// Capture frontmatter wrapped with `---` or `+++` near the top of the file — +// either at the very start (allowing a BOM) or after leading whitespace, with +// the fence still starting on a new line. const frontmatterRE = /(?:^\uFEFF?|^\s*\n)(?:---|\+\+\+)([\s\S]*?\n)(?:---|\+\+\+)/; const frontmatterTypeRE = /(?:^\uFEFF?|^\s*\n)(---|\+\+\+)/; + export function extractFrontmatter(code: string): string | undefined { return frontmatterRE.exec(code)?.[1]; } @@ -24,6 +24,7 @@ export function extractFrontmatter(code: string): string | undefined { function getFrontmatterParser(code: string): [string, (str: string) => unknown] { return frontmatterTypeRE.exec(code)?.[1] === '+++' ? ['+++', toml.parse] : ['---', yaml.load]; } + export interface ParseFrontmatterOptions { /** * How the frontmatter should be handled in the returned `content` string. diff --git a/packages/internal-helpers/src/markdown.ts b/packages/internal-helpers/src/markdown.ts new file mode 100644 index 000000000000..bb21c274fe09 --- /dev/null +++ b/packages/internal-helpers/src/markdown.ts @@ -0,0 +1,158 @@ +import type * as hast from 'hast'; +import type * as mdast from 'mdast'; +import type { Options as SmartypantsOptions } from 'retext-smartypants'; +import type { + BuiltinTheme, + HighlighterCoreOptions, + LanguageRegistration, + ShikiTransformer, + ThemeRegistration, + ThemeRegistrationRaw, +} from 'shiki'; +import type { Plugin } from 'unified'; +import type { RemotePattern } from './remote.js'; + +// Processor-agnostic markdown contract types, shared between `astro` and the +// markdown processor packages. They live here because `astro` depends on the +// processor packages, so the shared types must sit below all of them. + +export type SyntaxHighlightConfigType = 'shiki' | 'prism'; + +export interface SyntaxHighlightConfig { + type: SyntaxHighlightConfigType; + excludeLangs?: string[]; +} + +type ThemePresets = BuiltinTheme | 'css-variables'; + +export interface ShikiConfig { + langs?: LanguageRegistration[]; + theme?: ThemePresets | ThemeRegistration | ThemeRegistrationRaw; + themes?: Record; + langAlias?: HighlighterCoreOptions['langAlias']; + defaultColor?: 'light' | 'dark' | string | false; + wrap?: boolean | null; + transformers?: ShikiTransformer[]; +} + +export type Smartypants = SmartypantsOptions; + +// Plugin/option shapes for the deprecated `markdown.remarkPlugins` / +// `rehypePlugins` / `remarkRehype` config fields. + +export type RemarkPlugin = Plugin< + PluginParameters, + mdast.Root +>; +export type RemarkPlugins = (string | [string, any] | RemarkPlugin | [RemarkPlugin, any])[]; +export type RehypePlugin = Plugin< + PluginParameters, + hast.Root +>; +export type RehypePlugins = (string | [string, any] | RehypePlugin | [RehypePlugin, any])[]; +export type RemarkRehype = Record; + +export interface MarkdownHeading { + depth: number; + slug: string; + text: string; +} + +export interface AstroMarkdownProcessorOptions { + syntaxHighlight?: SyntaxHighlightConfig | SyntaxHighlightConfigType | false; + shikiConfig?: ShikiConfig; + /** @deprecated Configure via the processor instead, e.g. `unified({ gfm: false })` or `satteri({ features: { gfm: false } })`. */ + gfm?: boolean; + /** @deprecated Configure via the processor instead, e.g. `unified({ smartypants: false })` or `satteri({ features: { smartPunctuation: false } })`. */ + smartypants?: boolean | SmartypantsOptions; + /** @deprecated Use `markdown.processor: unified({ remarkPlugins })` instead. */ + remarkPlugins?: RemarkPlugins; + /** @deprecated Use `markdown.processor: unified({ rehypePlugins })` instead. */ + rehypePlugins?: RehypePlugins; + /** @deprecated Use `markdown.processor: unified({ remarkRehype })` instead. */ + remarkRehype?: RemarkRehype; + image?: { + domains?: string[]; + remotePatterns?: RemotePattern[]; + }; +} + +export interface MarkdownProcessor { + render: ( + content: string, + opts?: MarkdownProcessorRenderOptions, + ) => Promise; +} + +export interface MarkdownProcessorRenderOptions { + fileURL?: URL; + frontmatter?: Record; +} + +export interface MarkdownProcessorRenderResult { + code: string; + metadata: { + headings: MarkdownHeading[]; + localImagePaths: string[]; + remoteImagePaths: string[]; + frontmatter: Record; + }; +} + +// MDX rendering metadata. Produced by `@astrojs/mdx`'s Sätteri and unified pipelines +// and surfaced on Vite's `meta.astro`; defined here so both pipelines share one shape. + +export interface AstroComponentMetadata { + exportName: string; + localName: string; + specifier: string; + resolvedPath: string; +} + +export interface AstroMetadata { + hydratedComponents: AstroComponentMetadata[]; + clientOnlyComponents: AstroComponentMetadata[]; + serverComponents: AstroComponentMetadata[]; + scripts: never[]; + propagation: 'none'; + containsHead: false; + pageOptions: Record; +} + +export function createDefaultAstroMetadata(): AstroMetadata { + return { + hydratedComponents: [], + clientOnlyComponents: [], + serverComponents: [], + scripts: [], + propagation: 'none', + containsHead: false, + pageOptions: {}, + }; +} + +// Languages never syntax-highlighted by default (e.g. `math`, handled elsewhere). +export const defaultExcludeLanguages = ['math']; + +export const syntaxHighlightDefaults: Required = { + type: 'shiki', + excludeLangs: defaultExcludeLanguages, +}; + +/** Default values for the markdown config, regardless of processor. */ +export const markdownConfigDefaults: Required> = { + syntaxHighlight: syntaxHighlightDefaults, + shikiConfig: { + langs: [], + theme: 'github-dark', + themes: {}, + wrap: false, + transformers: [], + langAlias: {}, + }, + gfm: true, + smartypants: true, + remarkPlugins: [], + rehypePlugins: [], + remarkRehype: {}, +}; diff --git a/packages/markdown/remark/src/shiki-engine-default.ts b/packages/internal-helpers/src/shiki-engine-default.ts similarity index 100% rename from packages/markdown/remark/src/shiki-engine-default.ts rename to packages/internal-helpers/src/shiki-engine-default.ts diff --git a/packages/markdown/remark/src/shiki-engine-workerd.ts b/packages/internal-helpers/src/shiki-engine-workerd.ts similarity index 100% rename from packages/markdown/remark/src/shiki-engine-workerd.ts rename to packages/internal-helpers/src/shiki-engine-workerd.ts diff --git a/packages/markdown/remark/src/shiki.ts b/packages/internal-helpers/src/shiki.ts similarity index 98% rename from packages/markdown/remark/src/shiki.ts rename to packages/internal-helpers/src/shiki.ts index 28d7124381b5..502b8d74a670 100644 --- a/packages/markdown/remark/src/shiki.ts +++ b/packages/internal-helpers/src/shiki.ts @@ -1,6 +1,7 @@ import type { Properties, Root } from 'hast'; import { type BuiltinLanguage, + type BuiltinTheme, type BundledLanguage, createCssVariablesTheme, createHighlighter, @@ -14,9 +15,10 @@ import { type ThemeRegistration, type ThemeRegistrationRaw, } from 'shiki'; -import type { ThemePresets } from './types.js'; import { loadShikiEngine } from '#shiki-engine'; +export type ThemePresets = BuiltinTheme | 'css-variables'; + export interface ShikiHighlighter { codeToHast( code: string, @@ -299,6 +301,3 @@ async function createShikiHighlighterInternal({ function normalizePropAsString(value: Properties[string]): string | null { return Array.isArray(value) ? value.join(' ') : (value as string | null); } - -// Re-export ThemePresets type for consumers -export type { ThemePresets }; diff --git a/packages/markdown/remark/test/frontmatter.test.ts b/packages/internal-helpers/test/frontmatter.test.ts similarity index 99% rename from packages/markdown/remark/test/frontmatter.test.ts rename to packages/internal-helpers/test/frontmatter.test.ts index ee365d19e4de..f9355035f2de 100644 --- a/packages/markdown/remark/test/frontmatter.test.ts +++ b/packages/internal-helpers/test/frontmatter.test.ts @@ -4,7 +4,7 @@ import { extractFrontmatter, parseFrontmatter, type ParseFrontmatterOptions, -} from '../dist/index.js'; +} from '../dist/frontmatter.js'; type FrontmatterStyle = ParseFrontmatterOptions['frontmatter']; diff --git a/packages/markdown/remark/package.json b/packages/markdown/remark/package.json index 247f62af7b34..39c59188ffac 100644 --- a/packages/markdown/remark/package.json +++ b/packages/markdown/remark/package.json @@ -13,17 +13,12 @@ "homepage": "https://astro.build", "main": "./dist/index.js", "exports": { - ".": "./dist/index.js", - "./shiki": "./dist/shiki.js" + ".": "./dist/index.js" }, "imports": { "#import-plugin": { "browser": "./dist/import-plugin-browser.js", "default": "./dist/import-plugin-default.js" - }, - "#shiki-engine": { - "workerd": "./dist/shiki-engine-workerd.js", - "default": "./dist/shiki-engine-default.js" } }, "files": [ @@ -42,7 +37,6 @@ "github-slugger": "^2.0.0", "hast-util-from-html": "^2.0.3", "hast-util-to-text": "^4.0.2", - "js-yaml": "^4.1.1", "mdast-util-definitions": "^6.0.0", "rehype-raw": "^7.0.0", "rehype-stringify": "^10.0.1", @@ -50,9 +44,6 @@ "remark-parse": "^11.0.0", "remark-rehype": "^11.1.2", "remark-smartypants": "^3.0.2", - "retext-smartypants": "^6.2.0", - "shiki": "^4.0.0", - "smol-toml": "^1.6.0", "unified": "^11.0.5", "unist-util-remove-position": "^5.0.0", "unist-util-visit": "^5.1.0", @@ -62,12 +53,12 @@ "devDependencies": { "@types/estree": "^1.0.8", "@types/hast": "^3.0.4", - "@types/js-yaml": "^4.0.9", "@types/mdast": "^4.0.4", "@types/unist": "^3.0.3", "astro-scripts": "workspace:*", "esbuild": "^0.28.0", - "mdast-util-mdx-expression": "^2.0.1" + "mdast-util-mdx-expression": "^2.0.1", + "shiki": "^4.0.0" }, "publishConfig": { "provenance": true diff --git a/packages/markdown/remark/src/highlight.ts b/packages/markdown/remark/src/highlight.ts index d4457c7d2015..dcde13e9fa69 100644 --- a/packages/markdown/remark/src/highlight.ts +++ b/packages/markdown/remark/src/highlight.ts @@ -1,3 +1,4 @@ +import { defaultExcludeLanguages } from '@astrojs/internal-helpers/markdown'; import type { Element, Parent, Root } from 'hast'; import { fromHtml } from 'hast-util-from-html'; import { toText } from 'hast-util-to-text'; @@ -11,8 +12,6 @@ type Highlighter = ( ) => Promise; const languagePattern = /\blanguage-(\S+)\b/; -// Don’t highlight math code blocks by default. -export const defaultExcludeLanguages = ['math']; /** * A hast utility to syntax highlight code blocks with a given syntax highlighter. diff --git a/packages/markdown/remark/src/index.ts b/packages/markdown/remark/src/index.ts index 4fcd8d304b72..807373d4c679 100644 --- a/packages/markdown/remark/src/index.ts +++ b/packages/markdown/remark/src/index.ts @@ -1,3 +1,4 @@ +import { markdownConfigDefaults } from '@astrojs/internal-helpers/markdown'; import rehypeRaw from 'rehype-raw'; import rehypeStringify from 'rehype-stringify'; import remarkGfm from 'remark-gfm'; @@ -6,7 +7,6 @@ import remarkRehype from 'remark-rehype'; import remarkSmartypants from 'remark-smartypants'; import { unified } from 'unified'; import { VFile } from 'vfile'; -import { defaultExcludeLanguages } from './highlight.js'; import { loadPlugins } from './load-plugins.js'; import { rehypeHeadingIds } from './rehype-collect-headings.js'; import { rehypeImages } from './rehype-images.js'; @@ -14,52 +14,63 @@ import { rehypePrism } from './rehype-prism.js'; import { rehypeShiki } from './rehype-shiki.js'; import { remarkCollectImages } from './remark-collect-images.js'; import type { - AstroMarkdownOptions, AstroMarkdownProcessorOptions, + MarkdownHeading, MarkdownProcessor, - SyntaxHighlightConfig, -} from './types.js'; +} from '@astrojs/internal-helpers/markdown'; + +// `vfile.data.astro` is populated by the unified pipeline as it renders. +declare module 'vfile' { + interface DataMap { + astro: { + headings?: MarkdownHeading[]; + localImagePaths?: string[]; + remoteImagePaths?: string[]; + frontmatter?: Record; + }; + } +} +export type { Node } from 'unist'; +// The markdown contract types live in `@astrojs/internal-helpers/markdown`; +// re-exported so consumers keep a single import surface. +export type { + AstroMarkdownProcessorOptions, + MarkdownHeading, + MarkdownProcessor, + MarkdownProcessorRenderOptions, + MarkdownProcessorRenderResult, + RehypePlugin, + RehypePlugins, + RemarkPlugin, + RemarkPlugins, + RemarkRehype, + ShikiConfig, + Smartypants, + SyntaxHighlightConfig, + SyntaxHighlightConfigType, +} from '@astrojs/internal-helpers/markdown'; export { extractFrontmatter, isFrontmatterValid, type ParseFrontmatterOptions, type ParseFrontmatterResult, parseFrontmatter, -} from './frontmatter.js'; +} from '@astrojs/internal-helpers/frontmatter'; export { rehypeHeadingIds } from './rehype-collect-headings.js'; export { rehypePrism } from './rehype-prism.js'; export { rehypeShiki } from './rehype-shiki.js'; export { remarkCollectImages } from './remark-collect-images.js'; export { - type CreateShikiHighlighterOptions, - createShikiHighlighter, - type ShikiHighlighter, - type ShikiHighlighterHighlightOptions, -} from './shiki.js'; -export * from './types.js'; - -export const syntaxHighlightDefaults: Required = { - type: 'shiki', - excludeLangs: defaultExcludeLanguages, -}; - -export const markdownConfigDefaults: Required = { - syntaxHighlight: syntaxHighlightDefaults, - shikiConfig: { - langs: [], - theme: 'github-dark', - themes: {}, - wrap: false, - transformers: [], - langAlias: {}, - }, - remarkPlugins: [], - rehypePlugins: [], - remarkRehype: {}, - gfm: true, - smartypants: true, -}; + isUnifiedProcessor, + type UnifiedProcessorDescriptor, + type UnifiedProcessorOptions, + unified, +} from './processor.js'; +export { + markdownConfigDefaults, + syntaxHighlightDefaults, +} from '@astrojs/internal-helpers/markdown'; // Skip nonessential plugins during performance benchmark runs const isPerformanceBenchmark = Boolean(process.env.ASTRO_PERFORMANCE_BENCHMARK); diff --git a/packages/markdown/remark/src/processor.ts b/packages/markdown/remark/src/processor.ts new file mode 100644 index 000000000000..7e363062b500 --- /dev/null +++ b/packages/markdown/remark/src/processor.ts @@ -0,0 +1,66 @@ +import type { + AstroMarkdownProcessorOptions, + MarkdownProcessor, + RehypePlugins, + RemarkPlugins, + RemarkRehype, + Smartypants, +} from '@astrojs/internal-helpers/markdown'; + +export interface UnifiedProcessorOptions { + remarkPlugins?: RemarkPlugins; + rehypePlugins?: RehypePlugins; + remarkRehype?: RemarkRehype; + /** Enable GitHub-Flavored Markdown. Defaults to `true`. */ + gfm?: boolean; + /** Enable SmartyPants typography. Defaults to `true`; pass an object to configure it. */ + smartypants?: boolean | Smartypants; +} + +/** + * The descriptor returned by `unified()`. Integrations extend the pipeline by mutating + * `descriptor.options.remarkPlugins` / `rehypePlugins` / `remarkRehype` directly. + */ +export interface UnifiedProcessorDescriptor { + readonly name: 'unified'; + options: { + remarkPlugins: RemarkPlugins; + rehypePlugins: RehypePlugins; + remarkRehype: RemarkRehype; + gfm?: boolean; + smartypants?: boolean | Smartypants; + }; + createRenderer(shared: AstroMarkdownProcessorOptions): Promise; +} + +export function unified(opts: UnifiedProcessorOptions = {}): UnifiedProcessorDescriptor { + const descriptor: UnifiedProcessorDescriptor = { + name: 'unified', + options: { + remarkPlugins: [...(opts.remarkPlugins ?? [])], + rehypePlugins: [...(opts.rehypePlugins ?? [])], + remarkRehype: { ...opts.remarkRehype }, + gfm: opts.gfm, + smartypants: opts.smartypants, + }, + async createRenderer(shared) { + // Lazy import to avoid a circular module load with `./index.js`. + const { createMarkdownProcessor } = await import('./index.js'); + return createMarkdownProcessor({ + ...shared, + remarkPlugins: descriptor.options.remarkPlugins, + rehypePlugins: descriptor.options.rehypePlugins, + remarkRehype: descriptor.options.remarkRehype, + // `unified({ gfm, smartypants })` wins; fall back to the deprecated + // top-level `markdown.gfm` / `markdown.smartypants` while they still exist. + gfm: descriptor.options.gfm ?? shared.gfm, + smartypants: descriptor.options.smartypants ?? shared.smartypants, + }); + }, + }; + return descriptor; +} + +export function isUnifiedProcessor(p: { name: string }): p is UnifiedProcessorDescriptor { + return p.name === 'unified'; +} diff --git a/packages/markdown/remark/src/rehype-collect-headings.ts b/packages/markdown/remark/src/rehype-collect-headings.ts index 8f4aa55d369e..4e65c44f581e 100644 --- a/packages/markdown/remark/src/rehype-collect-headings.ts +++ b/packages/markdown/remark/src/rehype-collect-headings.ts @@ -4,7 +4,7 @@ import type { MdxTextExpression } from 'mdast-util-mdx-expression'; import type { Node } from 'unist'; import { visit } from 'unist-util-visit'; import type { VFile } from 'vfile'; -import type { MarkdownHeading, RehypePlugin } from './types.js'; +import type { MarkdownHeading, RehypePlugin } from '@astrojs/internal-helpers/markdown'; import { FORBIDDEN_PATH_KEYS } from '@astrojs/internal-helpers/object'; diff --git a/packages/markdown/remark/src/rehype-shiki.ts b/packages/markdown/remark/src/rehype-shiki.ts index 9f05d65b89b7..e10137127874 100644 --- a/packages/markdown/remark/src/rehype-shiki.ts +++ b/packages/markdown/remark/src/rehype-shiki.ts @@ -1,8 +1,8 @@ import type { Root } from 'hast'; import type { Plugin } from 'unified'; import { highlightCodeBlocks } from './highlight.js'; -import { createShikiHighlighter, type ShikiHighlighter } from './shiki.js'; -import type { ShikiConfig } from './types.js'; +import { createShikiHighlighter, type ShikiHighlighter } from '@astrojs/internal-helpers/shiki'; +import type { ShikiConfig } from '@astrojs/internal-helpers/markdown'; export const rehypeShiki: Plugin<[ShikiConfig, string[]?], Root> = (config, excludeLangs) => { let highlighterAsync: Promise | undefined; diff --git a/packages/markdown/remark/src/remark-collect-images.ts b/packages/markdown/remark/src/remark-collect-images.ts index 0fa946670823..32f77ec19cc8 100644 --- a/packages/markdown/remark/src/remark-collect-images.ts +++ b/packages/markdown/remark/src/remark-collect-images.ts @@ -3,7 +3,7 @@ import type { Root } from 'mdast'; import { definitions } from 'mdast-util-definitions'; import { visit } from 'unist-util-visit'; import type { VFile } from 'vfile'; -import type { AstroMarkdownProcessorOptions } from './types.js'; +import type { AstroMarkdownProcessorOptions } from '@astrojs/internal-helpers/markdown'; export function remarkCollectImages(opts: AstroMarkdownProcessorOptions['image']) { const domains = opts?.domains ?? []; diff --git a/packages/markdown/remark/src/types.ts b/packages/markdown/remark/src/types.ts deleted file mode 100644 index 3fbba1ed9812..000000000000 --- a/packages/markdown/remark/src/types.ts +++ /dev/null @@ -1,105 +0,0 @@ -import type { RemotePattern } from '@astrojs/internal-helpers/remote'; -import type * as hast from 'hast'; -import type * as mdast from 'mdast'; -import type { Options as RemarkRehypeOptions } from 'remark-rehype'; -import type { Options as SmartypantsOptions } from 'retext-smartypants'; -import type { BuiltinTheme } from 'shiki'; -import type * as unified from 'unified'; -import type { CreateShikiHighlighterOptions, ShikiHighlighterHighlightOptions } from './shiki.js'; - -export type { Node } from 'unist'; - -declare module 'vfile' { - interface DataMap { - astro: { - headings?: MarkdownHeading[]; - localImagePaths?: string[]; - remoteImagePaths?: string[]; - frontmatter?: Record; - }; - } -} - -export type RemarkPlugin = unified.Plugin< - PluginParameters, - mdast.Root ->; - -export type RemarkPlugins = (string | [string, any] | RemarkPlugin | [RemarkPlugin, any])[]; - -export type RehypePlugin = unified.Plugin< - PluginParameters, - hast.Root ->; - -export type RehypePlugins = (string | [string, any] | RehypePlugin | [RehypePlugin, any])[]; - -export type RemarkRehype = RemarkRehypeOptions; - -export type Smartypants = SmartypantsOptions; - -export type ThemePresets = BuiltinTheme | 'css-variables'; - -export type SyntaxHighlightConfigType = 'shiki' | 'prism'; - -export interface SyntaxHighlightConfig { - type: SyntaxHighlightConfigType; - excludeLangs?: string[]; -} - -export interface ShikiConfig - extends Pick, - Pick {} - -/** - * Configuration options that end up in the markdown section of AstroConfig - */ -export interface AstroMarkdownOptions { - syntaxHighlight?: SyntaxHighlightConfig | SyntaxHighlightConfigType | false; - shikiConfig?: ShikiConfig; - remarkPlugins?: RemarkPlugins; - rehypePlugins?: RehypePlugins; - remarkRehype?: RemarkRehype; - gfm?: boolean; - smartypants?: boolean | SmartypantsOptions; -} - -/** - * Extra configuration options from other parts of AstroConfig that get injected into this plugin - */ -export interface AstroMarkdownProcessorOptions extends AstroMarkdownOptions { - image?: { - domains?: string[]; - remotePatterns?: RemotePattern[]; - }; -} - -export interface MarkdownProcessor { - render: ( - content: string, - opts?: MarkdownProcessorRenderOptions, - ) => Promise; -} - -export interface MarkdownProcessorRenderOptions { - /** The URL of the file being rendered, used for resolving relative image paths */ - fileURL?: URL; - /** Used for frontmatter injection plugins */ - frontmatter?: Record; -} - -export interface MarkdownProcessorRenderResult { - code: string; - metadata: { - headings: MarkdownHeading[]; - localImagePaths: string[]; - remoteImagePaths: string[]; - frontmatter: Record; - }; -} - -export interface MarkdownHeading { - depth: number; - slug: string; - text: string; -} diff --git a/packages/markdown/remark/test/shiki.test.ts b/packages/markdown/remark/test/shiki.test.ts index bc6d0dcb2fe3..51c2048be196 100644 --- a/packages/markdown/remark/test/shiki.test.ts +++ b/packages/markdown/remark/test/shiki.test.ts @@ -2,13 +2,10 @@ import assert from 'node:assert/strict'; import { describe, it } from 'node:test'; import type { Element } from 'hast'; import type { LanguageRegistration, ThemeRegistration } from 'shiki'; -import { - createMarkdownProcessor, - createShikiHighlighter, - type ShikiHighlighter, -} from '../dist/index.js'; +import { type ShikiHighlighter, createShikiHighlighter } from '@astrojs/internal-helpers/shiki'; // @ts-expect-error: `clearShikiHighlighterCache` is marked `@internal` and stripped from the `.d.ts`, but still exists at runtime. -import { clearShikiHighlighterCache } from '../dist/shiki.js'; +import { clearShikiHighlighterCache } from '@astrojs/internal-helpers/shiki'; +import { createMarkdownProcessor } from '../dist/index.js'; describe('shiki syntax highlighting', () => { it('does not add is:raw to the output', async () => { diff --git a/packages/markdown/satteri/package.json b/packages/markdown/satteri/package.json new file mode 100644 index 000000000000..3285ad05cf1c --- /dev/null +++ b/packages/markdown/satteri/package.json @@ -0,0 +1,40 @@ +{ + "name": "@astrojs/markdown-satteri", + "version": "0.1.0", + "type": "module", + "author": "withastro", + "license": "MIT", + "repository": { + "type": "git", + "url": "git+https://github.com/withastro/astro.git", + "directory": "packages/markdown/satteri" + }, + "bugs": "https://github.com/withastro/astro/issues", + "homepage": "https://astro.build", + "main": "./dist/index.js", + "exports": { + ".": "./dist/index.js" + }, + "files": [ + "dist" + ], + "scripts": { + "prepublish": "pnpm build", + "build": "astro-scripts build \"src/**/*.ts\" && tsc -p tsconfig.json", + "build:ci": "astro-scripts build \"src/**/*.ts\"", + "dev": "astro-scripts dev \"src/**/*.ts\"", + "test": "astro-scripts test \"test/**/*.test.ts\"", + "typecheck:tests": "tsc --build tsconfig.test.json" + }, + "dependencies": { + "@astrojs/internal-helpers": "workspace:*", + "github-slugger": "^2.0.0", + "satteri": "^0.6.3" + }, + "devDependencies": { + "astro-scripts": "workspace:*" + }, + "publishConfig": { + "provenance": true + } +} diff --git a/packages/markdown/satteri/src/index.ts b/packages/markdown/satteri/src/index.ts new file mode 100644 index 000000000000..be5e2458de85 --- /dev/null +++ b/packages/markdown/satteri/src/index.ts @@ -0,0 +1,16 @@ +export { + createCollectImagesPlugin as satteriCollectImagesPlugin, + createHeadingIdsPlugin as satteriHeadingIdsPlugin, + createImageMarkerPlugin as satteriImageMarkerPlugin, + createShikiPlugin as satteriShikiPlugin, + collectHastText as satteriCollectHastText, + makeFragmentNode as satteriMakeFragmentNode, + createSatteriMarkdownProcessor, + type SatteriMarkdownProcessorOptions, +} from './satteri-processor.js'; +export { + isSatteriProcessor, + satteri, + type SatteriProcessorDescriptor, + type SatteriProcessorOptions, +} from './processor.js'; diff --git a/packages/markdown/satteri/src/processor.ts b/packages/markdown/satteri/src/processor.ts new file mode 100644 index 000000000000..bc668eafe516 --- /dev/null +++ b/packages/markdown/satteri/src/processor.ts @@ -0,0 +1,52 @@ +import { createSatteriMarkdownProcessor } from './satteri-processor.js'; +import type { + AstroMarkdownProcessorOptions, + MarkdownProcessor, +} from '@astrojs/internal-helpers/markdown'; +import type { MdastPluginDefinition, HastPluginDefinition, Features } from 'satteri'; + +export interface SatteriProcessorOptions { + mdastPlugins?: MdastPluginDefinition[]; + hastPlugins?: HastPluginDefinition[]; + features?: Features; +} + +/** + * The descriptor returned by `satteri()`. Integrations extend the pipeline by mutating + * `descriptor.options.mdastPlugins` / `hastPlugins` / `features` directly. + */ +export interface SatteriProcessorDescriptor { + readonly name: 'satteri'; + options: { + mdastPlugins: MdastPluginDefinition[]; + hastPlugins: HastPluginDefinition[]; + features: Features; + }; + createRenderer(shared: AstroMarkdownProcessorOptions): Promise; +} + +export function satteri(opts: SatteriProcessorOptions = {}): SatteriProcessorDescriptor { + const descriptor: SatteriProcessorDescriptor = { + name: 'satteri', + options: { + mdastPlugins: [...(opts.mdastPlugins ?? [])], + hastPlugins: [...(opts.hastPlugins ?? [])], + // Default to `{}` so integrations can write `options.features.gfm = false` + // without an `??=` check. + features: { ...opts.features }, + }, + createRenderer(shared) { + return createSatteriMarkdownProcessor({ + ...shared, + mdastPlugins: descriptor.options.mdastPlugins, + hastPlugins: descriptor.options.hastPlugins, + features: descriptor.options.features, + }); + }, + }; + return descriptor; +} + +export function isSatteriProcessor(p: { name: string }): p is SatteriProcessorDescriptor { + return p.name === 'satteri'; +} diff --git a/packages/markdown/satteri/src/satteri-processor.ts b/packages/markdown/satteri/src/satteri-processor.ts new file mode 100644 index 000000000000..c5694f4f4956 --- /dev/null +++ b/packages/markdown/satteri/src/satteri-processor.ts @@ -0,0 +1,310 @@ +import { isRemoteAllowed } from '@astrojs/internal-helpers/remote'; +import { + defaultExcludeLanguages, + markdownConfigDefaults, + syntaxHighlightDefaults, +} from '@astrojs/internal-helpers/markdown'; +import Slugger from 'github-slugger'; +import type { Features, HastNode, HastPluginDefinition, MdastPluginDefinition } from 'satteri'; +import { createShikiHighlighter } from '@astrojs/internal-helpers/shiki'; +import type { + AstroMarkdownProcessorOptions, + MarkdownHeading, + MarkdownProcessor, +} from '@astrojs/internal-helpers/markdown'; + +type HighlightFn = (code: string, lang: string, meta?: string) => Promise; + +let satteri: typeof import('satteri') | undefined; + +// Loaded lazily so the Sätteri Rust/WASM binary isn't pulled in unless the +// Sätteri processor actually runs. +async function loadSatteri(): Promise { + satteri ??= await import('satteri'); + return satteri; +} + +export function createCollectImagesPlugin( + localImagePaths: Set, + remoteImagePaths: Set, + image: AstroMarkdownProcessorOptions['image'] = {}, +): MdastPluginDefinition { + const domains = image?.domains ?? []; + const remotePatterns = image?.remotePatterns ?? []; + return { + name: 'collect-images', + image(node) { + const url = node.url ? decodeURI(node.url) : undefined; + if (!url) return; + + if (URL.canParse(url)) { + if (isRemoteAllowed(url, { domains, remotePatterns })) { + remoteImagePaths.add(url); + } + } else if (!url.startsWith('/')) { + localImagePaths.add(url); + } + }, + }; +} + +function resolveFrontmatterExpression( + expr: string, + frontmatter: Record, +): string | undefined { + if (!expr.startsWith('frontmatter.') && !expr.startsWith('frontmatter[')) return undefined; + + const pathStr = expr.slice('frontmatter'.length); + const parts: string[] = []; + const pathRegex = /\.(\w+)|\[(\d+)\]|\["([^"]+)"\]|\['([^']+)'\]/g; + let match; + while ((match = pathRegex.exec(pathStr)) !== null) { + parts.push(match[1] ?? match[2] ?? match[3] ?? match[4]); + } + + if (parts.length === 0) return undefined; + + let value: any = frontmatter; + for (const key of parts) { + if (value == null || typeof value !== 'object') return undefined; + value = value[key]; + } + + return typeof value === 'string' ? value : undefined; +} + +export function collectHastText( + node: HastNode, + frontmatter: Record | undefined, +): string { + let text = ''; + if (node.type === 'mdxFlowExpression' || node.type === 'mdxTextExpression') { + if (node.value != null && frontmatter) { + const resolved = resolveFrontmatterExpression(node.value.trim(), frontmatter); + text += resolved ?? node.value; + } + } else if ('value' in node && node.value != null) { + text += node.value; + } + if ('children' in node && node.children) { + for (const child of node.children) { + text += collectHastText(child as HastNode, frontmatter); + } + } + return text; +} + +export function makeFragmentNode(html: string): HastNode { + return { + type: 'mdxJsxFlowElement', + name: 'Fragment', + attributes: [{ type: 'mdxJsxAttribute', name: 'set:html', value: html }], + children: [], + } as unknown as HastNode; +} + +export function createHeadingIdsPlugin( + headings: MarkdownHeading[], + frontmatter: Record | undefined, +): HastPluginDefinition { + const slugger = new Slugger(); + return { + name: 'heading-ids', + element: { + filter: ['h1', 'h2', 'h3', 'h4', 'h5', 'h6'], + visit(node, ctx) { + const rawText = ctx.textContent(node); + const text = + frontmatter && rawText.includes('frontmatter') + ? collectHastText(node, frontmatter) + : rawText; + const existingId = node.properties?.id; + const slug = typeof existingId === 'string' ? existingId : slugger.slug(text); + const depth = Number.parseInt(node.tagName[1], 10); + headings.push({ depth, slug, text }); + if (typeof existingId !== 'string') { + ctx.setProperty(node, 'id', slug); + } + }, + }, + }; +} + +function makeRawNode(html: string): HastNode { + return { type: 'raw', value: html } as unknown as HastNode; +} + +const HAST_PRESERVED_PROPERTIES = new Set(['className', 'htmlFor']); + +/** Tags with `__ASTRO_IMAGE_` for astro's vite-plugin-markdown to replace with the processed image's attributes. */ +export function createImageMarkerPlugin( + localImagePaths: Set, + remoteImagePaths: Set, +): HastPluginDefinition { + const indexBySrc = new Map(); + return { + name: 'image-marker', + element: { + filter: ['img'], + visit(node, ctx) { + const props = (node.properties ?? {}) as Record; + const rawSrc = typeof props.src === 'string' ? props.src : undefined; + if (!rawSrc) return; + const src = decodeURI(rawSrc); + + const isLocal = localImagePaths.has(src); + const isRemote = !isLocal && remoteImagePaths.has(src); + if (!isLocal && !isRemote) return; + + const { src: _src, ...rest } = props; + const index = indexBySrc.get(rawSrc) ?? 0; + indexBySrc.set(rawSrc, index + 1); + + const imageProperties: Record = { ...rest, src, index }; + if (isRemote && !('width' in props) && !('height' in props)) { + imageProperties.inferSize = true; + } + + ctx.setProperty(node, '__ASTRO_IMAGE_', JSON.stringify(imageProperties)); + for (const key of Object.keys(rest)) { + if (!HAST_PRESERVED_PROPERTIES.has(key)) { + ctx.setProperty(node, key, null); + } + } + ctx.setProperty(node, 'src', null); + }, + }, + }; +} + +export function createShikiPlugin( + highlight: HighlightFn, + excludeLangs: string[] | undefined, + options?: { mdx?: boolean }, +): HastPluginDefinition { + const wrapResult = options?.mdx ? makeFragmentNode : makeRawNode; + return { + name: 'shiki-highlight', + element: { + filter: ['pre'], + async visit(node, ctx) { + const codeChild = node.children?.find( + (c: HastNode) => c.type === 'element' && c.tagName === 'code', + ) as HastNode | undefined; + if (!codeChild || codeChild.type !== 'element') return; + + const lang = (codeChild.data as any)?.lang ?? 'plaintext'; + const meta = (codeChild.data as any)?.meta ?? undefined; + + if ( + (excludeLangs && excludeLangs.includes(lang)) || + defaultExcludeLanguages.includes(lang) + ) { + return; + } + + const code = ctx.textContent(codeChild).replace(/\n$/, ''); + const html = await highlight(code, lang, meta); + return wrapResult(html); + }, + }, + }; +} + +export interface SatteriMarkdownProcessorOptions extends AstroMarkdownProcessorOptions { + mdastPlugins?: MdastPluginDefinition[]; + hastPlugins?: HastPluginDefinition[]; + features?: Features; +} + +export async function createSatteriMarkdownProcessor( + opts?: SatteriMarkdownProcessorOptions, +): Promise { + const s = await loadSatteri(); + + const { + syntaxHighlight = syntaxHighlightDefaults, + shikiConfig = markdownConfigDefaults.shikiConfig, + gfm = markdownConfigDefaults.gfm, + smartypants = markdownConfigDefaults.smartypants, + mdastPlugins: userMdastPlugins = [], + hastPlugins: userHastPlugins = [], + features: userFeatures, + } = opts ?? {}; + + const syntaxHighlightType = + typeof syntaxHighlight === 'string' + ? syntaxHighlight + : syntaxHighlight + ? syntaxHighlight.type + : undefined; + + if (syntaxHighlightType === 'prism') { + throw new Error( + 'Prism syntax highlighting is not supported by the `satteri()` markdown processor. Use shiki instead, or switch to `markdown.processor: unified({...})`.', + ); + } + + let highlightFn: HighlightFn | undefined; + if (syntaxHighlightType === 'shiki') { + const hl = await createShikiHighlighter({ + langs: shikiConfig.langs, + theme: shikiConfig.theme, + themes: shikiConfig.themes, + langAlias: shikiConfig.langAlias, + }); + highlightFn = (code, lang, meta) => + hl.codeToHtml(code, lang, { + meta, + wrap: shikiConfig.wrap, + defaultColor: shikiConfig.defaultColor, + transformers: shikiConfig.transformers, + }); + } + + const syntaxHighlightExcludeLangs = + typeof syntaxHighlight === 'object' ? syntaxHighlight.excludeLangs : undefined; + + return { + async render(content, renderOpts) { + const headings: MarkdownHeading[] = []; + const localImagePaths = new Set(); + const remoteImagePaths = new Set(); + const frontmatter = renderOpts?.frontmatter ?? {}; + + const allMdastPlugins: MdastPluginDefinition[] = [ + createCollectImagesPlugin(localImagePaths, remoteImagePaths, opts?.image), + ...userMdastPlugins, + ]; + + const hastPlugins: HastPluginDefinition[] = []; + if (highlightFn) { + hastPlugins.push(createShikiPlugin(highlightFn, syntaxHighlightExcludeLangs)); + } + hastPlugins.push(createHeadingIdsPlugin(headings, frontmatter)); + hastPlugins.push(...userHastPlugins); + hastPlugins.push(createImageMarkerPlugin(localImagePaths, remoteImagePaths)); + + const { html } = await s.markdownToHtml(content, { + mdastPlugins: allMdastPlugins, + hastPlugins, + features: { + gfm: gfm !== false, + smartPunctuation: smartypants !== false, + ...userFeatures, + }, + filename: renderOpts?.fileURL?.pathname, + }); + + return { + code: html, + metadata: { + headings, + localImagePaths: Array.from(localImagePaths), + remoteImagePaths: Array.from(remoteImagePaths), + frontmatter, + }, + }; + }, + }; +} diff --git a/packages/markdown/satteri/test/highlight.test.ts b/packages/markdown/satteri/test/highlight.test.ts new file mode 100644 index 000000000000..9ff091fbc3c2 --- /dev/null +++ b/packages/markdown/satteri/test/highlight.test.ts @@ -0,0 +1,32 @@ +import assert from 'node:assert/strict'; +import { describe, it } from 'node:test'; +import { createSatteriMarkdownProcessor } from '../dist/index.js'; + +describe('satteri highlight', () => { + it('highlights using shiki by default', async () => { + const processor = await createSatteriMarkdownProcessor(); + const { code } = await processor.render('```js\nconsole.log("Hello, world!");\n```'); + assert.match(code, /background-color:/); + }); + + it('does not highlight math code blocks by default', async () => { + const processor = await createSatteriMarkdownProcessor(); + const { code } = await processor.render('```math\n\\frac{1}{2}\n```'); + assert.ok(!code.includes('background-color:')); + }); + + it('supports excludeLangs', async () => { + const processor = await createSatteriMarkdownProcessor({ + syntaxHighlight: { type: 'shiki', excludeLangs: ['mermaid'] }, + }); + const { code } = await processor.render('```mermaid\ngraph TD\nA --> B\n```'); + assert.ok(!code.includes('background-color:')); + }); + + it('rejects prism highlighting', async () => { + await assert.rejects( + createSatteriMarkdownProcessor({ syntaxHighlight: { type: 'prism' } }), + /Prism syntax highlighting is not supported/, + ); + }); +}); diff --git a/packages/markdown/satteri/test/markdown.test.ts b/packages/markdown/satteri/test/markdown.test.ts new file mode 100644 index 000000000000..949a00b06ac6 --- /dev/null +++ b/packages/markdown/satteri/test/markdown.test.ts @@ -0,0 +1,51 @@ +import assert from 'node:assert/strict'; +import { describe, it } from 'node:test'; +import { createSatteriMarkdownProcessor } from '../dist/index.js'; + +describe('satteri markdown', () => { + it('renders basic markdown', async () => { + const processor = await createSatteriMarkdownProcessor(); + const { code } = await processor.render('# Title\n\nSome **bold** text.'); + assert.match(code, /

Title<\/h1>/); + assert.match(code, /bold<\/strong>/); + }); + + it('collects headings into metadata', async () => { + const processor = await createSatteriMarkdownProcessor(); + const { metadata } = await processor.render('# One\n\n## Two'); + assert.deepEqual(metadata.headings, [ + { depth: 1, slug: 'one', text: 'One' }, + { depth: 2, slug: 'two', text: 'Two' }, + ]); + }); + + it('applies GFM by default', async () => { + const processor = await createSatteriMarkdownProcessor(); + const { code } = await processor.render('Visit https://example.com today'); + assert.match(code, / { + const processor = await createSatteriMarkdownProcessor({ gfm: false }); + const { code } = await processor.render('Visit https://example.com today'); + assert.ok(!code.includes(' { + const processor = await createSatteriMarkdownProcessor(); + const { code } = await processor.render('He said "hello"'); + assert.match(code, /“hello”/); + }); + + it('disables smart punctuation when `smartypants: false`', async () => { + const processor = await createSatteriMarkdownProcessor({ smartypants: false }); + const { code } = await processor.render('He said "hello"'); + assert.ok(code.includes('"hello"')); + }); + + it('collects local image paths into metadata', async () => { + const processor = await createSatteriMarkdownProcessor(); + const { metadata } = await processor.render('![alt](./local.png)'); + assert.deepEqual(metadata.localImagePaths, ['./local.png']); + }); +}); diff --git a/packages/markdown/satteri/tsconfig.build.json b/packages/markdown/satteri/tsconfig.build.json new file mode 100644 index 000000000000..2a10b0fd387a --- /dev/null +++ b/packages/markdown/satteri/tsconfig.build.json @@ -0,0 +1,4 @@ +{ + "extends": "../../../configs/tsconfig.build.json", + "references": [{ "path": "../../internal-helpers/tsconfig.json" }] +} diff --git a/packages/markdown/satteri/tsconfig.json b/packages/markdown/satteri/tsconfig.json new file mode 100644 index 000000000000..c0e0de315104 --- /dev/null +++ b/packages/markdown/satteri/tsconfig.json @@ -0,0 +1,5 @@ +{ + "extends": "../../../configs/tsconfig.base.json", + "files": [], + "references": [{ "path": "./tsconfig.build.json" }, { "path": "./tsconfig.test.json" }] +} diff --git a/packages/markdown/satteri/tsconfig.test.json b/packages/markdown/satteri/tsconfig.test.json new file mode 100644 index 000000000000..9af37150bd9d --- /dev/null +++ b/packages/markdown/satteri/tsconfig.test.json @@ -0,0 +1,4 @@ +{ + "extends": "../../../configs/tsconfig.test.json", + "references": [{ "path": "./tsconfig.build.json" }] +} diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 9ce1705a33de..5234ea1d8f94 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -529,9 +529,9 @@ importers: '@astrojs/internal-helpers': specifier: workspace:* version: link:../internal-helpers - '@astrojs/markdown-remark': + '@astrojs/markdown-satteri': specifier: workspace:* - version: link:../markdown/remark + version: link:../markdown/satteri '@astrojs/telemetry': specifier: workspace:* version: link:../telemetry @@ -692,6 +692,9 @@ importers: '@astrojs/check': specifier: workspace:* version: link:../language-tools/astro-check + '@astrojs/markdown-remark': + specifier: workspace:* + version: link:../markdown/remark '@playwright/test': specifier: 1.58.2 version: 1.58.2 @@ -740,12 +743,6 @@ importers: hono: specifier: ^4.12.14 version: 4.12.16 - mdast-util-mdx: - specifier: ^3.0.0 - version: 3.0.0 - mdast-util-mdx-jsx: - specifier: ^3.2.0 - version: 3.2.0 node-mocks-http: specifier: ^1.17.2 version: 1.17.2(@types/express@5.0.6)(@types/node@25.2.3) @@ -5160,9 +5157,12 @@ importers: packages/integrations/mdx: dependencies: - '@astrojs/markdown-remark': + '@astrojs/internal-helpers': specifier: workspace:* - version: link:../../markdown/remark + version: link:../../internal-helpers + '@astrojs/markdown-satteri': + specifier: workspace:* + version: link:../../markdown/satteri '@mdx-js/mdx': specifier: ^3.1.1 version: 3.1.1 @@ -5190,6 +5190,9 @@ importers: remark-smartypants: specifier: ^3.0.2 version: 3.0.2 + satteri: + specifier: ^0.6.3 + version: 0.6.3 source-map: specifier: ^0.7.6 version: 0.7.6 @@ -5200,11 +5203,14 @@ importers: specifier: ^6.0.3 version: 6.0.3 devDependencies: + '@astrojs/markdown-remark': + specifier: workspace:* + version: link:../../markdown/remark '@shikijs/rehype': - specifier: ^4.0.2 + specifier: ^4.0.0 version: 4.0.2 '@shikijs/twoslash': - specifier: ^4.0.2 + specifier: ^4.0.0 version: 4.0.2(typescript@6.0.3) '@types/estree': specifier: ^1.0.8 @@ -5299,6 +5305,9 @@ importers: packages/integrations/mdx/test/fixtures/mdx-frontmatter-injection: dependencies: + '@astrojs/markdown-remark': + specifier: workspace:* + version: link:../../../../../markdown/remark '@astrojs/mdx': specifier: workspace:* version: link:../../.. @@ -6447,10 +6456,34 @@ importers: packages/internal-helpers: dependencies: + '@types/hast': + specifier: ^3.0.4 + version: 3.0.4 + '@types/mdast': + specifier: ^4.0.4 + version: 4.0.4 + js-yaml: + specifier: ^4.1.1 + version: 4.1.1 picomatch: specifier: ^4.0.4 version: 4.0.4 + retext-smartypants: + specifier: ^6.2.0 + version: 6.2.0 + shiki: + specifier: ^4.0.2 + version: 4.0.2 + smol-toml: + specifier: ^1.6.0 + version: 1.6.1 + unified: + specifier: ^11.0.5 + version: 11.0.5 devDependencies: + '@types/js-yaml': + specifier: ^4.0.9 + version: 4.0.9 '@types/picomatch': specifier: ^4.0.2 version: 4.0.2 @@ -6754,9 +6787,6 @@ importers: hast-util-to-text: specifier: ^4.0.2 version: 4.0.2 - js-yaml: - specifier: ^4.1.1 - version: 4.1.1 mdast-util-definitions: specifier: ^6.0.0 version: 6.0.0 @@ -6778,15 +6808,6 @@ importers: remark-smartypants: specifier: ^3.0.2 version: 3.0.2 - retext-smartypants: - specifier: ^6.2.0 - version: 6.2.0 - shiki: - specifier: ^4.0.0 - version: 4.0.2 - smol-toml: - specifier: ^1.6.0 - version: 1.6.1 unified: specifier: ^11.0.5 version: 11.0.5 @@ -6809,9 +6830,6 @@ importers: '@types/hast': specifier: ^3.0.4 version: 3.0.4 - '@types/js-yaml': - specifier: ^4.0.9 - version: 4.0.9 '@types/mdast': specifier: ^4.0.4 version: 4.0.4 @@ -6827,6 +6845,25 @@ importers: mdast-util-mdx-expression: specifier: ^2.0.1 version: 2.0.1 + shiki: + specifier: ^4.0.0 + version: 4.0.2 + + packages/markdown/satteri: + dependencies: + '@astrojs/internal-helpers': + specifier: workspace:* + version: link:../../internal-helpers + github-slugger: + specifier: ^2.0.0 + version: 2.0.0 + satteri: + specifier: ^0.6.3 + version: 0.6.3 + devDependencies: + astro-scripts: + specifier: workspace:* + version: link:../../../scripts packages/telemetry: dependencies: @@ -7459,6 +7496,32 @@ packages: '@borewit/text-codec@0.2.2': resolution: {integrity: sha512-DDaRehssg1aNrH4+2hnj1B7vnUGEjU6OIlyRdkMd0aUdIUvKXrJfXsy8LVtXAy7DRvYVluWbMspsRhz2lcW0mQ==} + '@bruits/satteri-darwin-arm64@0.6.3': + resolution: {integrity: sha512-oKgMfpmNzQ8vaqmkE37PBu8tOyVjoOc4s+DV2tLpWvO6WO467qn/+Nbcirm/ceer7wUM8v4vMLGcZO0gkRzBEg==} + cpu: [arm64] + os: [darwin] + + '@bruits/satteri-darwin-x64@0.6.3': + resolution: {integrity: sha512-9I5pbwZRWH5LvhoCtwpRr4rYSDe43/dLvps6zO70ipVF2XbH4rJ20T+EfvPcmou5jsWMsq9Ybn5GX3PwlSrBaw==} + cpu: [x64] + os: [darwin] + + '@bruits/satteri-linux-x64-gnu@0.6.3': + resolution: {integrity: sha512-aFfw2DL2HpIcAQ8I3ZEtKuz+/GoF0H0sq387jAlvr00Q7buiiFbvARFuQlvTk00N7u3SJIh/3+YkKssTJDT+yQ==} + cpu: [x64] + os: [linux] + libc: [glibc] + + '@bruits/satteri-wasm32-wasi@0.6.3': + resolution: {integrity: sha512-DdpfnJ+04Mb4YtHaxAeETUvhdxFKg3URnroGY39FvweJEgeXx8cFNutuF5w904BhqaiblhmF76AoBnJwNLxsXg==} + engines: {node: '>=14.0.0'} + cpu: [wasm32] + + '@bruits/satteri-win32-x64-msvc@0.6.3': + resolution: {integrity: sha512-J/CZqACnBbv77eImx/JeO5RmCuCyliihiC81u3M4VobA8eupsygaPen3/UFFqf3yfeHvMlE3myilouh/2iHMOA==} + cpu: [x64] + os: [win32] + '@capsizecss/unpack@4.0.0': resolution: {integrity: sha512-VERIM64vtTP1C4mxQ5thVT9fK0apjPFobqybMtA1UdUujWka24ERHbRHFGmpbbhp73MhV+KSsHQH9C6uOTdEQA==} engines: {node: '>=18'} @@ -7969,9 +8032,18 @@ packages: '@emnapi/core@1.10.0': resolution: {integrity: sha512-yq6OkJ4p82CAfPl0u9mQebQHKPJkY7WrIuk205cTYnYe+k2Z8YBh11FrbRG/H6ihirqcacOgl2BIO8oyMQLeXw==} + '@emnapi/core@1.9.1': + resolution: {integrity: sha512-mukuNALVsoix/w1BJwFzwXBN/dHeejQtuVzcDsfOEsdpCumXb/E9j8w11h5S54tT1xhifGfbbSm/ICrObRb3KA==} + '@emnapi/runtime@1.10.0': resolution: {integrity: sha512-ewvYlk86xUoGI0zQRNq/mC+16R1QeDlKQy21Ki3oSYXNgLb45GV1P6A0M+/s6nyCuNDqe5VpaY84BzXGwVbwFA==} + '@emnapi/runtime@1.9.1': + resolution: {integrity: sha512-VYi5+ZVLhpgK4hQ0TAjiQiZ6ol0oe4mBx7mVv7IflsiEp0OWoVsp/+f9Vc1hOhE0TtkORVrI1GvzyreqpgWtkA==} + + '@emnapi/wasi-threads@1.2.0': + resolution: {integrity: sha512-N10dEJNSsUx41Z6pZsXU8FjPjpBEplgH24sfkmITrBED1/U2Esum9F3lfLrMjKHHjmi557zQn7kR9R+XWXu5Rg==} + '@emnapi/wasi-threads@1.2.1': resolution: {integrity: sha512-uTII7OYF+/Mes/MrcIOYp5yOtSMLBWSIoLPpcgwipoiKbli6k322tcoFsxoIIxPDqW01SQGAgko4EzZi2BNv2w==} @@ -14921,6 +14993,9 @@ packages: engines: {node: '>=14.0.0'} hasBin: true + satteri@0.6.3: + resolution: {integrity: sha512-iY5xd2tBDQveYRFkL1F0cegkabWSVoXHi64e1p49SiCs1bZDaqTQPGQI+PqEQIaWkz0iqK80CuQQUYvhsssviw==} + sax@1.6.0: resolution: {integrity: sha512-6R3J5M4AcbtLUdZmRv2SygeVaM7IhrLXu9BmnOGmmACak8fiUtOsYNWUS4uK7upbmHIBbLBeFeI//477BKLBzA==} engines: {node: '>=11.0.0'} @@ -17277,6 +17352,25 @@ snapshots: '@borewit/text-codec@0.2.2': {} + '@bruits/satteri-darwin-arm64@0.6.3': + optional: true + + '@bruits/satteri-darwin-x64@0.6.3': + optional: true + + '@bruits/satteri-linux-x64-gnu@0.6.3': + optional: true + + '@bruits/satteri-wasm32-wasi@0.6.3': + dependencies: + '@emnapi/core': 1.9.1 + '@emnapi/runtime': 1.9.1 + '@napi-rs/wasm-runtime': 1.1.4(@emnapi/core@1.9.1)(@emnapi/runtime@1.9.1) + optional: true + + '@bruits/satteri-win32-x64-msvc@0.6.3': + optional: true + '@capsizecss/unpack@4.0.0': dependencies: fontkitten: 1.0.2 @@ -17895,11 +17989,27 @@ snapshots: tslib: 2.8.1 optional: true + '@emnapi/core@1.9.1': + dependencies: + '@emnapi/wasi-threads': 1.2.0 + tslib: 2.8.1 + optional: true + '@emnapi/runtime@1.10.0': dependencies: tslib: 2.8.1 optional: true + '@emnapi/runtime@1.9.1': + dependencies: + tslib: 2.8.1 + optional: true + + '@emnapi/wasi-threads@1.2.0': + dependencies: + tslib: 2.8.1 + optional: true + '@emnapi/wasi-threads@1.2.1': dependencies: tslib: 2.8.1 @@ -18778,6 +18888,13 @@ snapshots: '@tybys/wasm-util': 0.10.1 optional: true + '@napi-rs/wasm-runtime@1.1.4(@emnapi/core@1.9.1)(@emnapi/runtime@1.9.1)': + dependencies: + '@emnapi/core': 1.9.1 + '@emnapi/runtime': 1.9.1 + '@tybys/wasm-util': 0.10.1 + optional: true + '@neon-rs/load@0.0.4': {} '@netlify/ai@0.4.1': @@ -25805,6 +25922,19 @@ snapshots: optionalDependencies: '@parcel/watcher': 2.5.6 + satteri@0.6.3: + dependencies: + '@types/estree-jsx': 1.0.5 + '@types/hast': 3.0.4 + '@types/mdast': 4.0.4 + '@types/unist': 3.0.3 + optionalDependencies: + '@bruits/satteri-darwin-arm64': 0.6.3 + '@bruits/satteri-darwin-x64': 0.6.3 + '@bruits/satteri-linux-x64-gnu': 0.6.3 + '@bruits/satteri-wasm32-wasi': 0.6.3 + '@bruits/satteri-win32-x64-msvc': 0.6.3 + sax@1.6.0: {} scheduler@0.23.2: diff --git a/pnpm-workspace.yaml b/pnpm-workspace.yaml index cceed3a6128e..95a77a55d91c 100644 --- a/pnpm-workspace.yaml +++ b/pnpm-workspace.yaml @@ -29,6 +29,8 @@ minimumReleaseAgeExclude: # TODO: remove once more stable - '@flue/*' - '@astrojs/*' + - satteri + - '@bruits/satteri-*' peerDependencyRules: allowAny: - 'astro'