From 74dc3edb305c49feec49c39082fa836485da8a92 Mon Sep 17 00:00:00 2001 From: Bjorn Lu Date: Thu, 14 Sep 2023 20:05:38 +0800 Subject: [PATCH] Improve MDX rendering performance (#8533) --- .changeset/thin-starfishes-love.md | 5 + packages/integrations/mdx/package.json | 1 + packages/integrations/mdx/src/index.ts | 50 ++---- packages/integrations/mdx/src/plugins.ts | 143 ++++-------------- .../mdx/src/recma-inject-import-meta-env.ts | 65 ++++++++ .../src/rehype-apply-frontmatter-export.ts | 49 ++++++ .../remark/src/frontmatter-injection.ts | 8 +- packages/markdown/remark/src/index.ts | 6 +- packages/markdown/remark/src/internal.ts | 1 - pnpm-lock.yaml | 3 + 10 files changed, 175 insertions(+), 156 deletions(-) create mode 100644 .changeset/thin-starfishes-love.md create mode 100644 packages/integrations/mdx/src/recma-inject-import-meta-env.ts create mode 100644 packages/integrations/mdx/src/rehype-apply-frontmatter-export.ts diff --git a/.changeset/thin-starfishes-love.md b/.changeset/thin-starfishes-love.md new file mode 100644 index 000000000000..0bb465ee52fd --- /dev/null +++ b/.changeset/thin-starfishes-love.md @@ -0,0 +1,5 @@ +--- +'@astrojs/mdx': patch +--- + +Improve MDX rendering performance by sharing processor instance diff --git a/packages/integrations/mdx/package.json b/packages/integrations/mdx/package.json index a534c767bc4f..0eca06bb7373 100644 --- a/packages/integrations/mdx/package.json +++ b/packages/integrations/mdx/package.json @@ -74,6 +74,7 @@ "remark-rehype": "^10.1.0", "remark-shiki-twoslash": "^3.1.3", "remark-toc": "^8.0.1", + "unified": "^10.1.2", "vite": "^4.4.9" }, "engines": { diff --git a/packages/integrations/mdx/src/index.ts b/packages/integrations/mdx/src/index.ts index 438372e87767..fd330625e137 100644 --- a/packages/integrations/mdx/src/index.ts +++ b/packages/integrations/mdx/src/index.ts @@ -1,6 +1,4 @@ -import { markdownConfigDefaults } from '@astrojs/markdown-remark'; -import { toRemarkInitializeAstroData } from '@astrojs/markdown-remark/dist/internal.js'; -import { compile as mdxCompile, type CompileOptions } from '@mdx-js/mdx'; +import { markdownConfigDefaults, setVfileFrontmatter } from '@astrojs/markdown-remark'; import type { PluggableList } from '@mdx-js/mdx/lib/core.js'; import type { AstroIntegration, ContentEntryType, HookParameters, SSRError } from 'astro'; import astroJSXRenderer from 'astro/jsx/renderer.js'; @@ -8,10 +6,9 @@ import { parse as parseESM } from 'es-module-lexer'; import fs from 'node:fs/promises'; import { fileURLToPath } from 'node:url'; import type { Options as RemarkRehypeOptions } from 'remark-rehype'; -import { SourceMapGenerator } from 'source-map'; import { VFile } from 'vfile'; import type { Plugin as VitePlugin } from 'vite'; -import { getRehypePlugins, getRemarkPlugins, recmaInjectImportMetaEnvPlugin } from './plugins.js'; +import { createMdxProcessor } from './plugins.js'; import type { OptimizeOptions } from './rehype-optimize-static.js'; import { ASTRO_IMAGE_ELEMENT, @@ -84,21 +81,7 @@ export default function mdx(partialMdxOptions: Partial = {}): AstroI ), }); - const mdxPluginOpts: CompileOptions = { - remarkPlugins: await getRemarkPlugins(mdxOptions), - rehypePlugins: getRehypePlugins(mdxOptions), - recmaPlugins: mdxOptions.recmaPlugins, - remarkRehypeOptions: mdxOptions.remarkRehype, - jsx: true, - jsxImportSource: 'astro', - // Note: disable `.md` (and other alternative extensions for markdown files like `.markdown`) support - format: 'mdx', - mdExtensions: [], - }; - - let importMetaEnv: Record = { - SITE: config.site, - }; + let processor: ReturnType; updateConfig({ vite: { @@ -107,7 +90,10 @@ export default function mdx(partialMdxOptions: Partial = {}): AstroI name: '@mdx-js/rollup', enforce: 'pre', configResolved(resolved) { - importMetaEnv = { ...importMetaEnv, ...resolved.env }; + processor = createMdxProcessor(mdxOptions, { + sourcemap: !!resolved.build.sourcemap, + importMetaEnv: { SITE: config.site, ...resolved.env }, + }); // HACK: move ourselves before Astro's JSX plugin to transform things in the right order const jsxPluginIndex = resolved.plugins.findIndex((p) => p.name === 'astro:jsx'); @@ -134,23 +120,13 @@ export default function mdx(partialMdxOptions: Partial = {}): AstroI const code = await fs.readFile(fileId, 'utf-8'); const { data: frontmatter, content: pageContent } = parseFrontmatter(code, id); + + const vfile = new VFile({ value: pageContent, path: id }); + // Ensure `data.astro` is available to all remark plugins + setVfileFrontmatter(vfile, frontmatter); + try { - const compiled = await mdxCompile(new VFile({ value: pageContent, path: id }), { - ...mdxPluginOpts, - elementAttributeNameCase: 'html', - remarkPlugins: [ - // Ensure `data.astro` is available to all remark plugins - toRemarkInitializeAstroData({ userFrontmatter: frontmatter }), - ...(mdxPluginOpts.remarkPlugins ?? []), - ], - recmaPlugins: [ - ...(mdxPluginOpts.recmaPlugins ?? []), - () => recmaInjectImportMetaEnvPlugin({ importMetaEnv }), - ], - SourceMapGenerator: config.vite.build?.sourcemap - ? SourceMapGenerator - : undefined, - }); + const compiled = await processor.process(vfile); return { code: escapeViteEnvReferences(String(compiled.value)), diff --git a/packages/integrations/mdx/src/plugins.ts b/packages/integrations/mdx/src/plugins.ts index a3d9e4ff30ee..3286a9fd877b 100644 --- a/packages/integrations/mdx/src/plugins.ts +++ b/packages/integrations/mdx/src/plugins.ts @@ -4,101 +4,49 @@ import { remarkPrism, remarkShiki, } from '@astrojs/markdown-remark'; -import { - InvalidAstroDataError, - safelyGetAstroData, -} from '@astrojs/markdown-remark/dist/internal.js'; -import { nodeTypes } from '@mdx-js/mdx'; +import { createProcessor, nodeTypes } from '@mdx-js/mdx'; import type { PluggableList } from '@mdx-js/mdx/lib/core.js'; -import type { Literal, MemberExpression } from 'estree'; -import { visit as estreeVisit } from 'estree-util-visit'; import rehypeRaw from 'rehype-raw'; import remarkGfm from 'remark-gfm'; import remarkSmartypants from 'remark-smartypants'; -import type { VFile } from 'vfile'; +import { SourceMapGenerator } from 'source-map'; +import type { Processor } from 'unified'; import type { MdxOptions } from './index.js'; +import { recmaInjectImportMetaEnv } from './recma-inject-import-meta-env.js'; +import { rehypeApplyFrontmatterExport } from './rehype-apply-frontmatter-export.js'; import { rehypeInjectHeadingsExport } from './rehype-collect-headings.js'; import rehypeMetaString from './rehype-meta-string.js'; import { rehypeOptimizeStatic } from './rehype-optimize-static.js'; import { remarkImageToComponent } from './remark-images-to-component.js'; -import { jsToTreeNode } from './utils.js'; // Skip nonessential plugins during performance benchmark runs const isPerformanceBenchmark = Boolean(process.env.ASTRO_PERFORMANCE_BENCHMARK); -export function recmaInjectImportMetaEnvPlugin({ - importMetaEnv, -}: { +interface MdxProcessorExtraOptions { + sourcemap: boolean; importMetaEnv: Record; -}) { - return (tree: any) => { - estreeVisit(tree, (node) => { - if (node.type === 'MemberExpression') { - // attempt to get "import.meta.env" variable name - const envVarName = getImportMetaEnvVariableName(node); - if (typeof envVarName === 'string') { - // clear object keys to replace with envVarLiteral - for (const key in node) { - delete (node as any)[key]; - } - const envVarLiteral: Literal = { - type: 'Literal', - value: importMetaEnv[envVarName], - raw: JSON.stringify(importMetaEnv[envVarName]), - }; - Object.assign(node, envVarLiteral); - } - } - }); - }; } -export function rehypeApplyFrontmatterExport() { - return function (tree: any, vfile: VFile) { - const astroData = safelyGetAstroData(vfile.data); - if (astroData instanceof InvalidAstroDataError) - throw new Error( - // Copied from Astro core `errors-data` - // TODO: find way to import error data from core - '[MDX] A remark or rehype plugin attempted to inject invalid frontmatter. Ensure "astro.frontmatter" is set to a valid JSON object that is not `null` or `undefined`.' - ); - const { frontmatter } = astroData; - const exportNodes = [ - jsToTreeNode(`export const frontmatter = ${JSON.stringify(frontmatter)};`), - ]; - if (frontmatter.layout) { - // NOTE(bholmesdev) 08-22-2022 - // Using an async layout import (i.e. `const Layout = (await import...)`) - // Preserves the dev server import cache when globbing a large set of MDX files - // Full explanation: 'https://github.com/withastro/astro/pull/4428' - exportNodes.unshift( - jsToTreeNode( - /** @see 'vite-plugin-markdown' for layout props reference */ - `import { jsx as layoutJsx } from 'astro/jsx-runtime'; - - export default async function ({ children }) { - const Layout = (await import(${JSON.stringify(frontmatter.layout)})).default; - const { layout, ...content } = frontmatter; - content.file = file; - content.url = url; - return layoutJsx(Layout, { - file, - url, - content, - frontmatter: content, - headings: getHeadings(), - 'server:root': true, - children, - }); - };` - ) - ); - } - tree.children = exportNodes.concat(tree.children); - }; +export function createMdxProcessor( + mdxOptions: MdxOptions, + extraOptions: MdxProcessorExtraOptions +): Processor { + return createProcessor({ + remarkPlugins: getRemarkPlugins(mdxOptions), + rehypePlugins: getRehypePlugins(mdxOptions), + recmaPlugins: getRecmaPlugins(mdxOptions, extraOptions.importMetaEnv), + remarkRehypeOptions: mdxOptions.remarkRehype, + jsx: true, + jsxImportSource: 'astro', + // Note: disable `.md` (and other alternative extensions for markdown files like `.markdown`) support + format: 'mdx', + mdExtensions: [], + elementAttributeNameCase: 'html', + SourceMapGenerator: extraOptions.sourcemap ? SourceMapGenerator : undefined, + }); } -export async function getRemarkPlugins(mdxOptions: MdxOptions): Promise { +function getRemarkPlugins(mdxOptions: MdxOptions): PluggableList { let remarkPlugins: PluggableList = [remarkCollectImages, remarkImageToComponent]; if (!isPerformanceBenchmark) { @@ -125,7 +73,7 @@ export async function getRemarkPlugins(mdxOptions: MdxOptions): Promise +): PluggableList { + return [...(mdxOptions.recmaPlugins ?? []), [recmaInjectImportMetaEnv, { importMetaEnv }]]; } diff --git a/packages/integrations/mdx/src/recma-inject-import-meta-env.ts b/packages/integrations/mdx/src/recma-inject-import-meta-env.ts new file mode 100644 index 000000000000..00578535de4e --- /dev/null +++ b/packages/integrations/mdx/src/recma-inject-import-meta-env.ts @@ -0,0 +1,65 @@ +import type { Literal, MemberExpression } from 'estree'; +import { visit as estreeVisit } from 'estree-util-visit'; + +export function recmaInjectImportMetaEnv({ + importMetaEnv, +}: { + importMetaEnv: Record; +}) { + return (tree: any) => { + estreeVisit(tree, (node) => { + if (node.type === 'MemberExpression') { + // attempt to get "import.meta.env" variable name + const envVarName = getImportMetaEnvVariableName(node); + if (typeof envVarName === 'string') { + // clear object keys to replace with envVarLiteral + for (const key in node) { + delete (node as any)[key]; + } + const envVarLiteral: Literal = { + type: 'Literal', + value: importMetaEnv[envVarName], + raw: JSON.stringify(importMetaEnv[envVarName]), + }; + Object.assign(node, envVarLiteral); + } + } + }); + }; +} + +/** + * Check if estree entry is "import.meta.env.VARIABLE" + * If it is, return the variable name (i.e. "VARIABLE") + */ +function getImportMetaEnvVariableName(node: MemberExpression): string | Error { + try { + // check for ".[ANYTHING]" + if (node.object.type !== 'MemberExpression' || node.property.type !== 'Identifier') + return new Error(); + + const nestedExpression = node.object; + // check for ".env" + if (nestedExpression.property.type !== 'Identifier' || nestedExpression.property.name !== 'env') + return new Error(); + + const envExpression = nestedExpression.object; + // check for ".meta" + if ( + envExpression.type !== 'MetaProperty' || + envExpression.property.type !== 'Identifier' || + envExpression.property.name !== 'meta' + ) + return new Error(); + + // check for "import" + if (envExpression.meta.name !== 'import') return new Error(); + + return node.property.name; + } catch (e) { + if (e instanceof Error) { + return e; + } + return new Error('Unknown parsing error'); + } +} diff --git a/packages/integrations/mdx/src/rehype-apply-frontmatter-export.ts b/packages/integrations/mdx/src/rehype-apply-frontmatter-export.ts new file mode 100644 index 000000000000..3a1098800226 --- /dev/null +++ b/packages/integrations/mdx/src/rehype-apply-frontmatter-export.ts @@ -0,0 +1,49 @@ +import { InvalidAstroDataError } from '@astrojs/markdown-remark'; +import { safelyGetAstroData } from '@astrojs/markdown-remark/dist/internal.js'; +import type { VFile } from 'vfile'; +import { jsToTreeNode } from './utils.js'; + +export function rehypeApplyFrontmatterExport() { + return function (tree: any, vfile: VFile) { + const astroData = safelyGetAstroData(vfile.data); + if (astroData instanceof InvalidAstroDataError) + throw new Error( + // Copied from Astro core `errors-data` + // TODO: find way to import error data from core + '[MDX] A remark or rehype plugin attempted to inject invalid frontmatter. Ensure "astro.frontmatter" is set to a valid JSON object that is not `null` or `undefined`.' + ); + const { frontmatter } = astroData; + const exportNodes = [ + jsToTreeNode(`export const frontmatter = ${JSON.stringify(frontmatter)};`), + ]; + if (frontmatter.layout) { + // NOTE(bholmesdev) 08-22-2022 + // Using an async layout import (i.e. `const Layout = (await import...)`) + // Preserves the dev server import cache when globbing a large set of MDX files + // Full explanation: 'https://github.com/withastro/astro/pull/4428' + exportNodes.unshift( + jsToTreeNode( + /** @see 'vite-plugin-markdown' for layout props reference */ + `import { jsx as layoutJsx } from 'astro/jsx-runtime'; + + export default async function ({ children }) { + const Layout = (await import(${JSON.stringify(frontmatter.layout)})).default; + const { layout, ...content } = frontmatter; + content.file = file; + content.url = url; + return layoutJsx(Layout, { + file, + url, + content, + frontmatter: content, + headings: getHeadings(), + 'server:root': true, + children, + }); + };` + ) + ); + } + tree.children = exportNodes.concat(tree.children); + }; +} diff --git a/packages/markdown/remark/src/frontmatter-injection.ts b/packages/markdown/remark/src/frontmatter-injection.ts index 4f5118ece2dd..4828873fd2c1 100644 --- a/packages/markdown/remark/src/frontmatter-injection.ts +++ b/packages/markdown/remark/src/frontmatter-injection.ts @@ -27,12 +27,14 @@ export function safelyGetAstroData(vfileData: Data): MarkdownAstroData | Invalid return astro; } -export function setAstroData(vfileData: Data, astroData: MarkdownAstroData) { - vfileData.astro = astroData; +export function setVfileFrontmatter(vfile: VFile, frontmatter: Record) { + vfile.data ??= {}; + vfile.data.astro ??= {}; + (vfile.data.astro as any).frontmatter = frontmatter; } /** - * @deprecated Use `setAstroData` instead + * @deprecated Use `setVfileFrontmatter` instead */ export function toRemarkInitializeAstroData({ userFrontmatter, diff --git a/packages/markdown/remark/src/index.ts b/packages/markdown/remark/src/index.ts index 41d08ec9a4ba..89c9ca8bdb65 100644 --- a/packages/markdown/remark/src/index.ts +++ b/packages/markdown/remark/src/index.ts @@ -9,7 +9,7 @@ import type { import { InvalidAstroDataError, safelyGetAstroData, - setAstroData, + setVfileFrontmatter, } from './frontmatter-injection.js'; import { loadPlugins } from './load-plugins.js'; import { rehypeHeadingIds } from './rehype-collect-headings.js'; @@ -27,7 +27,7 @@ import { unified } from 'unified'; import { VFile } from 'vfile'; import { rehypeImages } from './rehype-images.js'; -export { InvalidAstroDataError } from './frontmatter-injection.js'; +export { InvalidAstroDataError, setVfileFrontmatter } from './frontmatter-injection.js'; export { rehypeHeadingIds } from './rehype-collect-headings.js'; export { remarkCollectImages } from './remark-collect-images.js'; export { remarkPrism } from './remark-prism.js'; @@ -125,7 +125,7 @@ export async function createMarkdownProcessor( return { async render(content, renderOpts) { const vfile = new VFile({ value: content, path: renderOpts?.fileURL }); - setAstroData(vfile.data, { frontmatter: renderOpts?.frontmatter ?? {} }); + setVfileFrontmatter(vfile, renderOpts?.frontmatter ?? {}); const result: MarkdownVFile = await parser.process(vfile).catch((err) => { // Ensure that the error message contains the input filename diff --git a/packages/markdown/remark/src/internal.ts b/packages/markdown/remark/src/internal.ts index a0f344a3ae88..0ab7e34bb394 100644 --- a/packages/markdown/remark/src/internal.ts +++ b/packages/markdown/remark/src/internal.ts @@ -1,6 +1,5 @@ export { InvalidAstroDataError, safelyGetAstroData, - setAstroData, toRemarkInitializeAstroData, } from './frontmatter-injection.js'; diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 66287ebc73f5..35e5e84414f6 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -4108,6 +4108,9 @@ importers: remark-toc: specifier: ^8.0.1 version: 8.0.1 + unified: + specifier: ^10.1.2 + version: 10.1.2 vite: specifier: ^4.4.9 version: 4.4.9(@types/node@18.17.8)(sass@1.66.1)